Merge pull request #8 from kuafuai/feat/abc

refactor(*):introduce interface design

backend/app/controllers/step_api.py

@@ -1,27 +1,22 @@
 from flask import request, session
 from app.controllers.common import json_response
 from flask import Blueprint
-from app.pkgs.prompt.api import clarifyAPI
-from app.pkgs.prompt.api_pro import clarifyAPIPro
 from app.pkgs.tools.i18b import getI18n
-from config import GRADE
+from app.pkgs.prompt.prompt import clarifyAPI
 
 bp = Blueprint('step_api', __name__, url_prefix='/step_api')
 
 @bp.route('/clarify', methods=['POST'])
 @json_response
 def gen_interface_doc():
     _ = getI18n("controllers")
-    user_prompt = request.json.get('user_prompt')
+    userPrompt = request.json.get('user_prompt')
     username = session["username"]
     apiDocUrl = session[username]['memory']['appconfig']['apiDocUrl']
 
-    if GRADE == "base":
-        msg, success = clarifyAPI(user_prompt, apiDocUrl)
-    else:
-        msg, success = clarifyAPIPro(user_prompt, apiDocUrl)
+    msg, success = clarifyAPI(userPrompt, apiDocUrl)
 
-    session[username]['memory']['originalPrompt'] = user_prompt
+    session[username]['memory']['originalPrompt'] = userPrompt
     session.update()
 
     if success:

backend/app/controllers/step_code.py

@@ -1,12 +1,8 @@
 from flask import request, session
-import json
-import re
 from app.controllers.common import json_response
-from app.pkgs.prompt.code import aiReferenceRepair
 from app.pkgs.tools.i18b import getI18n
-from config import GRADE
-from app.pkgs.prompt.code import aiGenCode, aiMergeCode, aiCheckCode, aiFixError
-from app.pkgs.devops.devops import get_file_content
+from app.pkgs.prompt.prompt import aiGenCode, aiMergeCode, aiCheckCode, aiFixError, aiReferenceRepair
+from app.pkgs.devops.local_tools import getFileContent
 from flask import Blueprint
 
 bp = Blueprint('step_code', __name__, url_prefix='/step_code')
@@ -68,7 +64,7 @@ def reference_repair():
     appName = session[userName]['memory']['appconfig']['appName']
     branch = session[userName]['memory']['appconfig']['sourceBranch']
 
-    hasGitCode, referenceCode =  get_file_content(referenceFile, branch, repo)
+    hasGitCode, referenceCode =  getFileContent(referenceFile, branch, repo)
     if not hasGitCode:
         raise Exception(_("Failed to reference repair no reference file found."))
 

backend/app/controllers/step_devops.py

@@ -1,10 +1,9 @@
 from flask import request, session
 from app.controllers.common import json_response
-from app.pkgs.prompt.code import aiAnalyzeError
-from app.pkgs.devops.gitlab_tools import get_pipeline_status
-from app.pkgs.devops.devops import trigger_pipeline
-from app.pkgs.devops.local_tools import compile_check, lint_check
+from app.pkgs.prompt.prompt import aiAnalyzeError
+from app.pkgs.devops.local_tools import compileCheck, lintCheck
 from app.pkgs.tools.i18b import getI18n
+from app.pkgs.devops.devops import triggerPipeline, getPipelineStatus
 from config import WORKSPACE_PATH
 from flask import Blueprint
 
@@ -18,7 +17,7 @@ def trigger_ci():
     username = session['username']
     branch = session[username]['memory']['appconfig']['featureBranch']
 
-    result, piplineID, piplineUrl, success = trigger_pipeline(branch, repo_path)
+    result, piplineID, piplineUrl, success = triggerPipeline(branch, repo_path)
     if success:
         return {"name": 'ci', "info": {"piplineID": piplineID, "repopath": repo_path, "piplineUrl": piplineUrl}}
     else:
@@ -31,7 +30,7 @@ def plugin_ci():
     pipeline_id = request.args.get('piplineID')
     repopath = request.args.get('repopath')
 
-    piplineJobs = get_pipeline_status(pipeline_id, repopath)
+    piplineJobs = getPipelineStatus(pipeline_id, repopath)
     print("piplineJobs:", piplineJobs)
 
     return {'piplineJobs': piplineJobs}
@@ -45,7 +44,7 @@ def check_compile():
     repo_path = request.json.get('repo_path')
     ws_path = WORKSPACE_PATH+task_id+'/'+repo_path
 
-    success, message = compile_check(ws_path, repo_path)
+    success, message = compileCheck(ws_path, repo_path)
 
     if success:
         reasoning = _("Compile check pass.")
@@ -67,7 +66,7 @@ def check_lint():
     repo_path = request.json.get('service_name')
     ws_path = WORKSPACE_PATH+task_id+'/'+repo_path
 
-    success, message = lint_check(ws_path, repo_path, file_path)
+    success, message = lintCheck(ws_path, repo_path, file_path)
 
     if success:
         reasoning = _("Static code scan passed.")

backend/app/controllers/step_requirement.py

@@ -1,29 +1,24 @@
 from flask import request, session
 from app.controllers.common import json_response
 from flask import Blueprint
-from app.pkgs.prompt.requirement import clarifyRequirement
-from app.pkgs.prompt.requirement_pro import clarifyRequirementPro
 from app.pkgs.tools.i18b import getI18n
-from config import GRADE
+from app.pkgs.prompt.prompt import clarifyRequirement
 
 bp = Blueprint('step_requirement', __name__, url_prefix='/step_requirement')
 
 @bp.route('/clarify', methods=['POST'])
 @json_response
 def clarify():
     _ = getI18n("controllers")
-    user_prompt = request.json.get('user_prompt')
-    global_context = request.json.get('global_context')
+    userPrompt = request.json.get('user_prompt')
+    globalContext = request.json.get('global_context')
     userName = session["username"]
 
     appName = session[userName]['memory']['appconfig']['appName']
     if len(appName) == 0:
         raise Exception(_("Please select the application you want to develop."))
-
-    if GRADE == "base":
-        msg, success = clarifyRequirement(user_prompt, global_context, appName)
-    else:
-        msg, success = clarifyRequirementPro(user_prompt, global_context, appName)
+
+    msg, success = clarifyRequirement(userPrompt, globalContext)
 
     if success:
         return {'message': msg, 'memory': session[userName]['memory']}

backend/app/controllers/step_subtask.py

@@ -1,15 +1,11 @@
-import time
 from flask import request, session
 from app.controllers.common import json_response
-from app.pkgs.prompt.subtask import splitTask
-from app.pkgs.prompt.subtask_java_pro import splitTaskJavaPro
-from app.pkgs.prompt.subtask_vue_pro import splitTaskVuePro
-from app.pkgs.devops.devops import get_file_content
+from app.pkgs.devops.local_tools import getFileContent
 from app.pkgs.tools.i18b import getI18n
 from app.pkgs.knowledge.app_info import getSwagger
-from config import GRADE
 from app.pkgs.tools.file_tool import get_ws_path
 from flask import Blueprint
+from app.pkgs.prompt.prompt import splitTask
 
 bp = Blueprint('step_subtask', __name__, url_prefix='/step_subtask')
 
@@ -39,25 +35,17 @@ def analysis():
     else:
         newfeature = requirementDoc
 
-    if GRADE == "base":
-        filesToEdit, success = splitTask(newfeature, serviceName, apiDocUrl)
-    else:
-        if "java" in serviceName:
-            filesToEdit, success = splitTaskJavaPro(newfeature, serviceName, wsPath)
-        if "vue" in serviceName:
-            filesToEdit, success = splitTaskVuePro(newfeature, serviceName)
-        else:
-            filesToEdit, success = splitTask(newfeature, serviceName)
+    filesToEdit, success = splitTask(newfeature, serviceName, apiDocUrl)
 
     if success:
         for serviceIdx, service in enumerate(filesToEdit):
             for index, file in enumerate(service["files"]):
-                isSuccess, oldCode = get_file_content(file["file-path"], sourceBranch, filesToEdit[serviceIdx]["service-name"])
+                isSuccess, oldCode = getFileContent(file["file-path"], sourceBranch, filesToEdit[serviceIdx]["service-name"])
                 filesToEdit[serviceIdx]["files"][index]["old-code"] = oldCode
                 if not isSuccess:
                     filesToEdit[serviceIdx]["files"][index]["old-code"] = ''
 
-                isSuccess, referenceCode = get_file_content(file["reference-file"], sourceBranch, filesToEdit[serviceIdx]["service-name"])
+                isSuccess, referenceCode = getFileContent(file["reference-file"], sourceBranch, filesToEdit[serviceIdx]["service-name"])
                 filesToEdit[serviceIdx]["files"][index]["reference-code"] = referenceCode
                 if not isSuccess:
                     filesToEdit[serviceIdx]["files"][index]["reference-code"] = ''

backend/app/controllers/task.py

@@ -2,7 +2,6 @@
 from app.controllers.common import json_response
 from app.models.task import getTaskInfo, getEmptyTaskInfo
 from app.pkgs.tools.i18b import getI18n
-from app.pkgs.tools.i18b import getFrontendText
 from config import GRADE
 
 bp = Blueprint('task', __name__, url_prefix='/task')

backend/app/controllers/workspace.py

@@ -1,9 +1,8 @@
 import re
 from flask import request, session
 from app.controllers.common import json_response
-from app.pkgs.devops.gitlab_tools import pull_code
-from app.pkgs.devops.gitlab_tools import check_and_create_branch, update_file_content
 from app.pkgs.tools.i18b import getI18n
+from app.pkgs.devops.git_tools import pullCode, createBranch, pushCode
 from config import GRADE
 from config import WORKSPACE_PATH
 from app.pkgs.tools.file_tool import get_ws_path, write_file_content
@@ -34,10 +33,11 @@ def create():
     fature_branch = request.json.get('feature_branch')
     ws_path = get_ws_path(task_id)
 
+    # todo clone template from git(by independent config)
     if GRADE == "base":
         success = True
     else:
-        success = pull_code(ws_path, repo_path, base_branch, fature_branch)
+        success = pullCode(ws_path, repo_path, base_branch, fature_branch)
 
     if success:
         return _("Create workspace successfully.")
@@ -76,9 +76,9 @@ def gitpush():
 
     plugin = {"name": 'push_code', "uuid": frontUuid}
 
-    check_and_create_branch(session[userName]['memory']['appconfig']['sourceBranch'],
+    createBranch(session[userName]['memory']['appconfig']['sourceBranch'],
                             session[userName]['memory']['appconfig']['featureBranch'], parts[0])
-    result, success = update_file_content(
+    result, success = pushCode(
         parts[2], session[userName]['memory']['appconfig']['featureBranch'], parts[0], code, commitMsg)
     newPrompt = "基于 " + \
         session[userName]['memory']["repoPath"]+" 代码分支进行自动化集成测试"

backend/app/pkgs/devops/cd_interface.py

@@ -0,0 +1,17 @@
+from abc import ABC, abstractmethod
+
+from config import GRADE
+
+# todo finish CD and cloud services
+class CDInterface(ABC):
+    @abstractmethod
+    def triggerPipeline(self, branchName, repoPath):
+        pass
+
+    @abstractmethod
+    def getPipelineStatus(self, pipelineId, repoPath):
+        pass
+
+    @abstractmethod
+    def getPipelineLogs(self, repopath, pipeline_id, job_id):
+        pass

backend/app/pkgs/devops/devops.py

@@ -1,24 +1,27 @@
+from app.pkgs.devops.devops_gitlab import DevopsGitlab
+from app.pkgs.devops.devops_local import DevopsLocal
 from config import DEVOPS_TOOLS
-from app.pkgs.devops.gitlab_tools import get_file_content as gitlab_get_file_content
-from app.pkgs.devops.local_tools import get_file_content as local_get_file_content
-from app.pkgs.devops.gitlab_tools import trigger_pipeline as gitlab_trigger_pipeline
-from app.pkgs.devops.local_tools import trigger_pipeline as local_trigger_pipeline
 
+def triggerPipeline(filePath, branchName, repoPath):
+    if DEVOPS_TOOLS == 'local':
+        obj = DevopsLocal()
+    elif DEVOPS_TOOLS == 'gitlab':
+        obj = DevopsGitlab()
+
+    return obj.triggerPipeline(obj, filePath, branchName, repoPath)
 
-def get_file_content(file_path, branch_name, repopath):
-    try: 
-        if DEVOPS_TOOLS == 'local':
-            return local_get_file_content(file_path, branch_name, repopath)
-        elif DEVOPS_TOOLS == 'gitlab':
-            return gitlab_get_file_content(file_path, branch_name, repopath)
-    except Exception as e:
-        return False, ""
+def getPipelineStatus(piplineId, repoPath):
+    if DEVOPS_TOOLS == 'local':
+        obj = DevopsLocal()
+    elif DEVOPS_TOOLS == 'gitlab':
+        obj = DevopsGitlab()
+
+    return obj.getPipelineStatus(piplineId, repoPath)
 
-def trigger_pipeline(branch_name, repopath):
-    try: 
-        if DEVOPS_TOOLS == 'local':
-            return local_trigger_pipeline(branch_name, repopath)
-        elif DEVOPS_TOOLS == 'gitlab':
-            return gitlab_trigger_pipeline(branch_name, repopath)
-    except Exception as e:
-        return False, ""
+def getPipelineJobLogs(repopath, pipeline_id, job_id):
+    if DEVOPS_TOOLS == 'local':
+        obj = DevopsLocal()
+    elif DEVOPS_TOOLS == 'gitlab':
+        obj = DevopsGitlab()
+
+    return obj.getPipelineJobLogs(obj, repopath, pipeline_id, job_id)

backend/app/pkgs/devops/devops_gitlab.py

@@ -0,0 +1,91 @@
+import gitlab
+import html
+import re
+from app.pkgs.devops.devops_interface import DevopsInterface
+from config import GITLAB_TOKEN, GITLAB_URL, GITLAB_CLONE_URL
+
+class DevopsGitlab(DevopsInterface):
+    def triggerPipeline(self, branch_name, repopath):
+        try:
+            gl = gitlab.Gitlab(GITLAB_URL, GITLAB_TOKEN, api_version='4')
+
+            project = gl.projects.get(repopath)
+            pipeline = project.pipelines.create({'ref': branch_name})
+            pipeline_url = GITLAB_URL + \
+                repopath + '/-/pipelines/' + str(pipeline.id)
+            return "Get pipline status...", str(pipeline.get_id()), pipeline_url, True
+        except Exception as e:
+            return "Failed to trigger pipline:" + str(e), 0, "", False
+
+
+    def getPipelineStatus(self, pipline_id, repopath):
+        try:
+            gl = gitlab.Gitlab(GITLAB_URL, GITLAB_TOKEN, api_version='4')
+
+            project = gl.projects.get(repopath)
+
+            pipeline = project.pipelines.get(pipline_id)
+
+            print("pipeline:", pipeline.status)
+
+            jobs = pipeline.jobs.list()
+
+            job_info = []
+            for job in jobs:
+                print("job:", job)
+                job_info.append({
+                    'job_id': job.id,
+                    'job_name': job.name,
+                    'status': job.status,
+                    'duration': job.duration,
+                    'log': get_pipeline_job_logs(repopath, pipline_id, job.id)
+                })
+
+            return list(reversed(job_info))
+        except Exception as e:
+            return "Failed to get pipline status:" + str(e)
+
+    def getPipelineJobLogs(self, repopath, pipeline_id, job_id):
+        try:
+            gl = gitlab.Gitlab(GITLAB_URL, GITLAB_TOKEN, api_version='4')
+
+            project = gl.projects.get(repopath)
+            job = project.jobs.get(job_id)
+            logs = job.trace()
+
+            return self.remove_color_codes(logs)
+        except Exception as e:
+            return "Failed to get log: " + str(e)
+
+
+    def remove_color_codes(log_string):
+        unicode_string = log_string.decode('unicode_escape')
+        color_regex = re.compile(r'\x1b\[[0-9;]*m')
+        cleaned_string = re.sub(color_regex, '', unicode_string)
+        cleaned_string = re.sub(r'\n', '<br>', unicode_string)
+        cleaned_string = re.sub(r'\r', '<br>', cleaned_string)
+        cleaned_string = re.sub('"', ' ', cleaned_string)
+        cleaned_string = re.sub("'", ' ', cleaned_string)
+        cleaned_string = re.sub(
+            r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]', ' ', cleaned_string)
+        cleaned_string = html.escape(cleaned_string)
+        return cleaned_string
+
+    # def getFileContent(self, file_path, branch_name, repopath):
+    #     try:
+    #         print("get_file_content-repopath:" + repopath)
+    #         print("get_file_content-branch_name:" + branch_name)
+    #         print("get_file_content-file_path:" + file_path)
+    #         gl = gitlab.Gitlab(GITLAB_URL, private_token=GITLAB_TOKEN, api_version='4')
+
+    #         project = gl.projects.get(repopath)
+
+    #         file = project.files.get(file_path, ref=branch_name)
+
+    #         content = codecs.decode(file.decode(), 'utf-8')
+
+    #         print(content)
+
+    #         return True, content
+    #     except Exception as e:
+    #         return False, str(e)