From 5e80166b63a0f1b0d793f0bffbe0a016650fe80c Mon Sep 17 00:00:00 2001 From: Arkajit-Datta Date: Fri, 21 Jul 2023 18:52:00 +0530 Subject: [PATCH 01/23] Added functions to write various file types and a file handler too --- superagi/resource_manager/file_manager.py | 58 +++++++++++++++++++---- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/superagi/resource_manager/file_manager.py b/superagi/resource_manager/file_manager.py index e9059ac20..7a48da53d 100644 --- a/superagi/resource_manager/file_manager.py +++ b/superagi/resource_manager/file_manager.py @@ -7,11 +7,16 @@ from superagi.models.agent import Agent from superagi.models.agent_execution import AgentExecution from superagi.types.storage_types import StorageType + +class UnsupportedFileTypeError(Exception): + pass + class FileManager: def __init__(self, session: Session, agent_id: int = None, agent_execution_id: int = None): self.session = session self.agent_id = agent_id self.agent_execution_id = agent_execution_id + def write_binary_file(self, file_name: str, data): if self.agent_id is not None: final_path = ResourceHelper.get_agent_write_resource_path(file_name, @@ -31,6 +36,7 @@ def write_binary_file(self, file_name: str, data): return f"Binary {file_name} saved successfully" except Exception as err: return f"Error write_binary_file: {err}" + def write_to_s3(self, file_name, final_path): with open(final_path, 'rb') as img: resource = ResourceHelper.make_written_file_resource(file_name=file_name, @@ -54,15 +60,9 @@ def write_file(self, file_name: str, content): self.agent_execution_id)) else: final_path = ResourceHelper.get_resource_path(file_name) - try: - with open(final_path, mode="w") as file: - file.write(content) - file.close() - self.write_to_s3(file_name, final_path) - logger.info(f"{file_name} - File written successfully") - return f"{file_name} - File written successfully" - except Exception as err: - return f"Error write_file: {err}" + + self.save_file_by_type(file_name=file_name, file_path=final_path, content=content) + def write_csv_file(self, file_name: str, csv_data): if self.agent_id is not None: final_path = ResourceHelper.get_agent_write_resource_path(file_name, @@ -84,13 +84,30 @@ def write_csv_file(self, file_name: str, csv_data): except Exception as err: return f"Error write_csv_file: {err}" - + def write_pdf_file(self, file_name: str ,file_path: str, content): + pass + + def write_docx_file(self, file_name: str ,file_path: str, content): + pass + + def write_txt_file(self, file_name: str ,file_path: str, content): + try: + with open(file_path, mode="w") as file: + file.write(content) + file.close() + self.write_to_s3(file_name, file_path) + logger.info(f"{file_name} - File written successfully") + return f"{file_name} - File written successfully" + except Exception as err: + return f"Error write_file: {err}" + def get_agent_resource_path(self, file_name: str): return ResourceHelper.get_agent_write_resource_path(file_name, agent=Agent.get_agent_from_id(self.session, self.agent_id), agent_execution=AgentExecution .get_agent_execution_from_id(self.session, self.agent_execution_id)) + def read_file(self, file_name: str): if self.agent_id is not None: final_path = self.get_agent_resource_path(file_name) @@ -104,6 +121,7 @@ def read_file(self, file_name: str): return content except Exception as err: return f"Error while reading file {file_name}: {err}" + def get_files(self): """ Gets all file names generated by the CodingTool. @@ -122,3 +140,23 @@ def get_files(self): logger.error(f"Error while accessing files in {final_path}: {err}") files = [] return files + + def save_file_by_type(self, file_name: str, file_path: str, content): + + # Extract the file type from the file_name + file_type = file_name.split('.')[-1].lower() + + # Dictionary to map file extensions to corresponding functions + file_type_handlers = { + 'txt': write_txt_file, + 'pdf': write_pdf_file, + 'docx': write_docx_file, + 'csv': write_csv_file + # NOTE: Add more file types and corresponding functions as needed, These functions should be defined + } + + if file_path in file_type_handlers: + file_type_handlers[file_type](file_name, file_path, content) + else: + raise UnsupportedFileTypeError(f"Unsupported file type: {file_type}. Cannot save the file.") + \ No newline at end of file From 2738a9a6c0a28512712691268b6bfe96ecd6755c Mon Sep 17 00:00:00 2001 From: Arkajit-Datta Date: Wed, 26 Jul 2023 11:53:51 +0530 Subject: [PATCH 02/23] FileManager updated to handle and save HTMLs. --- superagi/resource_manager/file_manager.py | 32 +++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/superagi/resource_manager/file_manager.py b/superagi/resource_manager/file_manager.py index 7a48da53d..5ebf1215f 100644 --- a/superagi/resource_manager/file_manager.py +++ b/superagi/resource_manager/file_manager.py @@ -50,7 +50,7 @@ def write_to_s3(self, file_name, final_path): s3_helper = S3Helper() s3_helper.upload_file(img, path=resource.path) - def write_file(self, file_name: str, content): + def write_file(self, file_name: str, content, return_file_path: bool = False): if self.agent_id is not None: final_path = ResourceHelper.get_agent_write_resource_path(file_name, agent=Agent.get_agent_from_id(self.session, @@ -61,18 +61,17 @@ def write_file(self, file_name: str, content): else: final_path = ResourceHelper.get_resource_path(file_name) - self.save_file_by_type(file_name=file_name, file_path=final_path, content=content) + try: + self.save_file_by_type(file_name=file_name, file_path=final_path, content=content) + except Exception as err: + return f"Error write_file: {err}" - def write_csv_file(self, file_name: str, csv_data): - if self.agent_id is not None: - final_path = ResourceHelper.get_agent_write_resource_path(file_name, - agent=Agent.get_agent_from_id(self.session, - self.agent_id), - agent_execution=AgentExecution - .get_agent_execution_from_id(self.session, - self.agent_execution_id)) + if return_file_path: + return final_path else: - final_path = ResourceHelper.get_resource_path(file_name) + return f"{file_name} - File written successfully" + + def write_csv_file(self, file_name: str, final_path: str, csv_data) -> str: try: with open(final_path, mode="w") as file: writer = csv.writer(file, lineterminator="\n") @@ -90,14 +89,14 @@ def write_pdf_file(self, file_name: str ,file_path: str, content): def write_docx_file(self, file_name: str ,file_path: str, content): pass - def write_txt_file(self, file_name: str ,file_path: str, content): + def write_txt_file(self, file_name: str ,file_path: str, content) -> str: try: with open(file_path, mode="w") as file: file.write(content) file.close() self.write_to_s3(file_name, file_path) logger.info(f"{file_name} - File written successfully") - return f"{file_name} - File written successfully" + return file_path except Exception as err: return f"Error write_file: {err}" @@ -146,17 +145,18 @@ def save_file_by_type(self, file_name: str, file_path: str, content): # Extract the file type from the file_name file_type = file_name.split('.')[-1].lower() - # Dictionary to map file extensions to corresponding functions + # Dictionary to map file types to corresponding functions file_type_handlers = { 'txt': write_txt_file, 'pdf': write_pdf_file, 'docx': write_docx_file, - 'csv': write_csv_file + 'csv': write_csv_file, + 'html': write_txt_file # NOTE: Add more file types and corresponding functions as needed, These functions should be defined } if file_path in file_type_handlers: - file_type_handlers[file_type](file_name, file_path, content) + return file_type_handlers[file_type](file_name, file_path, content) else: raise UnsupportedFileTypeError(f"Unsupported file type: {file_type}. Cannot save the file.") \ No newline at end of file From 0fdeff9ead9065a6a8494388d2fda99a5d863199 Mon Sep 17 00:00:00 2001 From: Arkajit-Datta Date: Fri, 28 Jul 2023 18:25:06 +0530 Subject: [PATCH 03/23] adding PDF + DOCX support to save images --- Dockerfile | 3 + DockerfileCelery | 3 + requirements.txt | 2 + superagi/resource_manager/file_manager.py | 66 +++++++++++++++---- .../tools/file/prompts/add_images_to_html.txt | 4 ++ .../file/prompts/content_to_html_prompt.txt | 7 ++ superagi/tools/file/write_file.py | 60 ++++++++++++++++- .../stable_diffusion_image_gen.py | 10 ++- 8 files changed, 135 insertions(+), 20 deletions(-) create mode 100644 superagi/tools/file/prompts/add_images_to_html.txt create mode 100644 superagi/tools/file/prompts/content_to_html_prompt.txt diff --git a/Dockerfile b/Dockerfile index 775a0d5dc..9d0804851 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,9 @@ WORKDIR /app COPY requirements.txt . #RUN apt-get update && apt-get install --no-install-recommends -y git wget libpq-dev gcc python3-dev && pip install psycopg2 +RUN apt-get update +RUN apt-get install -y wkhtmltopdf + RUN pip install --upgrade pip RUN pip install --no-cache-dir -r requirements.txt diff --git a/DockerfileCelery b/DockerfileCelery index 0f8625679..a09c6160b 100644 --- a/DockerfileCelery +++ b/DockerfileCelery @@ -3,6 +3,9 @@ FROM python:3.9 WORKDIR /app #RUN apt-get update && apt-get install --no-install-recommends -y git wget libpq-dev gcc python3-dev && pip install psycopg2 +RUN apt-get update +RUN apt-get install -y wkhtmltopdf + RUN pip install --upgrade pip COPY requirements.txt . diff --git a/requirements.txt b/requirements.txt index f393a996d..c5a4cc8c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -153,3 +153,5 @@ EbookLib==0.18 html2text==2020.1.16 duckduckgo-search==3.8.3 google-generativeai==0.1.0 +pdfkit==1.0.0 +htmldocx==0.0.6 diff --git a/superagi/resource_manager/file_manager.py b/superagi/resource_manager/file_manager.py index 5ebf1215f..fe7c74558 100644 --- a/superagi/resource_manager/file_manager.py +++ b/superagi/resource_manager/file_manager.py @@ -7,10 +7,15 @@ from superagi.models.agent import Agent from superagi.models.agent_execution import AgentExecution from superagi.types.storage_types import StorageType +import pdfkit +from htmldocx import HtmlToDocx class UnsupportedFileTypeError(Exception): pass +class FileNotCreatedError(Exception): + pass + class FileManager: def __init__(self, session: Session, agent_id: int = None, agent_execution_id: int = None): self.session = session @@ -62,7 +67,8 @@ def write_file(self, file_name: str, content, return_file_path: bool = False): final_path = ResourceHelper.get_resource_path(file_name) try: - self.save_file_by_type(file_name=file_name, file_path=final_path, content=content) + res = self.save_file_by_type(file_name=file_name, file_path=final_path, content=content) + print(f"---------------{res}-------------") except Exception as err: return f"Error write_file: {err}" @@ -81,24 +87,55 @@ def write_csv_file(self, file_name: str, final_path: str, csv_data) -> str: logger.info(f"{file_name} - File written successfully") return f"{file_name} - File written successfully" except Exception as err: - return f"Error write_csv_file: {err}" + raise FileNotCreatedError(f"Error write_file: {err}") from err def write_pdf_file(self, file_name: str ,file_path: str, content): - pass - + logger.info("Inside writing PDF") + # Saving the HTML file + html_file_path = f"{file_path[:-4]}.html" + self.write_txt_file(file_name=html_file_path.split('/')[-1], file_path=html_file_path, content=content) + + # Convert HTML file to a PDF file + try: + options = { + 'quiet': '', + 'page-size': 'Letter', + 'margin-top': '0.75in', + 'margin-right': '0.75in', + 'margin-bottom': '0.75in', + 'margin-left': '0.75in', + 'enable-local-file-access': '' + } + config = pdfkit.configuration(wkhtmltopdf = "/usr/bin/wkhtmltopdf") + pdfkit.from_file(html_file_path, file_path, options = options, configuration = config) + return file_path + + except Exception as err: + raise FileNotCreatedError(f"Error write_file: {err}") from err + def write_docx_file(self, file_name: str ,file_path: str, content): - pass - + # Saving the HTML file + html_file_path = f"{file_path[:-4]}.html" + self.write_txt_file(file_name=html_file_path.split('/')[-1], file_path=html_file_path, content=content) + + # Convert HTML file to a DOCx file + try: + new_parser = HtmlToDocx() + new_parser.parse_html_file(html_file_path, file_path) + + return file_path + except Exception as err: + raise FileNotCreatedError(f"Error write_file: {err}") from err + def write_txt_file(self, file_name: str ,file_path: str, content) -> str: try: with open(file_path, mode="w") as file: file.write(content) file.close() self.write_to_s3(file_name, file_path) - logger.info(f"{file_name} - File written successfully") return file_path except Exception as err: - return f"Error write_file: {err}" + raise FileNotCreatedError(f"Error write_file: {err}") from err def get_agent_resource_path(self, file_name: str): return ResourceHelper.get_agent_write_resource_path(file_name, agent=Agent.get_agent_from_id(self.session, @@ -147,15 +184,16 @@ def save_file_by_type(self, file_name: str, file_path: str, content): # Dictionary to map file types to corresponding functions file_type_handlers = { - 'txt': write_txt_file, - 'pdf': write_pdf_file, - 'docx': write_docx_file, - 'csv': write_csv_file, - 'html': write_txt_file + 'txt': self.write_txt_file, + 'pdf': self.write_pdf_file, + 'docx': self.write_docx_file, + 'doc': self.write_docx_file, + 'csv': self.write_csv_file, + 'html': self.write_txt_file # NOTE: Add more file types and corresponding functions as needed, These functions should be defined } - if file_path in file_type_handlers: + if file_type in file_type_handlers: return file_type_handlers[file_type](file_name, file_path, content) else: raise UnsupportedFileTypeError(f"Unsupported file type: {file_type}. Cannot save the file.") diff --git a/superagi/tools/file/prompts/add_images_to_html.txt b/superagi/tools/file/prompts/add_images_to_html.txt new file mode 100644 index 000000000..57dda0a49 --- /dev/null +++ b/superagi/tools/file/prompts/add_images_to_html.txt @@ -0,0 +1,4 @@ +Now, you will be provided with few image path locations. You will have to attach the following images in appropriate locations inside the html code. +Remember to maintain the elegancy and styling of the User Interface generated. Make sure you attach all the images provided to you. + +The relevant paths of the images are provided below: diff --git a/superagi/tools/file/prompts/content_to_html_prompt.txt b/superagi/tools/file/prompts/content_to_html_prompt.txt new file mode 100644 index 000000000..e450f46e4 --- /dev/null +++ b/superagi/tools/file/prompts/content_to_html_prompt.txt @@ -0,0 +1,7 @@ +You are an HTML code generating AI Agent. Your task is to generate a well formatted and well styled HTML file for a given content. +Remember to style the HTML beautifully, for which you can add the