Merge pull request #179 from MLSysOps/feat/improve-web

[MRG] Improve the web UI to use the sync backend API

mle/agents/reporter.py

@@ -23,23 +23,20 @@ def __init__(self, model, console=None):
  self.sys_prompt = """
  You are writing a weekly progress report for an engineer working on a project. Your capabilities include:
  
- 1. Based on the user's input information, you need to organize the information and generate a report from the
+ 1. Based on the user's input information, you need to organize the information and generate more details from the
  user's perspective.
- 2. The first section should be the "Business Goal" to summarize the project's business goal.
- 3. You need to generate a section called "Development Progress" based on the user's Github
- summary given by the user. You may need to generate some more details based on the issues/PRs/commits
- to make the report more informative.
- 4. You need to generate a section called "Communication / Design Progress" based on the user's Google Calendar
- events given by the user. You may need to generate some more details based on the events to make the report.
- 5. You need to generate a section called "Development To-do" based on the user's Github information, and the
+ 2. You need to generate a section called "Development Progress" based on the user's Github
+ summary given by the user.
+ 3. You need to generate a section called "Communication / Design Progress" based on the user's Google Calendar
+ events (if any). Not all events are related to the project but you need to filter out the related ones.
+ 4. You need to generate a section called "Development To-do" based on the user's Github information, and the
  task priority, with the highest priority first and generate more details.
- 6. You need to generate a section called "Communication / Design To-do" based on the user's future
- Google Calendar events.
- 7. You need to generate a section called "Existing Hard Parts" to summarize the hard parts of the project.
- 8. Based on the hard parts and the project information, you need to generate a section called
- "Require Manager' / Others’s help", to indicate the parts that need help.
- 9. You should put some related work and suggestions in the "Other Progress / Thoughts" section.
- 10. You can generate as more as possible details to make sure the report is informative and has great progress.
+ 5. You need to generate a section called "Communication / Design To-do" based on the user's future
+ Google Calendar events (if any).
+ 6. You need to generate a section called "Existing Hard Parts" to summarize/infer the hard parts of the project.
+ 7. Based on the hard parts and the project information, you need to generate a section called
+ "Require Manager' / Others’ help", to indicate the parts that may need help.
+ 8. You can generate as more as possible details to make sure the report is informative and has great progress.
  
  """
  self.json_mode_prompt = """
@@ -57,7 +54,7 @@ def __init__(self, model, console=None):
  "hard_parts": ["The project may face the challenge of ...", ...],
  "require_manager_help": ["The project needs help from the design team to ...", ...],
  "suggestions_to_user": ["Increase more meeting with design team...", ...],
- "reference": [{"title": "xxxx", "link":"https://arxiv.org/abs/1409.0575"}, {"title": "github-project-xxx", "link": "https://github.com/xxx"}, ...],
+ "reference": [{"title": "xxxx", "link":"https://arxiv.org/abs/xxx.xxxx"}, {"title": "xxx", "link": "https://github.com/xxx"}, ...],
  }
 
  """
@@ -104,7 +101,7 @@ def process_knowledge(self, github_summary: dict, calendar_events: list = None,
 
  info_prompt += f"\n## The project's related work: \n"
  for work in github_summary.get("related_work", []):
- info_prompt += f"- {work}\n"
+ info_prompt += f"- {work['title']} ({work['link']})\n"
 
  activities = github_summary.get("user_activity")
  info_prompt += f"""

mle/agents/summarizer.py

@@ -13,9 +13,9 @@ def __init__(self, model, github_repo: str, username: str, github_token: str = N
 
  Args:
  model: the model to use.
- username: the Github username of the user.
- github_token: the Github token to use, if None, will fetch from the environment variable.
- github_repo: the Github repo to summarize.
+ username: the GitHub username of the user.
+ github_token: the GitHub token to use, if None, will fetch from the environment variable.
+ github_repo: the GitHub repo to summarize.
  console: the console to use.
  """
  self.report = None
@@ -57,7 +57,7 @@ def __init__(self, model, github_repo: str, username: str, github_token: str = N
  "tech_stack": ["Python", "PyTorch", "MLFlow", ...],
  "roadmap": [{"task": "fix ...", "priority": "high"}, {"task": "support ...", "priority": "medium"}, ...],
  "hard_parts": ["The project may face the challenge of ...", ...],
- "related_work": ["https://arxiv.org/abs/1409.0575", "https://github.com/MLSysOps/MLE-Agent", ...],
+ "related_work": [{"title": "xxxx", "link":"https://arxiv.org/abs/xxx.xxxx"}, {"title": "xxx", "link": "https://github.com/xxx"}, ...],
  }
 
  """
@@ -72,7 +72,7 @@ def __init__(self, model, github_repo: str, username: str, github_token: str = N
 
  def process_knowledge(self):
  """
- Process the knowledge from the Github repo.
+ Process the knowledge from the GitHub repo.
  Args: None
  """
  info_str = f"""

mle/cli.py

@@ -205,10 +205,8 @@ def integrate():
  "What is your GitHub token? (https://github.com/settings/tokens)"
  ).ask()
 
- username = questionary.text("What is your GitHub username?").ask()
  config["integration"]["github"] = {
- "token": token,
- "username": username,
+ "token": token
  }
  write_config(config)
 

mle/server/app.py

@@ -69,9 +69,9 @@ def read_latest_report():
 
 
 @app.post("/gen_report")
-async def gen_report(report_request: ReportRequest, background_tasks: BackgroundTasks):
+def gen_report(report_request: ReportRequest):
  """
- Generate a report based on the provided GitHub repository and username.
+ Generate a report synchronously based on the provided GitHub repository and username.
  Optionally includes OKR text.
 
  Example payload:
@@ -84,6 +84,43 @@ async def gen_report(report_request: ReportRequest, background_tasks: Background
  "okr": "Improve system efficiency by 20% this quarter"
  }'
  """
+ try:
+ # Run report generation synchronously
+ result = report(
+ os.getcwd(),
+ report_request.repo,
+ report_request.username,
+ okr_str=report_request.okr,
+ model="gpt-4o",
+ )
+
+ return {
+ "message": "Report generation completed",
+ "repo": report_request.repo,
+ "username": report_request.username,
+ "okr_provided": report_request.okr is not None,
+ "result": result # Assuming the report function returns some result
+ }
+ except Exception as e:
+ raise HTTPException(status_code=500, detail=f"Error in report generation process: {e}")
+
+
+@app.post("/gen_report_async")
+async def gen_report_async(report_request: ReportRequest, background_tasks: BackgroundTasks):
+ """
+ Generate a report (async) based on the provided GitHub repository and username.
+ Optionally includes OKR text.
+
+ Example payload:
+
+ curl -X POST http://localhost:8000/gen_report_async \
+ -H "Content-Type: application/json" \
+ -d '{
+ "repo": "MLSysOps/MLE-agent",
+ "username": "huangyz0918",
+ "okr": "Improve system efficiency by 20% this quarter"
+ }'
+ """
  try:
  # Trigger report generation in the background
  background_tasks.add_task(

mle/workflow/report.py

@@ -80,6 +80,4 @@ def report(
  reporter = ReportAgent(model, console)
 
  github_summary = summarizer.summarize()
- proj_report = reporter.gen_report(github_summary, events, okr=okr_str)
- # print_in_box(proj_report, console, title="Github Summarizer", color="green")
- print(proj_report)
+ return reporter.gen_report(github_summary, events, okr=okr_str)

web/app/page.tsx

@@ -1,6 +1,6 @@
 "use client";
 import React, { useState, useEffect } from 'react';
-import { Layout, Card, Input, Button, message, Form, Select } from 'antd';
+import { Layout, Card, Input, Button, message, Form, Select, Spin } from 'antd';
 import dynamic from "next/dynamic";
 
 const MDEditor = dynamic(
@@ -72,9 +72,9 @@ export default function Home() {
  markdown += `### Work Finished This Week\n\n`;
  markdown += `#### Development Progress\n${data.dev_progress.map(progress => `- ${progress}`).join('\n')}\n\n`;
  markdown += `#### Communication/Design Progress\n${data.communicate_progress.map(progress => `- ${progress}`).join('\n')}\n\n`;
- markdown += `### Work TODO in the Next Week\n\n`;
- markdown += `#### Development TODO\n${data.dev_todo.map(todo => `- **${todo.task}** (${todo.priority}): ${todo.description}`).join('\n')}\n\n`;
- markdown += `#### Communication TODO\n${data.communicate_todo.map(todo => `- **${todo.task}** (${todo.priority})`).join('\n')}\n\n`;
+ markdown += `### Work TODOs in the Next Week\n\n`;
+ markdown += `#### Development TODOs\n${data.dev_todo.map(todo => `- ${todo.task} **(${todo.priority})**: ${todo.description}`).join('\n')}\n\n`;
+ markdown += `#### Communication TODOs\n${data.communicate_todo.map(todo => `- ${todo.task} **(${todo.priority})**`).join('\n')}\n\n`;
  markdown += `### Hard Problems\n\n`;
  markdown += `#### Challenges\n${data.hard_parts.map(part => `- ${part}`).join('\n')}\n\n`;
  markdown += `#### Manager Help Required\n${data.require_manager_help.map(help => `- ${help}`).join('\n')}\n\n`;
@@ -100,8 +100,15 @@ export default function Home() {
  }
 
  const result = await response.json();
- message.success('Report generation started successfully');
- setTimeout(fetchLatestReport, 5000);
+
+ if (result.result) {
+ setReportData(result.result);
+ const markdownContent = convertToMarkdown(result.result);
+ setReportContent(markdownContent);
+ message.success('Report generated successfully');
+ } else {
+ message.info('Report generation completed, but no data returned');
+ }
  } catch (error) {
  console.error('Error generating report:', error);
  message.error('Failed to generate report');
@@ -119,7 +126,9 @@ export default function Home() {
  <Content className="p-8">
  <div className="flex gap-4" style={{ alignItems: 'flex-start' }}>
  <Card className="w-[70%]" style={{ maxHeight: 'calc(100vh - 50px)', overflowY: 'auto' }}>
+ <Spin size="large" spinning={loading} tip="Generating. This may take a few minutes...">
  <MDEditor value={reportContent} preview='preview' onChange={setReportContent} height='calc(100vh - 50px)'/>
+ </Spin>
  </Card>
 
  <Card className="w-[30%]" title="Settings" style={{ position: 'sticky', top: '20px' }}>

web/package.json

@@ -9,7 +9,7 @@
  "lint": "next lint"
  },
  "dependencies": {
- "@uiw/react-md-editor": "^4.0.4",
+ "@uiw/react-md-editor": "v3.6.0",
  "antd": "^5.20.5",
  "next": "14.2.8",
  "react": "^18",