Ingest repo (C# only) (autogenhub#7)

* ingest WIP function * ingest WIP * analyze WIP * WIP * ingester WIP * ingester WIP * ingester WIP * fix github download * working ingester
odoochain · Nov 3, 2023 · ddd5fa0 · ddd5fa0
1 parent f423e15
commit ddd5fa0
Show file tree

Hide file tree

Showing 21 changed files with 1,309 additions and 102 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,13 +1,11 @@
 FROM mcr.microsoft.com/devcontainers/dotnet:0-7.0
 # Install the xz-utils package
-RUN apt-get update && apt-get install -y xz-utils nodejs npm
+RUN apt-get update && apt-get install -y xz-utils ca-certificates curl gnupg
 
 RUN curl -fsSL https://aka.ms/install-azd.sh | bash
 
 RUN curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | \
   sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null && \
   echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | \
   sudo tee /etc/apt/sources.list.d/ngrok.list && \
-  sudo apt update && sudo apt install ngrok
-
-RUN npm i -g azure-functions-core-tools@4 --unsafe-perm true
+  sudo apt update && sudo apt install ngrok
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -10,7 +10,16 @@
 		"ghcr.io/devcontainers/features/azure-cli:1": {},
 		"ghcr.io/devcontainers/features/common-utils:2": {},
 		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
-		"ghcr.io/azure/azure-dev/azd:latest": {}
+		"ghcr.io/azure/azure-dev/azd:latest": {},
+		"ghcr.io/devcontainers/features/node:1": {
+			"nodeGypDependencies": true,
+			"version": "18",
+			"nvmVersion": "latest"
+		},
+		"ghcr.io/azure/azure-dev/azd:0": {
+			"version": "stable"
+		},
+		"ghcr.io/jlaundry/devcontainer-features/azure-functions-core-tools:1": {}
 	},
 	"postCreateCommand": "bash .devcontainer/startup.sh",
 	"hostRequirements": {

diff --git a/.gitignore b/.gitignore
@@ -495,4 +495,5 @@ elsa.sqlite.*
 elsa-core/
 sk-azfunc-server/local.settings.json
 .azure
-temp
+temp
+
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -1,11 +1,18 @@
 {
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Attach to .NET Functions",
-            "type": "coreclr",
-            "request": "attach",
-            "processId": "${command:azureFunctions.pickProcess}"
-        }
-    ]
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Attach to .NET Functions",
+      "type": "coreclr",
+      "request": "attach",
+      "processId": "${command:azureFunctions.pickProcess}"
+    },
+    {
+      "name": "Attach to Node Functions",
+      "type": "node",
+      "request": "attach",
+      "port": 9229,
+      "preLaunchTask": "func: host start"
+    }
+  ]
 }
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,8 +1,11 @@
 {
     "dotnet.defaultSolution": "sk-dev-team.sln",
-    "azureFunctions.deploySubpath": "sk-azfunc-server/bin/Release/net7.0/publish",
-    "azureFunctions.projectLanguage": "C#",
+    "azureFunctions.deploySubpath": "src/apps/ingest",
+    "azureFunctions.projectLanguage": "JavaScript",
     "azureFunctions.projectRuntime": "~4",
     "debug.internalConsoleOptions": "neverOpen",
-    "azureFunctions.preDeployTask": "publish (functions)"
+    "azureFunctions.preDeployTask": "npm prune (functions)",
+    "azureFunctions.postDeployTask": "npm install (functions)",
+    "azureFunctions.projectLanguageModel": 4,
+    "azureFunctions.projectSubpath": "src/apps/ingest"
 }
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -1,81 +1,99 @@
 {
-	"version": "2.0.0",
-	"tasks": [
-		{
-			"label": "clean (functions)",
-			"command": "dotnet",
-			"args": [
-				"clean",
-				"/property:GenerateFullPaths=true",
-				"/consoleloggerparameters:NoSummary"
-			],
-			"type": "process",
-			"problemMatcher": "$msCompile",
-			"options": {
-				"cwd": "${workspaceFolder}/sk-azfunc-server"
-			}
-		},
-		{
-			"label": "build (functions)",
-			"command": "dotnet",
-			"args": [
-				"build",
-				"/property:GenerateFullPaths=true",
-				"/consoleloggerparameters:NoSummary"
-			],
-			"type": "process",
-			"dependsOn": "clean (functions)",
-			"group": {
-				"kind": "build",
-				"isDefault": true
-			},
-			"problemMatcher": "$msCompile",
-			"options": {
-				"cwd": "${workspaceFolder}/sk-azfunc-server"
-			}
-		},
-		{
-			"label": "clean release (functions)",
-			"command": "dotnet",
-			"args": [
-				"clean",
-				"--configuration",
-				"Release",
-				"/property:GenerateFullPaths=true",
-				"/consoleloggerparameters:NoSummary"
-			],
-			"type": "process",
-			"problemMatcher": "$msCompile",
-			"options": {
-				"cwd": "${workspaceFolder}/sk-azfunc-server"
-			}
-		},
-		{
-			"label": "publish (functions)",
-			"command": "dotnet",
-			"args": [
-				"publish",
-				"--configuration",
-				"Release",
-				"/property:GenerateFullPaths=true",
-				"/consoleloggerparameters:NoSummary"
-			],
-			"type": "process",
-			"dependsOn": "clean release (functions)",
-			"problemMatcher": "$msCompile",
-			"options": {
-				"cwd": "${workspaceFolder}/sk-azfunc-server"
-			}
-		},
-		{
-			"type": "func",
-			"dependsOn": "build (functions)",
-			"options": {
-				"cwd": "${workspaceFolder}/sk-azfunc-server/bin/Debug/net7.0"
-			},
-			"command": "host start",
-			"isBackground": true,
-			"problemMatcher": "$func-dotnet-watch"
-		}
-	]
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "clean (functions)",
+      "command": "dotnet",
+      "args": [
+        "clean",
+        "/property:GenerateFullPaths=true",
+        "/consoleloggerparameters:NoSummary"
+      ],
+      "type": "process",
+      "problemMatcher": "$msCompile",
+      "options": {
+        "cwd": "${workspaceFolder}/sk-azfunc-server"
+      }
+    },
+    {
+      "label": "build (functions)",
+      "command": "dotnet",
+      "args": [
+        "build",
+        "/property:GenerateFullPaths=true",
+        "/consoleloggerparameters:NoSummary"
+      ],
+      "type": "process",
+      "dependsOn": "clean (functions)",
+      "group": {
+        "kind": "build",
+        "isDefault": true
+      },
+      "problemMatcher": "$msCompile",
+      "options": {
+        "cwd": "${workspaceFolder}/sk-azfunc-server"
+      }
+    },
+    {
+      "label": "clean release (functions)",
+      "command": "dotnet",
+      "args": [
+        "clean",
+        "--configuration",
+        "Release",
+        "/property:GenerateFullPaths=true",
+        "/consoleloggerparameters:NoSummary"
+      ],
+      "type": "process",
+      "problemMatcher": "$msCompile",
+      "options": {
+        "cwd": "${workspaceFolder}/sk-azfunc-server"
+      }
+    },
+    {
+      "label": "publish (functions)",
+      "command": "dotnet",
+      "args": [
+        "publish",
+        "--configuration",
+        "Release",
+        "/property:GenerateFullPaths=true",
+        "/consoleloggerparameters:NoSummary"
+      ],
+      "type": "process",
+      "dependsOn": "clean release (functions)",
+      "problemMatcher": "$msCompile",
+      "options": {
+        "cwd": "${workspaceFolder}/sk-azfunc-server"
+      }
+    },
+    {
+      "type": "func",
+      "label": "func: host start",
+      "command": "host start",
+      "problemMatcher": "$func-node-watch",
+      "isBackground": true,
+      "dependsOn": "npm install (functions)",
+      "options": {
+        "cwd": "${workspaceFolder}/src/apps/ingest"
+      }
+    },
+    {
+      "type": "shell",
+      "label": "npm install (functions)",
+      "command": "npm install",
+      "options": {
+        "cwd": "${workspaceFolder}/src/apps/ingest"
+      }
+    },
+    {
+      "type": "shell",
+      "label": "npm prune (functions)",
+      "command": "npm prune --production",
+      "problemMatcher": [],
+      "options": {
+        "cwd": "${workspaceFolder}/src/apps/ingest"
+      }
+    }
+  ]
 }
diff --git a/src/apps/gh-flow-df/Activities/PullRequestActivities.cs b/src/apps/gh-flow-df/Activities/PullRequestActivities.cs
@@ -8,7 +8,6 @@
 using Azure.ResourceManager.ContainerInstance.Models;
 using Azure.ResourceManager.Resources;
 using Azure.Storage.Files.Shares;
-using Microsoft.AspNetCore.Http;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Azure.Functions.Worker.Http;
 using Microsoft.DurableTask.Client;

diff --git a/src/apps/gh-flow/Program.cs b/src/apps/gh-flow/Program.cs
@@ -12,6 +12,7 @@
 var builder = WebApplication.CreateBuilder(args);
 builder.Services.AddSingleton<WebhookEventProcessor, GithubWebHookProcessor>();
 builder.Services.AddTransient(CreateKernel);
+builder.Services.AddHttpClient();
 
 builder.Services.AddSingleton(s =>
 {
@@ -43,9 +44,15 @@
     {
         configuration.GetSection("QdrantOptions").Bind(settings);
     });
+builder.Services.AddOptions<ServiceOptions>()
+    .Configure<IConfiguration>((settings, configuration) =>
+    {
+        configuration.GetSection("ServiceOptions").Bind(settings);
+    });
 
 builder.Services.AddSingleton<IManageAzure, AzureService>();
 builder.Services.AddSingleton<IManageGithub, GithubService>();
+builder.Services.AddSingleton<IAnalyzeCode, CodeAnalyzer>();
 
 
 builder.Host.UseOrleans(siloBuilder =>
@@ -130,6 +137,7 @@
     endpoints.MapGitHubWebhooks();
 });
 
+
 app.Run();
 
 static IKernel CreateKernel(IServiceProvider provider)

diff --git a/src/apps/gh-flow/Services/GithubWebHookProcessor.cs b/src/apps/gh-flow/Services/GithubWebHookProcessor.cs
@@ -2,6 +2,7 @@
 using Microsoft.AI.DevTeam.Skills;
 using Octokit.Webhooks;
 using Octokit.Webhooks.Events;
+using Octokit.Webhooks.Events.CommitComment;
 using Octokit.Webhooks.Events.IssueComment;
 using Octokit.Webhooks.Events.Issues;
 using Octokit.Webhooks.Models;
@@ -62,6 +63,19 @@ protected override async Task ProcessIssueCommentWebhookAsync(
         }
     }
 
+    // TODO: implement
+    protected override Task ProcessPushWebhookAsync(WebhookHeaders headers, PushEvent pushEvent)
+    {
+        var org = pushEvent.Organization.Login;
+        var repo = pushEvent.Repository.Name;
+        // Assumes the label follows the following convention: Skill.Function example: PM.Readme
+
+        var suffix = $"{org}-{repo}";
+        var ingester = _grains.GetGrain<IIngestRepo>(suffix);
+
+        return Task.CompletedTask;
+    }
+
     private async Task HandleClosingIssue(long issueNumber, string skillName, string functionName, string suffix, string org, string repo)
     {
         if (skillName == nameof(PM) && functionName == nameof(PM.Readme))
@@ -183,6 +197,11 @@ private async Task HandleNewAsk(long issueNumber, string skillName, string funct
             var conductor = _grains.GetGrain<IOrchestrateWorkflows>(issueNumber, suffix);
             await conductor.InitialFlow(input, org, repo, issueNumber);
         }
+        else if (skillName == "Repo" && functionName == "Ingest")
+        {
+            var ingestor = _grains.GetGrain<IIngestRepo>(suffix);
+            await ingestor.IngestionFlow(org, repo, "main");
+        }
         else if (skillName == nameof(PM) && functionName == nameof(PM.Readme))
         {
             var pm = _grains.GetGrain<IManageProduct>(issueNumber, suffix);

diff --git a/src/apps/gh-flow/gh-flow.csproj b/src/apps/gh-flow/gh-flow.csproj
@@ -21,6 +21,7 @@
     <PackageReference Include="Microsoft.Orleans.Clustering.Cosmos" Version="7.2.1" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.21.0" />
     <PackageReference Include="Microsoft.Orleans.Reminders.Cosmos" Version="7.2.1" />
+    <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
   </ItemGroup>
 
 

diff --git a/src/apps/ingest/.funcignore b/src/apps/ingest/.funcignore
@@ -0,0 +1,10 @@
+*.js.map
+*.ts
+.git*
+.vscode
+__azurite_db*__.json
+__blobstorage__
+__queuestorage__
+local.settings.json
+test
+tsconfig.json