Merge pull request #1 from ZanzyTHEbar/main

Bring feature branch up to date with main

Makefile

@@ -0,0 +1,75 @@
+# Go parameters
+GOCMD = go
+GOBUILD = $(GOCMD) build
+GOCLEAN = $(GOCMD) clean
+GOTEST = $(GOCMD) test
+GOGET = $(GOCMD) get
+
+# Main package name
+MAIN_PACKAGE = main
+
+# Output binary name
+BINARY_NAME = plandex
+
+# Check the PLANDEX_ENVIRONMENT environment variable, reassign the BINARY_NAME if necessary
+ifeq ($(PLANDEX_ENV),development)
+BINARY_NAME = plandex-dev
+endif
+
+# create a dev cmd that runs a shell script
+dev:
+	@cd app/scripts && ./dev.sh
+
+# Build target
+build:
+	@$(GOBUILD) -o $(BINARY_NAME) -v $(MAIN_PACKAGE)
+
+# Clean target
+clean:
+	@$(GOCLEAN)
+	@rm -f $(BINARY_NAME)
+
+# Test target
+test: render
+	@$(GOTEST) -v ./...
+
+#### Evals and Providers ####
+
+# TODO: Implement eval all
+eval:
+	@cd test/evals/promptfoo-poc/$(filter-out $@,$(MAKECMDGOALS)) && promptfoo eval --no-cache
+
+view-eval:
+	@cd test/evals/promptfoo-poc/$(filter-out $@,$(MAKECMDGOALS)) && promptfoo view
+
+gen-eval:
+	@$(GOCMD) run app/scripts/cmd/gen/gen.go test/evals/promptfoo-poc/$(filter-out $@,$(MAKECMDGOALS))
+
+gen-provider:
+	@$(GOCMD) run app/scripts/cmd/provider/gen_provider.go
+
+#### End Evals and Providers ####
+
+# Get dependencies
+deps:
+	$(GOGET) -v ./...
+
+# Default target
+default: build
+
+# Usage
+help:
+	@echo "Usage:"
+	@echo "  make dev - to run the development scripts"
+	@echo "  make eval <directory_name> - to run the promptfoo eval command on a specific directory"
+	@echo "  make view-eval - to view the promptfoo eval output"
+	@echo "  make gen-eval <directory_name> - to create a new promptfoo eval directory structure"
+	@echo "  make gen-provider - to create a new promptfoo provider file from the promptfoo diretory structure"
+	@echo "  make clean - to remove generated files and directories"
+	@echo "  make help - to display this help message"
+
+# Prevents make from interpreting the arguments as targets
+%:
+	@:
+
+.PHONY: all render build clean test deps

app/cli/dev.sh

@@ -4,8 +4,9 @@ OUT="${PLANDEX_DEV_CLI_OUT_DIR:-/usr/local/bin}"
 NAME="${PLANDEX_DEV_CLI_NAME:-plandex-dev}"
 ALIAS="${PLANDEX_DEV_CLI_ALIAS:-pdxd}"
 
-go build -o $NAME && \
-rm -f $OUT/$NAME && \
-cp $NAME $OUT/$NAME && \
-ln -sf $OUT/$NAME $OUT/$ALIAS && \
-echo built $NAME cli and added $ALIAS alias to $OUT
+# Double quote to prevent globbing and word splitting.
+sudo go build -o "$NAME" &&
+    sudo rm -f "$OUT"/"$NAME" &&
+    sudo cp "$NAME" "$OUT"/"$NAME" &&
+    sudo ln -sf "$OUT"/"$NAME" "$OUT"/"$ALIAS" &&
+    echo built "$NAME" cli and added "$ALIAS" alias to "$OUT"

app/scripts/cmd/gen/gen.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"text/template"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		log.Fatalf("Usage: %s <path/to/directory>", os.Args[0])
+	}
+
+	dirPath := os.Args[1]
+	dirName := filepath.Base(dirPath)
+
+	// Create the main directory
+	if err := os.MkdirAll(dirPath, 0755); err != nil {
+		log.Fatalf("Error creating directory: %s", err)
+	}
+
+	f, err := os.Create(fmt.Sprintf("%s/%s", dirPath, "promptfooconfig.yaml"))
+	if err != nil {
+		log.Fatalf("Error creating file: %s", err)
+	}
+	f.Close()
+
+	// Create files inside the directory
+	files := []string{
+		"parameters.json",
+		"config.properties",
+		"prompt.txt",
+	}
+
+	for _, file := range files {
+		f, err := os.Create(fmt.Sprintf("%s/%s.%s", dirPath, dirName, file))
+		if err != nil {
+			log.Fatalf("Error creating file: %s", err)
+		}
+		f.Close()
+	}
+
+	// Create assets and tests directories
+	subDirs := []string{"assets", "tests"}
+
+	for _, subDir := range subDirs {
+		if err := os.Mkdir(fmt.Sprintf("%s/%s", dirPath, subDir), 0755); err != nil {
+			log.Fatalf("Error creating subdirectory: %s", err)
+		}
+	}
+
+	// Template for promptfooconfig.yaml
+	ymlTemplate := `description: "{{ .Name }}"
+
+prompts:
+  - file://{{ .Name }}.prompt.txt
+
+providers:
+  - file://{{ .Name }}.provider.yml
+
+tests: tests/*.tests.yml
+`
+
+	// Populate promptfooconfig.yaml
+	promptFooConfigTmpl, err := template.New("yml").Parse(ymlTemplate)
+	if err != nil {
+		log.Fatalf("Error creating template: %s", err)
+	}
+
+	// Template for config.properties
+	propertiesTemplate := `provider_id=openai:gpt-4o
+function_name=
+tool_type=function
+function_param_type=object
+tool_choice_type=function
+tool_choice_function_name=
+nested_parameters_json={{ .Name }}.parameters.json
+`
+
+	// Populate config.properties
+	configPropertiesTmpl, err := template.New("properties").Parse(propertiesTemplate)
+	if err != nil {
+		log.Fatalf("Error creating template: %s", err)
+	}
+
+	configFile, err := os.Create(fmt.Sprintf("%s/%s.%s", dirPath, dirName, "config.properties"))
+	if err != nil {
+		log.Fatalf("Error creating config.properties: %s", err)
+	}
+	defer configFile.Close()
+
+	file, err := os.Create(fmt.Sprintf("%s/promptfooconfig.yaml", dirPath))
+	if err != nil {
+		log.Fatalf("Error creating promptfooconfig.yaml: %s", err)
+	}
+	defer file.Close()
+
+	data := struct {
+		Name string
+	}{
+		Name: dirName,
+	}
+
+	if err := promptFooConfigTmpl.Execute(file, data); err != nil {
+		log.Fatalf("Error executing template: %s", err)
+	}
+
+	if err := configPropertiesTmpl.Execute(configFile, data); err != nil {
+		log.Fatalf("Error executing template: %s", err)
+	}
+
+	fmt.Println("Directory created successfully!")
+	fmt.Println("Please check the contents of the directory and proceed with the implementation.")
+}

app/scripts/cmd/provider/gen_provider.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+)
+
+var testDir = "test/evals/promptfoo-poc"
+var templFile = testDir + "/templates/" + "/provider.template.yml"
+
+func main() {
+
+	testAbsPath, _ := filepath.Abs(testDir)
+	templAbsPath, _ := filepath.Abs(templFile)
+
+	// Function to walk through directories and find required values
+	err := filepath.Walk(testAbsPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && filepath.Ext(path) == ".properties" {
+			dirName := filepath.Base(filepath.Dir(path))
+			outputFileName := filepath.Join(filepath.Dir(path), dirName+".provider.yml")
+
+			// Read the template file
+			templateContent, err := os.ReadFile(templAbsPath)
+			if err != nil {
+				log.Fatalf("Error reading template file: %v", err)
+			}
+
+			// Prepare variables (this assumes properties file is a simple key=value format)
+			variables := map[string]interface{}{}
+			properties, err := os.ReadFile(path)
+			if err != nil {
+				log.Fatalf("Error reading properties file: %v", err)
+			}
+			for _, line := range strings.Split(string(properties), "\n") {
+				if len(line) == 0 {
+					continue
+				}
+				parts := strings.SplitN(line, "=", 2)
+
+				if len(parts) > 2 {
+					log.Fatalf("Invalid line in properties file: %s", line)
+				}
+
+				if len(parts) < 2 {
+					log.Fatalf("Invalid line in properties file: %s", line)
+				}
+
+				key := strings.TrimSpace(parts[0])
+				value := strings.TrimSpace(parts[1])
+
+				if key != "nested_parameters_json" {
+					variables[key] = value
+					continue
+				}
+
+				// Read the file path from the nested_parameters_json key
+				parametersJsonFile := filepath.Join(filepath.Dir(path), value)
+				jsonParameters, err := os.ReadFile(parametersJsonFile)
+				if err != nil {
+					log.Fatalf("Error reading nested parameters JSON file: %v", err)
+				}
+				// Parse the JSON string
+				var nestedParameters map[string]interface{}
+
+				// We marshal and unmarshal the JSON to ensure that the nested properties are properly formatted 
+				// for the template, and to ensure that the data is correct json
+
+				err = json.Unmarshal(jsonParameters, &nestedParameters)
+
+				if err != nil {
+					log.Fatalf("Error un-marshalling nested parameters JSON: %v", err)
+				}
+
+				parameters, err := json.Marshal(nestedParameters)
+				if err != nil {
+					log.Fatalf("Error marshalling nested parameters JSON: %v", err)
+				}
+
+				// Add the nested properties to the variables
+				variables["parameters"] = string(parameters)
+			}
+
+			// Parse and execute the template
+			tmpl, err := template.New("yamlTemplate").Parse(string(templateContent))
+			if err != nil {
+				log.Fatalf("Error parsing template: %v", err)
+			}
+			outputFile, err := os.Create(outputFileName)
+			if err != nil {
+				log.Fatalf("Error creating output file: %v", err)
+			}
+			defer outputFile.Close()
+
+			err = tmpl.Execute(outputFile, variables)
+			if err != nil {
+				log.Fatalf("Error executing template: %v", err)
+			}
+			log.Printf("Template rendered and saved to '%s'", outputFileName)
+		}
+		return nil
+	})
+	if err != nil {
+		log.Fatalf("Error walking the path: %v", err)
+	}
+}

app/scripts/dev.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Detect zsh and trigger it if its the shell
+if [ -n "$ZSH_VERSION" ]; then
+  # shell is zsh
+  echo "Detected zsh"
+  zsh -c "source ~/.zshrc && $*"
+fi
+
+# Detect if reflex is installed and install it if not
+if ! [ -x "$(command -v reflex)" ]; then
+
+  # Check if the $GOPATH is empty
+  if [ -z "$GOPATH" ]; then
+    echo "Error: $GOPATH is not set. Please set it to continue..." >&2
+    exit 1
+  fi
+
+  echo 'Error: reflex is not installed. Installing it now...' >&2
+  go get -u github.com/cespare/reflex
+fi
+
+terminate() {
+  pkill -f 'plandex-server' # Assuming plandex-server is the name of your process
+  kill -TERM "$pid1" 2>/dev/null
+  kill -TERM "$pid2" 2>/dev/null
+}
+
+trap terminate SIGTERM SIGINT
+
+(cd .. && cd cli && ./dev.sh)
+
+reflex -r '^(cli|shared)/.*\.(go|mod|sum)$' -- sh -c 'cd cli && ./dev.sh' &
+pid1=$!
+
+reflex -r '^(server|shared)/.*\.(go|mod|sum)$' -s -- sh -c 'cd server && go build && ./plandex-server' &
+pid2=$!
+
+wait $pid1
+wait $pid2