An initial version of ResultManager. Write the SQL for use with gener…

…atec.

app/api/experiment.go

@@ -13,6 +13,9 @@ type Experiment struct {
 }
 
 type ExperimentSpec struct {
+	// AgentAddress is the address of the agent to use to generate completions
+	AgentAddress string `json:"agentAddress" yaml:"agentAddress"`
+
 	// EvalDir is the directory containing the evaluation the evaluation input
 	EvalDir string `json:"evalDir" yaml:"evalDir"`
 

app/pkg/analyze/fsql/eval_query.sql

@@ -0,0 +1,9 @@
+-- name: UpdateResult :exec
+INSERT OR REPLACE INTO results
+(id, time, proto_json)
+VALUES
+(?, ?, ?);
+
+-- name: GetResult :one
+SELECT * FROM results
+WHERE id = ?;

app/pkg/analyze/fsql/query.sql

@@ -20,4 +20,4 @@ WHERE contextID = ?;
 INSERT OR REPLACE INTO sessions 
 (contextID, startTime, endTime, selectedId, selectedKind, total_input_tokens, total_output_tokens, num_generate_traces, proto)
 VALUES 
-(?, ?, ?, ?, ?, ?, ?, ?, ?);
+(?, ?, ?, ?, ?, ?, ?, ?, ?);

app/pkg/analyze/fsql/schema.sql

@@ -23,3 +23,14 @@ CREATE TABLE IF NOT EXISTS sessions (
     -- TODO(jeremy): Should we store the proto in JSON format so that we can run SQL queries on values in it?
     proto BLOB
 );
+
+-- Results contains evaluation results
+CREATE TABLE IF NOT EXISTS results (
+    id VARCHAR(255) PRIMARY KEY,
+    -- time is the time of the evaluation example
+    -- protobufs can't have null timestamps so no point allowing nulls
+    time TIMESTAMP NOT NULL,
+
+    -- The JSON serialization of the proto.
+    proto_json TEXT NOT NULL
+);

app/pkg/analyze/fsql/sqlc.yaml

@@ -1,7 +1,9 @@
 version: "2"
 sql:
   - engine: "sqlite"
-    queries: "query.sql"
+    queries:
+      - "eval_query.sql"
+      - "query.sql"
     schema: "schema.sql"
     gen:
       go:

app/pkg/analyze/session_manager.go

@@ -283,6 +283,10 @@ func (m *SessionsManager) DumpExamples(ctx context.Context, request *connect.Req
 }
 
 // protoToRow converts from the proto representation of a session to the database row representation.
+//
+// TODO(jeremy): I think it would be better to make the return type fsql.UpdateSessionParams. Right now the only
+// place this function gets called is in the Update method and the returned value is immediately converted to
+// fsql.UpdateSessionParams.
 func protoToRow(session *logspb.Session) (*fsql.Session, error) {
 	log := logs.NewLogger()
 	protoBytes, err := proto.Marshal(session)
@@ -303,7 +307,6 @@ func protoToRow(session *logspb.Session) (*fsql.Session, error) {
 		}
 	}
 
-	// TODO: How do we deal with the end/starttime? In sqlc should we specify the type as timestamp?
 	return &fsql.Session{
 		Contextid:         session.ContextId,
 		Starttime:         session.StartTime.AsTime(),