Merge pull request #186 from mikepsinn/develop

text2measurements & image2measurements endpoints

apps/nextjs/app/api/chat-with-vision/route.ts

@@ -0,0 +1,44 @@
+// ./app/api/chat/route.ts
+import OpenAI from 'openai';
+import { OpenAIStream, StreamingTextResponse } from 'ai';
+
+// Create an OpenAI API client (that's edge friendly!)
+const openai = new OpenAI({
+  apiKey: process.env.OPENAI_API_KEY || '',
+});
+
+// IMPORTANT! Set the runtime to edge
+export const runtime = 'edge';
+
+export async function POST(req: Request) {
+  // Extract the `prompt` from the body of the request
+  const { messages, data } = await req.json();
+
+  const initialMessages = messages.slice(0, -1);
+  const currentMessage = messages[messages.length - 1];
+
+  // Ask OpenAI for a streaming chat completion given the prompt
+  const response = await openai.chat.completions.create({
+    model: 'gpt-4-vision-preview',
+    stream: true,
+    max_tokens: 150,
+    messages: [
+      ...initialMessages,
+      {
+        ...currentMessage,
+        content: [
+          { type: 'text', text: currentMessage.content },
+          {
+            type: 'image_url',
+            image_url: data.imageUrl,
+          },
+        ],
+      },
+    ],
+  });
+
+  // Convert the response into a friendly text-stream
+  const stream = OpenAIStream(response);
+  // Respond with the stream
+  return new StreamingTextResponse(stream);
+}

apps/nextjs/app/api/image2measurements/route.ts

@@ -0,0 +1,160 @@
+/**
+ * API Route - Image Processing
+ *
+ * This API route is designed for processing images within an application using the OpenAI API.
+ * It handles the reception of an image file (in base64 format) and an optional custom prompt through a POST request.
+ * The route then sends this data to OpenAI for analysis, typically involving image description or any other
+ * relevant vision-based task. The response from OpenAI, containing the analysis of the image, is then returned
+ * to the user.
+ *
+ * Path: /api/image2measurements
+ */
+
+import { NextRequest, NextResponse } from 'next/server';
+import OpenAI from "openai";
+
+// Initialize the OpenAI client with the API key. This key is essential for authenticating
+// the requests with OpenAI's API services.
+const openai = new OpenAI({
+  apiKey: process.env.OPENAI_API_KEY,
+});
+
+export async function POST(request: NextRequest) {
+  // Logging the start of the image processing API call
+  console.log('Starting the image processing API call');
+
+  // Extracting the file (in base64 format) and an optional custom prompt
+  // from the request body. This is essential for processing the image using OpenAI's API.
+  const { file: base64Image, prompt: customPrompt, detail, max_tokens } = await request.json();
+
+  // Check if the image file is included in the request. If not, return an error response.
+  if (!base64Image) {
+    console.error('No file found in the request');
+    return NextResponse.json({ success: false, message: 'No file found' });
+  }
+
+  // Log the receipt of the image in base64 format
+  console.log('Received image in base64 format');
+
+  // Utilize the provided custom prompt or a default prompt if it's not provided.
+  // This prompt guides the analysis of the image by OpenAI's model.
+  let promptText = `
+  Analyze the provided image and estimate the macro and micronutrient content of any food items, and extract data about any medications or nutritional supplements present. Return the results as an array of structured JSON data with the following format:
+
+[
+  {
+    "type": "food",
+    "food_item": "string",
+    "serving_size": "string",
+    "calories": number,
+    "macronutrients": {
+      "protein": {
+        "value": number,
+        "unit": "string"
+      },
+      "carbohydrates": {
+        "value": number,
+        "unit": "string"
+      },
+      "fat": {
+        "value": number,
+        "unit": "string"
+      }
+    },
+    "micronutrients": [
+      {
+        "name": "string",
+        "value": number,
+        "unit": "string"
+      },
+      ...
+    ]
+  },
+  {
+    "type": "medication",
+    "name": "string",
+    "dosage": "string",
+    "frequency": "string",
+    "purpose": "string"
+  },
+  {
+    "type": "supplement",
+    "name": "string",
+    "brand": "string",
+    "dosage": "string",
+    "ingredients": [
+      {
+        "name": "string",
+        "amount": "string"
+      },
+      ...
+    ]
+  },
+  ...
+]
+
+For food items:
+- The "type" field should be set to "food".
+- The "food_item", "serving_size", "calories", "macronutrients", and "micronutrients" fields should be populated as described in the previous prompt.
+
+For medications:
+- The "type" field should be set to "medication".
+- The "name" field should contain the name of the medication.
+- The "dosage" field should specify the dosage information.
+- The "frequency" field should indicate how often the medication should be taken.
+- The "purpose" field should briefly describe the purpose or condition the medication is intended to treat.
+
+For nutritional supplements:
+- The "type" field should be set to "supplement".
+- The "name" field should contain the name of the supplement.
+- The "brand" field should specify the brand or manufacturer of the supplement, if available.
+- The "dosage" field should indicate the recommended dosage.
+- The "ingredients" array should contain objects representing the ingredients and their amounts in the supplement.
+
+Please provide the JSON output without any additional text or explanations.
+  `
+  // Log the chosen prompt
+  console.log(`Using prompt: ${promptText}`);
+
+  // Sending the image and prompt to OpenAI for processing. This step is crucial for the image analysis.
+  console.log('Sending request to OpenAI');
+  try {
+    const response = await openai.chat.completions.create({
+      model: "gpt-4-turbo",
+      messages: [
+        {
+          role: "user",
+          content: [
+            { type: "text", text: promptText },
+            {
+              type: "image_url",
+              image_url: {
+                url: base64Image,
+                ...(detail && { detail: detail }) // Include the detail field only if it exists
+              }
+            }
+          ]
+        }
+      ],
+      response_format: {
+        type: "json_object"
+      },
+      max_tokens: max_tokens
+    });
+
+    // Log the response received from OpenAI, which includes the analysis of the image.
+    console.log('Received response from OpenAI');
+    //console.log('Response:', JSON.stringify(response, null, 2)); // Log the response for debugging
+
+    // Extract and log the analysis from the response
+    const analysis = response?.choices[0]?.message?.content;
+    console.log('Analysis:', analysis);
+
+    // Return the analysis in the response
+    return NextResponse.json({ success: true, analysis: analysis });
+  } catch (error) {
+    // Log and handle any errors encountered during the request to OpenAI
+    console.error('Error sending request to OpenAI:', error);
+    return NextResponse.json({ success: false, message: 'Error sending request to OpenAI' });
+  }
+}

apps/nextjs/app/api/text2measurements/measurementSchema.ts

@@ -0,0 +1,170 @@
+export const VariableCategoryNames = [
+  'Emotions',
+  'Physique',
+  'Physical Activity',
+  'Locations',
+  'Miscellaneous',
+  'Sleep',
+  'Social Interactions',
+  'Vital Signs',
+  'Cognitive Performance',
+  'Symptoms',
+  'Nutrients',
+  'Goals',
+  'Treatments',
+  'Activities',
+  'Foods',
+  'Conditions',
+  'Environment',
+  'Causes of Illness',
+  'Books',
+  'Software',
+  'Payments',
+  'Movies and TV',
+  'Music',
+  'Electronics',
+  'IT Metrics',
+  'Economic Indicators',
+  'Investment Strategies'
+] as const;  // 'as const' makes it a readonly array
+
+// Then you can use this array to define your type:
+export type VariableCategoryName = typeof VariableCategoryNames[number];  // This will be a union of the array values
+
+export const UnitNames = [
+  'per Minute',
+  'Yes/No',
+  'Units',
+  'Torr',
+  'Tablets',
+  'Sprays',
+  'Serving',
+  'Seconds',
+  'Quarts',
+  'Puffs',
+  'Pounds',
+  'Pills',
+  'Pieces',
+  'Percent',
+  'Pascal',
+  'Parts per Million',
+  'Ounces',
+  'Minutes',
+  'Milliseconds',
+  'Millimeters Merc',
+  'Millimeters',
+  'Milliliters',
+  'Milligrams',
+  'Millibar',
+  'Miles per Hour',
+  'Miles',
+  'Micrograms per decilitre',
+  'Micrograms',
+  'Meters per Second',
+  'Meters',
+  'Liters',
+  'Kilometers',
+  'Kilograms',
+  'Kilocalories',
+  'International Units',
+  'Index',
+  'Inches',
+  'Hours',
+  'Hectopascal',
+  'Grams',
+  'Gigabecquerel',
+  'Feet',
+  'Event',
+  'Drops',
+  'Doses',
+  'Dollars',
+  'Degrees North',
+  'Degrees Fahrenheit',
+  'Degrees East',
+  'Degrees Celsius',
+  'Decibels',
+  'Count',
+  'Centimeters',
+  'Capsules',
+  'Calories',
+  'Beats per Minute',
+  'Applications',
+  '1 to 5 Rating',
+  '1 to 3 Rating',
+  '1 to 10 Rating',
+  '0 to 5 Rating',
+  '0 to 1 Rating',
+  '-4 to 4 Rating',
+  '% Recommended Daily Allowance'
+] as const;  // 'as const' makes it a readonly array
+
+// Then you can use this array to define your type:
+export type UnitName = typeof UnitNames[number];  // This will be a union of the array values
+
+
+// a set of measurements logged by the user
+export type MeasurementSet = {
+  measurements: (Measurement | UnknownText)[];
+};
+
+export interface Measurement {
+  itemType: 'measurement',
+  variableName: string;  // variableName is the name of the treatment, symptom, food, drink, etc.
+  // For example, if the answer is "I took 5 mg of NMN", then this variableName is "NMN".
+  // For example, if the answer is "I have been having trouble concentrating today", then this variableName is "Concentration".
+  value: number; // value is the number of units of the treatment, symptom, food, drink, etc.
+  // For example, if the answer is "I took 5 mg of NMN", then this value is 5.
+  // For example, if the answer is "I have been feeling very tired and fatigued today", you would return two measurements
+  // with the value 5 like this:
+  // {variableName: "Tiredness", value: 5, unitName: "1 to 5 Rating", startAt: "00:00:00", endAt: "23:59:59", combinationOperation: "MEAN", variableCategoryName: "Symptoms"}
+  // {variableName: "Fatigue", value: 5, unitName: "1 to 5 Rating", startAt: "00:00:00", endAt: "23:59:59", combinationOperation: "MEAN", variableCategoryName: "Symptoms"}
+  // For example, if the answer is "I have been having trouble concentrating today", then this value is 1 and the object
+  // would be {variableName: "Concentration", value: 1, unitName: "1 to 5 Rating", startAt: "00:00:00", endAt: "23:59:59", combinationOperation: "MEAN", variableCategoryName: "Symptoms"}
+  // For example, if the answer is "I also took magnesium 200mg, Omega3 one capsule 500mg", then the measurements would be:
+  // {variableName: "Magnesium", value: 200, unitName: "Milligrams", startAt: "00:00:00", endAt: "23:59:59", combinationOperation: "SUM", variableCategoryName: "Treatments"}
+  // {variableName: "Omega3", value: 500, unitName: "Milligrams", startAt: "00:00:00", endAt: "23:59:59", combinationOperation: "SUM", variableCategoryName: "Treatments"}
+  unitName: UnitName;
+  // unitName is the unit of the treatment, symptom, food, drink, etc.
+  // For example, if the answer is "I took 5 mg of NMN", then this unitName is "Milligrams".
+  startDateLocal: string|null;  // startDate should be the date the measurement was taken in the format "YYYY-MM-DD" or null if no date is known
+  startTimeLocal: string|null;  // startAt should be the time the measurement was taken in
+  // the format "HH:MM:SS".  For instance, midday would be "12:00:00".
+  // ex. The term `breakfast` would be a typical breakfast time of "08:00:00".
+  // ex. The term `lunch` would be a typical lunchtime of "12:00:00".
+  // ex. The term `dinner` would be a typical dinner time of "18:00:00".
+  // If no time or date is known, then startTime should be null.
+  endDateLocal: string|null;
+  endTimeLocal: string|null;
+  // If a time range is given, then endAt should be the end of that period. It should also be in the format "HH:MM:SS".
+  combinationOperation: "SUM" | "MEAN"; // combinationOperation is the operation used to combine multiple measurements of the same variableName
+  variableCategoryName: VariableCategoryName; // variableCategoryName is the category of the variableName
+  // For example, if the answer is "I took 5 mg of NMN", then this variableCategoryName is "Treatments".
+  originalText: string; // the text fragment that was used to create this measurement
+}
+
+// Use this type for measurement items that match nothing else
+export interface UnknownText {
+  itemType: 'unknown',
+  text: string; // The text that wasn't understood
+}
+
+export type Food = Measurement & {
+  variableCategoryName: "Foods";
+  combinationOperation: "SUM";
+};
+
+export type Drink = Measurement & {
+  variableCategoryName: "Foods";
+  combinationOperation: "SUM";
+};
+
+export type Treatment = Measurement & {
+  variableCategoryName: "Treatments";
+  combinationOperation: "SUM";
+};
+
+export type Symptom = Measurement & {
+  variableCategoryName: "Symptoms";
+  combinationOperation: "MEAN";
+  unitName: '/5';
+};