Call scraper lambda and save results to db

.env.example

@@ -1,2 +1 @@
 DATABASE_URL=
-LAMBDA_URL=

prisma/migrations/20240523092504_add_scrape_success_and_info/migration.sql

@@ -0,0 +1,14 @@
+/*
+ Warnings:
+
+ - You are about to drop the column `imageBase64` on the `ProductScraperHistory` table. All the data in the column will be lost.
+ - Added the required column `imageUrl` to the `ProductScraperHistory` table without a default value. This is not possible if the table is not empty.
+ - Added the required column `scrapeInfo` to the `ProductScraperHistory` table without a default value. This is not possible if the table is not empty.
+ - Added the required column `successfulScrape` to the `ProductScraperHistory` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "ProductScraperHistory" DROP COLUMN "imageBase64",
+ADD COLUMN "imageUrl" TEXT NOT NULL,
+ADD COLUMN "scrapeInfo" TEXT NOT NULL,
+ADD COLUMN "successfulScrape" BOOLEAN NOT NULL;

prisma/migrations/20240523092616_add_scrape_info_can_empty/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "ProductScraperHistory" ALTER COLUMN "scrapeInfo" DROP NOT NULL;

prisma/migrations/20240523092846_add_price_stock_can_empty/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "ProductScraperHistory" ALTER COLUMN "price" DROP NOT NULL,
+ALTER COLUMN "inStock" DROP NOT NULL;

prisma/migrations/20240523114429_add_image_xpath/migration.sql

@@ -0,0 +1,6 @@
+-- AlterTable
+ALTER TABLE "ProductScraper" ADD COLUMN "imageXPATH" TEXT NOT NULL DEFAULT '';
+
+-- AlterTable
+ALTER TABLE "ProductScraperHistory" ALTER COLUMN "imageUrl" DROP NOT NULL,
+ALTER COLUMN "imageUrl" SET DEFAULT 'https://loremflickr.com/1000/1000/cat';

prisma/migrations/20240523114540_remove_image_xpath_default/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "ProductScraper" ALTER COLUMN "imageXPATH" DROP DEFAULT;

prisma/migrations/20240524194123_refactor/migration.sql

@@ -0,0 +1,46 @@
+/*
+ Warnings:
+
+ - You are about to drop the column `imageXPATH` on the `ProductScraper` table. All the data in the column will be lost.
+ - You are about to drop the column `inStockString` on the `ProductScraper` table. All the data in the column will be lost.
+ - You are about to drop the column `outOfStockString` on the `ProductScraper` table. All the data in the column will be lost.
+ - You are about to drop the column `priceXPATH` on the `ProductScraper` table. All the data in the column will be lost.
+ - You are about to drop the column `url` on the `ProductScraper` table. All the data in the column will be lost.
+ - You are about to drop the column `currencyType` on the `ScraperLambda` table. All the data in the column will be lost.
+ - Added the required column `url` to the `Product` table without a default value. This is not possible if the table is not empty.
+ - Added the required column `currency` to the `ProductScraper` table without a default value. This is not possible if the table is not empty.
+ - Added the required column `scraperPropertiesId` to the `ProductScraper` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "Product" ADD COLUMN "url" TEXT NOT NULL;
+
+-- AlterTable
+ALTER TABLE "ProductScraper" DROP COLUMN "imageXPATH",
+DROP COLUMN "inStockString",
+DROP COLUMN "outOfStockString",
+DROP COLUMN "priceXPATH",
+DROP COLUMN "url",
+ADD COLUMN "currency" TEXT NOT NULL,
+ADD COLUMN "scraperPropertiesId" INTEGER NOT NULL;
+
+-- AlterTable
+ALTER TABLE "ScraperLambda" DROP COLUMN "currencyType";
+
+-- CreateTable
+CREATE TABLE "ScraperProperties" (
+ "id" SERIAL NOT NULL,
+ "inStockString" TEXT NOT NULL,
+ "outOfStockString" TEXT NOT NULL,
+ "priceXPATH" TEXT NOT NULL,
+ "imageXPATH" TEXT NOT NULL,
+ "companyId" INTEGER,
+
+ CONSTRAINT "ScraperProperties_pkey" PRIMARY KEY ("id")
+);
+
+-- AddForeignKey
+ALTER TABLE "ScraperProperties" ADD CONSTRAINT "ScraperProperties_companyId_fkey" FOREIGN KEY ("companyId") REFERENCES "Company"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ProductScraper" ADD CONSTRAINT "ProductScraper_scraperPropertiesId_fkey" FOREIGN KEY ("scraperPropertiesId") REFERENCES "ScraperProperties"("id") ON DELETE RESTRICT ON UPDATE CASCADE;

prisma/migrations/20240524195435_rename_image_add_names/migration.sql

@@ -0,0 +1,17 @@
+/*
+ Warnings:
+
+ - You are about to drop the column `imageUrl` on the `ProductScraperHistory` table. All the data in the column will be lost.
+ - Added the required column `name` to the `ProductScraper` table without a default value. This is not possible if the table is not empty.
+ - Added the required column `name` to the `ScraperProperties` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "ProductScraper" ADD COLUMN "name" TEXT NOT NULL;
+
+-- AlterTable
+ALTER TABLE "ProductScraperHistory" DROP COLUMN "imageUrl",
+ADD COLUMN "imageBase64" TEXT;
+
+-- AlterTable
+ALTER TABLE "ScraperProperties" ADD COLUMN "name" TEXT NOT NULL;

prisma/schema.prisma

@@ -8,15 +8,29 @@ datasource db {
 }
 
 model Company {
- id Int @id @default(autoincrement())
- name String
- slug String @unique
- Products Product[]
+ id Int @id @default(autoincrement())
+ name String
+ slug String @unique
+ Products Product[]
+ ScraperProperties ScraperProperties[]
+}
+
+model ScraperProperties {
+ id Int @id @default(autoincrement())
+ name String
+ inStockString String
+ outOfStockString String
+ priceXPATH String
+ imageXPATH String
+ companyId Int?
+ Company Company? @relation(fields: [companyId], references: [id])
+ ProductScraper ProductScraper[]
 }
 
 model Product {
  id Int @id @default(autoincrement())
  name String
+ url String
  created DateTime @default(now())
  slug String @unique
  companyId Int
@@ -26,12 +40,12 @@ model Product {
 
 model ProductScraper {
  id Int @id @default(autoincrement())
- url String
- inStockString String
- outOfStockString String
- priceXPATH String
+ name String
+ currency String
  productId Int
  product Product @relation(fields: [productId], references: [id])
+ scraperPropertiesId Int
+ ScraperProperties ScraperProperties @relation(fields: [scraperPropertiesId], references: [id])
  scraperLambdaId Int
  ScraperLambda ScraperLambda @relation(fields: [scraperLambdaId], references: [id])
  ProductScraperHistory ProductScraperHistory[]
@@ -41,16 +55,17 @@ model ScraperLambda {
  id Int @id @default(autoincrement())
  url String @unique
  region String
- currencyType String
  ProductScraper ProductScraper[]
 }
 
 model ProductScraperHistory {
  id Int @id @default(autoincrement())
- price Float
- inStock Boolean
+ price Float?
+ inStock Boolean?
+ imageBase64 String?
  created DateTime @default(now())
- imageBase64 String
+ successfulScrape Boolean
+ scrapeInfo String?
  productScraperId Int
  ProductScraper ProductScraper @relation(fields: [productScraperId], references: [id])
 }

src/app/company/[company_slug]/products/[product_slug]/getProduct.ts

@@ -1,4 +1,4 @@
-import prisma from '@/app/lib/db';
+import prisma from '@/lib/db';
 import type { Product } from '@prisma/client';
 
 export async function fetchProductData(
@@ -17,11 +17,6 @@ export async function fetchProductData(
  },
  take: 25,
  },
- ScraperLambda: {
- select: {
- currencyType: true,
- },
- },
  },
  },
  },

src/app/company/[company_slug]/products/[product_slug]/page.tsx

@@ -1,7 +1,7 @@
 import type { Product as ProductType } from '@prisma/client';
 import { notFound } from 'next/navigation';
 import { fetchProductData } from './getProduct';
-import Product from '@/app/components/Product';
+import Product from '@/components/Product';
 
 export const metadata = {
  title: '',

src/app/layout.tsx

@@ -1,8 +1,8 @@
 import type { Metadata } from 'next';
 import { Inter } from 'next/font/google';
 import './globals.css';
-import Nav from './components/Nav';
-import Footer from './components/Footer';
+import Nav from '../components/Nav';
+import Footer from '../components/Footer';
 
 const inter = Inter({ subsets: ['latin'] });
 

src/app/components/Product.tsx → src/components/Product.tsx

@@ -1,4 +1,4 @@
-import { Product as ProductType } from '@prisma/client';
+/* eslint-disable @next/next/no-img-element */
 import { Overpass_Mono } from 'next/font/google';
 
 const overpassMono = Overpass_Mono({ subsets: ['latin'] });
@@ -13,6 +13,7 @@ export default function Product({ product }: any) {
  src={
  product.ProductScraper[0].ProductScraperHistory[0].imageBase64
  }
+ alt={'Image of product named ' + product.name}
  />
  </div>
  </a>
@@ -37,7 +38,7 @@ export default function Product({ product }: any) {
  >
  <h1>
  <span className='font-semi-bold text-gray-800'> Price </span>
- {product.ProductScraper[0].ScraperLambda.currencyType}
+ {product.ProductScraper[0].currency}
  {product.ProductScraper[0].ProductScraperHistory[0].price}
  </h1>
  <h2 className='text-sm'>