From 8525e4efee67ac49bec06f1b137002e3c6b1a502 Mon Sep 17 00:00:00 2001 From: Zach Lyon Date: Tue, 29 Oct 2024 15:44:17 -0400 Subject: [PATCH 1/6] Add headless browser script --- .../run-script-in-headless-browser/README.md | 17 +++++++ .../run-script-in-headless-browser.js | 48 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/javascript-sdk/examples/run-script-in-headless-browser/README.md b/javascript-sdk/examples/run-script-in-headless-browser/README.md index e69de29..a1cf5b6 100644 --- a/javascript-sdk/examples/run-script-in-headless-browser/README.md +++ b/javascript-sdk/examples/run-script-in-headless-browser/README.md @@ -0,0 +1,17 @@ +# Example script: run the script in headless browser with AgentQL + +This example demonstrates how to run the script in headless browser. + +## Run the script + +- [Install AgentQL SDK](https://docs.agentql.com/javascript-sdk/installation) +- Save this Javascript file locally as **run_in_headless_browser.js** +- Run the following command from the project's folder: + +```bash +node run_in_headless_browser.js +``` + +## Play with the query + +Install the [AgentQL Debugger Chrome extension](https://docs.agentql.com/installation/chrome-extension-installation) to play with the AgentQL query. [Learn more about the AgentQL query language](https://docs.agentql.com/agentql-query/query-intro) diff --git a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js index e69de29..89bad3e 100644 --- a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js +++ b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js @@ -0,0 +1,48 @@ +/* This example demonstrates how to run the script in a headless browser. */ + +const { wrap, configure } = require('agentql'); +const { chromium } = require('playwright'); + +// Define the URL of the page to scrape. +const URL = "https://scrapeme.live/shop/"; + +// Define the queries to locate the search box and fetch the stock number. +const SEARCH_QUERY = ` +{ + search_products_box +} +`; + +const STOCK_NUMBER_QUERY = ` +{ + number_in_stock +} +`; + + +(async () => { + // Set the AgentQL API key via the `configure` method. + configure({ apiKey: process.env.AGENTQL_API_KEY }); + + // Launch a headless browser using Playwright. + const browser = await chromium.launch({ headless: true }); + + // Wrap a new page to use the AgentQL API. + const page = wrap(await browser.newPage()); + + await page.goto(URL); + + let response; + // Use queryElements() method to locate the search box from the page. + response = await page.queryElements(SEARCH_QUERY); + + // Use Playwright's API to fill the search box and press Enter. + await response.search_products_box.type("Charmander"); + page.keyboard.press("Enter"); + + // Use queryData() method to fetch the stock number from the page. + response = await page.queryData(STOCK_NUMBER_QUERY); + console.log(response); + + await browser.close(); +})(); From 958f48bce14f8ca7da95caff70b5e64e9276afca Mon Sep 17 00:00:00 2001 From: Zach Lyon Date: Tue, 29 Oct 2024 15:48:33 -0400 Subject: [PATCH 2/6] linting --- .../run-script-in-headless-browser.js | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js index 89bad3e..3a9e9a9 100644 --- a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js +++ b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js @@ -4,7 +4,7 @@ const { wrap, configure } = require('agentql'); const { chromium } = require('playwright'); // Define the URL of the page to scrape. -const URL = "https://scrapeme.live/shop/"; +const URL = 'https://scrapeme.live/shop/'; // Define the queries to locate the search box and fetch the stock number. const SEARCH_QUERY = ` @@ -19,30 +19,29 @@ const STOCK_NUMBER_QUERY = ` } `; - (async () => { - // Set the AgentQL API key via the `configure` method. - configure({ apiKey: process.env.AGENTQL_API_KEY }); + // Set the AgentQL API key via the `configure` method. + configure({ apiKey: process.env.AGENTQL_API_KEY }); - // Launch a headless browser using Playwright. - const browser = await chromium.launch({ headless: true }); + // Launch a headless browser using Playwright. + const browser = await chromium.launch({ headless: true }); - // Wrap a new page to use the AgentQL API. - const page = wrap(await browser.newPage()); + // Wrap a new page to use the AgentQL API. + const page = wrap(await browser.newPage()); - await page.goto(URL); + await page.goto(URL); - let response; - // Use queryElements() method to locate the search box from the page. - response = await page.queryElements(SEARCH_QUERY); + let response; + // Use queryElements() method to locate the search box from the page. + response = await page.queryElements(SEARCH_QUERY); - // Use Playwright's API to fill the search box and press Enter. - await response.search_products_box.type("Charmander"); - page.keyboard.press("Enter"); + // Use Playwright's API to fill the search box and press Enter. + await response.search_products_box.type('Charmander'); + page.keyboard.press('Enter'); - // Use queryData() method to fetch the stock number from the page. - response = await page.queryData(STOCK_NUMBER_QUERY); - console.log(response); + // Use queryData() method to fetch the stock number from the page. + response = await page.queryData(STOCK_NUMBER_QUERY); + console.log(response); - await browser.close(); + await browser.close(); })(); From 301e6946784aa501d5529aaf4ae570ccd1e1382b Mon Sep 17 00:00:00 2001 From: Zach Lyon Date: Tue, 29 Oct 2024 21:18:27 -0400 Subject: [PATCH 3/6] addressed pr comments --- .../run-script-in-headless-browser.js | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js index 3a9e9a9..1c37472 100644 --- a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js +++ b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js @@ -1,6 +1,6 @@ /* This example demonstrates how to run the script in a headless browser. */ -const { wrap, configure } = require('agentql'); +const { wrap } = require('agentql'); const { chromium } = require('playwright'); // Define the URL of the page to scrape. @@ -20,28 +20,24 @@ const STOCK_NUMBER_QUERY = ` `; (async () => { - // Set the AgentQL API key via the `configure` method. - configure({ apiKey: process.env.AGENTQL_API_KEY }); - // Launch a headless browser using Playwright. const browser = await chromium.launch({ headless: true }); + const normalPage = await browser.newPage(); + await normalPage.goto(URL); - // Wrap a new page to use the AgentQL API. - const page = wrap(await browser.newPage()); - - await page.goto(URL); + // Wrap a new page instance to use with AgentQL API + const wrappedPage = wrap(normalPage); - let response; // Use queryElements() method to locate the search box from the page. - response = await page.queryElements(SEARCH_QUERY); + const searchResponse = await wrappedPage.queryElements(SEARCH_QUERY); // Use Playwright's API to fill the search box and press Enter. - await response.search_products_box.type('Charmander'); - page.keyboard.press('Enter'); + await searchResponse.search_products_box.type('Charmander'); + await normalPage.keyboard.press('Enter'); - // Use queryData() method to fetch the stock number from the page. - response = await page.queryData(STOCK_NUMBER_QUERY); - console.log(response); + // Use queryData() method to locate the stock number from the page. + const stockResponse = await wrappedPage.queryData(STOCK_NUMBER_QUERY); + console.log(stockResponse); await browser.close(); })(); From e126dc6c1f5702d20b88bc06b170ee108af016ba Mon Sep 17 00:00:00 2001 From: Zach Lyon Date: Tue, 29 Oct 2024 21:19:07 -0400 Subject: [PATCH 4/6] nit --- .../run-script-in-headless-browser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js index 1c37472..174a921 100644 --- a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js +++ b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js @@ -35,7 +35,7 @@ const STOCK_NUMBER_QUERY = ` await searchResponse.search_products_box.type('Charmander'); await normalPage.keyboard.press('Enter'); - // Use queryData() method to locate the stock number from the page. + // Use queryData() method to fetch the stock number from the page. const stockResponse = await wrappedPage.queryData(STOCK_NUMBER_QUERY); console.log(stockResponse); From ebc150ecf1dd6de7e0cbac7d78dd89c5891cdbe5 Mon Sep 17 00:00:00 2001 From: Zach Lyon Date: Wed, 30 Oct 2024 21:26:45 -0400 Subject: [PATCH 5/6] revert --- .../run-script-in-headless-browser.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js index 174a921..5e7ee50 100644 --- a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js +++ b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js @@ -21,22 +21,20 @@ const STOCK_NUMBER_QUERY = ` (async () => { // Launch a headless browser using Playwright. - const browser = await chromium.launch({ headless: true }); - const normalPage = await browser.newPage(); - await normalPage.goto(URL); - - // Wrap a new page instance to use with AgentQL API - const wrappedPage = wrap(normalPage); + const browser = await chromium.launch({ headless: false }); + // Create a new page in the browser and wrap it to get access to the AgentQL's querying API + const page = await wrap(await browser.newPage()); + await page.goto(URL); // Use queryElements() method to locate the search box from the page. - const searchResponse = await wrappedPage.queryElements(SEARCH_QUERY); + const searchResponse = await page.queryElements(SEARCH_QUERY); // Use Playwright's API to fill the search box and press Enter. await searchResponse.search_products_box.type('Charmander'); await normalPage.keyboard.press('Enter'); // Use queryData() method to fetch the stock number from the page. - const stockResponse = await wrappedPage.queryData(STOCK_NUMBER_QUERY); + const stockResponse = await page.queryData(STOCK_NUMBER_QUERY); console.log(stockResponse); await browser.close(); From d7f3f04899a930f6c02157c14a1a402a63a93d31 Mon Sep 17 00:00:00 2001 From: Zach Lyon Date: Wed, 30 Oct 2024 21:30:15 -0400 Subject: [PATCH 6/6] change var name --- .../run-script-in-headless-browser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js index 5e7ee50..891fb76 100644 --- a/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js +++ b/javascript-sdk/examples/run-script-in-headless-browser/run-script-in-headless-browser.js @@ -31,7 +31,7 @@ const STOCK_NUMBER_QUERY = ` // Use Playwright's API to fill the search box and press Enter. await searchResponse.search_products_box.type('Charmander'); - await normalPage.keyboard.press('Enter'); + await page.keyboard.press('Enter'); // Use queryData() method to fetch the stock number from the page. const stockResponse = await page.queryData(STOCK_NUMBER_QUERY);