From 5856018646af795a23237b9f7825fbdf0da8d8b1 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sat, 13 Apr 2024 14:48:14 -0700 Subject: [PATCH 01/20] Set up 2023 & 2024 parquet file path and pointers --- components/db/DbProvider.jsx | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/components/db/DbProvider.jsx b/components/db/DbProvider.jsx index 993541021..4d51a2c35 100644 --- a/components/db/DbProvider.jsx +++ b/components/db/DbProvider.jsx @@ -8,8 +8,10 @@ import DbContext from '@db/DbContext'; const datasets = { parquet: { // huggingface - hfYtd: - 'https://huggingface.co/datasets/311-data/2024/resolve/main/2024.parquet', // year-to-date + hfYtd2024: + 'https://huggingface.co/datasets/311-data/2024/resolve/main/2024.parquet', // 2024 year-to-date + hfYtd2023: + 'https://huggingface.co/datasets/311-data/2023/resolve/main/2023.parquet', // 2023 year-to-date hfLastMonth: 'https://huggingface.co/datasets/edwinjue/311-data-last-month/resolve/refs%2Fconvert%2Fparquet/edwinjue--311-data-last-month/csv-train.parquet', // last month }, @@ -49,14 +51,20 @@ function DbProvider({ children }) { await newDb.instantiate( DUCKDB_CONFIG.mainModule, - DUCKDB_CONFIG.pthreadWorker + DUCKDB_CONFIG.pthreadWorker, ); // register parquet await newDb.registerFileURL( - 'requests.parquet', - datasets.parquet.hfYtd, - 4 // HTTP = 4. For more options: https://tinyurl.com/DuckDBDataProtocol + 'requests2024.parquet', + datasets.parquet.hfYtd2024, + 4, // HTTP = 4. For more options: https://tinyurl.com/DuckDBDataProtocol + ); + + await newDb.registerFileURL( + 'requests2023.parquet', + datasets.parquet.hfYtd2023, + 4, ); // Create db connection From 0bb9b1c1d422133eb42038f92ac7840122756588 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sat, 13 Apr 2024 14:49:15 -0700 Subject: [PATCH 02/20] Update 2024 file name in SQL query --- components/Map/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/Map/index.js b/components/Map/index.js index 9126014f0..18a9b1bd7 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -69,7 +69,7 @@ class MapContainer extends React.Component { // Create the 'requests' table. const createSQL = - 'CREATE TABLE requests AS SELECT * FROM "requests.parquet"'; // parquet + 'CREATE TABLE requests AS SELECT * FROM "requests2024.parquet"'; // parquet await conn.query(createSQL); }; From 780f5a12f3bb6a0f340c4818ffc0ba6658e81d52 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sat, 13 Apr 2024 14:59:41 -0700 Subject: [PATCH 03/20] Clarifying query comment --- components/Map/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/Map/index.js b/components/Map/index.js index 18a9b1bd7..6c6e880e5 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -69,7 +69,7 @@ class MapContainer extends React.Component { // Create the 'requests' table. const createSQL = - 'CREATE TABLE requests AS SELECT * FROM "requests2024.parquet"'; // parquet + 'CREATE TABLE requests AS SELECT * FROM "requests2024.parquet"'; // query from parquet await conn.query(createSQL); }; From e11c8a2022800514045db2b43a27fd51dd7bbd30 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sat, 13 Apr 2024 15:16:12 -0700 Subject: [PATCH 04/20] Renabled 2023 selection on day picker --- components/common/ReactDayPicker/ReactDayPicker.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/common/ReactDayPicker/ReactDayPicker.jsx b/components/common/ReactDayPicker/ReactDayPicker.jsx index 5d44ba0ad..ece94e899 100644 --- a/components/common/ReactDayPicker/ReactDayPicker.jsx +++ b/components/common/ReactDayPicker/ReactDayPicker.jsx @@ -222,7 +222,7 @@ function ReactDayPicker({ onDayClick={handleDayClick} onDayMouseEnter={handleDayMouseEnter} weekdayElement={} - fromMonth={new Date(2023, 12)} + fromMonth={new Date(2022, 12)} /> ); From 2491cf2d50e411e67f9f1e3ddaee0cc0d5a21a94 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sat, 13 Apr 2024 15:56:08 -0700 Subject: [PATCH 05/20] Used startDate prop to extract year and query dataset file by year dynamically --- components/Map/index.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/components/Map/index.js b/components/Map/index.js index 6c6e880e5..b44580779 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -66,10 +66,15 @@ class MapContainer extends React.Component { createRequestsTable = async () => { const { conn } = this.context; + const startDate = this.props.startDate; // directly use the startDate prop transformed for redux store + const year = moment(startDate).year(); // extrac the year + + const datasetYear = year === 2024 ? '2024' : '2023'; + const datasetFileName = `requests${datasetYear}.parquet` // Create the 'requests' table. const createSQL = - 'CREATE TABLE requests AS SELECT * FROM "requests2024.parquet"'; // query from parquet + `CREATE TABLE requests AS SELECT * FROM "${datasetFileName}"`; // query from parquet await conn.query(createSQL); }; @@ -443,6 +448,7 @@ MapContainer.propTypes = {}; MapContainer.defaultProps = {}; +// connect MapContainer to Redux store export default connect( mapStateToProps, mapDispatchToProps From 040bb2dda32934ee706f57cea61ff1020a99188e Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sat, 13 Apr 2024 18:55:29 -0700 Subject: [PATCH 06/20] Fix: add year changed check to set new table --- components/Map/index.js | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/components/Map/index.js b/components/Map/index.js index b44580779..7bb2a91e2 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -66,17 +66,23 @@ class MapContainer extends React.Component { createRequestsTable = async () => { const { conn } = this.context; + const startDate = this.props.startDate; // directly use the startDate prop transformed for redux store const year = moment(startDate).year(); // extrac the year - - const datasetYear = year === 2024 ? '2024' : '2023'; - const datasetFileName = `requests${datasetYear}.parquet` + const datasetFileName = `requests${year}.parquet`; + const tableName = `requests_${year}`; // Create the 'requests' table. const createSQL = - `CREATE TABLE requests AS SELECT * FROM "${datasetFileName}"`; // query from parquet + `CREATE TABLE IF NOT EXISTS ${tableName} AS SELECT * FROM "${datasetFileName}"`; // query from parquet + try { + await conn.query(createSQL); + console.log("Table created and dataset registered successfully."); + } catch (error) { + console.error("Error in creating table or registering dataset:", error); + } - await conn.query(createSQL); + // await conn.query(createSQL); }; async componentDidMount(props) { @@ -88,19 +94,20 @@ class MapContainer extends React.Component { async componentDidUpdate(prevProps) { const { activeMode, pins, startDate, endDate } = this.props; - function didDateRangeChange() { - // Check that endDate is not null since we only want to retrieve data - // when both the startDate and endDate are selected. - return ( - (prevProps.startDate != startDate || prevProps.endDate != endDate) && - endDate != null - ); - } + const yearChanged = moment(prevProps.startDate).year() !== moment(startDate).year(); + const startDateChanged = prevProps.startDate !== startDate; + const endDateChanged = prevProps.endDate !== endDate; + + // Check that endDate is not null since we only want to retrieve data + // when both the startDate and endDate are selected. + const didDateRangeChange = (yearChanged || startDateChanged || endDateChanged) && endDate !== null; + if ( prevProps.activeMode !== activeMode || prevProps.pins !== pins || - didDateRangeChange() + didDateRangeChange ) { + await this.createRequestsTable(); await this.setData(); } } @@ -298,9 +305,11 @@ class MapContainer extends React.Component { getAllRequests = async (startDate, endDate) => { try { const { conn } = this.context; + const year = moment(startDate).year(); + const tableName = `requests_${year}`; // Execute a SELECT query from 'requests' table - const selectSQL = `SELECT * FROM requests WHERE CreatedDate between '${startDate}' and '${endDate}'`; + const selectSQL = `SELECT * FROM ${tableName} WHERE CreatedDate between '${startDate}' and '${endDate}'`; const requestsAsArrowTable = await conn.query(selectSQL); From d369db240743e0a8021740014c2e37216ab5b589 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sat, 13 Apr 2024 18:55:54 -0700 Subject: [PATCH 07/20] Fix: update table name --- components/Footer/LastUpdated.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/Footer/LastUpdated.jsx b/components/Footer/LastUpdated.jsx index 1705fe28c..3f3d70349 100644 --- a/components/Footer/LastUpdated.jsx +++ b/components/Footer/LastUpdated.jsx @@ -21,7 +21,7 @@ function LastUpdated() { useEffect(() => { const getLastUpdated = async () => { - const getLastUpdatedSQL = 'select max(createddate) from requests;'; + const getLastUpdatedSQL = 'select max(createddate) from requests_2024;'; const lastUpdatedAsArrowTable = await conn.query(getLastUpdatedSQL); const results = ddbh.getTableData(lastUpdatedAsArrowTable); From ad21b027d3c5f0f0fc3cb77e2ea4c9ba678e63da Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sat, 13 Apr 2024 19:55:07 -0700 Subject: [PATCH 08/20] Fix: add isTableLoading state to have loading model cover table creation time --- components/Map/index.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/components/Map/index.js b/components/Map/index.js index 7bb2a91e2..2be46122d 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -53,6 +53,7 @@ class MapContainer extends React.Component { position: props.position, lastUpdated: props.lastUpdated, selectedTypes: this.getSelectedTypes(), + isTableLoading: false, }; // We store the raw requests from the API call here, but eventually they aremap/inde @@ -65,14 +66,14 @@ class MapContainer extends React.Component { } createRequestsTable = async () => { + this.setState({ isTableLoading: true }); const { conn } = this.context; - const startDate = this.props.startDate; // directly use the startDate prop transformed for redux store const year = moment(startDate).year(); // extrac the year const datasetFileName = `requests${year}.parquet`; const tableName = `requests_${year}`; - // Create the 'requests' table. + // Create the year data table. const createSQL = `CREATE TABLE IF NOT EXISTS ${tableName} AS SELECT * FROM "${datasetFileName}"`; // query from parquet try { @@ -80,6 +81,8 @@ class MapContainer extends React.Component { console.log("Table created and dataset registered successfully."); } catch (error) { console.error("Error in creating table or registering dataset:", error); + } finally { + this.setState({ isTableLoading: false}); } // await conn.query(createSQL); @@ -394,7 +397,7 @@ class MapContainer extends React.Component { isMapLoading, isDbLoading, } = this.props; - const { ncCounts, ccCounts, selectedTypes } = this.state; + const { ncCounts, ccCounts, selectedTypes, isTableLoading } = this.state; return (
- {(isDbLoading || isMapLoading) ? ( + {(isDbLoading || isMapLoading || isTableLoading) ? ( <> From 60a2d262fe5ffd497f9839ca98dad885e475a139 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sun, 14 Apr 2024 20:35:00 -0700 Subject: [PATCH 09/20] docs: add comment for dynamic table creation by year and year change checking for re-render --- components/Map/index.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/components/Map/index.js b/components/Map/index.js index 2be46122d..ff30b4068 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -71,10 +71,11 @@ class MapContainer extends React.Component { const startDate = this.props.startDate; // directly use the startDate prop transformed for redux store const year = moment(startDate).year(); // extrac the year const datasetFileName = `requests${year}.parquet`; - const tableName = `requests_${year}`; + const tableName = `requests_${year}`; // dynamic table name by year extracted from startDate // Create the year data table. const createSQL = + // create new table only if year changed `CREATE TABLE IF NOT EXISTS ${tableName} AS SELECT * FROM "${datasetFileName}"`; // query from parquet try { await conn.query(createSQL); @@ -84,8 +85,6 @@ class MapContainer extends React.Component { } finally { this.setState({ isTableLoading: false}); } - - // await conn.query(createSQL); }; async componentDidMount(props) { @@ -97,6 +96,8 @@ class MapContainer extends React.Component { async componentDidUpdate(prevProps) { const { activeMode, pins, startDate, endDate } = this.props; + + // create conditions to check if year or startDate or endDate changed const yearChanged = moment(prevProps.startDate).year() !== moment(startDate).year(); const startDateChanged = prevProps.startDate !== startDate; const endDateChanged = prevProps.endDate !== endDate; From 21e3f79ea9f6043ba28d33c4449eabe9607bf300 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sun, 14 Apr 2024 21:20:41 -0700 Subject: [PATCH 10/20] test: add log to track table creation time --- components/Map/index.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/components/Map/index.js b/components/Map/index.js index ff30b4068..1ded5613f 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -73,13 +73,16 @@ class MapContainer extends React.Component { const datasetFileName = `requests${year}.parquet`; const tableName = `requests_${year}`; // dynamic table name by year extracted from startDate - // Create the year data table. + // Create the year data table if not exist already const createSQL = - // create new table only if year changed `CREATE TABLE IF NOT EXISTS ${tableName} AS SELECT * FROM "${datasetFileName}"`; // query from parquet + + const startTime = performance.now(); // start the time tracker + try { await conn.query(createSQL); - console.log("Table created and dataset registered successfully."); + const endTime = performance.now() // end the timer + console.log(`Table created and dataset registered. Time taken: ${Math.floor(endTime - startTime)} ms.`); } catch (error) { console.error("Error in creating table or registering dataset:", error); } finally { @@ -322,7 +325,7 @@ class MapContainer extends React.Component { this.endTime = performance.now(); // end bnechmark console.log( - `Time taken to bootstrap db: ${this.endTime - this.startTime}ms` + `Time taken to bootstrap db: ${Math.floor(this.endTime - this.startTime)} ms` ); return requests; } catch (e) { From e2c93d9fa62b2391c4d7db6d594b4a10762a5ef8 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sun, 14 Apr 2024 22:49:21 -0700 Subject: [PATCH 11/20] refactor: move dynamic table name logic to DbContext so it can be access across application --- components/db/DbContext.jsx | 7 ++++++- components/db/DbProvider.jsx | 30 ++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/components/db/DbContext.jsx b/components/db/DbContext.jsx index cc4ef43c8..d409d7fba 100644 --- a/components/db/DbContext.jsx +++ b/components/db/DbContext.jsx @@ -1,5 +1,10 @@ import React from 'react'; -const DbContext = React.createContext(); +const DbContext = React.createContext({ + db: null, + conn: null, + worker: null, + tableName: '', +}); export default DbContext; diff --git a/components/db/DbProvider.jsx b/components/db/DbProvider.jsx index 4d51a2c35..6855f99ce 100644 --- a/components/db/DbProvider.jsx +++ b/components/db/DbProvider.jsx @@ -1,8 +1,10 @@ import React, { useEffect, useState } from 'react'; -import PropTypes from 'proptypes'; +import { connect } from 'react-redux'; +import PropTypes from 'prop-types'; import * as duckdb from '@duckdb/duckdb-wasm'; import Worker from 'web-worker'; import DbContext from '@db/DbContext'; +import moment from 'moment'; // List of remote dataset locations used by db.registerFileURL const datasets = { @@ -22,10 +24,11 @@ const datasets = { }, }; -function DbProvider({ children }) { +function DbProvider({ children, startDate }) { const [db, setDb] = useState(null); const [conn, setConn] = useState(null); const [worker, setWorker] = useState(null); + const [tableNameByYear, setTableNameByYear] = useState(''); useEffect(() => { const dbInitialize = async () => { @@ -108,13 +111,25 @@ function DbProvider({ children }) { // Important: dependency array must be empty or you will get the following error // "cannot send a message since the worker is not set" and app will infinite loop + // This useEffect specifically handle dynamic table name generation + // separated from the previous useEffect that handles db initialization and teardown + useEffect(() => { + if (startDate) { + const year = moment(startDate).year(); + setTableNameByYear(`requests_${year}`); + } + }, [startDate]); // Depend on startDate + // block until db, conn, worker are available if (!db || !conn || !worker) { return null; } return ( - + {children} ); @@ -122,10 +137,17 @@ function DbProvider({ children }) { DbProvider.propTypes = { children: PropTypes.node, + startDate: PropTypes.string, }; DbProvider.defaultProps = { children: null, + startDate: PropTypes.string, }; -export default DbProvider; +// connect DbProvider to Redux to get startDate +const mapStateToProps = state => ({ + startDate: state.filters.startDate, +}); + +export default connect(mapStateToProps)(DbProvider); From fdc8ec95000f62a2d9aa25d8d66c939b7bc46443 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sun, 14 Apr 2024 22:54:37 -0700 Subject: [PATCH 12/20] refactor: use context managed `tableNameByYear` instead of local state `tableName` to centralize state management across other files that need access to the dynamic table name --- components/Map/index.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/components/Map/index.js b/components/Map/index.js index 1ded5613f..33e89fa9d 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -67,15 +67,15 @@ class MapContainer extends React.Component { createRequestsTable = async () => { this.setState({ isTableLoading: true }); - const { conn } = this.context; + const { conn, tableNameByYear } = this.context; const startDate = this.props.startDate; // directly use the startDate prop transformed for redux store const year = moment(startDate).year(); // extrac the year const datasetFileName = `requests${year}.parquet`; - const tableName = `requests_${year}`; // dynamic table name by year extracted from startDate + // const tableName = `requests_${year}`; // dynamic table name by year extracted from startDate // Create the year data table if not exist already const createSQL = - `CREATE TABLE IF NOT EXISTS ${tableName} AS SELECT * FROM "${datasetFileName}"`; // query from parquet + `CREATE TABLE IF NOT EXISTS ${tableNameByYear} AS SELECT * FROM "${datasetFileName}"`; // query from parquet const startTime = performance.now(); // start the time tracker @@ -311,12 +311,11 @@ class MapContainer extends React.Component { getAllRequests = async (startDate, endDate) => { try { - const { conn } = this.context; + const { conn, tableNameByYear } = this.context; const year = moment(startDate).year(); - const tableName = `requests_${year}`; // Execute a SELECT query from 'requests' table - const selectSQL = `SELECT * FROM ${tableName} WHERE CreatedDate between '${startDate}' and '${endDate}'`; + const selectSQL = `SELECT * FROM ${tableNameByYear} WHERE CreatedDate between '${startDate}' and '${endDate}'`; const requestsAsArrowTable = await conn.query(selectSQL); From e70bcb831ead6838fafd7ac80190d701676416fd Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sun, 14 Apr 2024 22:56:03 -0700 Subject: [PATCH 13/20] fix: update table name with dynamic context state `tableNameByYear` --- components/Map/RequestDetail.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/Map/RequestDetail.jsx b/components/Map/RequestDetail.jsx index 77860e5ca..e3befbb73 100644 --- a/components/Map/RequestDetail.jsx +++ b/components/Map/RequestDetail.jsx @@ -63,10 +63,10 @@ function RequestDetail({ // dispatchGetPinInfoRequest, dispatchUpdatePinInfo, }) { - const { conn } = useContext(DbContext); + const { conn, tableNameByYear } = useContext(DbContext); const getPinInfo = useCallback(async () => { try { - const getPinsInfoSQL = `SELECT * FROM requests WHERE TRIM(SRNumber) = '${requestId}'`; + const getPinsInfoSQL = `SELECT * FROM ${tableNameByYear} WHERE TRIM(SRNumber) = '${requestId}'`; const pinsInfoAsArrowTable = await conn.query(getPinsInfoSQL); const newPinsInfo = ddbh.getTableData(pinsInfoAsArrowTable); From 9cfe1ab5df6552fdcac30617c39ff755f1593188 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Sun, 14 Apr 2024 23:02:20 -0700 Subject: [PATCH 14/20] style: remove unused comment out code --- components/Map/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/components/Map/index.js b/components/Map/index.js index 33e89fa9d..4e97b13f2 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -71,7 +71,6 @@ class MapContainer extends React.Component { const startDate = this.props.startDate; // directly use the startDate prop transformed for redux store const year = moment(startDate).year(); // extrac the year const datasetFileName = `requests${year}.parquet`; - // const tableName = `requests_${year}`; // dynamic table name by year extracted from startDate // Create the year data table if not exist already const createSQL = From 4e8f9f48182f11de87964233232c39f6afa279e3 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Thu, 18 Apr 2024 19:12:41 -0700 Subject: [PATCH 15/20] Refactor: remove unused variable --- components/Map/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/components/Map/index.js b/components/Map/index.js index 4e97b13f2..79a2ccff5 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -311,7 +311,6 @@ class MapContainer extends React.Component { getAllRequests = async (startDate, endDate) => { try { const { conn, tableNameByYear } = this.context; - const year = moment(startDate).year(); // Execute a SELECT query from 'requests' table const selectSQL = `SELECT * FROM ${tableNameByYear} WHERE CreatedDate between '${startDate}' and '${endDate}'`; From 88985442fc6dccdc331981e4c5076a1c5689f5c5 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Mon, 29 Apr 2024 18:22:04 -0700 Subject: [PATCH 16/20] fix: handle cross-year data request with union of two table queries --- components/Map/index.js | 62 ++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/components/Map/index.js b/components/Map/index.js index 2a56ebda1..30496ab9f 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -309,27 +309,63 @@ class MapContainer extends React.Component { return dateArray; }; - getAllRequests = async (startDate, endDate) => { - try { - const { conn, tableNameByYear } = this.context; + // getAllRequests = async (startDate, endDate) => { + // try { + // const { conn, tableNameByYear } = this.context; - // Execute a SELECT query from 'requests' table - const selectSQL = `SELECT * FROM ${tableNameByYear} WHERE CreatedDate between '${startDate}' and '${endDate}'`; + // // Execute a SELECT query from 'requests' table + // const selectSQL = `SELECT * FROM ${tableNameByYear} WHERE CreatedDate between '${startDate}' and '${endDate}'`; - const requestsAsArrowTable = await conn.query(selectSQL); + // const requestsAsArrowTable = await conn.query(selectSQL); - const requests = ddbh.getTableData(requestsAsArrowTable); + // const requests = ddbh.getTableData(requestsAsArrowTable); - this.endTime = performance.now(); // end bnechmark + // this.endTime = performance.now(); // end bnechmark + + // console.log( + // `Time taken to bootstrap db: ${Math.floor(this.endTime - this.startTime)} ms` + // ); + // return requests; + // } catch (e) { + // console.error(e); + // } + // }; + + async getAllRequests(startDate, endDate) { + const { conn } = this.context; + const startYear = moment(startDate).year(); + const endYear = moment(endDate).year(); + + let selectSQL = ''; + + try { + if (startYear === endYear) { + // If the dates are within the same year, query that single year's table. + const tableName = `requests_${startYear}`; + selectSQL = `SELECT * FROM ${tableName} WHERE CreatedDate BETWEEN '${startDate}' AND '${endDate}'`; + } else { + // If the dates span multiple years, create two queries and union them. + const tableNameStartYear = `requests_${startYear}`; + const endOfStartYear = moment(startDate).endOf('year').format('YYYY-MM-DD'); + const tableNameEndYear = `requests_${endYear}`; + const startOfEndYear = moment(endDate).startOf('year').format('YYYY-MM-DD'); + + selectSQL = ` + (SELECT * FROM ${tableNameStartYear} WHERE CreatedDate BETWEEN '${startDate}' AND '${endOfStartYear}') + UNION ALL + (SELECT * FROM ${tableNameEndYear} WHERE CreatedDate BETWEEN '${startOfEndYear}' AND '${endDate}') + `; + } + + const requestsAsArrowTable = await conn.query(selectSQL); + const requests = ddbh.getTableData(requestsAsArrowTable); - console.log( - `Time taken to bootstrap db: ${Math.floor(this.endTime - this.startTime)} ms` - ); return requests; } catch (e) { - console.error(e); + console.error("Error during database query execution:", e); } - }; + } + setData = async () => { const { startDate, endDate, dispatchGetDbRequest, dispatchGetDataRequest } = From 661a2661835a4a8902f16e513de8a0ada3c60c41 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Mon, 29 Apr 2024 18:23:31 -0700 Subject: [PATCH 17/20] fix: add query to handle cross-year search date for pin details --- components/Map/RequestDetail.jsx | 33 ++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/components/Map/RequestDetail.jsx b/components/Map/RequestDetail.jsx index e3befbb73..5f9257d81 100644 --- a/components/Map/RequestDetail.jsx +++ b/components/Map/RequestDetail.jsx @@ -62,11 +62,34 @@ function RequestDetail({ agencies, // dispatchGetPinInfoRequest, dispatchUpdatePinInfo, + startDate, + endDate, }) { - const { conn, tableNameByYear } = useContext(DbContext); + const { conn } = useContext(DbContext); const getPinInfo = useCallback(async () => { + if (!requestId) return; + try { - const getPinsInfoSQL = `SELECT * FROM ${tableNameByYear} WHERE TRIM(SRNumber) = '${requestId}'`; + const startYear = moment(startDate).year(); + const endYear = moment(endDate).year(); + + let getPinsInfoSQL = ''; + + if (startYear === endYear) { + // If search date range is within the same year + const tableName = `requests_${startYear}`; + getPinsInfoSQL = `SELECT * FROM ${tableName} WHERE TRIM(SRNumber) = '${requestId}'`; + } else { + // If search date range is across two different years + const tableNameStartYear = `requests_${startYear}`; + const tableNameEndYear = `requests_${endYear}`; + + getPinsInfoSQL = ` + (SELECT * FROM ${tableNameStartYear} WHERE TRIM(SRNumber) = '${requestId}') + UNION ALL + (SELECT * FROM ${tableNameEndYear} WHERE TRIM(SRNumber) = '${requestId}') + `; + } const pinsInfoAsArrowTable = await conn.query(getPinsInfoSQL); const newPinsInfo = ddbh.getTableData(pinsInfoAsArrowTable); @@ -76,12 +99,12 @@ function RequestDetail({ && Array.isArray(newPinsInfo) && newPinsInfo.length > 0 ) { - dispatchUpdatePinInfo(newPinsInfo[0]); + dispatchUpdatePinInfo(newPinsInfo[0]); // Assumes first entry is correct, adjust as needed } } catch (e) { console.error('RequestDetail: Error occurred: ', e); } - }, [requestId, conn, dispatchUpdatePinInfo]); + }, [requestId, conn, dispatchUpdatePinInfo, startDate, endDate]); useEffect(() => { async function fetchPins() { @@ -244,6 +267,8 @@ const mapStateToProps = state => ({ pinsInfo: state.data.pinsInfo, requestTypes: state.metadata.requestTypes, agencies: state.metadata.agencies, + startDate: state.filters.startDate, + endDate: state.filters.endDate, }); const mapDispatchToProps = dispatch => ({ From f9c091b574aaf501f78a2e6c7fdc6d27688c94ba Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Fri, 3 May 2024 13:34:30 -0700 Subject: [PATCH 18/20] refactor: clean up old code and added comment explanation --- components/Map/index.js | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/components/Map/index.js b/components/Map/index.js index 30496ab9f..d80820388 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -309,27 +309,9 @@ class MapContainer extends React.Component { return dateArray; }; - // getAllRequests = async (startDate, endDate) => { - // try { - // const { conn, tableNameByYear } = this.context; - - // // Execute a SELECT query from 'requests' table - // const selectSQL = `SELECT * FROM ${tableNameByYear} WHERE CreatedDate between '${startDate}' and '${endDate}'`; - - // const requestsAsArrowTable = await conn.query(selectSQL); - - // const requests = ddbh.getTableData(requestsAsArrowTable); - - // this.endTime = performance.now(); // end bnechmark - - // console.log( - // `Time taken to bootstrap db: ${Math.floor(this.endTime - this.startTime)} ms` - // ); - // return requests; - // } catch (e) { - // console.error(e); - // } - // }; + // To handle cross-year date ranges, we check if the startDate and endDate year are the same year + // if same year, we simply query from that year's table + // if different years, we query both startDate year and endDate year, then union the result async getAllRequests(startDate, endDate) { const { conn } = this.context; From e0859fed99ebbdf09b07bf72ed74a19f117203f7 Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Wed, 8 May 2024 17:53:10 -0700 Subject: [PATCH 19/20] feat: add time performance console logs for table creation, data loading, and map loading --- components/Map/index.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/components/Map/index.js b/components/Map/index.js index d80820388..c67d587be 100644 --- a/components/Map/index.js +++ b/components/Map/index.js @@ -82,7 +82,7 @@ class MapContainer extends React.Component { try { await conn.query(createSQL); const endTime = performance.now() // end the timer - console.log(`Table created and dataset registered. Time taken: ${Math.floor(endTime - startTime)} ms.`); + console.log(`Dataset registration & table creation (by year) time: ${Math.floor(endTime - startTime)} ms.`); } catch (error) { console.error("Error in creating table or registering dataset:", error); } finally { @@ -339,8 +339,16 @@ class MapContainer extends React.Component { `; } + const dataLoadStartTime = performance.now(); const requestsAsArrowTable = await conn.query(selectSQL); + const dataLoadEndTime = performance.now(); + + console.log(`Data loading time: ${Math.floor(dataLoadEndTime - dataLoadStartTime)} ms`); + const requests = ddbh.getTableData(requestsAsArrowTable); + const mapLoadEndTime = performance.now(); + + console.log(`Map loading time: ${Math.floor(mapLoadEndTime - dataLoadEndTime)} ms`); return requests; } catch (e) { From 19e5bf503d10f062513682423285bcf9b4e3b62f Mon Sep 17 00:00:00 2001 From: Johnny Wu Date: Wed, 8 May 2024 17:53:40 -0700 Subject: [PATCH 20/20] fix: update correct proptypes --- components/db/DbContext.jsx | 2 +- components/db/DbProvider.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/db/DbContext.jsx b/components/db/DbContext.jsx index d409d7fba..2116ab102 100644 --- a/components/db/DbContext.jsx +++ b/components/db/DbContext.jsx @@ -4,7 +4,7 @@ const DbContext = React.createContext({ db: null, conn: null, worker: null, - tableName: '', + tableNameByYear: '', }); export default DbContext; diff --git a/components/db/DbProvider.jsx b/components/db/DbProvider.jsx index 6855f99ce..94ec0cc0d 100644 --- a/components/db/DbProvider.jsx +++ b/components/db/DbProvider.jsx @@ -142,7 +142,7 @@ DbProvider.propTypes = { DbProvider.defaultProps = { children: null, - startDate: PropTypes.string, + startDate: null, }; // connect DbProvider to Redux to get startDate