1666 pull data from previous years

components/Footer/LastUpdated.jsx

@@ -21,7 +21,7 @@ function LastUpdated() {
 
   useEffect(() => {
     const getLastUpdated = async () => {
-      const getLastUpdatedSQL = 'select max(createddate) from requests;';
+      const getLastUpdatedSQL = 'select max(createddate) from requests_2024;';
 
       const lastUpdatedAsArrowTable = await conn.query(getLastUpdatedSQL);
       const results = ddbh.getTableData(lastUpdatedAsArrowTable);

components/Map/RequestDetail.jsx

@@ -63,10 +63,10 @@ function RequestDetail({
   // dispatchGetPinInfoRequest,
   dispatchUpdatePinInfo,
 }) {
-  const { conn } = useContext(DbContext);
+  const { conn, tableNameByYear } = useContext(DbContext);
   const getPinInfo = useCallback(async () => {
     try {
-      const getPinsInfoSQL = `SELECT * FROM requests WHERE TRIM(SRNumber) = '${requestId}'`;
+      const getPinsInfoSQL = `SELECT * FROM ${tableNameByYear} WHERE TRIM(SRNumber) = '${requestId}'`;
 
       const pinsInfoAsArrowTable = await conn.query(getPinsInfoSQL);
       const newPinsInfo = ddbh.getTableData(pinsInfoAsArrowTable);

components/Map/index.js

@@ -53,6 +53,7 @@ class MapContainer extends React.Component {
       position: props.position,
       lastUpdated: props.lastUpdated,
       selectedTypes: this.getSelectedTypes(),
+      isTableLoading: false,
     };
 
     // We store the raw requests from the API call here, but eventually they aremap/inde
@@ -65,13 +66,27 @@ class MapContainer extends React.Component {
   }
 
   createRequestsTable = async () => {
-    const { conn } = this.context;
+    this.setState({ isTableLoading: true });
+    const { conn, tableNameByYear } = this.context;
+    const startDate = this.props.startDate; // directly use the startDate prop transformed for redux store
+    const year = moment(startDate).year(); // extrac the year
+    const datasetFileName = `requests${year}.parquet`;
 
-    // Create the 'requests' table.
+    // Create the year data table if not exist already
     const createSQL =
-      'CREATE TABLE requests AS SELECT * FROM "requests.parquet"'; // parquet
-
-    await conn.query(createSQL);
+      `CREATE TABLE IF NOT EXISTS ${tableNameByYear} AS SELECT * FROM "${datasetFileName}"`; // query from parquet
+
+    const startTime = performance.now(); // start the time tracker
+
+      try {
+        await conn.query(createSQL);
+        const endTime = performance.now() // end the timer
+        console.log(`Table created and dataset registered. Time taken: ${Math.floor(endTime - startTime)} ms.`);
+      } catch (error) {
+        console.error("Error in creating table or registering dataset:", error);
+      } finally {
+        this.setState({ isTableLoading: false});
+      }
   };
 
   async componentDidMount(props) {
@@ -83,19 +98,22 @@ class MapContainer extends React.Component {
 
   async componentDidUpdate(prevProps) {
     const { activeMode, pins, startDate, endDate } = this.props;
-    function didDateRangeChange() {
-      // Check that endDate is not null since we only want to retrieve data
-      // when both the startDate and endDate are selected.
-      return (
-        (prevProps.startDate != startDate || prevProps.endDate != endDate) &&
-        endDate != null
-      );
-    }
+
+    // create conditions to check if year or startDate or endDate changed
+    const yearChanged = moment(prevProps.startDate).year() !== moment(startDate).year();
+    const startDateChanged = prevProps.startDate !== startDate;
+    const endDateChanged = prevProps.endDate !== endDate;
+
+    // Check that endDate is not null since we only want to retrieve data
+    // when both the startDate and endDate are selected.
+    const didDateRangeChange = (yearChanged || startDateChanged || endDateChanged) && endDate !== null;
+
     if (
       prevProps.activeMode !== activeMode ||
       prevProps.pins !== pins ||
-      didDateRangeChange()
+      didDateRangeChange
     ) {
+      await this.createRequestsTable();
       await this.setData();
     }
   }
@@ -292,10 +310,11 @@ class MapContainer extends React.Component {
 
   getAllRequests = async (startDate, endDate) => {
     try {
-      const { conn } = this.context;
+      const { conn, tableNameByYear } = this.context;
+      const year = moment(startDate).year();
 
       // Execute a SELECT query from 'requests' table
-      const selectSQL = `SELECT * FROM requests WHERE CreatedDate between '${startDate}' and '${endDate}'`;
+      const selectSQL = `SELECT * FROM ${tableNameByYear} WHERE CreatedDate between '${startDate}' and '${endDate}'`;
 
       const requestsAsArrowTable = await conn.query(selectSQL);
 
@@ -304,7 +323,7 @@ class MapContainer extends React.Component {
       this.endTime = performance.now(); // end bnechmark
 
       console.log(
-        `Time taken to bootstrap db: ${this.endTime - this.startTime}ms`
+        `Time taken to bootstrap db: ${Math.floor(this.endTime - this.startTime)} ms`
       );
       return requests;
     } catch (e) {
@@ -380,7 +399,7 @@ class MapContainer extends React.Component {
       isMapLoading,
       isDbLoading,
     } = this.props;
-    const { ncCounts, ccCounts, selectedTypes } = this.state;
+    const { ncCounts, ccCounts, selectedTypes, isTableLoading } = this.state;
     return (
       <div className={classes.root}>
         <Map
@@ -395,7 +414,7 @@ class MapContainer extends React.Component {
           initialState={this.initialState}
         />
         <CookieNotice />
-        {(isDbLoading || isMapLoading) ? (
+        {(isDbLoading || isMapLoading || isTableLoading) ? (
           <>
             <LoadingModal />
             <FunFactCard />
@@ -443,6 +462,7 @@ MapContainer.propTypes = {};
 
 MapContainer.defaultProps = {};
 
+// connect MapContainer to Redux store
 export default connect(
   mapStateToProps,
   mapDispatchToProps

components/common/ReactDayPicker/ReactDayPicker.jsx

@@ -222,7 +222,7 @@ function ReactDayPicker({
         onDayClick={handleDayClick}
         onDayMouseEnter={handleDayMouseEnter}
         weekdayElement={<WeekDay />}
-        fromMonth={new Date(2023, 12)}
+        fromMonth={new Date(2022, 12)}
       />
     </>
   );

components/db/DbContext.jsx

@@ -1,5 +1,10 @@
 import React from 'react';
 
-const DbContext = React.createContext();
+const DbContext = React.createContext({
+  db: null,
+  conn: null,
+  worker: null,
+  tableName: '',
+});
 
 export default DbContext;

components/db/DbProvider.jsx

@@ -1,15 +1,19 @@
 import React, { useEffect, useState } from 'react';
-import PropTypes from 'proptypes';
+import { connect } from 'react-redux';
+import PropTypes from 'prop-types';
 import * as duckdb from '@duckdb/duckdb-wasm';
 import Worker from 'web-worker';
 import DbContext from '@db/DbContext';
+import moment from 'moment';
 
 // List of remote dataset locations used by db.registerFileURL
 const datasets = {
   parquet: {
     // huggingface
-    hfYtd:
-      'https://huggingface.co/datasets/311-data/2024/resolve/main/2024.parquet', // year-to-date
+    hfYtd2024:
+      'https://huggingface.co/datasets/311-data/2024/resolve/main/2024.parquet', // 2024 year-to-date
+    hfYtd2023:
+      'https://huggingface.co/datasets/311-data/2023/resolve/main/2023.parquet', // 2023 year-to-date
     hfLastMonth:
       'https://huggingface.co/datasets/edwinjue/311-data-last-month/resolve/refs%2Fconvert%2Fparquet/edwinjue--311-data-last-month/csv-train.parquet', // last month
   },
@@ -20,10 +24,11 @@ const datasets = {
   },
 };
 
-function DbProvider({ children }) {
+function DbProvider({ children, startDate }) {
   const [db, setDb] = useState(null);
   const [conn, setConn] = useState(null);
   const [worker, setWorker] = useState(null);
+  const [tableNameByYear, setTableNameByYear] = useState('');
 
   useEffect(() => {
     const dbInitialize = async () => {
@@ -49,14 +54,20 @@ function DbProvider({ children }) {
 
         await newDb.instantiate(
           DUCKDB_CONFIG.mainModule,
-          DUCKDB_CONFIG.pthreadWorker
+          DUCKDB_CONFIG.pthreadWorker,
         );
 
         // register parquet
         await newDb.registerFileURL(
-          'requests.parquet',
-          datasets.parquet.hfYtd,
-          4 // HTTP = 4. For more options: https://tinyurl.com/DuckDBDataProtocol
+          'requests2024.parquet',
+          datasets.parquet.hfYtd2024,
+          4, // HTTP = 4. For more options: https://tinyurl.com/DuckDBDataProtocol
+        );
+
+        await newDb.registerFileURL(
+          'requests2023.parquet',
+          datasets.parquet.hfYtd2023,
+          4,
         );
 
         // Create db connection
@@ -100,24 +111,43 @@ function DbProvider({ children }) {
   // Important: dependency array must be empty or you will get the following error
   // "cannot send a message since the worker is not set" and app will infinite loop
 
+  // This useEffect specifically handle dynamic table name generation
+  // separated from the previous useEffect that handles db initialization and teardown
+  useEffect(() => {
+    if (startDate) {
+      const year = moment(startDate).year();
+      setTableNameByYear(`requests_${year}`);
+    }
+  }, [startDate]); // Depend on startDate
+
   //   block until db, conn, worker are available
   if (!db || !conn || !worker) {
     return null;
   }
 
   return (
-    <DbContext.Provider value={{ db, conn, worker }}>
+    <DbContext.Provider value={{
+      db, conn, worker, tableNameByYear,
+    }}
+    >
       {children}
     </DbContext.Provider>
   );
 }
 
 DbProvider.propTypes = {
   children: PropTypes.node,
+  startDate: PropTypes.string,
 };
 
 DbProvider.defaultProps = {
   children: null,
+  startDate: PropTypes.string,
 };
 
-export default DbProvider;
+// connect DbProvider to Redux to get startDate
+const mapStateToProps = state => ({
+  startDate: state.filters.startDate,
+});
+
+export default connect(mapStateToProps)(DbProvider);