Merge pull request #1357 from cmu-delphi/release/delphi-epidata-4.1.15

Release Delphi Epidata 4.1.15

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 4.1.14
+current_version = 4.1.15
 commit = False
 tag = False
 

dev/local/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = Delphi Development
-version = 4.1.14
+version = 4.1.15
 
 [options]
 packages =

requirements.dev.txt

@@ -1,4 +1,4 @@
-aiohttp==3.8.6
+aiohttp==3.9.0
 black>=20.8b1
 bump2version==1.0.1
 covidcast==0.1.5

src/acquisition/covid_hosp/common/database.py

@@ -184,19 +184,16 @@ def nan_safe_dtype(dtype, value):
     for csv_name in self.key_columns:
       dataframe.loc[:, csv_name] = dataframe[csv_name].map(self.columns_and_types[csv_name].dtype)
 
-    num_columns = 2 + len(dataframe_columns_and_types) + len(self.additional_fields)
-    value_placeholders = ', '.join(['%s'] * num_columns)
     col_names = [f'`{i.sql_name}`' for i in dataframe_columns_and_types + self.additional_fields]
-    columns = ', '.join(col_names)
-    updates = ', '.join(f'{c}=new_values.{c}' for c in col_names)
-    # NOTE: list in `updates` presumes `publication_col_name` is part of the unique key and thus not needed in UPDATE
-    sql = f'INSERT INTO `{self.table_name}` (`id`, `{self.publication_col_name}`, {columns}) ' \
-          f'VALUES ({value_placeholders}) AS new_values ' \
-          f'ON DUPLICATE KEY UPDATE {updates}'
+    value_placeholders = ', '.join(['%s'] * (2 + len(col_names))) # extra 2 for `id` and `self.publication_col_name` cols
+    columnstring = ', '.join(col_names)
+    sql = f'REPLACE INTO `{self.table_name}` (`id`, `{self.publication_col_name}`, {columnstring}) VALUES ({value_placeholders})'
     id_and_publication_date = (0, publication_date)
+    num_values = len(dataframe.index)
     if logger:
-      logger.info('updating values', count=len(dataframe.index))
+      logger.info('updating values', count=num_values)
     n = 0
+    rows_affected = 0
     many_values = []
     with self.new_cursor() as cursor:
       for index, row in dataframe.iterrows():
@@ -212,6 +209,7 @@ def nan_safe_dtype(dtype, value):
         if n % 5_000 == 0:
           try:
             cursor.executemany(sql, many_values)
+            rows_affected += cursor.rowcount
             many_values = []
           except Exception as e:
             if logger:
@@ -220,6 +218,11 @@ def nan_safe_dtype(dtype, value):
       # insert final batch
       if many_values:
         cursor.executemany(sql, many_values)
+        rows_affected += cursor.rowcount
+      if logger:
+        # NOTE: REPLACE INTO marks 2 rows affected for a "replace" (one for a delete and one for a re-insert)
+        # which allows us to count rows which were updated
+        logger.info('rows affected', total=rows_affected, updated=rows_affected-num_values)
 
     # deal with non/seldomly updated columns used like a fk table (if this database needs it)
     if hasattr(self, 'AGGREGATE_KEY_COLS'):

src/client/delphi_epidata.R

@@ -15,7 +15,7 @@ Epidata <- (function() {
   # API base url
   BASE_URL <- getOption('epidata.url', default = 'https://api.delphi.cmu.edu/epidata/')
 
-  client_version <- '4.1.14'
+  client_version <- '4.1.15'
 
   auth <- getOption("epidata.auth", default = NA)
 

src/client/delphi_epidata.js

@@ -22,7 +22,7 @@
   }
 })(this, function (exports, fetchImpl, jQuery) {
   const BASE_URL = "https://api.delphi.cmu.edu/epidata/";
-  const client_version = "4.1.14";
+  const client_version = "4.1.15";
 
   // Helper function to cast values and/or ranges to strings
   function _listitem(value) {

src/client/packaging/npm/package.json

@@ -2,7 +2,7 @@
   "name": "delphi_epidata",
   "description": "Delphi Epidata API Client",
   "authors": "Delphi Group",
-  "version": "4.1.14",
+  "version": "4.1.15",
   "license": "MIT",
   "homepage": "https://github.com/cmu-delphi/delphi-epidata",
   "bugs": {

src/client/packaging/pypi/delphi_epidata/__init__.py

@@ -1,4 +1,4 @@
 from .delphi_epidata import Epidata
 
 name = "delphi_epidata"
-__version__ = "4.1.14"
+__version__ = "4.1.15"

src/client/packaging/pypi/setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="delphi_epidata",
-    version="4.1.14",
+    version="4.1.15",
     author="David Farrow",
     author_email="dfarrow0@gmail.com",
     description="A programmatic interface to Delphi's Epidata API.",

src/server/_config.py

@@ -7,7 +7,7 @@
 
 load_dotenv()
 
-VERSION = "4.1.14"
+VERSION = "4.1.15"
 
 MAX_RESULTS = int(10e6)
 MAX_COMPATIBILITY_RESULTS = int(3650)

tests/acquisition/covid_hosp/common/test_database.py

@@ -148,7 +148,7 @@ def test_insert_dataset(self):
 
     actual_sql = mock_cursor.executemany.call_args[0][0]
     self.assertIn(
-      'INSERT INTO `test_table` (`id`, `publication_date`, `sql_str_col`, `sql_int_col`, `sql_float_col`)',
+      'REPLACE INTO `test_table` (`id`, `publication_date`, `sql_str_col`, `sql_int_col`, `sql_float_col`)',
       actual_sql)
 
     expected_values = [