Upgrade database libraries

.gitignore

@@ -1,28 +1,30 @@
+# Local config files
+*.cfg
+.env*
+
 # Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
 *$py.class
+*.py[cod]
+__pycache__/
 
 # C extensions
 *.so
 
 # Distribution / packaging
+*.egg
+*.egg-info/
 .Python
-env/
+.eggs/
+.installed.cfg
+.venv/
 build/
-src/
 develop-eggs/
 dist/
 downloads/
 eggs/
-.eggs/
 lib/
 lib64/
 parts/
 sdist/
-var/
+venv/
 wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-

README.md

@@ -24,17 +24,17 @@ variable, for example:
     PYTHONPATH=$(pwd) alembic upgrade head
 
 ### Coding standards
-The `freezing-web` code is intended to be [PEP-8](https://www.python.org/dev/peps/pep-0008/) compliant. Code formatting is done with [black](https://black.readthedocs.io/en/stable/) and can be linted with [flake8](http://flake8.pycqa.org/en/latest/). See the [.flake8](.flake8) file and install the test dependencies to get these tools (`pip install -r test-requirements.txt`).
+The `freezing-model` code is intended to be [PEP-8](https://www.python.org/dev/peps/pep-0008/) compliant. Code formatting is done with [black](https://black.readthedocs.io/en/stable/) and can be linted with [flake8](http://flake8.pycqa.org/en/latest/). See the [.flake8](.flake8) file and install the test dependencies to get these tools (`pip install -r test-requirements.txt`).
 
 Useful Queries
 --------------
 (TODO: This is probably not the best place for this documentation, but I'm not sure where else to put it)
 
 Beyond the model definitions there are a few other useful SQL utilities and queries that can help in operations:
 
-The script [bin/registrants.ps1](bin/registrants.ps1), given a CSV export from the WordPress registration site for Freezing Saddles, can generate a `registrants` table in the `freezing` database that is useful for determining who has registered but has not authorized properly in the database.
+The script [bin/registrants.py](bin/registrants.py), given a CSV export from the WordPress registration site for Freezing Saddles, can generate a `registrants` table in the `freezing` database that is useful for determining who has registered but has not authorized properly in the database.
 
-These queries can find users who need to authorize and generate a list of emails 
+These queries can find users who need to authorize and generate a list of emails for those users:
 ```
 select regnum, id, username, name, email, registered_on from registrants r where id not in (select id from athletes); /* Athletes who have never authorized with the freezingsaddles.org site */
 

bin/registrants.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""
+This script reads a CSV file of registrants from https://freezingsaddles.info and outputs SQL commands to create a table and insert the registrants into the table.
+
+Auto-generated from registrants.ps1 by GitHub Copilot.
+Author: @obscurerichard
+Usage: python registrants.py <csvfile>
+"""
+import csv
+import sys
+from datetime import datetime
+
+from pymysql.converters import escape_string
+
+
+def main(csvfile):
+    print("drop table if exists registrants;")
+    print(
+        "create table registrants (regnum int(11), id int(11), username varchar(255), name varchar(255), email varchar(255), registered_on datetime);"
+    )
+    print("begin;")
+
+    with open(csvfile, newline="") as csvfile:
+        reader = csv.DictReader(csvfile)
+        for row in reader:
+            regnum = int(row["#"])
+            id = int(row["Strava user ID"])
+            firstname = escape_string(row["First Name"])
+            lastname = escape_string(row["Last Name"])
+            username = escape_string(
+                row["Your user name on the Washington Area Bike Forum"]
+            )
+            email = escape_string(row["E-mail"])
+            datesubmitted = datetime.strptime(
+                row["Date Submitted"], "%B %d, %Y %I:%M %p"
+            )
+            datesubmitted_str = datesubmitted.strftime("%Y-%m-%d %H:%M:%S")
+            print(
+                f"insert into registrants values({regnum}, {id}, '{username}', '{firstname} {lastname}', '{email}', '{datesubmitted_str}');"
+            )
+    print("commit;")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python registrants.py <csvfile>")
+        sys.exit(1)
+    main(sys.argv[1])

bin/reinstall-env.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# reinstall-env.sh
+#
+# Reinstall the pip virtualenv, useful when troubleshooting
+# package dependencies.
+
+#shellcheck disable=SC1091
+rm -rf .venv/ \
+    && python3 -mvenv .venv \
+    && source .venv/bin/activate \
+    && pip install -r requirements.txt \
+    && pip install -r requirements-test.txt \
+    && pip install -e .

bin/reset-database.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# recreate-database.sh
+#
+# This script is used to recreate the database for the Freezing Model service.
+#
+# Typically this script is used to reset the database to a clean state for testing.
+#
+# Usage:
+#   APPSETTINGS=test.cfg bin/reset-database.sh
+#
+#   # or
+#
+#   MYSQL_ROOT_PASSWORD=your_password MYSQL_DATABASE=your_database SQLALCHEMY_URL=your_url ./recreate-database.sh
+
+set -euo pipefail
+
+APPSETTINGS=${APPSETTINGS:-}
+
+if [[ -n "$APPSETTINGS" ]] && [[ -f "$APPSETTINGS" ]]; then
+    # shellcheck disable=SC1090
+    source "$APPSETTINGS"
+fi
+
+MYSQL_VERSION=${MYSQL_VERSION:-8.0}
+MYSQL_HOST=${MYSQL_HOST:-127.0.0.1}
+MYSQL_PORT=${MYSQL_PORT:-3306}
+MYSQL_USER=${MYSQL_USER:-freezing}
+MYSQL_ROOT_USER=${MYSQL_ROOT_USER:-root}
+MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:?You must set a MYSQL_ROOT_PASSWORD environment variable.}
+MYSQL_DATABASE=${MYSQL_DATABASE:?You must set a MYSQL_DATABASE environment variable.}
+SQLALCHEMY_URL=${SQLALCHEMY_URL:?You must set a SQLALCHEMY_URL environment variable.}
+
+function mysql-freezing-root-non-interactive() {
+    docker run -i \
+        --rm \
+        --network=host \
+        mysql:"$MYSQL_VERSION"  \
+        mysql \
+        --host="$MYSQL_HOST" \
+        --port="$MYSQL_PORT" \
+        --user="$MYSQL_ROOT_USER" \
+        --password="$MYSQL_ROOT_PASSWORD" \
+        --default-character-set=utf8mb4
+}
+
+mysql-freezing-root-non-interactive <<EOF
+drop database if exists $MYSQL_DATABASE;
+create database $MYSQL_DATABASE character set utf8mb4;
+grant all on freezing_model_test.* to freezing;
+EOF
+freezing-model-init-db
+alembic upgrade head
+alembic current

development.cfg

@@ -0,0 +1,21 @@
+# You can use a file containing environment vars like this:
+#   APP_SETTINGS=/path/to/envfile.cfg freezing-model-init-db
+
+DEBUG=true
+
+# MySQL settings needed for initializing the database - see bin/freezing-model-init-db
+MYSQL_VERSION=8.0
+MYSQL_HOST=127.0.0.1
+MYSQL_PORT=3306
+MYSQL_ROOT_USER=root
+MYSQL_ROOT_PASSWORD=terrible-root-password-which-should-be-changed
+MYSQL_DATABASE=freezing
+
+# SQLAlchemy URL for the database.  
+# Note that the pymysql driver must be explicitly specified.
+SQLALCHEMY_URL='mysql+pymysql://freezing:please-change-me-as-this-is-a-default@127.0.0.1/freezing?charset=utf8mb4&binary_prefix=true'
+
+# Python Time zone for competition days. 
+# See https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+TIMEZONE=America/New_York
+

freezing/model/__init__.py

@@ -13,7 +13,7 @@
 
 from freezing.model import meta, migrationsutil
 from freezing.model.autolog import log
-from freezing.model.config import config as model_config
+from freezing.model.config import config
 from freezing.model.monkeypatch import collections
 from freezing.model.orm import (
     Athlete,
@@ -45,6 +45,10 @@
 ]
 
 
+def init_db():
+    init_model(config.SQLALCHEMY_URL)
+
+
 def init_model(sqlalchemy_url: str, drop: bool = False, check_version: bool = True):
     """
     Initializes the tables and classes of the model using configured engine.
@@ -163,7 +167,7 @@ def create_supplemental_db_objects(engine: Engine):
           ride_date
         ;
     """.format(
-            model_config.TIMEZONE
+            config.TIMEZONE
         )
     )
 

freezing/model/migrations/env.py

@@ -6,7 +6,7 @@
 from alembic import context
 from sqlalchemy import create_engine, engine_from_config, pool
 
-from freezing.model import config, meta
+from freezing.model import config, meta, orm
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -21,7 +21,8 @@
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-target_metadata = None
+# target_metadata = None
+target_metadata = orm.Base.metadata
 
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
@@ -43,7 +44,7 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     if not url:
-        url = freezing.model.config.SQLALCHEMY_URL
+        url = config.SQLALCHEMY_URL
     context.configure(url=url, target_metadata=target_metadata, literal_binds=True)
 
     with context.begin_transaction():
@@ -58,13 +59,15 @@ def run_migrations_online():
 
     """
 
-    if meta.engine:
-        # This is the path taken when migrating from freezing-web
-        print("run_migrations_online: use meta.engine to get connection")
-        connectable = meta.engine
     if environ["SQLALCHEMY_URL"]:
-        print("run_migrations_online: use SQLALCHEMY_URL var for connection")
+        print(
+            "run_migrations_online: using SQLALCHEMY_URL environment variable for connection"
+        )
         connectable = create_engine(environ["SQLALCHEMY_URL"], poolclass=pool.NullPool)
+    elif meta.engine:
+        # This is the path taken when migrating from freezing-web
+        print("run_migrations_online: using meta.engine to get connection")
+        connectable = meta.engine
     else:
         print("run_migrations_online: use engine_from_config for connection")
         connectable = engine_from_config(

freezing/model/migrations/versions/17b73a90925d_ride_tracks.py

@@ -10,7 +10,7 @@
 revision = "17b73a90925d"
 down_revision = "54627e8199c9"
 
-import geoalchemy as ga
+import geoalchemy2 as ga
 import sqlalchemy as sa
 from alembic import op
 

freezing/model/orm.py

@@ -1,7 +1,7 @@
 import re
 import warnings
 
-from geoalchemy import GeometryColumn, GeometryDDL, LineString, Point
+from geoalchemy2 import Geometry
 from sqlalchemy import (
     BigInteger,
     Boolean,
@@ -15,7 +15,7 @@
     Time,
     orm,
 )
-from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import declarative_base
 
 from . import meta, satypes
 
@@ -154,13 +154,13 @@ class Ride(StravaEntity):
 class RideGeo(Base):
     __tablename__ = "ride_geo"
     __table_args__ = {
-        "mysql_engine": "MyISAM",
+        "mysql_engine": "InnoDB",
         "mysql_charset": "utf8",
     }  # MyISAM for spatial indexes
 
     ride_id = Column(BigInteger, ForeignKey("rides.id"), primary_key=True)
-    start_geo = GeometryColumn(Point(2), nullable=False)
-    end_geo = GeometryColumn(Point(2), nullable=False)
+    start_geo = Column(Geometry("POINT"), nullable=False)
+    end_geo = Column(Geometry("POINT"), nullable=False)
 
     def __repr__(self):
         return "<{0} ride_id={1} start={2}>".format(
@@ -177,7 +177,7 @@ class RideTrack(Base):
     }  # MyISAM for spatial indexes
 
     ride_id = Column(BigInteger, ForeignKey("rides.id"), primary_key=True)
-    gps_track = GeometryColumn(LineString(2), nullable=False)
+    gps_track = Column(Geometry("LINESTRING"), nullable=False)
     elevation_stream = Column(satypes.JSONEncodedText, nullable=True)
     time_stream = Column(satypes.JSONEncodedText, nullable=True)
 
@@ -288,17 +288,3 @@ def __repr__(self):
         return "<{0} id={1} tribe_name={2}>".format(
             self.__class__.__name__, self.id, self.tribe_name
         )
-
-
-# Setup Geometry columns
-GeometryDDL(RideGeo.__table__)
-GeometryDDL(RideTrack.__table__)
-
-# Opting for a more explicit approach to specifyign which tables are to be managed by SA.
-#
-# _MANAGED_TABLES = [obj.__table__ for name, obj in inspect.getmembers(sys.modules[__name__])
-#                   if inspect.isclass(obj) and (issubclass(obj, Base) and obj is not Base)
-#                   and hasattr(obj, '__table__')
-#                   and not issubclass(obj, _SqlView)]
-#
-# register_managed_tables(_MANAGED_TABLES)