Update Packages

.python-version

@@ -1 +1 @@
-3.8.3
+3.10.2

Pipfile

@@ -4,6 +4,7 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
+setuptools = "57.5.0"
 pipdeptree = "*"
 decorator = "*"
 Faker = "*"
@@ -28,7 +29,6 @@ py = "*"
 
 [packages]
 alembic = "*"
-anyjson = "*"
 appdirs = "*"
 asn1crypto = "*"
 bcrypt = "*"
@@ -44,7 +44,7 @@ cryptography = "*"
 cssselect2 = "*"
 defusedxml = "*"
 dominate = "*"
-elasticsearch = "*"
+elasticsearch = {version = ">=7.0,<8.0"}
 holidays = "*"
 html5lib = "*"
 idna = "*"
@@ -82,7 +82,6 @@ business_calendar = "*"
 CairoSVG = "*"
 Flask = "*"
 Flask-Bootstrap = "*"
-Flask-Elasticsearch = "*"
 Flask-Login = "*"
 Flask-Mail = "*"
 Flask-Migrate = "*"
@@ -117,4 +116,4 @@ python-magic = "*"
 gunicorn = "*"
 
 [requires]
-python_version = "3.8.3"
+python_version = "3.10.2"

app/__init__.py

@@ -11,25 +11,25 @@
 from celery import Celery
 from flask import (Flask, abort, redirect, render_template, request as flask_request, session, url_for)
 from flask_bootstrap import Bootstrap
-from flask_elasticsearch import FlaskElasticsearch
 from flask_login import LoginManager, current_user
 from flask_mail import Mail
 from flask_moment import Moment
 from flask_sqlalchemy import SQLAlchemy
 from flask_tracy import Tracy
-from flask_wtf import CsrfProtect
+from flask_wtf.csrf import CSRFProtect
 from flask_session import Session
 from raven.contrib.flask import Sentry
 
 from app import celery_config
 from app.constants import OPENRECORDS_DL_EMAIL
 from app.lib import NYCHolidays, jinja_filters
 from config import Config, config
+from elasticsearch import Elasticsearch
 
 bootstrap = Bootstrap()
-es = FlaskElasticsearch()
+es = Elasticsearch(Config.ELASTICSEARCH_HOST)
 db = SQLAlchemy()
-csrf = CsrfProtect()
+csrf = CSRFProtect()
 moment = Moment()
 mail = Mail()
 tracy = Tracy()
@@ -101,9 +101,6 @@ def create_app(config_name='default'):
     app.jinja_env.filters['format_ultimate_determination_reason'] = jinja_filters.format_ultimate_determination_reason
 
     bootstrap.init_app(app)
-    es.init_app(app,
-                use_ssl=app.config['ELASTICSEARCH_USE_SSL'],
-                verify_certs=app.config['ELASTICSEARCH_VERIFY_CERTS'])
     db.init_app(app)
     csrf.init_app(app)
     moment.init_app(app)
@@ -113,6 +110,7 @@ def create_app(config_name='default'):
     celery.config_from_object(celery_config)
     sentry.init_app(app, logging=app.config["USE_SENTRY"], level=logging.INFO)
     sess.init_app(app)
+    app.elasticsearch = Elasticsearch(Config.ELASTICSEARCH_HOST)
 
     with app.app_context():
         from app.models import Anonymous

app/admin/forms.py

@@ -1,5 +1,5 @@
 from flask_login import current_user
-from flask_wtf import Form
+from flask_wtf import FlaskForm
 from wtforms import (
     SelectField,
     StringField,
@@ -15,7 +15,7 @@
 from app.models import Agencies
 
 
-class ActivateAgencyUserForm(Form):
+class ActivateAgencyUserForm(FlaskForm):
     users = SelectField('Add Agency Users')
 
     def __init__(self, agency_ein):
@@ -25,7 +25,7 @@ def __init__(self, agency_ein):
             for u in Agencies.query.filter_by(ein=agency_ein).one().inactive_users]
 
 
-class SelectAgencyForm(Form):
+class SelectAgencyForm(FlaskForm):
     agencies = SelectField('Current Agency')
 
     def __init__(self, agency_ein=None):
@@ -72,7 +72,7 @@ def __init__(self, agency_ein=None):
 # TODO: Add forms to modify agency_features (see models.py:183)
 
 
-class AddAgencyUserForm(Form):
+class AddAgencyUserForm(FlaskForm):
     agency = SelectField('Agency', choices=None, validators=[DataRequired()])
     first_name = StringField('First Name', validators=[Length(max=32), DataRequired()])
     last_name = StringField('Last Name', validators=[Length(max=64), DataRequired()])

app/agency/api/views.py

@@ -91,6 +91,8 @@ def get_custom_request_form_options(agency_ein):
                                                                   CustomRequestForms.category,
                                                                   CustomRequestForms.minimum_required).filter_by(
         agency_ein=agency_ein).order_by(asc(CustomRequestForms.category), asc(CustomRequestForms.id)).all()
+    # Convert the results of with_entities back to tuple format so that jsonify can be used
+    custom_request_forms = [tuple(form) for form in custom_request_forms]
     return jsonify(custom_request_forms), 200
 
 

app/auth/forms.py

@@ -5,15 +5,14 @@
     :synopsis: Defines the forms used to manage Authentication requests
 """
 
-from flask_wtf import Form
+from flask_wtf import FlaskForm
 from wtforms import StringField, SelectField, PasswordField, SubmitField
 from wtforms.validators import Length, Email, Optional, DataRequired
 
 from app.constants import STATES
-from app.models import Agencies
 
 
-class StripFieldsForm(Form):
+class StripFieldsForm(FlaskForm):
     """
     Any field data that can be stripped, will be stripped.
     http://stackoverflow.com/questions/26232165/automatically-strip-all-values-in-wtforms
@@ -174,7 +173,7 @@ def validate(self):
         )
 
 
-class BasicLoginForm(Form):
+class BasicLoginForm(FlaskForm):
     email = StringField("Email")
     password = PasswordField("Password")
 

app/models.py

@@ -527,6 +527,8 @@ def is_agency_active(self, ein=None):
 
     def agencies_for_forms(self):
         agencies = self.agencies.with_entities(Agencies.ein, Agencies._name).all()
+        # Convert the results of with_entities back to tuple format so that agencies can be processed
+        agencies = [tuple(agency) for agency in agencies]
         agencies.insert(
             0,
             agencies.pop(
@@ -590,7 +592,6 @@ def es_update(self):
                 es,
                 actions,
                 index=current_app.config["ELASTICSEARCH_INDEX"],
-                doc_type="request",
                 chunk_size=current_app.config["ELASTICSEARCH_CHUNK_SIZE"],
             )
 
@@ -968,7 +969,6 @@ def es_update(self):
             if self.agency.is_active:
                 es.update(
                     index=current_app.config["ELASTICSEARCH_INDEX"],
-                    doc_type="request",
                     id=self.id,
                     body={
                         "doc": {
@@ -1004,7 +1004,6 @@ def es_create(self):
         if current_app.config["ELASTICSEARCH_ENABLED"]:
             es.create(
                 index=current_app.config["ELASTICSEARCH_INDEX"],
-                doc_type="request",
                 id=self.id,
                 body={
                     "title": self.title,
@@ -1040,7 +1039,6 @@ def es_delete(self):
         if current_app.config["ELASTICSEARCH_ENABLED"]:
             es.delete(
                 index=current_app.config["ELASTICSEARCH_INDEX"],
-                doc_type="request",
                 id=self.id,
             )
 

app/report/forms.py

@@ -6,15 +6,15 @@
 from datetime import date, timedelta
 
 from flask_login import current_user
-from flask_wtf import Form
+from flask_wtf import FlaskForm
 from wtforms import DateField, SelectField, SubmitField
 from wtforms.validators import DataRequired
 
 from app.lib.db_utils import get_agency_choices
 from app.constants.dates import MONTHS, PORTAL_START_YEAR
 
 
-class ReportFilterForm(Form):
+class ReportFilterForm(FlaskForm):
     """
     Form for users to filter different report statistics.
 
@@ -37,7 +37,7 @@ def __init__(self):
                 self.agency.choices.insert(1, self.agency.choices.pop(self.agency.choices.index(user_agency)))
 
 
-class AcknowledgmentForm(Form):
+class AcknowledgmentForm(FlaskForm):
     """Form to generate a report with acknowledgment data."""
     date_from = DateField('Date From (required)', format='%m/%d/%Y', validators=[DataRequired()])
     date_to = DateField('Date To (required)', format='%m/%d/%Y', validators=[DataRequired()])
@@ -60,7 +60,7 @@ def validate(self):
         return is_valid
 
 
-class MonthlyMetricsReportForm(Form):
+class MonthlyMetricsReportForm(FlaskForm):
     """Form to generate a monthly metrics report."""
     year = SelectField('Year (required)', choices=None, validators=[DataRequired()])
     month = SelectField('Month (required)', choices=MONTHS, validators=[DataRequired()])
@@ -76,7 +76,7 @@ def __init__(self):
         self.year.choices.insert(0, ('', ''))
 
 
-class OpenDataReportForm(Form):
+class OpenDataReportForm(FlaskForm):
     """Form to generate a report with Open Data compliance data."""
     date_from = DateField('Date From (required)', id='open-data-date-from', format='%m/%d/%Y', validators=[DataRequired()])
     date_to = DateField('Date To (required)', id='open-data-date-to', format='%m/%d/%Y', validators=[DataRequired()])

app/request/forms.py

@@ -7,7 +7,7 @@
 from datetime import datetime
 
 from flask_login import current_user
-from flask_wtf import Form
+from flask_wtf import FlaskForm
 from flask_wtf.file import FileField
 from wtforms import (
     StringField,
@@ -32,7 +32,7 @@
 from app.lib.recaptcha_utils import Recaptcha3Field
 
 
-class PublicUserRequestForm(Form):
+class PublicUserRequestForm(FlaskForm):
     """
     Form for public users to create a new FOIL request.
     For a public user, the required fields are:
@@ -65,7 +65,7 @@ def __init__(self):
         self.request_agency.choices.insert(0, ("", ""))
 
 
-class AgencyUserRequestForm(Form):
+class AgencyUserRequestForm(FlaskForm):
     """
     Form for agency users to create a new FOIL request.
     For an agency user, the required fields are:
@@ -133,7 +133,7 @@ def __init__(self):
             self.request_agency.choices = current_user.agencies_for_forms()
 
 
-class AnonymousRequestForm(Form):
+class AnonymousRequestForm(FlaskForm):
     """
     Form for anonymous users to create a new FOIL request.
     For a anonymous user, the required fields are:
@@ -189,7 +189,7 @@ def __init__(self):
         self.request_agency.choices.insert(0, ("", ""))
 
 
-class EditRequesterForm(Form):
+class EditRequesterForm(FlaskForm):
     # TODO: Add class docstring
     email = StringField("Email")
     phone = StringField("Phone Number")
@@ -220,7 +220,7 @@ def __init__(self, requester):
             self.zipcode.data = requester.mailing_address.get("zip") or ""
 
 
-class DeterminationForm(Form):
+class DeterminationForm(FlaskForm):
     # TODO: Add class docstring
     def __init__(self, agency_ein):
         super(DeterminationForm, self).__init__()
@@ -309,7 +309,7 @@ class ReopenRequestForm(DeterminationForm):
     ultimate_determination_type = [determination_type.REOPENING]
 
 
-class GenerateEnvelopeForm(Form):
+class GenerateEnvelopeForm(FlaskForm):
     # TODO: Add class docstring
     template = SelectField("Template")
     recipient_name = StringField("Recipient Name")
@@ -341,7 +341,7 @@ def __init__(self, agency_ein, requester):
             self.zipcode.data = requester.mailing_address.get("zip") or ""
 
 
-class GenerateLetterForm(Form):
+class GenerateLetterForm(FlaskForm):
     # TODO: Add class docstring
     def __init__(self, agency_ein):
         super(GenerateLetterForm, self).__init__()
@@ -439,7 +439,7 @@ class GenerateResponseLetterForm(GenerateLetterForm):
     letter_type = [response_type.LETTER]
 
 
-class SearchRequestsForm(Form):
+class SearchRequestsForm(FlaskForm):
     # TODO: Add class docstring
     agency_ein = SelectField("Agency")
     agency_user = SelectField("User")
@@ -507,7 +507,7 @@ def __init__(self):
             self.process()
 
 
-class ContactAgencyForm(Form):
+class ContactAgencyForm(FlaskForm):
     # TODO: Add class docstring
     first_name = StringField(
         u"First Name", validators=[InputRequired(), Length(max=32)]
@@ -531,5 +531,5 @@ def __init__(self, request):
         self.subject.data = "Inquiry about {}".format(request.id)
 
 
-class TechnicalSupportForm(Form):
+class TechnicalSupportForm(FlaskForm):
     recaptcha = Recaptcha3Field(action="TestAction", execute_on_load=True)

app/request/views.py

@@ -20,7 +20,7 @@
 from flask_login import current_user
 from sqlalchemy import any_
 from sqlalchemy.orm.exc import NoResultFound
-from werkzeug.utils import escape
+from markupsafe import escape
 
 from app.constants import request_status, permission, HIDDEN_AGENCIES
 from app.lib.date_utils import DEFAULT_YEARS_HOLIDAY_LIST, get_holidays_date_list
@@ -79,7 +79,7 @@ def new():
     kiosk_mode = eval_request_bool(escape(flask_request.args.get("kiosk_mode", False)))
     category = str(escape(flask_request.args.get("category", None)))
     agency = str(escape(flask_request.args.get("agency", None)))
-    title = str(escape(flask_request.args.get("title", None)))
+    title = str(escape(flask_request.args.get("title", "")))
 
     if current_user.is_public:
         form = PublicUserRequestForm()