Merge pull request #125 from effective-altruism-australia/master

Make debuggin/dev improvements and add internal reports and redacted partner reports

.devcontainer/default/devcontainer.json

@@ -0,0 +1,31 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+	"name": "ubuntu16.04",
+	"build": {
+		// Sets the run context to one level up instead of the .devcontainer folder.
+		"context": "../..",
+		// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+		"dockerfile": "../../Dockerfile"
+	},
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [8000],
+
+	// Uncomment the next line to run commands after the container is created.
+	"postCreateCommand": "cat /etc/os-release",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	"extensions": [
+    "ms-python.python@2021.9.1246542782",
+    "ms-python.debugpy@2023.1.12492010"
+  ]
+
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "devcontainer"
+}

.devcontainer/ubuntu24.04/devcontainer.json

@@ -0,0 +1,26 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
+{
+	"name": "ubuntu24.04",
+	"build": {
+		// Sets the run context to one level up instead of the .devcontainer folder.
+		"context": "../..",
+		// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
+		"dockerfile": "../../Dockerfile.ubuntu24.04"
+	},
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [8000],
+
+	// Uncomment the next line to run commands after the container is created.
+	"postCreateCommand": "cat /etc/os-release"
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "devcontainer"
+}

.vscode/launch.json

@@ -0,0 +1,32 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Django: Runserver",
+      "type": "python",
+      "request": "launch",
+      "stopOnEntry": false,
+      "python": "${workspaceRoot}/donation_portal_env/bin/python",
+      "program": "${workspaceRoot}/manage.py",
+      "args": [
+        "runserver",
+        "--no-color",
+        "--noreload"
+      ],
+    },
+    {
+      "name": "Django: Shell",
+      "type": "python",
+      "request": "launch",
+      "stopOnEntry": false,
+      "python": "${workspaceRoot}/donation_portal_env/bin/python",
+      "program": "${workspaceRoot}/manage.py",
+      "args": [
+        "shell"
+      ]
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "python.pythonPath": "/workspaces/donation_portal-2/donation_portal_env/bin/python2.7"
+}

Dockerfile

@@ -2,8 +2,23 @@
 
 FROM ubuntu:16.04
 
+# Install sudo
+RUN apt-get update && apt-get install -y sudo
+
+# Create a new user "devuser" with user ID 1000
+RUN useradd -m -s /bin/bash -u 1000 devuser
+
+# Add the user to the sudo group
+RUN usermod -aG sudo devuser
+
+# Set up password-less sudo for the user
+RUN echo "devuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+# Switch to the new user
+USER devuser
+
 # Install OS level dependencies
-RUN apt-get update && apt-get install -y \
+RUN sudo apt-get update && sudo apt-get install -y \
   python \
   python-pip \
   build-essential \
@@ -25,30 +40,30 @@ USER postgres
 RUN service postgresql start && \
   psql -c "ALTER USER postgres PASSWORD 'password';" && \
   psql -c "CREATE DATABASE eaa;"
-USER root
+USER devuser
 
 # Upgrade pip to the last version that supports Python 2.7 and install the last
 # version of virtualenv that works for Python 2.7
 RUN pip install --upgrade "pip<21.0" && \
   pip install virtualenv==16.7.10
 
 # Set the working directory
-WORKDIR /usr/src/app
+# WORKDIR /usr/src/app
 
 # Copy over all the files from this repo
-COPY . .
+# COPY . .
 
 # Install nvm and node
-ENV NVM_DIR /usr/src/app/.nvm
+ENV NVM_DIR=/home/devuser/.nvm
 RUN mkdir $NVM_DIR && \
   curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash && \
   . $NVM_DIR/nvm.sh && nvm install 8.14.0
 
 # Create and activate a Python virtual environment and install the dependencies
-RUN virtualenv donation_portal_env && \
-  . donation_portal_env/bin/activate && \
-  pip install -r deps/pip.base && \
-  pip install -r deps/pip
+# RUN virtualenv donation_portal_env && \
+#   . donation_portal_env/bin/activate && \
+#   pip install -r deps/pip.base && \
+#   pip install -r deps/pip
 
 # Document that the service listens on port 8000. Note, this doesn't actually
 # open the port - you'll need to run `docker run -p 8000:8000 <image name>`.

Dockerfile.ubuntu24.04

@@ -0,0 +1,88 @@
+# For local development and testing only.
+
+FROM ubuntu:24.04
+
+# Avoid prompts from apt
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Set environment variables
+ENV PYENV_ROOT=/root/.pyenv
+ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH
+
+# Update and install dependencies
+RUN apt-get update && apt-get upgrade -y \
+    && apt-get install -y \
+        # Essential build tools
+        build-essential \
+        # Libraries for building Python
+        libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev \
+        # Additional libraries that some Python packages might need
+        libncursesw5-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
+        # Useful tools
+        wget curl git vim sudo \
+        # App specific dependencies
+        wkhtmltopdf \
+        xauth \
+        xvfb \
+        # PostgreSQL
+        postgresql \
+        libpq-dev \
+        # Redis
+        redis-server \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create a new user "devuser" with user ID 1000
+RUN useradd -m -s /bin/bash -u 1000 devuser
+
+# Add the user to the sudo group
+RUN usermod -aG sudo devuser
+
+# Set up password-less sudo for the user
+RUN echo "devuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+# Switch to the new user
+USER devuser
+
+# Install pyenv
+RUN curl https://pyenv.run | bash
+
+# Add pyenv initialization to bash profile
+RUN echo 'eval "$(pyenv init -)"' >> ~/.bashrc
+
+# Install Python versions
+RUN pyenv install 2.7.18
+
+# Set global Python version
+RUN pyenv global 2.7.18
+
+# Start PostgreSQL, set a password and create eaa database
+USER postgres
+RUN service postgresql start && \
+  psql -c "ALTER USER postgres PASSWORD 'password';" && \
+  psql -c "CREATE DATABASE eaa;"
+USER devuser
+
+# Upgrade pip to the last version that supports Python 2.7 and install the last
+# version of virtualenv that works for Python 2.7
+RUN pip install --upgrade "pip<21.0" && \
+  pip install virtualenv==16.7.10
+
+# Install nvm and node
+ENV NVM_DIR=/root/.nvm
+RUN mkdir $NVM_DIR && \
+  curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash && \
+  . $NVM_DIR/nvm.sh && nvm install 8.14.0
+
+# Copy over python dependencies
+# COPY deps deps
+
+# Create and activate a Python virtual environment and install the dependencies
+# RUN virtualenv donation_portal_env && \
+#   . donation_portal_env/bin/activate && \
+#   pip install -r deps/pip.base && \
+#   pip install -r deps/pip
+
+# Document that the service listens on port 8000. Note, this doesn't actually
+# open the port - you'll need to run `docker run -p 8000:8000 <image name>`.
+EXPOSE 8000

donation/emails/partner_charity_report.py

@@ -17,8 +17,9 @@
 def send_partner_charity_reports(test=True):
     # Create list of partners, combined the GiveDirectly entries
     partners = {partner.name: [partner.id] for partner in PartnerCharity.objects.all()}
-    partners['GiveDirectly'] += partners['GiveDirectly Basic income research']
-    del partners['GiveDirectly Basic income research']
+    if 'GiveDirectly' in partners and 'GiveDirectly Basic income research' in partners:
+        partners['GiveDirectly'] += partners['GiveDirectly Basic income research']
+        del partners['GiveDirectly Basic income research']
 
     for partner, ids in partners.iteritems():
         # Start time is when we last reported
@@ -82,14 +83,16 @@ def send_partner_charity_reports(test=True):
             ('Source', 'pledge__how_did_you_hear_about_us_db__reason'),
         ])
 
-        filename = 'EAA donation report - {0} - {1}.xlsx'.format(partner,
-                                                                 # Avoid collisions of filename while testing
-                                                                 timezone.now())
+        # We add timezone.now() to avoid collisions of filename while testing
+        filename = 'EAA donation report - {0} - {1}.xlsx'.format(partner, timezone.now())
+        filename_internal = 'EAA donation report - {0} - {1} - internal.xlsx'.format(partner, timezone.now())
         location = os.path.join(settings.MEDIA_ROOT, 'partner_reports', filename)
+        location_internal = os.path.join(settings.MEDIA_ROOT, 'partner_reports', filename_internal)
 
-        write_spreadsheet(location, querysets, template)
+        write_spreadsheet(location, querysets, template, cleaned=True)
+        write_spreadsheet(location_internal, querysets, template, cleaned=False)
 
-        # Create email
+        # Create email for partner charity (personal information removed)
         try:
             partner_obj = PartnerCharity.objects.get(id=ids[0])
             to = [partner_obj.email]
@@ -117,6 +120,31 @@ def send_partner_charity_reports(test=True):
             print e.message
             client.captureException()
 
+        # Create internal report (personal information included)
+        try:
+            partner_obj = PartnerCharity.objects.get(id=ids[0])
+            to = [settings.EAA_INFO_EMAIL]
+            cc = []
+            body = "Hi team, here's the internal report for {0} from {1} to {2}".format(partner, start.strftime('%d-%m-%Y'), end.strftime('%d-%m-%Y'))
+            message = EmailMessage(
+                subject="Internal report for {0} from {1} to {2}".format(partner, start.strftime('%d-%m-%Y'), end.strftime('%d-%m-%Y')),
+                body=body,
+                to=to if not test else [settings.TESTING_EMAIL],
+                cc=cc if not test else [settings.TESTING_EMAIL],
+                # There is a filter in info@eaa.org.au
+                #   from:(donations @ eaa.org.au) deliveredto:(info + receipts @ eaa.org.au)
+                # that automatically archives messages sent to info+receipt and adds the label 'receipts'
+                # bcc=["info+receipt@eaa.org.au", ],
+                bcc=[],
+                from_email=settings.POSTMARK_SENDER,
+            )
+            message.attach_file(location_internal, mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
+            message.send()
+
+        except Exception as e:
+            print e.message
+            client.captureException()
+
         if not test:
             partner = PartnerCharity.objects.get(id=ids[0])
             PartnerCharityReport(partner=partner, date=end).save()

donation/views/export.py

@@ -76,13 +76,20 @@ def download_spreadsheet(request, extra_fields=None):
         return response
 
 
-def write_spreadsheet(location, querysets, template):
+def write_spreadsheet(location, querysets, template, cleaned):
     with xlsxwriter.Workbook(location, {'default_date_format': 'dd mmm yyyy'}) as wb:
         for name, queryset in querysets.iteritems():
             ws = wb.add_worksheet(name=name)
             ws.write_row(0, 0, template.keys())
             row_number = 0
             for row in queryset.values_list(*template.values()):
+                if cleaned:
+                    row_list = list(row)
+                    if not row_list[9]:
+                        row_list[5] = "anonymous"
+                        row_list[6] = "anonymous"
+                        row_list[7] = "anonymous"    
+                    row = tuple(row_list)
                 row_number += 1
                 # Resolve any Enums
                 row = [value.label if isinstance(value, Enum) else value for value in row]

donation/views/pledge.py

@@ -162,26 +162,19 @@ def post(self, request):
                 )
 
 
-
-
             stripe.api_key = settings.STRIPE_API_KEY_DICT.get("eaae" if pledge.is_eaae else "eaa")
+            session_options = {
+                'payment_method_types': ['card'],
+                'line_items': line_items,
+                'success_url': 'http://localhost:8000/pledge_new/?thankyou' if settings.DEBUG else 'https://effectivealtruism.org.au/donate/?thankyou',
+                'cancel_url': 'http://localhost:8000/pledge_new/' if settings.DEBUG else 'https://effectivealtruism.org.au/donate/',
+            }
             if (pledge.recurring_frequency == RecurringFrequency.MONTHLY):
-                session = stripe.checkout.Session.create(
-                    payment_method_types=['card'],
-                    line_items=line_items,
-                    mode='subscription',
-                    success_url='https://effectivealtruism.org.au/donate/?thankyou',
-                    cancel_url='https://effectivealtruism.org.au/donate/',
-                )
+                session_options['mode'] = 'subscription'
             else:
-                session = stripe.checkout.Session.create(
-                    payment_method_types=['card'],
-                    line_items=line_items,
-                    submit_type="donate",
-                    mode='payment',
-                    success_url='https://effectivealtruism.org.au/donate/?thankyou',
-                    cancel_url='https://effectivealtruism.org.au/donate/',
-                )
+                session_options['mode'] = 'payment'
+                session_options['submit_type'] = 'donate'
+            session = stripe.checkout.Session.create(**session_options)
             print(session.__dict__)
             pledge.stripe_checkout_id = session.id
             pledge.save()

start_services_dev.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Run stripe webhook listener
+echo "Remember to locally run: stripe listen --forward-to localhost:8000/stripe-webhooks/"
+
+# Start PostgreSQL service
+sudo service postgresql start
+
+# Start Redis service
+sudo service redis-server start
+
+# Start celery
+celery -A donation_portal worker -l info &
+
+# Apply database migrations
+python manage.py migrate
+
+# Run the Django development server
+python manage.py runserver 0.0.0.0:8000