From 91bf1cbf1877ee0e507cb72bf4082350f1f57030 Mon Sep 17 00:00:00 2001
From: Chris Briggs <confusedmatrix@gmail.com>
Date: Thu, 25 Jan 2024 16:39:17 +0000
Subject: [PATCH] Added requets and beautifulsoup deps for web scraping

---
 poetry.lock               | 34 +++++++++++++++++++++++++++++++++-
 pyproject.toml            |  2 ++
 ruvnl_consumer_app/app.py | 13 +++++--------
 3 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 41faf67..dd90add 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -30,6 +30,27 @@ tests = ["attrs[tests-no-zope]", "zope-interface"]
 tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"]
 tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"]
 
+[[package]]
+name = "beautifulsoup4"
+version = "4.12.3"
+description = "Screen-scraping library"
+optional = false
+python-versions = ">=3.6.0"
+files = [
+    {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"},
+    {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"},
+]
+
+[package.dependencies]
+soupsieve = ">1.2"
+
+[package.extras]
+cchardet = ["cchardet"]
+chardet = ["chardet"]
+charset-normalizer = ["charset-normalizer"]
+html5lib = ["html5lib"]
+lxml = ["lxml"]
+
 [[package]]
 name = "certifi"
 version = "2023.11.17"
@@ -1063,6 +1084,17 @@ files = [
     {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
 ]
 
+[[package]]
+name = "soupsieve"
+version = "2.5"
+description = "A modern CSS selector implementation for Beautiful Soup."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"},
+    {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"},
+]
+
 [[package]]
 name = "sqlalchemy"
 version = "1.4.46"
@@ -1279,4 +1311,4 @@ files = [
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.11"
-content-hash = "51011b8b39bb44352244c774498c96be14f9d5626357a09e04aa81e2a2d9ba0d"
+content-hash = "85acc49e20cea3963ac71395b09891408e92b208a9461ca0f270d66569c673de"
diff --git a/pyproject.toml b/pyproject.toml
index ff81823..fa2c59b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,6 +10,8 @@ python = "^3.11"
 click = "^8.1.7"
 pvsite-datamodel = "^1.0.10"
 pandas = "1.5.3"
+requests = "^2.31.0"
+beautifulsoup4 = "^4.12.3"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.4"
diff --git a/ruvnl_consumer_app/app.py b/ruvnl_consumer_app/app.py
index 0896d0b..4d21312 100644
--- a/ruvnl_consumer_app/app.py
+++ b/ruvnl_consumer_app/app.py
@@ -4,12 +4,9 @@
 
 import datetime as dt
 import logging
-import os
 import sys
 
 import click
-from pvsite_datamodel import DatabaseConnection
-
 
 log = logging.getLogger(__name__)
 
@@ -50,12 +47,12 @@ def app(timestamp: dt.datetime | None, write_to_db: bool, log_level: str):
         timestamp.replace(tzinfo=dt.UTC)
         
     # 0. Initialise DB connection
-    url = os.environ["DB_URL"]
+    # url = os.environ["DB_URL"]
+
+    # db_conn = DatabaseConnection(url, echo=False)
+    # with db_conn.get_session() as session:
 
-    db_conn = DatabaseConnection(url, echo=False)
-    
-    with db_conn.get_session() as session:
-        pass
+    pass
 
 
 if __name__ == "__main__":