Skip to content

Commit

Permalink
Sources availability
Browse files Browse the repository at this point in the history
  • Loading branch information
JonanOribe committed Jan 30, 2023
1 parent 50365b2 commit cd0a48b
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 1 deletion.
5 changes: 4 additions & 1 deletion scrapper/data_scrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,7 @@

class DataScrapper:
def __init__(self, source) -> None:
self.source:str = {section: dict(config.items(section)) for section in config.sections()}['URLS'][source.lower()]
self.source:str = {section: dict(config.items(section)) for section in config.sections()}['URLS'][source.lower()]

def get_scrapper_sources(self):
return {section: dict(config.items(section)) for section in config.sections()}['URLS']
7 changes: 7 additions & 0 deletions scrapper/supreme_court_united_states.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,13 @@ def get_data(self, output_path:str, format:str='json', save_data_on_file:bool=Fa

return 'Data was saved! on {}'.format(output_path) if save_data_on_file else response

def get_areas(self):
completed_url:str = self.source
headers = {
'Content-Type': 'application/json'
}
return requests.request("GET", completed_url, headers=headers, data={}).json()

def __save_data(self, file_name:str, data,format:str, output_path:str):
if format == 'json':
formatted_output_path:str = '{}{}.{}'.format(output_path, file_name, format)
Expand Down
8 changes: 8 additions & 0 deletions tests/data_scrapper_test.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from typing import List
import unittest
from scrapper import DataScrapper
import configparser
Expand All @@ -12,6 +13,13 @@ class DataScrapperTestCase(unittest.TestCase):
def setUp(self):
self.DataScrapper = DataScrapper(source)

def test_get_scrapper_sources_counter(self):
self.assertEqual(len(self.DataScrapper.get_scrapper_sources()),2)

def test_get_scrapper_sources_options(self):
options:List = self.DataScrapper.get_scrapper_sources()
self.assertEqual([key for key in options.keys()][0],source.lower())

def test_get_valid_url(self):
object_source:str = self.DataScrapper.source
valid_url:str = config['URLS'][source.lower()]
Expand Down
4 changes: 4 additions & 0 deletions tests/supreme_court_united_states_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ class SupremeCourtUnitedStatesTestCase(unittest.TestCase):
def setUp(self):
self.SupremeCourtUnitedStates = SupremeCourtUnitedStates(source,area)

def test_get_areas(self):
options = self.SupremeCourtUnitedStates.get_areas()
self.assertGreater(len([key for key in options.keys()]),1)

def test_get_first_element(self):
result = self.SupremeCourtUnitedStates.get_data(output_path,'json',False)
self.assertEqual(result['results'][0]['id'], 'scotus')
Expand Down

0 comments on commit cd0a48b

Please sign in to comment.