Merge pull request #12 from bsa7/SMLP-011-implement-data-fetcher

SMLP-011 Implement data fetcher class

.pylintrc

@@ -144,15 +144,17 @@ confidence=HIGH,
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use "--disable=all --enable=classes
 # --disable=W".
-disable=raw-checker-failed,
-        bad-inline-option,
-        locally-disabled,
+disable=bad-inline-option,
+        deprecated-pragma,
         file-ignored,
-        suppressed-message,
         import-error,
+        locally-disabled,
+        no-name-in-module,
+        raw-checker-failed,
+        suppressed-message,
+        use-symbolic-message-instead,
         useless-suppression,
-        deprecated-pragma,
-        use-symbolic-message-instead
+        wrong-import-order
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

app/lib/api_client.py

@@ -1,20 +1,52 @@
 ''' This file contains class for fetch data from abstract API '''
 
+from abc import ABC, abstractmethod
+from app.types.api_exmo_responses import CandlesHistory
 import requests
 
-class ApiClient:
+class ApiClient(ABC):
   ''' This class is wrapper for API classes '''
-  def __init__(self, api_url: str):
-    self.__api_url = api_url
 
-  def get(self, api_path: str, request_attributes: dict) -> dict:
+  @abstractmethod
+  def _api_url(self) -> str:
+    ''' Returns API url '''
+    return ''
+
+  @abstractmethod
+  def fetch_data(self, symbol: str, from_timestamp: int, to_timestamp: int) -> CandlesHistory:
+    ''' This common method name for api_client implementation. It load data portion from API '''
+    return
+
+  def fetch_data_in_batches(self,
+                            symbol: str,
+                            from_timestamp: int,
+                            to_timestamp: int,
+                            batch_size_in_milliseconds: int) -> CandlesHistory:
+    ''' This common method name for api_client implementation. It load any volume of data in batches from API '''
+    time_intervals = self.__time_intervals(from_timestamp, to_timestamp, batch_size_in_milliseconds)
+    result: CandlesHistory = []
+    for [start_timestamp, finish_timestamp] in time_intervals:
+      result += self.fetch_data(symbol, from_timestamp = start_timestamp, to_timestamp = finish_timestamp)
+
+    return result
+
+  def _get(self, api_path: str, request_attributes: dict) -> dict:
     ''' This method implement GET request to API '''
     query: str = self.__query_string(request_attributes)
-    uri: str = f'{self.__api_url}{api_path}?{query}'
-    print(f'{uri=}')
+    uri: str = f'{self._api_url}{api_path}?{query}'
     response = requests.get(uri, timeout = 1)
     return response.json()
 
+  def __time_intervals(self,
+                       from_timestamp: int,
+                       to_timestamp: int,
+                       batch_size_in_milliseconds: int) -> list[list[int, int]]:
+    ''' This method splits a large time interval into parts of a valid size '''
+    intervals = [*range(from_timestamp, to_timestamp, batch_size_in_milliseconds), to_timestamp]
+    zipped = zip(intervals[:-1], intervals[1:])
+    return list(map(lambda x: [x[0] + 1, x[1]], zipped))
+
+
   def __query_string(self, attributes: dict) -> str:
     ''' Builds a query string for GET request '''
     query: list[str] = []

app/lib/api_exmo_client.py

@@ -3,28 +3,26 @@
 from app.lib.env import Env
 from app.types.api_exmo_responses import CandlesHistory
 
-class ApiExmoClient:
+class ApiExmoClient(ApiClient):
   ''' This class contains methods for fetching data from API '''
-  def __init__(self):
-    self.__api_client = ApiClient(self.__api_url)
 
-  def candles_history(self, symbol: str, from_timestamp: int, to_timestamp: int) -> CandlesHistory:
+  def fetch_data(self, symbol: str, from_timestamp: int, to_timestamp: int) -> CandlesHistory:
     ''' This method get candles of symbol from API for exact period '''
     request_attributes: dict = {
       'symbol': symbol,
       'from': from_timestamp,
       'to': to_timestamp,
       'resolution': 1 }
 
-    result = self.__api_client.get(self.__candles_history_path, request_attributes)
+    result = self._get(self.__candles_history_path, request_attributes)
     return result['candles']
 
+  @property
+  def _api_url(self) -> str:
+    ''' Returns exmo API url '''
+    return Env().get('API_EXMO_HOST')
+
   @property
   def __candles_history_path(self) -> str:
     ''' Returns candles history API path '''
     return '/candles_history'
-
-  @property
-  def __api_url(self) -> str:
-    ''' Returns exmo API url '''
-    return Env().get('API_EXMO_HOST')

app/lib/data_fetcher.py

@@ -0,0 +1,44 @@
+''' This file contains a DataFetcher class - this class working with data: load data, store data '''
+from datetime import datetime
+# from app.lib.env import Env
+
+class DataFetcher():
+  ''' This class fetches new portion of data from desired source. '''
+  DEFAULT_FETCH_INTERVAL = 1 * 60000 # 1.minute
+
+  def __init__(self,
+               api_client,
+               from_timestamp = None,
+               to_timestamp = None,
+               fetch_interval_size = DEFAULT_FETCH_INTERVAL):
+    self.__from_timestamp = from_timestamp
+    self.__to_timestamp = to_timestamp
+    self.__api_client = api_client
+    self.__fetch_interval_size = fetch_interval_size
+
+  def update(self, symbol: str):
+    ''' This method look over previous stored data and fetch new data '''
+    start_timestamp = self.__start_timestamp(symbol)
+    return self.__api_client.fetch_data_in_batches(symbol = symbol,
+                                                   from_timestamp = start_timestamp,
+                                                   to_timestamp = self.__finish_timestamp,
+                                                   batch_size_in_milliseconds = self.__fetch_interval_size)
+
+  def __start_timestamp(self, symbol: str) -> int:
+    ''' This method determines the last point in time beyond which the required
+        data is stored in the system. In fact, this moment plus one millisecond
+        is the start for the time interval for which the data will be received. '''
+    # In this point we would to find last data of time series in our db
+    print(f'{symbol=}')
+    if self.__from_timestamp is not None:
+      return self.__from_timestamp
+
+    return self.__finish_timestamp - self.__fetch_interval_size
+
+  @property
+  def __finish_timestamp(self) -> int:
+    ''' This method always returns current timestamp '''
+    if self.__to_timestamp is not None:
+      return self.__to_timestamp
+
+    return int(datetime.now().timestamp())

app/lib/db_client.py

@@ -0,0 +1,19 @@
+''' This file contains client for time series database '''
+from datetime import datetime
+from app.lib.singleton import Singleton
+
+class DbClient(Singleton):
+  ''' This class contains client for database '''
+  def __init__(self):
+    ''' Initialize connection '''
+    self.connection = None
+
+  def find_latest_timestamp_for_symbol(self, symbol: str) -> dict:
+    ''' This method finds the most recently stored value for a given pair and
+        returns a timestamp of that value. '''
+    result = {
+      'symbol': symbol,
+      'timestamp': datetime.now().timestamp() - 2 * 60000 # 2 minutes from now
+    }
+
+    return result

data_creation.py

@@ -1,9 +1,8 @@
 ''' This file run methods for data creation '''
 from app.lib.api_exmo_client import ApiExmoClient
-# from app.lib.utils import timestamp_to_formatteddatetime
+from app.lib.data_fetcher import DataFetcher
 
-# print(timestamp_to_formatteddatetime(1585557000000))
-
-result = ApiExmoClient().candles_history('BTC_USDT', 1585551900, 1585552000)
+data_fetcher = DataFetcher(api_client = ApiExmoClient(), fetch_interval_size = 60 * 1000)
+result = data_fetcher.update('BTC_USDT')
 
 print(f'{result=}')

scripts/run_pytest

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-python -m pytest test
+python -m pytest test --capture=sys

test/lib/api_exmo_client_test.py

@@ -3,6 +3,7 @@
 import responses
 from app.types.api_exmo_responses import CandlesHistory
 from app.lib.api_exmo_client import ApiExmoClient
+from test.support.mock_helper import stub_get_request
 
 class TestApiExmoClient(unittest.TestCase):
   ''' This class contains tests for ApiExmoClient class '''
@@ -12,11 +13,6 @@ def test_candles_history_request_with_valid_params(self):
     uri = 'https://api.exmo.com/v1.1/candles_history?from=1585551900&resolution=1&symbol=BTC_USDT&to=1585552000'
     expected_response: CandlesHistory = { 'candles': [{ 't': 1 }] }
     with responses.RequestsMock() as rsps:
-      self.__stub_get_request(rsps, uri, expected_response)
-      response = ApiExmoClient().candles_history('BTC_USDT', 1585551900, 1585552000)
+      stub_get_request(rsps, uri, expected_response)
+      response = ApiExmoClient().fetch_data('BTC_USDT', 1585551900, 1585552000)
       self.assertEqual(response, expected_response['candles'])
-
-  def __stub_get_request(self, rsps, api_uri: str, response: dict):
-    ''' This method creates a stub for a specific api endpoint and emulates a
-        successful data fetch '''
-    rsps.get(api_uri, json = response, status = 200)

test/lib/data_fetcher_test.py

@@ -0,0 +1,27 @@
+''' This file contains unit tests for app/lib/data_fetcher.py '''
+import unittest
+import responses
+from app.lib.data_fetcher import DataFetcher
+from app.lib.api_exmo_client import ApiExmoClient
+from app.types.api_exmo_responses import CandlesHistory
+from test.support.mock_helper import stub_get_request
+
+class TestDataFetcher(unittest.TestCase):
+  ''' This class contains tests for DataFetcher class '''
+  def test_update(self):
+    ''' This case runs the Update method to ensure that the third party API
+        call is being made and that the data received from both requests is
+        being accumulated. '''
+    uri1 = 'https://api.exmo.com/v1.1/candles_history?from=1585551901&resolution=1&symbol=BTC_USDT&to=1585551910'
+    uri2 = 'https://api.exmo.com/v1.1/candles_history?from=1585551911&resolution=1&symbol=BTC_USDT&to=1585551920'
+    expected_response1: CandlesHistory = { 'candles': [{ 't': 1 }] }
+    expected_response2: CandlesHistory = { 'candles': [{ 't': 2 }] }
+    data_fetcher = DataFetcher(api_client = ApiExmoClient(),
+                               from_timestamp = 1585551900,
+                               to_timestamp = 1585551920,
+                               fetch_interval_size = 10)
+    with responses.RequestsMock() as rsps:
+      stub_get_request(rsps, uri1, expected_response1)
+      stub_get_request(rsps, uri2, expected_response2)
+      result = data_fetcher.update(symbol = 'BTC_USDT')
+      self.assertEqual(result, expected_response1['candles'] + expected_response2['candles'])

test/support/mock_helper.py

@@ -0,0 +1,6 @@
+''' This file contain helpers for mock external service responses '''
+
+def stub_get_request(rsps, api_uri: str, response: dict):
+  ''' This method creates a stub for a specific api endpoint and emulates a
+      successful data fetch '''
+  rsps.get(api_uri, json = response, status = 200)