SMLP-013 Convert all used timestamps to timestamps with milliseconds

app/lib/api_client.py

@@ -2,10 +2,13 @@
 
 from abc import ABC, abstractmethod
 from app.types.api_exmo_responses import CandlesHistory
+from app.lib.utils import seconds, timestamp_to_formatted_datetime
 import requests
 
 class ApiClient(ABC):
   ''' This class is wrapper for API classes '''
+  def __init__(self, resolution: int):
+    self._resolution = resolution
 
   @abstractmethod
   def _api_url(self) -> str:
@@ -26,6 +29,7 @@ def fetch_data_in_batches(self,
     time_intervals = self.__time_intervals(from_timestamp, to_timestamp, batch_size_in_milliseconds)
     result: CandlesHistory = []
     for [start_timestamp, finish_timestamp] in time_intervals:
+      self.__log_interval(start_timestamp, finish_timestamp)
       result += self.fetch_data(symbol, from_timestamp = start_timestamp, to_timestamp = finish_timestamp)
 
     return result
@@ -37,14 +41,19 @@ def _get(self, api_path: str, request_attributes: dict) -> dict:
     response = requests.get(uri, timeout = 1)
     return response.json()
 
+  def __log_interval(self, start_timestamp: int, finish_timestamp: int):
+    start_time = timestamp_to_formatted_datetime(start_timestamp)
+    finish_time = timestamp_to_formatted_datetime(finish_timestamp)
+    print(f'Fetch data for interval from {start_time} to {finish_time}')
+
   def __time_intervals(self,
                        from_timestamp: int,
                        to_timestamp: int,
                        batch_size_in_milliseconds: int) -> list[list[int, int]]:
     ''' This method splits a large time interval into parts of a valid size '''
     intervals = [*range(from_timestamp, to_timestamp, batch_size_in_milliseconds), to_timestamp]
     zipped = zip(intervals[:-1], intervals[1:])
-    return list(map(lambda x: [x[0] + 1, x[1]], zipped))
+    return list(map(lambda x: [x[0] + seconds(1), x[1]], zipped))
 
 
   def __query_string(self, attributes: dict) -> str:

app/lib/api_exmo_client.py

@@ -10,9 +10,9 @@ def fetch_data(self, symbol: str, from_timestamp: int, to_timestamp: int) -> Can
     ''' This method get candles of symbol from API for exact period '''
     request_attributes: dict = {
       'symbol': symbol,
-      'from': from_timestamp,
-      'to': to_timestamp,
-      'resolution': 1 }
+      'from': int(from_timestamp / 1000),
+      'to': int(to_timestamp / 1000),
+      'resolution': self._resolution }
 
     result = self._get(self.__candles_history_path, request_attributes)
     return result['candles']

app/lib/data_fetcher.py

@@ -1,28 +1,27 @@
 ''' This file contains a DataFetcher class - this class working with data: load data, store data '''
-from datetime import datetime
-# from app.lib.env import Env
+from app.lib.utils import current_timestamp, minutes
 
 class DataFetcher():
   ''' This class fetches new portion of data from desired source. '''
-  DEFAULT_FETCH_INTERVAL = 1 * 60000 # 1.minute
+  DEFAULT_FETCH_INTERVAL = minutes(1)
 
   def __init__(self,
                api_client,
                from_timestamp = None,
                to_timestamp = None,
-               fetch_interval_size = DEFAULT_FETCH_INTERVAL):
+               batch_size_in_milliseconds = DEFAULT_FETCH_INTERVAL):
     self.__from_timestamp = from_timestamp
     self.__to_timestamp = to_timestamp
     self.__api_client = api_client
-    self.__fetch_interval_size = fetch_interval_size
+    self.__batch_size_in_milliseconds = batch_size_in_milliseconds
 
   def update(self, symbol: str):
     ''' This method look over previous stored data and fetch new data '''
     start_timestamp = self.__start_timestamp(symbol)
     return self.__api_client.fetch_data_in_batches(symbol = symbol,
                                                    from_timestamp = start_timestamp,
                                                    to_timestamp = self.__finish_timestamp,
-                                                   batch_size_in_milliseconds = self.__fetch_interval_size)
+                                                   batch_size_in_milliseconds = self.__batch_size_in_milliseconds)
 
   def __start_timestamp(self, symbol: str) -> int:
     ''' This method determines the last point in time beyond which the required
@@ -33,12 +32,12 @@ def __start_timestamp(self, symbol: str) -> int:
     if self.__from_timestamp is not None:
       return self.__from_timestamp
 
-    return self.__finish_timestamp - self.__fetch_interval_size
+    return self.__finish_timestamp - self.__batch_size_in_milliseconds
 
   @property
   def __finish_timestamp(self) -> int:
     ''' This method always returns current timestamp '''
     if self.__to_timestamp is not None:
       return self.__to_timestamp
 
-    return int(datetime.now().timestamp())
+    return current_timestamp()

app/lib/utils.py

@@ -5,3 +5,23 @@ def timestamp_to_formatted_datetime(timestamp: int) -> str:
   ''' Converts a unix timestamp with milliseconds (1585557000000) to formatted date like:
       2020-03-30 08:30:00.231000 '''
   return datetime.utcfromtimestamp(timestamp / 1000.0).strftime('%Y-%m-%d %H:%M:%S.%f')
+
+def current_timestamp() -> int:
+  ''' This method returns the current time in unix timestamp (in milliseconds) '''
+  return int(datetime.now().timestamp() * 1000)
+
+def seconds(value: float) -> int:
+  ''' This method returns milliseconds for value of seconds '''
+  return int(value * 1000)
+
+def minutes(value: float) -> int:
+  ''' This method returns milliseconds for value of minutes '''
+  return seconds(value * 60)
+
+def hours(value: float) -> int:
+  ''' This method returns milliseconds for value of hours '''
+  return minutes(value * 60)
+
+def days(value: float) -> int:
+  ''' This method returns milliseconds for value of days '''
+  return hours(value * 24)

data_creation.py

@@ -1,8 +1,14 @@
 ''' This file run methods for data creation '''
 from app.lib.api_exmo_client import ApiExmoClient
 from app.lib.data_fetcher import DataFetcher
+from app.lib.utils import current_timestamp, days
+
+from_timestamp = current_timestamp() - days(365)
+data_fetcher = DataFetcher(api_client = ApiExmoClient(resolution = 'D'),
+                           from_timestamp = from_timestamp,
+                           to_timestamp = current_timestamp(),
+                           batch_size_in_milliseconds = days(25))
 
-data_fetcher = DataFetcher(api_client = ApiExmoClient(), fetch_interval_size = 60 * 1000)
 result = data_fetcher.update('BTC_USDT')
 
 print(f'{result=}')

test/lib/api_exmo_client_test.py

@@ -12,7 +12,10 @@ def test_candles_history_request_with_valid_params(self):
         with correct request parameters '''
     uri = 'https://api.exmo.com/v1.1/candles_history?from=1585551900&resolution=1&symbol=BTC_USDT&to=1585552000'
     expected_response: CandlesHistory = { 'candles': [{ 't': 1 }] }
+    api_exmo_client = ApiExmoClient(resolution = 1)
     with responses.RequestsMock() as rsps:
       stub_get_request(rsps, uri, expected_response)
-      response = ApiExmoClient().fetch_data('BTC_USDT', 1585551900, 1585552000)
+      response = api_exmo_client.fetch_data(symbol = 'BTC_USDT',
+                                            from_timestamp = 1585551900000,
+                                            to_timestamp = 1585552000000)
       self.assertEqual(response, expected_response['candles'])

test/lib/data_fetcher_test.py

@@ -3,23 +3,44 @@
 import responses
 from app.lib.data_fetcher import DataFetcher
 from app.lib.api_exmo_client import ApiExmoClient
+from app.lib.utils import days, seconds
 from app.types.api_exmo_responses import CandlesHistory
 from test.support.mock_helper import stub_get_request
 
 class TestDataFetcher(unittest.TestCase):
   ''' This class contains tests for DataFetcher class '''
-  def test_update(self):
+  def test_update_with_default_resolution(self):
     ''' This case runs the Update method to ensure that the third party API
         call is being made and that the data received from both requests is
         being accumulated. '''
     uri1 = 'https://api.exmo.com/v1.1/candles_history?from=1585551901&resolution=1&symbol=BTC_USDT&to=1585551910'
     uri2 = 'https://api.exmo.com/v1.1/candles_history?from=1585551911&resolution=1&symbol=BTC_USDT&to=1585551920'
     expected_response1: CandlesHistory = { 'candles': [{ 't': 1 }] }
     expected_response2: CandlesHistory = { 'candles': [{ 't': 2 }] }
-    data_fetcher = DataFetcher(api_client = ApiExmoClient(),
-                               from_timestamp = 1585551900,
-                               to_timestamp = 1585551920,
-                               fetch_interval_size = 10)
+    data_fetcher = DataFetcher(api_client = ApiExmoClient(resolution = 1),
+                               from_timestamp = 1585551900000,
+                               to_timestamp = 1585551920000,
+                               batch_size_in_milliseconds = seconds(10))
+
+    with responses.RequestsMock() as rsps:
+      stub_get_request(rsps, uri1, expected_response1)
+      stub_get_request(rsps, uri2, expected_response2)
+      result = data_fetcher.update(symbol = 'BTC_USDT')
+      self.assertEqual(result, expected_response1['candles'] + expected_response2['candles'])
+
+  def test_update_with_long_resolution(self):
+    ''' This case runs the Update method to ensure that the third party API
+        call is being made and that the data received from both requests is
+        being accumulated. Also, resolution of candles is a days '''
+    uri1 = 'https://api.exmo.com/v1.1/candles_history?from=1583219101&resolution=D&symbol=BTC_USDT&to=1585379100'
+    uri2 = 'https://api.exmo.com/v1.1/candles_history?from=1585379101&resolution=D&symbol=BTC_USDT&to=1585551900'
+    expected_response1: CandlesHistory = { 'candles': [{ 't': 1 }] }
+    expected_response2: CandlesHistory = { 'candles': [{ 't': 2 }] }
+    data_fetcher = DataFetcher(api_client = ApiExmoClient(resolution = 'D'),
+                               from_timestamp = 1583219100000,
+                               to_timestamp = 1585551900000,
+                               batch_size_in_milliseconds = days(25))
+
     with responses.RequestsMock() as rsps:
       stub_get_request(rsps, uri1, expected_response1)
       stub_get_request(rsps, uri2, expected_response2)

test/lib/utils/current_timestamp_test.py

@@ -0,0 +1,11 @@
+''' This file contains unit tests for app/lib/utils.py '''
+import unittest
+from datetime import datetime
+from app.lib.utils import current_timestamp
+
+class TestCurrentTimestamp(unittest.TestCase):
+  ''' This class runs all tests for current_timestamp method '''
+  def test_current_timestamp_returns_current_time(self):
+    ''' This case checks the result for a valid timestamp '''
+    expected_timestamp = int(datetime.now().timestamp() * 1000)
+    self.assertEqual(current_timestamp(), expected_timestamp)

test/lib/utils/days_test.py

@@ -0,0 +1,17 @@
+''' This file contains unit tests for app/lib/utils.py '''
+import unittest
+from app.lib.utils import days
+
+class TestDays(unittest.TestCase):
+  ''' This class runs all tests for days method '''
+  def test_days_returns_zero(self):
+    ''' This case checks the result for a valid zero timestamp '''
+    self.assertEqual(days(0), 0)
+
+  def test_days_returns_for_integer_argument(self):
+    ''' This case checks the correct result for integer argument '''
+    self.assertEqual(days(11), 950400000)
+
+  def test_days_returns_for_float_argument(self):
+    ''' This case checks the correct result for float argument '''
+    self.assertEqual(days(1.45), 125280000)

test/lib/utils/hours_test.py

@@ -0,0 +1,17 @@
+''' This file contains unit tests for app/lib/utils.py '''
+import unittest
+from app.lib.utils import hours
+
+class TestHours(unittest.TestCase):
+  ''' This class runs all tests for hours method '''
+  def test_hours_returns_zero(self):
+    ''' This case checks the result for a valid zero timestamp '''
+    self.assertEqual(hours(0), 0)
+
+  def test_hours_returns_for_integer_argument(self):
+    ''' This case checks the correct result for integer argument '''
+    self.assertEqual(hours(11), 39600000)
+
+  def test_hours_returns_for_float_argument(self):
+    ''' This case checks the correct result for float argument '''
+    self.assertEqual(hours(1.45), 5220000)

test/lib/utils/minutes_test.py

@@ -0,0 +1,17 @@
+''' This file contains unit tests for app/lib/utils.py '''
+import unittest
+from app.lib.utils import minutes
+
+class TestMinutes(unittest.TestCase):
+  ''' This class runs all tests for minutes method '''
+  def test_minutes_returns_zero(self):
+    ''' This case checks the result for a valid zero timestamp '''
+    self.assertEqual(minutes(0), 0)
+
+  def test_minutes_returns_for_integer_argument(self):
+    ''' This case checks the correct result for integer argument '''
+    self.assertEqual(minutes(11), 660000)
+
+  def test_minutes_returns_for_float_argument(self):
+    ''' This case checks the correct result for float argument '''
+    self.assertEqual(minutes(1.45), 87000)

test/lib/utils/seconds_test.py

@@ -0,0 +1,17 @@
+''' This file contains unit tests for app/lib/utils.py '''
+import unittest
+from app.lib.utils import seconds
+
+class TestSeconds(unittest.TestCase):
+  ''' This class runs all tests for seconds method '''
+  def test_seconds_returns_zero(self):
+    ''' This case checks the result for a valid zero timestamp '''
+    self.assertEqual(seconds(0), 0)
+
+  def test_seconds_returns_for_integer_argument(self):
+    ''' This case checks the correct result for integer argument '''
+    self.assertEqual(seconds(11), 11000)
+
+  def test_seconds_returns_for_float_argument(self):
+    ''' This case checks the correct result for float argument '''
+    self.assertEqual(seconds(1.45), 1450)