⌨️ fix: pytest ⌨️ (#119)

* pytest

* update deps

* fix ohlc and intraday tests?

* stocktwits

* update sentiment script

* test token

.github/workflows/build.yml

@@ -58,7 +58,7 @@ jobs:
       - name: Run all unit tests
         env:
           S3_BUCKET: ${{ secrets.S3_DEV_BUCKET }}
-        run: coverage run -m pytest -vv
+        run: coverage run -m pytest -vv -s
 
       - name: Generate test coverage report
         run: coverage report -m --fail-under=90

.github/workflows/deps.yml

@@ -65,7 +65,7 @@ jobs:
       - name: Run all unit tests
         env:
           S3_BUCKET: ${{ secrets.S3_DEV_BUCKET }}
-        run: coverage run -m pytest -vv
+        run: coverage run -m pytest -vv -s
 
       - name: Generate test coverage report
         run: coverage report -m --fail-under=90
@@ -93,4 +93,4 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          tag_name: ${{ steps.bumpVersion.outputs.new_tag }}
+          tag_name: ${{ steps.bumpVersion.outputs.new_tag }}

.github/workflows/sandbox.yml

@@ -58,7 +58,7 @@ jobs:
           flake8 . --count --max-complexity=10 --max-line-length=127 --statistics
 
       - name: Run all unit tests
-        run: coverage run -m pytest -vv
+        run: coverage run -m pytest -vv -s
 
       - name: Generate test coverage report
         run: coverage report -m --fail-under=90
@@ -77,7 +77,7 @@ jobs:
 
       - name: Update social sentiment
         run: python scripts/update_sentiment.py
-      
+
       # - name: Update intraday
       #   run: python scripts/update_intraday.py
 

requirements.txt

@@ -1,7 +1,7 @@
 python-dotenv == 0.19.0
-pandas == 1.3.1
+pandas == 1.3.2
 robin-stocks == 2.0.4
-boto3 == 1.18.11
+boto3 == 1.18.36
 polygon-api-client == 0.2.11
 pytz == 2021.1
-vectorbt == 0.19.2
+vectorbt == 0.21.0

scripts/update_sentiment.py

@@ -7,6 +7,8 @@
 
 
 twit = StockTwits()
+no_auth_twit = StockTwits()
+no_auth_twit.token = ''
 symbols = twit.get_symbols()
 crypto_symbols = ['BTC-X', 'ETH-X', 'LTC-X', 'XMR-X', 'IOT-X']
 if C.TEST:
@@ -18,17 +20,30 @@
 # better solution is to dynamically choose 175 most outdated symbols
 
 # First batch
-for symbol in symbols[C.TWIT_RATE*(BATCH-1):C.TWIT_RATE*BATCH]:
+
+current_batch = symbols[C.TWIT_RATE*(BATCH-1):C.TWIT_RATE*BATCH]
+num_failed = 0
+
+for symbol in current_batch:
     if symbol in C.SENTIMENT_SYMBOLS_IGNORE:
         continue
     try:
-        twit.save_social_sentiment(symbol=symbol, timeframe='1d',
-                                   retries=1 if C.TEST else 2)
+        twit.save_social_sentiment(symbol=symbol, timeframe='30d',
+                                   retries=1)
     except Exception as e:
-        print(f'Stocktwits sentiment update failed for {symbol}.')
-        print(e)
+        try:
+            no_auth_twit.save_social_sentiment(symbol=symbol, timeframe='30d',
+                                               retries=1)
+        except Exception as e2:
+            num_failed += 1
+            print(f'Stocktwits sentiment update failed for {symbol}.')
+            print(e)
+            print(e2)
     finally:
         filename = PathFinder().get_sentiment_path(
             symbol=symbol, provider=twit.provider)
         if C.CI and os.path.exists(filename):
             os.remove(filename)
+
+if num_failed / len(current_batch) > 0.5:
+    raise Exception('Majority failure.')

src/DataSource.py

@@ -734,13 +734,17 @@ def _get_social_volume(symbol, timeframe='max'):
             url = '/'.join(parts)
             params = {'access_token': self.token}
             vol_res = requests.get(url, params=params)
+            json_res = vol_res.json()
             empty = pd.DataFrame()
 
             if vol_res.ok:
-                vol_data = vol_res.json()['data']
+                vol_data = json_res['data']
             else:
+                if 'errors' in json_res:
+                    errors = '\n'.join([error['message']
+                                        for error in json_res['errors']])
                 raise Exception(
-                    f'Invalid response from Stocktwits for {symbol}')
+                    f'Invalid response from Stocktwits for {symbol}\n{errors}')
 
             if vol_data == []:
                 return empty
@@ -772,13 +776,17 @@ def _get_social_sentiment(symbol, timeframe='max'):
             url = '/'.join(parts)
             params = {'access_token': self.token}
             sen_res = requests.get(url, params=params)
+            json_res = sen_res.json()
             empty = pd.DataFrame()
 
             if sen_res.ok:
-                sen_data = sen_res.json()['data']
+                sen_data = json_res['data']
             else:
+                if 'errors' in json_res:
+                    errors = '\n'.join([error['message']
+                                        for error in json_res['errors']])
                 raise Exception(
-                    f'Invalid response from Stocktwits for {symbol}.')
+                    f'Invalid response from Stocktwits for {symbol}\n{errors}')
 
             if sen_data == []:
                 return empty

test/test_DataSource.py

@@ -15,6 +15,7 @@
 iex = IEXCloud()
 poly = Polygon()
 twit = StockTwits()
+twit.token = ''
 bls = LaborStats()
 glass = Glassnode()
 flow = Flow()
@@ -276,13 +277,13 @@ def test_save_intraday(self):
             os.remove(path)
 
     def test_get_ohlc(self):
-        df = md.get_ohlc('TSLA', '2m')
+        df = md.get_ohlc('NFLX', '2m')
         assert {C.TIME, C.OPEN, C.HIGH, C.LOW,
                 C.CLOSE, C.VOL}.issubset(df.columns)
         assert len(df) > 0
 
     def test_get_intraday(self):
-        df = pd.concat(md.get_intraday(symbol='TSLA', timeframe='2m'))
+        df = pd.concat(md.get_intraday(symbol='NFLX', timeframe='2m'))
         assert {C.TIME, C.OPEN, C.HIGH, C.LOW,
                 C.CLOSE, C.VOL}.issubset(df.columns)
         assert len(df) > 0