From 8dd39147ad6f0a0291fe66285ba922b72f023f29 Mon Sep 17 00:00:00 2001 From: Kevin Sheppard Date: Wed, 24 Jan 2018 17:38:06 +0000 Subject: [PATCH] ENH: Add readers for Tiingo data Add daily, quote and metadata readers for Tiingo closes #390 --- docs/source/readers/index.rst | 1 + docs/source/readers/tiingo.rst | 18 +++ docs/source/remote_data.rst | 18 +++ docs/source/whatsnew/v0.6.0.txt | 7 + pandas_datareader/data.py | 14 ++ pandas_datareader/tests/test_tiingo.py | 44 ++++++ pandas_datareader/tiingo.py | 191 +++++++++++++++++++++++++ requirements.txt | 1 + 8 files changed, 294 insertions(+) create mode 100644 docs/source/readers/tiingo.rst create mode 100644 pandas_datareader/tests/test_tiingo.py create mode 100644 pandas_datareader/tiingo.py diff --git a/docs/source/readers/index.rst b/docs/source/readers/index.rst index bcedd791..c25eebdb 100644 --- a/docs/source/readers/index.rst +++ b/docs/source/readers/index.rst @@ -17,5 +17,6 @@ Data Readers quandl robinhood stooq + tiingo tsp world-bank diff --git a/docs/source/readers/tiingo.rst b/docs/source/readers/tiingo.rst new file mode 100644 index 00000000..e6ca873d --- /dev/null +++ b/docs/source/readers/tiingo.rst @@ -0,0 +1,18 @@ +Tiingo +------ + +.. py:module:: pandas_datareader.tiingo + +.. autoclass:: TiingoDailyReader + :members: + :inherited-members: + +.. autoclass:: TiingoQuoteReader + :members: + :inherited-members: + +.. autoclass:: TiingoMetaDataReader + :members: + :inherited-members: + +.. autofunction:: get_tiingo_symbols diff --git a/docs/source/remote_data.rst b/docs/source/remote_data.rst index 481e2141..ea66156c 100644 --- a/docs/source/remote_data.rst +++ b/docs/source/remote_data.rst @@ -66,6 +66,24 @@ Google Finance f = web.DataReader('F', 'google', start, end) f.ix['2010-01-04'] +.. _remote_data.tiingo: + +Tiingo +====== +`Tiingo `__ is a tracing platform that provides a data +api with historical end-of-day prices on equities, mutual funds and ETFs. +Free registration is required to get an API key. Free accounts are rate +limited and can access a limited number of symbols (500 at the time of +writing). + +.. ipython:: python + + import os + import pandas_datareader as pdr + + df = pdr.get_data_tiingo('GOOG', api_key=os.getenv('TIINGO_API_KEY')) + df.head() + .. _remote_data.morningstar: Morningstar diff --git a/docs/source/whatsnew/v0.6.0.txt b/docs/source/whatsnew/v0.6.0.txt index 88065746..fd3b40df 100644 --- a/docs/source/whatsnew/v0.6.0.txt +++ b/docs/source/whatsnew/v0.6.0.txt @@ -24,6 +24,10 @@ Highlights include: have been removed. PDR would like to restore these features, and pull requests are welcome. +- A new connector for Tiingo was introduced. Tiingo provides + historical end-of-day data for a large set of equities, ETFs and mutual + funds. Free registration is required to get an API key (:issue:`478`). + - A new connector for Robinhood was introduced. This provides up to 1 year of historical end-of-day data. It also provides near real-time quotes. (:issue:`477`). @@ -72,6 +76,9 @@ Enhancements - A new data connector for stock pricing data provided by `Robinhood `__ was introduced. (:issue:`477`) +- A new data connector for stock pricing data provided by + `Tiingo `__ was introduced. (:issue:`478`) + .. _whatsnew_060.api_breaking: Backwards incompatible API changes diff --git a/pandas_datareader/data.py b/pandas_datareader/data.py index d7926c96..0007bdd7 100644 --- a/pandas_datareader/data.py +++ b/pandas_datareader/data.py @@ -27,6 +27,7 @@ from pandas_datareader.robinhood import RobinhoodHistoricalReader, \ RobinhoodQuoteReader from pandas_datareader.stooq import StooqDailyReader +from pandas_datareader.tiingo import TiingoDailyReader, TiingoQuoteReader from pandas_datareader.yahoo.actions import (YahooActionReader, YahooDivReader) from pandas_datareader.yahoo.components import _get_data as \ get_components_yahoo @@ -114,6 +115,14 @@ def get_quotes_robinhood(*args, **kwargs): return RobinhoodQuoteReader(*args, **kwargs).read() +def get_data_tiingo(*args, **kwargs): + return TiingoDailyReader(*args, **kwargs).read() + + +def get_quotes_tiingo(*args, **kwargs): + return TiingoQuoteReader(*args, **kwargs).read() + + def get_markets_iex(*args, **kwargs): """ Returns near-real time volume data across markets segregated by tape @@ -384,6 +393,11 @@ def DataReader(name, data_source=None, start=None, end=None, return RobinhoodHistoricalReader(symbols=name, start=start, end=end, retry_count=retry_count, pause=pause, session=session).read() + elif data_source == 'tiingo': + return TiingoDailyReader(symbols=name, start=start, end=end, + retry_count=retry_count, pause=pause, + session=session, + api_key=access_key).read() else: msg = "data_source=%r is not implemented" % data_source raise NotImplementedError(msg) diff --git a/pandas_datareader/tests/test_tiingo.py b/pandas_datareader/tests/test_tiingo.py new file mode 100644 index 00000000..29f6057a --- /dev/null +++ b/pandas_datareader/tests/test_tiingo.py @@ -0,0 +1,44 @@ +import os + +import pandas as pd +import pytest + +from pandas_datareader.tiingo import TiingoDailyReader, TiingoMetaDataReader, \ + TiingoQuoteReader + +TEST_API_KEY = os.getenv('TIINGO_API_KEY') + +syms = ['GOOG', ['GOOG', 'XOM']] +ids = list(map(str, syms)) + + +@pytest.fixture(params=syms, ids=ids) +def symbols(request): + return request.param + + +@pytest.mark.skipif(TEST_API_KEY is None, reason="TIINGO_API_KEY not set") +def test_tiingo_quote(symbols): + df = TiingoQuoteReader(symbols=symbols).read() + assert isinstance(df, pd.DataFrame) + if isinstance(symbols, str): + symbols = [symbols] + assert df.shape[0] == len(symbols) + + +@pytest.mark.skipif(TEST_API_KEY is None, reason="TIINGO_API_KEY not set") +def test_tiingo_historical(symbols): + df = TiingoDailyReader(symbols=symbols).read() + assert isinstance(df, pd.DataFrame) + if isinstance(symbols, str): + symbols = [symbols] + assert df.index.levels[0].shape[0] == len(symbols) + + +@pytest.mark.skipif(TEST_API_KEY is None, reason="TIINGO_API_KEY not set") +def test_tiingo_metadata(symbols): + df = TiingoMetaDataReader(symbols=symbols).read() + assert isinstance(df, pd.DataFrame) + if isinstance(symbols, str): + symbols = [symbols] + assert df.shape[1] == len(symbols) diff --git a/pandas_datareader/tiingo.py b/pandas_datareader/tiingo.py new file mode 100644 index 00000000..eaed50e0 --- /dev/null +++ b/pandas_datareader/tiingo.py @@ -0,0 +1,191 @@ +import os + +import pandas as pd + +from pandas_datareader.base import _BaseReader + + +def get_tiingo_symbols(): + """ + Get the set of stock symbols supported by Tiingo + + Returns + ------- + symbols : DataFrame + DataFrame with symbols (ticker), exchange, asset type, currency and + start and end dates + + Notes + ----- + Reads https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip + """ + url = 'https://apimedia.tiingo.com/docs/tiingo/daily/supported_tickers.zip' + return pd.read_csv(url) + + +class TiingoDailyReader(_BaseReader): + """ + Historical daily data from Tiingo on equities, ETFs and mutual funds + + Parameters + ---------- + symbols : {str, List[str]} + String symbol of like of symbols + start : str, (defaults to '1/1/2010') + Starting date, timestamp. Parses many different kind of date + representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980') + end : str, (defaults to today) + Ending date, timestamp. Same format as starting date. + retry_count : int, default 3 + Number of times to retry query request. + pause : float, default 0.1 + Time, in seconds, of the pause between retries. + session : Session, default None + requests.sessions.Session instance to be used + freq : {str, None} + Not used. + api_key : str, optional + Tiingo API key . If not provided the environmental variable + TIINGO_API_KEY is read. The API key is *required*. + """ + + def __init__(self, symbols, start=None, end=None, retry_count=3, pause=0.1, + timeout=30, session=None, freq=None, api_key=None): + super(TiingoDailyReader, self).__init__(symbols, start, end, + retry_count, pause, timeout, + session, freq) + if isinstance(self.symbols, str): + self.symbols = [self.symbols] + self._symbol = '' + if api_key is None: + api_key = os.environ.get('TIINGO_API_KEY', None) + if api_key is None: + raise ValueError('The tiingo API key must be provided either ' + 'through the api_key variable or through the ' + 'environmental variable TIINGO_API_KEY.') + self.api_key = api_key + self._concat_axis = 0 + + @property + def url(self): + """API URL""" + _url = 'https://api.tiingo.com/tiingo/daily/{ticker}/prices' + return _url.format(ticker=self._symbol) + + @property + def params(self): + """Parameters to use in API calls""" + return {'startDate': self.start.strftime('%Y-%m-%d'), + 'endDate': self.end.strftime('%Y-%m-%d'), + 'format': 'json'} + + def _get_crumb(self, *args): + pass + + def _read_one_data(self, url, params): + """ read one data from specified URL """ + headers = {'Content-Type': 'application/json', + 'Authorization': 'Token ' + self.api_key} + out = self._get_response(url, params=params, headers=headers).json() + return self._read_lines(out) + + def _read_lines(self, out): + df = pd.DataFrame(out) + df['symbol'] = self._symbol + df['date'] = pd.to_datetime(df['date']) + df = df.set_index(['symbol', 'date']) + return df + + def read(self): + """Read data from connector""" + dfs = [] + for symbol in self.symbols: + self._symbol = symbol + try: + dfs.append(self._read_one_data(self.url, self.params)) + finally: + self.close() + return pd.concat(dfs, self._concat_axis) + + +class TiingoMetaDataReader(TiingoDailyReader): + """ + Read metadata about symbols from Tiingo + + Parameters + ---------- + symbols : {str, List[str]} + String symbol of like of symbols + start : str, (defaults to '1/1/2010') + Not used. + end : str, (defaults to today) + Not used. + retry_count : int, default 3 + Number of times to retry query request. + pause : float, default 0.1 + Time, in seconds, of the pause between retries. + session : Session, default None + requests.sessions.Session instance to be used + freq : {str, None} + Not used. + api_key : str, optional + Tiingo API key . If not provided the environmental variable + TIINGO_API_KEY is read. The API key is *required*. + """ + + def __init__(self, symbols, start=None, end=None, retry_count=3, pause=0.1, + timeout=30, session=None, freq=None, api_key=None): + super(TiingoMetaDataReader, self).__init__(symbols, start, end, + retry_count, pause, timeout, + session, freq, api_key) + self._concat_axis = 1 + + @property + def url(self): + """API URL""" + _url = 'https://api.tiingo.com/tiingo/daily/{ticker}' + return _url.format(ticker=self._symbol) + + @property + def params(self): + return None + + def _read_lines(self, out): + s = pd.Series(out) + s.name = self._symbol + return s + + +class TiingoQuoteReader(TiingoDailyReader): + """ + Read quotes (latest prices) from Tiingo + + Parameters + ---------- + symbols : {str, List[str]} + String symbol of like of symbols + start : str, (defaults to '1/1/2010') + Not used. + end : str, (defaults to today) + Not used. + retry_count : int, default 3 + Number of times to retry query request. + pause : float, default 0.1 + Time, in seconds, of the pause between retries. + session : Session, default None + requests.sessions.Session instance to be used + freq : {str, None} + Not used. + api_key : str, optional + Tiingo API key . If not provided the environmental variable + TIINGO_API_KEY is read. The API key is *required*. + + Notes + ----- + This is a special case of the daily reader which automatically selected + the latest data available for each symbol. + """ + + @property + def params(self): + return None diff --git a/requirements.txt b/requirements.txt index 3131045e..1f225b18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ matplotlib ipython sphinx sphinx_rtd_theme +requests-cache \ No newline at end of file