diff --git a/docs/fangraphs.md b/docs/fangraphs.md index b1b20369..097caf9d 100644 --- a/docs/fangraphs.md +++ b/docs/fangraphs.md @@ -18,6 +18,7 @@ | team | str | Team to filter data by.
Specify "0,ts" to get aggregate team data. | position | str | Position to filter data by.
Default = ALL | max_results | int | The maximum number of results to return.
Default = 1000000 (In effect, all results) +| legacy | bool | Flag to select whether to get data from legacy leaderboard page or not
Default = False ## Usage diff --git a/pybaseball/datasources/fangraphs.py b/pybaseball/datasources/fangraphs.py index 4867a475..46c278a3 100644 --- a/pybaseball/datasources/fangraphs.py +++ b/pybaseball/datasources/fangraphs.py @@ -11,7 +11,8 @@ stat_list_from_str, stat_list_to_str) from .html_table_processor import HTMLTableProcessor, RowIdFunction -_FG_LEADERS_URL = "/leaders-legacy.aspx" +_FG_LEADERS_URL = "/api/leaders/major-league/data" +_LEGACY_FG_LEADERS_URL = "/leaders-legacy.aspx" MIN_AGE = 0 MAX_AGE = 100 @@ -43,6 +44,7 @@ class FangraphsDataTable(ABC): DATA_ROWS_XPATH: str = "({TABLE_XPATH}/tbody//tr)" DATA_CELLS_XPATH: str = "td[position()>1]/descendant-or-self::*/text()" QUERY_ENDPOINT: str = _FG_LEADERS_URL + LEGACY_QUERY_ENDPOINT: str = _LEGACY_FG_LEADERS_URL STATS_CATEGORY: FangraphsStatsCategory = FangraphsStatsCategory.NONE DEFAULT_STAT_COLUMNS: List[FangraphsStatColumn] = [] KNOWN_PERCENTAGES: List[str] = [] @@ -77,7 +79,7 @@ def fetch(self, start_season: int, end_season: Optional[int] = None, league: str stat_columns: Union[str, List[str]] = 'ALL', qual: Optional[int] = None, split_seasons: bool = True, month: str = 'ALL', on_active_roster: bool = False, minimum_age: int = MIN_AGE, maximum_age: int = MAX_AGE, team: str = '', _filter: str = '', players: str = '', - position: str = 'ALL', max_results: int = 1000000,) -> pd.DataFrame: + position: str = 'ALL', max_results: int = 1000000, legacy: bool = False) -> pd.DataFrame: """ Get leaderboard data from Fangraphs. @@ -146,22 +148,26 @@ def fetch(self, start_season: int, end_season: Optional[int] = None, league: str 'age': f"{minimum_age},{maximum_age}", 'filter': _filter, 'players': players, - 'page': f'1_{max_results}' + 'page': f'1_{max_results}', + 'pageitems': max_results # New Fangraphs Leaderboard uses pageitems to get maximum results per page } - return self._validate( - self._postprocess( - self.html_accessor.get_tabular_data_from_options( - self.QUERY_ENDPOINT, - query_params=url_options, - # TODO: Remove the type: ignore after this is fixed: https://github.com/python/mypy/issues/5485 - column_name_mapper=self.COLUMN_NAME_MAPPER, # type: ignore - known_percentages=self.KNOWN_PERCENTAGES, - row_id_func=self.ROW_ID_FUNC, - row_id_name=self.ROW_ID_NAME, - ) - ) - ) + # Add `legacy` flag to let users decide whether use legacy api or not + tabular_data = self.html_accessor.get_tabular_data_from_options( + self.LEGACY_QUERY_ENDPOINT, + query_params=url_options, + # TODO: Remove the type: ignore after this is fixed: https://github.com/python/mypy/issues/5485 + column_name_mapper=self.COLUMN_NAME_MAPPER, # type: ignore + known_percentages=self.KNOWN_PERCENTAGES, + row_id_func=self.ROW_ID_FUNC, + row_id_name=self.ROW_ID_NAME, + ) if legacy else self.html_accessor.get_tabular_data_from_api( + f"{self.ROOT_URL}{self.QUERY_ENDPOINT}", + query_params=url_options + ) + + + return self._validate(self._postprocess(tabular_data)) class FangraphsBattingStatsTable(FangraphsDataTable): STATS_CATEGORY: FangraphsStatsCategory = FangraphsStatsCategory.BATTING diff --git a/pybaseball/datasources/html_table_processor.py b/pybaseball/datasources/html_table_processor.py index 0884a072..4f972429 100644 --- a/pybaseball/datasources/html_table_processor.py +++ b/pybaseball/datasources/html_table_processor.py @@ -4,6 +4,8 @@ import lxml.etree import pandas as pd import requests +import json +import re from ..datahelpers import postprocessing from ..datahelpers.column_mapper import ColumnListMapperFunction @@ -95,3 +97,21 @@ def get_tabular_data_from_options(self, base_url: str, query_params: Dict[str, U row_id_func=row_id_func, row_id_name=row_id_name, ) + + def get_tabular_data_from_api(self, base_url: str, query_params: Dict[str, Union[str, int]]): + # Newest Fangraphs Leaderboard API will return html tag in `Name` and `Team` column + # Therefore we need to extract the name and team from response result + def extract_text_from_html(text): + try: + return re.search('>(.+?)<', text).group(1) + except AttributeError: + return text + + data = requests.get(base_url, query_params).content + data = json.loads(data) + + df = pd.DataFrame(data['data']) + df['Name'] = df['Name'].apply(extract_text_from_html) + df['Team'] = df['Team'].apply(extract_text_from_html) + + return df diff --git a/test.ipynb b/test.ipynb new file mode 100644 index 00000000..f54a3140 --- /dev/null +++ b/test.ipynb @@ -0,0 +1,786 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDfgSeasonNameTeamAgeGABPAH1B...maxEVHardHitHardHit%EventsCStr%CSW%xBAxSLGxwOBAL-WAR
1184012023Ronald Acuna Jr.ATL25159643735217137...121.23090.5505620.1370.2140.3560.6600.4638.6
3136112023Mookie BettsLAD3015258469317999...110.12320.4814820.2050.2620.2950.5650.4118.6
553612023Freddie FreemanLAD33161637730211121...110.62180.4185210.1140.2080.3200.5640.4097.6
4143442023Matt OlsonATL2916260872017288...118.62450.5514450.1310.2490.2630.5610.3946.8
0197552023Shohei OhtaniLAA2813549759915173...118.61930.5333620.1250.2650.2940.6340.4286.6
..................................................................
132151722023Tim AndersonCHW30123493524121100...109.61460.3913730.1400.2630.2500.3300.279-0.8
125156762023Jose AbreuHOU3614154059412886...113.31730.4164160.1700.2910.2430.4000.310-0.5
115179882023Tyler StephensonCIN2614246551711378...111.11430.4323310.1910.3000.2420.3950.314-0.8
129104722023Enrique Hernandez- - -3114046550811076...109.21250.3343740.1770.2800.2220.3330.271-0.7
116108152023Jurickson Profar- - -3012545952111173...108.81190.3173750.1510.2360.2450.3440.305-0.9
\n", + "

133 rows × 320 columns

\n", + "
" + ], + "text/plain": [ + " IDfg Season Name Team Age G AB PA H 1B \\\n", + "1 18401 2023 Ronald Acuna Jr. ATL 25 159 643 735 217 137 \n", + "3 13611 2023 Mookie Betts LAD 30 152 584 693 179 99 \n", + "5 5361 2023 Freddie Freeman LAD 33 161 637 730 211 121 \n", + "4 14344 2023 Matt Olson ATL 29 162 608 720 172 88 \n", + "0 19755 2023 Shohei Ohtani LAA 28 135 497 599 151 73 \n", + ".. ... ... ... ... ... ... ... ... ... ... \n", + "132 15172 2023 Tim Anderson CHW 30 123 493 524 121 100 \n", + "125 15676 2023 Jose Abreu HOU 36 141 540 594 128 86 \n", + "115 17988 2023 Tyler Stephenson CIN 26 142 465 517 113 78 \n", + "129 10472 2023 Enrique Hernandez - - - 31 140 465 508 110 76 \n", + "116 10815 2023 Jurickson Profar - - - 30 125 459 521 111 73 \n", + "\n", + " ... maxEV HardHit HardHit% Events CStr% CSW% xBA xSLG xwOBA \\\n", + "1 ... 121.2 309 0.550 562 0.137 0.214 0.356 0.660 0.463 \n", + "3 ... 110.1 232 0.481 482 0.205 0.262 0.295 0.565 0.411 \n", + "5 ... 110.6 218 0.418 521 0.114 0.208 0.320 0.564 0.409 \n", + "4 ... 118.6 245 0.551 445 0.131 0.249 0.263 0.561 0.394 \n", + "0 ... 118.6 193 0.533 362 0.125 0.265 0.294 0.634 0.428 \n", + ".. ... ... ... ... ... ... ... ... ... ... \n", + "132 ... 109.6 146 0.391 373 0.140 0.263 0.250 0.330 0.279 \n", + "125 ... 113.3 173 0.416 416 0.170 0.291 0.243 0.400 0.310 \n", + "115 ... 111.1 143 0.432 331 0.191 0.300 0.242 0.395 0.314 \n", + "129 ... 109.2 125 0.334 374 0.177 0.280 0.222 0.333 0.271 \n", + "116 ... 108.8 119 0.317 375 0.151 0.236 0.245 0.344 0.305 \n", + "\n", + " L-WAR \n", + "1 8.6 \n", + "3 8.6 \n", + "5 7.6 \n", + "4 6.8 \n", + "0 6.6 \n", + ".. ... \n", + "132 -0.8 \n", + "125 -0.5 \n", + "115 -0.8 \n", + "129 -0.7 \n", + "116 -0.9 \n", + "\n", + "[133 rows x 320 columns]" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pybaseball.batting_leaders import batting_stats\n", + "\n", + "legacy_data = batting_stats(2023, legacy=True)\n", + "\n", + "legacy_data\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
BatsxMLBAMIDNameTeamSeasonAgeAgeRSeasonMinSeasonMaxG...QTGTPAPlayerNameRoutePlayerNameplayeridTeamNameTeamNameAbbteamidPos
0R660670Ronald Acuna Jr.ATL202325.025 - 2520232023159.0...4.5370162.0735.0Ronald Acuna Jr.Ronald Acuña Jr.18401ATLATL160.0127
1R605141Mookie BettsLAD202330.030 - 3020232023152.0...4.2778162.0693.0Mookie BettsMookie Betts13611LADLAD220.0000
2L518692Freddie FreemanLAD202333.033 - 3320232023161.0...4.5062162.0730.0Freddie FreemanFreddie Freeman5361LADLAD220.0000
3L621566Matt OlsonATL202329.029 - 2920232023162.0...4.4444162.0720.0Matt OlsonMatt Olson14344ATLATL160.0000
4L660271Shohei OhtaniLAA202328.028 - 2820232023135.0...3.6975162.0599.0Shohei OhtaniShohei Ohtani19755LAALAA10.8535
..................................................................
128R641313Tim AndersonCHW202330.030 - 3020232023123.0...3.2346162.0524.0Tim AndersonTim Anderson15172CHWCHW40.0000
129R547989Jose AbreuHOU202336.036 - 3620232023141.0...3.6667162.0594.0Jose AbreuJosé Abreu15676HOUHOU210.0496
130R663886Tyler StephensonCIN202326.026 - 2620232023142.0...3.1914162.0517.0Tyler StephensonTyler Stephenson17988CINCIN180.3007
131R571771Enrique Hernandez- - -202331.031 - 3120232023140.0...3.1358162.0508.0Enrique HernandezEnrique Hernández10472- - -2 Tms00.0000
132B595777Jurickson Profar- - -202330.030 - 3020232023125.0...3.2160162.0521.0Jurickson ProfarJurickson Profar10815- - -2 Tms00.1488
\n", + "

133 rows × 357 columns

\n", + "
" + ], + "text/plain": [ + " Bats xMLBAMID Name Team Season Age AgeR \\\n", + "0 R 660670 Ronald Acuna Jr. ATL 2023 25.0 25 - 25 \n", + "1 R 605141 Mookie Betts LAD 2023 30.0 30 - 30 \n", + "2 L 518692 Freddie Freeman LAD 2023 33.0 33 - 33 \n", + "3 L 621566 Matt Olson ATL 2023 29.0 29 - 29 \n", + "4 L 660271 Shohei Ohtani LAA 2023 28.0 28 - 28 \n", + ".. ... ... ... ... ... ... ... \n", + "128 R 641313 Tim Anderson CHW 2023 30.0 30 - 30 \n", + "129 R 547989 Jose Abreu HOU 2023 36.0 36 - 36 \n", + "130 R 663886 Tyler Stephenson CIN 2023 26.0 26 - 26 \n", + "131 R 571771 Enrique Hernandez - - - 2023 31.0 31 - 31 \n", + "132 B 595777 Jurickson Profar - - - 2023 30.0 30 - 30 \n", + "\n", + " SeasonMin SeasonMax G ... Q TG TPA \\\n", + "0 2023 2023 159.0 ... 4.5370 162.0 735.0 \n", + "1 2023 2023 152.0 ... 4.2778 162.0 693.0 \n", + "2 2023 2023 161.0 ... 4.5062 162.0 730.0 \n", + "3 2023 2023 162.0 ... 4.4444 162.0 720.0 \n", + "4 2023 2023 135.0 ... 3.6975 162.0 599.0 \n", + ".. ... ... ... ... ... ... ... \n", + "128 2023 2023 123.0 ... 3.2346 162.0 524.0 \n", + "129 2023 2023 141.0 ... 3.6667 162.0 594.0 \n", + "130 2023 2023 142.0 ... 3.1914 162.0 517.0 \n", + "131 2023 2023 140.0 ... 3.1358 162.0 508.0 \n", + "132 2023 2023 125.0 ... 3.2160 162.0 521.0 \n", + "\n", + " PlayerNameRoute PlayerName playerid TeamName TeamNameAbb \\\n", + "0 Ronald Acuna Jr. Ronald Acuña Jr. 18401 ATL ATL \n", + "1 Mookie Betts Mookie Betts 13611 LAD LAD \n", + "2 Freddie Freeman Freddie Freeman 5361 LAD LAD \n", + "3 Matt Olson Matt Olson 14344 ATL ATL \n", + "4 Shohei Ohtani Shohei Ohtani 19755 LAA LAA \n", + ".. ... ... ... ... ... \n", + "128 Tim Anderson Tim Anderson 15172 CHW CHW \n", + "129 Jose Abreu José Abreu 15676 HOU HOU \n", + "130 Tyler Stephenson Tyler Stephenson 17988 CIN CIN \n", + "131 Enrique Hernandez Enrique Hernández 10472 - - - 2 Tms \n", + "132 Jurickson Profar Jurickson Profar 10815 - - - 2 Tms \n", + "\n", + " teamid Pos \n", + "0 16 0.0127 \n", + "1 22 0.0000 \n", + "2 22 0.0000 \n", + "3 16 0.0000 \n", + "4 1 0.8535 \n", + ".. ... ... \n", + "128 4 0.0000 \n", + "129 21 0.0496 \n", + "130 18 0.3007 \n", + "131 0 0.0000 \n", + "132 0 0.1488 \n", + "\n", + "[133 rows x 357 columns]" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = batting_stats(2023)\n", + "\n", + "data\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/test.py b/test.py new file mode 100644 index 00000000..a44b25cc --- /dev/null +++ b/test.py @@ -0,0 +1,6 @@ +from pybaseball.batting_leaders import batting_stats + +data = batting_stats(2023) + +# Use `legacy` flag to get legacy leaderboard page data +legacy_data = batting_stats(2023, legacy=True)