From 0f70991351e4dee667dd7e8c68bc32d87c99b863 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Mon, 31 Aug 2020 21:56:52 -0500 Subject: [PATCH 1/9] TYP/CLN: cleanup, add type annotation #36021 --- pandas/io/excel/_openpyxl.py | 56 +++++++++--------------------------- setup.cfg | 3 -- 2 files changed, 13 insertions(+), 46 deletions(-) diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index c2730536af8a3..058d0c4092c54 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Any, Dict, List, Optional import numpy as np @@ -22,53 +22,22 @@ def __init__(self, path, engine=None, mode="w", **engine_kwargs): if self.mode == "a": # Load from existing workbook from openpyxl import load_workbook - book = load_workbook(self.path) - self.book = book + self.book = load_workbook(self.path) else: # Create workbook object with default optimized_write=True. self.book = Workbook() if self.book.worksheets: - try: - self.book.remove(self.book.worksheets[0]) - except AttributeError: - - # compat - for openpyxl <= 2.4 - self.book.remove_sheet(self.book.worksheets[0]) + self.book.remove(self.book.worksheets[0]) def save(self): """ Save workbook to disk. """ - return self.book.save(self.path) + self.book.save(self.path) @classmethod - def _convert_to_style(cls, style_dict): - """ - Converts a style_dict to an openpyxl style object. - - Parameters - ---------- - style_dict : style dictionary to convert - """ - from openpyxl.style import Style - - xls_style = Style() - for key, value in style_dict.items(): - for nk, nv in value.items(): - if key == "borders": - ( - xls_style.borders.__getattribute__(nk).__setattr__( - "border_style", nv - ) - ) - else: - xls_style.__getattribute__(key).__setattr__(nk, nv) - - return xls_style - - @classmethod - def _convert_to_style_kwargs(cls, style_dict): + def _convert_to_style_kwargs(cls, style_dict: dict) -> Dict[str, Any]: """ Convert a style_dict to a set of kwargs suitable for initializing or updating-on-copy an openpyxl v2 style object. @@ -93,7 +62,7 @@ def _convert_to_style_kwargs(cls, style_dict): """ _style_key_map = {"borders": "border"} - style_kwargs = {} + style_kwargs: Dict[str, Any] = {} for k, v in style_dict.items(): if k in _style_key_map: k = _style_key_map[k] @@ -404,7 +373,7 @@ def write_cells( # Write the frame cells using openpyxl. sheet_name = self._get_sheet_name(sheet_name) - _style_cache = {} + _style_cache: Dict[str, Dict[str, Any]] = {} if sheet_name in self.sheets: wks = self.sheets[sheet_name] @@ -426,7 +395,7 @@ def write_cells( if fmt: xcell.number_format = fmt - style_kwargs = {} + style_kwargs: Optional[Dict[str, Any]] = {} if cell.style: key = str(cell.style) style_kwargs = _style_cache.get(key) @@ -515,16 +484,17 @@ def get_sheet_by_index(self, index: int): def _convert_cell(self, cell, convert_float: bool) -> Scalar: - # TODO: replace with openpyxl constants + from openpyxl.cell.cell import TYPE_BOOL, TYPE_ERROR, TYPE_NUMERIC + if cell.is_date: return cell.value - elif cell.data_type == "e": + elif cell.data_type == TYPE_ERROR: return np.nan - elif cell.data_type == "b": + elif cell.data_type == TYPE_BOOL: return bool(cell.value) elif cell.value is None: return "" # compat with xlrd - elif cell.data_type == "n": + elif cell.data_type == TYPE_NUMERIC: # GH5394 if convert_float: val = int(cell.value) diff --git a/setup.cfg b/setup.cfg index c10624d60aaff..e346a625911c5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -217,9 +217,6 @@ check_untyped_defs=False [mypy-pandas.io.excel._base] check_untyped_defs=False -[mypy-pandas.io.excel._openpyxl] -check_untyped_defs=False - [mypy-pandas.io.excel._util] check_untyped_defs=False From bba25865983912702668c7b221d76d306203a700 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Mon, 31 Aug 2020 22:30:42 -0500 Subject: [PATCH 2/9] TYP/CLN: bump openpyxl vertion to fix import error #36021 --- ci/deps/azure-37-locale_slow.yaml | 2 +- ci/deps/azure-37-minimum_versions.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/deps/azure-37-locale_slow.yaml b/ci/deps/azure-37-locale_slow.yaml index 8000f3e6b9a9c..fbb1ea671d696 100644 --- a/ci/deps/azure-37-locale_slow.yaml +++ b/ci/deps/azure-37-locale_slow.yaml @@ -18,7 +18,7 @@ dependencies: - lxml - matplotlib=3.0.0 - numpy=1.16.* - - openpyxl=2.5.7 + - openpyxl=2.6.0 - python-dateutil - python-blosc - pytz=2017.3 diff --git a/ci/deps/azure-37-minimum_versions.yaml b/ci/deps/azure-37-minimum_versions.yaml index 05b1957198bc4..31f82f3304db3 100644 --- a/ci/deps/azure-37-minimum_versions.yaml +++ b/ci/deps/azure-37-minimum_versions.yaml @@ -19,7 +19,7 @@ dependencies: - numba=0.46.0 - numexpr=2.6.8 - numpy=1.16.5 - - openpyxl=2.5.7 + - openpyxl=2.6.0 - pytables=3.4.4 - python-dateutil=2.7.3 - pytz=2017.3 From b81bae37b2d241a6c5075566a37f44c33c5e01ed Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 1 Sep 2020 08:55:27 -0500 Subject: [PATCH 3/9] import type --- pandas/io/excel/_openpyxl.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index 058d0c4092c54..c25aee1706e25 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Dict, List, Optional import numpy as np @@ -8,6 +8,9 @@ from pandas.io.excel._base import ExcelWriter, _BaseExcelReader from pandas.io.excel._util import _validate_freeze_panes +if TYPE_CHECKING: + from openpyxl.descriptors.serialisable import Serialisable as StyleObject + class _OpenpyxlWriter(ExcelWriter): engine = "openpyxl" @@ -37,7 +40,7 @@ def save(self): self.book.save(self.path) @classmethod - def _convert_to_style_kwargs(cls, style_dict: dict) -> Dict[str, Any]: + def _convert_to_style_kwargs(cls, style_dict: dict) -> Dict[str, StyleObject]: """ Convert a style_dict to a set of kwargs suitable for initializing or updating-on-copy an openpyxl v2 style object. @@ -62,7 +65,7 @@ def _convert_to_style_kwargs(cls, style_dict: dict) -> Dict[str, Any]: """ _style_key_map = {"borders": "border"} - style_kwargs: Dict[str, Any] = {} + style_kwargs: Dict[str, StyleObject] = {} for k, v in style_dict.items(): if k in _style_key_map: k = _style_key_map[k] @@ -373,7 +376,7 @@ def write_cells( # Write the frame cells using openpyxl. sheet_name = self._get_sheet_name(sheet_name) - _style_cache: Dict[str, Dict[str, Any]] = {} + _style_cache: Dict[str, Dict[str, StyleObject]] = {} if sheet_name in self.sheets: wks = self.sheets[sheet_name] @@ -395,7 +398,7 @@ def write_cells( if fmt: xcell.number_format = fmt - style_kwargs: Optional[Dict[str, Any]] = {} + style_kwargs: Optional[Dict[str, StyleObject]] = {} if cell.style: key = str(cell.style) style_kwargs = _style_cache.get(key) From 915183cafbb266fe12498465b0516b7cd01ddad9 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 1 Sep 2020 10:44:31 -0500 Subject: [PATCH 4/9] fix importr error --- pandas/io/excel/_openpyxl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index c25aee1706e25..7fc9921b43ebf 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -40,7 +40,7 @@ def save(self): self.book.save(self.path) @classmethod - def _convert_to_style_kwargs(cls, style_dict: dict) -> Dict[str, StyleObject]: + def _convert_to_style_kwargs(cls, style_dict: dict): """ Convert a style_dict to a set of kwargs suitable for initializing or updating-on-copy an openpyxl v2 style object. From baf096669b65e00754d281092f64d0a1749bde84 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 1 Sep 2020 10:53:20 -0500 Subject: [PATCH 5/9] fix importr error --- pandas/io/excel/_openpyxl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index 7fc9921b43ebf..2834e2d9cba38 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -40,7 +40,7 @@ def save(self): self.book.save(self.path) @classmethod - def _convert_to_style_kwargs(cls, style_dict: dict): + def _convert_to_style_kwargs(cls, style_dict: dict) -> Dict[str, 'StyleObject']: """ Convert a style_dict to a set of kwargs suitable for initializing or updating-on-copy an openpyxl v2 style object. From 48da88e85f1b83627181fdec257c6d9c0959c2ab Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 1 Sep 2020 11:09:49 -0500 Subject: [PATCH 6/9] remove alias --- pandas/io/excel/_openpyxl.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index 2834e2d9cba38..745a2ec2e2fd2 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -9,7 +9,7 @@ from pandas.io.excel._util import _validate_freeze_panes if TYPE_CHECKING: - from openpyxl.descriptors.serialisable import Serialisable as StyleObject + from openpyxl.descriptors.serialisable import Serialisable class _OpenpyxlWriter(ExcelWriter): @@ -40,7 +40,7 @@ def save(self): self.book.save(self.path) @classmethod - def _convert_to_style_kwargs(cls, style_dict: dict) -> Dict[str, 'StyleObject']: + def _convert_to_style_kwargs(cls, style_dict: dict) -> Dict[str, 'Serialisable']: """ Convert a style_dict to a set of kwargs suitable for initializing or updating-on-copy an openpyxl v2 style object. @@ -65,7 +65,7 @@ def _convert_to_style_kwargs(cls, style_dict: dict) -> Dict[str, 'StyleObject']: """ _style_key_map = {"borders": "border"} - style_kwargs: Dict[str, StyleObject] = {} + style_kwargs: Dict[str, Serialisable] = {} for k, v in style_dict.items(): if k in _style_key_map: k = _style_key_map[k] @@ -376,7 +376,7 @@ def write_cells( # Write the frame cells using openpyxl. sheet_name = self._get_sheet_name(sheet_name) - _style_cache: Dict[str, Dict[str, StyleObject]] = {} + _style_cache: Dict[str, Dict[str, Serialisable]] = {} if sheet_name in self.sheets: wks = self.sheets[sheet_name] @@ -398,7 +398,7 @@ def write_cells( if fmt: xcell.number_format = fmt - style_kwargs: Optional[Dict[str, StyleObject]] = {} + style_kwargs: Optional[Dict[str, Serialisable]] = {} if cell.style: key = str(cell.style) style_kwargs = _style_cache.get(key) From 4f0a843172b2ff79bf7f233f51663e9a36d2b9ae Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 1 Sep 2020 11:13:13 -0500 Subject: [PATCH 7/9] move import to module scope --- pandas/io/excel/_openpyxl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index 745a2ec2e2fd2..0474ad4d32f71 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional import numpy as np +from openpyxl.cell.cell import TYPE_BOOL, TYPE_ERROR, TYPE_NUMERIC from pandas._typing import FilePathOrBuffer, Scalar, StorageOptions from pandas.compat._optional import import_optional_dependency @@ -40,7 +41,7 @@ def save(self): self.book.save(self.path) @classmethod - def _convert_to_style_kwargs(cls, style_dict: dict) -> Dict[str, 'Serialisable']: + def _convert_to_style_kwargs(cls, style_dict: dict) -> Dict[str, "Serialisable"]: """ Convert a style_dict to a set of kwargs suitable for initializing or updating-on-copy an openpyxl v2 style object. @@ -487,8 +488,6 @@ def get_sheet_by_index(self, index: int): def _convert_cell(self, cell, convert_float: bool) -> Scalar: - from openpyxl.cell.cell import TYPE_BOOL, TYPE_ERROR, TYPE_NUMERIC - if cell.is_date: return cell.value elif cell.data_type == TYPE_ERROR: From 2340da6a8c02e8af63e21c93f91e1b0c8b6dd664 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 1 Sep 2020 11:19:45 -0500 Subject: [PATCH 8/9] bump openpyxl version in doc --- doc/source/getting_started/install.rst | 2 +- doc/source/whatsnew/v1.2.0.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst index 4c270117e079e..c9ac1b0d284a3 100644 --- a/doc/source/getting_started/install.rst +++ b/doc/source/getting_started/install.rst @@ -274,7 +274,7 @@ html5lib 1.0.1 HTML parser for read_html (see :ref lxml 4.3.0 HTML parser for read_html (see :ref:`note `) matplotlib 2.2.3 Visualization numba 0.46.0 Alternative execution engine for rolling operations -openpyxl 2.5.7 Reading / writing for xlsx files +openpyxl 2.6.0 Reading / writing for xlsx files pandas-gbq 0.12.0 Google Big Query access psycopg2 2.7 PostgreSQL engine for sqlalchemy pyarrow 0.15.0 Parquet, ORC, and feather reading / writing diff --git a/doc/source/whatsnew/v1.2.0.rst b/doc/source/whatsnew/v1.2.0.rst index 1617bf66c4f04..76bebd4a9a1cb 100644 --- a/doc/source/whatsnew/v1.2.0.rst +++ b/doc/source/whatsnew/v1.2.0.rst @@ -109,7 +109,7 @@ Optional libraries below the lowest tested version may still work, but are not c +-----------------+-----------------+---------+ | numba | 0.46.0 | | +-----------------+-----------------+---------+ -| openpyxl | 2.5.7 | | +| openpyxl | 2.6.0 | X | +-----------------+-----------------+---------+ | pyarrow | 0.15.0 | X | +-----------------+-----------------+---------+ From 3d0781a41cb95222a6ded6279e6981f12df207b6 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 1 Sep 2020 11:45:17 -0500 Subject: [PATCH 9/9] move import back to function scope --- pandas/io/excel/_openpyxl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/io/excel/_openpyxl.py b/pandas/io/excel/_openpyxl.py index 0474ad4d32f71..3c67902d41baa 100644 --- a/pandas/io/excel/_openpyxl.py +++ b/pandas/io/excel/_openpyxl.py @@ -1,7 +1,6 @@ from typing import TYPE_CHECKING, Dict, List, Optional import numpy as np -from openpyxl.cell.cell import TYPE_BOOL, TYPE_ERROR, TYPE_NUMERIC from pandas._typing import FilePathOrBuffer, Scalar, StorageOptions from pandas.compat._optional import import_optional_dependency @@ -488,6 +487,8 @@ def get_sheet_by_index(self, index: int): def _convert_cell(self, cell, convert_float: bool) -> Scalar: + from openpyxl.cell.cell import TYPE_BOOL, TYPE_ERROR, TYPE_NUMERIC + if cell.is_date: return cell.value elif cell.data_type == TYPE_ERROR: