Skip to content

CI: avoid file leaks in sas_xport tests #35693

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Aug 13, 2020
10 changes: 8 additions & 2 deletions pandas/io/sas/sasreader.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from pandas._typing import FilePathOrBuffer, Label

from pandas.io.common import stringify_path
from pandas.io.common import get_filepath_or_buffer, stringify_path

if TYPE_CHECKING:
from pandas import DataFrame # noqa: F401
Expand Down Expand Up @@ -109,6 +109,10 @@ def read_sas(
else:
raise ValueError("unable to infer format of SAS file")

filepath_or_buffer, _, _, should_close = get_filepath_or_buffer(
filepath_or_buffer, encoding
)

reader: ReaderBase
if format.lower() == "xport":
from pandas.io.sas.sas_xport import XportReader
Expand All @@ -129,5 +133,7 @@ def read_sas(
return reader

data = reader.read()
reader.close()

if should_close:
reader.close()
return data
13 changes: 11 additions & 2 deletions pandas/tests/io/sas/test_xport.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import numpy as np
import pytest

import pandas.util._test_decorators as td

import pandas as pd
import pandas._testing as tm

Expand All @@ -26,10 +28,12 @@ def setup_method(self, datapath):
self.dirpath = datapath("io", "sas", "data")
self.file01 = os.path.join(self.dirpath, "DEMO_G.xpt")
self.file02 = os.path.join(self.dirpath, "SSHSV1_A.xpt")
self.file02b = open(os.path.join(self.dirpath, "SSHSV1_A.xpt"), "rb")
self.file03 = os.path.join(self.dirpath, "DRXFCD_G.xpt")
self.file04 = os.path.join(self.dirpath, "paxraw_d_short.xpt")

with td.file_leak_context():
yield

def test1_basic(self):
# Tests with DEMO_G.xpt (all numeric file)

Expand Down Expand Up @@ -127,7 +131,12 @@ def test2_binary(self):
data_csv = pd.read_csv(self.file02.replace(".xpt", ".csv"))
numeric_as_float(data_csv)

data = read_sas(self.file02b, format="xport")
with open(self.file02, "rb") as fd:
with td.file_leak_context():
# GH#35693 ensure that if we pass an open file, we
# dont incorrectly close it in read_sas
data = read_sas(fd, format="xport")

tm.assert_frame_equal(data, data_csv)

def test_multiple_types(self):
Expand Down
33 changes: 23 additions & 10 deletions pandas/util/_test_decorators.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ def test_foo():

For more information, refer to the ``pytest`` documentation on ``skipif``.
"""
from contextlib import contextmanager
from distutils.version import LooseVersion
from functools import wraps
import locale
from typing import Callable, Optional

Expand Down Expand Up @@ -237,23 +237,36 @@ def documented_fixture(fixture):

def check_file_leaks(func) -> Callable:
"""
Decorate a test function tot check that we are not leaking file descriptors.
Decorate a test function to check that we are not leaking file descriptors.
"""
psutil = safe_import("psutil")
if not psutil:
with file_leak_context():
return func

@wraps(func)
def new_func(*args, **kwargs):

@contextmanager
def file_leak_context():
"""
ContextManager analogue to check_file_leaks.
"""
psutil = safe_import("psutil")
if not psutil:
yield
else:
proc = psutil.Process()
flist = proc.open_files()
conns = proc.connections()

func(*args, **kwargs)
yield

flist2 = proc.open_files()
assert flist2 == flist

return new_func
# on some builds open_files includes file position, which we _dont_
# expect to remain unchanged, so we need to compare excluding that
flist_ex = [(x.path, x.fd) for x in flist]
flist2_ex = [(x.path, x.fd) for x in flist2]
assert flist2_ex == flist_ex, (flist2, flist)

conns2 = proc.connections()
assert conns2 == conns, (conns2, conns)


def async_mark():
Expand Down