Skip to content

PERF faster head, tail and size groupby methods #5518

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pandas/compat/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ def wrapper(cls):


class _OrderedDict(dict):

"""Dictionary that remembers insertion order"""
# An inherited dict maps keys to values.
# The inherited dict provides __getitem__, __len__, __contains__, and get.
Expand Down Expand Up @@ -505,6 +506,7 @@ def viewitems(self):


class _Counter(dict):

"""Dict subclass for counting hashable objects. Sometimes called a bag
or multiset. Elements are stored as dictionary keys and their counts
are stored as dictionary values.
Expand Down
3 changes: 3 additions & 0 deletions pandas/computation/engines.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@


class AbstractEngine(object):

"""Object serving as a base class for all engines."""

__metaclass__ = abc.ABCMeta
Expand Down Expand Up @@ -73,6 +74,7 @@ def _evaluate(self):


class NumExprEngine(AbstractEngine):

"""NumExpr engine class"""
has_neg_frac = True

Expand Down Expand Up @@ -105,6 +107,7 @@ def _evaluate(self):


class PythonEngine(AbstractEngine):

"""Evaluate an expression in Python space.

Mostly for testing purposes.
Expand Down
8 changes: 8 additions & 0 deletions pandas/computation/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ def _raw_hex_id(obj, pad_size=2):


class Scope(StringMixin):

"""Object to hold scope, with a few bells to deal with some custom syntax
added by pandas.

Expand Down Expand Up @@ -324,6 +325,7 @@ def _node_not_implemented(node_name, cls):
"""Return a function that raises a NotImplementedError with a passed node
name.
"""

def f(self, *args, **kwargs):
raise NotImplementedError("{0!r} nodes are not "
"implemented".format(node_name))
Expand Down Expand Up @@ -356,6 +358,7 @@ def _op_maker(op_class, op_symbol):
-------
f : callable
"""

def f(self, node, *args, **kwargs):
"""Return a partial function with an Op subclass with an operator
already passed.
Expand Down Expand Up @@ -389,6 +392,7 @@ def f(cls):
@disallow(_unsupported_nodes)
@add_ops(_op_classes)
class BaseExprVisitor(ast.NodeVisitor):

"""Custom ast walker. Parsers of other engines should subclass this class
if necessary.

Expand Down Expand Up @@ -710,19 +714,22 @@ def visitor(x, y):
(_boolop_nodes | frozenset(['BoolOp', 'Attribute', 'In', 'NotIn',
'Tuple'])))
class PandasExprVisitor(BaseExprVisitor):

def __init__(self, env, engine, parser,
preparser=lambda x: _replace_locals(_replace_booleans(x))):
super(PandasExprVisitor, self).__init__(env, engine, parser, preparser)


@disallow(_unsupported_nodes | _python_not_supported | frozenset(['Not']))
class PythonExprVisitor(BaseExprVisitor):

def __init__(self, env, engine, parser, preparser=lambda x: x):
super(PythonExprVisitor, self).__init__(env, engine, parser,
preparser=preparser)


class Expr(StringMixin):

"""Object encapsulating an expression.

Parameters
Expand All @@ -734,6 +741,7 @@ class Expr(StringMixin):
truediv : bool, optional, default True
level : int, optional, default 2
"""

def __init__(self, expr, engine='numexpr', parser='pandas', env=None,
truediv=True, level=2):
self.expr = expr
Expand Down
12 changes: 12 additions & 0 deletions pandas/computation/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@


class UndefinedVariableError(NameError):

"""NameError subclass for local variables."""

def __init__(self, *args):
msg = 'name {0!r} is not defined'
subbed = _TAG_RE.sub('', args[0])
Expand All @@ -51,6 +53,7 @@ def _possibly_update_key(d, value, old_key, new_key=None):


class Term(StringMixin):

def __new__(cls, name, env, side=None, encoding=None):
klass = Constant if not isinstance(name, string_types) else cls
supr_new = super(Term, klass).__new__
Expand Down Expand Up @@ -195,6 +198,7 @@ def ndim(self):


class Constant(Term):

def __init__(self, value, env, side=None, encoding=None):
super(Constant, self).__init__(value, env, side=side,
encoding=encoding)
Expand All @@ -211,8 +215,10 @@ def name(self):


class Op(StringMixin):

"""Hold an operator of unknown arity
"""

def __init__(self, op, operands, *args, **kwargs):
self.op = _bool_op_map.get(op, op)
self.operands = operands
Expand Down Expand Up @@ -328,6 +334,7 @@ def is_term(obj):


class BinOp(Op):

"""Hold a binary operator and its operands

Parameters
Expand All @@ -336,6 +343,7 @@ class BinOp(Op):
left : Term or Op
right : Term or Op
"""

def __init__(self, op, lhs, rhs, **kwargs):
super(BinOp, self).__init__(op, (lhs, rhs))
self.lhs = lhs
Expand Down Expand Up @@ -452,6 +460,7 @@ def _disallow_scalar_only_bool_ops(self):


class Div(BinOp):

"""Div operator to special case casting.

Parameters
Expand All @@ -462,6 +471,7 @@ class Div(BinOp):
Whether or not to use true division. With Python 3 this happens
regardless of the value of ``truediv``.
"""

def __init__(self, lhs, rhs, truediv=True, *args, **kwargs):
super(Div, self).__init__('/', lhs, rhs, *args, **kwargs)

Expand All @@ -475,6 +485,7 @@ def __init__(self, lhs, rhs, truediv=True, *args, **kwargs):


class UnaryOp(Op):

"""Hold a unary operator and its operands

Parameters
Expand All @@ -489,6 +500,7 @@ class UnaryOp(Op):
ValueError
* If no function associated with the passed operator token is found.
"""

def __init__(self, op, operand):
super(UnaryOp, self).__init__(op, (operand,))
self.operand = operand
Expand Down
6 changes: 4 additions & 2 deletions pandas/computation/pytables.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ def __init__(self, gbls=None, lcls=None, queryables=None, level=1):


class Term(ops.Term):

def __new__(cls, name, env, side=None, encoding=None):
klass = Constant if not isinstance(name, string_types) else cls
supr_new = StringMixin.__new__
Expand Down Expand Up @@ -57,6 +58,7 @@ def value(self):


class Constant(Term):

def __init__(self, value, env, side=None, encoding=None):
super(Constant, self).__init__(value, env, side=side,
encoding=encoding)
Expand Down Expand Up @@ -292,9 +294,9 @@ def __unicode__(self):

def invert(self):
""" invert the condition """
#if self.condition is not None:
# if self.condition is not None:
# self.condition = "~(%s)" % self.condition
#return self
# return self
raise NotImplementedError("cannot use an invert condition when "
"passing to numexpr")

Expand Down
Loading