PERF: Quick Shift Implementation (GH5609)

doc/source/release.rst

@@ -182,7 +182,8 @@ Improvements to existing features
 - Support passing ``encoding`` with xlwt (:issue:`3710`)
 - Performance improvement when converting ``DatetimeIndex`` to floating ordinals
   using ``DatetimeConverter`` (:issue:`6636`)
-
+- Performance improvement for  ``DataFrame.shift`` (:issue: `5609`)
+
 .. _release.bug_fixes-0.14.0:
 
 Bug Fixes

pandas/core/common.py

@@ -2003,18 +2003,6 @@ def intersection(*seqs):
     return type(seqs[0])(list(result))
 
 
-def _shift_indexer(N, periods):
-    # small reusable utility
-    indexer = np.zeros(N, dtype=int)
-
-    if periods > 0:
-        indexer[periods:] = np.arange(N - periods)
-    else:
-        indexer[:periods] = np.arange(-periods, N)
-
-    return indexer
-
-
 def _asarray_tuplesafe(values, dtype=None):
     from pandas.core.index import Index
 

pandas/core/generic.py

@@ -3222,9 +3222,7 @@ def shift(self, periods=1, freq=None, axis=0, **kwds):
             return self
 
         if freq is None and not len(kwds):
-            block_axis = self._get_block_manager_axis(axis)
-            indexer = com._shift_indexer(len(self._get_axis(axis)), periods)
-            new_data = self._data.shift(indexer=indexer, periods=periods, axis=block_axis)
+            new_data = self._data.shift(periods=periods, axis=axis)
         else:
             return self.tshift(periods, freq, **kwds)
 

pandas/core/internals.py

@@ -961,23 +961,20 @@ def diff(self, n):
         return [make_block(new_values, self.items, self.ref_items,
                            ndim=self.ndim, fastpath=True)]
 
-    def shift(self, indexer, periods, axis=0):
+    def shift(self, periods, axis=0):
         """ shift the block by periods, possibly upcast """
-
-        new_values = self.values.take(indexer, axis=axis)
         # convert integer to float if necessary. need to do a lot more than
         # that, handle boolean etc also
-        new_values, fill_value = com._maybe_upcast(new_values)
-
+        new_values, fill_value = com._maybe_upcast(self.values)
+        new_values = np.roll(new_values.T,periods,axis=axis)
         axis_indexer = [ slice(None) ] * self.ndim
         if periods > 0:
             axis_indexer[axis] = slice(None,periods)
         else:
-            axis_indexer = [ slice(None) ] * self.ndim
             axis_indexer[axis] = slice(periods,None)
         new_values[tuple(axis_indexer)] = fill_value
 
-        return [make_block(new_values, self.items, self.ref_items,
+        return [make_block(new_values.T, self.items, self.ref_items,
                            ndim=self.ndim, fastpath=True)]
 
     def eval(self, func, other, raise_on_error=True, try_cast=False):
@@ -1910,9 +1907,15 @@ def fillna(self, value, limit=None, inplace=False, downcast=None):
         values = self.values if inplace else self.values.copy()
         return [self.make_block(values.get_values(value), fill_value=value)]
 
-    def shift(self, indexer, periods, axis=0):
+
+    def shift(self, periods, axis=0):
         """ shift the block by periods """
-
+        N = len(self.values.T)
+        indexer = np.zeros(N, dtype=int)
+        if periods > 0:
+            indexer[periods:] = np.arange(N - periods)
+        else:
+            indexer[:periods] = np.arange(-periods, N)
         new_values = self.values.to_dense().take(indexer)
         # convert integer to float if necessary. need to do a lot more than
         # that, handle boolean etc also

vb_suite/frame_methods.py

@@ -429,3 +429,16 @@ def test_unequal(name):
                                               setup,
                                               start_date=datetime(2014, 2, 7))
 
+
+#-------------------------------------------------------------------------
+# frame shift speedup issue-5609
+
+setup = common_setup + """
+df = pd.DataFrame(np.random.rand(10000,500))
+"""
+frame_shift_axis0 = Benchmark('df.shift(1,axis=0)', setup,
+                    name = 'frame_shift_axis_0',
+                    start_date=datetime(2014,1,1))
+frame_shift_axis1 = Benchmark('df.shift(1,axis=1)', setup,
+                    name = 'frame_shift_axis_1',
+                    start_date=datetime(2014,1,1))