diff --git a/crates/polars-core/src/chunked_array/ops/sort/arg_sort_multiple.rs b/crates/polars-core/src/chunked_array/ops/sort/arg_sort_multiple.rs
index 538e5164bef1..734d6cab79a2 100644
--- a/crates/polars-core/src/chunked_array/ops/sort/arg_sort_multiple.rs
+++ b/crates/polars-core/src/chunked_array/ops/sort/arg_sort_multiple.rs
@@ -100,7 +100,8 @@ pub fn _get_rows_encoded_compat_array(by: &Series) -> PolarsResult<ArrayRef> {
                 ca.physical().chunks[0].clone()
             }
         },
-        _ => by.to_arrow(0, CompatLevel::newest()),
+        // Take physical
+        _ => by.chunks()[0].clone(),
     };
     Ok(out)
 }
diff --git a/py-polars/tests/unit/test_rows.py b/py-polars/tests/unit/test_rows.py
index 1a66f6dc7fb3..91d001a58c96 100644
--- a/py-polars/tests/unit/test_rows.py
+++ b/py-polars/tests/unit/test_rows.py
@@ -1,3 +1,5 @@
+from datetime import date
+
 import pytest
 
 import polars as pl
@@ -246,3 +248,23 @@ def test_row_constructor_uint64() -> None:
         data=[[0], [int(2**63) + 1]], schema={"x": pl.UInt64}, orient="row"
     )
     assert df.rows() == [(0,), (9223372036854775809,)]
+
+
+def test_physical_row_encoding() -> None:
+    dt_str = [
+        {
+            "ts": date(2023, 7, 1),
+            "files": "AGG_202307.xlsx",
+            "period_bins": [date(2023, 7, 1), date(2024, 1, 1)],
+        },
+    ]
+
+    df = pl.from_dicts(dt_str)
+    df_groups = df.group_by("period_bins")
+    assert df_groups.all().to_dicts() == [
+        {
+            "period_bins": [date(2023, 7, 1), date(2024, 1, 1)],
+            "ts": [date(2023, 7, 1)],
+            "files": ["AGG_202307.xlsx"],
+        }
+    ]