Inconsistency in the result of groupby with multiple columns

[Eisbrenner]

I have a snippet of code (below) which unfortunately I couldn't create a small working example from.

However, I binby three columns to get uniques and export the result to a xarray object and this process sometimes works and sometimes messes up the coordinates. In the figures below I think the process succeeded until it concatenated some data below the rest even though it was supposed to go above.

For reference I have the plots of the expression without the process of filtering for uniques.

and then when filtering here the right one is "broken". It should be a subset of the right plot in the first figure, but it got this weird bar on the bottom which should go above I think.

So what am I messing up? :-/

non-reproducable code below:

# this function is the issue
def hist(df, expression, iiminmax, ijminmax):
    return (
        df[expression]
        .groupby(["id", "ii", "ij"], agg={"id": vaex.agg.first("id", "id")})
        .count(
            binby=[df[expression].ii, df[expression].ij],
            limits=[
                [np.min(iiminmax), np.max(iiminmax)],
                [np.min(ijminmax), np.max(ijminmax)],
            ],
            shape=[
                int(np.max(iiminmax) + 1 - np.min(iiminmax)),
                int(np.max(ijminmax) + 1 - np.min(ijminmax)),
            ],
            array_type="xarray",
        )
    )


def coarsen(da, coarse=8):
    return da.coarsen(
        dim={"ii": coarse, "ij": coarse // 2},
        boundary="pad",
        side="left",
        coord_func="mean",
    ).mean()


expression = c_indomain & c_not_land & c_exceeds & c_maxage & c_exact_age
expression1 = expression & c_in_month_1
expression2 = expression & c_in_month_2

width, frac = 13, 1 / 5
f = plt.figure(figsize=(width, frac * width))
data.df.viz.heatmap(
    "i",
    "j",
    what=np.log(vaex.stat.count() + 1),
    limits="100%",
    shape=[50, 50],
    visual=dict(column="selection"),
    selection=[expression1, expression2],
)
f.patch.set_facecolor("w")

h1 = coarsen(hist(data.df, expression1, ienw, jenn)).T
h2 = coarsen(hist(data.df, expression2, ienw, jenn)).T

vlim = np.quantile(np.abs(dh), 0.995)
f, axs = plt.subplots(ncols=2, nrows=1, figsize=(width, frac * width))
np.log(h1+1).plot(ax=axs.flat[0], cmap=cmr.sunburst)
np.log(h2+1).plot(ax=axs.flat[1], cmap=cmr.sunburst)
f.patch.set_facecolor("w")

[Eisbrenner]

I tested the result now when using numpy instead (aside from the grouping for uniques which is still done by vaex) and the issue is persistent, so it must be inside the way I use vaex for binning of uniques.

def nphist(data, expression, iminmax, jminmax):
    groups = data.df[expression].groupby(
        ["id", "ii", "ij"],
        agg={
            "id": vaex.agg.first("id", "id"),
            "subvol": vaex.agg.first("subvol", "id"),
        },
    )
    z, y, x = np.histogram2d(
        groups.ii.values,
        groups.ij.values,
        bins=(
            int(np.max(iminmax) + 1 - np.min(iminmax)),
            int(np.max(jminmax) + 1 - np.min(jminmax)),
        ),
        weights=groups.subvol.values,
        range=[
            [np.min(iminmax), np.max(iminmax)],
            [np.min(jminmax), np.max(jminmax)],
        ],
    )
    return z.T


f, axs = plt.subplots(ncols=2, nrows=1, figsize=(10, 4))
axs.flat[0].contourf(nphist(data, expression1, ienw, jenn))
axs.flat[1].contourf(nphist(data, expression2, ienw, jenn))
f.patch.set_facecolor("w")

[Eisbrenner]

Well and for completeness when done with pandas + numpy (now the result as expected)

def nphist(data, expression, iminmax, jminmax):
    df = data.df[expression].to_pandas_df()
    df = df.drop_duplicates(subset=['id', 'ii', 'ij'])
    z, y, x = np.histogram2d(
        df.ii.values,
        df.ij.values,
        bins=(
            int(np.max(iminmax) + 1 - np.min(iminmax)),
            int(np.max(jminmax) + 1 - np.min(jminmax)),
        ),
        weights=df.subvol.values,
        range=[
            [np.min(iminmax), np.max(iminmax)],
            [np.min(jminmax), np.max(jminmax)],
        ],
    )
    return z.T


f, axs = plt.subplots(ncols=2, nrows=1, figsize=(10, 4))
axs.flat[0].contourf(nphist(data, expression1, ienw, jenn))
axs.flat[1].contourf(nphist(data, expression2, ienw, jenn))
f.patch.set_facecolor("w")