From 221840b8bae0fabfd488ffe966095b58b4661a85 Mon Sep 17 00:00:00 2001 From: Charlie Fox Date: Wed, 3 Sep 2025 14:39:14 +0100 Subject: [PATCH 1/3] fix: lp-polars api --- linopy/io.py | 78 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 63 insertions(+), 15 deletions(-) diff --git a/linopy/io.py b/linopy/io.py index ba28694c..4e8e5f48 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -40,6 +40,7 @@ concat_kwargs = dict(dim=CONCAT_DIM, coords="minimal") TQDM_COLOR = "#80bfff" +COEFF_THRESHOLD = 1e-12 def handle_batch(batch: list[str], f: TextIOWrapper, batch_size: int) -> list[str]: @@ -550,8 +551,10 @@ def objective_to_file_polars( f.write(f"{sense}\n\nobj:\n\n".encode()) df = m.objective.to_polars() + # Filter out zero coefficients like the regular LP version does if m.is_linear: - objective_write_linear_terms_polars(f, df, print_variable) + df_filtered = df.filter(pl.col("coeffs").abs() > COEFF_THRESHOLD) + objective_write_linear_terms_polars(f, df_filtered, print_variable) elif m.is_quadratic: linear_terms = df.filter(pl.col("vars1").eq(-1) | pl.col("vars2").eq(-1)) @@ -561,9 +564,13 @@ def objective_to_file_polars( .otherwise(pl.col("vars1")) .alias("vars") ) + # Filter out zero coefficients + linear_terms = linear_terms.filter(pl.col("coeffs").abs() > COEFF_THRESHOLD) objective_write_linear_terms_polars(f, linear_terms, print_variable) quads = df.filter(pl.col("vars1").ne(-1) & pl.col("vars2").ne(-1)) + # Filter out zero coefficients + quads = quads.filter(pl.col("coeffs").abs() > COEFF_THRESHOLD) objective_write_quadratic_terms_polars(f, quads, print_variable) @@ -731,28 +738,69 @@ def constraints_to_file_polars( for con_slice in con.iterate_slices(slice_size): df = con_slice.to_polars() - # df = df.lazy() - # filter out repeated label values - df = df.with_columns( - pl.when(pl.col("labels").is_first_distinct()) - .then(pl.col("labels")) - .otherwise(pl.lit(None)) - .alias("labels") + # Filter out rows with zero coefficients or invalid variables - but KEEP RHS rows + # RHS rows have null coeffs/vars but contain the constraint sign/rhs + df = df.filter( + # Keep RHS rows (have sign/rhs but null coeffs/vars) + (pl.col("sign").is_not_null() & pl.col("rhs").is_not_null()) + | + # OR keep valid coefficient rows + ( + (pl.col("coeffs").abs() > COEFF_THRESHOLD) + & (pl.col("vars").is_not_null()) + & (pl.col("vars") >= 0) + ) + ) + + if df.height == 0: + continue + + # Ensure each constraint has both coefficient and RHS terms + analysis = df.group_by("labels").agg( + [ + pl.col("coeffs").is_not_null().sum().alias("coeff_rows"), + pl.col("sign").is_not_null().sum().alias("rhs_rows"), + ] + ) + + valid = analysis.filter( + (pl.col("coeff_rows") > 0) & (pl.col("rhs_rows") > 0) + ) + + if valid.height == 0: + continue + + df = df.join(valid.select("labels"), on="labels", how="inner") + + # Sort by labels for proper grouping and mark first/last occurrences + df = df.sort("labels").with_columns( + [ + pl.when(pl.col("labels").is_first_distinct()) + .then(pl.col("labels")) + .otherwise(pl.lit(None)) + .alias("labels_first"), + (pl.col("labels") != pl.col("labels").shift(-1)) + .fill_null(True) + .alias("is_last_in_group"), + ] ) - row_labels = print_constraint(pl.col("labels")) + # Build output columns + row_labels = print_constraint(pl.col("labels_first")) col_labels = print_variable(pl.col("vars")) columns = [ - pl.when(pl.col("labels").is_not_null()).then(row_labels[0]), - pl.when(pl.col("labels").is_not_null()).then(row_labels[1]), - pl.when(pl.col("labels").is_not_null()).then(pl.lit(":\n")).alias(":"), + pl.when(pl.col("labels_first").is_not_null()).then(row_labels[0]), + pl.when(pl.col("labels_first").is_not_null()).then(row_labels[1]), + pl.when(pl.col("labels_first").is_not_null()) + .then(pl.lit(":\n")) + .alias(":"), pl.when(pl.col("coeffs") >= 0).then(pl.lit("+")), pl.col("coeffs").cast(pl.String), pl.when(pl.col("vars").is_not_null()).then(col_labels[0]), pl.when(pl.col("vars").is_not_null()).then(col_labels[1]), - "sign", - pl.lit(" "), - pl.col("rhs").cast(pl.String), + pl.when(pl.col("is_last_in_group")).then(pl.col("sign")), + pl.when(pl.col("is_last_in_group")).then(pl.lit(" ")), + pl.when(pl.col("is_last_in_group")).then(pl.col("rhs").cast(pl.String)), ] kwargs: Any = dict( From 36b036e7f4ea9be37ff8c64ad6c22165755578b2 Mon Sep 17 00:00:00 2001 From: Charlie Fox Date: Thu, 4 Sep 2025 12:27:43 +0100 Subject: [PATCH 2/3] chore: remove small val filtering --- linopy/io.py | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/linopy/io.py b/linopy/io.py index 4e8e5f48..3ff39838 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -40,7 +40,6 @@ concat_kwargs = dict(dim=CONCAT_DIM, coords="minimal") TQDM_COLOR = "#80bfff" -COEFF_THRESHOLD = 1e-12 def handle_batch(batch: list[str], f: TextIOWrapper, batch_size: int) -> list[str]: @@ -551,10 +550,8 @@ def objective_to_file_polars( f.write(f"{sense}\n\nobj:\n\n".encode()) df = m.objective.to_polars() - # Filter out zero coefficients like the regular LP version does if m.is_linear: - df_filtered = df.filter(pl.col("coeffs").abs() > COEFF_THRESHOLD) - objective_write_linear_terms_polars(f, df_filtered, print_variable) + objective_write_linear_terms_polars(f, df, print_variable) elif m.is_quadratic: linear_terms = df.filter(pl.col("vars1").eq(-1) | pl.col("vars2").eq(-1)) @@ -564,13 +561,9 @@ def objective_to_file_polars( .otherwise(pl.col("vars1")) .alias("vars") ) - # Filter out zero coefficients - linear_terms = linear_terms.filter(pl.col("coeffs").abs() > COEFF_THRESHOLD) objective_write_linear_terms_polars(f, linear_terms, print_variable) quads = df.filter(pl.col("vars1").ne(-1) & pl.col("vars2").ne(-1)) - # Filter out zero coefficients - quads = quads.filter(pl.col("coeffs").abs() > COEFF_THRESHOLD) objective_write_quadratic_terms_polars(f, quads, print_variable) @@ -738,20 +731,6 @@ def constraints_to_file_polars( for con_slice in con.iterate_slices(slice_size): df = con_slice.to_polars() - # Filter out rows with zero coefficients or invalid variables - but KEEP RHS rows - # RHS rows have null coeffs/vars but contain the constraint sign/rhs - df = df.filter( - # Keep RHS rows (have sign/rhs but null coeffs/vars) - (pl.col("sign").is_not_null() & pl.col("rhs").is_not_null()) - | - # OR keep valid coefficient rows - ( - (pl.col("coeffs").abs() > COEFF_THRESHOLD) - & (pl.col("vars").is_not_null()) - & (pl.col("vars") >= 0) - ) - ) - if df.height == 0: continue @@ -770,9 +749,10 @@ def constraints_to_file_polars( if valid.height == 0: continue + # Keep only constraints that have both parts df = df.join(valid.select("labels"), on="labels", how="inner") - # Sort by labels for proper grouping and mark first/last occurrences + # Sort by labels and mark first/last occurrences df = df.sort("labels").with_columns( [ pl.when(pl.col("labels").is_first_distinct()) From c0deea9b948c1fe4210ea6a21cc47ff1ee69ebbd Mon Sep 17 00:00:00 2001 From: Charlie Fox Date: Fri, 5 Sep 2025 11:18:55 +0100 Subject: [PATCH 3/3] chore: remove redundant comment --- linopy/io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/linopy/io.py b/linopy/io.py index 3ff39838..69cfd295 100644 --- a/linopy/io.py +++ b/linopy/io.py @@ -765,7 +765,6 @@ def constraints_to_file_polars( ] ) - # Build output columns row_labels = print_constraint(pl.col("labels_first")) col_labels = print_variable(pl.col("vars")) columns = [