Merge multiple filter queries and apply once

src/copairs/compute.py

@@ -151,7 +151,7 @@ def compute_p_values(ap_scores, null_confs, null_size: int, seed):
     p_values = np.empty(len(ap_scores), dtype=np.float32)
     for i, (ap_score, ix) in enumerate(zip(ap_scores, rev_ix)):
         # Reverse to get from hi to low
-        num = null_size - np.searchsorted(null_dists[ix], ap_score)
+        num = null_size - np.searchsorted(null_dists[ix], ap_score, side='right')
         p_values[i] = (num + 1) / (null_size + 1)
     return p_values
 

src/copairs/map.py

@@ -12,26 +12,45 @@
 logger = logging.getLogger('copairs')
 
 
-def evaluate_and_filter(df, columns) -> list:
-    '''Evaluate the query and filter the dataframe'''
+def extract_filters(columns, df_columns) -> list:
+    '''Extract and validate filters from columns'''
     parsed_cols = []
+    queries_to_eval = []
+
     for col in columns:
-        if col in df.columns:
+        if col in df_columns:
             parsed_cols.append(col)
             continue
-
         column_names = re.findall(r'(\w+)\s*[=<>!]+', col)
-        valid_column_names = [col for col in column_names if col in df.columns]
+
+        valid_column_names = [col for col in column_names if col in df_columns]
         if not valid_column_names:
             raise ValueError(f"Invalid query or column name: {col}")
+
+        queries_to_eval.append(col)
+        parsed_cols.extend(valid_column_names)
+
+        if len(parsed_cols) != len(set(parsed_cols)):
+            raise ValueError(f"Duplicate queries for column: {col}")
+
+    return queries_to_eval, parsed_cols
+
+
+def apply_filters(df, query_list):
+    '''Combine and apply filters to dataframe'''
+    if not query_list:
+        return df
+
+    combined_query = " & ".join(f"({query})" for query in query_list)
+    try:
+        df_filtered = df.query(combined_query)
+    except Exception as e:
+        raise ValueError(f"Invalid combined query expression: {combined_query}. Error: {e}")
 
-        try:
-            df = df.query(col)
-            parsed_cols.extend(valid_column_names)
-        except:
-            raise ValueError(f"Invalid query expression: {col}")
+    if df_filtered.empty:
+        raise ValueError(f"Empty dataframe after processing combined query: {combined_query}")
 
-    return df, parsed_cols
+    return df_filtered
 
 
 def flatten_str_list(*args):
@@ -55,7 +74,9 @@ def create_matcher(obs: pd.DataFrame,
                    neg_diffby,
                    multilabel_col=None):
     columns = flatten_str_list(pos_sameby, pos_diffby, neg_sameby, neg_diffby)
-    obs, columns = evaluate_and_filter(obs, columns)
+    query_list, columns = extract_filters(columns, obs.columns)
+    obs = apply_filters(obs, query_list)
+
     if multilabel_col:
         return MatcherMultilabel(obs, columns, multilabel_col, seed=0)
     return Matcher(obs, columns, seed=0)