feat: piloting of the framework (#227)

* Initial template

* Updated a bit on the framework

* Revised to enable feat training

* Fixed Accuracy

* fixed mcc

* change some files for feature

* update some logics in feat eng

* update some codes

* update some codes

* Revised further to successfully completed a cycle

* Revised further

* Delete two CSVs

* fix some ci errors

---------

Co-authored-by: Young <afe.young@gmail.com>
Co-authored-by: Xisen-Wang <xisen_application@163.com>

rdagent/scenarios/kaggle/experiment/README.md

@@ -0,0 +1,6 @@
+
+# Meta template
+It is an example of how we organize the workspace of a competition.
+We expect all the competitions to align with it so the knowledge in modules (model, feature) can transfer.
+
+The generation process of the initial template is hoped to be conducted by LLM (however, it is based on human efforts currently).

rdagent/scenarios/kaggle/experiment/meta_tpl/fea_share_preprocess.py

@@ -0,0 +1,47 @@
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OneHotEncoder
+
+
+def preprocess(X: pd.DataFrame):
+ """
+ We want the X_train & X_test & X_valid to contain the same number of columns & maintain feature consistency.
+ """
+ # Identify numerical and categorical features
+ numerical_cols = [cname for cname in X.columns if X[cname].dtype in ["int64", "float64"]]
+ categorical_cols = [cname for cname in X.columns if X[cname].dtype == "object"]
+
+ # Define preprocessors for numerical and categorical features
+ categorical_transformer = Pipeline(
+ steps=[
+ ("imputer", SimpleImputer(strategy="most_frequent")),
+ ("onehot", OneHotEncoder(handle_unknown="ignore")),
+ ]
+ )
+
+ numerical_transformer = Pipeline(steps=[("imputer", SimpleImputer(strategy="mean"))])
+
+ # Combine preprocessing steps
+ preprocessor = ColumnTransformer(
+ transformers=[
+ ("cat", categorical_transformer, categorical_cols),
+ ("num", numerical_transformer, numerical_cols),
+ ]
+ )
+
+ # Fit the preprocessor on the data and transform it
+ preprocessor.fit(X) # TODO depend on its input shape
+ X_array = preprocessor.transform(X).toarray()
+
+ # Get feature names for the columns in the transformed data
+ feature_names = (
+ preprocessor.named_transformers_["cat"]["onehot"].get_feature_names_out(categorical_cols).tolist()
+ + numerical_cols
+ )
+
+ # Convert arrays back to DataFrames
+ X_transformed = pd.DataFrame(X_array, columns=feature_names, index=X.index)
+
+ return X_transformed

rdagent/scenarios/kaggle/experiment/meta_tpl/feat.py

@@ -0,0 +1,13 @@
+import pandas as pd
+
+"""
+Here is the feature engineering code for each task, with the function name specified as feat_eng. 
+The file name should start with feat_, followed by the specific task name.
+"""
+
+
+def feat_eng(X: pd.DataFrame):
+ """
+ return the selected features
+ """
+ return X

rdagent/scenarios/kaggle/experiment/meta_tpl/model.py

@@ -0,0 +1,36 @@
+"""
+motivation of the model
+"""
+import pandas as pd
+import xgboost as xgb
+
+
+def select(X):
+ """
+ Select relevant features. To be used in fit & predict function
+ """
+ return X
+
+
+def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_valid: pd.DataFrame):
+ """Define and train the model. Merge feature_select"""
+ dtrain = xgb.DMatrix(X_train, label=y_train)
+ dvalid = xgb.DMatrix(X_valid, label=y_valid)
+
+ # TODO: for quick running....
+ params = {}
+ num_round = 2
+
+ evallist = [(dtrain, "train"), (dvalid, "eval")]
+ bst = xgb.train(params, dtrain, num_round, evallist)
+
+ return bst
+
+
+def predict(model, X):
+ """
+ Keep feature select's consistency.
+ """
+ dtest = xgb.DMatrix(X)
+ y_pred_prob = model.predict(dtest)
+ return y_pred_prob > 0.5 # Apply threshold to get boolean predictions

rdagent/scenarios/kaggle/experiment/meta_tpl/train.py

@@ -0,0 +1,125 @@
+import os
+import random
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import xgboost as xgb
+from fea_share_preprocess import preprocess
+from sklearn.compose import ColumnTransformer
+from sklearn.impute import SimpleImputer
+from sklearn.metrics import accuracy_score, matthews_corrcoef
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import LabelEncoder, OneHotEncoder
+
+from rdagent.scenarios.kaggle.experiment.meta_tpl.fea_share_preprocess import preprocess
+
+# Set random seed for reproducibility
+SEED = 42
+random.seed(SEED)
+np.random.seed(SEED)
+DIRNAME = Path(__file__).absolute().resolve().parent
+
+
+# support various method for metrics calculation
+def compute_metrics_for_classification(y_true, y_pred):
+ """Compute accuracy metric for classification."""
+ accuracy = accuracy_score(y_true, y_pred)
+ return accuracy
+
+
+def compute_metrics_for_classification(y_true, y_pred):
+ """Compute MCC for classification."""
+ mcc = matthews_corrcoef(y_true, y_pred)
+ return mcc
+
+
+# Load and preprocess the data
+data_df = pd.read_csv("/home/v-xisenwang/git_ignore_folder/data/playground-series-s4e8/train.csv")
+data_df = data_df.drop(["id"], axis=1)
+
+X = data_df.drop(["class"], axis=1)
+y = data_df[["class"]]
+
+label_encoder = LabelEncoder()
+y = label_encoder.fit_transform(y) # 将类别标签转换为数值
+X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.10, random_state=SEED)
+
+# 1) Preprocess the data
+X_train = preprocess(X_train)
+X_valid = preprocess(X_valid)
+
+submission_df = pd.read_csv("/home/v-xisenwang/git_ignore_folder/data/playground-series-s4e8/test.csv")
+passenger_ids = submission_df["id"]
+submission_df = submission_df.drop(["id"], axis=1)
+X_test = preprocess(submission_df)
+
+# 2) Auto feature engineering
+X_train_l, X_valid_l = [], []
+X_test_l = []
+for f in DIRNAME.glob("feat*.py"):
+ m = __import__(f.name.strip(".py"))
+ X_train = m.feat_eng(X_train)
+ X_valid = m.feat_eng(X_valid)
+ X_test = m.feat_eng(X_test)
+
+ X_train_l.append(X_train)
+ X_valid_l.append(X_valid)
+ X_test_l.append(X_test)
+
+X_train = pd.concat(X_train_l, axis=1)
+X_valid = pd.concat(X_valid_l, axis=1)
+X_test = pd.concat(X_test_l, axis=1)
+
+
+def align_features(train_df, valid_df):
+ # Align the features of validation data to the training data
+ valid_df = valid_df.reindex(columns=train_df.columns, fill_value=0)
+ return valid_df
+
+
+X_valid = align_features(X_train, X_valid)
+X_test = align_features(X_train, X_test)
+
+# 3) Train the model
+model_l = [] # list[tuple[model, predict_func,]]
+for f in DIRNAME.glob("model*.py"):
+ # TODO put select() in model.py: fit(X_train, y_train, X_valid, y_valid)
+ m = __import__(f.name.strip(".py"))
+ model_l.append((m.fit(X_train, y_train, X_valid, y_valid), m.predict))
+
+# Evaluate the model on the validation set
+y_valid_pred_l = []
+for model, predict_func in model_l:
+ y_valid_pred_l.append(predict_func(model, X_valid))
+
+# Ensemble
+# TODO: ensemble method in a script
+# Average the predictions and apply a threshold to determine class labels
+y_valid_pred = np.mean(y_valid_pred_l, axis=0)
+y_valid_pred = (y_valid_pred > 0.5).astype(int)
+
+mcc = compute_metrics_for_classification(y_valid, y_valid_pred)
+print("Final on validation set: ", mcc)
+
+# Save the validation accuracy
+pd.Series(data=[mcc], index=["MCC"]).to_csv(
+ "/home/v-xisenwang/RD-Agent/rdagent/scenarios/kaggle/experiment/meta_tpl/submission_score.csv"
+)
+
+# Make predictions on the test set and save them
+y_test_pred_bool_l = []
+for m, m_pred in model_l:
+ y_test_pred_bool_l.append(
+ m_pred(m, X_test).astype(int)
+ ) # TODO Make this an ensemble. Currently it uses the last prediction
+
+y_test_pred = np.mean(y_test_pred_bool_l, axis=0)
+y_test_pred = (y_test_pred > 0.5).astype(int) # TODO Make it a module. Ensemble prediction
+
+y_test_pred_labels = label_encoder.inverse_transform(y_test_pred) # 将整数转换回 'e' 或 'p'
+submission_result = pd.DataFrame({"id": passenger_ids, "class": y_test_pred_labels})
+
+# submit predictions for the test set
+submission_result.to_csv("./submission.csv", index=False)

rdagent/scenarios/kaggle/experiment/prompts.yaml

@@ -4,7 +4,7 @@ kg_description_template:
 
  user: |-
  Based on the following competition description, please extract the following details:
- 1. Competition Type
+ 1. Competition Type 
  2. Competition Description
  3. Target Description
  4. Competition Features