preprocessing.ipynb

# Advanced missing value handling
from sklearn.impute import SimpleImputer

# Separate numerical and categorical columns
numerical_cols = data.select_dtypes(include=['float64', 'int64']).columns
categorical_cols = data.select_dtypes(include=['object']).columns

# Impute numerical columns with median
num_imputer = SimpleImputer(strategy='median')
data[numerical_cols] = num_imputer.fit_transform(data[numerical_cols])

# Impute categorical columns with most frequent value
cat_imputer = SimpleImputer(strategy='most_frequent')
data[categorical_cols] = cat_imputer.fit_transform(data[categorical_cols])

print("Missing values imputed successfully!")


# Outlier detection and removal using IQR
def remove_outliers(df, col):
    Q1 = df[col].quantile(0.25)
    Q3 = df[col].quantile(0.75)
    IQR = Q3 - Q1
    lower_bound = Q1 - 1.5 * IQR
    upper_bound = Q3 + 1.5 * IQR
    return df[(df[col] >= lower_bound) & (df[col] <= upper_bound)]

for col in numerical_cols:
    data = remove_outliers(data, col)

print("Outliers removed successfully!")

from sklearn.preprocessing import OneHotEncoder

# One-hot encoding with threshold to keep only frequent categories
encoder = OneHotEncoder(handle_unknown='ignore', min_frequency=10, sparse=False)
encoded_cols = pd.DataFrame(encoder.fit_transform(data[categorical_cols]), 
                            columns=encoder.get_feature_names_out(categorical_cols))
data = data.drop(categorical_cols, axis=1).reset_index(drop=True)
data = pd.concat([data, encoded_cols], axis=1)

print("Categorical variables encoded successfully!")

# Apply log transformation for skewed columns
import numpy as np

for col in numerical_cols:
    if data[col].skew() > 1:  # Apply to highly skewed features
        data[col] = np.log1p(data[col])

print("Skewness reduced using log transformation!")

from sklearn.preprocessing import PolynomialFeatures

# Generate polynomial features
poly = PolynomialFeatures(degree=2, interaction_only=True, include_bias=False)
poly_features = poly.fit_transform(data[numerical_cols])

# Add to original dataframe
poly_df = pd.DataFrame(poly_features, columns=poly.get_feature_names_out(numerical_cols))
data = pd.concat([data.reset_index(drop=True), poly_df.reset_index(drop=True)], axis=1)

print("Polynomial features added successfully!")

# Target-based mean encoding for high-cardinality categorical features
for col in categorical_cols:
    if data[col].nunique() > 10:  # Apply only to high-cardinality features
        target_mean = data.groupby(col)['Salary'].mean()
        data[col] = data[col].map(target_mean)

print("High-cardinality features encoded using target mean!")

from sklearn.preprocessing import RobustScaler

# Scale numerical columns
scaler = RobustScaler()
data[numerical_cols] = scaler.fit_transform(data[numerical_cols])

print("Numerical features scaled successfully!")

from sklearn.feature_selection import VarianceThreshold

# Apply variance threshold
selector = VarianceThreshold(threshold=0.01)  # Adjust threshold as needed
selected_features = selector.fit_transform(data)

print("Low-variance features removed successfully!")

# Custom feature engineering
data['Experience_Level'] = pd.cut(data['Experience'], bins=[0, 5, 10, 20, np.inf], labels=['Junior', 'Mid', 'Senior', 'Expert'])
data['Age_Group'] = pd.cut(data['Age'], bins=[18, 30, 45, 60, np.inf], labels=['Young', 'Adult', 'Middle-Age', 'Senior'])

print("Custom features engineered successfully!")


from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline

# Pipeline for preprocessing
preprocessor = ColumnTransformer(
    transformers=[
        ('num', Pipeline([
            ('imputer', SimpleImputer(strategy='median')),
            ('scaler', RobustScaler())
        ]), numerical_cols),
        ('cat', Pipeline([
            ('imputer', SimpleImputer(strategy='most_frequent')),
            ('encoder', OneHotEncoder(handle_unknown='ignore'))
        ]), categorical_cols)
    ]
)

data_processed = preprocessor.fit_transform(data)
print("Data preprocessing pipeline applied successfully!")