Feature Selection vs Feature Extraction

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_digits

# Pima Diabetes — load with zero-imputation (same as Decision Tree and Random Forest posts)
url = "https://raw.githubusercontent.com/jbrownlee/Datasets/master/pima-indians-diabetes.data.csv"
cols = ['Pregnancies','Glucose','BloodPressure','SkinThickness',
        'Insulin','BMI','DiabetesPedigree','Age','Outcome']
df = pd.read_csv(url, names=cols)
for col in ['Glucose','BloodPressure','SkinThickness','Insulin','BMI']:
    df[col] = df[col].replace(0, np.nan)
    df[col].fillna(df[col].median(), inplace=True)

X = df.drop('Outcome', axis=1)
y = df['Outcome']
feature_names = X.columns.tolist()
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y)
print(f"Train: {X_train.shape}, Test: {X_test.shape}")

from sklearn.feature_selection import VarianceThreshold

sel_var = VarianceThreshold(threshold=0.1)
X_var = sel_var.fit_transform(X_train)
print(f"Features before: {X_train.shape[1]}, after: {X_var.shape[1]}")
print(f"Removed features: {np.array(feature_names)[~sel_var.get_support()].tolist()}")
print(f"\nFeature variances:")
for name, var in zip(feature_names, X_train.var()):
    print(f"  {name:20s}: {var:.3f}")

from sklearn.feature_selection import SelectKBest, f_classif

sel_kbest = SelectKBest(score_func=f_classif, k=5)
X_k5 = sel_kbest.fit_transform(X_train, y_train)

scores = pd.DataFrame({
    'Feature': feature_names,
    'F_score': sel_kbest.scores_,
    'p_value': sel_kbest.pvalues_,
    'Selected': sel_kbest.get_support()
}).sort_values('F_score', ascending=False)
print(scores.round(4))

from sklearn.feature_selection import mutual_info_classif

mi_scores_arr = mutual_info_classif(X_train, y_train, random_state=42)
mi_df = pd.DataFrame({
    'Feature': feature_names,
    'MI_score': mi_scores_arr,
}).sort_values('MI_score', ascending=False)
print(mi_df.round(4))

from sklearn.feature_selection import RFE
from sklearn.linear_model import LogisticRegression

lr = LogisticRegression(max_iter=500, random_state=42)
rfe = RFE(estimator=lr, n_features_to_select=5, step=1)
rfe.fit(X_train, y_train)

print("RFE feature ranking (rank 1 = selected):")
for feat, rank, sel in zip(feature_names, rfe.ranking_, rfe.support_):
    marker = "✓ selected" if sel else f"rank {rank}"
    print(f"  {feat:20s}: {marker}")

from sklearn.feature_selection import RFECV
from sklearn.model_selection import StratifiedKFold

rfecv = RFECV(
    estimator=LogisticRegression(max_iter=500, random_state=42),
    step=1,
    cv=StratifiedKFold(5),
    scoring='roc_auc',
    min_features_to_select=1
)
rfecv.fit(X_train, y_train)
print(f"Optimal n_features: {rfecv.n_features_}")
print(f"Selected: {np.array(feature_names)[rfecv.support_].tolist()}")
cv_aucs = rfecv.cv_results_['mean_test_score']
print(f"\nCV AUC per n_features: {cv_aucs.round(4)}")

from sklearn.linear_model import LassoCV
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
X_tr_sc = scaler.fit_transform(X_train)
X_te_sc = scaler.transform(X_test)

lasso = LassoCV(cv=5, random_state=42, max_iter=2000)
lasso.fit(X_tr_sc, y_train)

print(f"Best alpha: {lasso.alpha_:.4f}")
lasso_coefs = pd.DataFrame({
    'Feature': feature_names,
    'Coefficient': lasso.coef_
}).sort_values('Coefficient', key=abs, ascending=False)
print(lasso_coefs.round(4))
print(f"\nNonzero features: {(lasso.coef_ != 0).sum()}")

from sklearn.decomposition import PCA
from sklearn.datasets import load_digits

digits = load_digits()
X_d = digits.data  # (1797, 64)

pca = PCA(n_components=2)
X_2d = pca.fit_transform(X_d)
print(f"Original: {X_d.shape} → PCA 2D: {X_2d.shape}")
print(f"PC1 explains: {pca.explained_variance_ratio_[0]:.4f}")
print(f"PC2 explains: {pca.explained_variance_ratio_[1]:.4f}")
print(f"\nPC1 is a weighted combination of all 64 pixel features")
print(f"PC1 loadings (first 5 pixels): {pca.components_[0, :5].round(4)}")

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score

results = []

# 1. All 8 features
lr = LogisticRegression(max_iter=500, random_state=42)
lr.fit(X_train, y_train)
auc = roc_auc_score(y_test, lr.predict_proba(X_test)[:, 1])
results.append(('All features (8)', 8, auc))

# 2. Filter top-5 (f_classif)
X_k5_test = sel_kbest.transform(X_test)
lr.fit(X_k5, y_train)
auc = roc_auc_score(y_test, lr.predict_proba(X_k5_test)[:, 1])
results.append(('Filter top-5 (f_classif)', 5, auc))

# 3. RFE top-5
X_rfe_test = rfe.transform(X_test)
lr.fit(rfe.transform(X_train), y_train)
auc = roc_auc_score(y_test, lr.predict_proba(X_rfe_test)[:, 1])
results.append(('RFE top-5', 5, auc))

# 4. Lasso nonzero features (6)
mask = lasso.coef_ != 0
lr_sc = LogisticRegression(max_iter=500, random_state=42)
lr_sc.fit(X_tr_sc[:, mask], y_train)
auc = roc_auc_score(y_test, lr_sc.predict_proba(X_te_sc[:, mask])[:, 1])
results.append(('Lasso nonzero (6)', int(mask.sum()), auc))

print(f"{'Method':28s} | {'n_feat':>7} | {'Test AUC':>10}")
for name, n, auc in results:
    print(f"{name:28s} | {n:>7} | {auc:>10.4f}")