Capstone - Complete Analytics System

# Python: Introduction to Player Clustering
import pandas as pd
import numpy as np

# Example: Same position, different players
midfielders = player_stats[
    (player_stats["position"] == "Midfielder") &
    (player_stats["minutes"] >= 900)
].copy()

# Look at variance within "Midfielder"
metrics = ["tackles_90", "progressive_passes_90", "xa_90", "npxg_90"]
midfielder_variance = midfielders[metrics].agg(["mean", "std", lambda x: x.max() - x.min()])
midfielder_variance.index = ["mean", "std", "range"]

print("Variance within Midfielders:")
print(midfielder_variance)

# Create a simple style score
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()

midfielders["style_score"] = (
    scaler.fit_transform(midfielders[["xa_90"]]).flatten() -
    scaler.fit_transform(midfielders[["tackles_90"]]).flatten()
)

extremes = midfielders.sort_values("style_score")[
    ["player", "team", "tackles_90", "xa_90", "style_score"]
]

print("\nMost Defensive Midfielders:")
print(extremes.head())

print("\nMost Creative Midfielders:")
print(extremes.tail())

# Python: Feature Engineering for Player Clustering
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler

def create_clustering_features(player_stats):
    """Create and scale features for player clustering."""
    # Filter for minimum minutes
    df = player_stats[player_stats["minutes"] >= 900].copy()

    # Create composite features
    df["goal_threat"] = df["npxg_90"] + (df["shots_90"] * 0.1)
    df["chance_creation"] = df["xa_90"] + (df["key_passes_90"] * 0.05)

    df["pass_risk"] = df["long_balls_90"] / (df["passes_90"] + 1)
    df["progressive_rate"] = df["progressive_passes_90"] / (df["passes_90"] + 1)

    df["defensive_actions"] = (
        df["tackles_90"] + df["interceptions_90"] + df["blocks_90"]
    )
    df["pressing_intensity"] = df["pressures_90"]

    df["ball_progression"] = df["progressive_carries_90"] + df["dribbles_90"]
    df["box_presence"] = df["touches_att_pen_90"]

    df["total_involvement"] = df["touches_90"]

    # Select features for clustering
    feature_cols = [
        "goal_threat", "chance_creation",
        "pass_completion_pct", "pass_risk", "progressive_rate",
        "defensive_actions", "pressing_intensity",
        "ball_progression", "box_presence",
        "total_involvement"
    ]

    # Extract features
    features_raw = df[["player", "team", "position", "minutes"] + feature_cols].copy()

    # Scale features
    scaler = StandardScaler()
    features_scaled = features_raw.copy()
    features_scaled[feature_cols] = scaler.fit_transform(features_raw[feature_cols])

    return {
        "raw": features_raw,
        "scaled": features_scaled,
        "feature_cols": feature_cols,
        "scaler": scaler
    }

clustering_data = create_clustering_features(player_stats)
print(clustering_data["scaled"].head())

# Python: K-Means Clustering for Player Roles
import pandas as pd
import numpy as np
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
import matplotlib.pyplot as plt

def perform_kmeans_clustering(data, feature_cols, max_k=15):
    """Perform K-Means clustering with optimal K selection."""
    # Extract feature matrix
    X = data["scaled"][feature_cols].values

    # Find optimal K
    wss = []
    sil_scores = []

    for k in range(2, max_k + 1):
        km = KMeans(n_clusters=k, n_init=25, max_iter=100, random_state=42)
        km.fit(X)
        wss.append(km.inertia_)
        sil_scores.append(silhouette_score(X, km.labels_))

    # Plot elbow and silhouette
    fig, axes = plt.subplots(1, 2, figsize=(14, 5))

    # Elbow plot
    axes[0].plot(range(2, max_k + 1), wss, "bo-")
    axes[0].set_xlabel("Number of Clusters")
    axes[0].set_ylabel("Within-cluster SS")
    axes[0].set_title("Elbow Method")

    # Silhouette plot
    axes[1].plot(range(2, max_k + 1), sil_scores, "go-")
    axes[1].set_xlabel("Number of Clusters")
    axes[1].set_ylabel("Silhouette Score")
    axes[1].set_title("Silhouette Analysis")

    plt.tight_layout()
    plt.show()

    # Best K based on silhouette
    best_k = np.argmax(sil_scores) + 2

    # Final clustering
    final_km = KMeans(n_clusters=best_k, n_init=50, max_iter=100, random_state=42)
    data["scaled"]["cluster"] = final_km.fit_predict(X)

    return {
        "model": final_km,
        "data": data["scaled"],
        "best_k": best_k,
        "silhouette": max(sil_scores)
    }

km_result = perform_kmeans_clustering(clustering_data, clustering_data["feature_cols"])

print(f"Optimal number of clusters: {km_result['best_k']}")
print(f"Silhouette score: {km_result['silhouette']:.3f}")
print("\nCluster distribution:")
print(km_result["data"]["cluster"].value_counts().sort_index())

# Python: Interpret and Label Clusters
import pandas as pd
import numpy as np

def interpret_clusters(km_result, feature_cols):
    """Interpret clusters and assign meaningful labels."""
    # Get cluster centers
    centers = pd.DataFrame(
        km_result["model"].cluster_centers_,
        columns=feature_cols
    )
    centers["cluster"] = range(len(centers))

    # Cluster summaries
    data = km_result["data"]
    cluster_summary = data.groupby("cluster").agg(
        n_players=("player", "count"),
        common_positions=("position", lambda x: ", ".join(
            x.value_counts().head(2).index.tolist()
        )),
        example_players=("player", lambda x: ", ".join(x.head(3).tolist()))
    ).reset_index()

    # Get distinguishing features
    def get_key_features(row):
        features = []
        for col in feature_cols:
            z = row[col]
            if z > 0.5:
                features.append(f"{col}: HIGH")
            elif z < -0.5:
                features.append(f"{col}: LOW")

        return "; ".join(sorted(features, key=lambda x: -abs(row[x.split(":")[0].strip()]))[:3])

    centers["key_features"] = centers.apply(get_key_features, axis=1)

    # Auto-label clusters
    def label_cluster(profile):
        if "goal_threat: HIGH" in profile:
            if "chance_creation: HIGH" in profile:
                return "Complete Forward"
            return "Goal Poacher"
        if "chance_creation: HIGH" in profile:
            return "Creative Playmaker"
        if "defensive_actions: HIGH" in profile:
            if "ball_progression: HIGH" in profile:
                return "Box-to-Box"
            return "Defensive Anchor"
        if "ball_progression: HIGH" in profile:
            return "Progressive Carrier"
        if "pressing_intensity: HIGH" in profile:
            return "High Press Forward"
        return "Utility Player"

    centers["label"] = centers["key_features"].apply(label_cluster)

    # Combine
    result = cluster_summary.merge(
        centers[["cluster", "key_features", "label"]],
        on="cluster"
    )

    return result

cluster_interpretation = interpret_clusters(km_result, clustering_data["feature_cols"])
print(cluster_interpretation[["cluster", "label", "n_players", "example_players"]])

# Python: Hierarchical Clustering for Player Roles
import pandas as pd
import numpy as np
from scipy.cluster.hierarchy import dendrogram, linkage, fcluster
from scipy.spatial.distance import pdist
import matplotlib.pyplot as plt

def perform_hierarchical_clustering(data, feature_cols, method="ward"):
    """Perform hierarchical clustering and create dendrogram."""
    # Extract feature matrix
    X = data[feature_cols].values
    players = data["player"].values

    # Calculate linkage
    linkage_matrix = linkage(X, method=method, metric="euclidean")

    # Plot dendrogram
    fig, ax = plt.subplots(figsize=(16, 8))
    dendrogram(
        linkage_matrix,
        labels=players,
        leaf_rotation=90,
        leaf_font_size=6,
        color_threshold=7,
        ax=ax
    )
    ax.set_title("Player Role Hierarchy")
    ax.set_ylabel("Distance")
    plt.tight_layout()
    plt.show()

    # Cut at different levels
    cuts = {
        "k4": fcluster(linkage_matrix, 4, criterion="maxclust"),
        "k8": fcluster(linkage_matrix, 8, criterion="maxclust"),
        "k12": fcluster(linkage_matrix, 12, criterion="maxclust")
    }

    # Create progression table
    progression = pd.DataFrame({
        "player": players,
        "level_4": cuts["k4"],
        "level_8": cuts["k8"],
        "level_12": cuts["k12"]
    })

    # Show nesting
    nesting = progression.groupby(["level_4", "level_8"]).size().reset_index(name="count")

    return {
        "linkage": linkage_matrix,
        "cuts": cuts,
        "progression": progression,
        "nesting": nesting
    }

hc_result = perform_hierarchical_clustering(
    km_result["data"],
    clustering_data["feature_cols"]
)

print("Cluster Nesting (4 -> 8):")
print(hc_result["nesting"])

# Show hierarchy for specific cluster
print("\nPlayers in k=4 Cluster 1, by k=8 cluster:")
k4_c1 = hc_result["progression"][hc_result["progression"]["level_4"] == 1]
print(k4_c1["level_8"].value_counts().sort_index())

# Python: Gaussian Mixture Models for Soft Clustering
import pandas as pd
import numpy as np
from sklearn.mixture import GaussianMixture
from sklearn.metrics import silhouette_score

def perform_gmm_clustering(data, feature_cols, max_k=12):
    """Perform GMM clustering with soft assignments."""
    X = data[feature_cols].values

    # Find optimal number of components using BIC
    bic_scores = []
    for k in range(2, max_k + 1):
        gmm = GaussianMixture(n_components=k, covariance_type="full",
                              random_state=42, n_init=10)
        gmm.fit(X)
        bic_scores.append(gmm.bic(X))

    best_k = np.argmin(bic_scores) + 2
    print(f"Optimal components (by BIC): {best_k}")

    # Fit final model
    final_gmm = GaussianMixture(n_components=best_k, covariance_type="full",
                                random_state=42, n_init=10)
    final_gmm.fit(X)

    # Get assignments and probabilities
    data = data.copy()
    data["cluster"] = final_gmm.predict(X)
    probabilities = final_gmm.predict_proba(X)

    # Identify hybrid players
    max_prob = probabilities.max(axis=1)
    sorted_probs = np.sort(probabilities, axis=1)[:, ::-1]
    second_prob = sorted_probs[:, 1]

    data["max_prob"] = max_prob
    data["second_prob"] = second_prob
    data["is_hybrid"] = second_prob > 0.25

    # Get hybrid player details
    hybrid_mask = data["is_hybrid"]
    hybrid_players = data[hybrid_mask][["player", "cluster", "max_prob", "second_prob"]].copy()

    second_clusters = []
    for i, (idx, row) in enumerate(data[hybrid_mask].iterrows()):
        probs = probabilities[data.index.get_loc(idx)]
        sorted_indices = np.argsort(probs)[::-1]
        second_clusters.append(sorted_indices[1])

    hybrid_players["second_cluster"] = second_clusters

    return {
        "model": final_gmm,
        "data": data,
        "probabilities": probabilities,
        "hybrid_players": hybrid_players
    }

gmm_result = perform_gmm_clustering(km_result["data"], clustering_data["feature_cols"])

print("\nHybrid Players (blend multiple roles):")
print(gmm_result["hybrid_players"].head(10))

# Python: Position-Specific Clustering
import pandas as pd
import numpy as np
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler

def create_position_clusters(player_stats):
    """Create position-specific clusters with tailored features."""
    results = {}

    # Forward-specific clustering
    forwards = player_stats[
        player_stats["position"].isin(["Forward", "Winger"]) &
        (player_stats["minutes"] >= 900)
    ].copy()

    forward_features = [
        "npxg_90", "shots_90", "xa_90", "dribbles_90",
        "touches_att_pen_90", "progressive_carries_90",
        "pressures_att_third_90", "aerial_wins_90"
    ]

    fwd_X = StandardScaler().fit_transform(forwards[forward_features])
    fwd_km = KMeans(n_clusters=5, random_state=42, n_init=25)
    forwards["cluster"] = fwd_km.fit_predict(fwd_X)

    fwd_labels = {
        0: "Goal Poacher", 1: "Complete Forward", 2: "Creator Winger",
        3: "Inside Forward", 4: "Target Man"
    }
    forwards["role"] = forwards["cluster"].map(fwd_labels)
    results["forwards"] = forwards

    # Midfielder-specific clustering
    midfielders = player_stats[
        (player_stats["position"] == "Midfielder") &
        (player_stats["minutes"] >= 900)
    ].copy()

    mid_features = [
        "progressive_passes_90", "xa_90", "tackles_90", "interceptions_90",
        "pressures_90", "pass_completion_pct", "progressive_carries_90",
        "through_balls_90"
    ]

    mid_X = StandardScaler().fit_transform(midfielders[mid_features])
    mid_km = KMeans(n_clusters=5, random_state=42, n_init=25)
    midfielders["cluster"] = mid_km.fit_predict(mid_X)

    mid_labels = {
        0: "Defensive Anchor", 1: "Deep Playmaker", 2: "Box-to-Box",
        3: "Advanced Playmaker", 4: "Pressing Midfielder"
    }
    midfielders["role"] = midfielders["cluster"].map(mid_labels)
    results["midfielders"] = midfielders

    # Defender-specific clustering
    defenders = player_stats[
        player_stats["position"].isin(["Defender", "Center Back", "Full Back"]) &
        (player_stats["minutes"] >= 900)
    ].copy()

    def_features = [
        "tackles_90", "interceptions_90", "blocks_90", "clearances_90",
        "aerial_wins_90", "progressive_passes_90", "progressive_carries_90",
        "pass_completion_pct"
    ]

    def_X = StandardScaler().fit_transform(defenders[def_features])
    def_km = KMeans(n_clusters=5, random_state=42, n_init=25)
    defenders["cluster"] = def_km.fit_predict(def_X)

    def_labels = {
        0: "Ball-Playing CB", 1: "Traditional Stopper", 2: "Attacking FB",
        3: "Defensive FB", 4: "Sweeper"
    }
    defenders["role"] = defenders["cluster"].map(def_labels)
    results["defenders"] = defenders

    return results

position_clusters = create_position_clusters(player_stats)

print("Forward Archetypes:")
print(position_clusters["forwards"]["role"].value_counts())

print("\nMidfielder Archetypes:")
print(position_clusters["midfielders"]["role"].value_counts())

print("\nDefender Archetypes:")
print(position_clusters["defenders"]["role"].value_counts())

# Python: Comprehensive Cluster Validation
import pandas as pd
import numpy as np
from sklearn.cluster import KMeans
from sklearn.metrics import (silhouette_score, calinski_harabasz_score,
                            davies_bouldin_score, adjusted_rand_score)
from scipy.spatial.distance import cdist

def validate_clusters(data, feature_cols, k_range=range(2, 13)):
    """Comprehensive cluster validation."""
    X = data[feature_cols].values
    results = {}

    # 1. Internal validation metrics
    internal_metrics = []

    for k in k_range:
        km = KMeans(n_clusters=k, n_init=25, random_state=42)
        labels = km.fit_predict(X)

        metrics = {
            "k": k,
            "silhouette": silhouette_score(X, labels),
            "calinski_harabasz": calinski_harabasz_score(X, labels),
            "davies_bouldin": davies_bouldin_score(X, labels)
        }
        internal_metrics.append(metrics)

    results["internal"] = pd.DataFrame(internal_metrics)

    # 2. Stability validation (bootstrap)
    n_boot = 50
    k_test = 8

    original_km = KMeans(n_clusters=k_test, n_init=25, random_state=42)
    original_labels = original_km.fit_predict(X)

    stability_scores = []
    for b in range(n_boot):
        # Bootstrap sample
        boot_idx = np.random.choice(len(X), size=len(X), replace=True)
        X_boot = X[boot_idx]

        # Cluster bootstrap sample
        boot_km = KMeans(n_clusters=k_test, n_init=10, random_state=b)
        boot_labels = boot_km.fit_predict(X_boot)

        # Map back to original indices for comparison
        unique_idx = np.unique(boot_idx)
        if len(unique_idx) > 10:
            orig_labels_subset = original_labels[unique_idx]
            # Find corresponding boot labels
            boot_labels_mapped = []
            for idx in unique_idx:
                positions = np.where(boot_idx == idx)[0]
                boot_labels_mapped.append(boot_labels[positions[0]])

            ari = adjusted_rand_score(orig_labels_subset, boot_labels_mapped)
            stability_scores.append(ari)

    results["stability"] = np.mean(stability_scores)

    # 3. Cluster separation
    km_final = KMeans(n_clusters=8, n_init=25, random_state=42)
    km_final.fit(X)
    center_distances = cdist(km_final.cluster_centers_, km_final.cluster_centers_)
    np.fill_diagonal(center_distances, np.inf)

    results["min_center_distance"] = center_distances.min()
    results["avg_center_distance"] = center_distances[center_distances < np.inf].mean()

    return results

validation = validate_clusters(km_result["data"], clustering_data["feature_cols"])

print("Internal Validation Metrics:")
print(validation["internal"])

print(f"\nCluster Stability (Adjusted Rand Index): {validation[\"stability\"]:.3f}")
print(f"Min Center Distance: {validation[\"min_center_distance\"]:.3f}")
print(f"Avg Center Distance: {validation[\"avg_center_distance\"]:.3f}")

# Python: Visualizing Clusters with Dimensionality Reduction
import numpy as np
import pandas as pd
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import umap
import matplotlib.pyplot as plt

def visualize_clusters(data, feature_cols, cluster_labels):
    """Visualize clusters using different dimensionality reduction techniques."""
    X = data[feature_cols].values

    fig, axes = plt.subplots(1, 3, figsize=(18, 5))

    # PCA
    pca = PCA(n_components=2)
    pca_coords = pca.fit_transform(X)
    explained_var = sum(pca.explained_variance_ratio_) * 100

    scatter1 = axes[0].scatter(
        pca_coords[:, 0], pca_coords[:, 1],
        c=cluster_labels, cmap="tab10", alpha=0.7
    )
    axes[0].set_xlabel("PC1")
    axes[0].set_ylabel("PC2")
    axes[0].set_title(f"PCA (Explained var: {explained_var:.1f}%)")

    # t-SNE
    tsne = TSNE(n_components=2, perplexity=30, random_state=42)
    tsne_coords = tsne.fit_transform(X)

    axes[1].scatter(
        tsne_coords[:, 0], tsne_coords[:, 1],
        c=cluster_labels, cmap="tab10", alpha=0.7
    )
    axes[1].set_xlabel("Dimension 1")
    axes[1].set_ylabel("Dimension 2")
    axes[1].set_title("t-SNE")

    # UMAP
    reducer = umap.UMAP(n_neighbors=15, min_dist=0.1, random_state=42)
    umap_coords = reducer.fit_transform(X)

    axes[2].scatter(
        umap_coords[:, 0], umap_coords[:, 1],
        c=cluster_labels, cmap="tab10", alpha=0.7
    )
    axes[2].set_xlabel("UMAP 1")
    axes[2].set_ylabel("UMAP 2")
    axes[2].set_title("UMAP")

    plt.colorbar(scatter1, ax=axes, label="Cluster", shrink=0.6)
    plt.tight_layout()
    plt.show()

    return {"pca": pca_coords, "tsne": tsne_coords, "umap": umap_coords}

viz_coords = visualize_clusters(
    km_result["data"],
    clustering_data["feature_cols"],
    km_result["data"]["cluster"]
)

# Python: Player Similarity Search
import pandas as pd
import numpy as np
from scipy.spatial.distance import cdist

def create_similarity_engine(data, feature_cols):
    """Create a player similarity search function."""
    # Build feature matrix
    X = data[feature_cols].values
    players = data["player"].values

    # Calculate pairwise distances
    dist_matrix = cdist(X, X, metric="euclidean")
    player_to_idx = {p: i for i, p in enumerate(players)}

    def find_similar(player_name, n=10, same_cluster_only=False,
                     position_filter=None):
        """Find most similar players to target."""
        if player_name not in player_to_idx:
            raise ValueError(f"Player not found: {player_name}")

        idx = player_to_idx[player_name]
        distances = dist_matrix[idx]

        # Create results
        results = pd.DataFrame({
            "player": players,
            "distance": distances
        }).merge(
            data[["player", "team", "position", "cluster"]],
            on="player"
        )

        # Remove self
        results = results[results["player"] != player_name]

        # Apply filters
        if same_cluster_only:
            target_cluster = data.loc[data["player"] == player_name, "cluster"].iloc[0]
            results = results[results["cluster"] == target_cluster]

        if position_filter:
            results = results[results["position"].isin(position_filter)]

        # Sort and return
        results = results.nsmallest(n, "distance")
        results["similarity"] = 1 / (1 + results["distance"])

        return results[["player", "team", "position", "cluster", "distance", "similarity"]]

    return find_similar

find_similar = create_similarity_engine(km_result["data"], clustering_data["feature_cols"])

# Find players similar to Mo Salah
print("Players most similar to Mo Salah:")
print(find_similar("Mo Salah", n=10))

# Find similar midfielders to De Bruyne
print("\nPlayers similar to De Bruyne (same cluster):")
print(find_similar("Kevin De Bruyne", n=5, same_cluster_only=True))

# Python: Recruitment Use Case
import pandas as pd

def find_replacement_targets(departing_player, find_similar_fn, player_info,
                              max_age=28, max_value=50, exclude_leagues=None):
    """Find replacement targets for a departing player."""
    if exclude_leagues is None:
        exclude_leagues = []

    # Find similar players
    similar = find_similar_fn(departing_player, n=50)

    # Merge with player info
    targets = similar.merge(
        player_info[["player", "age", "market_value", "league"]],
        on="player",
        how="left"
    )

    # Apply recruitment filters
    targets = targets[
        (targets["age"] <= max_age) &
        (targets["market_value"] <= max_value) &
        (~targets["league"].isin(exclude_leagues))
    ]

    # Calculate value score
    targets["value_score"] = targets["similarity"] / (targets["market_value"] / 10 + 1)
    targets["fit_score"] = targets["similarity"] * 100

    return targets.nlargest(10, "value_score")[
        ["player", "team", "age", "market_value", "fit_score", "value_score"]
    ]

# Find replacement for Mo Salah
replacement_targets = find_replacement_targets(
    departing_player="Mo Salah",
    find_similar_fn=find_similar,
    player_info=player_info,
    max_age=26,
    max_value=60,
    exclude_leagues=["Premier League"]
)

print("Replacement Targets for Mo Salah:")
print(replacement_targets)