Update src/streamlit_app.py

src/streamlit_app.py

@@ -1,40 +1,350 @@
-import altair as alt
+import streamlit as st
 import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import norm
+from scipy.optimize import minimize
 import pandas as pd
-import streamlit as st
 
-"""
-# Welcome to Streamlit!
-
-Edit `/streamlit_app.py` to customize this app to your heart's desire :heart:.
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
-
-In the meantime, below is an example of what you can do with just a few lines of code:
-"""
-
-num_points = st.slider("Number of points in spiral", 1, 10000, 1100)
-num_turns = st.slider("Number of turns in spiral", 1, 300, 31)
-
-indices = np.linspace(0, 1, num_points)
-theta = 2 * np.pi * num_turns * indices
-radius = indices
-
-x = radius * np.cos(theta)
-y = radius * np.sin(theta)
-
-df = pd.DataFrame({
-    "x": x,
-    "y": y,
-    "idx": indices,
-    "rand": np.random.randn(num_points),
-})
-
-st.altair_chart(alt.Chart(df, height=700, width=700)
-    .mark_point(filled=True)
-    .encode(
-        x=alt.X("x", axis=None),
-        y=alt.Y("y", axis=None),
-        color=alt.Color("idx", legend=None, scale=alt.Scale()),
-        size=alt.Size("rand", legend=None, scale=alt.Scale(range=[1, 150])),
-    ))
+# Set page config
+st.set_page_config(page_title="Gaussian Distribution & Overfitting Demo", layout="wide")
+
+st.title("Gaussian Distribution & Overfitting in ML")
+st.markdown("Interactive demonstration of concepts from PRML Chapter 1")
+
+# Sidebar for navigation
+page = st.sidebar.selectbox("Select Demo", 
+    ["Gaussian Distribution Basics", 
+     "Maximum Likelihood Bias", 
+     "Polynomial Curve Fitting",
+     "Probabilistic Curve Fitting",
+     "Regularized Curve Fitting"])
+
+if page == "Gaussian Distribution Basics":
+    st.header("1.2.4 The Gaussian Distribution")
+    
+    col1, col2 = st.columns(2)
+    
+    with col1:
+        st.subheader("Parameters")
+        mu = st.slider("Mean (μ)", -5.0, 5.0, 0.0, 0.1)
+        sigma = st.slider("Standard Deviation (σ)", 0.1, 5.0, 1.0, 0.1)
+        
+        st.latex(r"N(x|\mu, \sigma^2) = \frac{1}{(2\pi\sigma^2)^{1/2}} \exp\left\{-\frac{1}{2\sigma^2}(x-\mu)^2\right\}")
+    
+    with col2:
+        st.subheader("Gaussian Distribution Plot")
+        x = np.linspace(mu - 4*sigma, mu + 4*sigma, 1000)
+        y = norm.pdf(x, mu, sigma)
+        
+        fig, ax = plt.subplots(figsize=(8, 6))
+        ax.plot(x, y, 'b-', linewidth=2, label=f'N({mu:.1f}, {sigma:.1f}²)')
+        ax.fill_between(x, y, alpha=0.3)
+        ax.axvline(mu, color='r', linestyle='--', label=f'Mean = {mu:.1f}')
+        ax.axvline(mu - sigma, color='g', linestyle='--', alpha=0.5)
+        ax.axvline(mu + sigma, color='g', linestyle='--', alpha=0.5, label=f'±σ = ±{sigma:.1f}')
+        ax.set_xlabel('x')
+        ax.set_ylabel('p(x)')
+        ax.legend()
+        ax.grid(True, alpha=0.3)
+        st.pyplot(fig)
+
+elif page == "Maximum Likelihood Bias":
+    st.header("Maximum Likelihood Bias in Variance Estimation")
+    st.markdown("This demonstrates how ML systematically underestimates the true variance")
+    
+    col1, col2 = st.columns(2)
+    
+    with col1:
+        st.subheader("Simulation Parameters")
+        true_mu = st.slider("True Mean", -2.0, 2.0, 0.0, 0.1)
+        true_sigma = st.slider("True Std Dev", 0.5, 3.0, 1.0, 0.1)
+        n_samples = st.slider("Number of Samples (N)", 2, 100, 10, 1)
+        n_experiments = st.slider("Number of Experiments", 100, 1000, 500, 100)
+        
+        if st.button("Run Simulation"):
+            # Run multiple experiments
+            ml_means = []
+            ml_vars = []
+            unbiased_vars = []
+            
+            for _ in range(n_experiments):
+                # Generate random samples
+                samples = np.random.normal(true_mu, true_sigma, n_samples)
+                
+                # ML estimates
+                ml_mean = np.mean(samples)
+                ml_var = np.var(samples, ddof=0)  # ML estimate
+                unbiased_var = np.var(samples, ddof=1)  # Unbiased estimate
+                
+                ml_means.append(ml_mean)
+                ml_vars.append(ml_var)
+                unbiased_vars.append(unbiased_var)
+            
+            # Store results in session state
+            st.session_state.ml_means = ml_means
+            st.session_state.ml_vars = ml_vars
+            st.session_state.unbiased_vars = unbiased_vars
+            st.session_state.true_var = true_sigma**2
+            st.session_state.n_samples_used = n_samples
+        
+        # Results section below parameters
+        if 'ml_vars' in st.session_state:
+            st.markdown("---")  # Separator line
+            st.subheader("Results")
+            
+            # Calculate averages
+            avg_ml_var = np.mean(st.session_state.ml_vars)
+            avg_unbiased_var = np.mean(st.session_state.unbiased_vars)
+            true_var = st.session_state.true_var
+            n_samples_used = st.session_state.n_samples_used
+            expected_ml_var = (n_samples_used - 1) / n_samples_used * true_var
+            
+            # Display metrics
+            col3, col4, col5, col6 = st.columns(4)
+            with col3:
+                st.metric("Average ML Mean", f"{np.mean(st.session_state.ml_means):.4f}")
+            with col4:
+                st.metric("Average Unbiased Mean", f"{np.mean(st.session_state.unbiased_vars):.4f}")
+            with col5:
+                st.metric("True Mean", f"{true_mu:.4f}")
+            with col6:
+                st.metric("Expected ML Variance", f"{expected_ml_var:.4f}",
+                         f"{(expected_ml_var - true_var) / true_var * 100:.1f}%")
+            
+            # Bias factor
+            st.info(f"Bias Factor: (N-1)/N = {n_samples_used-1}/{n_samples_used} = {(n_samples_used-1)/n_samples_used:.3f}")
+    
+    with col2:
+        if 'ml_vars' in st.session_state:
+            st.subheader("Variance Distribution")
+            
+            # Get values for plotting
+            true_var = st.session_state.true_var
+            n_samples_used = st.session_state.n_samples_used
+            expected_ml_var = (n_samples_used - 1) / n_samples_used * true_var
+            
+            # Histogram
+            fig, ax = plt.subplots(figsize=(10, 8))
+            ax.hist(st.session_state.ml_vars, bins=30, alpha=0.5, label='ML Variance', density=True)
+            ax.hist(st.session_state.unbiased_vars, bins=30, alpha=0.5, label='Unbiased Variance', density=True)
+            ax.axvline(true_var, color='r', linestyle='--', linewidth=2, label='True Variance')
+            ax.axvline(expected_ml_var, color='g', linestyle='--', linewidth=2, label='Expected ML Variance')
+            ax.set_xlabel('Variance Estimate', fontsize=12)
+            ax.set_ylabel('Density', fontsize=12)
+            ax.legend(fontsize=11)
+            ax.grid(True, alpha=0.3)
+            ax.set_title(f'Distribution of Variance Estimates (N={n_samples_used})', fontsize=14)
+            st.pyplot(fig)
+
+elif page == "Polynomial Curve Fitting":
+    st.header("Polynomial Curve Fitting and Overfitting")
+    
+    # Generate true function
+    def true_function(x):
+        return np.sin(2 * np.pi * x)
+    
+    col1, col2 = st.columns([1, 2])
+    
+    with col1:
+        st.subheader("Parameters")
+        n_data_points = st.slider("Number of Data Points", 5, 50, 15, 1)
+        noise_level = st.slider("Noise Level", 0.0, 0.5, 0.2, 0.05)
+        polynomial_degree = st.slider("Polynomial Degree (M)", 0, 15, 3, 1)
+        
+        if st.button("Generate New Data"):
+            np.random.seed(None)  # Random seed
+            x_train = np.random.uniform(0, 1, n_data_points)
+            y_train = true_function(x_train) + np.random.normal(0, noise_level, n_data_points)
+            st.session_state.x_train = x_train
+            st.session_state.y_train = y_train
+    
+    # Initialize data if not exists
+    if 'x_train' not in st.session_state:
+        np.random.seed(42)
+        x_train = np.random.uniform(0, 1, n_data_points)
+        y_train = true_function(x_train) + np.random.normal(0, noise_level, n_data_points)
+        st.session_state.x_train = x_train
+        st.session_state.y_train = y_train
+    
+    with col2:
+        st.subheader("Polynomial Fit")
+        
+        # Fit polynomial
+        X_train = np.vander(st.session_state.x_train, polynomial_degree + 1, increasing=True)
+        w = np.linalg.lstsq(X_train, st.session_state.y_train, rcond=None)[0]
+        
+        # Plot
+        x_plot = np.linspace(0, 1, 200)
+        X_plot = np.vander(x_plot, polynomial_degree + 1, increasing=True)
+        y_pred = X_plot @ w
+        y_true = true_function(x_plot)
+        
+        fig, ax = plt.subplots(figsize=(10, 6))
+        ax.plot(x_plot, y_true, 'g-', linewidth=2, label='True Function')
+        ax.plot(x_plot, y_pred, 'r-', linewidth=2, label=f'Polynomial (M={polynomial_degree})')
+        ax.scatter(st.session_state.x_train, st.session_state.y_train, 
+                  c='blue', s=50, alpha=0.8, edgecolors='black', label='Training Data')
+        ax.set_xlabel('x')
+        ax.set_ylabel('y')
+        ax.set_ylim(-1.5, 1.5)
+        ax.legend()
+        ax.grid(True, alpha=0.3)
+        ax.set_title(f'Polynomial Degree M = {polynomial_degree}')
+        st.pyplot(fig)
+        
+        # Calculate training error
+        y_train_pred = X_train @ w
+        train_rmse = np.sqrt(np.mean((st.session_state.y_train - y_train_pred)**2))
+        st.metric("Training RMSE", f"{train_rmse:.4f}")
+
+elif page == "Probabilistic Curve Fitting":
+    st.header("Probabilistic View of Curve Fitting")
+    st.latex(r"p(t|x,\mathbf{w},\beta) = N(t|y(x,\mathbf{w}), \beta^{-1})")
+    
+    col1, col2 = st.columns([1, 2])
+    
+    with col1:
+        st.subheader("Parameters")
+        n_data_points = st.slider("Number of Data Points", 5, 50, 20, 1)
+        true_noise = st.slider("True Noise (σ)", 0.1, 0.5, 0.2, 0.05)
+        polynomial_degree = st.slider("Polynomial Degree", 0, 9, 3, 1)
+        show_uncertainty = st.checkbox("Show Predictive Distribution", True)
+        
+        if st.button("Generate Data"):
+            np.random.seed(None)
+            x_train = np.random.uniform(0, 1, n_data_points)
+            y_train = np.sin(2 * np.pi * x_train) + np.random.normal(0, true_noise, n_data_points)
+            st.session_state.prob_x_train = x_train
+            st.session_state.prob_y_train = y_train
+    
+    # Initialize data
+    if 'prob_x_train' not in st.session_state:
+        np.random.seed(42)
+        x_train = np.random.uniform(0, 1, n_data_points)
+        y_train = np.sin(2 * np.pi * x_train) + np.random.normal(0, true_noise, n_data_points)
+        st.session_state.prob_x_train = x_train
+        st.session_state.prob_y_train = y_train
+    
+    with col2:
+        st.subheader("Maximum Likelihood Fit")
+        
+        # Fit polynomial and estimate noise
+        X_train = np.vander(st.session_state.prob_x_train, polynomial_degree + 1, increasing=True)
+        w_ml = np.linalg.lstsq(X_train, st.session_state.prob_y_train, rcond=None)[0]
+        
+        # Estimate noise variance (beta^-1)
+        y_train_pred = X_train @ w_ml
+        residuals = st.session_state.prob_y_train - y_train_pred
+        sigma_ml = np.sqrt(np.mean(residuals**2))
+        beta_ml = 1 / (sigma_ml**2)
+        
+        # Plot
+        x_plot = np.linspace(0, 1, 200)
+        X_plot = np.vander(x_plot, polynomial_degree + 1, increasing=True)
+        y_mean = X_plot @ w_ml
+        
+        fig, ax = plt.subplots(figsize=(10, 6))
+        
+        # Plot uncertainty bands if requested
+        if show_uncertainty:
+            y_std = np.sqrt(1 / beta_ml)
+            ax.fill_between(x_plot, y_mean - 2*y_std, y_mean + 2*y_std, 
+                           alpha=0.3, color='red', label='±2σ predictive')
+        
+        ax.plot(x_plot, np.sin(2 * np.pi * x_plot), 'g-', linewidth=2, label='True Function')
+        ax.plot(x_plot, y_mean, 'r-', linewidth=2, label=f'ML Fit (M={polynomial_degree})')
+        ax.scatter(st.session_state.prob_x_train, st.session_state.prob_y_train, 
+                  c='blue', s=50, alpha=0.8, edgecolors='black', label='Training Data')
+        
+        ax.set_xlabel('x')
+        ax.set_ylabel('t')
+        ax.legend()
+        ax.grid(True, alpha=0.3)
+        st.pyplot(fig)
+        
+        # Display estimated parameters
+        col3, col4 = st.columns(2)
+        with col3:
+            st.metric("ML Noise Estimate (σ)", f"{sigma_ml:.3f}")
+        with col4:
+            st.metric("True Noise (σ)", f"{true_noise:.3f}")
+
+elif page == "Regularized Curve Fitting":
+    st.header("Regularized Curve Fitting (MAP Estimation)")
+    st.latex(r"E(\mathbf{w}) = \frac{\beta}{2}\sum_{n=1}^{N}\{y(x_n,\mathbf{w})-t_n\}^2 + \frac{\alpha}{2}\mathbf{w}^T\mathbf{w}")
+    
+    col1, col2 = st.columns([1, 2])
+    
+    with col1:
+        st.subheader("Parameters")
+        n_data_points = st.slider("Data Points", 10, 50, 15, 1)
+        noise_level = st.slider("Noise", 0.1, 0.5, 0.3, 0.05)
+        polynomial_degree = st.slider("Degree (M)", 0, 15, 9, 1)
+        log_lambda = st.slider("log₁₀(λ)", -8.0, 2.0, -3.0, 0.5)
+        regularization = 10**log_lambda
+        
+        if st.button("New Data"):
+            np.random.seed(None)
+            x_train = np.random.uniform(0, 1, n_data_points)
+            y_train = np.sin(2 * np.pi * x_train) + np.random.normal(0, noise_level, n_data_points)
+            st.session_state.reg_x_train = x_train
+            st.session_state.reg_y_train = y_train
+    
+    # Initialize
+    if 'reg_x_train' not in st.session_state:
+        np.random.seed(42)
+        x_train = np.random.uniform(0, 1, n_data_points)
+        y_train = np.sin(2 * np.pi * x_train) + np.random.normal(0, noise_level, n_data_points)
+        st.session_state.reg_x_train = x_train
+        st.session_state.reg_y_train = y_train
+    
+    with col2:
+        st.subheader("Regularized Fit")
+        
+        # Fit with regularization
+        X_train = np.vander(st.session_state.reg_x_train, polynomial_degree + 1, increasing=True)
+        
+        # Ridge regression (L2 regularization)
+        XtX = X_train.T @ X_train
+        Xty = X_train.T @ st.session_state.reg_y_train
+        w_reg = np.linalg.solve(XtX + regularization * np.eye(polynomial_degree + 1), Xty)
+        
+        # Plot
+        x_plot = np.linspace(0, 1, 200)
+        X_plot = np.vander(x_plot, polynomial_degree + 1, increasing=True)
+        y_pred = X_plot @ w_reg
+        
+        fig, ax = plt.subplots(figsize=(10, 6))
+        ax.plot(x_plot, np.sin(2 * np.pi * x_plot), 'g-', linewidth=2, label='True Function')
+        ax.plot(x_plot, y_pred, 'r-', linewidth=2, label=f'Regularized (λ={regularization:.1e})')
+        ax.scatter(st.session_state.reg_x_train, st.session_state.reg_y_train, 
+                  c='blue', s=50, alpha=0.8, edgecolors='black', label='Training Data')
+        ax.set_xlabel('x')
+        ax.set_ylabel('t')
+        ax.set_ylim(-1.5, 1.5)
+        ax.legend()
+        ax.grid(True, alpha=0.3)
+        ax.set_title(f'M = {polynomial_degree}, λ = {regularization:.1e}')
+        st.pyplot(fig)
+        
+        # Metrics
+        train_pred = X_train @ w_reg
+        train_rmse = np.sqrt(np.mean((st.session_state.reg_y_train - train_pred)**2))
+        weight_norm = np.linalg.norm(w_reg)
+        
+        col3, col4 = st.columns(2)
+        with col3:
+            st.metric("Training RMSE", f"{train_rmse:.4f}")
+        with col4:
+            st.metric("||w||²", f"{weight_norm:.2f}")
+
+# Add information footer
+st.markdown("---")
+st.markdown("### Key Concepts Demonstrated:")
+st.markdown("""
+- **Gaussian Distribution**: Fundamental probability distribution with mean μ and variance σ²
+- **Maximum Likelihood Bias**: ML estimation systematically underestimates variance by factor (N-1)/N
+- **Overfitting**: High-degree polynomials fit training data perfectly but generalize poorly
+- **Probabilistic Curve Fitting**: View regression as estimating conditional distribution p(t|x)
+- **Regularization**: Adding penalty term α||w||² prevents overfitting (equivalent to MAP with Gaussian prior)
+""")