🧠 Naive Bayes Fundamentals

🎯 What is Naive Bayes?

Naive Bayes is a family of probabilistic classifiers based on Bayes' theorem with a "naive" assumption of conditional independence between features. Despite its simplicity, it's remarkably effective for text classification, spam filtering, sentiment analysis, and medical diagnosis. It's fast, requires little training data, and works well with high-dimensional data.

Why Naive Bayes Matters

Simple: Easy to understand and implement. Fast: Training and prediction are O(n). Effective: Works surprisingly well for text. Low data: Needs less training data than complex models. Probabilistic: Provides confidence scores. Baseline: Excellent baseline for any classification task.

📌 Bayes' Theorem📌 Probabilistic📌 Fast📌 Text Classification

# Naive Bayes intuition (conceptual)
class NaiveBayes:
    def __init__(self):
        self.priors = {}      # P(class)
        self.likelihoods = {} # P(feature | class)
    
    def train(self, X, y):
        # 1. Compute class priors: P(class)
        for c in set(y):
            self.priors[c] = (y == c).mean()
        
        # 2. Compute likelihoods: P(feature | class)
        for c in set(y):
            X_c = X[y == c]
            self.likelihoods[c] = {
                'mean': X_c.mean(axis=0),
                'std': X_c.std(axis=0)
            }
    
    def predict(self, x):
        # 3. Compute posterior: P(class | x) ∝ P(class) * Π P(feature | class)
        posteriors = {}
        for c in self.priors:
            prior = self.priors[c]
            likelihood = self.compute_likelihood(x, c)
            posteriors[c] = prior * likelihood
        
        # 4. Return class with highest posterior
        return max(posteriors, key=posteriors.get)
    
    def compute_likelihood(self, x, c):
        # Gaussian likelihood
        mean = self.likelihoods[c]['mean']
        std = self.likelihoods[c]['std']
        
        # P(x | c) = Π P(x_i | c)
        likelihood = 1.0
        for i, x_i in enumerate(x):
            p = (1 / (np.sqrt(2 * np.pi) * std[i])) * \
                np.exp(-((x_i - mean[i]) ** 2) / (2 * std[i] ** 2))
            likelihood *= p
        
        return likelihood

📐 Bayes' Theorem

🎯 The Foundation

Bayes' theorem describes the probability of an event based on prior knowledge of conditions related to the event. It's the mathematical foundation of Naive Bayes.

The Formula

📊 Posterior📈 Likelihood📉 Prior📋 Evidence

# Bayes' theorem from scratch
import numpy as np
import pandas as pd
from collections import defaultdict

def bayes_theorem(P_B_given_A, P_A, P_B):
    """P(A|B) = P(B|A) * P(A) / P(B)"""
    return (P_B_given_A * P_A) / P_B

# Example: Medical diagnosis
# Disease affects 1% of population
P_disease = 0.01
P_no_disease = 0.99

# Test accuracy: 99% sensitivity, 95% specificity
P_positive_given_disease = 0.99
P_positive_given_no_disease = 0.05

# Total probability of positive test
P_positive = (P_positive_given_disease * P_disease + 
              P_positive_given_no_disease * P_no_disease)

# Bayes: P(disease | positive)
P_disease_given_positive = bayes_theorem(
    P_positive_given_disease, P_disease, P_positive
)

print(f"P(Disease | Positive Test) = {P_disease_given_positive:.4f}")
print(f"Even with 99% accurate test, only {P_disease_given_positive*100:.1f}% chance of disease")

# Naive Bayes classification formula
# P(class | x_1, ..., x_n) ∝ P(class) × Π P(x_i | class)
def naive_bayes_posterior(class_prior, feature_likelihoods):
    """
    class_prior: P(class)
    feature_likelihoods: [P(x_1|class), ..., P(x_n|class)]
    """
    posterior = class_prior
    for likelihood in feature_likelihoods:
        posterior *= likelihood
    return posterior

🔗 Conditional Independence

🎯 The "Naive" Assumption

Naive Bayes assumes that features are conditionally independent given the class. This means P(x_i | class, x_j) = P(x_i | class). It's "naive" because this is rarely true in reality, but it works surprisingly well.

Why It Works

Simplification: Reduces parameters from exponential to linear. Robustness: Even if independence is violated, classification often works. Overconfidence: Probabilities are often miscalibrated but rankings are correct. Real-world: Document classification, spam filtering — features are correlated but NB still works.

🔗 Independence📉 Fewer Parameters💪 Robust🎯 Good Rankings

# Conditional independence illustration
import numpy as np

# Example: Email classification
# Features: contains "free", contains "money", contains "viagra"
# Class: spam or not spam

# WITHOUT independence assumption:
# Need P(free, money, viagra | spam) — 2^3 = 8 combinations

# WITH independence assumption:
# P(free, money, viagra | spam) = P(free|spam) × P(money|spam) × P(viagra|spam)
# Only need 3 × 2 = 6 probabilities

def compute_joint_probability(features, class_probs):
    """
    Compute P(features | class) assuming independence.
    """
    prob = 1.0
    for feature, p in zip(features, class_probs):
        if feature:
            prob *= p
        else:
            prob *= (1 - p)
    return prob

# Spam probabilities for each feature
spam_probs = [0.8, 0.7, 0.6]  # P(feature=True | spam)
ham_probs = [0.2, 0.1, 0.05]  # P(feature=True | ham)

# Test email containing "free" and "money" but not "viagra"
features = [True, True, False]

P_email_given_spam = compute_joint_probability(features, spam_probs)
P_email_given_ham = compute_joint_probability(features, ham_probs)

# Assume 30% of emails are spam
P_spam = 0.3
P_ham = 0.7

# Posterior (unnormalized)
P_spam_given_email = P_email_given_spam * P_spam
P_ham_given_email = P_email_given_ham * P_ham

print(f"P(email | spam) = {P_email_given_spam:.6f}")
print(f"P(email | ham)  = {P_email_given_ham:.6f}")
print(f"P(spam | email) ∝ {P_spam_given_email:.6f}")
print(f"P(ham | email)  ∝ {P_ham_given_email:.6f}")
print(f"Classification: {'SPAM' if P_spam_given_email > P_ham_given_email else 'HAM'}")

🔀 Naive Bayes Variants

🎯 Different Data, Different Assumptions

Naive Bayes has several variants depending on the type of data: Gaussian (continuous), Multinomial (counts), Bernoulli (binary), Categorical (discrete), Complement (imbalanced).

Variant Comparison

Gaussian NB: Features follow normal distribution. Multinomial NB: Feature counts (word frequencies). Bernoulli NB: Binary features (word present/absent). Categorical NB: Discrete categories. Complement NB: For imbalanced datasets.

📈 Gaussian📝 Multinomial🎲 Bernoulli🏷️ Categorical

# Naive Bayes variants with scikit-learn
from sklearn.naive_bayes import (
    GaussianNB, MultinomialNB, BernoulliNB, 
    CategoricalNB, ComplementNB
)
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

# Generate data
X, y = make_classification(n_samples=1000, n_features=20, 
                           n_informative=10, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# 1. Gaussian NB — for continuous features
gnb = GaussianNB()
gnb.fit(X_train, y_train)
print(f"Gaussian NB: {accuracy_score(y_test, gnb.predict(X_test)):.4f}")

# 2. Multinomial NB — for count data (text)
X_counts = np.abs(X).astype(int)  # Simulate counts
X_train_c, X_test_c, y_train_c, y_test_c = train_test_split(X_counts, y, test_size=0.2)
mnb = MultinomialNB()
mnb.fit(X_train_c, y_train_c)
print(f"Multinomial NB: {accuracy_score(y_test_c, mnb.predict(X_test_c)):.4f}")

# 3. Bernoulli NB — for binary features
X_binary = (X > 0).astype(int)
X_train_b, X_test_b, y_train_b, y_test_b = train_test_split(X_binary, y, test_size=0.2)
bnb = BernoulliNB()
bnb.fit(X_train_b, y_train_b)
print(f"Bernoulli NB: {accuracy_score(y_test_b, bnb.predict(X_test_b)):.4f}")

# 4. Categorical NB — for discrete categories
X_cat = np.abs(X).astype(int) % 5  # 5 categories
X_train_cat, X_test_cat, y_train_cat, y_test_cat = train_test_split(X_cat, y, test_size=0.2)
cnb = CategoricalNB()
cnb.fit(X_train_cat, y_train_cat)
print(f"Categorical NB: {accuracy_score(y_test_cat, cnb.predict(X_test_cat)):.4f}")

# 5. Complement NB — for imbalanced data
cnb2 = ComplementNB()
cnb2.fit(X_train_c, y_train_c)
print(f"Complement NB: {accuracy_score(y_test_c, cnb2.predict(X_test_c)):.4f}")

📈 Gaussian Naive Bayes

🎯 Continuous Features

Gaussian NB assumes features follow a normal distribution within each class. It's used for continuous data like measurements, sensor readings, and financial indicators.

Gaussian Likelihood

Formula: P(x_i | class) = (1 / √(2πσ²)) × exp(-(x_i - μ)² / (2σ²)). Parameters: Mean (μ) and variance (σ²) per feature per class. Training: Compute mean and variance from data. Prediction: Evaluate Gaussian PDF at test point.

🔔 Normal📊 Mean📉 Variance📈 Continuous

# Gaussian Naive Bayes from scratch
import numpy as np

class GaussianNB:
    def __init__(self):
        self.classes = None
        self.means = {}
        self.variances = {}
        self.priors = {}
    
    def fit(self, X, y):
        self.classes = np.unique(y)
        n_samples = len(y)
        
        for c in self.classes:
            X_c = X[y == c]
            self.means[c] = X_c.mean(axis=0)
            self.variances[c] = X_c.var(axis=0) + 1e-9  # avoid /0
            self.priors[c] = len(X_c) / n_samples
    
    def gaussian_pdf(self, x, mean, var):
        """P(x | class) for continuous x."""
        return (1 / np.sqrt(2 * np.pi * var)) * \
               np.exp(-((x - mean) ** 2) / (2 * var))
    
    def predict(self, X):
        predictions = []
        for x in X:
            posteriors = {}
            for c in self.classes:
                # Log posterior = log P(c) + Σ log P(x_i | c)
                log_posterior = np.log(self.priors[c])
                for i, x_i in enumerate(x):
                    log_posterior += np.log(
                        self.gaussian_pdf(x_i, self.means[c][i], 
                                          self.variances[c][i])
                    )
                posteriors[c] = log_posterior
            # Return class with highest log posterior
            predictions.append(max(posteriors, key=posteriors.get))
        return np.array(predictions)

# Usage
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split

iris = load_iris()
X_train, X_test, y_train, y_test = train_test_split(
    iris.data, iris.target, test_size=0.2, random_state=42
)

gnb = GaussianNB()
gnb.fit(X_train, y_train)
predictions = gnb.predict(X_test)
accuracy = (predictions == y_test).mean()
print(f"Gaussian NB accuracy: {accuracy:.4f}")

📝 Multinomial Naive Bayes

🎯 Count Data & Text

Multinomial NB is designed for count data, especially text where features are word frequencies. It's the go-to algorithm for document classification and spam filtering.

Multinomial Likelihood

Formula: P(document | class) ∝ Π P(word_i | class)^count_i. Parameters: Word probabilities per class. Training: Count word occurrences per class. Laplace smoothing: Add 1 to all counts to handle unseen words. Bag of words: Ignores word order.

📊 Counts📧 Spam📰 Documents🎒 Bag of Words

# Multinomial Naive Bayes from scratch
import numpy as np
from collections import Counter

class MultinomialNB:
    def __init__(self, alpha=1.0):
        self.alpha = alpha  # Laplace smoothing
        self.class_priors = {}
        self.word_probs = {}
        self.vocab = set()
    
    def fit(self, X, y):
        """X: list of documents (lists of words), y: labels"""
        classes = set(y)
        n_docs = len(X)
        
        for c in classes:
            # Get documents of this class
            docs_c = [X[i] for i in range(n_docs) if y[i] == c]
            
            # Class prior
            self.class_priors[c] = len(docs_c) / n_docs
            
            # Word counts
            word_counts = Counter()
            for doc in docs_c:
                word_counts.update(doc)
            
            # Vocabulary from all documents
            self.vocab.update(word_counts.keys())
            
            # Total words in class
            total_words = sum(word_counts.values())
            
            # Word probabilities with Laplace smoothing
            self.word_probs[c] = {}
            for word in self.vocab:
                self.word_probs[c][word] = (
                    (word_counts.get(word, 0) + self.alpha) / 
                    (total_words + self.alpha * len(self.vocab))
                )
    
    def predict(self, X):
        predictions = []
        for doc in X:
            posteriors = {}
            for c in self.class_priors:
                # Log posterior = log P(c) + Σ log P(word | c)
                log_posterior = np.log(self.class_priors[c])
                for word in doc:
                    if word in self.vocab:
                        log_posterior += np.log(self.word_probs[c][word])
                posteriors[c] = log_posterior
            predictions.append(max(posteriors, key=posteriors.get))
        return predictions

# Example: Spam classification
training_docs = [
    ['free', 'money', 'now', 'click'],
    ['meeting', 'tomorrow', 'at', 'noon'],
    ['free', 'viagra', 'click', 'now'],
    ['project', 'update', 'attached'],
    ['win', 'free', 'prize', 'money'],
    ['lunch', 'with', 'team', 'tomorrow']
]
labels = ['spam', 'ham', 'spam', 'ham', 'spam', 'ham']

mnb = MultinomialNB(alpha=1.0)
mnb.fit(training_docs, labels)

# Test
test_docs = [
    ['free', 'money'],
    ['meeting', 'tomorrow'],
    ['click', 'here', 'now']
]
predictions = mnb.predict(test_docs)
for doc, pred in zip(test_docs, predictions):
    print(f"{doc} → {pred}")

🎲 Bernoulli Naive Bayes

🎯 Binary Features

Bernoulli NB assumes features are binary (present/absent). Unlike Multinomial, it explicitly penalizes absence of features — good for short texts and binary data.

Bernoulli vs Multinomial

Bernoulli: Binary features, penalizes absence. Multinomial: Count features, ignores absence. Formula: P(x | c) = Π P(x_i=1|c)^x_i × (1-P(x_i=1|c))^(1-x_i). Use cases: Short texts, binary vectors, presence/absence data.

🔘 Binary📊 Presence🚫 Absence Penalty📄 Short Texts

# Bernoulli Naive Bayes from scratch
import numpy as np
from collections import Counter

class BernoulliNB:
    def __init__(self, alpha=1.0):
        self.alpha = alpha
        self.class_priors = {}
        self.feature_probs = {}
        self.vocab = set()
    
    def fit(self, X, y):
        """X: binary feature matrix (n_samples, n_features)"""
        classes = set(y)
        n_samples, n_features = X.shape
        
        for c in classes:
            X_c = X[y == c]
            self.class_priors[c] = len(X_c) / n_samples
            
            # Probability of each feature being 1 in this class
            self.feature_probs[c] = (X_c.sum(axis=0) + self.alpha) / \
                                     (len(X_c) + 2 * self.alpha)
    
    def predict(self, X):
        predictions = []
        for x in X:
            posteriors = {}
            for c in self.class_priors:
                log_posterior = np.log(self.class_priors[c])
                for i, x_i in enumerate(x):
                    p = self.feature_probs[c][i]
                    if x_i == 1:
                        log_posterior += np.log(p)
                    else:
                        log_posterior += np.log(1 - p)
                posteriors[c] = log_posterior
            predictions.append(max(posteriors, key=posteriors.get))
        return predictions

# Example: Document classification with binary features
# Vocabulary: [free, money, meeting, tomorrow, click, project]
vocab = ['free', 'money', 'meeting', 'tomorrow', 'click', 'project']

# Documents as binary vectors
X_train = np.array([
    [1, 1, 0, 0, 1, 0],  # "free money click" → spam
    [0, 0, 1, 1, 0, 0],  # "meeting tomorrow" → ham
    [1, 1, 0, 0, 1, 0],  # "free money click" → spam
    [0, 0, 0, 0, 0, 1],  # "project" → ham
])
y_train = np.array(['spam', 'ham', 'spam', 'ham'])

bnb = BernoulliNB(alpha=1.0)
bnb.fit(X_train, y_train)

X_test = np.array([
    [1, 1, 0, 0, 0, 0],  # "free money"
    [0, 0, 1, 0, 0, 0],  # "meeting"
])
predictions = bnb.predict(X_test)
print(f"Predictions: {predictions}")

# Scikit-learn version
from sklearn.naive_bayes import BernoulliNB
from sklearn.feature_extraction.text import CountVectorizer

documents = [
    "free money now click here",
    "meeting tomorrow at noon",
    "win free prize money now",
    "project update attached file",
]
labels = ['spam', 'ham', 'spam', 'ham']

vectorizer = CountVectorizer(binary=True)
X = vectorizer.fit_transform(documents).toarray()

bnb = BernoulliNB(alpha=1.0)
bnb.fit(X, labels)

test_docs = ["free money", "meeting tomorrow"]
X_test = vectorizer.transform(test_docs).toarray()
print(f"Sklearn predictions: {bnb.predict(X_test)}")

🎓 Training & Parameter Estimation

🎯 Learning from Data

Training Naive Bayes means estimating two sets of parameters: class priors P(class) and feature likelihoods P(feature | class). Both are computed via Maximum Likelihood Estimation (MLE).

Training Steps

1. Compute priors: P(c) = count(c) / N. 2. Compute likelihoods: P(x_i | c) via MLE. 3. Store parameters: Means/variances (Gaussian) or probabilities (Multinomial/Bernoulli). 4. Smoothing: Add small constant to avoid zeros. 5. Prediction: Use Bayes' rule with stored parameters.

📊 MLE📈 Priors📉 Likelihoods🎯 Parameters

# Training Naive Bayes — step by step
import numpy as np
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split

class NaiveBayesTrainer:
    def __init__(self):
        self.class_priors = {}
        self.feature_stats = {}
    
    def fit(self, X, y):
        """Train on data."""
        classes = np.unique(y)
        n_samples = len(y)
        
        print("=== Training Naive Bayes ===")
        
        for c in classes:
            X_c = X[y == c]
            
            # 1. Class prior: P(c)
            prior = len(X_c) / n_samples
            self.class_priors[c] = prior
            print(f"\nClass {c}:")
            print(f"  Prior P(c) = {len(X_c)}/{n_samples} = {prior:.4f}")
            
            # 2. Feature statistics: mean, variance per feature
            mean = X_c.mean(axis=0)
            var = X_c.var(axis=0) + 1e-9
            self.feature_stats[c] = {'mean': mean, 'var': var}
            
            print(f"  Feature means: {mean.round(3)}")
            print(f"  Feature variances: {var.round(3)}")
        
        print("\n=== Training complete ===")
    
    def predict_proba(self, X):
        """Compute posterior probabilities."""
        n_samples = len(X)
        n_classes = len(self.class_priors)
        probs = np.zeros((n_samples, n_classes))
        
        for i, x in enumerate(X):
            for j, c in enumerate(self.class_priors):
                log_p = np.log(self.class_priors[c])
                mean = self.feature_stats[c]['mean']
                var = self.feature_stats[c]['var']
                
                # Gaussian log-likelihood
                log_likelihood = -0.5 * np.sum(
                    np.log(2 * np.pi * var) + ((x - mean) ** 2) / var
                )
                probs[i, j] = log_p + log_likelihood
            
            # Normalize (log-sum-exp)
            probs[i] = np.exp(probs[i] - np.max(probs[i]))
            probs[i] /= probs[i].sum()
        
        return probs
    
    def predict(self, X):
        return np.argmax(self.predict_proba(X), axis=1)

# Load data
iris = load_iris()
X_train, X_test, y_train, y_test = train_test_split(
    iris.data, iris.target, test_size=0.2, random_state=42
)

# Train
trainer = NaiveBayesTrainer()
trainer.fit(X_train, y_train)

# Predict
predictions = trainer.predict(X_test)
accuracy = (predictions == y_test).mean()
print(f"\nTest accuracy: {accuracy:.4f}")

# Show probabilities for first test sample
probs = trainer.predict_proba(X_test[:1])
print(f"\nProbabilities for first test sample: {probs[0]}")
print(f"Predicted class: {predictions[0]}, Actual: {y_test[0]}")

🧂 Smoothing (Laplace, Lidstone)

🎯 Handling Zero Probabilities

Smoothing adds a small constant to counts to avoid zero probabilities. Without it, a single unseen feature makes the entire likelihood zero, breaking classification.

Smoothing Types

Laplace (α=1): Add 1 to all counts. Lidstone (α<1): Add small α (e.g., 0.1). Formula: P(x_i | c) = (count(x_i, c) + α) / (count(c) + α × V). Effect: Non-zero probabilities for all features. Trade-off: Higher α = more smoothing = less overfitting.

🧂 Laplace🔬 Lidstone🚫 No Zeros⚖️ Trade-off

# Smoothing comparison
import numpy as np
from collections import Counter

def word_probability(word, class_docs, vocab_size, alpha=1.0):
    """
    Compute P(word | class) with Laplace/Lidstone smoothing.
    alpha=1.0 → Laplace
    alpha=0.1 → Lidstone
    alpha=0.0 → MLE (no smoothing)
    """
    word_count = sum(doc.count(word) for doc in class_docs)
    total_words = sum(len(doc) for doc in class_docs)
    
    return (word_count + alpha) / (total_words + alpha * vocab_size)

# Example: Spam classification
spam_docs = [
    ['free', 'money', 'now'],
    ['click', 'here', 'free'],
    ['win', 'money', 'free']
]
ham_docs = [
    ['meeting', 'tomorrow'],
    ['project', 'update'],
    ['lunch', 'tomorrow']
]

vocab = set()
for doc in spam_docs + ham_docs:
    vocab.update(doc)
V = len(vocab)

print(f"Vocabulary size: {V}")
print(f"Vocabulary: {vocab}")

# Test word probabilities with different smoothing
test_words = ['free', 'money', 'meeting', 'unknown_word']

print("\n=== Word probabilities ===")
print(f"{'Word':<15} {'Spam (α=1)':<12} {'Spam (α=0.1)':<14} {'Spam (α=0)':<12}")
print("-" * 55)

for word in test_words:
    p_laplace = word_probability(word, spam_docs, V, alpha=1.0)
    p_lidstone = word_probability(word, spam_docs, V, alpha=0.1)
    p_mle = word_probability(word, spam_docs, V, alpha=0.0)
    print(f"{word:<15} {p_laplace:<12.4f} {p_lidstone:<14.4f} {p_mle:<12.4f}")

# Effect of smoothing on classification
def classify_email(email, spam_docs, ham_docs, alpha=1.0):
    """Classify email with given smoothing."""
    V = len(set(w for doc in spam_docs + ham_docs for w in doc))
    
    # Priors (assuming equal priors)
    P_spam = 0.5
    P_ham = 0.5
    
    # Likelihoods with smoothing
    log_P_spam = np.log(P_spam)
    log_P_ham = np.log(P_ham)
    
    for word in email:
        log_P_spam += np.log(word_probability(word, spam_docs, V, alpha))
        log_P_ham += np.log(word_probability(word, ham_docs, V, alpha))
    
    return 'spam' if log_P_spam > log_P_ham else 'ham', log_P_spam, log_P_ham

# Test with unseen word
email = ['free', 'money', 'crypto']  # "crypto" is unseen

for alpha in [0.0, 0.1, 1.0]:
    label, log_spam, log_ham = classify_email(email, spam_docs, ham_docs, alpha)
    print(f"\nα = {alpha}: {label}")
    print(f"  log P(spam) = {log_spam:.4f}")
    print(f"  log P(ham)  = {log_ham:.4f}")

🎯 Classification & Prediction

🎯 Making Predictions

Classification with Naive Bayes uses Bayes' rule: pick the class with the highest posterior probability. In practice, we use log probabilities to avoid numerical underflow.

Prediction Process

1. Compute log priors: log P(c). 2. Compute log likelihoods: Σ log P(x_i | c). 3. Sum log posteriors: log P(c) + Σ log P(x_i | c). 4. Pick max: argmax over classes. 5. Optional: Convert to probabilities via softmax.

📊 Log Probs🎯 Argmax🛡️ Underflow📈 Softmax

# Classification with Naive Bayes
import numpy as np

class NaiveBayesClassifier:
    def __init__(self):
        self.classes = None
        self.log_priors = {}
        self.log_likelihoods = {}
    
    def fit(self, X, y):
        """Train the classifier."""
        self.classes = np.unique(y)
        n_samples = len(y)
        
        for c in self.classes:
            X_c = X[y == c]
            
            # Log prior
            self.log_priors[c] = np.log(len(X_c) / n_samples)
            
            # Log likelihood (Gaussian)
            mean = X_c.mean(axis=0)
            var = X_c.var(axis=0) + 1e-9
            self.log_likelihoods[c] = {
                'mean': mean, 'var': var
            }
    
    def predict_log_proba(self, X):
        """Return log posterior probabilities."""
        n_samples = len(X)
        n_classes = len(self.classes)
        log_probs = np.zeros((n_samples, n_classes))
        
        for i, x in enumerate(X):
            for j, c in enumerate(self.classes):
                # Log prior
                log_p = self.log_priors[c]
                
                # Log likelihood (sum of per-feature log probs)
                mean = self.log_likelihoods[c]['mean']
                var = self.log_likelihoods[c]['var']
                log_p += -0.5 * np.sum(
                    np.log(2 * np.pi * var) + ((x - mean) ** 2) / var
                )
                
                log_probs[i, j] = log_p
        
        return log_probs
    
    def predict_proba(self, X):
        """Return posterior probabilities (normalized)."""
        log_probs = self.predict_log_proba(X)
        
        # Convert to probabilities via softmax
        # Subtract max for numerical stability
        log_probs = log_probs - log_probs.max(axis=1, keepdims=True)
        probs = np.exp(log_probs)
        probs /= probs.sum(axis=1, keepdims=True)
        
        return probs
    
    def predict(self, X):
        """Return predicted classes."""
        log_probs = self.predict_log_proba(X)
        return self.classes[np.argmax(log_probs, axis=1)]

# Example: Iris classification
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split

iris = load_iris()
X_train, X_test, y_train, y_test = train_test_split(
    iris.data, iris.target, test_size=0.2, random_state=42
)

# Train
clf = NaiveBayesClassifier()
clf.fit(X_train, y_train)

# Predict
predictions = clf.predict(X_test)
probabilities = clf.predict_proba(X_test)

print(f"Accuracy: {(predictions == y_test).mean():.4f}")
print(f"\nFirst 5 predictions:")
print(f"Predicted: {predictions[:5]}")
print(f"Actual:    {y_test[:5]}")
print(f"\nProbabilities (first 5):")
for i in range(5):
    print(f"  Sample {i}: {probabilities[i].round(4)} → {clf.classes[np.argmax(probabilities[i])]}")

📧 Text Classification (Spam, Sentiment)

🎯 The Killer App of Naive Bayes

Text classification is where Naive Bayes shines — spam filtering, sentiment analysis, topic classification, language detection. It's fast, accurate, and handles high-dimensional data well.

Text Classification Pipeline

1. Tokenize: Split text into words. 2. Vectorize: CountVectorizer or TF-IDF. 3. Train: Multinomial or Bernoulli NB. 4. Predict: Classify new documents. 5. Evaluate: Accuracy, precision, recall, F1.

📧 Spam😊 Sentiment📰 Topics🌍 Language

# Spam classification with Multinomial NB
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import numpy as np

# Sample SMS spam dataset
messages = [
    # Spam
    "WINNER! You have won a FREE iPhone! Click here now!",
    "URGENT! Your account has been compromised. Verify now!",
    "Congratulations! Claim your $1000 prize today!",
    "FREE entry into our competition. Text WIN to 80082",
    "You have won a lottery of $1,000,000. Send bank details.",
    "Limited time offer! Buy 1 get 1 free. Order now!",
    "Your mobile number has won £2000. Call now to claim.",
    "Hot singles in your area want to meet you!",
    "Get cheap loans now! No credit check required!",
    "Claim your free gift card now! Click this link.",
    # Ham (not spam)
    "Hey, are we still meeting for lunch tomorrow?",
    "The project deadline has been extended to Friday.",
    "Can you send me the report when you get a chance?",
    "Happy birthday! Hope you have a great day!",
    "Don't forget to pick up milk on your way home.",
    "The meeting has been rescheduled to 3pm.",
    "Thanks for your help with the presentation.",
    "I'll be late tonight, don't wait up for dinner.",
    "Did you see the game last night? What a match!",
    "Please review the attached document before tomorrow."
]
labels = ['spam'] * 10 + ['ham'] * 10

# Split data
X_train, X_test, y_train, y_test = train_test_split(
    messages, labels, test_size=0.3, random_state=42, stratify=labels
)

# Vectorize
vectorizer = CountVectorizer(stop_words='english', lowercase=True)
X_train_vec = vectorizer.fit_transform(X_train)
X_test_vec = vectorizer.transform(X_test)

# Train
clf = MultinomialNB(alpha=1.0)
clf.fit(X_train_vec, y_train)

# Predict
y_pred = clf.predict(X_test_vec)

# Evaluate
print(f"Accuracy: {accuracy_score(y_test, y_pred):.4f}")
print("\nClassification Report:")
print(classification_report(y_test, y_pred))

# Test with new messages
new_messages = [
    "FREE iPhone! Click now to claim!",
    "Hey, are you free for coffee tomorrow?",
    "URGENT: Your account needs verification",
    "Meeting at 3pm in conference room B"
]

new_vec = vectorizer.transform(new_messages)
predictions = clf.predict(new_vec)
probabilities = clf.predict_proba(new_vec)

print("\n=== New Predictions ===")
for msg, pred, prob in zip(new_messages, predictions, probabilities):
    print(f"\nMessage: {msg}")
    print(f"Prediction: {pred}")
    print(f"Probability: spam={prob[0]:.4f}, ham={prob[1]:.4f}")

# Sentiment analysis example
sentiment_reviews = [
    ("This movie was absolutely fantastic! Great acting and story.", "positive"),
    ("Best product I've ever bought. Highly recommend!", "positive"),
    ("Amazing quality and fast shipping. Very happy!", "positive"),
    ("I love this! Works perfectly and looks great.", "positive"),
    ("Excellent service and friendly staff.", "positive"),
    ("Terrible experience. Would not recommend.", "negative"),
    ("Worst product ever. Broke after one day.", "negative"),
    ("Very disappointed. Poor quality and slow shipping.", "negative"),
    ("I hate this. Complete waste of money.", "negative"),
    ("Awful customer service. Never buying again.", "negative"),
]

texts, sentiments = zip(*sentiment_reviews)
X_train_s, X_test_s, y_train_s, y_test_s = train_test_split(
    texts, sentiments, test_size=0.3, random_state=42, stratify=sentiments
)

vec_s = TfidfVectorizer(stop_words='english', ngram_range=(1, 2))
X_train_sv = vec_s.fit_transform(X_train_s)
X_test_sv = vec_s.transform(X_test_s)

clf_s = MultinomialNB(alpha=1.0)
clf_s.fit(X_train_sv, y_train_s)

y_pred_s = clf_s.predict(X_test_sv)
print(f"\n\nSentiment Accuracy: {accuracy_score(y_test_s, y_pred_s):.4f}")

# Test sentiment
test_reviews = [
    "This is the best thing ever!",
    "I really hate this product.",
    "It's okay, nothing special."
]
test_vec = vec_s.transform(test_reviews)
test_pred = clf_s.predict(test_vec)
print("\n=== Sentiment Predictions ===")
for review, pred in zip(test_reviews, test_pred):
    print(f"'{review}' → {pred}")

⚠️ Zero Frequency Problem

🎯 When One Zero Ruins Everything

The zero frequency problem occurs when a feature value never appears in the training data for a class. Without smoothing, its probability is 0, which makes the entire likelihood 0 — regardless of other features.

The Problem

Example: If "crypto" never appears in spam training data, P(crypto | spam) = 0. Effect: Any email containing "crypto" gets P(email | spam) = 0, so it can never be classified as spam. Solution: Smoothing (Laplace/Lidstone). Trade-off: Smoothing slightly reduces accuracy on seen features but handles unseen ones.

⚠️ Zero🚫 Unseen🧂 Smoothing✅ Fix

# Zero frequency problem demonstration
import numpy as np

# Training data
spam_docs = [
    ['free', 'money', 'now'],
    ['click', 'here', 'free'],
    ['win', 'money', 'free']
]
ham_docs = [
    ['meeting', 'tomorrow'],
    ['project', 'update'],
    ['lunch', 'tomorrow']
]

def classify_without_smoothing(email, spam_docs, ham_docs):
    """Classify without smoothing — shows zero frequency problem."""
    # Compute likelihoods (MLE)
    P_spam = 0.5
    P_ham = 0.5
    
    for word in email:
        # P(word | spam)
        spam_count = sum(doc.count(word) for doc in spam_docs)
        spam_total = sum(len(doc) for doc in spam_docs)
        P_word_spam = spam_count / spam_total if spam_total > 0 else 0
        
        # P(word | ham)
        ham_count = sum(doc.count(word) for doc in ham_docs)
        ham_total = sum(len(doc) for doc in ham_docs)
        P_word_ham = ham_count / ham_total if ham_total > 0 else 0
        
        P_spam *= P_word_spam
        P_ham *= P_word_ham
    
    return 'spam' if P_spam > P_ham else 'ham', P_spam, P_ham

def classify_with_smoothing(email, spam_docs, ham_docs, alpha=1.0):
    """Classify with Laplace smoothing."""
    vocab = set(w for doc in spam_docs + ham_docs for w in doc)
    V = len(vocab)
    
    P_spam = 0.5
    P_ham = 0.5
    
    for word in email:
        # P(word | spam) with Laplace
        spam_count = sum(doc.count(word) for doc in spam_docs)
        spam_total = sum(len(doc) for doc in spam_docs)
        P_word_spam = (spam_count + alpha) / (spam_total + alpha * V)
        
        # P(word | ham) with Laplace
        ham_count = sum(doc.count(word) for doc in ham_docs)
        ham_total = sum(len(doc) for doc in ham_docs)
        P_word_ham = (ham_count + alpha) / (ham_total + alpha * V)
        
        P_spam *= P_word_spam
        P_ham *= P_word_ham
    
    return 'spam' if P_spam > P_ham else 'ham', P_spam, P_ham

# Test with unseen word
email = ['free', 'money', 'crypto']

print("=== Without Smoothing ===")
label, p_spam, p_ham = classify_without_smoothing(email, spam_docs, ham_docs)
print(f"Email: {email}")
print(f"P(spam | email) ∝ {p_spam}")
print(f"P(ham | email)  ∝ {p_ham}")
print(f"Classification: {label}")
print(f"Problem: 'crypto' is unseen → probabilities become 0")

print("\n=== With Laplace Smoothing (α=1) ===")
label, p_spam, p_ham = classify_with_smoothing(email, spam_docs, ham_docs, alpha=1.0)
print(f"Email: {email}")
print(f"P(spam | email) ∝ {p_spam:.6f}")
print(f"P(ham | email)  ∝ {p_ham:.6f}")
print(f"Classification: {label}")
print(f"Fixed: 'crypto' gets non-zero probability")

# With scikit-learn (uses smoothing by default)
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB

docs = [
    "free money now",
    "click here free",
    "win money free",
    "meeting tomorrow",
    "project update",
    "lunch tomorrow"
]
labels = ['spam', 'spam', 'spam', 'ham', 'ham', 'ham']

vec = CountVectorizer()
X = vec.fit_transform(docs)

# Test with unseen word
test_doc = ["free money crypto"]  # crypto is unseen
X_test = vec.transform(test_doc)

# Without smoothing (alpha=0)
clf_no_smooth = MultinomialNB(alpha=0.0)
clf_no_smooth.fit(X, labels)
try:
    pred = clf_no_smooth.predict(X_test)
    print(f"\nNo smoothing prediction: {pred}")
except Exception as e:
    print(f"\nNo smoothing failed: {e}")

# With smoothing (alpha=1)
clf_smooth = MultinomialNB(alpha=1.0)
clf_smooth.fit(X, labels)
pred = clf_smooth.predict(X_test)
probs = clf_smooth.predict_proba(X_test)
print(f"\nWith smoothing prediction: {pred}")
print(f"Probabilities: {probs}")

⚖️ Advantages & Limitations

🎯 When to Use Naive Bayes

Naive Bayes is a powerful baseline. It excels at text classification, handles small datasets well, and trains quickly. But its independence assumption is limiting for correlated features.

Trade-offs

Pros: Fast training/prediction, works with small data, handles high dimensions, probabilistic outputs, simple to implement. Cons: Independence assumption often violated, poor probability estimates, sensitive to feature scaling, can't learn feature interactions. Best for: Text, spam, sentiment, medical diagnosis, real-time classification.

⚡ Fast📊 Small Data🔗 Independence📧 Text

# Comparing Naive Bayes with other classifiers
from sklearn.naive_bayes import GaussianNB, MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.datasets import load_iris, fetch_20newsgroups
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import accuracy_score, f1_score
from sklearn.feature_extraction.text import TfidfVectorizer
import time

def benchmark_classifiers(X, y, classifiers, dataset_name):
    """Benchmark multiple classifiers."""
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=42
    )
    
    print(f"\n=== {dataset_name} ===")
    print(f"{'Classifier':<25} {'Train':<8} {'Predict':<8} {'Accuracy':<10} {'F1':<8}")
    print("-" * 65)
    
    results = {}
    for name, clf in classifiers.items():
        # Time training
        t0 = time.time()
        clf.fit(X_train, y_train)
        train_time = time.time() - t0
        
        # Time prediction
        t0 = time.time()
        y_pred = clf.predict(X_test)
        predict_time = time.time() - t0
        
        acc = accuracy_score(y_test, y_pred)
        f1 = f1_score(y_test, y_pred, average='weighted')
        
        results[name] = {'acc': acc, 'f1': f1, 'train': train_time}
        print(f"{name:<25} {train_time:<8.4f} {predict_time:<8.4f} {acc:<10.4f} {f1:<8.4f}")
    
    return results

# 1. Iris dataset (small, continuous)
iris = load_iris()
classifiers = {
    'Gaussian NB': GaussianNB(),
    'Logistic Regression': LogisticRegression(max_iter=1000),
    'Decision Tree': DecisionTreeClassifier(),
    'Random Forest': RandomForestClassifier(n_estimators=100),
    'SVM (RBF)': SVC(),
    'KNN (k=5)': KNeighborsClassifier(n_neighbors=5)
}
benchmark_classifiers(iris.data, iris.target, classifiers, "Iris (150 samples, 4 features)")

# 2. Text dataset (20 newsgroups subset)
categories = ['alt.atheism', 'comp.graphics', 'sci.med', 'soc.religion.christian']
newsgroups = fetch_20newsgroups(
    subset='train', categories=categories, 
    shuffle=True, random_state=42,
    remove=('headers', 'footers', 'quotes')
)

# Vectorize text
vectorizer = TfidfVectorizer(max_features=5000, stop_words='english')
X_text = vectorizer.fit_transform(newsgroups.data)
y_text = newsgroups.target

# Classifiers for text
text_classifiers = {
    'Multinomial NB': MultinomialNB(alpha=0.1),
    'Logistic Regression': LogisticRegression(max_iter=1000, C=1.0),
    'Random Forest': RandomForestClassifier(n_estimators=100),
    'SVM (Linear)': SVC(kernel='linear'),
    'KNN (k=5)': KNeighborsClassifier(n_neighbors=5)
}
benchmark_classifiers(X_text.toarray(), y_text, text_classifiers, 
                      "20 Newsgroups Text (sparse)")

# Summary: When to use what
print("\n\n=== Summary ===")
print("""
Naive Bayes is best for:
  ✓ Text classification (spam, sentiment, topics)
  ✓ High-dimensional data (many features)
  ✓ Small datasets
  ✓ Real-time predictions
  ✓ Baseline model

Naive Bayes struggles with:
  ✗ Correlated features
  ✗ Complex decision boundaries
  ✗ Continuous features with non-Gaussian distributions
  ✗ When you need accurate probability estimates
""")

🚀 Future: Bayesian Deep Learning

🎯 The Best of Both Worlds

Bayesian methods are making a comeback in deep learning: Bayesian Neural Networks, variational inference, Bayesian optimization, and uncertainty quantification. Combining Naive Bayes' probabilistic foundations with deep learning's representational power.

Bayesian Advances

Bayesian Neural Networks: Weight distributions instead of point estimates. Variational Inference: Approximate posterior for scalability. Monte Carlo Dropout: Simple Bayesian approximation. Bayesian Optimization: Hyperparameter tuning. Uncertainty: Know when the model doesn't know.

🧠 BNN🎯 Variational📊 Uncertainty🔬 MC Dropout

# Bayesian methods in modern ML
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

# 1. Monte Carlo Dropout — simple Bayesian approximation
class MCDropoutNet(nn.Module):
    """Neural network with dropout at test time for uncertainty."""
    def __init__(self, input_dim, hidden_dim, output_dim):
        super().__init__()
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        self.fc3 = nn.Linear(hidden_dim, output_dim)
        self.dropout = nn.Dropout(0.5)
    
    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = F.relu(self.fc2(x))
        x = self.dropout(x)
        return self.fc3(x)
    
    def predict_with_uncertainty(self, x, n_iter=100):
        """Make predictions with uncertainty estimates."""
        self.train()  # Keep dropout ON
        predictions = []
        with torch.no_grad():
            for _ in range(n_iter):
                predictions.append(F.softmax(self.forward(x), dim=-1))
        
        predictions = torch.stack(predictions)
        mean = predictions.mean(dim=0)
        std = predictions.std(dim=0)
        
        return mean, std

# Example
model = MCDropoutNet(input_dim=10, hidden_dim=64, output_dim=3)
x_test = torch.randn(5, 10)
mean, std = model.predict_with_uncertainty(x_test)

print("Predictions with uncertainty:")
for i in range(5):
    print(f"  Sample {i}: mean={mean[i].numpy().round(3)}, std={std[i].numpy().round(3)}")

# 2. Bayesian Linear Regression
class BayesianLinearRegression:
    """Bayesian linear regression with closed-form posterior."""
    def __init__(self, alpha=1.0, beta=1.0):
        self.alpha = alpha  # prior precision
        self.beta = beta    # noise precision
        self.mean = None
        self.cov = None
    
    def fit(self, X, y):
        """Compute posterior over weights."""
        n_features = X.shape[1]
        prior_cov = self.alpha * np.eye(n_features)
        noise_cov = (1 / self.beta) * np.eye(len(y))
        
        # Posterior covariance
        self.cov = np.linalg.inv(prior_cov + self.beta * X.T @ X)
        
        # Posterior mean
        self.mean = self.beta * self.cov @ X.T @ y
    
    def predict(self, X, return_std=False):
        """Predict with uncertainty."""
        y_mean = X @ self.mean
        
        if return_std:
            y_var = (1 / self.beta) + np.sum(X @ self.cov * X, axis=1)
            return y_mean, np.sqrt(y_var)
        
        return y_mean

# Example
np.random.seed(42)
X_train = np.random.randn(50, 3)
true_w = np.array([1.0, -2.0, 0.5])
y_train = X_train @ true_w + 0.1 * np.random.randn(50)

blr = BayesianLinearRegression(alpha=1.0, beta=100.0)
blr.fit(X_train, y_train)

X_test = np.random.randn(5, 3)
y_pred, y_std = blr.predict(X_test, return_std=True)

print("\nBayesian Linear Regression:")
for i in range(5):
    print(f"  Prediction: {y_pred[i]:.3f} ± {y_std[i]:.3f}")

# 3. Bayesian Optimization for hyperparameters
class BayesianOptimizer:
    """Simple 1D Bayesian optimization with Gaussian Process."""
    def __init__(self, bounds, n_iter=20):
        self.bounds = bounds
        self.n_iter = n_iter
        self.X_observed = []
        self.y_observed = []
    
    def _kernel(self, x1, x2, length_scale=0.3):
        """RBF kernel."""
        return np.exp(-0.5 * ((x1 - x2) / length_scale) ** 2)
    
    def _gp_predict(self, X_new):
        """Predict mean and variance using GP."""
        if len(self.X_observed) == 0:
            return np.zeros(len(X_new)), np.ones(len(X_new))
        
        X_obs = np.array(self.X_observed).reshape(-1, 1)
        y_obs = np.array(self.y_observed)
        X_new = X_new.reshape(-1, 1)
        
        # Kernel matrices
        K = np.array([[self._kernel(x1[0], x2[0]) 
                       for x2 in X_obs] for x1 in X_obs])
        K_s = np.array([[self._kernel(x1[0], x2[0]) 
                         for x2 in X_obs] for x1 in X_new])
        K_ss = np.array([[self._kernel(x1[0], x2[0]) 
                          for x2 in X_new] for x1 in X_new])
        
        # Add noise
        K += 1e-6 * np.eye(len(X_obs))
        
        # Posterior
        K_inv = np.linalg.inv(K)
        mu = K_s @ K_inv @ y_obs
        cov = K_ss - K_s @ K_inv @ K_s.T
        
        return mu, np.diag(cov)
    
    def _acquisition(self, X):
        """Upper Confidence Bound acquisition."""
        mu, sigma = self._gp_predict(X)
        kappa = 2.0
        return mu + kappa * np.sqrt(np.maximum(sigma, 1e-9))
    
    def optimize(self, objective_fn):
        """Run Bayesian optimization."""
        for i in range(self.n_iter):
            # Grid search over acquisition function
            X_candidates = np.linspace(self.bounds[0], self.bounds[1], 100)
            acq = self._acquisition(X_candidates)
            x_next = X_candidates[np.argmax(acq)]
            
            # Evaluate objective
            y_next = objective_fn(x_next)
            self.X_observed.append(x_next)
            self.y_observed.append(y_next)
        
        # Return best
        best_idx = np.argmax(self.y_observed)
        return self.X_observed[best_idx], self.y_observed[best_idx]

# Example: minimize a function
def objective(x):
    return -(np.sin(3*x) + 0.5*x + 0.5*np.cos(5*x))

optimizer = BayesianOptimizer(bounds=(-2, 2), n_iter=15)
best_x, best_y = optimizer.optimize(objective)
print(f"\nBayesian Optimization:")
print(f"  Best x: {best_x:.4f}")
print(f"  Best y: {best_y:.4f}")