机器学习基础系列——数学基础：微积分

机器学习基础系列

• 学习资源
• 核心概念
• 线性代数基础
• 概率统计基础
• 微积分基础
• 损失函数与优化
• 线性回归
• 多项式回归与正则化
• 逻辑回归
• 决策树
• 随机森林
• 支持向量机
• K近邻算法
• 朴素贝叶斯
• 聚类算法
• 降维技术
• 模型评估指标
• 交叉验证
• 过拟合与欠拟合
• 超参数调优
• 神经网络基础
• 前馈神经网络
• 激活函数
• 反向传播
• 优化算法详解
• 批归一化
• Dropout详解
• 卷积神经网络
• 循环神经网络
• 注意力机制
• Transformer
• 迁移学习
• 实战项目
• 集成学习
• XGBoost与LightGBM
• 自编码器
• 生成对抗网络
• 图神经网络
• 混合专家模型
• 强化学习基础
• 自监督学习
• 知识蒸馏
• LoRA与参数高效微调
• 模型量化与剪枝

import numpy as np

def numerical_derivative(f, x, h=1e-5):
    """数值求导（中心差分法）"""
    return (f(x + h) - f(x - h)) / (2 * h)

# 示例：f(x) = x^2
f = lambda x: x ** 2
x = 3

derivative = numerical_derivative(f, x)
print(f"f'({x}) ≈ {derivative:.4f}")  # 约为 6

def gradient(f, x, h=1e-5):
    """计算梯度"""
    grad = np.zeros_like(x)
    for i in range(len(x)):
        x_plus = x.copy()
        x_minus = x.copy()
        x_plus[i] += h
        x_minus[i] -= h
        grad[i] = (f(x_plus) - f(x_minus)) / (2 * h)
    return grad

# 示例：f(x, y) = x^2 + y^2
f = lambda x: x[0]**2 + x[1]**2
point = np.array([3.0, 4.0])

grad = gradient(f, point)
print(f"梯度: {grad}")  # [6, 8]

import torch

# 定义变量
x = torch.tensor([3.0, 4.0], requires_grad=True)

# 前向计算
y = x[0]**2 + x[1]**2

# 反向传播
y.backward()

print(f"梯度: {x.grad}")  # tensor([6., 8.])

# f(x) = (x^2 + 1)^3
# 令 u = x^2 + 1, f = u^3
# df/dx = df/du * du/dx = 3u^2 * 2x = 6x(x^2 + 1)^2

def f(x):
    return (x**2 + 1)**3

def df_analytical(x):
    """解析导数"""
    return 6 * x * (x**2 + 1)**2

x = 2.0
print(f"数值导数: {numerical_derivative(f, x):.4f}")
print(f"解析导数: {df_analytical(x):.4f}")  # 150

# 前向传播
w, x, b, y = 2.0, 3.0, 1.0, 10.0

# 计算图节点
z1 = w * x      # z1 = 6
z2 = z1 + b     # z2 = 7
z3 = z2 - y     # z3 = -3
L = z3 ** 2     # L = 9

print(f"损失: {L}")

# 反向传播（链式法则）
dL_dz3 = 2 * z3           # dL/dz3 = -6
dz3_dz2 = 1               # dz3/dz2 = 1
dz2_dz1 = 1               # dz2/dz1 = 1
dz1_dw = x                # dz1/dw = 3

# 完整链式
dL_dw = dL_dz3 * dz3_dz2 * dz2_dz1 * dz1_dw
print(f"dL/dw = {dL_dw}")  # -18

# 验证
w_tensor = torch.tensor(2.0, requires_grad=True)
L_torch = (w_tensor * x + b - y) ** 2
L_torch.backward()
print(f"PyTorch梯度: {w_tensor.grad}")  # -18

import torch

# 批量数据
X = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], requires_grad=False)
W = torch.tensor([[0.1, 0.2], [0.3, 0.4]], requires_grad=True)
y = torch.tensor([[1.0], [2.0], [3.0]])

# 前向传播
pred = X @ W @ torch.ones(2, 1)
loss = ((pred - y) ** 2).mean()

# 反向传播
loss.backward()
print(f"W的梯度:\n{W.grad}")

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_derivative(x):
    s = sigmoid(x)
    return s * (1 - s)

x = np.linspace(-5, 5, 100)
plt.plot(x, sigmoid(x), label='sigmoid')
plt.plot(x, sigmoid_derivative(x), label='derivative')
plt.legend()
plt.title('Sigmoid及其导数')
plt.show()

def relu(x):
    return np.maximum(0, x)

def relu_derivative(x):
    return (x > 0).astype(float)

def softmax(x):
    exp_x = np.exp(x - np.max(x))  # 数值稳定
    return exp_x / exp_x.sum()

def softmax_jacobian(x):
    s = softmax(x)
    return np.diag(s) - np.outer(s, s)

x = np.array([1.0, 2.0, 3.0])
print(f"Softmax: {softmax(x)}")
print(f"Jacobian:\n{softmax_jacobian(x)}")

def gradient_descent(f, grad_f, x0, lr=0.1, n_iters=100):
    """基本梯度下降"""
    x = x0.copy()
    history = [x.copy()]
    
    for _ in range(n_iters):
        g = grad_f(x)
        x = x - lr * g
        history.append(x.copy())
    
    return x, np.array(history)

# 最小化 f(x,y) = x^2 + y^2
f = lambda x: x[0]**2 + x[1]**2
grad_f = lambda x: np.array([2*x[0], 2*x[1]])

x0 = np.array([5.0, 5.0])
x_min, history = gradient_descent(f, grad_f, x0, lr=0.1, n_iters=50)
print(f"最优解: {x_min}")  # 接近 [0, 0]

learning_rates = [0.01, 0.1, 0.5, 0.9]
x0 = np.array([5.0, 5.0])

for lr in learning_rates:
    x_min, history = gradient_descent(f, grad_f, x0, lr=lr, n_iters=20)
    print(f"lr={lr}: 最终值={f(x_min):.6f}")

def hessian(f, x, h=1e-5):
    """数值计算海森矩阵"""
    n = len(x)
    H = np.zeros((n, n))
    
    for i in range(n):
        for j in range(n):
            x_pp = x.copy(); x_pp[i] += h; x_pp[j] += h
            x_pm = x.copy(); x_pm[i] += h; x_pm[j] -= h
            x_mp = x.copy(); x_mp[i] -= h; x_mp[j] += h
            x_mm = x.copy(); x_mm[i] -= h; x_mm[j] -= h
            
            H[i, j] = (f(x_pp) - f(x_pm) - f(x_mp) + f(x_mm)) / (4 * h * h)
    
    return H

# 示例
f = lambda x: x[0]**2 + 2*x[1]**2 + x[0]*x[1]
x = np.array([1.0, 1.0])
H = hessian(f, x)
print(f"海森矩阵:\n{H}")

def newton_method(f, grad_f, hess_f, x0, n_iters=10):
    x = x0.copy()
    for _ in range(n_iters):
        g = grad_f(x)
        H = hess_f(x)
        x = x - np.linalg.solve(H, g)
    return x