第9章：开发工具与框架

本章介绍AI开发中最常用的Python工具库和深度学习框架，帮助你搭建高效的开发环境，快速实现AI项目。

9.1 Python生态工具

Python是AI开发的首选语言，拥有丰富的第三方库生态。以下是核心工具库的介绍和使用示例。

9.1.1 NumPy：数组操作基础

NumPy是Python科学计算的基础库，提供高效的多维数组对象和各种数学函数。

import numpy as np

# 创建数组
a = np.array([1, 2, 3, 4, 5])
b = np.zeros((3, 3))
c = np.ones((2, 4))
d = np.random.randn(3, 3)  # 标准正态分布

# 数组运算
print(a + 10)        # [11 12 13 14 15]
print(a * 2)         # [ 2  4  6  8 10]
print(np.sum(a))     # 15
print(np.mean(a))    # 3.0

# 矩阵运算
A = np.array([[1, 2], [3, 4]])
B = np.array([[5, 6], [7, 8]])
print(np.dot(A, B))  # 矩阵乘法
print(A.T)           # 转置

9.1.2 Pandas：数据处理与分析

Pandas提供DataFrame数据结构，是处理表格数据的利器。

import pandas as pd

# 创建DataFrame
data = {
    'name': ['Alice', 'Bob', 'Charlie'],
    'age': [25, 30, 35],
    'score': [85.5, 90.2, 78.9]
}
df = pd.DataFrame(data)

# 基本操作
print(df.head())           # 查看前几行
print(df.describe())       # 统计摘要
print(df['age'].mean())    # 计算平均值

# 数据筛选
adults = df[df['age'] >= 30]
high_score = df[df['score'] > 80]

# 读取CSV文件
df = pd.read_csv('data.csv')
df.to_csv('output.csv', index=False)

9.1.3 Matplotlib/Seaborn：数据可视化

数据可视化是理解数据和展示结果的重要手段。

import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

# Matplotlib基础绘图
x = np.linspace(0, 10, 100)
y = np.sin(x)

plt.figure(figsize=(10, 6))
plt.plot(x, y, label='sin(x)', color='blue')
plt.xlabel('X轴')
plt.ylabel('Y轴')
plt.title('正弦函数')
plt.legend()
plt.grid(True)
plt.savefig('sine_wave.png')
plt.show()

# Seaborn高级绘图
tips = sns.load_dataset('tips')
sns.scatterplot(data=tips, x='total_bill', y='tip', hue='sex')
plt.show()

# 热力图
corr_matrix = tips.corr(numeric_only=True)
sns.heatmap(corr_matrix, annot=True, cmap='coolwarm')
plt.show()

9.1.4 Scikit-learn：机器学习工具包

Scikit-learn是最流行的机器学习库，提供完整的机器学习流程工具。

from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, classification_report

# 加载数据
iris = load_iris()
X, y = iris.data, iris.target

# 数据划分
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 特征标准化
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 训练模型
model = LogisticRegression(max_iter=200)
model.fit(X_train_scaled, y_train)

# 预测与评估
y_pred = model.predict(X_test_scaled)
print(f"准确率: {accuracy_score(y_test, y_pred):.4f}")
print(classification_report(y_test, y_pred, target_names=iris.target_names))

9.2 深度学习框架对比

深度学习框架是构建神经网络的基础设施。本节对比三大主流框架的特点和适用场景。

9.2.1 框架概览

9.2.2 PyTorch详解（重点推荐）

PyTorch是当下学术界最流行的深度学习框架，以其直观的动态图机制和优秀的调试体验著称。

import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader, TensorDataset

# 定义神经网络
class NeuralNetwork(nn.Module):
    def __init__(self, input_size, hidden_size, num_classes):
        super(NeuralNetwork, self).__init__()
        self.layer1 = nn.Linear(input_size, hidden_size)
        self.relu = nn.ReLU()
        self.dropout = nn.Dropout(0.2)
        self.layer2 = nn.Linear(hidden_size, num_classes)
    
    def forward(self, x):
        x = self.layer1(x)
        x = self.relu(x)
        x = self.dropout(x)
        x = self.layer2(x)
        return x

# 超参数
input_size = 784
hidden_size = 256
num_classes = 10
batch_size = 64
learning_rate = 0.001
num_epochs = 5

# 创建模型、损失函数、优化器
model = NeuralNetwork(input_size, hidden_size, num_classes)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)

# 模拟训练（实际使用时替换为真实数据）
X_dummy = torch.randn(1000, input_size)
y_dummy = torch.randint(0, num_classes, (1000,))
dataset = TensorDataset(X_dummy, y_dummy)
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)

# 训练循环
for epoch in range(num_epochs):
    for batch_idx, (data, target) in enumerate(dataloader):
        # 前向传播
        outputs = model(data)
        loss = criterion(outputs, target)
        
        # 反向传播
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
        if batch_idx % 10 == 0:
            print(f'Epoch [{epoch+1}/{num_epochs}], '
                  f'Step [{batch_idx}/{len(dataloader)}], '
                  f'Loss: {loss.item():.4f}')

# 保存模型
torch.save(model.state_dict(), 'model.pth')

# 加载模型
model = NeuralNetwork(input_size, hidden_size, num_classes)
model.load_state_dict(torch.load('model.pth'))
model.eval()

PyTorch常用模块速查

# torch.nn - 神经网络层
nn.Linear(in_features, out_features)      # 全连接层
nn.Conv2d(in_ch, out_ch, kernel_size)     # 卷积层
nn.LSTM(input_size, hidden_size)          # LSTM层
nn.ReLU() / nn.Sigmoid() / nn.Tanh()      # 激活函数
nn.Dropout(p=0.5)                         # Dropout正则化
nn.BatchNorm2d(num_features)              # 批归一化
nn.CrossEntropyLoss() / nn.MSELoss()      # 损失函数

# torch.optim - 优化器
optim.SGD(params, lr=0.01, momentum=0.9)  # SGD
optim.Adam(params, lr=0.001)              # Adam
optim.AdamW(params, lr=0.001)             # AdamW（推荐）

# torch.utils.data - 数据处理
Dataset / DataLoader                       # 数据加载

# GPU加速
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
data = data.to(device)

9.2.3 TensorFlow/Keras

TensorFlow是工业界部署的主流选择，Keras作为其高级API大大降低了使用门槛。

import tensorflow as tf
from tensorflow import keras

# Keras Sequential API
model = keras.Sequential([
    keras.layers.Dense(256, activation='relu', input_shape=(784,)),
    keras.layers.Dropout(0.2),
    keras.layers.Dense(10, activation='softmax')
])

model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)

# 训练
# model.fit(x_train, y_train, epochs=5, batch_size=64, validation_split=0.2)

# Keras Functional API（更灵活）
inputs = keras.Input(shape=(784,))
x = keras.layers.Dense(256, activation='relu')(inputs)
x = keras.layers.Dropout(0.2)(x)
outputs = keras.layers.Dense(10, activation='softmax')(x)
model = keras.Model(inputs=inputs, outputs=outputs)

9.2.4 JAX简介

JAX是Google推出的高性能机器学习框架，结合了NumPy的易用性和XLA的编译优化能力。

import jax
import jax.numpy as jnp
from jax import grad, jit, vmap

# JAX NumPy - 类似NumPy的API
def predict(params, inputs):
    for W, b in params:
        outputs = jnp.dot(inputs, W) + b
        inputs = jnp.maximum(outputs, 0)  # ReLU
    return outputs

# 自动微分
grad_fn = grad(lambda params, x, y: jnp.sum((predict(params, x) - y) ** 2))

# JIT编译加速
fast_predict = jit(predict)

# 向量化（自动批处理）
batch_predict = vmap(predict, in_axes=(None, 0))

9.3 Hugging Face Transformers

Hugging Face的Transformers库是目前最流行的大语言模型工具库，让使用预训练模型变得极其简单。

9.3.1 Transformers库介绍

Transformers库提供了数万个预训练模型，涵盖NLP、CV、音频等多个领域。

9.3.2 Pipeline API快速使用

Pipeline是最高级的API，封装了预处理、模型推理和后处理全流程。

from transformers import pipeline

# 情感分析
classifier = pipeline("sentiment-analysis")
result = classifier("I love using Hugging Face Transformers!")
print(result)
# [{'label': 'POSITIVE', 'score': 0.9998}]

# 文本生成
generator = pipeline("text-generation", model="gpt2")
text = generator("In the future, AI will", max_length=50, num_return_sequences=1)
print(text[0]['generated_text'])

# 翻译
translator = pipeline("translation_en_to_zh", model="Helsinki-NLP/opus-mt-en-zh")
result = translator("Hello, how are you?")
print(result[0]['translation_text'])

# 问答
qa_pipeline = pipeline("question-answering")
context = "Hugging Face is a company that develops tools for building applications using machine learning."
question = "What does Hugging Face do?"
result = qa_pipeline(question=question, context=context)
print(result['answer'])

9.3.3 Model Hub模型下载与使用

Model Hub收录了超过50万个模型，可直接通过模型ID加载使用。

from transformers import AutoModel, AutoTokenizer

# 自动加载模型和分词器
model_name = "bert-base-chinese"  # 中文BERT
# model_name = "distilbert-base-uncased"  # 英文DistilBERT
# model_name = "meta-llama/Llama-2-7b"  # 需要登录

tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)

# 使用
inputs = tokenizer("你好，世界！", return_tensors="pt")
outputs = model(**inputs)
print(outputs.last_hidden_state.shape)

# 使用AutoModelForSequenceClassification进行具体任务
from transformers import AutoModelForSequenceClassification

model = AutoModelForSequenceClassification.from_pretrained(
    "distilbert-base-uncased-finetuned-sst-2-english"
)
classifier = pipeline("sentiment-analysis", model=model, tokenizer=tokenizer)

9.3.4 Datasets库使用

Datasets库提供了高效的数据集加载和处理功能。

from datasets import load_dataset, Dataset

# 加载内置数据集
dataset = load_dataset("imdb")  # 影评情感分析
print(dataset)
# DatasetDict({
#     train: Dataset({ features: ['text', 'label'], num_rows: 25000 })
#     test: Dataset({ features: ['text', 'label'], num_rows: 25000 })
# })

# 查看样本
print(dataset['train'][0])

# 数据预处理
from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

def tokenize_function(examples):
    return tokenizer(examples["text"], padding="max_length", truncation=True, max_length=512)

tokenized_datasets = dataset.map(tokenize_function, batched=True)

# 创建自己的数据集
data = {
    "text": ["This is great!", "This is terrible."],
    "label": [1, 0]
}
custom_dataset = Dataset.from_dict(data)

9.3.5 完整示例：文本分类微调

下面展示如何使用Transformers进行BERT模型的微调训练。

from transformers import (
    AutoTokenizer, 
    AutoModelForSequenceClassification,
    TrainingArguments, 
    Trainer,
    DataCollatorWithPadding
)
from datasets import load_dataset
import numpy as np
from sklearn.metrics import accuracy_score, f1_score

# 1. 加载数据和模型
dataset = load_dataset("imdb")
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
model = AutoModelForSequenceClassification.from_pretrained(
    "distilbert-base-uncased", 
    num_labels=2
)

# 2. 预处理
def preprocess(examples):
    return tokenizer(examples["text"], truncation=True, max_length=512)

tokenized = dataset.map(preprocess, batched=True)

# 3. 数据收集器
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

# 4. 评估函数
def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    predictions = np.argmax(predictions, axis=1)
    return {
        "accuracy": accuracy_score(labels, predictions),
        "f1": f1_score(labels, predictions)
    }

# 5. 训练参数
training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    weight_decay=0.01,
    save_strategy="epoch",
    load_best_model_at_end=True,
)

# 6. 创建Trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized["train"].shuffle(seed=42).select(range(10000)),
    eval_dataset=tokenized["test"].shuffle(seed=42).select(range(2000)),
    tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_metrics,
)

# 7. 训练
# trainer.train()

# 8. 保存
# trainer.save_model("./sentiment_model")

9.3.6 使用Transformers进行多任务

from transformers import pipeline

# 命名实体识别(NER)
ner = pipeline("ner", model="dslim/bert-base-NER", aggregation_strategy="simple")
text = "My name is John and I work at Google in New York."
print(ner(text))

# 摘要生成
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
article = """Hugging Face Inc. is a company based in New York City. 
Its headquarters are in DUMBO, therefore very close to the Manhattan Bridge."""
print(summarizer(article, max_length=30, min_length=10))

# 特征提取
feature_extractor = pipeline("feature-extraction")
features = feature_extractor("Transformers are amazing!")
print(len(features[0][0]))  # 向量维度

本章小结