Python 数据科学高级应用指南

Python 数据科学高级应用指南

1. 数据科学基础

Python 的数据科学主要通过 NumPy、Pandas 和 scikit-learn 等库实现。

import numpy as np
import pandas as pd
from sklearn.datasets import load_iris

# 加载数据集
iris = load_iris()
data = pd.DataFrame(data=iris.data, columns=iris.feature_names)
data['target'] = iris.target

# 查看数据
t print(data.head())
print(data.describe())

2. NumPy 高级应用

2.1 数组操作

import numpy as np

# 创建数组
arr = np.array([1, 2, 3, 4, 5])
print(f"Original array: {arr}")

# 数组运算
print(f"Array + 1: {arr + 1}")
print(f"Array * 2: {arr * 2}")
print(f"Array sum: {arr.sum()}")
print(f"Array mean: {arr.mean()}")
print(f"Array max: {arr.max()}")
print(f"Array min: {arr.min()}")

# 多维数组
arr_2d = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
print(f"2D array:\\n{arr_2d}")
print(f"2D array shape: {arr_2d.shape}")
print(f"2D array sum: {arr_2d.sum()}")
print(f"2D array sum axis 0: {arr_2d.sum(axis=0)}")
print(f"2D array sum axis 1: {arr_2d.sum(axis=1)}")

2.2 矩阵运算

import numpy as np

# 创建矩阵
A = np.array([[1, 2], [3, 4]])
B = np.array([[5, 6], [7, 8]])

print(f"Matrix A:\\n{A}")
print(f"Matrix B:\\n{B}")

# 矩阵加法
print(f"A + B:\\n{A + B}")

# 矩阵乘法
print(f"A @ B:\\n{A @ B}")

# 矩阵转置
print(f"A transpose:\\n{A.T}")

# 矩阵逆
print(f"A inverse:\\n{np.linalg.inv(A)}")

# 矩阵行列式
print(f"A determinant: {np.linalg.det(A)}")

2.3 广播

import numpy as np

# 广播
arr = np.array([1, 2, 3])
scalar = 2
print(f"Array: {arr}")
print(f"Scalar: {scalar}")
print(f"Array * scalar: {arr * scalar}")

# 广播规则
arr1 = np.array([[1, 2, 3], [4, 5, 6]])
arr2 = np.array([10, 20, 30])
print(f"Array 1:\\n{arr1}")
print(f"Array 2: {arr2}")
print(f"Array 1 + Array 2:\\n{arr1 + arr2}")

3. Pandas 高级应用

3.1 数据清洗

import pandas as pd
import numpy as np

# 创建数据
data = pd.DataFrame({
'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],
'age': [25, 30, np.nan, 40, 45],
'salary': [50000, 60000, 70000, np.nan, 90000],
'department': ['IT', 'HR', 'IT', 'Finance', 'HR']
})

print(f"Original data:\\n{data}")

# 处理缺失值
print(f"Missing values:\\n{data.isnull().sum()}")
data['age'].fillna(data['age'].mean(), inplace=True)
data['salary'].fillna(data['salary'].median(), inplace=True)
print(f"Data after filling missing values:\\n{data}")

# 数据转换
data['salary'] = data['salary'].astype(int)
print(f"Data after type conversion:\\n{data}")

# 数据过滤
it_department = data[data['department'] == 'IT']
print(f"IT department employees:\\n{it_department}")

# 数据排序
data_sorted = data.sort_values('salary', ascending=False)
print(f"Data sorted by salary:\\n{data_sorted}")

3.2 数据聚合

import pandas as pd
import numpy as np

# 创建数据
data = pd.DataFrame({
'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve', 'Frank'],
'age': [25, 30, 35, 40, 45, 50],
'salary': [50000, 60000, 70000, 80000, 90000, 100000],
'department': ['IT', 'HR', 'IT', 'Finance', 'HR', 'IT']
})

# 按部门聚合
department_stats = data.groupby('department').agg({
'age': ['mean', 'min', 'max'],
'salary': ['mean', 'sum', 'count']
})
print(f"Department stats:\\n{department_stats}")

# 透视表
pivot_table = pd.pivot_table(data, values='salary', index='department', aggfunc=np.mean)
print(f"Pivot table:\\n{pivot_table}")

3.3 时间序列分析

import pandas as pd
import numpy as np

# 创建时间序列数据
dates = pd.date_range('2023-01-01', '2023-12-31', freq='D')
data = pd.DataFrame({
'date': dates,
'value': np.random.randn(365) * 100 + 1000
})

print(f"Original data:\\n{data.head()}")

# 设置日期索引
data.set_index('date', inplace=True)
print(f"Data with date index:\\n{data.head()}")

# 时间序列操作
print(f"Monthly mean:\\n{data.resample('M').mean()}")
print(f"Quarterly sum:\\n{data.resample('Q').sum()}")

# 移动平均
data['MA7'] = data['value'].rolling(window=7).mean()
data['MA30'] = data['value'].rolling(window=30).mean()
print(f"Data with moving averages:\\n{data.head(10)}")

4. scikit-learn 高级应用

4.1 分类

from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report

# 加载数据集
iris = load_iris()
X = iris.data
y = iris.target

# 数据分割
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 数据标准化
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 模型训练
model = LogisticRegression()
model.fit(X_train_scaled, y_train)

# 模型预测
y_pred = model.predict(X_test_scaled)

# 模型评估
print(f"Accuracy: {accuracy_score(y_test, y_pred)}")
print(f"Confusion Matrix:\\n{confusion_matrix(y_test, y_pred)}")
print(f"Classification Report:\\n{classification_report(y_test, y_pred)}")

4.2 回归

from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score

# 加载数据集
boston = load_boston()
X = boston.data
y = boston.target

# 数据分割
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 数据标准化
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 模型训练
model = LinearRegression()
model.fit(X_train_scaled, y_train)

# 模型预测
y_pred = model.predict(X_test_scaled)

# 模型评估
print(f"Mean Squared Error: {mean_squared_error(y_test, y_pred)}")
print(f"R2 Score: {r2_score(y_test, y_pred)}")

4.3 聚类

from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt

# 生成数据
X, y = make_blobs(n_samples=300, centers=4, random_state=42)

# 可视化数据
plt.scatter(X[:, 0], X[:, 1], c=y)
plt.title("Original Data")
plt.show()

# 模型训练
kmeans = KMeans(n_clusters=4, random_state=42)
y_pred = kmeans.fit_predict(X)

# 可视化聚类结果
plt.scatter(X[:, 0], X[:, 1], c=y_pred)
plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1], s=100, c='red')
plt.title("K-Means Clustering")
plt.show()

5. 实际应用场景

5.1 房价预测

import pandas as pd
import numpy as np
from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error, r2_score

# 加载数据集
boston = load_boston()
data = pd.DataFrame(data=boston.data, columns=boston.feature_names)
data['PRICE'] = boston.target

# 特征和目标变量
X = data.drop('PRICE', axis=1)
y = data['PRICE']

# 数据分割
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 数据标准化
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 模型训练和调参
param_grid = {
'n_estimators': [100, 200, 300],
'max_depth': [3, 5, 7],
'min_samples_split': [2, 4, 6]
}

grid_search = GridSearchCV(RandomForestRegressor(), param_grid, cv=5, scoring='r2')
grid_search.fit(X_train_scaled, y_train)

# 最佳模型
best_model = grid_search.best_estimator_
print(f"Best parameters: {grid_search.best_params_}")

# 模型预测
y_pred = best_model.predict(X_test_scaled)

# 模型评估
print(f"Mean Squared Error: {mean_squared_error(y_test, y_pred)}")
print(f"R2 Score: {r2_score(y_test, y_pred)}")

# 特征重要性
feature_importance = pd.DataFrame({
'feature': boston.feature_names,
'importance': best_model.feature_importances_
})
feature_importance.sort_values('importance', ascending=False, inplace=True)
print(f"Feature Importance:\\n{feature_importance}")

5.2 客户分类

import pandas as pd
import numpy as np
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report

# 生成数据
X, y = make_classification(n_samples=1000, n_features=20, n_classes=2, random_state=42)

# 数据分割
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 数据标准化
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 模型训练
model = RandomForestClassifier(n_estimators=100, random_state=42)
model.fit(X_train_scaled, y_train)

# 模型预测
y_pred = model.predict(X_test_scaled)

# 模型评估
print(f"Accuracy: {accuracy_score(y_test, y_pred)}")
print(f"Confusion Matrix:\\n{confusion_matrix(y_test, y_pred)}")
print(f"Classification Report:\\n{classification_report(y_test, y_pred)}")

5.3 异常检测

import pandas as pd
import numpy as np
from sklearn.datasets import make_blobs
from sklearn.ensemble import IsolationForest
import matplotlib.pyplot as plt

# 生成数据
X, _ = make_blobs(n_samples=1000, centers=1, n_features=2, random_state=42)

# 添加异常值
outliers = np.random.uniform(low=-10, high=10, size=(20, 2))
X = np.vstack([X, outliers])

# 模型训练
model = IsolationForest(contamination=0.02, random_state=42)
y_pred = model.fit_predict(X)

# 可视化结果
plt.scatter(X[:, 0], X[:, 1], c=y_pred)
plt.title("Isolation Forest Anomaly Detection")
plt.show()

6. 最佳实践

7. 总结

Python 的数据科学库提供了丰富的功能，从数据处理和分析到机器学习建模。通过掌握这些高级应用，我们可以更有效地解决实际问题。

在实际应用中，数据科学可以用于房价预测、客户分类、异常检测等多种场景，帮助我们做出更明智的决策。

希望本文对你理解和应用 Python 数据科学有所帮助！