【Bug已解决】Converting python list to pytorch tensor 解决方案

【Bug已解决】Converting python list to pytorch tensor 解决方案

问题描述

在 PyTorch 开发中，将 Python 列表转换为张量是最常见的操作之一。然而，开发者经常遇到数据类型不匹配、维度错误、性能问题等各种问题。

import torch

# 基本转换
python_list = [1, 2, 3, 4, 5]
tensor = torch.tensor(python_list)
print(f"Tensor: {tensor}, dtype: {tensor.dtype}")
# tensor([1, 2, 3, 4, 5]), dtype: int64

# 常见问题1：浮点数被转为整数
float_list = [1.0, 2.0, 3.0]
tensor = torch.tensor(float_list)
print(f"dtype: {tensor.dtype}") # float32

# 常见问题2：混合类型
mixed_list = [1, 2.0, 3]
tensor = torch.tensor(mixed_list)
print(f"dtype: {tensor.dtype}") # float64（自动提升）

# 常见问题3：嵌套列表
nested_list = [[1, 2], [3, 4]]
tensor = torch.tensor(nested_list)
print(f"Shape: {tensor.shape}") # [2, 2]

常见困惑：

错误复现

import torch

# 复现1：不规则列表无法转换
print("=" * 50)
print("复现1：不规则列表")
print("=" * 50)

ragged_list = [[1, 2, 3], [4, 5], [6, 7, 8, 9]]
try:
tensor = torch.tensor(ragged_list)
except ValueError as e:
print(f"Error: {e}")
# ValueError: expected sequence of length 3 but got 2

# 复现2：dtype 不匹配
print("\\n" + "=" * 50)
print("复现2：dtype 不匹配")
print("=" * 50)

int_list = [1, 2, 3]
try:
tensor = torch.tensor(int_list, dtype=torch.float32)
print(f"OK: {tensor}, dtype: {tensor.dtype}")
except TypeError as e:
print(f"Error: {e}")

# torch.Tensor 构造器的问题
tensor2 = torch.Tensor(int_list)
print(f"torch.Tensor: {tensor2}, dtype: {tensor2.dtype}") # float32（默认）
# 注意：torch.Tensor 默认 float32，而 torch.tensor 推断类型

# 复现3：性能问题
print("\\n" + "=" * 50)
print("复现3：大列表性能")
print("=" * 50)

import time

# 大列表
large_list = list(range(1000000))

start = time.time()
tensor = torch.tensor(large_list)
print(f"torch.tensor: {time.time() – start:.4f}s")

start = time.time()
tensor2 = torch.tensor(large_list, dtype=torch.int32)
print(f"torch.tensor with dtype: {time.time() – start:.4f}s")

根因分析

1. torch.tensor vs torch.Tensor

# torch.tensor：推荐使用
# – 推断数据类型
# – 无数据时创建空张量
# – 总是复制数据

# torch.Tensor：不推荐
# – 默认 float32
# – 是构造器，行为不一致
# – 可能不复制数据

list_data = [1, 2, 3]

# torch.tensor 推断类型
t1 = torch.tensor(list_data)
print(f"torch.tensor: {t1.dtype}") # int64

# torch.Tensor 默认 float32
t2 = torch.Tensor(list_data)
print(f"torch.Tensor: {t2.dtype}") # float32

# 指定 dtype
t3 = torch.tensor(list_data, dtype=torch.float32)
print(f"with dtype: {t3.dtype}") # float32

2. 数据类型推断规则

# PyTorch 的类型推断规则：
# 整数 -> int64
# 浮点数 -> float32
# 布尔 -> bool
# 混合整数和浮点 -> float64（类型提升）

print(torch.tensor([1, 2, 3]).dtype) # int64
print(torch.tensor([1.0, 2.0]).dtype) # float32
print(torch.tensor([True, False]).dtype) # bool
print(torch.tensor([1, 2.0]).dtype) # float64（提升）
print(torch.tensor([1+2j, 3+4j]).dtype) # complex64

3. 嵌套列表的维度推断

# PyTorch 根据嵌套深度推断维度
# 但要求每层长度一致

# 1D
t1 = torch.tensor([1, 2, 3])
print(f"1D: {t1.shape}") # [3]

# 2D
t2 = torch.tensor([[1, 2], [3, 4]])
print(f"2D: {t2.shape}") # [2, 2]

# 3D
t3 = torch.tensor([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
print(f"3D: {t3.shape}") # [2, 2, 2]

# 不规则列表报错
try:
torch.tensor([[1, 2], [3, 4, 5]])
except ValueError as e:
print(f"Error: {e}")

4. 内存布局与复制

# torch.tensor 总是复制数据
import numpy as np

arr = np.array([1, 2, 3])
t1 = torch.tensor(arr)
print(f"Copy: {t1.data_ptr() != arr.ctypes.data}") # True

# torch.from_numpy 共享内存
t2 = torch.from_numpy(arr)
print(f"Share: {t2.data_ptr() == arr.ctypes.data}") # True

# 修改 t2 会影响 arr
t2[0] = 99
print(f"arr after modify: {arr}") # [99, 2, 3]

解决方案

方案一：基本转换方法

import torch

# 1D 列表
list_1d = [1, 2, 3, 4, 5]
t1 = torch.tensor(list_1d)
print(f"1D: {t1.shape}, dtype: {t1.dtype}")

# 指定 dtype
t2 = torch.tensor(list_1d, dtype=torch.float32)
print(f"Float: {t2.dtype}")

# 2D 列表
list_2d = [[1, 2, 3], [4, 5, 6]]
t3 = torch.tensor(list_2d)
print(f"2D: {t3.shape}")

# 3D 列表
list_3d = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
t4 = torch.tensor(list_3d)
print(f"3D: {t4.shape}")

# 布尔列表
bool_list = [True, False, True]
t5 = torch.tensor(bool_list)
print(f"Bool: {t5.dtype}")

方案二：处理不规则列表

import torch

# 方法1：padding 到相同长度
def pad_ragged_list(ragged_list, pad_value=0):
"""将不规则列表 padding 为规则列表"""
max_len = max(len(sublist) for sublist in ragged_list)
padded = [sublist + [pad_value] * (max_len – len(sublist))
for sublist in ragged_list]
return torch.tensor(padded)

ragged = [[1, 2, 3], [4, 5], [6, 7, 8, 9]]
padded_tensor = pad_ragged_list(ragged, pad_value=0)
print(f"Padded:\\n{padded_tensor}")
# tensor([[1, 2, 3, 0],
# [4, 5, 0, 0],
# [6, 7, 8, 9]])

# 方法2：使用 mask
def ragged_to_tensor_with_mask(ragged_list, pad_value=0):
"""转换不规则列表并返回 mask"""
max_len = max(len(sublist) for sublist in ragged_list)
padded = []
masks = []
for sublist in ragged_list:
pad_len = max_len – len(sublist)
padded.append(sublist + [pad_value] * pad_len)
masks.append([1] * len(sublist) + [0] * pad_len)

tensor = torch.tensor(padded)
mask = torch.tensor(masks, dtype=torch.bool)
return tensor, mask

tensor, mask = ragged_to_tensor_with_mask(ragged)
print(f"Tensor:\\n{tensor}")
print(f"Mask:\\n{mask}")

# 方法3：分别存储为 1D 张量
flat_tensor = torch.tensor([item for sublist in ragged for item in sublist])
lengths = torch.tensor([len(sublist) for sublist in ragged])
print(f"Flat: {flat_tensor}")
print(f"Lengths: {lengths}")

方案三：高效转换大列表

import torch
import numpy as np
import time

def efficient_list_to_tensor(python_list, dtype=None):
"""高效转换大列表"""
# 方法1：通过 numpy 中转（最快）
arr = np.array(python_list, dtype=dtype) if dtype else np.array(python_list)
return torch.from_numpy(arr)

# 性能对比
large_list = list(range(1000000))

# torch.tensor
start = time.time()
t1 = torch.tensor(large_list)
print(f"torch.tensor: {time.time() – start:.4f}s")

# 通过 numpy
start = time.time()
t2 = efficient_list_to_tensor(large_list, dtype=np.int64)
print(f"via numpy: {time.time() – start:.4f}s")

# 验证结果一致
print(f"Equal: {torch.equal(t1, t2)}")

方案四：处理各种数据类型

import torch

# 整数列表
int_list = [1, 2, 3]
t_int = torch.tensor(int_list, dtype=torch.long)
print(f"Long: {t_int.dtype}")

# 浮点列表
float_list = [1.0, 2.0, 3.0]
t_float = torch.tensor(float_list, dtype=torch.float32)
print(f"Float: {t_float.dtype}")

# 字符串列表（需要编码）
str_list = ["hello", "world"]
# 不能直接转为张量，需要先编码
# 方法1：转为字节再转为张量
encoded = [s.encode('utf-8') for s in str_list]
print(f"Encoded: {encoded}")

# 方法2：使用词汇表映射
vocab = {"hello": 1, "world": 2}
indices = [vocab[s] for s in str_list]
t_str = torch.tensor(indices)
print(f"Indices: {t_str}")

# 混合类型处理
mixed = [1, 2.5, 3, 4.0]
# 自动提升为 float64
t_mixed = torch.tensor(mixed)
print(f"Mixed dtype: {t_mixed.dtype}") # float64

# 强制 float32
t_mixed_f32 = torch.tensor(mixed, dtype=torch.float32)
print(f"Forced float32: {t_mixed_f32}")

完整修复代码

"""
完整代码：Python 列表转 PyTorch 张量的各种场景
"""

import torch
import numpy as np
from typing import List, Any, Optional

class ListToTensorConverter:
"""列表转张量工具类"""

@staticmethod
def convert(list_data: List, dtype: Optional[torch.dtype] = None) -> torch.Tensor:
"""基本转换"""
return torch.tensor(list_data, dtype=dtype)

@staticmethod
def convert_ragged(ragged_list: List[List], pad_value: int = 0):
"""转换不规则列表，返回张量和 mask"""
max_len = max(len(sublist) for sublist in ragged_list)
padded = []
masks = []
for sublist in ragged_list:
pad_len = max_len – len(sublist)
padded.append(sublist + [pad_value] * pad_len)
masks.append([1] * len(sublist) + [0] * pad_len)

tensor = torch.tensor(padded)
mask = torch.tensor(masks, dtype=torch.bool)
return tensor, mask

@staticmethod
def convert_efficient(list_data: List, dtype: Optional[torch.dtype] = None) -> torch.Tensor:
"""高效转换（通过 numpy）"""
np_dtype = None
if dtype == torch.float32:
np_dtype = np.float32
elif dtype == torch.float64:
np_dtype = np.float64
elif dtype == torch.int32:
np_dtype = np.int32
elif dtype == torch.int64:
np_dtype = np.int64

arr = np.array(list_data, dtype=np_dtype)
tensor = torch.from_numpy(arr)
if dtype is not None and tensor.dtype != dtype:
tensor = tensor.to(dtype)
return tensor

@staticmethod
def convert_with_vocab(str_list: List[str], vocab: dict) -> torch.Tensor:
"""通过词汇表转换字符串列表"""
unk_idx = len(vocab)
indices = [vocab.get(s, unk_idx) for s in str_list]
return torch.tensor(indices, dtype=torch.long)

def test_conversions():
"""测试各种转换场景"""
print("=" * 60)
print("Testing List to Tensor Conversions")
print("=" * 60)

converter = ListToTensorConverter()

# 测试1：基本转换
print("\\n— Test 1: Basic Conversion —")
t = converter.convert([1, 2, 3, 4, 5])
print(f"1D: {t.shape}, dtype: {t.dtype}")

t = converter.convert([[1, 2], [3, 4]])
print(f"2D: {t.shape}")

# 测试2：指定 dtype
print("\\n— Test 2: Specify dtype —")
t = converter.convert([1, 2, 3], dtype=torch.float32)
print(f"Float32: {t.dtype}")

t = converter.convert([1.0, 2.0], dtype=torch.int32)
print(f"Int32: {t.dtype}, values: {t}")

# 测试3：不规则列表
print("\\n— Test 3: Ragged List —")
ragged = [[1, 2, 3], [4, 5], [6, 7, 8, 9]]
tensor, mask = converter.convert_ragged(ragged)
print(f"Tensor:\\n{tensor}")
print(f"Mask:\\n{mask}")

# 测试4：高效转换
print("\\n— Test 4: Efficient Conversion —")
import time
large_list = list(range(100000))

start = time.time()
t1 = converter.convert(large_list)
t1_time = time.time() – start

start = time.time()
t2 = converter.convert_efficient(large_list, dtype=torch.int64)
t2_time = time.time() – start

print(f"Standard: {t1_time:.4f}s")
print(f"Efficient: {t2_time:.4f}s")
print(f"Equal: {torch.equal(t1, t2)}")

# 测试5：字符串列表
print("\\n— Test 5: String List —")
vocab = {"hello": 1, "world": 2, "foo": 3}
t = converter.convert_with_vocab(["hello", "world", "unknown"], vocab)
print(f"Indices: {t}")

print("\\n" + "=" * 60)
print("All tests passed!")
print("=" * 60)

if __name__ == "__main__":
test_conversions()

常见陷阱与注意事项

1. torch.tensor vs torch.Tensor

# 推荐：torch.tensor（小写）
t = torch.tensor([1, 2, 3]) # int64

# 不推荐：torch.Tensor（大写）
t = torch.Tensor([1, 2, 3]) # float32（默认）

# 区别：
# torch.tensor 推断类型，总是复制
# torch.Tensor 默认 float32，行为不一致

2. dtype 自动提升

# 混合类型会自动提升
t = torch.tensor([1, 2.0]) # float64
# 整数和浮点混合 -> float64

# 显式指定避免意外
t = torch.tensor([1, 2.0], dtype=torch.float32) # float32

3. 内存共享

import numpy as np

# torch.from_numpy 共享内存
arr = np.array([1, 2, 3])
t = torch.from_numpy(arr)
t[0] = 99
print(arr) # [99, 2, 3] # 也被修改了

# torch.tensor 不共享
t2 = torch.tensor(arr)
t2[0] = 0
print(arr) # [99, 2, 3] # 不受影响

4. GPU 转换

# 先转 CPU 张量再移到 GPU
list_data = [1, 2, 3]
t = torch.tensor(list_data, device='cuda') # 直接在 GPU 创建

# 或先 CPU 再移动
t = torch.tensor(list_data).cuda()

5. 空列表

# 空列表
t = torch.tensor([])
print(f"Empty: {t.shape}, dtype: {t.dtype}") # [], float32

# 指定 dtype
t = torch.tensor([], dtype=torch.int64)
print(f"Empty int: {t.dtype}")

总结

将 Python 列表转换为 PyTorch 张量时的关键要点：

最佳实践：

• 始终使用 torch.tensor（小写）
• 明确指定 dtype 避免类型推断意外
• 大列表使用 numpy 中转提升性能
• 不规则列表先 padding 再转换
• 注意 from_numpy 的内存共享特性
• GPU 张量直接指定 device='cuda'

通过掌握这些转换技巧，可以在实际项目中高效、正确地将 Python 数据转换为 PyTorch 张量。