Python 生成器表达式优化：内存效率对比分析

Python 生成器表达式优化：内存效率对比分析

1. 技术分析

1.1 生成器表达式定义

生成器表达式是一种创建迭代器的简洁方式，使用圆括号：

gen = (x ** 2 for x in range(10))

1.2 生成器 vs 列表对比

1.3 生成器表达式语法

(expression for item in iterable if condition)

2. 核心功能实现

2.1 基础生成器表达式

def fibonacci(n):
a, b = 0, 1
for _ in range(n):
yield a
a, b = b, a + b

def prime_generator(limit):
for num in range(2, limit):
if all(num % i != 0 for i in range(2, int(num ** 0.5) + 1)):
yield num

def chunk_generator(data, chunk_size):
for i in range(0, len(data), chunk_size):
yield data[i:i + chunk_size]

def flatten(nested):
for item in nested:
if isinstance(item, (list, tuple)):
yield from flatten(item)
else:
yield item

2.2 生成器表达式优化技巧

import itertools

def optimized_filter(data, condition):
return (x for x in data if condition(x))

def optimized_map(data, func):
return (func(x) for x in data)

def optimized_chain(*iterables):
for iterable in iterables:
yield from iterable

def optimized_groupby(data, keyfunc):
groups = {}
for item in data:
key = keyfunc(item)
if key not in groups:
groups[key] = []
groups[key].append(item)

for key, items in groups.items():
yield key, items

def lazy_range(start, stop=None, step=1):
if stop is None:
start, stop = 0, start

current = start
while current < stop:
yield current
current += step

def take(n, iterable):
for i, item in enumerate(iterable):
if i >= n:
break
yield item

def drop(n, iterable):
iterator = iter(iterable)
for _ in range(n):
next(iterator, None)
yield from iterator

2.3 生成器组合

def pipeline(data):
step1 = (x.strip() for x in data)
step2 = (x for x in step1 if x)
step3 = (x.lower() for x in step2)
return step3

def compose_generators(*generators):
def composed(data):
result = data
for gen_func in generators:
result = gen_func(result)
return result
return composed

def process_large_file(filepath):
with open(filepath, 'r') as f:
lines = (line.strip() for line in f)
filtered = (line for line in lines if line and not line.startswith('#'))
parsed = (json.loads(line) for line in filtered)
yield from parsed

class DataPipeline:
def __init__(self):
self._steps = []

def add_step(self, step_func):
self._steps.append(step_func)
return self

def process(self, data):
result = iter(data)
for step in self._steps:
result = step(result)
yield from result

2.4 生成器与协程

def consumer():
while True:
item = yield
print(f"Consumed: {item}")

def producer(consumer, items):
for item in items:
consumer.send(item)
consumer.close()

def pipeline_with_coroutines():
def stage1(data):
for item in data:
yield item * 2

def stage2(data):
for item in data:
if item % 3 == 0:
yield item

def stage3(data):
for item in data:
yield f"Result: {item}"

return stage3(stage2(stage1(range(10))))

3. 性能对比

3.1 内存占用对比

3.2 执行时间对比

3.3 生成器类型性能

4. 最佳实践

4.1 生成器设计模式

class LazyDataLoader:
def __init__(self, filepath, batch_size=1000):
self.filepath = filepath
self.batch_size = batch_size

def __iter__(self):
with open(self.filepath, 'r') as f:
batch = []
for line in f:
batch.append(json.loads(line))
if len(batch) >= self.batch_size:
yield batch
batch = []
if batch:
yield batch

class InfiniteGenerator:
def __init__(self, generator_func):
self.generator_func = generator_func
self.generator = None

def __iter__(self):
while True:
if self.generator is None:
self.generator = self.generator_func()

try:
yield next(self.generator)
except StopIteration:
self.generator = None

def sliding_window(iterable, window_size):
iterator = iter(iterable)
window = []

for _ in range(window_size):
try:
window.append(next(iterator))
except StopIteration:
if window:
yield tuple(window)
return

yield tuple(window)

for item in iterator:
window = window[1:] + [item]
yield tuple(window)

4.2 生成器性能优化

def optimized_generator(data):
data_iter = iter(data)

try:
while True:
item = next(data_iter)
if item % 2 == 0:
yield item * 2
except StopIteration:
pass

def vectorized_generator(data):
import numpy as np

arr = np.array(list(data))
mask = arr % 2 == 0
result = arr[mask] * 2

for item in result:
yield item

5. 总结

生成器表达式是 Python 中高效处理大数据的工具：

对比数据如下：

• 生成器内存占用比列表低 99%
• 创建时间比列表快 93%
• 单次遍历比列表慢 67%
• 多次遍历列表更优