2GB python文本文件编辑性能

Question

我在python3中运行以下代码以接收.txt文件，编辑每一行，并存储编辑后的.txt文件。它对小文件很有效，但我的文件是~2GB，而且需要太长时间。在

有人对如何修改代码以提高效率和速度有什么建议吗？在

newData = ""
i=0
run=0
j=0
k=1
seqFile = open('temp100.txt', 'r')
seqData = seqFile.readlines()
while i < 14371315:
    sLine = seqData[j] 
    editLine = seqData[k]
    tempLine = editLine[0:20]
    newLine = editLine.replace(editLine, tempLine)
    newData = newData + sLine + newLine
    if len(seqData[k]) > 20:
        newData += '\n'
i=i+1
j=j+2
k=k+2
run=run+1
print(run)

seqFile.close()

new = open("new_temp100.txt", "w")
sys.stdout = new
print(newData)

Answer 1

我建议这样做：

# if python 2.x
#from itertools import tee, izip
# if python 3
from itertols import tee
# http://docs.python.org/2/library/itertools.html#recipes
def pairwise(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = tee(iterable)
    next(b, None)
    # if python 2.x
    #return izip(a, b)
    return zip(a, b)

new_data = []
with open('temp100.txt', 'r') as sqFile:
    for sLine, edit_line  in pairwise(seqFile):
        # I think this is just new_line = tempLine
        #tempLine = edit_line[:20]
        #new_line = editLine.replace(editLine, tempLine)
        new_data.append(sLine + editLine[:20])
        if len(sLine) > 20:
            new_data.append('\n')



with open("new_temp100.txt", "w") as new:
    new.write(''.join(new_data))

如果直接流式传输到磁盘上，您可能会做得更好

^{pr2}$

所以你不必把文件的全部内容保存在内存中