<p>这里有一种方法可以像您想要的那样做,但是字符串列表作为单独的行存储在一个文件中。在</p>
<pre><code>import luigi
import requests
BATCH_SIZE = 10
class BatchProcessor(luigi.Task):
items = luigi.ListParameter()
max = luigi.IntParameter()
def requires(self):
return None
def output(self):
return luigi.LocalTarget('processed'+str(max)+'.txt')
def run(self):
for item in self.items:
req = requests.get('http://www.'+item+'.org')
# do something useful here
req.contents
open("processed"+str(max)+".txt",'w').close()
class BatchCreator(luigi.Task):
file_with_urls = luigi.Parameter()
def requires(self):
required_tasks = []
f = open(self.file_with_urls)
batch_index = 0
total_index = 0
lines = []
while True:
line = f.readline()
if not line: break
total_index += 1
if batch_index < BATCH_SIZE:
lines.append(line)
batch_index += 1
else:
required_tasks.append(BatchProcessor(batch=lines))
lines = [line]
batch_index = 1
return required_tasks
def output(self):
return luigi.LocalTarget(str(self.file_with_urls) + 'processed')
def run(self):
open(str(self.file_with_urls) + 'processed', 'w').close()
</code></pre>