使用Pysftp和多线程将下载文件的结果插入DB表

Question

仍与以下问题有关。。。 Parallel downloads with Multiprocessing and PySftp

我想知道如何打印下载的文件？我的意图是在数据库表中添加一条记录，以便创建一个包含文件名、日期和时间的下载文件日志。你知道吗

有什么想法吗？我已经搜索了一些示例并进行了一些测试，但是我的下载模块似乎无法返回任何内容，或者我没有使用正确的代码来读取结果并打印它。你知道吗

下载功能

import pysftp
import os

def fdownload(vfileaux):

    vtmpspl = vfileaux.split(',')

    vfile = vtmpspl[0]
    vhost = vtmpspl[1]
    vlogin = vtmpspl[2]
    vpwd = vtmpspl[3]
    vftppath = vtmpspl[4]
    vlocalpath = vtmpspl[5]

    os.chdir(vlocalpath)
    os.getcwd()

    cnopts = pysftp.CnOpts()
    cnopts.hostkeys = None

    vfilecheck = vlocalpath + '/' + vfile

    if not os.path.isfile(vfilecheck):

        vftpaux = pysftp.Connection(host=vhost, username=vlogin, password=vpwd, cnopts=cnopts)
        vftpaux.cwd(vftppath)
        vftpaux.get(vfile, preserve_mtime=True)
        vftpaux.close()

        return vnename + "_" + vdatetime

    else:
        pass

主要功能

from datetime import *
from ffilelist import *
from ffilefilter import *
from developing.fdownload import *
import pymysql.cursors
from concurrent.futures import ThreadPoolExecutor, wait, as_completed

def main():

    print(datetime.datetime.now(), 'Loading variables...')

    vhostlist = {}
    vloginlist = {}
    vpwdlist = {}
    vftppathlist = {}
    vlocalpathlist = {}

    vhostaux = '10.11.12.13'
    vhostlist[vhostaux] = vhostaux
    vloginlist[vhostaux] = 'admin'
    vpwdlist[vhostaux] = 'pass1234'
    vftppathlist[vhostaux] = '/export/home'
    vlocalpathlist[vhostaux] = 'd:/test/'

    vfilelist1 = []

    global vfilelist2
    vfilelist2 = []

    for vhosttmp in vhostlist:

        print(datetime.datetime.now(), 'Starting to process ' + vhosttmp + "...")

        global vhost
        global vlogin
        global vpwd
        global vftppath
        global vlocalpath

        vhost = vhostlist[vhosttmp]
        vlogin = vloginlist[vhosttmp]
        vpwd = vpwdlist[vhosttmp]
        vftppath = vftppathlist[vhosttmp]
        vlocalpath = vlocalpathlist[vhosttmp]

        vfilelist1 = ffilelist(vhost, vlogin, vpwd, vftppath)

        print(datetime.datetime.now(), 'Vectorizing download file     list...')

        for vfile in vfilelist1:
            vfilelist2.append(vfile + ',' + vhost + ',' + vlogin + ',' +     vpwd + ',' + vftppath + ',' + vlocalpath)

    vfilelist0 = ffilefilter(vfilelist2)

    print(datetime.datetime.now(), 'Starting simultaneous downloads...')

    vpool = concurrent.futures.ProcessPoolExecutor(max_workers=8)
    vpool.map(fdownload, vfilelist0)
    vpool.shutdown()

    print(datetime.datetime.now(), 'Downloads finished!')

要存储在MARIADB中的日志的插入字符串是这样的。已经测试和工作。用在主函数中，我一找到解决方案就可以得到下载文件的列表。你知道吗

vconn = pymysql.connect(host='localhost', user='root', password='pass1234', db='test')
vcurs = vconn.cursor()
vsql = "INSERT INTO `logs_download` (`ne`, `datetime`) VALUES (\'" + vnename + "\', \'" + vdatetime + "\')"
vcurs.execute(vsql)
vconn.commit()

Answer 1

我试过亚历克斯的建议。。。所以我修改了部分代码：

vpool = concurrent.futures.ThreadPoolExecutor(max_workers=8) 
print(vpool.map(fdownload, vfilelist0)) 
vpool.shutdown()

…但得到了以下结果：

2018-05-08 12:44:25.115066 Loading variables...
2018-05-08 12:44:25.115066 Starting to process 10.11.12.13...
2018-05-08 12:44:25.115066 Disabling known hosts...
2018-05-08 12:44:25.115066 Opening FTP connection...
2018-05-08 12:44:26.567149 Reading objects list...
2018-05-08 12:45:30.580015 Separating files from folders...
2018-05-08 12:45:30.584015 Closing FTP connection...
2018-05-08 12:45:30.585015 Vectorizing download file list...
2018-05-08 12:45:30.596016 Filtering latest file for each object...
2018-05-08 12:45:30.648019 Vectorizing only latest files...
2018-05-08 12:45:30.652019 Starting simultaneous downloads...
<generator object Executor.map.<locals>.result_iterator at 0x04332D20>