如何使用Asyncio模块将数据库从Sqlite迁移到Postgres？

Question

我有一个将数据库从Sqlite迁移到Postgres的脚本。我原来的scipt可以工作，但当我尝试使用Asyncio来加速程序时，我的新代码甚至比原来的运行速度慢了几秒钟。表的传输速度非常慢。谁能告诉我，我错在哪里

我的原始代码：

import psycopg2, sqlite3, sys
import time

start_time = time.time()

sqdb="D://Python//SqliteToPostgreFull//testmydb6.db" #folder contain sqlite db
sqlike="table"
pgdb="testmydb7" #postgres db
pguser="postgres"
pgpswd="1234"
pghost="127.0.0.1"
pgport="5432"

consq=sqlite3.connect(sqdb)
cursq=consq.cursor()

tabnames=[]
print() 
cursq.execute('SELECT name FROM sqlite_master WHERE type="table" AND name LIKE "%table%";')
tabgrab = cursq.fetchall()
for item in tabgrab:
    tabnames.append(item[0])
print(tabgrab)
for table in tabnames:
    print(table)
    cursq.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name = ?;", (table,))
    create = cursq.fetchone()[0]
    cursq.execute("SELECT * FROM %s;" %table)
    rows=cursq.fetchall()
    colcount=len(rows[0])
    pholder='%s,'*colcount
    newholder=pholder[:-1]

    try:
        conpg = psycopg2.connect(database=pgdb, user=pguser, password=pgpswd,
                               host=pghost, port=pgport)
        curpg = conpg.cursor()
        curpg.execute("DROP TABLE IF EXISTS %s;" %table)
        create = create.replace("AUTOINCREMENT", "")
        curpg.execute(create)
        curpg.executemany("INSERT INTO %s VALUES (%s);" % (table, newholder),rows)
        conpg.commit()

        if conpg:
            conpg.close()

    except psycopg2.DatabaseError as e:
        print ('Error %s' % e) 
        sys.exit(1)

    finally:
        print("Complete")


consq.close()

duration = time.time() - start_time
print(f"Duration {duration} seconds")

我的异步IO模块代码：

import psycopg2, sqlite3, sys
import time
import asyncio


sqdb="D://Python//SqliteToPostgreFull//testmydb6.db"
sqlike="table"
pgdb="testmydb9"
pguser="postgres"
pgpswd="1234"
pghost="127.0.0.1"
pgport="5432"


consq=sqlite3.connect(sqdb)
cursq=consq.cursor()

tabnames=[]
print() 
cursq.execute('''SELECT name FROM sqlite_master WHERE type="table" AND name LIKE "'''+sqlike+'''%";''')
tabgrab = cursq.fetchall()
for item in tabgrab:
    tabnames.append(item[0])
print(tabgrab)

async def copyTable(table):
    cursq.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name = ?;", (table,))
    create = cursq.fetchone()[0]
    cursq.execute("SELECT * FROM %s;" %table)
    rows=cursq.fetchall()
    colcount=len(rows[0])
    pholder='%s,'*colcount
    newholder=pholder[:-1]

    try:

        conpg = psycopg2.connect(database=pgdb, user=pguser, password=pgpswd,
                               host=pghost, port=pgport)
        curpg = conpg.cursor()
        curpg.execute("DROP TABLE IF EXISTS %s;" %table)
        create = create.replace("AUTOINCREMENT", "")
        curpg.execute(create)
        curpg.executemany("INSERT INTO %s VALUES (%s);" % (table, newholder),rows)
        conpg.commit()

        if conpg:
            conpg.close()

    except psycopg2.DatabaseError as e:
        print ('Error %s' % e) 
        sys.exit(1)

    finally:
        print("Complete")

async def main():
    for table in tabnames:
        a = loop.create_task(copyTable(table,))
    await asyncio.wait([a])

if __name__ == "__main__":
    start_time = time.time()
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main())
    loop.close()      
    duration = time.time() - start_time
    print(f"Duration {duration} seconds")

Answer 1

如果两个数据库都位于同一台计算机上，asyncio将不会加快进程：没有需要并行化的网络开销。恰恰相反：使用协程的开销会使程序稍微慢一点

请阅读this answer了解详细解释