我面临一个问题,来自ThreadPoolExecutor的concurrent.futures.Future有时会挂起,我使用的是python 3.7,下面是我的代码:
import pandas as pd
import concurrent.futures
import requests
import time
import sys
from urllib3.util.retry import Retry
import os
from decimal import localcontext, Decimal, ROUND_HALF_UP
from datetime import datetime,timezone ,timedelta
import urllib3
import base64
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
class Test:
out = []
CONNECTIONS = 200
f1 = open('D:/NewProxeyUserPass.txt', 'r')
Proxies= f1.readlines()
f1.close()
f2 = open('D:/UserAgents.txt', 'r')
UserAgents= f2.readlines()
f2.close()
MAX_RETRIES = 3
ExcptionCount= int
ExcptionCount=0
BASE_URI = 'https://h.com/session.php'
def load_url(n):
Proxy_Port_User_Pass=Test.Proxies[n].split(":")
useragent=Test.UserAgents[n]
Proxy_Port_User_Pass[0].strip()
ProxyIP =Proxy_Port_User_Pass[0].strip()
ProxyPort =Proxy_Port_User_Pass[1].strip()
username = Proxy_Port_User_Pass[2].strip()
password =Proxy_Port_User_Pass[3].strip()
usrPass = username+":"+password
b64Val = base64.b64encode(usrPass.encode()).decode()
Timestamp =int((datetime.utcnow() - datetime(1970,1,1)).total_seconds())
headers = {
"user-agent": useragent.strip(),
"Connection":"close"
}
proxies = {
"http" :"http://" + username + ":" + password + "@" + ProxyIP + ":" + ProxyPort,
"https" :"https://" + username + ":" + password + "@" + ProxyIP + ":" +ProxyPort
}
JsonSession = "{'device':'id'}"
retries = Retry(total=3,
backoff_factor=0.1)
adapter = requests.adapters.HTTPAdapter(max_retries=retries)
with requests.Session() as session:
session.mount('https://', adapter)
session.keep_alive = False
session.trust_env=False
ans = session.post(Test.BASE_URI,
data=JsonSession,
headers=headers,
params={"timestamp":Timestamp},
proxies=proxies,
verify=False)
return ans.content
def DoWork():
with concurrent.futures.ThreadPoolExecutor(max_workers=Test.CONNECTIONS) as executor:
future_to_url = (executor.submit(Test.load_url, n) for n in range(0,150))
count=int
count=1
ExcptionCount = int
ExcptionCount = 0
time1 = time.time()
for future in concurrent.futures.as_completed(future_to_url):
try:
print( str(count)+" - "+str(future.result())+"\n")
count = count +1
except Exception as exc:
ExcptionCount=ExcptionCount+1
print("Exception count : "+ str(ExcptionCount) + " - "+str(exc))
time2 = time.time()
print(f'Took {time2-time1:.2f} s'+"\n Total Exceptions Count : "+ str(ExcptionCount))
pass
Test.DoWork()
我找到了这些 Why would a timeout avoid a tornado hang? 和 https://github.com/tornadoweb/tornado/issues/1595 但我不知道如何在代码中使用它, 那么如何防止concurrent.futures.from线程池执行器挂起,如果你能给我工作代码,谢谢你的帮助
目前没有回答
相关问题 更多 >
编程相关推荐