我正在尝试从请求库中删除日志记录,我已经阅读并尝试了有关堆栈溢出的其他答案,但它们似乎不起作用
我正在解析网页的html代码中的URL,然后从每个URL下载zip文件。下载所有zip文件后,我从请求库收到一条消息:
从以下网址下载数据:
http://viewfinderpanoramas.org/Coverage%20map%20viewfinderpanoramas_org3.htm
这是我的密码。我正在导入该类并从另一个文件运行download_data
方法
import os
import argparse
import requests
import re
import logging
from tqdm import tqdm
"""
Downloads all zip files from the viewfinder panoramas elevation dataset
"""
class ElevationScraper:
def __init__(self):
self.data_url = "http://viewfinderpanoramas.org/Coverage%20map%20viewfinderpanoramas_org3.htm"
self.base_dir = "./data/elevation/"
self.elevation_dir = os.path.join(self.base_dir, "elevation")
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("requests").propagate = True
def download_data(self):
# get html from self.data_url and get all urls in the html code
req = requests.get(self.data_url)
urls = re.findall(r"\"(https?://\S+)\"", str(req.text))
# first four are irrelevant links so remove those
urls = urls[4:]
with open(os.path.join(self.base_dir, "urls.txt"), "w") as url_file:
for url in urls:
url_file.write(url + "\n")
# downloading all zip files
if not os.path.exists(self.elevation_dir):
os.mkdir(self.elevation_dir)
print("Downloading " + str(len(urls)) + " data files to " + self.elevation_dir + " from " + self.data_url)
for url in tqdm(urls[:3]):
filename = url.split("/")[-1]
with open(os.path.join(self.base_dir, "elevation", filename), "wb") as data_file:
req = requests.get(url)
data_file.write(req.content)
输出:
shrey@Shrey-TPL480 MINGW64 ~/dev/Landslide-Prediction (main)
$ python download_elevation.py
Downloading 1140 data files to ./data/elevation/elevation from http://viewfinderpanoramas.org/Coverage%20map%20viewfinderpanoramas_org3.htm
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:08<00:00, 2.75s/it]
Downloading data from http://viewfinderpanoramas.org/Coverage%20map%20viewfinderpanoramas_org3.htm
您可以像这样将
setLevel
更改为临界值logging.getLogger("requests").setLevel(logging.CRITICAL)
,也可以使用requests.packages.urllib3.disable_warnings()
。请参见:How do I disable log messages from the Requests library?。像这样的问题总是被问到相关问题 更多 >
编程相关推荐