>>1205162Didn't work for me. I wrote this instead:
import requests
from bs4 import BeautifulSoup
import os
from tqdm import tqdm
your_region = "USA"
your_index_url = ' url of intest (can be one system) '
def download_files_with_progress(links):
for link in links:
file_url = requests.compat.urljoin(url, link['href'])
file_name = link['href'].split('/')[-1]
file_name = requests.compat.unquote(file_name)
print(f"Downloading {file_name}...")
response = requests.head(file_url)
file_size = int(response.headers.get('Content-Length', 0))
with requests.get(file_url, stream=True) as file_response:
file_response.raise_for_status()
with open(file_name, 'wb') as f:
with tqdm(total=file_size, unit='B', unit_scale=True, unit_divisor=1024) as pbar:
for chunk in file_response.iter_content(chunk_size=8192):
f.write(chunk)
pbar.update(len(chunk))
print(f"Downloaded {file_name} successfully.")
def download_files(url):
try:
response = requests.get(url)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
links = soup.find_all(
'a', title=lambda t: t and f'({your_region.upper})' in t)
download_files_with_progress(links)
except requests.RequestException as e:
print(f"An error occurred: {e}")
if __name__ == '__main__':
download_files(your_index_url)
execute in the directory you want the files
