Better error handling, add total statistics
This commit is contained in:
@@ -166,6 +166,7 @@ def main():
|
||||
tmin, tmsec = divmod(tsec, 60)
|
||||
avg_speed = round((fsize/tsec/1024/1024)*8, 1)
|
||||
logging.info(f"Downloaded {round((fsize/1024/1024), 2)}MB in {int(tmin)}m {int(tmsec)}s ({avg_speed} Mbps).")
|
||||
return last_size, tsec
|
||||
def wait_for_file(pattern, delay=1, max=15):
|
||||
#print(f"Waiting for file matching: {pattern}...")
|
||||
i = 0
|
||||
@@ -177,25 +178,37 @@ def main():
|
||||
# Return the first matching file
|
||||
return glob.glob(pattern)[0]
|
||||
# For each URL in the file, run program
|
||||
total_size = 0
|
||||
total_time = 0
|
||||
failed_urls = []
|
||||
for url in urls:
|
||||
# Open URL
|
||||
driver.get(url)
|
||||
# Wait to open URL
|
||||
wait = WebDriverWait(driver, page_load_time)
|
||||
# Click Download button
|
||||
download_button = wait.until(
|
||||
EC.element_to_be_clickable((By.XPATH, "//button[text()='Download']"))
|
||||
)
|
||||
download_button.click()
|
||||
# Output page title to console
|
||||
title = driver.title.replace("The Vault: ", f"{cur_url}/{url_length}: ")
|
||||
# Click Download button
|
||||
try:
|
||||
download_button = wait.until(
|
||||
EC.element_to_be_clickable((By.XPATH, "//button[text()='Download']"))
|
||||
)
|
||||
download_button.click()
|
||||
except:
|
||||
logging.warning(f"Download button not found for {title}!")
|
||||
failed_urls.append(url)
|
||||
# Get Vimm file size
|
||||
size_element = WebDriverWait(driver, 10).until(
|
||||
EC.presence_of_element_located((By.ID, "dl_size"))
|
||||
)
|
||||
size_raw = size_element.text
|
||||
logging.info(f"{title} {size_raw}")
|
||||
size = 0
|
||||
try:
|
||||
size_element = WebDriverWait(driver, 10).until(
|
||||
EC.presence_of_element_located((By.ID, "dl_size"))
|
||||
)
|
||||
size_raw = size_element.text
|
||||
except:
|
||||
logging.warning("Could not determine download size from webpage")
|
||||
size_raw = '1 GB'
|
||||
pass
|
||||
logging.info(f"{title} {size_raw}")
|
||||
if " KB" in size_raw:
|
||||
size = float(size_raw.replace(" KB", "")) * 1024
|
||||
elif " MB" in size_raw:
|
||||
@@ -203,7 +216,7 @@ def main():
|
||||
elif " GB" in size_raw:
|
||||
size = float(size_raw.replace(" GB", "")) * 1024 ** 3
|
||||
elif " TB" in size_raw:
|
||||
size = float(size_raw.replace(" GB", "")) * 1024 ** 4
|
||||
size = float(size_raw.replace(" TB", "")) * 1024 ** 4
|
||||
else:
|
||||
size = 500 * 1024 * 1024
|
||||
# Try to click Continue if it appears
|
||||
@@ -214,10 +227,11 @@ def main():
|
||||
continue_button.click()
|
||||
except:
|
||||
pass
|
||||
|
||||
# Monitor current download
|
||||
argmonitor = monitor
|
||||
argwait = wait_time
|
||||
ctime = 0
|
||||
csize = 0
|
||||
if size < 33554432:
|
||||
monitor = True
|
||||
wait_time = 17
|
||||
@@ -233,7 +247,9 @@ def main():
|
||||
logging.warning(f"{time.time() - tempTime}")
|
||||
time.sleep(refresh_rate)
|
||||
if monitor == False:
|
||||
monitor_download(download_dir, size+1024)
|
||||
csize, ctime = monitor_download(download_dir, size+1024)
|
||||
total_size += csize
|
||||
total_time += ctime
|
||||
# Wait to call next URL
|
||||
time.sleep(wait_time)
|
||||
cur_url += 1
|
||||
@@ -241,6 +257,10 @@ def main():
|
||||
wait_time = argwait
|
||||
# Close Chrome session
|
||||
driver.quit()
|
||||
if total_size > 0:
|
||||
ttmin, ttmsec = divmod(total_time, 60)
|
||||
tavg_speed = round((total_size/total_time/1024/1024)*8, 1)
|
||||
logging.info(f"Downloaded {round((total_size/1024/1024), 2)}MB in {int(ttmin)}m {int(ttmsec)}s ({tavg_speed} Mbps).")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user