Refactor, simplify, more error handling
This commit is contained in:
iamdoubz
@@ -1,5 +1,4 @@
|
||||
import argparse
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
@@ -16,11 +15,12 @@ from selenium.webdriver.support.ui import WebDriverWait, Select
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
|
||||
__version__ = "2026.3.27.0"
|
||||
__version__ = "2026.4.2.0"
|
||||
|
||||
# Helper for logging
|
||||
def setup_logging(mode="syslog", logfile=None):
|
||||
def setup_logging(folder, mode="syslog", logfile=None):
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.INFO)
|
||||
# remove default handlers
|
||||
@@ -39,60 +39,36 @@ def setup_logging(mode="syslog", logfile=None):
|
||||
if mode in ("file", "all"):
|
||||
if not logfile:
|
||||
raise ValueError("File logging requires a logfile path")
|
||||
file_handler = logging.FileHandler(logfile)
|
||||
file_handler = logging.FileHandler(os.path.join(folder, logfile))
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
# Setup pass in arguments
|
||||
def main():
|
||||
# Create the parser and add a description
|
||||
# Create the parser and add a description
|
||||
def args():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="V-dlp options and variables...",
|
||||
epilog="End of help documentation..."
|
||||
)
|
||||
#parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-l", type=str, help="Choose logging option", default="syslog", choices=["syslog","file","all","none"])
|
||||
parser.add_argument("-lf", help="If log is file/all need to specify file to log to")
|
||||
parser.add_argument("-d", type=str, help="Download directory to use", default=f"{Path.home() / 'Downloads'}")
|
||||
parser.add_argument("-u", type=str, help="File with links inside", default="urls.txt")
|
||||
parser.add_argument("-uh", type=bool, help="Use headless Chrome", default=False)
|
||||
parser.add_argument("-tl", type=float, help="How long to wait for webpage to load before timeout", default=10)
|
||||
parser.add_argument("-tl", type=float, help="How long to wait for webpage to load before timeout", default=8)
|
||||
parser.add_argument("-r", type=float, help="How often to refresh statistics on screen", default=2)
|
||||
parser.add_argument("-tw", type=float, help="Number of seconds to pause between downloads", default=4)
|
||||
parser.add_argument("-nm", type=bool, help="Do not monitor download statistics", default=False)
|
||||
parser.add_argument("-gc", type=int, help="Download cover image (0: don't download, 1: small, 2: large, 3: both)", default=0, choices=[0, 1, 2, 3])
|
||||
parser.add_argument("-gc", type=int, help="Download cover image (0: don't download, 1: small (avif), 2: large (webp), 3: both)", default=0, choices=[0, 1, 2, 3])
|
||||
parser.add_argument("-v", action='store_true', help="Display version information")
|
||||
args = parser.parse_args()
|
||||
return parser
|
||||
|
||||
if args.v:
|
||||
sys.exit(f"v{__version__}\n")
|
||||
|
||||
setup_logging(args.l, args.lf)
|
||||
|
||||
download_dir = args.d
|
||||
url_file = args.u
|
||||
headless = args.uh
|
||||
page_load_time = args.tl
|
||||
refresh_rate = args.r
|
||||
wait_time = args.tw
|
||||
monitor = args.nm
|
||||
cover = args.gc
|
||||
|
||||
logging.info(f"Download Directory: {download_dir}")
|
||||
logging.info(f"Reading URLs from: {url_file}")
|
||||
|
||||
emessage = f""
|
||||
# URLs to process
|
||||
# URLs to process
|
||||
def open_urls(file):
|
||||
try:
|
||||
with open(url_file) as f:
|
||||
with open(file) as f:
|
||||
urls = [line.strip() for line in f]
|
||||
except Exception as e:
|
||||
emessage = f"Could not find url file: {e}!"
|
||||
logging.error(emessage)
|
||||
sys.exit(emessage)
|
||||
urls = list(dict.fromkeys(urls))
|
||||
url_length = len(urls)
|
||||
cur_url = 1
|
||||
ess = "s"
|
||||
if url_length == 0:
|
||||
logging.warning("There are no URLs to process!")
|
||||
@@ -100,10 +76,18 @@ def main():
|
||||
if url_length == 1:
|
||||
ess = ""
|
||||
logging.info(f"Will process {url_length} URL{ess}...")
|
||||
# If directory does not exist, create it
|
||||
os.makedirs(download_dir, exist_ok=True)
|
||||
return urls, url_length
|
||||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
emessage = f"Error {exc_tb.tb_lineno}: Could not find url file: {e}!"
|
||||
logging.error(emessage)
|
||||
sys.exit(emessage)
|
||||
|
||||
# Launch Chrome
|
||||
def open_chrome(headless, download_dir):
|
||||
# Generate list of random screen resolutions
|
||||
display_resolutions = ["2560,1440","1920,1080","1600,1200"]
|
||||
|
||||
# Create and add Chrome options
|
||||
chrome_options = Options()
|
||||
if headless == True:
|
||||
@@ -112,6 +96,9 @@ def main():
|
||||
chrome_options.add_argument(f"--window-size=2560,1440")
|
||||
chrome_options.add_argument("--no-sandbox")
|
||||
chrome_options.add_argument("--disable-dev-shm-usage")
|
||||
chrome_options.add_argument("--simulate-outdated-no-au='Tue, 31 Dec 2099 23:59:59 GMT'")
|
||||
chrome_options.add_argument("--disable-background-networking")
|
||||
chrome_options.add_argument("--disable-component-update")
|
||||
prefs = {
|
||||
"download.default_directory": download_dir,
|
||||
"download.prompt_for_download": False,
|
||||
@@ -121,8 +108,8 @@ def main():
|
||||
|
||||
driver = webdriver.Chrome(options=chrome_options)
|
||||
|
||||
if headless == True:
|
||||
# Chrome sometimes blocks downloads in headless mode
|
||||
if headless == True:
|
||||
driver.execute_cdp_cmd(
|
||||
"Page.setDownloadBehavior",
|
||||
{
|
||||
@@ -130,16 +117,40 @@ def main():
|
||||
"downloadPath": download_dir
|
||||
}
|
||||
)
|
||||
# Helper to calculate percentages
|
||||
def percentage_of_total(part, whole):
|
||||
return driver
|
||||
|
||||
# Helper to calculate percentages
|
||||
def percentage_of_total(part, whole):
|
||||
if whole == 0:
|
||||
return 0 # Handle division by zero case
|
||||
return round(((part / whole) * 100), 1)
|
||||
# Display useful stats about ongoing downloads
|
||||
def monitor_download(folder, fsize):
|
||||
|
||||
# Check for temp files limiting amount of time spent before failing
|
||||
def wait_for_file(download_dir, check_interval_seconds=0.25, max_wait_seconds=10):
|
||||
start_time = time.time()
|
||||
while True:
|
||||
# Check if the file exists
|
||||
files = os.listdir(download_dir)
|
||||
partial = [f for f in files if f.endswith(".crdownload")]
|
||||
if partial:
|
||||
elapsed_time = time.time() - start_time
|
||||
return True
|
||||
# Calculate elapsed time and check if timeout is reached
|
||||
elapsed_time = time.time() - start_time
|
||||
if elapsed_time >= max_wait_seconds:
|
||||
print(f"Timed out after {max_wait_seconds} seconds. Temp file not found at {download_dir}.")
|
||||
return False
|
||||
# Wait for the specified interval before the next check
|
||||
remaining_time = max_wait_seconds - elapsed_time
|
||||
if remaining_time < check_interval_seconds:
|
||||
time_to_sleep = remaining_time
|
||||
# Time to sleep
|
||||
time.sleep(check_interval_seconds)
|
||||
|
||||
# Display useful stats about ongoing downloads
|
||||
def monitor_download(folder, fsize, refresh_rate, tstart=time.time()):
|
||||
downloading = True
|
||||
last_size = 0
|
||||
tstart = time.time()
|
||||
while downloading:
|
||||
files = os.listdir(folder)
|
||||
partial = [f for f in files if f.endswith(".crdownload")]
|
||||
@@ -166,32 +177,60 @@ def main():
|
||||
tsec = 1
|
||||
tmin, tmsec = divmod(tsec, 60)
|
||||
avg_speed = round((fsize/tsec/1024/1024)*8, 1)
|
||||
if avg_speed > 175:
|
||||
logging.warning("Download did not start (probably)")
|
||||
return -999, -999
|
||||
else:
|
||||
logging.info(f"Downloaded {round((fsize/1024/1024), 2)}MB in {int(tmin)}m {int(tmsec)}s ({avg_speed} Mbps).")
|
||||
return last_size, tsec
|
||||
def wait_for_file(pattern, delay=1, max=15):
|
||||
#print(f"Waiting for file matching: {pattern}...")
|
||||
i = 0
|
||||
while not glob.glob(pattern):
|
||||
time.sleep(delay)
|
||||
i += 1
|
||||
if i > max:
|
||||
break
|
||||
# Return the first matching file
|
||||
return glob.glob(pattern)[0]
|
||||
# For each URL in the file, run program
|
||||
total_size = 0
|
||||
total_time = 0
|
||||
failed_urls = []
|
||||
for url in urls:
|
||||
def download_it(monitor, wait_time, download_dir, total_size, total_time, cur_url, dtitle, durl, cover, ddisc=0):
|
||||
def download_img(download_dir, title, itype, baseurl):
|
||||
def save_img(download_dir, title, itype, iurl, baseurl, iform):
|
||||
|
||||
# Get cover art
|
||||
def download_img(driver, download_dir, title, itype, baseurl):
|
||||
try:
|
||||
img_element = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, '//img[@alt="Box"]'))
|
||||
)
|
||||
if img_element:
|
||||
if itype in [1,3]:
|
||||
img_url = img_element.get_attribute('src')
|
||||
if img_url:
|
||||
img_saved = save_img(download_dir, title, itype, img_url, baseurl, 1)
|
||||
if not img_saved:
|
||||
logging.warning("See above WARNING.")
|
||||
else:
|
||||
logging.warning("No box image url found.")
|
||||
if itype in [2,3]:
|
||||
body_element = driver.find_element(By.ID, "main")
|
||||
img_element.click()
|
||||
try:
|
||||
dialog_element = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.ID, "imageDialog"))
|
||||
)
|
||||
if dialog_element:
|
||||
img_element2 = dialog_element.find_element(By.TAG_NAME, "img")
|
||||
img_url2 = img_element2.get_attribute('src')
|
||||
if img_url2:
|
||||
save_img(download_dir, title, itype, img_url2, baseurl, 2)
|
||||
else:
|
||||
logging.warning("No large box image found.")
|
||||
actions = ActionChains(driver)
|
||||
actions.move_to_element_with_offset(body_element, random.randint(1, 100), random.randint(1, 100)).click().perform()
|
||||
time.sleep(1)
|
||||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
logging.warning(f"Error {exc_tb.tb_lineno}: No large box image found!")
|
||||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
logging.warning(f"Error {exc_tb.tb_lineno}: No box image exists. Skipping...")
|
||||
|
||||
# Save cover art
|
||||
def save_img(download_dir, title, itype, iurl, baseurl, iform):
|
||||
headers: dict[str, str] = {
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Encoding': 'gzip, deflate, br, zstd',
|
||||
'Connection': 'keep-alive',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0',
|
||||
'Referer': f'{url}'
|
||||
'Referer': f'{iurl}'
|
||||
}
|
||||
response = requests.get(iurl, headers=headers, allow_redirects=True, stream=True)
|
||||
response.raise_for_status()
|
||||
@@ -204,66 +243,85 @@ def main():
|
||||
with open(save_path, 'wb') as file:
|
||||
for chunk in response.iter_content(1024):
|
||||
file.write(chunk)
|
||||
time.sleep(1)
|
||||
return True
|
||||
else:
|
||||
logging.warning(f"Could not download box image: {response.status_code} - {response.reason}")
|
||||
pass
|
||||
try:
|
||||
img_element = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.XPATH, '//img[@alt="Box"]'))
|
||||
)
|
||||
if itype in [1,3]:
|
||||
img_url = img_element.get_attribute('src')
|
||||
if img_url:
|
||||
save_img(download_dir, title, itype, img_url, baseurl, 1)
|
||||
else:
|
||||
logging.warning("No box image url found.")
|
||||
if itype in [2,3]:
|
||||
body_element = driver.find_element(By.TAG_NAME, "body")
|
||||
img_element.click()
|
||||
try:
|
||||
dialog_element = WebDriverWait(driver, 5).until(
|
||||
EC.presence_of_element_located((By.ID, "imageDialog"))
|
||||
)
|
||||
img_element2 = dialog_element.find_element(By.TAG_NAME, "img")
|
||||
img_url2 = img_element2.get_attribute('src')
|
||||
if img_url2:
|
||||
save_img(download_dir, title, itype, img_url2, baseurl, 2)
|
||||
else:
|
||||
logging.warning("No large box image found.")
|
||||
actions = ActionChains(driver)
|
||||
actions.move_to_element_with_offset(body_element, random.randint(1, 100), random.randint(1, 100)).click().perform()
|
||||
time.sleep(1)
|
||||
except Exception as e:
|
||||
logging.warning(f"No large box image found! {e}")
|
||||
except:
|
||||
logging.warning("No box image exists. Skipping...")
|
||||
# Get Vimm file size
|
||||
return False
|
||||
|
||||
# Download logic
|
||||
def download_it(driver, wait, monitor, wait_time, download_dir, refresh_rate, total_size, total_time, cur_url, url_length, dtitle, durl, cover, failed_urls, ddisc=0):
|
||||
# Get file size
|
||||
size = 0
|
||||
try:
|
||||
size_element = WebDriverWait(driver, 5).until(
|
||||
size_element = wait.until(
|
||||
EC.presence_of_element_located((By.ID, "dl_size"))
|
||||
)
|
||||
size_raw = size_element.text
|
||||
except:
|
||||
logging.warning("Could not determine download size from webpage")
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
logging.warning(f"Error {exc_tb.tb_lineno}: Could not determine download size from webpage")
|
||||
size_raw = '1 GB'
|
||||
pass
|
||||
logging.info(f"{title} {size_raw}")
|
||||
|
||||
logging.info(f"{dtitle} {size_raw}")
|
||||
|
||||
# Download box cover
|
||||
if cover > 0:
|
||||
raw_title = driver.title.replace("The Vault: ", "")
|
||||
download_img(download_dir, raw_title, cover, url)
|
||||
download_img(driver, download_dir, raw_title, cover, durl)
|
||||
|
||||
# Click Download button
|
||||
dl_start = time.time()
|
||||
try:
|
||||
download_button = WebDriverWait(driver, 5).until(
|
||||
download_form = wait.until(
|
||||
EC.presence_of_element_located((By.ID, "dl_form"))
|
||||
)
|
||||
download_button = wait.until(
|
||||
EC.element_to_be_clickable((By.XPATH, "//button[text()='Download']"))
|
||||
)
|
||||
actions = ActionChains(driver)
|
||||
if download_form:
|
||||
actions.move_to_element_with_offset(download_form, random.randint(1, 50), random.randint(1, 11)).perform()
|
||||
time.sleep(1)
|
||||
actions.move_to_element_with_offset(download_button, random.randint(1, 25), random.randint(1, 6)).perform()
|
||||
time.sleep(0.5)
|
||||
#time.sleep(3)
|
||||
download_form = wait.until(
|
||||
EC.presence_of_element_located((By.ID, "dl_form"))
|
||||
)
|
||||
#logging.info("Clicked form submit")
|
||||
if download_form:
|
||||
download_form.submit()
|
||||
raw_title = driver.title
|
||||
if raw_title == "Vimm's Lair: Error 400":
|
||||
return -998, -998
|
||||
else:
|
||||
raise ValueError("Download form could not be submitted")
|
||||
else:
|
||||
actions.move_to_element_with_offset(download_button, random.randint(1, 13), random.randint(1, 3)).perform()
|
||||
time.sleep(1)
|
||||
#logging.info("Clicked download button")
|
||||
download_button = wait.until(
|
||||
EC.element_to_be_clickable((By.XPATH, "//button[text()='Download']"))
|
||||
)
|
||||
if download_button:
|
||||
download_button.click()
|
||||
except:
|
||||
logging.warning(f"Download button not found for {title}!")
|
||||
failed_urls.append(url)
|
||||
pass
|
||||
#driver.execute_script("arguments[0].click();", download_button)
|
||||
raw_title = driver.title
|
||||
if raw_title == "Vimm's Lair: Error 400":
|
||||
return -998, -998
|
||||
else:
|
||||
raise ValueError("Download button could not be submitted")
|
||||
dl_start = time.time()
|
||||
except Exception as e:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
logging.warning(f"Error {exc_tb.tb_lineno}: Download button not found for {dtitle}!")
|
||||
logging.error(f"{e}")
|
||||
failed_urls.append(dtitle)
|
||||
failed_urls.append(durl)
|
||||
return 0, 0
|
||||
|
||||
# Convert human readable size to bytes
|
||||
if " KB" in size_raw:
|
||||
size = float(size_raw.replace(" KB", "")) * 1024
|
||||
elif " MB" in size_raw:
|
||||
@@ -273,57 +331,73 @@ def main():
|
||||
elif " TB" in size_raw:
|
||||
size = float(size_raw.replace(" TB", "")) * 1024 ** 4
|
||||
else:
|
||||
size = 500 * 1024 * 1024
|
||||
# Try to click Continue if it appears
|
||||
try:
|
||||
continue_button = WebDriverWait(driver, 5).until(
|
||||
EC.element_to_be_clickable((By.XPATH, "//input[@value='Continue']"))
|
||||
)
|
||||
continue_button.click()
|
||||
except:
|
||||
pass
|
||||
size = 500 * 1024 ** 2
|
||||
|
||||
# Monitor current download
|
||||
argmonitor = monitor
|
||||
argwait = wait_time
|
||||
ctime = 0
|
||||
csize = 0
|
||||
|
||||
# If file size is >32MB, go ahead and monitor anyway
|
||||
if size > 33554431 and monitor:
|
||||
monitor = False
|
||||
|
||||
# If file size <32MB, force no monitor
|
||||
if size < 33554432:
|
||||
monitor = True
|
||||
wait_time = 15
|
||||
if size < 2097152 * 10:
|
||||
wait_time = 10
|
||||
if size < 2097152 * 8:
|
||||
wait_time = 8
|
||||
if size < 2097152 * 7:
|
||||
wait_time = 7
|
||||
if size < 2097152 * 6:
|
||||
wait_time = 6
|
||||
if size < 2097152 * 5:
|
||||
wait_time = 5
|
||||
if size < 2097152 * 4:
|
||||
wait_time = 4
|
||||
if size < 2097152 * 3:
|
||||
wait_time = 3
|
||||
if size < 2097152 * 2:
|
||||
wait_time = 2
|
||||
if size < 2097152:
|
||||
size_limit = 2097152
|
||||
size_factor = size // size_limit
|
||||
if size_factor < wait_time:
|
||||
wait_time = size_factor
|
||||
if size_factor == 0:
|
||||
wait_time = 1
|
||||
# If we are monitoring downloads
|
||||
if monitor == False:
|
||||
tempTime = time.time()
|
||||
file_pattern = download_dir + '/*.crdownload'
|
||||
while not os.path.exists(wait_for_file(file_pattern)):
|
||||
if time.time() - tempTime > wait_time:
|
||||
logging.warning("Never found temp file for download...")
|
||||
break
|
||||
if wait_for_file(download_dir, 0.25, 8):
|
||||
csize, ctime = monitor_download(download_dir, size*1.01, refresh_rate, dl_start)
|
||||
if csize == -999 and ctime == -999:
|
||||
failed_urls.append(dtitle)
|
||||
failed_urls.append(durl)
|
||||
elif csize == -998 and ctime == -998:
|
||||
logging.warning("Vimm's Lair Error 400: An unexpected browser error has occurred (we think you might be a bot)")
|
||||
failed_urls.append(dtitle)
|
||||
failed_urls.append(durl)
|
||||
else:
|
||||
logging.warning(f"{time.time() - tempTime}")
|
||||
time.sleep(refresh_rate)
|
||||
if monitor == False:
|
||||
csize, ctime = monitor_download(download_dir, size*1.01)
|
||||
total_size += csize
|
||||
total_time += ctime
|
||||
else:
|
||||
# Try to click Continue if it appears
|
||||
try:
|
||||
raw_title = driver.title
|
||||
if raw_title != "Vimm's Lair: Error 400":
|
||||
continue_button = WebDriverWait(driver, 4).until(
|
||||
EC.element_to_be_clickable((By.XPATH, "//input[@value='Continue']"))
|
||||
)
|
||||
continue_button.click()
|
||||
dl_start = time.time()
|
||||
logging.info(f"DL Click: {dl_start} Check: {round((time.time() - dl_start),1)}")
|
||||
if wait_for_file(files, 0.25, 8):
|
||||
csize, ctime = monitor_download(download_dir, files, size*1.01, refresh_rate, dl_start)
|
||||
total_size += csize
|
||||
total_time += ctime
|
||||
else:
|
||||
logging.warning("Never found temp file for download...")
|
||||
failed_urls.append(dtitle)
|
||||
failed_urls.append(durl)
|
||||
pass
|
||||
else:
|
||||
logging.warning("Vimm's Lair Error 400: An unexpected browser error has occurred (we think you might be a bot)")
|
||||
failed_urls.append(dtitle)
|
||||
failed_urls.append(durl)
|
||||
except:
|
||||
exc_type, exc_obj, exc_tb = sys.exc_info()
|
||||
logging.warning(f"Error {exc_tb.tb_lineno}: Could not click on Continue button...")
|
||||
failed_urls.append(dtitle)
|
||||
failed_urls.append(durl)
|
||||
pass
|
||||
|
||||
# Wait to call next URL
|
||||
if cur_url < url_length:
|
||||
if size < 33554432:
|
||||
@@ -332,15 +406,42 @@ def main():
|
||||
else:
|
||||
if size < 33554432:
|
||||
logging.warning(f"File size was too small to monitor! (Less than 32MB).")
|
||||
|
||||
cur_url += 1
|
||||
monitor = argmonitor
|
||||
wait_time = argwait
|
||||
return csize, ctime
|
||||
# Open URL
|
||||
|
||||
# Main program
|
||||
def main(args):
|
||||
download_dir = args.d
|
||||
url_file = args.u
|
||||
headless = args.uh
|
||||
page_load_time = args.tl
|
||||
refresh_rate = args.r
|
||||
wait_time = args.tw
|
||||
monitor = args.nm
|
||||
cover = args.gc
|
||||
|
||||
logging.info(f"Download Directory: {download_dir}")
|
||||
logging.info(f"Reading URLs from: {url_file}")
|
||||
|
||||
urls, url_length = open_urls(url_file)
|
||||
|
||||
# For each URL in the file
|
||||
total_size = 0
|
||||
total_time = 0
|
||||
cur_url = 1
|
||||
failed_urls = []
|
||||
for url in urls:
|
||||
# Launch chrome and open URL
|
||||
driver = open_chrome(headless, download_dir)
|
||||
driver.get(url)
|
||||
|
||||
# Wait to open URL
|
||||
wait = WebDriverWait(driver, page_load_time)
|
||||
# Multiple discs?
|
||||
|
||||
# Check for multiple discs
|
||||
try:
|
||||
disc_element = driver.find_element(By.ID, "disc_number")
|
||||
disc_select = Select(disc_element)
|
||||
@@ -353,32 +454,50 @@ def main():
|
||||
disc_replace = f"{cur_url}/{url_length}: "
|
||||
disc_select.select_by_value(disc_value)
|
||||
title = driver.title.replace("The Vault: ", disc_replace)
|
||||
osize, otime = download_it(monitor, wait_time, download_dir, total_size, total_time, cur_url, title, url, cover, disc_value)
|
||||
osize, otime = download_it(driver, wait, monitor, wait_time, download_dir, refresh_rate, total_size, total_time, cur_url, url_length, title, url, cover, failed_urls, disc_value)
|
||||
total_size += osize
|
||||
total_time += otime
|
||||
cur_url += 1
|
||||
except:
|
||||
title = driver.title.replace("The Vault: ", f"{cur_url}/{url_length}: ")
|
||||
osize, otime = download_it(monitor, wait_time, download_dir, total_size, total_time, cur_url, title, url, cover)
|
||||
total_size += osize
|
||||
total_time += otime
|
||||
cur_url += 1
|
||||
# Close Chrome session
|
||||
except KeyboardInterrupt:
|
||||
logging.info("\nSignal received. Shutting down gracefully...")
|
||||
driver.close()
|
||||
driver.quit()
|
||||
sys.exit(67)
|
||||
except Exception as e:
|
||||
logging.warning(f"Multiple discs error? {e}")
|
||||
title = driver.title.replace("The Vault: ", f"{cur_url}/{url_length}: ")
|
||||
osize, otime = download_it(driver, wait, monitor, wait_time, download_dir, refresh_rate, total_size, total_time, cur_url, url_length, title, url, cover, failed_urls)
|
||||
total_size += osize
|
||||
total_time += otime
|
||||
cur_url += 1
|
||||
finally:
|
||||
# Close and end Chrome session
|
||||
driver.close()
|
||||
driver.quit()
|
||||
|
||||
# If we were monitoring, display total statistics
|
||||
if total_size > 0:
|
||||
if total_time == 0:
|
||||
total_time = 1
|
||||
ttmin, ttmsec = divmod(total_time, 60)
|
||||
tavg_speed = round((total_size/total_time/1024/1024)*8, 1)
|
||||
logging.info(f"Downloaded {round((total_size/1024/1024), 2)}MB in {int(ttmin)}m {int(ttmsec)}s ({tavg_speed} Mbps).")
|
||||
|
||||
# If anything failed, write to file
|
||||
if failed_urls:
|
||||
# Add URL links to a file
|
||||
fn = f"failed_downloads.txt"
|
||||
logging.warning(f"Writing failed downloads to {fn}")
|
||||
fnt = 'a'
|
||||
fn = os.path.join(f"{download_dir}", "failed.txt")
|
||||
logging.warning(f"Appending failed downloads to {fn}")
|
||||
fnt = "a"
|
||||
with open(f"{fn}", fnt) as f:
|
||||
for item in failed_urls:
|
||||
f.write(item + '\n')
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
parser = args()
|
||||
args = parser.parse_args()
|
||||
if args.v:
|
||||
sys.exit(f"v{__version__}\n")
|
||||
# If directory does not exist, create it
|
||||
os.makedirs(args.d, exist_ok=True)
|
||||
setup_logging(args.d, args.l, args.lf)
|
||||
main(args)
|
||||
Reference in New Issue
Block a user