Add logic to download multiple discs
This commit is contained in:
iamdoubz
@@ -6,9 +6,11 @@ from pathlib import Path
|
||||
import random
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.common.exceptions import NoSuchElementException
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
from selenium.webdriver.support.ui import WebDriverWait, Select
|
||||
#from selenium.webdriver.support.ui import Select
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
import sys
|
||||
import time
|
||||
@@ -182,79 +184,100 @@ def main():
|
||||
total_time = 0
|
||||
failed_urls = []
|
||||
for url in urls:
|
||||
def download_it(monitor, wait_time, download_dir, total_size, total_time, cur_url, dtitle, durl, ddisc=0):
|
||||
# Click Download button
|
||||
try:
|
||||
download_button = wait.until(
|
||||
EC.element_to_be_clickable((By.XPATH, "//button[text()='Download']"))
|
||||
)
|
||||
download_button.click()
|
||||
except:
|
||||
logging.warning(f"Download button not found for {title}!")
|
||||
failed_urls.append(url)
|
||||
# Get Vimm file size
|
||||
size = 0
|
||||
try:
|
||||
size_element = WebDriverWait(driver, 10).until(
|
||||
EC.presence_of_element_located((By.ID, "dl_size"))
|
||||
)
|
||||
size_raw = size_element.text
|
||||
except:
|
||||
logging.warning("Could not determine download size from webpage")
|
||||
size_raw = '1 GB'
|
||||
pass
|
||||
logging.info(f"{title} {size_raw}")
|
||||
if " KB" in size_raw:
|
||||
size = float(size_raw.replace(" KB", "")) * 1024
|
||||
elif " MB" in size_raw:
|
||||
size = float(size_raw.replace(" MB", "")) * 1024 ** 2
|
||||
elif " GB" in size_raw:
|
||||
size = float(size_raw.replace(" GB", "")) * 1024 ** 3
|
||||
elif " TB" in size_raw:
|
||||
size = float(size_raw.replace(" TB", "")) * 1024 ** 4
|
||||
else:
|
||||
size = 500 * 1024 * 1024
|
||||
# Try to click Continue if it appears
|
||||
try:
|
||||
continue_button = WebDriverWait(driver, 4).until(
|
||||
EC.element_to_be_clickable((By.XPATH, "//input[@value='Continue']"))
|
||||
)
|
||||
continue_button.click()
|
||||
except:
|
||||
pass
|
||||
# Monitor current download
|
||||
argmonitor = monitor
|
||||
argwait = wait_time
|
||||
ctime = 0
|
||||
csize = 0
|
||||
if size < 33554432:
|
||||
monitor = True
|
||||
wait_time = 17
|
||||
logging.warning("File size was too small to monitor! (Less than 32MB)")
|
||||
if monitor == False:
|
||||
tempTime = time.time()
|
||||
file_pattern = download_dir + '/*.crdownload'
|
||||
while not os.path.exists(wait_for_file(file_pattern)):
|
||||
if time.time() - tempTime > wait_time:
|
||||
logging.warning("Never found temp file for download...")
|
||||
break
|
||||
else:
|
||||
logging.warning(f"{time.time() - tempTime}")
|
||||
time.sleep(refresh_rate)
|
||||
if monitor == False:
|
||||
csize, ctime = monitor_download(download_dir, size+1024)
|
||||
total_size += csize
|
||||
total_time += ctime
|
||||
# Wait to call next URL
|
||||
time.sleep(wait_time)
|
||||
cur_url += 1
|
||||
monitor = argmonitor
|
||||
wait_time = argwait
|
||||
return csize, ctime
|
||||
# Open URL
|
||||
driver.get(url)
|
||||
# Wait to open URL
|
||||
wait = WebDriverWait(driver, page_load_time)
|
||||
# Output page title to console
|
||||
title = driver.title.replace("The Vault: ", f"{cur_url}/{url_length}: ")
|
||||
# Click Download button
|
||||
# Multiple discs?
|
||||
try:
|
||||
download_button = wait.until(
|
||||
EC.element_to_be_clickable((By.XPATH, "//button[text()='Download']"))
|
||||
)
|
||||
download_button.click()
|
||||
except:
|
||||
logging.warning(f"Download button not found for {title}!")
|
||||
failed_urls.append(url)
|
||||
# Get Vimm file size
|
||||
size = 0
|
||||
try:
|
||||
size_element = WebDriverWait(driver, 10).until(
|
||||
EC.presence_of_element_located((By.ID, "dl_size"))
|
||||
)
|
||||
size_raw = size_element.text
|
||||
except:
|
||||
logging.warning("Could not determine download size from webpage")
|
||||
size_raw = '1 GB'
|
||||
pass
|
||||
logging.info(f"{title} {size_raw}")
|
||||
if " KB" in size_raw:
|
||||
size = float(size_raw.replace(" KB", "")) * 1024
|
||||
elif " MB" in size_raw:
|
||||
size = float(size_raw.replace(" MB", "")) * 1024 ** 2
|
||||
elif " GB" in size_raw:
|
||||
size = float(size_raw.replace(" GB", "")) * 1024 ** 3
|
||||
elif " TB" in size_raw:
|
||||
size = float(size_raw.replace(" TB", "")) * 1024 ** 4
|
||||
else:
|
||||
size = 500 * 1024 * 1024
|
||||
# Try to click Continue if it appears
|
||||
try:
|
||||
continue_button = WebDriverWait(driver, 4).until(
|
||||
EC.element_to_be_clickable((By.XPATH, "//input[@value='Continue']"))
|
||||
)
|
||||
continue_button.click()
|
||||
except:
|
||||
pass
|
||||
# Monitor current download
|
||||
argmonitor = monitor
|
||||
argwait = wait_time
|
||||
ctime = 0
|
||||
csize = 0
|
||||
if size < 33554432:
|
||||
monitor = True
|
||||
wait_time = 17
|
||||
logging.warning("File size was too small to monitor! (Less than 32MB)")
|
||||
if monitor == False:
|
||||
tempTime = time.time()
|
||||
file_pattern = download_dir + '/*.crdownload'
|
||||
while not os.path.exists(wait_for_file(file_pattern)):
|
||||
if time.time() - tempTime > wait_time:
|
||||
logging.warning("Never found temp file for download...")
|
||||
break
|
||||
else:
|
||||
logging.warning(f"{time.time() - tempTime}")
|
||||
time.sleep(refresh_rate)
|
||||
if monitor == False:
|
||||
csize, ctime = monitor_download(download_dir, size+1024)
|
||||
total_size += csize
|
||||
total_time += ctime
|
||||
# Wait to call next URL
|
||||
time.sleep(wait_time)
|
||||
cur_url += 1
|
||||
monitor = argmonitor
|
||||
wait_time = argwait
|
||||
#disc_elements = driver.find_elements(By.CSS_SELECTOR, "select[id^='disc_number']")
|
||||
disc_element = driver.find_element(By.ID, "disc_number")
|
||||
disc_select = Select(disc_element)
|
||||
url_length += len(disc_select.options) - 1
|
||||
for option in disc_select.options:
|
||||
disc_value = option.get_attribute("value")
|
||||
disc_text = option.get_attribute("text")
|
||||
disc_select.select_by_value(disc_value)
|
||||
title = driver.title.replace("The Vault: ", f"{cur_url}/{url_length}: ({disc_text}) ")
|
||||
osize, otime = download_it(monitor, wait_time, download_dir, total_size, total_time, cur_url, title, url, disc_value)
|
||||
total_size += osize
|
||||
total_time += otime
|
||||
cur_url += 1
|
||||
except NoSuchElementException:
|
||||
title = driver.title.replace("The Vault: ", f"{cur_url}/{url_length}: ")
|
||||
osize, otime = download_it(monitor, wait_time, download_dir, total_size, total_time, cur_url, title, url)
|
||||
total_size += osize
|
||||
total_time += otime
|
||||
cur_url += 1
|
||||
# Close Chrome session
|
||||
driver.quit()
|
||||
if total_size > 0:
|
||||
@@ -266,6 +289,7 @@ def main():
|
||||
if failed_urls:
|
||||
# Add URL links to a file
|
||||
fn = f"failed_downloads.txt"
|
||||
logging.warning(f"Writing failed downloads to {fn}")
|
||||
fnt = 'a'
|
||||
with open(f"{fn}", fnt) as f:
|
||||
for item in failed_urls:
|
||||
|
||||
Reference in New Issue
Block a user