Warning popup when encountering network errors

LinkGenerator now handles if the called DomainLinkGenerator children throw request's connection errors in scrape_tags() and informs the user via popup and cancels the scraping.
dev
Peery 2 years ago
parent c5a02b9736
commit 2c66002d64
Signed by: pandro
SSH Key Fingerprint: SHA256:iBUZSuDxqYr4hYpe9U3BA9NJmXKpbGt4H0S8hUwIbrA

@ -1,8 +1,9 @@
import re import logging
import requests import requests
import lxml.html
from urllib.parse import urlparse from urllib.parse import urlparse
from PyQt5.QtWidgets import QMessageBox
DOMAIN_UNKNOWN = -1 DOMAIN_UNKNOWN = -1
@ -43,6 +44,10 @@ class DomainIdentifier:
class DomainLinkGenerator: class DomainLinkGenerator:
"""
Base class for classes that generate a link to the file on their domain given a sample of a filename pattern and
implement a method to scrape the available metadata.
"""
def __init__(self, domain: DomainIdentifier): def __init__(self, domain: DomainIdentifier):
self.__identifier = domain self.__identifier = domain
@ -166,8 +171,16 @@ class LinkGenerator:
url_domain = urlparse(url).netloc url_domain = urlparse(url).netloc
for g in self.__link_generators: for g in self.__link_generators:
if g.get_identifier() == domain or g.get_domain_name() == url_domain: if g.get_identifier() == domain or g.get_domain_name() == url_domain:
try:
try: try:
return g.scrape_tags(url=url, headers=headers, file_name=file_name) return g.scrape_tags(url=url, headers=headers, file_name=file_name)
except requests.exceptions.ConnectionError as e:
logging.warning(f"Encountered connection error when trying to scrape tags from \"{url}\".\n"
f"See also:\n{e}")
QMessageBox.warning(None, "Connection Error",
f"The http connection to \"{url}\" ran into an error. Check your network.\n"
f"\n{e}")
break
except NotImplementedError: except NotImplementedError:
pass pass
return None return None

Loading…
Cancel
Save