Skip to content

Commit

Permalink
added error handling for exceptions raised during download
Browse files Browse the repository at this point in the history
  • Loading branch information
9FS committed Sep 26, 2023
1 parent 53d515a commit a964825
Show file tree
Hide file tree
Showing 5 changed files with 39 additions and 45 deletions.
8 changes: 4 additions & 4 deletions poetry.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ license = "MIT"
name = "x" # can't leave empty because of bug with `poetry install` from poetry.lock file
readme = "readme.md"
repository = "https://github.com/9-FS/2021-11-15-nHentai-to-PDF"
version = "1.0.1"
version = "1.0.3"

[tool.poetry.dependencies]
kfsconfig = "^1.0.0"
kfsfstr = "^1.0.0"
kfslog = "^1.0.0"
kfsmedia = "^2.0.0"
kfsmedia = "^2.1.0"
python = "^3.11.0"

[build-system]
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ kfsconfig==1.0.2 ; python_full_version >= "3.11.0" and python_full_version < "4.
kfsfstr==1.0.2 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
kfslog==1.0.1 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
kfsmath==1.0.1 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
kfsmedia==2.0.3 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
kfsmedia==2.1.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
pillow==10.0.1 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
requests==2.31.0 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
urllib3==2.0.5 ; python_full_version >= "3.11.0" and python_full_version < "4.0.0"
67 changes: 30 additions & 37 deletions src/Hentai.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,27 +95,24 @@ def _increment_fails(self, image_list: list[str]) -> None:
Takes list of filepaths that could not be downloaded or converted and increments appropiate failure counter.
"""

# re_match_1: re.Match|None
re_match_2: re.Match|None


for image in image_list:
# re_match_1=re.match("https://nhentai.net/g/[0-9]+/(?P<page_no>[0-9]+)/", image) # parse page number in URL
re_match_2=re.match("[0-9]+-(?P<page_no>[0-9]+)", os.path.splitext(os.path.basename(image))[0]) # parse page number in filepath

# if re_match_1!=None:
# self._fails[int(re_match_1.groupdict()["page_no"])-1]+=1 # increment appropiate counter
# if 10<=self._fails[int(re_match_1.groupdict()["page_no"])-1]: # if any counter 10 or above: give hentai up
# self._give_up=True

if re_match_2!=None:
self._fails[int(re_match_2.groupdict()["page_no"])-1]+=1 # increment appropiate counter
if 10<=self._fails[int(re_match_2.groupdict()["page_no"])-1]: # if any counter 10 or above: give hentai up
self._give_up=True

else: # if naming unexpected: skip
logging.error(f"Incrementing fails counter of \"{image}\" failed.")
continue
PATTERNS: list[str]=[
r"^((?P<page_no>[0-9]+)\.(jpg|png))$", # page URL pattern
r"^([0-9]+-(?P<page_no>[0-9]+)\.(jpg|png))$", # image filepath pattern
]
re_match: re.Match|None


for image in image_list: # for each image:
for pattern in PATTERNS: # with each pattern:
re_match=re.search(pattern, image.split("/")[-1]) # try to parse page number, use only filename not path
if re_match!=None: # if page number could be parsed:
self._fails[int(re_match.groupdict()["page_no"])-1]+=1 # increment appropiate fails counter
if 10<=self._fails[int(re_match.groupdict()["page_no"])-1]: # if any counter 10 or above:
self._give_up=True # give hentai up
break
else: # if page number can't be parsed:
logging.critical(f"Incrementing fails counter of \"{image}\" failed.") # don't know which counter to increment, critical error because should not happen
raise RuntimeError(f"Error in {self._increment_fails.__name__}{inspect.signature(self._increment_fails)}: Incrementing fails counter of \"{image}\" failed.")

return

Expand Down Expand Up @@ -156,23 +153,27 @@ def download(self) -> None:
raise FileExistsError(f"File \"{PDF_filepath}\" already exists. Skipped download.") # raise exception to skip upload in main
if os.path.isdir(PDF_filepath)==True: # if PDF already exists as directory: skip download, append to failures
logging.error(f"\"{PDF_filepath}\" already exists as directory. Skipped download.")
raise self.DownloadError(f"Error in {self.download.__name__}{inspect.signature(self.download)}: \"{PDF_filepath}\" already exists as directory. Skipped download.")
raise KFSmedia.DownloadError(f"Error in {self.download.__name__}{inspect.signature(self.download)}: \"{PDF_filepath}\" already exists as directory. Skipped download.")


while self._give_up==False: # while not giving up: try to download and convert
KFSmedia.download_medias(pages_URL, images_filepath) # download images # type:ignore

while self._give_up==False: # while not giving up: try to download and convert
try:
KFSmedia.download_medias(pages_URL, images_filepath) # download images # type:ignore
except KFSmedia.DownloadError as e:
self._increment_fails(e.args[0]) # increment fails, may trigger giving up
continue

try:
KFSmedia.convert_images_to_PDF(images_filepath, PDF_filepath) # convert images to PDF
except KFSmedia.ConversionError as e:
self._increment_fails(e.args[0]) # increment conversion fails, may trigger giving up
self._increment_fails(e.args[0]) # increment fails, may trigger giving up
continue
else: # if conversion successful:
self.PDF_filepath=PDF_filepath # save PDF filepath
break # break out
else: # if giving up:
logging.error(f"Tried to convert hentai \"{self}\" several times, but failed. Giving up.")
raise self.DownloadError(f"Error in {self.download.__name__}{inspect.signature(self.download)}: Tried to convert hentai \"{self}\" several times, but failed. Giving up.")
logging.error(f"Tried to download and convert hentai \"{self}\" several times, but failed. Giving up.")
raise KFSmedia.DownloadError(f"Error in {self.download.__name__}{inspect.signature(self.download)}: Tried to download and convert hentai \"{self}\" several times, but failed. Giving up.")


if os.path.isdir(f"./hentai/{self.ID}") and len(os.listdir(f"./hentai/{self.ID}"))==0: # if cache folder still exists and is empty:
Expand All @@ -181,12 +182,4 @@ def download(self) -> None:
except PermissionError: # may fail if another process is still using directory like dropbox
pass # don't warn because will be retried in main

return


class DownloadError(Exception):
"""
Raised when self.download(...) fails.
"""

pass
return
3 changes: 2 additions & 1 deletion src/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from KFSconfig import KFSconfig
from KFSfstr import KFSfstr
from KFSlog import KFSlog
from KFSmedia import KFSmedia
import logging
import os
from get_hentai_ID_list import get_hentai_ID_list
Expand Down Expand Up @@ -48,7 +49,7 @@ def main():
hentai.download() # download hentai
except FileExistsError: # if hentai already exists:
continue # skip to next hentai
except Hentai.DownloadError:
except KFSmedia.DownloadError:
with open("FAILURES.txt", "at") as fails_file: # append in failure file
fails_file.write(f"{hentai.ID}\n")
logging.info("--------------------------------------------------")
Expand Down

0 comments on commit a964825

Please sign in to comment.