From 85e2c85acae5b9e0ac2e1f6f77f911709acdddee Mon Sep 17 00:00:00 2001 From: WorldTeacher Date: Wed, 30 Jul 2025 16:47:40 +0200 Subject: [PATCH] replaced serie.name with serie.metadata.title to use actual series name instead of local folder name, various smaller changes --- src/logic/{cli.py => search.py} | 476 ++++++++++++++++++++------------ 1 file changed, 299 insertions(+), 177 deletions(-) rename src/logic/{cli.py => search.py} (54%) diff --git a/src/logic/cli.py b/src/logic/search.py similarity index 54% rename from src/logic/cli.py rename to src/logic/search.py index 84d03ba..0316d9e 100644 --- a/src/logic/cli.py +++ b/src/logic/search.py @@ -4,7 +4,6 @@ import shutil import time import jaro from src.data.komga import KomgaAPI -from komgapi import komgapi from komgapi.schemas.Series import Series from src.data.Feeds.nyaasi import NyaaFeed from komsuite_nyaapy import Torrent @@ -31,10 +30,14 @@ from src.logic.db_schemas import ( LASTCHECKED_KOMGRABBER, GET_LASTCHECKED_KOMGRABBER, ) +from src.logic.db_schemas_mariadb import ( + MARIADB_KOMGRABBER_TABLE +) import loguru from pathlib import Path from alive_progress import alive_it -from typing import Any +from typing import Any, Optional +from anilistapi import AnilistAPI config = KomConfig() @@ -49,6 +52,7 @@ Komga = KomgaAPI() LINE_CLEAR = "\x1b[2K" failed_items: list[str] = [] +incomplete: list[str] = [] class mangaCli: @@ -68,11 +72,9 @@ class mangaCli: self.series_data: Series self.volumes = [] self.download_path = config.komgrabber.download_location - self.cache = KomCache | None - if config.komgrabber.use_cache: - self.cache = KomCache() - self.cache.create_table(KOMGRABBER_TABLE) - # self.allSeries = Komga.getAllSeries() + self.library_id = library_id + self.cache = KomCache() + pass def download(self, feed_url: str): @@ -110,6 +112,7 @@ class mangaCli: # print(f"Filename: {file}") file_move = False + new_folder = None if file.endswith(".cbz") or file.endswith(".cbr"): new_folder = Path(self.download_path, self.serie) os.makedirs(new_folder, exist_ok=True) @@ -134,9 +137,14 @@ class mangaCli: # if progress remains the same for 30 seconds, stop the download progress = self.dl.check_progress() - time.sleep(45) + time.sleep(90) n_progress = self.dl.check_progress() - dl_name = self.dl.api.get_downloads()[0].name + try: + dl_name = self.dl.api.get_downloads()[0].name + except IndexError: + log.error("No downloads found, skipping...") + return False + if not folder_similarity(self.serie.lower(), dl_name.lower()) > 0.8: log.error( f"Folder name {dl_name} does not match {self.serie}, skipping download" @@ -144,63 +152,14 @@ class mangaCli: self.dl.api.get_downloads()[0].remove(force=True) dl_complete = False break - if not check_done: - local_files = os.listdir(f"{self.download_path}") - for f in local_files: - # print(f) - if os.path.isdir(f"{self.download_path}/{f}"): - local_files.extend( - [ - f"{self.download_path}/{f}/{file}" - for file in os.listdir(f"{self.download_path}/{f}") - ] - ) - local_files = [ - file - for file in local_files - if file.endswith(".cbz") or file.endswith(".cbr") - ] - local_volumes = Komga.getVolumes(self.series_data.id) - # if not local_files: - # dl_complete=False - # break - local_files_volumes = [] - for file in local_files: - vol_regex = r"(v\d{1,3}(-\d{1,3})?)|(Vol\. \d{1,3})" - # if the file does not match the naming convention, skip it - if re.search(vol_regex, file): - match = re.search(vol_regex, file) - if match: - vol = match.group(0).replace("v", "").replace("Vol. ", "") - if "-" in vol: - local_files_volumes.extend( - [int(volume) for volume in vol.split("-")] - ) - continue - vol = int(vol) - local_files_volumes.append(vol) - log.info( - "Grabbed volumes: {}, Komga volumes: {}".format( - sorted(local_files_volumes), local_volumes - ) - ) - if local_files_volumes == []: - pass - # check íf any local_file_volumes are not in local_volumes - if all([vol in local_volumes for vol in local_files_volumes]): - log.info("all volumes downloaded, stopping...") - dl_complete = False - break - else: - log.info("not all volumes downloaded, continuing...") - check_done = True if progress == n_progress: log.debug( "Progress has not changed for 30 seconds, stopping the download" ) self.dl.api.get_downloads()[0].remove(force=True) dl_complete = False + incomplete.append(dl_name) break else: pass @@ -244,14 +203,28 @@ class mangaCli: - list[dict]: a list of dictionaries containing the entries to download """ serie = data.name + series_id = data.id - vols = Komga.getVolumes(series_id=series_id, unpaged=True) + + vols = ( + Komga.getVolumes(series_id=series_id, unpaged=True) + if series_id is not None + else [] + ) feed_titles = NyaaFeed().search(serie) f_d = [] if feed_titles == []: failed_items.append(serie) added_max_vols = vols if vols else [0] for entry in feed_titles: + if not any( + filetype in entry.filetypes + for filetype in config.komgrabber.manga.valid_file_extensions + ): + log.info( + f"Skipping {entry.name}, Reason: Filetype not in valid filetypes, wanted: {config.komgrabber.manga.valid_file_extensions}, found: {entry.filetypes}" + ) + continue if entry.seeders > 0: if ( serie.lower() in entry.name.lower() @@ -275,16 +248,22 @@ class mangaCli: # return entry with the most volumes return f_d - def media_grabber(self, serie: Series, bar: Any = None) -> bool: + def media_grabber(self, serie: Series, bar: Optional[Any] = None) -> bool: result = self.process_serie(serie) total_new_volumes: list[tuple[Torrent, list[int]]] = [] fs_per_volume = config.komgrabber.manga.min_filesize - series_volumes = Komga.getVolumes(series_id=serie.id, unpaged=True) + series_volumes = ( + Komga.getVolumes(series_id=serie.id, unpaged=True) + if serie.id is not None + else [0] + ) + max_new_volume: int = 0 if result is None or result == []: - log.info(f"Could not find any new volumes for {serie.name}") + log.info(f"Could not find any new volumes for {serie.metadata.title}") return False - bar.text(f"Downloading new volumes for {serie.name}...") + if bar: + bar.text(f"Downloading new volumes for {serie.metadata.title}...") for res in result: log.info(f"{res.name}, Volumes: {res.volumes}") if res.volumes != [0]: @@ -308,78 +287,127 @@ class mangaCli: total_new_volumes = sorted( total_new_volumes, key=lambda x: len(x[1]), reverse=True ) - res = total_new_volumes[0][0] - log.info(f"Found {len(total_new_volumes[0][1])} new entries for {serie.name}") + log.info(f"Found {len(total_new_volumes)} new results for {serie.name}") + for res, new_volumes in total_new_volumes: + if "epub" in res.filetypes and len(res.filetypes) == 1: + log.info( + f"Skipping {res.name}, Reason: Epub file, no other filetypes present" + ) + continue + if ( + max(new_volumes) > max(series_volumes) + and max(new_volumes) > max_new_volume + ): + max_new_volume = max(new_volumes) + log.info( + "Found new volumes: {} for series: {}, downloading".format( + new_volumes, serie.name + ) + ) - # log.info( - # f"Found {len(new_volumes)} new {'volume' if len(new_volumes) == 1 else 'volumes'} for {serie.name}" - # ) - # # check if the new volumes were aleady downloaded - # log.info(f"current volumes: {series_volumes}, new volumes: {new_volumes}") - # # print(result) + # log.info( + # f"Found {len(new_volumes)} new {'volume' if len(new_volumes) == 1 else 'volumes'} for {serie.name}" + # ) + # # check if the new volumes were aleady downloaded + # log.info(f"current volumes: {series_volumes}, new volumes: {new_volumes}") + # # print(result) - if self.download(res.download_url) is True: - log.success(f"Downloaded {res.name}") - # self.rename_folder_and_files(self.file, komga_data=serie, remove=True) - # self.move_to_komga(serie=entry) - log.info("Renaming and tagging files") - rename() - if not config.komgrabber.get_chapters: - detect_chapters() - tag_folder() - if rename_folder(series=serie): - move(self.download_path, config.komga.media_path) - else: - log.info("Seems like we grabbed the wrong series, oops") - failed_items.append(serie.name) - # clear folder - # remove the download dir and create it anew - remove_empty_folders(self.download_path) - safe_remove_directory(self.download_path) + if self.download(res.download_url) is True: + log.success(f"Downloaded {res.name}") + # self.rename_folder_and_files(self.file, komga_data=serie, remove=True) + # self.move_to_komga(serie=entry) + log.info("Renaming and tagging files") + rename() + if not config.komgrabber.get_chapters: + detect_chapters() + tag_folder() + if rename_folder(series=serie): + move(self.download_path, config.komga.media_path) + else: + log.info("Seems like we grabbed the wrong series, oops") + failed_items.append(serie.metadata.title) + # clear folder + # remove the download dir and create it anew + remove_empty_folders(self.download_path) + safe_remove_directory(self.download_path) return True - def search_for_new_volumes(self): - cache_present = False - if self.cache: - cache_present = True - series = Komga.series_controller.getAllSeries( - body={ - "condition": { - "anyOf": [ - {"seriesStatus": {"operator": "is", "value": "ONGOING"}}, - {"seriesStatus": {"operator": "is", "value": "HIATUS"}}, - {"seriesStatus": {"operator": "is", "value": "ENDED"}}, - ] + def search_for_new_volumes(self, all: bool = False): + query = { + "condition": { + "allOf": [ + { + "anyOf": [ + { + "seriesStatus": { + "operator": "is", + "value": "ONGOING" + } + }, + { + "seriesStatus": { + "operator": "is", + "value": "HIATUS" + } + }, + { + "allOf": [ + { + "seriesStatus": { + "operator": "is", + "value": "ENDED" + } + }, + { + "complete": { + "operator": "isFalse" + } + } + ] + } + ] + } + ] + } +} + if self.library_id != "": + query["condition"]["allOf"].append( + { + "libraryId": { + "operator": "is", + "value": self.library_id, + } } - } + ) + series = Komga.series_controller.getAllSeries( + body= query ) + komga_series: list[Series] = [] shutil.rmtree(self.download_path, ignore_errors=True) - os.mkdir(self.download_path) - - log.info(f"{len(series)} series found") + # log.debug(f"Series: {series}") today = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) today = time.mktime(time.strptime(today, "%Y-%m-%d %H:%M:%S")) - def cache_ending(bar): - bar.title("Updating cache entries") - bar.text("Cache updated, continuing...") + def cache_ending(bar): # type:ignore + bar.title("Updating cache entries") # type:ignore + bar.text("Cache updated, continuing...") # type:ignore - def series_ending(bar): - bar.title("Completed searching for new volumes") - bar.text("All series checked, exiting...") + def series_ending(bar): # type:ignore + bar.title("Completed searching for new volumes") # type:ignore + bar.text("All series checked, exiting...") # type:ignore - def skip_ending(bar): - bar.title("Skipping series") - bar.text("Skipped series, continuing...") + def skip_ending(bar): # type:ignore + bar.title("Skipping series") # type:ignore + bar.text("Skipped series, continuing...") # type:ignore - def ended_ending(bar): - bar.title("Skipping finished series") - bar.text("Finished check, continuing to search new volumes...") + def ended_ending(bar): # type:ignore + bar.title("Skipping finished series") # type:ignore + bar.text("Finished check, continuing to search new volumes...") # type:ignore - if cache_present: + if config.komgrabber.use_cache: log.info("Cache present, checking for missing entries") cacheBar = alive_it( series, @@ -389,9 +417,9 @@ class mangaCli: receipt_text=True, ) for serie in cacheBar: - data = self.cache.query(SELECT_KOMGRABBER, (serie.id,)) + data = self.cache.fetch_one(SELECT_KOMGRABBER, {"series_id": serie.id}) log.debug( - f"Cache data: {data}, Serie: {serie.name}, Status: {serie.metadata.status}" + f"Cache data: {data}, Serie: {serie.metadata.title}, Status: {serie.metadata.status}" ) if data: if data[3] == serie.metadata.status: @@ -399,52 +427,69 @@ class mangaCli: elif data and data[3] != serie.metadata.status: self.cache.update( UPDATE_KOMGRABBER, - (serie.name, serie.metadata.status, serie.id), + { + "name": serie.metadata.title, + "status": serie.metadata.status, + "series_id": serie.id, + }, ) log.info(f"Serie {serie.name} updated") time.sleep(0.05) else: self.cache.insert( - INSERT_KOMGRABBER, (serie.name, serie.id, serie.metadata.status) + INSERT_KOMGRABBER, + { + "name": serie.name, + "series_id": serie.id, + "status": serie.metadata.status, + } ) - log.info(f"Serie {serie.name} added to cache") + log.info(f"Serie {serie.metadata.title} added to cache") log.debug("Cache created, added missing entries") time.sleep(0.5) - if cache_present: - skipBar = alive_it( - series, - bar="smooth", - spinner="dots", - receipt_text=True, - finalize=skip_ending, - ) - for serie in skipBar: - last_checked = self.cache.query( - GET_LASTCHECKED_KOMGRABBER, (serie.id,) - )[0] - # convert timestamp to epoch float for comparison - if last_checked: + if all is True: + log.info("Searching for all series in the database") + komga_series = series + else: + if config.komgrabber.use_cache: + skipBar = alive_it( + series, + bar="smooth", + spinner="dots", + receipt_text=True, + finalize=skip_ending, + ) + for serie in skipBar: + last_checked = self.cache.fetch_one( + GET_LASTCHECKED_KOMGRABBER, {"series_id": serie.id} + )[0] + log.debug( + f"Last checked: {last_checked}, Serie: {serie.name}, Status: {serie.metadata.status}" + ) + # convert timestamp to epoch float for comparison + + if last_checked == 0 or last_checked is None: + last_checked = "2024-01-01 00:00:00" last_checked = time.mktime( - time.strptime(last_checked, "%Y-%m-%d %H:%M:%S") + time.strptime(str(last_checked), "%Y-%m-%d %H:%M:%S") ) # if difference between last_checked and today is less than config.komgrabber.cache_check_interval, skip entry time_difference = time_checker(last_checked, today) - # if time difference is less than set in the settings and the series status is not ended and the book count is not the same as the total book count, skip the entry - if time_difference < config.komgrabber.cache_check_interval: - komga_series.append(serie) - log.debug(f"Added {serie.name} to the list") - if ( - serie.metadata.status == "ENDED" - and serie.booksCount == serie.metadata.totalBookCount - ): + # if time difference is less than set in the settings and the series status is not ended and the book count is not the same as the total book count, skip the entry + if time_difference >= config.komgrabber.cache_check_interval: + komga_series.append(serie) log.debug( - f"Serie {serie.name} if finished and has all volumes present, skipping..." + f"Added {serie.name} to the checking list, as the last check was {time_difference} days ago" ) else: - komga_series.append(serie) - time.sleep(0.005) + log.debug( + f"Skipped {serie.name} as the last check was {time_difference} days ago, whereas the set interval is {config.komgrabber.cache_check_interval} days" + ) + + time.sleep(0.005) + log.debug(len(komga_series)) log.info("Finished checking cache, continuing...") log.info("There are {} series to check".format(len(komga_series))) @@ -455,39 +500,104 @@ class mangaCli: title="Searching for new volumes", ) for serie in pBar: - pBar.text(f"Searching for new volumes for {serie.name}") + pBar.text(f"Searching for new volumes for {serie.metadata.title}") log.info( - f"searching for new volumes for {serie.name}, currently at {serie.booksCount} volumes" + f"searching for new volumes for {serie.metadata.title}, currently at {serie.booksCount} volumes" ) self.series_data = serie - self.serie = serie.name + self.serie = ( + serie.metadata.title + ) # replaced serie.name with serie.metadata.title self.serie_id = serie.id - self.media_grabber(serie, bar=pBar) - if cache_present: - self.cache.update(LASTCHECKED_KOMGRABBER, (serie.id,)) - time.sleep(5) - # print("done", serie.name) + found = self.media_grabber(serie, bar=pBar) + if config.komgrabber.use_cache: + if found: + self.cache.update(LASTCHECKED_KOMGRABBER, {"series_id": serie.id}) + log.info( + f"Cache updated for {serie.metadata.title}, new volumes found" + ) # updated to use serie.metadata.title + else: + self.cache.update(LASTCHECKED_KOMGRABBER, {"series_id": serie.id}) + + log.info( + f"Cache updated for {serie.metadata.title}, no new volumes found" + ) # updated to use serie.metadata.title + # self.cache.update(LASTCHECKED_KOMGRABBER, {"series_id": serie.id}) + # log.info("Cache updated") return self - def search_for_series(self, series: list[str]): - cache_present = False - if self.cache: - cache_present = True + def search_for_new_series(self, series_id: str) -> bool: + anilist = AnilistAPI() + series = anilist.get_manga(series_id) + if series is None: + log.error(f"Could not find series with id {series_id}") + return False + komga_results = Komga.series_controller.getAllSeries( + body={ + "condition": { + "anyOf": [ + { + "title": { + "operator": "contains", + "value": series.title.english + if series.title.english + else series.title.romaji, + } + }, + ] + } + } + ) + if not komga_results: + log.error(f"Could not find series with title {series.title.english}") + Komga_fake = Series( + name=series.title.english + if series.title.english + else series.title.romaji, + booksCount=0, + metadata={}, + booksMetadata={}, + ) + rbar = alive_it( + [Komga_fake], + title="Searching for new volumes", + bar="smooth", + spinner="dots", + receipt_text=True, + ) + for Komga_fake in rbar: + rbar.text(f"Searching for new volumes for {Komga_fake.name}") + log.info( + f"searching for new volumes for {Komga_fake.name}, currently at {Komga_fake.booksCount} volumes" + ) + self.serie = Komga_fake.name + result = self.media_grabber(Komga_fake, bar=rbar) + if result is False: + rbar.title("No new volumes found") + log.error(f"Could not find any new volumes for {Komga_fake.name}") + return False + + return False + else: + return True + + def search_for_series(self, serie: list[str]): + cache_present = config.komgrabber.use_cache shutil.rmtree(self.download_path, ignore_errors=True) os.mkdir(self.download_path) series_request = [] - for serie in series: + for series in serie: series_request.append( - {"title": {"operator": "is", "value": serie}}, + {"title": {"operator": "is", "value": series}}, ) request_body = {"condition": {"anyOf": series_request}} - series = Komga.series_controller.getAllSeries(body=request_body) + komga_series = Komga.series_controller.getAllSeries(body=request_body) def series_ending(bar): bar.title("Completed searching for new volumes") bar.text("All series checked, exiting...") - pBar = alive_it(series, finalize=series_ending) + pBar = alive_it(komga_series, finalize=series_ending) for serie in pBar: pBar.text(f"Searching for new volumes for {serie.name}") log.info( @@ -497,13 +607,12 @@ class mangaCli: self.serie = serie.name self.serie_id = serie.id - self.media_grabber(serie) + self.media_grabber(serie, bar=pBar) if cache_present: - self.cache.update(LASTCHECKED_KOMGRABBER, (serie.id,)) + self.cache.update(LASTCHECKED_KOMGRABBER, {"series_id": serie.id}) time.sleep(5) # print("done", serie.name) - return self pass @@ -530,15 +639,13 @@ def avail_check(): return (True, komga_avail) -def search_all(): - mangaCli().search_for_new_volumes() - komga = komgapi(config.komga.user, config.komga.password, config.komga.url) - libraries = komga.library_controller.getLibraries() - for library in libraries: - komga.library_controller.scanLibrary(library.id) - print(f"Initialized scan for library {library.name}") - # update_state() +def search_all(libraryName: str ="", all: bool = False): + libid = config.komga.libraries.get(libraryName) + mangaCli(library_id=libid).search_for_new_volumes(all) + Komga.library_controller.scanLibrary(libid) + print(f"Initialized scan for library {libraryName}") print("Failed series:\n", failed_items) + print("Incomplete series:\n", incomplete) def search_series(series: list[str]): @@ -547,5 +654,20 @@ def search_series(series: list[str]): print("Failed series:\n", failed_items) +def search_requested(): + cache = KomCache() + series = cache.query("SELECT manga_id from manga_requests WHERE grabbed = 0") + if series: + for serie in series: + result = mangaCli().search_for_new_series(int(serie[0])) + if result: + cache.update( + "UPDATE manga_requests SET grabbed = 1 WHERE manga_id = :manga_id", + {"manga_id": serie[0]}, + ) + else: + print("No series found to grab") + + if __name__ == "__main__": search_all()