diff --git a/src/backend/database.py b/src/backend/database.py index bc15c38..1ef0477 100644 --- a/src/backend/database.py +++ b/src/backend/database.py @@ -280,6 +280,21 @@ class Database: conn.commit() self.close_connection(conn) + def getWebADISAuth(self) -> Tuple[str, str]: + """ + Get the WebADIS authentication data from the database + + Returns: + Tuple[str, str]: The username and password for WebADIS + """ + result = self.query_db( + "SELECT username, password FROM webadis_login WHERE effective_range='SAP'", + one=True, + ) + if result is None: + return ("", "") + return (result[0], result[1]) + @log.catch def query_db( self, diff --git a/src/backend/migrations/V002__create_table_webadis_login.sql b/src/backend/migrations/V002__create_table_webadis_login.sql new file mode 100644 index 0000000..5e1b3a8 --- /dev/null +++ b/src/backend/migrations/V002__create_table_webadis_login.sql @@ -0,0 +1,10 @@ +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS webadis_login ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + username TEXT NOT NULL, + password TEXT NOT NULL + ); + +COMMIT; + diff --git a/src/backend/migrations/V003_update_webadis_add_user_area.sql b/src/backend/migrations/V003_update_webadis_add_user_area.sql new file mode 100644 index 0000000..1b5567b --- /dev/null +++ b/src/backend/migrations/V003_update_webadis_add_user_area.sql @@ -0,0 +1,6 @@ +BEGIN TRANSACTION; + +ALTER TABLE webadis_login +ADD COLUMN effective_range TEXT; + +COMMIT; \ No newline at end of file diff --git a/src/backend/threads_availchecker.py b/src/backend/threads_availchecker.py index 669af02..451df6e 100644 --- a/src/backend/threads_availchecker.py +++ b/src/backend/threads_availchecker.py @@ -3,7 +3,8 @@ from PySide6.QtCore import QThread from PySide6.QtCore import Signal as Signal from src.backend.database import Database -from src.logic.webrequest import BibTextTransformer, WebRequest +from src.backend.webadis import get_book_medianr +from src.logic.webrequest import BibTextTransformer, TransformerType, WebRequest from src.shared.logging import log @@ -37,7 +38,7 @@ class AvailChecker(QThread): ) # Pre-create reusable request and transformer to avoid per-item overhead self._request = WebRequest().set_apparat(self.appnumber) - self._rds_transformer = BibTextTransformer("RDS") + self._rds_transformer = BibTextTransformer(TransformerType.RDS) def run(self): self.db = Database() @@ -65,6 +66,12 @@ class AvailChecker(QThread): break log.info(f"State of {link}: " + str(state)) # #print("Updating availability of " + str(book_id) + " to " + str(state)) + # use get_book_medianr to update the medianr of the book in the database + auth = self.db.getWebADISAuth + medianr = get_book_medianr(rds.items[0].callnumber, self.appnumber, auth) + book_data = book["bookdata"] + book_data.medianr = medianr + self.db.updateBookdata(book["id"], book_data) self.db.setAvailability(book_id, state) count += 1 self.updateProgress.emit(count, len(self.links)) diff --git a/src/backend/webadis.py b/src/backend/webadis.py new file mode 100644 index 0000000..af79650 --- /dev/null +++ b/src/backend/webadis.py @@ -0,0 +1,35 @@ +from playwright.sync_api import sync_playwright + + +def get_book_medianr(signature: str, semesterapparat_nr: int, auth: tuple) -> str: + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=True) + context = browser.new_context() + page = context.new_page() + page.goto( + "https://bsz.ibs-bw.de:22998/aDISWeb/app?service=direct/0/Home/$DirectLink&sp=SDAP42" + ) + page.get_by_role("textbox", name="Benutzer").fill(auth[0]) + page.get_by_role("textbox", name="Benutzer").press("Tab") + page.get_by_role("textbox", name="Kennwort").fill(auth[1]) + page.get_by_role("textbox", name="Kennwort").press("Enter") + page.get_by_role("button", name="Katalog").click() + page.get_by_role("textbox", name="Signatur").click() + page.get_by_role("textbox", name="Signatur").fill(signature) + page.get_by_role("textbox", name="Signatur").press("Enter") + book_list = page.locator("iframe").content_frame.get_by_role( + "cell", name="Bibliothek der Pädagogischen" + ) + # this will always find one result, we need to split the resulting text based on the entries that start with "* " + book_entries = book_list.inner_text().split("\n") + books = [] + for entry in book_entries: + if entry.startswith("* "): + books.append(entry) + for book in books: + if f"Semesterapparat: {semesterapparat_nr}" in book: + return book.split("* ")[1].split(":")[0] + + # --------------------- + context.close() + browser.close() diff --git a/src/logic/dataclass.py b/src/logic/dataclass.py index ffe22f0..a7d4688 100644 --- a/src/logic/dataclass.py +++ b/src/logic/dataclass.py @@ -80,6 +80,7 @@ class BookData: old_book: Any | None = None media_type: str | None = None # in_library: bool | None = None # whether the book is in the library or not + medianr: int | None = None # Media number in the library system def __post_init__(self): self.library_location = ( diff --git a/src/logic/webrequest.py b/src/logic/webrequest.py index acd93b4..893e994 100644 --- a/src/logic/webrequest.py +++ b/src/logic/webrequest.py @@ -1,3 +1,4 @@ +from enum import Enum from typing import Any, Optional, Union import requests @@ -33,6 +34,14 @@ RATE_LIMIT = 20 RATE_PERIOD = 30 +class TransformerType(Enum): + ARRAY = "ARRAY" + COinS = "COinS" + BibTeX = "BibTeX" + RIS = "RIS" + RDS = "RDS" + + class WebRequest: def __init__(self) -> None: """Request data from the web, and format it depending on the mode.""" @@ -185,10 +194,16 @@ class BibTextTransformer: ValueError: Raised if mode is not in valid_modes """ - valid_modes = ["ARRAY", "COinS", "BibTeX", "RIS", "RDS"] + valid_modes = [ + TransformerType.ARRAY, + TransformerType.COinS, + TransformerType.BibTeX, + TransformerType.RIS, + TransformerType.RDS, + ] - def __init__(self, mode: str = "ARRAY") -> None: - self.mode = mode + def __init__(self, mode: TransformerType = TransformerType.ARRAY) -> None: + self.mode = mode.value self.field = None self.signature = None if mode not in self.valid_modes: diff --git a/src/transformers/transformers.py b/src/transformers/transformers.py index a7ef88a..ade70b7 100644 --- a/src/transformers/transformers.py +++ b/src/transformers/transformers.py @@ -2,14 +2,15 @@ from __future__ import annotations import json import re +import sys from dataclasses import dataclass from dataclasses import field as dataclass_field from typing import Any, List +import loguru + from src import LOG_DIR from src.logic.dataclass import BookData -import loguru -import sys log = loguru.logger log.remove() @@ -36,6 +37,7 @@ class Item: department: str | None = dataclass_field(default_factory=str) locationhref: str | None = dataclass_field(default_factory=str) location: str | None = dataclass_field(default_factory=str) + ktrl_nr: str | None = dataclass_field(default_factory=str) def from_dict(self, data: dict): """Import data from dict""" @@ -382,6 +384,8 @@ class RDSData: def transform(self, data: str): # rds_availability = RDS_AVAIL_DATA() # rds_data = RDS_GENERIC_DATA() + print(data) + def __get_raw_data(data: str) -> list: # create base data to be turned into pydantic classes data = data.split("RDS ----------------------------------")[1] diff --git a/src/ui/widgets/signature_update.py b/src/ui/widgets/signature_update.py index 056540b..b0b49fe 100644 --- a/src/ui/widgets/signature_update.py +++ b/src/ui/widgets/signature_update.py @@ -1,14 +1,133 @@ +import os +import time +from concurrent.futures import ThreadPoolExecutor +from queue import Empty, Queue + from PySide6 import QtCore, QtWidgets from PySide6.QtMultimedia import QAudioOutput, QMediaPlayer from src.backend.catalogue import Catalogue from src.backend.database import Database +from src.backend.webadis import get_book_medianr from src.logic.SRU import SWB from src.shared.logging import log from .widget_sources.admin_update_signatures_ui import Ui_Dialog +class MedianrThread(QtCore.QThread): + progress = QtCore.Signal(int) + currtot = QtCore.Signal(int, int) + + def __init__(self, books=None, thread_count=6): + super().__init__() + self.books = books or [] + self.total = 0 + # Database instances are not always thread-safe across threads; create one per worker. + # Keep a shared auth that is safe to read. + db_main = Database() + self.auth = db_main.getWebADISAuth() + if self.auth == ("", ""): + raise ValueError("WebADIS authentication not set in database.") + self.thread_count = max(1, thread_count) + self._stop_requested = False + + def run(self): + # Use ThreadPoolExecutor to parallelize I/O-bound tasks. + total_books = len(self.books) + if total_books == 0: + log.debug("MedianrThread: no books to process") + return + + chunk_size = (total_books + self.thread_count - 1) // self.thread_count + chunks = [ + self.books[i : i + chunk_size] for i in range(0, total_books, chunk_size) + ] + + # queue for worker results and progress + q = Queue() + + def medianrworker(chunk: list[dict]): + # Each worker creates its own Database instance for thread-safety + db = Database() + for book in chunk: + if self._stop_requested: + break + try: + booknr = get_book_medianr( + book["bookdata"].signature, + db.getApparatNrByBookId(book["id"]), + self.auth, + ) + log.debug( + f"Book ID {book['id']} - Signature {book['bookdata'].signature} - Medianr {booknr}" + ) + book_data = book["bookdata"] + book_data.medianr = booknr + db.updateBookdata(book["id"], book_data) + q.put(("progress", 1)) + except Exception as e: + log.error(f"Medianr worker error for book {book}: {e}") + q.put(("progress", 1)) + time.sleep(10) + q.put(("done", None)) + + processed = 0 + finished_workers = 0 + + with ThreadPoolExecutor(max_workers=len(chunks)) as ex: + futures = [ex.submit(medianrworker, ch) for ch in chunks] + + log.info( + f"Launched {len(futures)} worker thread(s) for {total_books} entries: {[len(ch) for ch in chunks]} entries per thread." + ) + + # aggregate progress + while finished_workers < len(chunks): + try: + kind, payload = q.get(timeout=0.1) + except Empty: + continue + + if kind == "progress": + processed += int(payload) + self.progress.emit(processed) + # emit currtot with processed and current chunk total as best-effort + self.currtot.emit(processed, total_books) + elif kind == "done": + finished_workers += 1 + + # ensure final progress reached total_books + self.progress.emit(total_books) + self.currtot.emit(total_books, total_books) + + def stop(self): + """Request the thread to stop early.""" + self._stop_requested = True + + def process_chunk(self, books_chunk): + # kept for backward compatibility but not used by run(); still usable externally + db = Database() + for index, book in enumerate(books_chunk): + try: + booknr = get_book_medianr( + book["bookdata"].signature, + db.getApparatNrByBookId(book["id"]), + self.auth, + ) + log.debug( + f"Book ID {book['id']} - Signature {book['bookdata'].signature} - Medianr {booknr}" + ) + book_data = book["bookdata"] + book_data.medianr = booknr + db.updateBookdata(book["id"], book_data) + except Exception as e: + log.error(f"Medianr process_chunk error for book {book}: {e}") + self.progress.emit(index + 1) + self.currtot.emit(self.total + 1, len(books_chunk)) + self.total += 1 + + class UpdaterThread(QtCore.QThread): progress = QtCore.Signal(int) currtot = QtCore.Signal(int, int) @@ -98,6 +217,8 @@ class UpdateSignatures(QtWidgets.QDialog, Ui_Dialog): self.catalogue = Catalogue() self.player = QMediaPlayer() self.audio_output = QAudioOutput() + self.spin_thread_count.setMaximum(os.cpu_count()) + self.btn_add_medianr.clicked.connect(self.add_medianr) def play_sound(self, sound_file: str): self.player.setAudioOutput(self.audio_output) @@ -114,6 +235,16 @@ class UpdateSignatures(QtWidgets.QDialog, Ui_Dialog): self.updater.finished.connect(self.updater.deleteLater) self.updater.start() + def add_medianr(self): + books = self.db.getAllBooks() + books = [book for book in books if book["bookdata"].medianr is None] + total_books = len(books) + self.progressBar.setMaximum(total_books) + self.medianr_thread = MedianrThread(books, self.spin_thread_count.value()) + self.medianr_thread.progress.connect(self.update_progress) + self.medianr_thread.finished.connect(self.medianr_thread.deleteLater) + self.medianr_thread.start() + def add_missing(self): books = self.db.getAllBooks() total_books = len(books) diff --git a/src/ui/widgets/widget_sources/admin_update_signatures.ui b/src/ui/widgets/widget_sources/admin_update_signatures.ui index 15afca4..8541410 100644 --- a/src/ui/widgets/widget_sources/admin_update_signatures.ui +++ b/src/ui/widgets/widget_sources/admin_update_signatures.ui @@ -30,6 +30,47 @@ + + + + + + Anzahl Parraleler Aktionen + + + + + + + Mediennummern ergänzen + + + + + + + 1 + + + 6 + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + diff --git a/src/ui/widgets/widget_sources/admin_update_signatures_ui.py b/src/ui/widgets/widget_sources/admin_update_signatures_ui.py index 25101c6..6ce7841 100644 --- a/src/ui/widgets/widget_sources/admin_update_signatures_ui.py +++ b/src/ui/widgets/widget_sources/admin_update_signatures_ui.py @@ -15,8 +15,9 @@ from PySide6.QtGui import (QBrush, QColor, QConicalGradient, QCursor, QFont, QFontDatabase, QGradient, QIcon, QImage, QKeySequence, QLinearGradient, QPainter, QPalette, QPixmap, QRadialGradient, QTransform) -from PySide6.QtWidgets import (QApplication, QDialog, QFormLayout, QProgressBar, - QPushButton, QSizePolicy, QVBoxLayout, QWidget) +from PySide6.QtWidgets import (QApplication, QDialog, QFormLayout, QLabel, + QProgressBar, QPushButton, QSizePolicy, QSpacerItem, + QSpinBox, QVBoxLayout, QWidget) class Ui_Dialog(object): def setupUi(self, Dialog): @@ -37,6 +38,32 @@ class Ui_Dialog(object): self.formLayout.setWidget(0, QFormLayout.ItemRole.FieldRole, self.btn_add_missing_data) + self.formLayout_2 = QFormLayout() + self.formLayout_2.setObjectName(u"formLayout_2") + self.label = QLabel(Dialog) + self.label.setObjectName(u"label") + + self.formLayout_2.setWidget(0, QFormLayout.ItemRole.LabelRole, self.label) + + self.btn_add_medianr = QPushButton(Dialog) + self.btn_add_medianr.setObjectName(u"btn_add_medianr") + + self.formLayout_2.setWidget(1, QFormLayout.ItemRole.FieldRole, self.btn_add_medianr) + + self.spin_thread_count = QSpinBox(Dialog) + self.spin_thread_count.setObjectName(u"spin_thread_count") + self.spin_thread_count.setMinimum(1) + self.spin_thread_count.setValue(6) + + self.formLayout_2.setWidget(0, QFormLayout.ItemRole.FieldRole, self.spin_thread_count) + + + self.formLayout.setLayout(1, QFormLayout.ItemRole.FieldRole, self.formLayout_2) + + self.verticalSpacer = QSpacerItem(20, 40, QSizePolicy.Policy.Minimum, QSizePolicy.Policy.Expanding) + + self.formLayout.setItem(2, QFormLayout.ItemRole.FieldRole, self.verticalSpacer) + self.verticalLayout.addLayout(self.formLayout) @@ -56,5 +83,7 @@ class Ui_Dialog(object): Dialog.setWindowTitle(QCoreApplication.translate("Dialog", u"Dialog", None)) self.btn_update_signatures.setText(QCoreApplication.translate("Dialog", u"Signaturen aktualisieren", None)) self.btn_add_missing_data.setText(QCoreApplication.translate("Dialog", u"Fehlende Daten hinzuf\u00fcgen", None)) + self.label.setText(QCoreApplication.translate("Dialog", u"Anzahl Parraleler Aktionen", None)) + self.btn_add_medianr.setText(QCoreApplication.translate("Dialog", u"Mediennummern erg\u00e4nzen", None)) # retranslateUi diff --git a/tests/test_migrations_runner.py b/tests/test_migrations_runner.py new file mode 100644 index 0000000..c86561c --- /dev/null +++ b/tests/test_migrations_runner.py @@ -0,0 +1,20 @@ +import sqlite3 as sql +from pathlib import Path + +from src.backend.database import Database + +p = Path("devtests_test_migrations.db") +if p.exists(): + p.unlink() + +print("Creating Database at", p) +db = Database(db_path=p) + +conn = sql.connect(p) +c = conn.cursor() +c.execute("SELECT name FROM sqlite_master WHERE type='table'") +print("Tables:", sorted([r[0] for r in c.fetchall()])) + +c.execute("SELECT id, applied_at FROM schema_migrations") +print("Migrations applied:", c.fetchall()) +conn.close()