rebase codebase, delete trunk, move threads to backend

This commit is contained in:
2025-01-14 16:20:08 +01:00
parent fba652006f
commit 08cd18f3f1
106 changed files with 1604 additions and 1057 deletions

View File

@@ -1,6 +1,7 @@
import requests
from bs4 import BeautifulSoup
from src import logger
# import sleep_and_retry decorator to retry requests
from ratelimit import limits, sleep_and_retry
@@ -27,11 +28,12 @@ HEADERS = {
RATE_LIMIT = 20
RATE_PERIOD = 30
class WebRequest:
def __init__(self) -> None:
"""Request data from the web, and format it depending on the mode."""
self.apparat = None
self.use_any = False # use any book that matches the search term
self.use_any = False # use any book that matches the search term
self.signature = None
self.ppn = None
self.data = None
@@ -44,6 +46,7 @@ class WebRequest:
self.use_any = True
logger.info("Using any book")
return self
def set_apparat(self, apparat):
self.apparat = apparat
if int(self.apparat) < 10:
@@ -59,6 +62,7 @@ class WebRequest:
signature = signature.split("/")[-1]
self.ppn = signature
return self
@sleep_and_retry
@limits(calls=RATE_LIMIT, period=RATE_PERIOD)
def search_book(self, searchterm: str):
@@ -73,6 +77,7 @@ class WebRequest:
for link in links:
res.append(BASE + link["href"])
return res
@sleep_and_retry
@limits(calls=RATE_LIMIT, period=RATE_PERIOD)
def search(self, link: str):
@@ -82,6 +87,7 @@ class WebRequest:
except requests.exceptions.RequestException as e:
logger.error(f"Request failed: {e}")
return None
def get_data(self):
links = self.get_book_links(self.ppn)
for link in links:
@@ -170,11 +176,11 @@ class BibTextTransformer:
COinS_IDENT = "ctx_ver"
BIBTEX_IDENT = "@book"
RDS_IDENT = "RDS ---------------------------------- "
if data is None:
self.data = None
return self
if self.mode == "RIS":
for line in data:
if RIS_IDENT in line:
@@ -207,7 +213,7 @@ class BibTextTransformer:
BookData: a dataclass containing data about the book
"""
if self.data is None:
return None
return None
match self.mode:
case "ARRAY":
return ARRAYData(self.signature).transform(self.data)
@@ -218,11 +224,10 @@ class BibTextTransformer:
case "RIS":
return RISData().transform(self.data)
case "RDS":
return RDSData().transform(self.data).return_data(option)
return RDSData().transform(self.data).return_data(option)
case None:
return None
# if self.mode == "ARRAY":
# return ARRAYData().transform(self.data)
# elif self.mode == "COinS":
@@ -252,4 +257,3 @@ if __name__ == "__main__":
data = WebRequest(71).get_ppn(link).get_data()
bib = BibTextTransformer("ARRAY").get_data().return_data()
print(bib)