rebase codebase, delete trunk, move threads to backend
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from src import logger
|
||||
|
||||
# import sleep_and_retry decorator to retry requests
|
||||
from ratelimit import limits, sleep_and_retry
|
||||
|
||||
@@ -27,11 +28,12 @@ HEADERS = {
|
||||
RATE_LIMIT = 20
|
||||
RATE_PERIOD = 30
|
||||
|
||||
|
||||
class WebRequest:
|
||||
def __init__(self) -> None:
|
||||
"""Request data from the web, and format it depending on the mode."""
|
||||
self.apparat = None
|
||||
self.use_any = False # use any book that matches the search term
|
||||
self.use_any = False # use any book that matches the search term
|
||||
self.signature = None
|
||||
self.ppn = None
|
||||
self.data = None
|
||||
@@ -44,6 +46,7 @@ class WebRequest:
|
||||
self.use_any = True
|
||||
logger.info("Using any book")
|
||||
return self
|
||||
|
||||
def set_apparat(self, apparat):
|
||||
self.apparat = apparat
|
||||
if int(self.apparat) < 10:
|
||||
@@ -59,6 +62,7 @@ class WebRequest:
|
||||
signature = signature.split("/")[-1]
|
||||
self.ppn = signature
|
||||
return self
|
||||
|
||||
@sleep_and_retry
|
||||
@limits(calls=RATE_LIMIT, period=RATE_PERIOD)
|
||||
def search_book(self, searchterm: str):
|
||||
@@ -73,6 +77,7 @@ class WebRequest:
|
||||
for link in links:
|
||||
res.append(BASE + link["href"])
|
||||
return res
|
||||
|
||||
@sleep_and_retry
|
||||
@limits(calls=RATE_LIMIT, period=RATE_PERIOD)
|
||||
def search(self, link: str):
|
||||
@@ -82,6 +87,7 @@ class WebRequest:
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Request failed: {e}")
|
||||
return None
|
||||
|
||||
def get_data(self):
|
||||
links = self.get_book_links(self.ppn)
|
||||
for link in links:
|
||||
@@ -170,11 +176,11 @@ class BibTextTransformer:
|
||||
COinS_IDENT = "ctx_ver"
|
||||
BIBTEX_IDENT = "@book"
|
||||
RDS_IDENT = "RDS ---------------------------------- "
|
||||
|
||||
|
||||
if data is None:
|
||||
self.data = None
|
||||
return self
|
||||
|
||||
|
||||
if self.mode == "RIS":
|
||||
for line in data:
|
||||
if RIS_IDENT in line:
|
||||
@@ -207,7 +213,7 @@ class BibTextTransformer:
|
||||
BookData: a dataclass containing data about the book
|
||||
"""
|
||||
if self.data is None:
|
||||
return None
|
||||
return None
|
||||
match self.mode:
|
||||
case "ARRAY":
|
||||
return ARRAYData(self.signature).transform(self.data)
|
||||
@@ -218,11 +224,10 @@ class BibTextTransformer:
|
||||
case "RIS":
|
||||
return RISData().transform(self.data)
|
||||
case "RDS":
|
||||
return RDSData().transform(self.data).return_data(option)
|
||||
return RDSData().transform(self.data).return_data(option)
|
||||
case None:
|
||||
return None
|
||||
|
||||
|
||||
|
||||
# if self.mode == "ARRAY":
|
||||
# return ARRAYData().transform(self.data)
|
||||
# elif self.mode == "COinS":
|
||||
@@ -252,4 +257,3 @@ if __name__ == "__main__":
|
||||
data = WebRequest(71).get_ppn(link).get_data()
|
||||
bib = BibTextTransformer("ARRAY").get_data().return_data()
|
||||
print(bib)
|
||||
|
||||
Reference in New Issue
Block a user