diff --git a/.gitignore b/.gitignore index 6f7eb31..39ed182 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ dist/ nyaapy.egg-info .vscode env/ -*.pyc \ No newline at end of file +*.pyc +test_files \ No newline at end of file diff --git a/NyaaPy/nyaa.py b/NyaaPy/nyaa.py index c4ec563..3719089 100644 --- a/NyaaPy/nyaa.py +++ b/NyaaPy/nyaa.py @@ -1,12 +1,11 @@ import requests -import urllib.parse from NyaaPy import utils class Nyaa: def __init__(self): - self.URI = "http://nyaa.si" + self.URI = "https://nyaa.si" def last_uploads(self, number_of_results): r = requests.get(self.URI) diff --git a/NyaaPy/sukebei.py b/NyaaPy/sukebei.py index d0223c2..40340a3 100644 --- a/NyaaPy/sukebei.py +++ b/NyaaPy/sukebei.py @@ -1,8 +1,12 @@ import requests -from bs4 import BeautifulSoup from NyaaPy import utils + class SukebeiNyaa: + + def __init__(self): + self.URI = "https://sukebei.nyaa.si" + def search(self, keyword, **kwargs): category = kwargs.get('category', 0) subcategory = kwargs.get('subcategory', 0) @@ -11,37 +15,37 @@ class SukebeiNyaa: if page > 0: r = requests.get("{}/?f={}&c={}_{}&q={}&p={}".format( - "http://sukebei.nyaa.si", filters, category, subcategory, + self.URI, filters, category, subcategory, keyword, page)) else: r = requests.get("{}/?f={}&c={}_{}&q={}".format( - "http://sukebei.nyaa.si", filters, category, subcategory, + self.URI, filters, category, subcategory, keyword)) - soup = BeautifulSoup(r.text, 'html.parser') - rows = soup.select('table tr') - - return utils.parse_nyaa(rows, limit=None) + r.raise_for_status() + return utils.parse_nyaa(r.text, limit=None, sukebei=True) def get(self, id): - r = requests.get("http://sukebei.nyaa.si/view/{}".format(id)) - soup = BeautifulSoup(r.text, 'html.parser') - content = soup.findAll("div", {"class": "panel", "id": None}) + r = requests.get("{}/view/{}".format(self.URI, id)) + r.raise_for_status() - return utils.parse_single(content) + return utils.parse_single(r.text, sukebei=True) def get_user(self, username): - r = requests.get("http://sukebei.nyaa.si/user/{}".format(username)) - soup = BeautifulSoup(r.text, 'html.parser') + r = requests.get("{}/user/{}".format(self.URI, username)) + r.raise_for_status() - return utils.parse_nyaa(soup.select('table tr'), limit=None) + return utils.parse_nyaa(r.text, limit=None, sukebei=True) - def news(self, number_of_results): - r = requests.get("http://sukebei.nyaa.si/") - soup = BeautifulSoup(r.text, 'html.parser') - rows = soup.select('table tr') + def last_uploads(self, number_of_results): + r = requests.get(self.URI) + r.raise_for_status() - return utils.parse_sukebei(rows, limit=number_of_results + 1) + return utils.parse_nyaa( + r.text, + limit=number_of_results + 1, + sukebei=True + ) class SukebeiPantsu: diff --git a/NyaaPy/utils.py b/NyaaPy/utils.py index 0c837b1..e221761 100644 --- a/NyaaPy/utils.py +++ b/NyaaPy/utils.py @@ -72,10 +72,15 @@ def nyaa_categories(b): return category_name -def parse_nyaa(request_text, limit): +def parse_nyaa(request_text, limit, sukebei=False): parser = etree.HTMLParser() tree = etree.fromstring(request_text, parser) + if sukebei is False: + uri = "https://nyaa.si" + else: + uri = "https://sukebei.nyaa.si" + torrents = [] # Going through table rows @@ -109,10 +114,10 @@ def parse_nyaa(request_text, limit): try: torrent = { 'id': block[1], - 'category': nyaa_categories(block[0]), - 'url': "https://nyaa.si/view/{}".format(block[1]), + 'category': nyaa_categories(block[0]) if sukebei is False else sukebei_categories(block[0]), + 'url': "{}/view/{}".format(uri, block[1]), 'name': block[2], - 'download_url': "https://nyaa.si/download/{}".format(block[3]), + 'download_url': "{}/download/{}".format(uri, block[3]), 'magnet': block[4], 'size': block[5], 'date': block[6], @@ -127,10 +132,15 @@ def parse_nyaa(request_text, limit): return torrents -def parse_single(request_text): +def parse_single(request_text, sukebei=False): parser = etree.HTMLParser() tree = etree.fromstring(request_text, parser) + if sukebei is False: + uri = "https://nyaa.si" + else: + uri = "https://sukebei.nyaa.si" + torrent = {} data = [] torrent_files = [] @@ -152,7 +162,7 @@ def parse_single(request_text): tree.xpath("//h3[@class='panel-title']/text()")[0].strip() torrent['category'] = data[0] torrent['uploader'] = data[4] - torrent['uploader_profile'] = "http://nyaa.si/user/{}".format(data[4]) + torrent['uploader_profile'] = "{}/user/{}".format(uri, data[4]) torrent['website'] = data[6] torrent['size'] = data[8] torrent['date'] = data[3] @@ -169,49 +179,8 @@ def parse_single(request_text): return torrent -def parse_sukebei(table_rows, limit): - if limit == 0: - limit = len(table_rows) - - torrents = [] - - for row in table_rows[:limit]: - block = [] - - for td in row.find_all('td'): - for link in td.find_all('a'): - if link.get('href')[-9:] != '#comments': - block.append(link.get('href')) - block.append(link.text.rstrip()) - - if td.text.rstrip(): - block.append(td.text.rstrip()) - - try: - torrent = { - 'id': block[1].replace("/view/", ""), - 'category': sukebei_categories(block[0]), - 'url': "http://sukebei.nyaa.si{}".format(block[1]), - 'name': block[2], - 'download_url': "http://sukebei.nyaa.si{}".format( - block[4]), - 'magnet': block[5], - 'size': block[6], - 'date': block[7], - 'seeders': block[8], - 'leechers': block[9], - 'completed_downloads': block[10], - } - except IndexError as ie: - pass - - torrents.append(torrent) - - return torrents - - def sukebei_categories(b): - c = b.replace('/?c=', '') + c = b.replace('?c=', '') cats = c.split('_') cat = cats[0] diff --git a/tests/test.py b/tests/test.py index 2a5f5a4..8b9215e 100644 --- a/tests/test.py +++ b/tests/test.py @@ -1,40 +1,51 @@ -from NyaaPy import Pantsu, Nyaa +from NyaaPy import Nyaa from pprint import pprint from datetime import datetime +import json +import sys +import os + +# Creating a folder for test_files +# ! not included in github project. +if not os.path.isdir("test_files"): + os.makedirs("test_files") -# pantsu = Pantsu() nyaa = Nyaa() # Get fresh torrents dt_latest_torrents_begin = datetime.now() latest_torrents = nyaa.last_uploads(100) dt_latest_torrents_end = datetime.now() +with open("test_files/nyaa_latest_torrent_test.json", 'w') as f: + json.dump(latest_torrents, f) -# I'd like to watch Tenki no ko, but not uploaded yet. +# Search some nasty stuff dt_search_begin = datetime.now() -test_search = nyaa.search("Kimi no Na wa") +test_search = nyaa.search("kimi no na wa") dt_search_end = datetime.now() -# pprint(test_search) +with open("test_files/nyaa_search_test.json", 'w') as f: + json.dump(test_search, f) # Get first torrent from found torrents -# print("First result torrent info:") dt_single_torrent_begin = datetime.now() single_torrent = nyaa.get(test_search[0]["id"]) dt_single_torrent_end = datetime.now() -#pprint(single_torrent) +with open("test_files/nyaa_single_torrent_test.json", 'w') as f: + json.dump(single_torrent, f) dt_user_begin = datetime.now() -user_torrents = nyaa.get_user("Lilith-Raws") +user_torrents = nyaa.get_user("HorribleSubs") dt_user_end = datetime.now() -#pprint(user_torrents) +with open("test_files/nyaa_single_user_test.json", 'w') as f: + json.dump(user_torrents, f) print( "Latest torrents time:", (dt_latest_torrents_end - dt_latest_torrents_begin).microseconds / 1000, "msec") print( - "Test search time:", - (dt_search_end - dt_search_begin).microseconds/ 1000, + "Test search time:", + (dt_search_end - dt_search_begin).microseconds / 1000, "msec" ) print( @@ -44,11 +55,6 @@ print( ) print( "Single user time:", - (dt_user_end - dt_user_begin ).microseconds / 1000, + (dt_user_end - dt_user_begin).microseconds / 1000, "msec" ) - -""" -print(pantsu.search(keyword='koe no katachi', - lang=["es", "ja"], category=[1, 3])) -""" diff --git a/tests/test_sukebei.py b/tests/test_sukebei.py new file mode 100644 index 0000000..5a2965a --- /dev/null +++ b/tests/test_sukebei.py @@ -0,0 +1,60 @@ +from NyaaPy import SukebeiNyaa +from pprint import pprint +from datetime import datetime +import json +import sys +import os + +# Creating a folder for test_files +# ! not included in github project. +if not os.path.isdir("test_files"): + os.makedirs("test_files") + +nyaa = SukebeiNyaa() + +# Get fresh torrents +dt_latest_torrents_begin = datetime.now() +latest_torrents = nyaa.last_uploads(100) +dt_latest_torrents_end = datetime.now() +with open("test_files/sukebei_latest_torrent_test.json", 'w') as f: + json.dump(latest_torrents, f) + +# Search some nasty stuff +dt_search_begin = datetime.now() +test_search = nyaa.search("G Senjou no maou") +dt_search_end = datetime.now() +with open("test_files/sukebei_search_test.json", 'w') as f: + json.dump(test_search, f) + +# Get first torrent from found torrents +dt_single_torrent_begin = datetime.now() +single_torrent = nyaa.get(test_search[0]["id"]) +dt_single_torrent_end = datetime.now() +with open("test_files/sukebei_single_torrent_test.json", 'w') as f: + json.dump(single_torrent, f) + +dt_user_begin = datetime.now() +user_torrents = nyaa.get_user("RUNBKK") +dt_user_end = datetime.now() +with open("test_files/sukebei_single_user_test.json", 'w') as f: + json.dump(user_torrents, f) + +print( + "Latest torrents time:", + (dt_latest_torrents_end - dt_latest_torrents_begin).microseconds / 1000, + "msec") +print( + "Test search time:", + (dt_search_end - dt_search_begin).microseconds / 1000, + "msec" +) +print( + "Single torrent time:", + (dt_single_torrent_end - dt_single_torrent_begin).microseconds / 1000, + "msec" +) +print( + "Single user time:", + (dt_user_end - dt_user_begin).microseconds / 1000, + "msec" +)