KomSuite/KomSuite-NyaaPy
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 3 Wiki Activity

dev improvements

Browse Source
This commit is contained in:
Juanjo Salvador
2019-10-05 14:50:47 +02:00
parent 1af3681542
commit 3287efbb4b
5 changed files with 438 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
NyaaPy/nyaa.py
View File

@@ -1,54 +1,54 @@
import requests import requests
import urllib.parse import urllib.parse
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from NyaaPy.utils import utils from NyaaPy import utils
class Nyaa: class Nyaa:
def __init__(self): def __init__(self):
self.URI = "http://nyaa.si" self.URI = "http://nyaa.si"
def search(self, keyword, **kwargs): def search(self, keyword, **kwargs):
user = kwargs.get('user', None) user = kwargs.get('user', None)
category = kwargs.get('category', 0) category = kwargs.get('category', 0)
subcategory = kwargs.get('subcategory', 0) subcategory = kwargs.get('subcategory', 0)
filters = kwargs.get('filters', 0) filters = kwargs.get('filters', 0)
page = kwargs.get('page', 0) page = kwargs.get('page', 0)
if user: if user:
user_uri = "user/{}".format(user) user_uri = "user/{}".format(user)
else: else:
user_uri = "" user_uri = ""
if page > 0: if page > 0:
r = requests.get("{}/{}?f={}&c={}_{}&q={}&p={}".format( r = requests.get("{}/{}?f={}&c={}_{}&q={}&p={}".format(
self.URI, user_uri, filters, category, subcategory, keyword, self.URI, user_uri, filters, category, subcategory, keyword,
page)) page))
else: else:
r = requests.get("{}/{}?f={}&c={}_{}&q={}".format( r = requests.get("{}/{}?f={}&c={}_{}&q={}".format(
self.URI, user_uri, filters, category, subcategory, keyword)) self.URI, user_uri, filters, category, subcategory, keyword))
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
rows = soup.select('table tr') rows = soup.select('table tr')
return utils.parse_nyaa(rows, limit=None) return utils.parse_nyaa(rows, limit=None)
def get(self, id): def get(self, id):
r = requests.get("{}/view/{}".format(self.URI, id)) r = requests.get("{}/view/{}".format(self.URI, id))
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
content = soup.findAll("div", {"class": "panel", "id": None}) content = soup.findAll("div", {"class": "panel", "id": None})
return utils.parse_single(content) return utils.parse_single(content)
def get_user(self, username): def get_user(self, username):
r = requests.get("{}/user/{}".format(self.URI, username)) r = requests.get("{}/user/{}".format(self.URI, username))
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
return utils.parse_nyaa(soup.select('table tr'), limit=None) return utils.parse_nyaa(soup.select('table tr'), limit=None)
def news(self, number_of_results): def news(self, number_of_results):
r = requests.get(self.URI) r = requests.get(self.URI)
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
rows = soup.select('table tr') rows = soup.select('table tr')
return utils.parse_nyaa(rows, limit=number_of_results + 1) return utils.parse_nyaa(rows, limit=number_of_results + 1)

80
NyaaPy/pantsu.py
View File

@@ -1,40 +1,40 @@
import requests import requests
from NyaaPy.utils import utils from NyaaPy import utils
class Pantsu: class Pantsu:
def __init__(self): def __init__(self):
self.BASE_URL = "https://nyaa.pantsu.cat/api" self.BASE_URL = "https://nyaa.pantsu.cat/api"
# Torrents - GET # Torrents - GET
def search(self, keyword, **kwargs): def search(self, keyword, **kwargs):
request = requests.get("{}/search{}".format( request = requests.get("{}/search{}".format(
self.BASE_URL, utils.query_builder(keyword, kwargs))) self.BASE_URL, utils.query_builder(keyword, kwargs)))
return request.json() return request.json()
def view(self, item_id): def view(self, item_id):
request = requests.get("{}/view/{}".format(self.BASE_URL, item_id)) request = requests.get("{}/view/{}".format(self.BASE_URL, item_id))
return request.json() return request.json()
# Torrents - POST # Torrents - POST
def upload(self): def upload(self):
return "Work in progress!" return "Work in progress!"
def update(self): def update(self):
return "Work in progress!" return "Work in progress!"
# Users # Users
def login(self, username, password): def login(self, username, password):
login = requests.post("{}/login/".format( login = requests.post("{}/login/".format(
self.BASE_URL), data={'username': username, 'password': password}) self.BASE_URL), data={'username': username, 'password': password})
return login.json() return login.json()
def profile(self, user_id): def profile(self, user_id):
profile = requests.post("{}/profile/".format( profile = requests.post("{}/profile/".format(
self.BASE_URL), data={'id': user_id}) self.BASE_URL), data={'id': user_id})
return profile.json() return profile.json()

168
NyaaPy/sukebei.py
View File

@@ -1,84 +1,84 @@
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from NyaaPy.utils import utils from NyaaPy import utils
class SukebeiNyaa: class SukebeiNyaa:
def search(self, keyword, **kwargs): def search(self, keyword, **kwargs):
category = kwargs.get('category', 0) category = kwargs.get('category', 0)
subcategory = kwargs.get('subcategory', 0) subcategory = kwargs.get('subcategory', 0)
filters = kwargs.get('filters', 0) filters = kwargs.get('filters', 0)
page = kwargs.get('page', 0) page = kwargs.get('page', 0)
if page > 0: if page > 0:
r = requests.get("{}/?f={}&c={}_{}&q={}&p={}".format( r = requests.get("{}/?f={}&c={}_{}&q={}&p={}".format(
"http://sukebei.nyaa.si", filters, category, subcategory, "http://sukebei.nyaa.si", filters, category, subcategory,
keyword, page)) keyword, page))
else: else:
r = requests.get("{}/?f={}&c={}_{}&q={}".format( r = requests.get("{}/?f={}&c={}_{}&q={}".format(
"http://sukebei.nyaa.si", filters, category, subcategory, "http://sukebei.nyaa.si", filters, category, subcategory,
keyword)) keyword))
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
rows = soup.select('table tr') rows = soup.select('table tr')
return utils.parse_nyaa(rows, limit=None) return utils.parse_nyaa(rows, limit=None)
def get(self, id): def get(self, id):
r = requests.get("http://sukebei.nyaa.si/view/{}".format(id)) r = requests.get("http://sukebei.nyaa.si/view/{}".format(id))
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
content = soup.findAll("div", {"class": "panel", "id": None}) content = soup.findAll("div", {"class": "panel", "id": None})
return utils.parse_single(content) return utils.parse_single(content)
def get_user(self, username): def get_user(self, username):
r = requests.get("http://sukebei.nyaa.si/user/{}".format(username)) r = requests.get("http://sukebei.nyaa.si/user/{}".format(username))
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
return utils.parse_nyaa(soup.select('table tr'), limit=None) return utils.parse_nyaa(soup.select('table tr'), limit=None)
def news(self, number_of_results): def news(self, number_of_results):
r = requests.get("http://sukebei.nyaa.si/") r = requests.get("http://sukebei.nyaa.si/")
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
rows = soup.select('table tr') rows = soup.select('table tr')
return utils.parse_sukebei(rows, limit=number_of_results + 1) return utils.parse_sukebei(rows, limit=number_of_results + 1)
class SukebeiPantsu: class SukebeiPantsu:
BASE_URL = "https://sukebei.pantsu.cat/api" BASE_URL = "https://sukebei.pantsu.cat/api"
# Torrents - GET # Torrents - GET
def search(self, keyword, **kwargs): def search(self, keyword, **kwargs):
request = requests.get("{}/search{}".format( request = requests.get("{}/search{}".format(
SukebeiPantsu.BASE_URL, utils.query_builder(keyword, kwargs))) SukebeiPantsu.BASE_URL, utils.query_builder(keyword, kwargs)))
return request.json() return request.json()
def view(self, item_id): def view(self, item_id):
request = requests.get("{}/view/{}".format( request = requests.get("{}/view/{}".format(
SukebeiPantsu.BASE_URL, item_id)) SukebeiPantsu.BASE_URL, item_id))
return request.json() return request.json()
# Torrents - POST # Torrents - POST
def upload(self): def upload(self):
return "Work in progress!" return "Work in progress!"
def update(self): def update(self):
return "Work in progress!" return "Work in progress!"
# Users # Users
def login(self, username, password): def login(self, username, password):
login = requests.post("{}/login/".format( login = requests.post("{}/login/".format(
SukebeiPantsu.BASE_URL), data={'username': username, SukebeiPantsu.BASE_URL), data={'username': username,
'password': password}) 'password': password})
return login.json() return login.json()
def profile(self, user_id): def profile(self, user_id):
profile = requests.post("{}/profile/".format( profile = requests.post("{}/profile/".format(
SukebeiPantsu.BASE_URL), data={'id': user_id}) SukebeiPantsu.BASE_URL), data={'id': user_id})
return profile.json() return profile.json()

506
NyaaPy/utils.py
View File

@@ -1,253 +1,253 @@
''' '''
Module utils Module utils
''' '''
import re import re
def nyaa_categories(b): def nyaa_categories(b):
c = b.replace('/?c=', '') c = b.replace('/?c=', '')
cats = c.split('_') cats = c.split('_')
cat = cats[0] cat = cats[0]
subcat = cats[1] subcat = cats[1]
categories = { categories = {
"1": { "1": {
"name": "Anime", "name": "Anime",
"subcats": { "subcats": {
"1": "Anime Music Video", "1": "Anime Music Video",
"2": "English-translated", "2": "English-translated",
"3": "Non-English-translated", "3": "Non-English-translated",
"4": "Raw" "4": "Raw"
} }
}, },
"2": { "2": {
"name": "Audio", "name": "Audio",
"subcats": { "subcats": {
"1": "Lossless", "1": "Lossless",
"2": "Lossy" "2": "Lossy"
} }
}, },
"3": { "3": {
"name": "Literature", "name": "Literature",
"subcats": { "subcats": {
"1": "English-translated", "1": "English-translated",
"2": "Non-English-translated", "2": "Non-English-translated",
"3": "Raw" "3": "Raw"
} }
}, },
"4": { "4": {
"name": "Live Action", "name": "Live Action",
"subcats": { "subcats": {
"1": "English-translated", "1": "English-translated",
"2": "Idol/Promotional Video", "2": "Idol/Promotional Video",
"3": "Non-English-translated", "3": "Non-English-translated",
"4": "Raw" "4": "Raw"
} }
}, },
"5": { "5": {
"name": "Pictures", "name": "Pictures",
"subcats": { "subcats": {
"1": "Graphics", "1": "Graphics",
"2": "Photos" "2": "Photos"
} }
}, },
"6": { "6": {
"name": "Software", "name": "Software",
"subcats": { "subcats": {
"1": "Applications", "1": "Applications",
"2": "Games" "2": "Games"
} }
} }
} }
try: try:
category_name = "{} - {}".format( category_name = "{} - {}".format(
categories[cat]['name'], categories[cat]['subcats'][subcat]) categories[cat]['name'], categories[cat]['subcats'][subcat])
except Exception: except Exception:
pass pass
return category_name return category_name
def parse_nyaa(table_rows, limit): def parse_nyaa(table_rows, limit):
if limit == 0: if limit == 0:
limit = len(table_rows) limit = len(table_rows)
torrents = [] torrents = []
for row in table_rows[:limit]: for row in table_rows[:limit]:
block = [] block = []
for td in row.find_all('td'): for td in row.find_all('td'):
if td.find_all('a'): if td.find_all('a'):
for link in td.find_all('a'): for link in td.find_all('a'):
if link.get('href')[-9:] != '#comments': if link.get('href')[-9:] != '#comments':
block.append(link.get('href')) block.append(link.get('href'))
if link.text.rstrip(): if link.text.rstrip():
block.append(link.text) block.append(link.text)
if td.text.rstrip(): if td.text.rstrip():
block.append(td.text.rstrip()) block.append(td.text.rstrip())
if row.has_attr('class'): if row.has_attr('class'):
if row['class'][0] == 'danger': if row['class'][0] == 'danger':
block.append("remake") block.append("remake")
elif row['class'][0] == 'success': elif row['class'][0] == 'success':
block.append("trusted") block.append("trusted")
else: else:
block.append("default") block.append("default")
try: try:
torrent = { torrent = {
'id': block[1].replace("/view/", ""), 'id': block[1].replace("/view/", ""),
'category': nyaa_categories(block[0]), 'category': nyaa_categories(block[0]),
'url': "http://nyaa.si{}".format(block[1]), 'url': "http://nyaa.si{}".format(block[1]),
'name': block[2], 'name': block[2],
'download_url': "http://nyaa.si{}".format(block[4]), 'download_url': "http://nyaa.si{}".format(block[4]),
'magnet': block[5], 'magnet': block[5],
'size': block[6], 'size': block[6],
'date': block[7], 'date': block[7],
'seeders': block[8], 'seeders': block[8],
'leechers': block[9], 'leechers': block[9],
'completed_downloads': block[10], 'completed_downloads': block[10],
'type': block[11], 'type': block[11],
} }
torrents.append(torrent) torrents.append(torrent)
except IndexError as ie: except IndexError as ie:
pass pass
return torrents return torrents
def parse_single(content): def parse_single(content):
torrent = {} torrent = {}
data = [] data = []
torrent_files = [] torrent_files = []
for row in content[0].find_all('div', {'class': 'row'}): for row in content[0].find_all('div', {'class': 'row'}):
for div in row.find_all('div', {'class': 'col-md-5'}): for div in row.find_all('div', {'class': 'col-md-5'}):
data.append(div.text.replace("\n", "")) data.append(div.text.replace("\n", ""))
files = content[2].find('div', files = content[2].find('div',
{'class', 'torrent-file-list'}).find_all('li') {'class', 'torrent-file-list'}).find_all('li')
for file in files: for file in files:
torrent_files.append(file.text) torrent_files.append(file.text)
torrent['title'] = re.sub('\n|\r|\t', '', content[0].find('h3', { torrent['title'] = re.sub('\n|\r|\t', '', content[0].find('h3', {
"class": "panel-title"}).text.replace("\n", "")) "class": "panel-title"}).text.replace("\n", ""))
torrent['category'] = data[0] torrent['category'] = data[0]
torrent['uploader'] = data[2] torrent['uploader'] = data[2]
torrent['uploader_profile'] = "https://nyaa.si/user/{}".format(data[2]) torrent['uploader_profile'] = "https://nyaa.si/user/{}".format(data[2])
torrent['website'] = re.sub('\t', '', data[4]) torrent['website'] = re.sub('\t', '', data[4])
torrent['size'] = data[6] torrent['size'] = data[6]
torrent['date'] = data[1] torrent['date'] = data[1]
torrent['seeders'] = data[3] torrent['seeders'] = data[3]
torrent['leechers'] = data[5] torrent['leechers'] = data[5]
torrent['completed'] = data[7] torrent['completed'] = data[7]
torrent['hash'] = data[8] torrent['hash'] = data[8]
torrent['description'] = re.sub('\t', '', content[1].find('div', { torrent['description'] = re.sub('\t', '', content[1].find('div', {
'id': 'torrent-description'}).text) 'id': 'torrent-description'}).text)
torrent['files'] = torrent_files torrent['files'] = torrent_files
return torrent return torrent
def parse_sukebei(table_rows, limit): def parse_sukebei(table_rows, limit):
if limit == 0: if limit == 0:
limit = len(table_rows) limit = len(table_rows)
torrents = [] torrents = []
for row in table_rows[:limit]: for row in table_rows[:limit]:
block = [] block = []
for td in row.find_all('td'): for td in row.find_all('td'):
for link in td.find_all('a'): for link in td.find_all('a'):
if link.get('href')[-9:] != '#comments': if link.get('href')[-9:] != '#comments':
block.append(link.get('href')) block.append(link.get('href'))
block.append(link.text.rstrip()) block.append(link.text.rstrip())
if td.text.rstrip(): if td.text.rstrip():
block.append(td.text.rstrip()) block.append(td.text.rstrip())
try: try:
torrent = { torrent = {
'id': block[1].replace("/view/", ""), 'id': block[1].replace("/view/", ""),
'category': sukebei_categories(block[0]), 'category': sukebei_categories(block[0]),
'url': "http://sukebei.nyaa.si{}".format(block[1]), 'url': "http://sukebei.nyaa.si{}".format(block[1]),
'name': block[2], 'name': block[2],
'download_url': "http://sukebei.nyaa.si{}".format( 'download_url': "http://sukebei.nyaa.si{}".format(
block[4]), block[4]),
'magnet': block[5], 'magnet': block[5],
'size': block[6], 'size': block[6],
'date': block[7], 'date': block[7],
'seeders': block[8], 'seeders': block[8],
'leechers': block[9], 'leechers': block[9],
'completed_downloads': block[10], 'completed_downloads': block[10],
} }
except IndexError as ie: except IndexError as ie:
pass pass
torrents.append(torrent) torrents.append(torrent)
return torrents return torrents
def sukebei_categories(b): def sukebei_categories(b):
c = b.replace('/?c=', '') c = b.replace('/?c=', '')
cats = c.split('_') cats = c.split('_')
cat = cats[0] cat = cats[0]
subcat = cats[1] subcat = cats[1]
categories = { categories = {
"1": { "1": {
"name": "Art", "name": "Art",
"subcats": { "subcats": {
"1": "Anime", "1": "Anime",
"2": "Doujinshi", "2": "Doujinshi",
"3": "Games", "3": "Games",
"4": "Manga", "4": "Manga",
"5": "Pictures", "5": "Pictures",
} }
}, },
"2": { "2": {
"name": "Real Life", "name": "Real Life",
"subcats": { "subcats": {
"1": "Photobooks & Pictures", "1": "Photobooks & Pictures",
"2": "Videos" "2": "Videos"
} }
} }
} }
try: try:
category_name = "{} - {}".format( category_name = "{} - {}".format(
categories[cat]['name'], categories[cat]['subcats'][subcat]) categories[cat]['name'], categories[cat]['subcats'][subcat])
except Exception: except Exception:
pass pass
return category_name return category_name
# Pantsu Utils # Pantsu Utils
def query_builder(q, params): def query_builder(q, params):
available_params = ["category", "page", "limit", "userID", "fromID", available_params = ["category", "page", "limit", "userID", "fromID",
"status", "maxage", "toDate", "fromDate", "status", "maxage", "toDate", "fromDate",
"dateType", "minSize", "maxSize", "sizeType", "dateType", "minSize", "maxSize", "sizeType",
"sort", "order", "lang"] "sort", "order", "lang"]
query = "?q={}".format(q.replace(" ", "+")) query = "?q={}".format(q.replace(" ", "+"))
for param, value in params.items(): for param, value in params.items():
if param in available_params: if param in available_params:
if (param != "category" and param != "status" and if (param != "category" and param != "status" and
param != "lang"): param != "lang"):
query += "&{}={}".format(param, value) query += "&{}={}".format(param, value)
elif param == "category": elif param == "category":
query += "&c={}_{}".format(value[0], value[1]) query += "&c={}_{}".format(value[0], value[1])
elif param == "status": elif param == "status":
query += "&s={}".format(value) query += "&s={}".format(value)
elif param == "lang": elif param == "lang":
for lang in value: for lang in value:
query += "&lang={}".format(lang) query += "&lang={}".format(lang)
return query return query

15
tests/test.py
View File

@@ -1,8 +1,7 @@
from NyaaPy import Pantsu, Nyaa from NyaaPy import Pantsu, Nyaa
pantsu = Pantsu() pantsu = Pantsu()
nyaa = Nyaa() nyaa = Nyaa()
# print(pantsu.search(keyword='koe no katachi', print(pantsu.search(keyword='koe no katachi',
# lang=["es", "ja"], category=[1, 3])) lang=["es", "ja"], category=[1, 3]))
print(nyaa.search(keyword='yuru camp'))