KomSuite/KomSuite-NyaaPy
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 3 Wiki Activity

Merged files

Browse Source
This commit is contained in:
Juanjo Salvador
2018-04-20 22:25:54 +02:00
parent b9622a2920 8df5b27bd4
commit 8d6c66a270
8 changed files with 134 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -1,4 +1,6 @@
build/ build/
dist/ dist/
nyaapy.egg-info nyaapy.egg-info
.vscode .vscode
env/
*.pyc

35
NyaaPy/nyaa.py
View File

@@ -1,41 +1,52 @@
import requests import requests
import urllib.parse
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from NyaaPy.utils import Utils as utils from NyaaPy.utils import Utils
utils = Utils()
class Nyaa: class Nyaa:
URI = "http://nyaa.si"
def __init__(self):
self.URI = "http://nyaa.si"
def search(keyword, **kwargs): def search(self, keyword, **kwargs):
user = kwargs.get('user', None)
category = kwargs.get('category', 0) category = kwargs.get('category', 0)
subcategory = kwargs.get('subcategory', 0) subcategory = kwargs.get('subcategory', 0)
filters = kwargs.get('filters', 0) filters = kwargs.get('filters', 0)
page = kwargs.get('page', 0) page = kwargs.get('page', 0)
if page > 0: if user:
r = requests.get("{}/?f={}&c={}_{}&q={}&p={}".format(Nyaa.URI, filters, category, subcategory, keyword, page)) user_uri = "user/{}".format(user)
else: else:
r = requests.get("{}/?f={}&c={}_{}&q={}".format(Nyaa.URI, filters, category, subcategory, keyword)) user_uri = ""
if page > 0:
r = requests.get("{}/{}?f={}&c={}_{}&q={}&p={}".format(self.URI, user_uri, filters, category, subcategory, keyword, page))
else:
r = requests.get("{}/{}?f={}&c={}_{}&q={}".format(self.URI, user_uri, filters, category, subcategory, keyword))
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
rows = soup.select('table tr') rows = soup.select('table tr')
return utils.parse_nyaa(rows, limit=None) return utils.parse_nyaa(rows, limit=None)
def get(id): def get(self, id):
r = requests.get("{}/view/{}".format(Nyaa.URI, id)) r = requests.get("{}/view/{}".format(self.URI, id))
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
content = soup.findAll("div", { "class": "panel", "id": None}) content = soup.findAll("div", { "class": "panel", "id": None})
return utils.parse_single(content) return utils.parse_single(content)
def get_user(username): def get_user(self, username):
r = requests.get("{}/user/{}".format(Nyaa.URI, username)) r = requests.get("{}/user/{}".format(self.URI, username))
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
return utils.parse_nyaa(soup.select('table tr'), limit=None) return utils.parse_nyaa(soup.select('table tr'), limit=None)
def news(number_of_results): def news(self, number_of_results):
r = requests.get(Nyaa.URI) r = requests.get(self.URI)
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
rows = soup.select('table tr') rows = soup.select('table tr')

29
NyaaPy/pantsu.py
View File

@@ -1,36 +1,39 @@
import requests import requests
from NyaaPy.utils import Utils as utils from NyaaPy.utils import Utils
utils = Utils()
class Pantsu: class Pantsu:
BASE_URL = "https://nyaa.pantsu.cat/api" def __init__(self):
self.BASE_URL = "https://nyaa.pantsu.cat/api"
# Torrents - GET # Torrents - GET
def search(keyword, **kwargs): def search(self, keyword, **kwargs):
request = requests.get("{}/search{}".format(Pantsu.BASE_URL, utils.query_builder(keyword, kwargs))) request = requests.get("{}/search{}".format(self.BASE_URL, utils.query_builder(keyword, kwargs)))
return request.json() return request.json()
def view(item_id): def view(self, item_id):
request = requests.get("{}/view/{}".format(Pantsu.BASE_URL, item_id)) request = requests.get("{}/view/{}".format(self.BASE_URL, item_id))
return request.json() return request.json()
# Torrents - POST # Torrents - POST
def upload(): def upload(self):
return "Work in progress!" return "Work in progress!"
def update(): def update(self):
return "Work in progress!" return "Work in progress!"
# Users # Users
def login(username, password): def login(self, username, password):
login = requests.post("{}/login/".format(Pantsu.BASE_URL), data={'username': username, 'password': password}) login = requests.post("{}/login/".format(self.BASE_URL), data={'username': username, 'password': password})
return login.json() return login.json()
def profile(user_id): def profile(self, user_id):
profile = requests.post("{}/profile/".format(Pantsu.BASE_URL), data={'id': user_id}) profile = requests.post("{}/profile/".format(self.BASE_URL), data={'id': user_id})
return profile.json() return profile.json()

20
NyaaPy/sukebei.py
View File

@@ -3,7 +3,7 @@ from bs4 import BeautifulSoup
from NyaaPy.utils import Utils as utils from NyaaPy.utils import Utils as utils
class SukebeiNyaa: class SukebeiNyaa:
def search(keyword, **kwargs): def search(self, keyword, **kwargs):
category = kwargs.get('category', 0) category = kwargs.get('category', 0)
subcategory = kwargs.get('subcategory', 0) subcategory = kwargs.get('subcategory', 0)
filters = kwargs.get('filters', 0) filters = kwargs.get('filters', 0)
@@ -19,20 +19,20 @@ class SukebeiNyaa:
return utils.parse_nyaa(rows, limit=None) return utils.parse_nyaa(rows, limit=None)
def get(id): def get(self, id):
r = requests.get("http://sukebei.nyaa.si/view/{}".format(id)) r = requests.get("http://sukebei.nyaa.si/view/{}".format(id))
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
content = soup.findAll("div", { "class": "panel", "id": None}) content = soup.findAll("div", { "class": "panel", "id": None})
return utils.parse_single(content) return utils.parse_single(content)
def get_user(username): def get_user(self, username):
r = requests.get("http://sukebei.nyaa.si/user/{}".format(username)) r = requests.get("http://sukebei.nyaa.si/user/{}".format(username))
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
return utils.parse_nyaa(soup.select('table tr'), limit=None) return utils.parse_nyaa(soup.select('table tr'), limit=None)
def news(number_of_results): def news(self, number_of_results):
r = requests.get("http://sukebei.nyaa.si/") r = requests.get("http://sukebei.nyaa.si/")
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
rows = soup.select('table tr') rows = soup.select('table tr')
@@ -43,32 +43,32 @@ class SukebeiPantsu:
BASE_URL = "https://sukebei.pantsu.cat/api" BASE_URL = "https://sukebei.pantsu.cat/api"
# Torrents - GET # Torrents - GET
def search(keyword, **kwargs): def search(self, keyword, **kwargs):
request = requests.get("{}/search{}".format(SukebeiPantsu.BASE_URL, utils.query_builder(keyword, kwargs))) request = requests.get("{}/search{}".format(SukebeiPantsu.BASE_URL, utils.query_builder(keyword, kwargs)))
return request.json() return request.json()
def view(item_id): def view(self, item_id):
request = requests.get("{}/view/{}".format(SukebeiPantsu.BASE_URL, item_id)) request = requests.get("{}/view/{}".format(SukebeiPantsu.BASE_URL, item_id))
return request.json() return request.json()
# Torrents - POST # Torrents - POST
def upload(): def upload(self):
return "Work in progress!" return "Work in progress!"
def update(): def update(self):
return "Work in progress!" return "Work in progress!"
# Users # Users
def login(username, password): def login(self, username, password):
login = requests.post("{}/login/".format(SukebeiPantsu.BASE_URL), data={'username': username, 'password': password}) login = requests.post("{}/login/".format(SukebeiPantsu.BASE_URL), data={'username': username, 'password': password})
return login.json() return login.json()
def profile(user_id): def profile(self, user_id):
profile = requests.post("{}/profile/".format(SukebeiPantsu.BASE_URL), data={'id': user_id}) profile = requests.post("{}/profile/".format(SukebeiPantsu.BASE_URL), data={'id': user_id})
return profile.json() return profile.json()

77
NyaaPy/utils.py
View File

@@ -5,8 +5,7 @@
import re import re
class Utils: class Utils:
def nyaa_categories(self, b):
def nyaa_categories(b):
c = b.replace('/?c=', '') c = b.replace('/?c=', '')
cats = c.split('_') cats = c.split('_')
@@ -70,13 +69,14 @@ class Utils:
return category_name return category_name
def parse_nyaa(table_rows, limit): def parse_nyaa(self, table_rows, limit):
if limit == 0: if limit == 0:
limit = len(table_rows) limit = len(table_rows)
torrents = [] torrents = []
for row in table_rows[:limit]: for row in table_rows[:limit]:
<<<<<<< HEAD
block = [] block = []
for td in row.find_all('td'): for td in row.find_all('td'):
@@ -103,10 +103,43 @@ class Utils:
} }
torrents.append(torrent) torrents.append(torrent)
=======
block = []
for td in row.find_all('td'):
if td.find_all('a'):
for link in td.find_all('a'):
if link.get('href')[-9:] != '#comments':
block.append(link.get('href'))
if link.text.rstrip():
block.append(link.text)
if td.text.rstrip():
block.append(td.text.rstrip())
try:
torrent = {
'id': block[1].replace("/view/", ""),
'category': Utils.nyaa_categories(self, block[0]),
'url': "http://nyaa.si{}".format(block[1]),
'name': block[2],
'download_url': "http://nyaa.si{}".format(block[4]),
'magnet': block[5],
'size': block[6],
'date': block[7],
'seeders': block[8],
'leechers': block[9],
'completed_downloads': block[10],
}
torrents.append(torrent)
except IndexError as ie:
pass
>>>>>>> 8df5b27bd43ffbfacbc58124f2082370e9287a92
return torrents return torrents
def parse_single(content): def parse_single(self, content):
torrent = {} torrent = {}
data = [] data = []
torrent_files = [] torrent_files = []
@@ -137,13 +170,14 @@ class Utils:
return torrent return torrent
def parse_sukebei(table_rows, limit): def parse_sukebei(self, table_rows, limit):
if limit == 0: if limit == 0:
limit = len(table_rows) limit = len(table_rows)
torrents = [] torrents = []
for row in table_rows[:limit]: for row in table_rows[:limit]:
<<<<<<< HEAD
block = [] block = []
for td in row.find_all('td'): for td in row.find_all('td'):
@@ -154,6 +188,35 @@ class Utils:
if td.text.rstrip(): if td.text.rstrip():
block.append(td.text.rstrip()) block.append(td.text.rstrip())
=======
block = []
for td in row.find_all('td'):
if td.find_all('a'):
for link in td.find_all('a'):
if link.get('href')[-9:] != '#comments':
block.append(link.get('href'))
if link.text.rstrip():
block.append(link.text)
if td.text.rstrip():
block.append(td.text.rstrip())
try:
torrent = {
'id': block[1].replace("/view/", ""),
'category': Utils.sukebei_categories(self, block[0]),
'url': "http://sukebei.nyaa.si{}".format(block[1]),
'name': block[2],
'download_url': "http://sukebei.nyaa.si{}".format(block[4]),
'magnet': block[5],
'size': block[6],
'date': block[7],
'seeders': block[8],
'leechers': block[9],
'completed_downloads': block[10],
}
>>>>>>> 8df5b27bd43ffbfacbc58124f2082370e9287a92
torrent = { torrent = {
'id': block[1].replace("/view/", ""), 'id': block[1].replace("/view/", ""),
@@ -173,7 +236,7 @@ class Utils:
return torrents return torrents
def sukebei_categories(b): def sukebei_categories(self, b):
c = b.replace('/?c=', '') c = b.replace('/?c=', '')
cats = c.split('_') cats = c.split('_')
@@ -208,7 +271,7 @@ class Utils:
return category_name return category_name
# Pantsu Utils # Pantsu Utils
def query_builder(q, params): def query_builder(self, q, params):
available_params = ["category", "page", "limit", "userID", "fromID", "status", "maxage", "toDate", "fromDate",\ available_params = ["category", "page", "limit", "userID", "fromID", "status", "maxage", "toDate", "fromDate",\
"dateType", "minSize", "maxSize", "sizeType", "sort", "order", "lang"] "dateType", "minSize", "maxSize", "sizeType", "sort", "order", "lang"]
query = "?q={}".format(q.replace(" ", "+")) query = "?q={}".format(q.replace(" ", "+"))

6
README.md
View File

@@ -1,11 +1,11 @@
![NyaaPy](https://github.com/JuanjoSalvador/NyaaPy/blob/master/nyaapy-logo.png?raw=true) <p align="center">
<img src="https://github.com/JuanjoSalvador/NyaaPy/blob/master/nyaapy-logo.png?raw=true" />
</p>
![](https://img.shields.io/badge/Python-3.5-green.svg) ![](https://img.shields.io/badge/Python-3.5-green.svg)
![](https://img.shields.io/badge/Nyaa.si-supported-green.svg) ![](https://img.shields.io/badge/Nyaa.si-supported-green.svg)
![](https://img.shields.io/badge/NyaaPantsu-supported-green.svg) ![](https://img.shields.io/badge/NyaaPantsu-supported-green.svg)
[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/JuanjoSalvador/NyaaPy/master/LICENSE.txt) [![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/JuanjoSalvador/NyaaPy/master/LICENSE.txt)
![](https://img.shields.io/badge/Version-0.5.0-blue.svg)
Unofficial Python module for Nyaa.si (WebScraping) and Nyaa.pantsu.cat (API wrapper) Unofficial Python module for Nyaa.si (WebScraping) and Nyaa.pantsu.cat (API wrapper)

6
setup.py
View File

@@ -1,13 +1,13 @@
from setuptools import setup, find_packages from setuptools import setup, find_packages
setup(name='nyaapy', setup(name='nyaapy',
version='0.6.1', version='0.6.2',
install_requires = [ install_requires = [
"requests", "requests",
"bs4", "beautifulsoup4",
], ],
url='https://github.com/juanjosalvador/nyaapy', url='https://github.com/juanjosalvador/nyaapy',
download_url = 'https://github.com/juanjosalvador/nyaapy/archive/0.6.1.tar.gz', download_url = 'https://github.com/juanjosalvador/nyaapy/archive/0.6.2.tar.gz',
license='MIT', license='MIT',
author='Juanjo Salvador', author='Juanjo Salvador',
author_email='juanjosalvador@netc.eu', author_email='juanjosalvador@netc.eu',

8
tests/test.py
View File

@@ -1,3 +1,7 @@
from NyaaPy import Pantsu from NyaaPy import Pantsu, Nyaa
print(Pantsu.search('koe no katachi', lang=["es", "ja"], category=[1, 3])) pantsu = Pantsu()
nyaa = Nyaa()
#print(pantsu.search(keyword='koe no katachi', lang=["es", "ja"], category=[1, 3]))
print(nyaa.search(keyword='yuru camp'))