Merge pull request #52 from Hungry-Dolphin/HowToContribute

Implemented issue #30, the nyaa search now returns a torrent object instead of a dict

.gitignore

@@ -4,3 +4,6 @@ nyaapy.egg-info
 .vscode
 env/
 *.pyc
+test_files
+venv
+.idea

NyaaPy/__init__.py

@@ -1,11 +1,7 @@
 # Info about the module
-__version__ = '0.6.0'
+__version__ = '0.6.3'
 __author__ = 'Juanjo Salvador'
 __email__ = 'juanjosalvador@netc.eu'
 __url__ = 'http://juanjosalvador.me'
 __copyright__ = '2017 Juanjo Salvador'
 __license__ = 'MIT license'
-
-from NyaaPy.nyaa import Nyaa
-from NyaaPy.pantsu import Pantsu
-from NyaaPy.sukebei import SukebeiNyaa, SukebeiPantsu

NyaaPy/nyaa.py

@@ -1,21 +1,30 @@
 import requests
-import urllib.parse
-from bs4 import BeautifulSoup
 from NyaaPy import utils
+from NyaaPy import torrent
+
 
 class Nyaa:
 
     def __init__(self):
-        self.URI = "http://nyaa.si"
+        self.SITE = utils.TorrentSite.NYAASI
+        self.URL = "https://nyaa.si"
 
     def last_uploads(self, number_of_results):
-        r = requests.get(self.URI)
+        r = requests.get(self.URL)
-        soup = BeautifulSoup(r.text, 'html.parser')
-        rows = soup.select('table tr')
 
-        return utils.parse_nyaa(table_rows=rows, limit=number_of_results + 1)
+        # If anything up with nyaa servers let the user know.
+        r.raise_for_status()
+
+        json_data = utils.parse_nyaa(
+            request_text=r.text,
+            limit=number_of_results + 1,
+            site=self.SITE
+        )
+        return torrent.json_to_class(json_data)
 
     def search(self, keyword, **kwargs):
+        url = self.URL
+
         user = kwargs.get('user', None)
         category = kwargs.get('category', 0)
         subcategory = kwargs.get('subcategory', 0)
@@ -23,32 +32,43 @@ class Nyaa:
         page = kwargs.get('page', 0)
 
         if user:
-            user_uri = "user/{}".format(user)
+            user_uri = f"user/{user}"
         else:
             user_uri = ""
 
         if page > 0:
             r = requests.get("{}/{}?f={}&c={}_{}&q={}&p={}".format(
-                self.URI, user_uri, filters, category, subcategory, keyword,
+                url, user_uri, filters, category, subcategory, keyword,
                 page))
         else:
             r = requests.get("{}/{}?f={}&c={}_{}&q={}".format(
-                self.URI, user_uri, filters, category, subcategory, keyword))
+                url, user_uri, filters, category, subcategory, keyword))
 
-        soup = BeautifulSoup(r.text, 'html.parser')
+        r.raise_for_status()
-        rows = soup.select('table tr')
 
-        return utils.parse_nyaa(rows, limit=None)
+        json_data = utils.parse_nyaa(
+            request_text=r.text,
+            limit=None,
+            site=self.SITE
+        )
 
-    def get(self, id):
+        return torrent.json_to_class(json_data)
-        r = requests.get("{}/view/{}".format(self.URI, id))
-        soup = BeautifulSoup(r.text, 'html.parser')
-        content = soup.findAll("div", {"class": "panel", "id": None})
 
-        return utils.parse_single(content)
+    def get(self, view_id):
+        r = requests.get(f'{self.URL}/view/{view_id}')
+        r.raise_for_status()
+
+        json_data = utils.parse_single(request_text=r.text, site=self.SITE)
+
+        return torrent.json_to_class(json_data)
 
     def get_user(self, username):
-        r = requests.get("{}/user/{}".format(self.URI, username))
+        r = requests.get(f'{self.URL}/user/{username}')
-        soup = BeautifulSoup(r.text, 'html.parser')
+        r.raise_for_status()
 
-        return utils.parse_nyaa(soup.select('table tr'), limit=None)
+        json_data = utils.parse_nyaa(
+            request_text=r.text,
+            limit=None,
+            site=self.SITE
+        )
+        return torrent.json_to_class(json_data)

NyaaPy/pantsu.py

@@ -1,18 +1,24 @@
 import requests
 from NyaaPy import utils
 
+
 class Pantsu:
 
     def __init__(self):
         self.BASE_URL = "https://nyaa.pantsu.cat/api"
+        self.SITE = utils.TorrentSite.NYAANET
 
     def last_uploads(self, number_of_results):
-       r = requests.get(self.URI)
+        r = requests.get(self.SITE.value)
-       soup = BeautifulSoup(r.text, 'html.parser')
+        r.raise_for_status()
-       rows = soup.select('table tr')
+        with open("test.html", "w") as f:
-
+            f.write(r.text)
-       return utils.parse_nyaa(rows, limit=number_of_results + 1)
 
+        return utils.parse_nyaa(
+            request_text=r.text,
+            limit=number_of_results + 1,
+            site=self.SITE
+        )
 
     # Torrents - GET
     def search(self, keyword, **kwargs):
@@ -23,10 +29,11 @@ class Pantsu:
     def view(self, item_id):
         request = requests.get("{}/view/{}".format(self.BASE_URL, item_id))
 
+        request.raise_for_status()
+
         return request.json()
 
     # Torrents - POST
-
     def upload(self):
         return "Work in progress!"
 
@@ -34,7 +41,6 @@ class Pantsu:
         return "Work in progress!"
 
     # Users
-
     def login(self, username, password):
         login = requests.post("{}/login/".format(
             self.BASE_URL), data={'username': username, 'password': password})

NyaaPy/sukebei.py

@@ -1,9 +1,14 @@
 import requests
-from bs4 import BeautifulSoup
 from NyaaPy import utils
 
+
 class SukebeiNyaa:
+
+    def __init__(self):
+        self.SITE = utils.TorrentSite.SUKEBEINYAASI
+
     def search(self, keyword, **kwargs):
+        uri = self.SITE.value
         category = kwargs.get('category', 0)
         subcategory = kwargs.get('subcategory', 0)
         filters = kwargs.get('filters', 0)
@@ -11,37 +16,37 @@ class SukebeiNyaa:
 
         if page > 0:
             r = requests.get("{}/?f={}&c={}_{}&q={}&p={}".format(
-                "http://sukebei.nyaa.si", filters, category, subcategory,
+                uri, filters, category, subcategory,
                 keyword, page))
         else:
             r = requests.get("{}/?f={}&c={}_{}&q={}".format(
-                "http://sukebei.nyaa.si", filters, category, subcategory,
+                uri, filters, category, subcategory,
                 keyword))
 
-        soup = BeautifulSoup(r.text, 'html.parser')
+        r.raise_for_status()
-        rows = soup.select('table tr')
+        return utils.parse_nyaa(r.text, limit=None, site=self.SITE)
-
-        return utils.parse_nyaa(rows, limit=None)
 
     def get(self, id):
-        r = requests.get("http://sukebei.nyaa.si/view/{}".format(id))
+        r = requests.get("{}/view/{}".format(self.SITE.value, id))
-        soup = BeautifulSoup(r.text, 'html.parser')
+        r.raise_for_status()
-        content = soup.findAll("div", {"class": "panel", "id": None})
 
-        return utils.parse_single(content)
+        return utils.parse_single(r.text, self.SITE)
 
     def get_user(self, username):
-        r = requests.get("http://sukebei.nyaa.si/user/{}".format(username))
+        r = requests.get("{}/user/{}".format(self.SITE.value, username))
-        soup = BeautifulSoup(r.text, 'html.parser')
+        r.raise_for_status()
 
-        return utils.parse_nyaa(soup.select('table tr'), limit=None)
+        return utils.parse_nyaa(r.text, limit=None, site=self.SITE)
 
-    def news(self, number_of_results):
+    def last_uploads(self, number_of_results):
-        r = requests.get("http://sukebei.nyaa.si/")
+        r = requests.get(self.SITE.value)
-        soup = BeautifulSoup(r.text, 'html.parser')
+        r.raise_for_status()
-        rows = soup.select('table tr')
 
-        return utils.parse_sukebei(rows, limit=number_of_results + 1)
+        return utils.parse_nyaa(
+            r.text,
+            limit=number_of_results + 1,
+            site=self.SITE
+        )
 
 
 class SukebeiPantsu:

NyaaPy/torrent.py

@@ -0,0 +1,17 @@
+def json_to_class(data):
+    # We check if the data passed is a list or not
+    if isinstance(data, list):
+        object_list = []
+        for item in data:
+            object_list.append(Torrent(item))
+            # Return a list of Torrent objects
+        return object_list
+    else:
+        return Torrent(data)
+
+
+# This deals with converting the dict to an object
+class Torrent(object):
+    def __init__(self, my_dict):
+        for key in my_dict:
+            setattr(self, key, my_dict[key])

NyaaPy/utils.py

@@ -1,20 +1,30 @@
-'''
+from enum import Enum
-    Module utils
+from lxml import etree
-'''
+
+
+class TorrentSite(Enum):
+    """
+    Contains torrent sites
+    """
+    NYAASI = "https://nyaa.si"
+    SUKEBEINYAASI = "https://sukebei.nyaa.si"
+
+    # * nyaa.pantsu.cat redirects to nyaa.net
+    NYAANET = "https://nyaa.net"
+    SUKEBEINYAANET = "https://sukebei.nyaa.net"
 
-import re
 
 def nyaa_categories(b):
-    c = b.replace('/?c=', '')
+    c = b.replace('?c=', '')
     cats = c.split('_')
 
     cat = cats[0]
-    subcat = cats[1]
+    sub_cat = cats[1]
 
     categories = {
         "1": {
             "name": "Anime",
-            "subcats": {
+            "sub_cats": {
                 "1": "Anime Music Video",
                 "2": "English-translated",
                 "3": "Non-English-translated",
@@ -23,14 +33,14 @@ def nyaa_categories(b):
         },
         "2": {
             "name": "Audio",
-            "subcats": {
+            "sub_cats": {
                 "1": "Lossless",
                 "2": "Lossy"
             }
         },
         "3": {
             "name": "Literature",
-            "subcats": {
+            "sub_cats": {
                 "1": "English-translated",
                 "2": "Non-English-translated",
                 "3": "Raw"
@@ -38,7 +48,7 @@ def nyaa_categories(b):
         },
         "4": {
             "name": "Live Action",
-            "subcats": {
+            "sub_cats": {
                 "1": "English-translated",
                 "2": "Idol/Promotional Video",
                 "3": "Non-English-translated",
@@ -47,14 +57,14 @@ def nyaa_categories(b):
         },
         "5": {
             "name": "Pictures",
-            "subcats": {
+            "sub_cats": {
                 "1": "Graphics",
                 "2": "Photos"
             }
         },
         "6": {
             "name": "Software",
-            "subcats": {
+            "sub_cats": {
                 "1": "Applications",
                 "2": "Games"
             }
@@ -62,138 +72,130 @@ def nyaa_categories(b):
     }
 
     try:
-        category_name = "{} - {}".format(
+        category_name = f"{categories[cat]['name']} - {categories[cat]['sub_cats'][sub_cat]}"
-            categories[cat]['name'], categories[cat]['subcats'][subcat])
+    except KeyError:
-    except Exception:
+        print("Unable to get Nyaa category name")
-        pass
+        return
 
     return category_name
 
-def parse_nyaa(table_rows, limit):
+
-    if limit == 0:
+def parse_nyaa(request_text, limit, site):
-        limit = len(table_rows)
+    parser = etree.HTMLParser()
+    tree = etree.fromstring(request_text, parser)
+
+    # Put proper domain here.
+    uri = site.value
 
     torrents = []
 
-    for row in table_rows[:limit]:
+    # Going through table rows
+    for tr in tree.xpath("//tbody//tr")[:limit]:
         block = []
 
-        for td in row.find_all('td'):
+        for td in tr.xpath("./td"):
-            if td.find_all('a'):
+            for link in td.xpath("./a"):
-                for link in td.find_all('a'):
-                    if link.get('href')[-9:] != '#comments':
-                        block.append(link.get('href'))
-                        if link.text.rstrip():
-                            block.append(link.text)
 
-            if td.text.rstrip():
+                href = link.attrib.get("href").split('/')[-1]
-                block.append(td.text.rstrip())
 
-        if row.has_attr('class'):
+                # Only caring about non-comment pages.
-            if row['class'][0] == 'danger':
+                if href[-9:] != "#comments":
+                    block.append(href)
+
+                    if link.text and link.text.strip():
+                        block.append(link.text.strip())
+
+            if td.text is not None and td.text.strip():
+                block.append(td.text.strip())
+
+        # Add type of torrent based on tr class.
+        if tr.attrib.get("class") is not None:
+            if 'danger' in tr.attrib.get("class"):
                 block.append("remake")
-            elif row['class'][0] == 'success':
+            elif 'success' in tr.attrib.get("class"):
                 block.append("trusted")
             else:
                 block.append("default")
+        else:
+            block.append("default")
 
+        # Decide category.
+        if site in [TorrentSite.NYAASI, TorrentSite.NYAANET]:
+            category = nyaa_categories(block[0])
+        elif site in [TorrentSite.SUKEBEINYAASI, TorrentSite.SUKEBEINYAANET]:
+            category = sukebei_categories(block[0])
+        else:
+            raise ValueError("Unknown TorrentSite received!")
+
+        # Create torrent object
         try:
             torrent = {
-                'id': block[1].replace("/view/", ""),
+                'id': block[1],
-                'category': nyaa_categories(block[0]),
+                'category': category,
-                'url': "http://nyaa.si{}".format(block[1]),
+                'url': "{}/view/{}".format(uri, block[1]),
                 'name': block[2],
-                'download_url': "http://nyaa.si{}".format(block[4]),
+                'download_url': "{}/download/{}".format(uri, block[3]),
-                'magnet': block[5],
+                'magnet': block[4],
-                'size': block[6],
+                'size': block[5],
-                'date': block[7],
+                'date': block[6],
-                'seeders': block[8],
+                'seeders': block[7],
-                'leechers': block[9],
+                'leechers': block[8],
-                'completed_downloads': block[10],
+                'completed_downloads': block[9],
-                'type': block[11],
+                'type': block[10]
             }
-
             torrents.append(torrent)
-        except IndexError as ie:
+        except IndexError:
             pass
-
     return torrents
 
-def parse_single(content):
+
+def parse_single(request_text, site):
+    parser = etree.HTMLParser()
+    tree = etree.fromstring(request_text, parser)
+
+    # Put proper domain here.
+    uri = site.value
+
     torrent = {}
     data = []
     torrent_files = []
 
-    for row in content[0].find_all('div', {'class': 'row'}):
+    # Find basic uploader info & torrent stats
-        for div in row.find_all('div', {'class': 'col-md-5'}):
+    for row in tree.xpath("//div[@class='row']"):
-            data.append(div.text.replace("\n", ""))
+        for div_text in row.xpath("./div[@class='col-md-5']//text()"):
+            d = div_text.strip()
+            if d:
+                data.append(d)
 
-    files = content[2].find('div',
+    # Find files, we need only text of the li element(s).
-                            {'class', 'torrent-file-list'}).find_all('li')
+    # Sorry about Pycodestyle aka PEP8 (E501) error
+    for el in tree.xpath("//div[contains(@class, 'torrent-file-list')]//li/text()"):
+        if el.rstrip():
+            torrent_files.append(el)
 
-    for file in files:
+    torrent['title'] = \
-        torrent_files.append(file.text)
+        tree.xpath("//h3[@class='panel-title']/text()")[0].strip()
-
-    torrent['title'] = re.sub('\n|\r|\t', '', content[0].find('h3', {
-        "class": "panel-title"}).text.replace("\n", ""))
     torrent['category'] = data[0]
-    torrent['uploader'] = data[2]
+    torrent['uploader'] = data[4]
-    torrent['uploader_profile'] = "https://nyaa.si/user/{}".format(data[2])
+    torrent['uploader_profile'] = "{}/user/{}".format(uri, data[4])
-    torrent['website'] = re.sub('\t', '', data[4])
+    torrent['website'] = data[6]
-    torrent['size'] = data[6]
+    torrent['size'] = data[8]
-    torrent['date'] = data[1]
+    torrent['date'] = data[3]
-    torrent['seeders'] = data[3]
+    torrent['seeders'] = data[5]
-    torrent['leechers'] = data[5]
+    torrent['leechers'] = data[7]
-    torrent['completed'] = data[7]
+    torrent['completed'] = data[9]
-    torrent['hash'] = data[8]
+    torrent['hash'] = data[10]
-    torrent['description'] = re.sub('\t', '', content[1].find('div', {
-        'id': 'torrent-description'}).text)
     torrent['files'] = torrent_files
 
+    torrent['description'] = ""
+    for s in tree.xpath("//div[@id='torrent-description']"):
+        torrent['description'] += s.text
+
     return torrent
 
-def parse_sukebei(table_rows, limit):
-    if limit == 0:
-        limit = len(table_rows)
-
-    torrents = []
-
-    for row in table_rows[:limit]:
-        block = []
-
-        for td in row.find_all('td'):
-            for link in td.find_all('a'):
-                if link.get('href')[-9:] != '#comments':
-                    block.append(link.get('href'))
-                    block.append(link.text.rstrip())
-
-            if td.text.rstrip():
-                block.append(td.text.rstrip())
-
-            try:
-                torrent = {
-                    'id': block[1].replace("/view/", ""),
-                    'category': sukebei_categories(block[0]),
-                    'url': "http://sukebei.nyaa.si{}".format(block[1]),
-                    'name': block[2],
-                    'download_url': "http://sukebei.nyaa.si{}".format(
-                        block[4]),
-                    'magnet': block[5],
-                    'size': block[6],
-                    'date': block[7],
-                    'seeders': block[8],
-                    'leechers': block[9],
-                    'completed_downloads': block[10],
-                }
-            except IndexError as ie:
-                pass
-
-            torrents.append(torrent)
-
-    return torrents
 
 def sukebei_categories(b):
-    c = b.replace('/?c=', '')
+    c = b.replace('?c=', '')
     cats = c.split('_')
 
     cat = cats[0]
@@ -220,13 +222,14 @@ def sukebei_categories(b):
     }
 
     try:
-        category_name = "{} - {}".format(
+        category_name = f"{categories[cat]['name']} - {categories[cat]['subcats'][subcat]}"
-            categories[cat]['name'], categories[cat]['subcats'][subcat])
+    except KeyError:
-    except Exception:
+        print("Unable to get Sukebei category name")
-        pass
+        return
 
     return category_name
 
+
 # Pantsu Utils
 def query_builder(q, params):
     available_params = ["category", "page", "limit", "userID", "fromID",

requirements.txt

@@ -1,2 +1,3 @@
 requests>=2.20.0
 beautifulsoup4==4.6.0
+lxml

tests/test.py

@@ -1,8 +1,79 @@
-from NyaaPy import Pantsu, Nyaa
+from NyaaPy.nyaa import Nyaa
+from pprint import pprint
+from datetime import datetime
+import json
+import sys
+import os
+
+# Creating a folder for test_files
+# ! not included in github project.
+if not os.path.isdir("test_files"):
+    os.makedirs("test_files")
 
-pantsu = Pantsu()
 nyaa = Nyaa()
 
-# print(pantsu.search(keyword='koe no katachi',
+# Get fresh torrents
-#                     lang=["es", "ja"], category=[1, 3]))
+dt_latest_torrents_begin = datetime.now()
-print(nyaa.search(keyword='yuru camp'))
+latest_torrents = nyaa.last_uploads(100)
+dt_latest_torrents_end = datetime.now()
+with open("test_files/nyaa_latest_torrent_test.json", 'w') as f:
+    for torrent in latest_torrents:
+        try:
+            # This prints it as byte like objects since unicode is fun
+            f.write(str(torrent.name.encode('utf-8')) + '\n')
+        except AttributeError:
+            f.write('No name found for this torrent')
+
+# Search some nasty stuff
+dt_search_begin = datetime.now()
+test_search = nyaa.search("kimi no na wa")
+dt_search_end = datetime.now()
+with open("test_files/nyaa_search_test.json", 'w') as f:
+    for torrent in test_search:
+        try:
+            # This prints it as byte like objects since unicode is fun
+            f.write(str(torrent.name.encode('utf-8')) + '\n')
+        except AttributeError:
+            f.write('No name found for this torrent')
+
+# Get first torrent from found torrents
+dt_single_torrent_begin = datetime.now()
+single_torrent = test_search[0]
+dt_single_torrent_end = datetime.now()
+with open("test_files/nyaa_single_torrent_test.json", 'w') as f:
+    try:
+        # This prints it as byte like objects since unicode is fun
+        f.write(str(torrent.name.encode('utf-8')) + '\n')
+    except AttributeError:
+        f.write('No name found for this torrent')
+
+dt_user_begin = datetime.now()
+user_torrents = nyaa.get_user("HorribleSubs")
+dt_user_end = datetime.now()
+with open("test_files/nyaa_single_user_test.json", 'w') as f:
+    for torrent in user_torrents:
+        try:
+            # This prints it as byte like objects since unicode is fun
+            f.write(str(torrent.name.encode('utf-8')) + '\n')
+        except AttributeError:
+            f.write('No name found for this torrent')
+
+print(
+    "Latest torrents time:",
+    (dt_latest_torrents_end - dt_latest_torrents_begin).microseconds / 1000,
+    "msec")
+print(
+    "Test search time:",
+    (dt_search_end - dt_search_begin).microseconds / 1000,
+    "msec"
+)
+print(
+    "Single torrent time:",
+    (dt_single_torrent_end - dt_single_torrent_begin).microseconds / 1000,
+    "msec"
+)
+print(
+    "Single user time:",
+    (dt_user_end - dt_user_begin).microseconds / 1000,
+    "msec"
+)

tests/test_pantsu.py

@@ -0,0 +1,6 @@
+"""
+* Pantsu need some serious work
+Regular data single_torrent parser not working from other Nyaa alternatives
+Needs some work
+"""
+print("TODO")

tests/test_sukebei.py

@@ -0,0 +1,58 @@
+from NyaaPy.sukebei import SukebeiNyaa
+from datetime import datetime
+import json
+import os
+
+# Creating a folder for test_files
+# ! not included in github project.
+if not os.path.isdir("test_files"):
+    os.makedirs("test_files")
+
+nyaa = SukebeiNyaa()
+
+# Get fresh torrents
+dt_latest_torrents_begin = datetime.now()
+latest_torrents = nyaa.last_uploads(100)
+dt_latest_torrents_end = datetime.now()
+with open("test_files/sukebei_latest_torrent_test.json", 'w') as f:
+    json.dump(latest_torrents, f)
+
+# Search some nasty stuff
+dt_search_begin = datetime.now()
+test_search = nyaa.search("G Senjou no maou")
+dt_search_end = datetime.now()
+with open("test_files/sukebei_search_test.json", 'w') as f:
+    json.dump(test_search, f)
+
+# Get first torrent from found torrents
+dt_single_torrent_begin = datetime.now()
+single_torrent = nyaa.get(test_search[0]["id"])
+dt_single_torrent_end = datetime.now()
+with open("test_files/sukebei_single_torrent_test.json", 'w') as f:
+    json.dump(single_torrent, f)
+
+dt_user_begin = datetime.now()
+user_torrents = nyaa.get_user("RUNBKK")
+dt_user_end = datetime.now()
+with open("test_files/sukebei_single_user_test.json", 'w') as f:
+    json.dump(user_torrents, f)
+
+print(
+    "Latest torrents time:",
+    (dt_latest_torrents_end - dt_latest_torrents_begin).microseconds / 1000,
+    "msec")
+print(
+    "Test search time:",
+    (dt_search_end - dt_search_begin).microseconds / 1000,
+    "msec"
+)
+print(
+    "Single torrent time:",
+    (dt_single_torrent_end - dt_single_torrent_begin).microseconds / 1000,
+    "msec"
+)
+print(
+    "Single user time:",
+    (dt_user_end - dt_user_begin).microseconds / 1000,
+    "msec"
+)