Nyaa.si parser converted to LXML (except view user info)

NyaaPy/utils.py

@@ -3,9 +3,12 @@
 '''
 
 import re
+from lxml import etree
+from pprint import pprint
+
 
 def nyaa_categories(b):
-    c = b.replace('/?c=', '')
+    c = b.replace('?c=', '')
     cats = c.split('_')
 
     cat = cats[0]
@@ -69,89 +72,104 @@ def nyaa_categories(b):
 
     return category_name
 
-def parse_nyaa(table_rows, limit):
-    if limit == 0:
-        limit = len(table_rows)
+
+def parse_nyaa(request_text, limit):
+    parser = etree.HTMLParser()
+    tree = etree.fromstring(request_text, parser)
 
     torrents = []
 
-    for row in table_rows[:limit]:
+    # Going through table rows
+    for tr in tree.xpath("//tbody//tr")[:limit]:
         block = []
 
-        for td in row.find_all('td'):
-            if td.find_all('a'):
-                for link in td.find_all('a'):
-                    if link.get('href')[-9:] != '#comments':
-                        block.append(link.get('href'))
-                        if link.text.rstrip():
-                            block.append(link.text)
+        for td in tr.xpath("./td"):
+            for link in td.xpath("./a"):
 
-            if td.text.rstrip():
-                block.append(td.text.rstrip())
+                href = link.attrib.get("href").split('/')[-1]
 
-        if row.has_attr('class'):
-            if row['class'][0] == 'danger':
-                block.append("remake")
-            elif row['class'][0] == 'success':
-                block.append("trusted")
-            else:
-                block.append("default")
+                # Only caring about non-comment pages.
+                if href[-9:] != "#comments":
+                    block.append(href)
 
+                    if link.text and link.text.strip():
+                        block.append(link.text.strip())
+
+            if td.text and td.text.strip():
+                block.append(td.text.strip())
+
+        # Add type of torrent based on tr class.
+        if 'danger' in tr.attrib.get("class"):
+            block.append("remake")
+        elif 'success' in tr.attrib.get("class"):
+            block.append("trusted")
+        else:
+            block.append("default")
+
+        # Create torrent object
         try:
             torrent = {
-                'id': block[1].replace("/view/", ""),
+                'id': block[1],
                 'category': nyaa_categories(block[0]),
-                'url': "http://nyaa.si{}".format(block[1]),
+                'url': "https://nyaa.si/view/{}".format(block[1]),
                 'name': block[2],
-                'download_url': "http://nyaa.si{}".format(block[4]),
-                'magnet': block[5],
-                'size': block[6],
-                'date': block[7],
-                'seeders': block[8],
-                'leechers': block[9],
-                'completed_downloads': block[10],
-                'type': block[11],
+                'download_url': "https://nyaa.si/download/{}".format(block[3]),
+                'magnet': block[4],
+                'size': block[5],
+                'date': block[6],
+                'seeders': block[7],
+                'leechers': block[8],
+                'completed_downloads': block[9],
+                'type': block[10]
             }
-
             torrents.append(torrent)
-        except IndexError as ie:
+        except IndexError:
             pass
-
     return torrents
 
-def parse_single(content):
+
+def parse_single(request_text):
+    parser = etree.HTMLParser()
+    tree = etree.fromstring(request_text, parser)
+
     torrent = {}
     data = []
     torrent_files = []
 
-    for row in content[0].find_all('div', {'class': 'row'}):
-        for div in row.find_all('div', {'class': 'col-md-5'}):
-            data.append(div.text.replace("\n", ""))
+    # Find basic uploader info & torrent stats
+    for row in tree.xpath("//div[@class='row']"):
+        for div_text in row.xpath("./div[@class='col-md-5']//text()"):
+            d = div_text.strip()
+            if d:
+                data.append(d)
 
-    files = content[2].find('div',
-                            {'class', 'torrent-file-list'}).find_all('li')
+    # Find files, we need only text of the li element(s).
+    # Sorry about Pycodestyle aka PEP8 (E501) error
+    for el in tree.xpath("//div[contains(@class, 'torrent-file-list')]//li/text()"):
+        if el.rstrip():
+            torrent_files.append(el)
 
-    for file in files:
-        torrent_files.append(file.text)
-
-    torrent['title'] = re.sub('\n|\r|\t', '', content[0].find('h3', {
-        "class": "panel-title"}).text.replace("\n", ""))
+    torrent['title'] = \
+        tree.xpath("//h3[@class='panel-title']/text()")[0].strip()
     torrent['category'] = data[0]
-    torrent['uploader'] = data[2]
-    torrent['uploader_profile'] = "https://nyaa.si/user/{}".format(data[2])
-    torrent['website'] = re.sub('\t', '', data[4])
-    torrent['size'] = data[6]
-    torrent['date'] = data[1]
-    torrent['seeders'] = data[3]
-    torrent['leechers'] = data[5]
-    torrent['completed'] = data[7]
-    torrent['hash'] = data[8]
-    torrent['description'] = re.sub('\t', '', content[1].find('div', {
-        'id': 'torrent-description'}).text)
+    torrent['uploader'] = data[4]
+    torrent['uploader_profile'] = "http://nyaa.si/user/{}".format(data[4])
+    torrent['website'] = data[6]
+    torrent['size'] = data[8]
+    torrent['date'] = data[3]
+    torrent['seeders'] = data[5]
+    torrent['leechers'] = data[7]
+    torrent['completed'] = data[9]
+    torrent['hash'] = data[10]
     torrent['files'] = torrent_files
 
+    torrent['description'] = ""
+    for s in tree.xpath("//div[@id='torrent-description']"):
+        torrent['description'] += s.text
+
     return torrent
 
+
 def parse_sukebei(table_rows, limit):
     if limit == 0:
         limit = len(table_rows)
@@ -192,6 +210,7 @@ def parse_sukebei(table_rows, limit):
 
     return torrents
 
+
 def sukebei_categories(b):
     c = b.replace('/?c=', '')
     cats = c.split('_')
@@ -227,6 +246,7 @@ def sukebei_categories(b):
 
     return category_name
 
+
 # Pantsu Utils
 def query_builder(q, params):
     available_params = ["category", "page", "limit", "userID", "fromID",