- Fixed unicode in mininova and btjunkie search engines

src/search_engine/engines/btjunkie.py

@@ -1,4 +1,4 @@
-#VERSION: 2.0
+#VERSION: 2.1
 #AUTHORS: Christophe Dumez (chris@qbittorrent.org)
 
 # Redistribution and use in source and binary forms, with or without
@@ -27,8 +27,8 @@
 
 
 from novaprinter import prettyPrinter
+from helpers import retrieve_url
 import sgmllib
-import urllib
 import re
 
 class btjunkie(object):
@@ -72,11 +72,6 @@ class btjunkie(object):
         if not self.current_item.has_key('leech'):
           self.current_item['leech'] = ''
         self.current_item['leech']+= data.strip()
-        
-    def start_font(self, attr):
-      if isinstance(self.th_counter,int):
-        if self.th_counter == 0:
-          self.current_item['name'] += ' '
       
     def start_th(self,attr):
         if isinstance(self.th_counter,int):
@@ -99,7 +94,12 @@ class btjunkie(object):
     while True and i<11:
       results = []
       parser = self.SimpleSGMLParser(results, self.url)
-      dat = urllib.urlopen(self.url+'/search?q=%s&o=52&p=%d'%(what,i)).read()
+      dat = retrieve_url(self.url+'/search?q=%s&o=52&p=%d'%(what,i))
+      # Remove <font> tags from page
+      p = re.compile( '<[/]?font.*?>')
+      dat = p.sub('', dat)
+      #print dat
+      #return
       results_re = re.compile('(?s)class="tab_results">.*')
       for match in results_re.finditer(dat):
         res_tab = match.group(0)

src/search_engine/engines/mininova.py

@@ -1,4 +1,4 @@
-#VERSION: 1.13
+#VERSION: 1.2
 #AUTHORS: Fabien Devaux (fab@gnux.info)
 
 # Redistribution and use in source and binary forms, with or without
@@ -26,7 +26,7 @@
 # POSSIBILITY OF SUCH DAMAGE.
 
 from novaprinter import prettyPrinter
-import urllib
+from helpers import retrieve_url
 from xml.dom import minidom
 import re
 
@@ -64,12 +64,15 @@ class mininova(object):
 				return ''.join([ get_text(n) for n in txt.childNodes])
 		page = 1
 		while True and page<11:
+			file = open('/home/chris/mytest.txt', 'w')
+			file.write(self.url+'/search/%s/seeds/%d'%(what, page))
+			file.close()
 			res = 0
-			dat = urllib.urlopen(self.url+'/search/%s/seeds/%d'%(what, page)).read().decode('utf-8', 'replace')
+			dat = retrieve_url(self.url+'/search/%s/seeds/%d'%(what, page))
 			dat = re.sub("<a href=\"http://www.boardreader.com/index.php.*\"", "<a href=\"plop\"", dat)
 			dat = re.sub("<=", "&lt;=", dat)
 			dat = re.sub("&\s", "&amp; ", dat)
-			x = minidom.parseString(dat.encode('utf-8', 'replace'))
+			x = minidom.parseString(dat)
 			table = x.getElementsByTagName('table').item(0)
 			if not table: return
 			for tr in table.getElementsByTagName('tr'):

src/search_engine/engines/versions.txt

@@ -1,5 +1,5 @@
 isohunt: 1.1
 torrentreactor: 1.02
-btjunkie: 2.0
-mininova: 1.13
+btjunkie: 2.1
+mininova: 1.2
 piratebay: 1.04