Merge branch 'master' into http1.1

searx/engines/__init__.py

@@ -57,11 +57,17 @@ def load_module(filename):
 
 
 def load_engine(engine_data):
-    engine_name = engine_data['engine']
+
+    if '_' in engine_data['name']:
+        logger.error('Engine name conains underscore: "{}"'.format(engine_data['name']))
+        sys.exit(1)
+
+    engine_module = engine_data['engine']
+
     try:
-        engine = load_module(engine_name + '.py')
+        engine = load_module(engine_module + '.py')
     except:
-        logger.exception('Cannot load engine "{}"'.format(engine_name))
+        logger.exception('Cannot load engine "{}"'.format(engine_module))
         return None
 
     for param_name in engine_data:

searx/engines/digbt.py

@@ -40,7 +40,7 @@ def response(resp):
     results = list()
     for result in search_res:
         url = urljoin(URL, result.xpath('.//a[@title]/@href')[0])
-        title = result.xpath('.//a[@title]/text()')[0]
+        title = extract_text(result.xpath('.//a[@title]'))
         content = extract_text(result.xpath('.//div[@class="files"]'))
         files_data = extract_text(result.xpath('.//div[@class="tail"]')).split()
         filesize = get_torrent_size(files_data[FILESIZE], files_data[FILESIZE_MULTIPLIER])

searx/engines/kickass.py

@@ -16,13 +16,14 @@ from urllib import quote
 from lxml import html
 from operator import itemgetter
 from searx.engines.xpath import extract_text
+from searx.utils import get_torrent_size, convert_str_to_int
 
 # engine dependent config
 categories = ['videos', 'music', 'files']
 paging = True
 
 # search-url
-url = 'https://kickass.to/'
+url = 'https://kickass.cd/'
 search_url = url + 'search/{search_term}/{pageno}/'
 
 # specific xpath variables
@@ -57,41 +58,16 @@ def response(resp):
         href = urljoin(url, link.attrib['href'])
         title = extract_text(link)
         content = escape(extract_text(result.xpath(content_xpath)))
-        seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]
-        leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]
-        filesize = result.xpath('.//td[contains(@class, "nobr")]/text()')[0]
-        filesize_multiplier = result.xpath('.//td[contains(@class, "nobr")]//span/text()')[0]
-        files = result.xpath('.//td[contains(@class, "center")][2]/text()')[0]
+        seed = extract_text(result.xpath('.//td[contains(@class, "green")]'))
+        leech = extract_text(result.xpath('.//td[contains(@class, "red")]'))
+        filesize_info = extract_text(result.xpath('.//td[contains(@class, "nobr")]'))
+        files = extract_text(result.xpath('.//td[contains(@class, "center")][2]'))
 
-        # convert seed to int if possible
-        if seed.isdigit():
-            seed = int(seed)
-        else:
-            seed = 0
+        seed = convert_str_to_int(seed)
+        leech = convert_str_to_int(leech)
 
-        # convert leech to int if possible
-        if leech.isdigit():
-            leech = int(leech)
-        else:
-            leech = 0
-
-        # convert filesize to byte if possible
-        try:
-            filesize = float(filesize)
-
-            # convert filesize to byte
-            if filesize_multiplier == 'TB':
-                filesize = int(filesize * 1024 * 1024 * 1024 * 1024)
-            elif filesize_multiplier == 'GB':
-                filesize = int(filesize * 1024 * 1024 * 1024)
-            elif filesize_multiplier == 'MB':
-                filesize = int(filesize * 1024 * 1024)
-            elif filesize_multiplier == 'KB':
-                filesize = int(filesize * 1024)
-        except:
-            filesize = None
-
-        # convert files to int if possible
+        filesize, filesize_multiplier = filesize_info.split()
+        filesize = get_torrent_size(filesize, filesize_multiplier)
         if files.isdigit():
             files = int(files)
         else:

searx/engines/pdbe.py

@@ -0,0 +1,109 @@
+"""
+ PDBe (Protein Data Bank in Europe)
+
+ @website       https://www.ebi.ac.uk/pdbe
+ @provide-api   yes (https://www.ebi.ac.uk/pdbe/api/doc/search.html),
+                unlimited
+ @using-api     yes
+ @results       python dictionary (from json)
+ @stable        yes
+ @parse         url, title, content, img_src
+"""
+
+from json import loads
+from flask_babel import gettext
+
+categories = ['science']
+
+hide_obsolete = False
+
+# status codes of unpublished entries
+pdb_unpublished_codes = ['HPUB', 'HOLD', 'PROC', 'WAIT', 'AUTH', 'AUCO', 'REPL', 'POLC', 'REFI', 'TRSF', 'WDRN']
+# url for api query
+pdbe_solr_url = 'https://www.ebi.ac.uk/pdbe/search/pdb/select?'
+# base url for results
+pdbe_entry_url = 'https://www.ebi.ac.uk/pdbe/entry/pdb/{pdb_id}'
+# link to preview image of structure
+pdbe_preview_url = 'https://www.ebi.ac.uk/pdbe/static/entry/{pdb_id}_deposited_chain_front_image-200x200.png'
+
+
+def request(query, params):
+
+    params['url'] = pdbe_solr_url
+    params['method'] = 'POST'
+    params['data'] = {
+        'q': query,
+        'wt': "json"  # request response in parsable format
+    }
+    return params
+
+
+def construct_body(result):
+    # set title
+    title = result['title']
+
+    # construct content body
+    content = """{title}<br />{authors} {journal} <strong>{volume}</strong>&nbsp;{page} ({year})"""
+
+    # replace placeholders with actual content
+    try:
+        if result['journal']:
+            content = content.format(
+                title=result['citation_title'],
+                authors=result['entry_author_list'][0], journal=result['journal'], volume=result['journal_volume'],
+                page=result['journal_page'], year=result['citation_year'])
+        else:
+            content = content.format(
+                title=result['citation_title'],
+                authors=result['entry_author_list'][0], journal='', volume='', page='', year=result['release_year'])
+        img_src = pdbe_preview_url.format(pdb_id=result['pdb_id'])
+    except (KeyError):
+        content = None
+        img_src = None
+
+    # construct url for preview image
+    try:
+        img_src = pdbe_preview_url.format(pdb_id=result['pdb_id'])
+    except (KeyError):
+        img_src = None
+
+    return [title, content, img_src]
+
+
+def response(resp):
+
+    results = []
+    json = loads(resp.text)['response']['docs']
+
+    # parse results
+    for result in json:
+        # catch obsolete entries and mark them accordingly
+        if result['status'] in pdb_unpublished_codes:
+            continue
+        if hide_obsolete:
+            continue
+        if result['status'] == 'OBS':
+            # expand title to add some sort of warning message
+            title = gettext('{title}&nbsp;(OBSOLETE)').format(title=result['title'])
+            superseded_url = pdbe_entry_url.format(pdb_id=result['superseded_by'])
+
+            # since we can't construct a proper body from the response, we'll make up our own
+            msg_superseded = gettext("This entry has been superseded by")
+            content = '<em>{msg_superseded} \<a href="{url}">{pdb_id}</a></em>'.format(
+                msg_superseded=msg_superseded,
+                url=superseded_url,
+                pdb_id=result['superseded_by'], )
+
+            # obsoleted entries don't have preview images
+            img_src = None
+        else:
+            title, content, img_src = construct_body(result)
+
+        results.append({
+            'url': pdbe_entry_url.format(pdb_id=result['pdb_id']),
+            'title': title,
+            'content': content,
+            'img_src': img_src
+        })
+
+    return results

searx/engines/seedpeer.py

@@ -0,0 +1,78 @@
+#  Seedpeer (Videos, Music, Files)
+#
+# @website     http://seedpeer.eu
+# @provide-api no (nothing found)
+#
+# @using-api   no
+# @results     HTML (using search portal)
+# @stable      yes (HTML can change)
+# @parse       url, title, content, seed, leech, magnetlink
+
+from urlparse import urljoin
+from cgi import escape
+from urllib import quote
+from lxml import html
+from operator import itemgetter
+from searx.engines.xpath import extract_text
+
+
+url = 'http://www.seedpeer.eu/'
+search_url = url + 'search/{search_term}/7/{page_no}.html'
+# specific xpath variables
+torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
+alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
+title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
+alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
+seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
+alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
+peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
+alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
+age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
+alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
+size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
+alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
+
+
+# do search-request
+def request(query, params):
+    params['url'] = search_url.format(search_term=quote(query),
+                                      page_no=params['pageno'] - 1)
+    return params
+
+
+# get response from search-request
+def response(resp):
+    results = []
+    dom = html.fromstring(resp.text)
+    torrent_links = dom.xpath(torrent_xpath)
+    if len(torrent_links) > 0:
+        seeds = dom.xpath(seeds_xpath)
+        peers = dom.xpath(peers_xpath)
+        titles = dom.xpath(title_xpath)
+        sizes = dom.xpath(size_xpath)
+        ages = dom.xpath(age_xpath)
+    else:  # under ~5 results uses a different xpath
+        torrent_links = dom.xpath(alternative_torrent_xpath)
+        seeds = dom.xpath(alternative_seeds_xpath)
+        peers = dom.xpath(alternative_peers_xpath)
+        titles = dom.xpath(alternative_title_xpath)
+        sizes = dom.xpath(alternative_size_xpath)
+        ages = dom.xpath(alternative_age_xpath)
+    # return empty array if nothing is found
+    if not torrent_links:
+        return []
+
+    # parse results
+    for index, result in enumerate(torrent_links):
+        link = result.attrib.get('href')
+        href = urljoin(url, link)
+        results.append({'url': href,
+                        'title': titles[index].text_content(),
+                        'content': '{}, {}'.format(sizes[index], ages[index]),
+                        'seed': seeds[index],
+                        'leech': peers[index],
+
+                        'template': 'torrent.html'})
+
+    # return results sorted by seeder
+    return sorted(results, key=itemgetter('seed'), reverse=True)

searx/settings.yml

@@ -18,6 +18,12 @@ ui:
     default_theme : oscar # ui theme
     default_locale : "" # Default interface locale - leave blank to detect from browser information or use codes from the 'locales' config section
 
+# searx supports result proxification using an external service: https://github.com/asciimoo/morty
+# uncomment below section if you have running morty proxy
+#result_proxy:
+#    url : http://127.0.0.1:3000/
+#    key : your_morty_proxy_key
+
 outgoing: # communication with search engines
     request_timeout : 2.0 # seconds
     useragent_suffix : "" # suffix of searx_useragent, could contain informations like an email address to the administrator
@@ -301,6 +307,12 @@ engines:
     timeout : 6.0
     disabled : True
 
+  - name: kickass
+    engine : kickass
+    shortcut : kc
+    timeout : 4.0
+    disabled : True
+
   - name : microsoft academic
     engine : json_engine
     paging : True
@@ -339,6 +351,13 @@ engines:
     disabled : True
     shortcut : or
 
+  - name : pdbe
+    engine : pdbe
+    shortcut : pdb
+# Hide obsolete PDB entries.
+# Default is not to hide obsolete structures
+#    hide_obsolete : False
+
   - name : photon
     engine : photon
     shortcut : ph
@@ -377,7 +396,7 @@ engines:
     timeout : 10.0
     disabled : True
 
-  - name : scanr_structures
+  - name : scanr structures
     shortcut: scs
     engine : scanr_structures
     disabled : True
@@ -495,6 +514,12 @@ engines:
     timeout: 6.0
     categories : science
 
+  - name : seedpeer
+    engine : seedpeer
+    shortcut: speu
+    categories: files, music, videos
+    disabled: True
+
   - name : dictzone
     engine : dictzone
     shortcut : dc

searx/settings_robot.yml

@@ -15,7 +15,7 @@ server:
 
 ui:
     themes_path : ""
-    default_theme : default
+    default_theme : legacy
     default_locale : ""
 
 outgoing:
@@ -23,12 +23,12 @@ outgoing:
     useragent_suffix : ""
 
 engines:
-  - name : general_dummy
+  - name : general dummy
     engine : dummy
     categories : general
     shortcut : gd
 
-  - name : dummy_dummy
+  - name : dummy dummy
     engine : dummy
     categories : dummy
     shortcut : dd

searx/templates/courgette/base.html

@@ -22,7 +22,7 @@
         {% endblock %}
         {% block meta %}{% endblock %}
         {% block head %}
-        <link title="searx" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
+        <link title="{{ instance_name }}" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
         {% endblock %}
         <script type="text/javascript">
             searx = {};

searx/templates/legacy/404.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block content %}
 <div class="center">
     <h1>{{ _('Page not found') }}</h1>

searx/templates/legacy/about.html

@@ -1,6 +1,6 @@
-{% extends 'default/base.html' %}
+{% extends 'legacy/base.html' %}
 {% block content %}
-{% include 'default/github_ribbon.html' %}
+{% include 'legacy/github_ribbon.html' %}
 <div class="row"{% if rtl %} dir="ltr"{% endif %}>
     <h1>About <a href="{{ url_for('index') }}">searx</a></h1>
 

searx/templates/legacy/base.html

@@ -17,7 +17,7 @@
         {% endblock %}
         {% block meta %}{% endblock %}
         {% block head %}
-        <link title="searx" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
+        <link title="{{ instance_name }}" type="application/opensearchdescription+xml" rel="search" href="{{ url_for('opensearch') }}"/>
         {% endblock %}
     </head>
     <body>

searx/templates/legacy/index.html

@@ -1,8 +1,8 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block content %}
 <div class="center">
     <div class="title"><h1>searx</h1></div>
-    {% include 'default/search.html' %}
+    {% include 'legacy/search.html' %}
     <p class="top_margin">
     	{% if rtl %}
     	<a href="{{ url_for('preferences') }}" class="hmarg">{{ _('preferences') }}</a>
@@ -13,6 +13,6 @@
         {% endif %}
     </p>
 </div>
-{% include 'default/github_ribbon.html' %}
+{% include 'legacy/github_ribbon.html' %}
 {% endblock %}
 

searx/templates/legacy/preferences.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <div class="row">
@@ -8,7 +8,7 @@
     <fieldset>
         <legend>{{ _('Default categories') }}</legend>
         {% set display_tooltip = false %}
-        {% include 'default/categories.html' %}
+        {% include 'legacy/categories.html' %}
     </fieldset>
     <fieldset>
         <legend>{{ _('Search language') }}</legend>

searx/templates/legacy/results.html

@@ -1,10 +1,10 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block title %}{{ q }} - {% endblock %}
 {% block meta %}<link rel="alternate" type="application/rss+xml" title="Searx search: {{ q }}" href="{{ url_for('index') }}?q={{ q|urlencode }}&amp;format=rss&amp;{% for category in selected_categories %}category_{{ category }}=1&amp;{% endfor %}pageno={{ pageno }}">{% endblock %}
 {% block content %}
 <div class="preferences_container right"><a href="{{ url_for('preferences') }}" id="preferences"><span>preferences</span></a></div>
 <div class="small search center">
-    {% include 'default/search.html' %}
+    {% include 'legacy/search.html' %}
 </div>
 <div id="results">
     <div id="sidebar">
@@ -55,16 +55,16 @@
     {% if infoboxes %}
     <div id="infoboxes">
       {% for infobox in infoboxes %}
-         {% include 'default/infobox.html' %}
+         {% include 'legacy/infobox.html' %}
       {% endfor %}
     </div>
     {% endif %}
 
     {% for result in results %}
         {% if result['template'] %}
-            {% include get_result_template('default', result['template']) %}
+            {% include get_result_template('legacy', result['template']) %}
         {% else %}
-            {% include 'default/result_templates/default.html' %}
+            {% include 'legacy/result_templates/default.html' %}
         {% endif %}
     {% endfor %}
 

searx/templates/legacy/search.html

@@ -4,5 +4,5 @@
         <input type="submit" value="search" id="search_submit" />
     </div>
     {% set display_tooltip = true %}
-    {% include 'default/categories.html' %}
+    {% include 'legacy/categories.html' %}
 </form>

searx/templates/legacy/stats.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <h2>{{ _('Engine stats') }}</h2>

searx/templates/oscar/macros.html

@@ -33,6 +33,9 @@
         <span class="label label-default">{{ engine }}</span>
     {% endfor %}
     <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
+    {% if proxify %}
+    <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
+    {% endif %}
 </div>
     <div class="text-muted"><small>{{ result.pretty_url }}</small></div>
 {%- endmacro %}
@@ -44,6 +47,9 @@
         <span class="label label-default">{{ engine }}</span>
     {% endfor %}
     <small>{{ result_link("https://web.archive.org/web/" + result.url, icon('link') + _('cached'), "text-info") }}</small>
+    {% if proxify %}
+    <small>{{ result_link(proxify(result.url), icon('sort') + _('proxied'), "text-info") }}</small>
+    {% endif %}
     <div class="text-muted"><small>{{ result.pretty_url }}</small></div>
 {%- endmacro %}
 

searx/templates/pix-art/preferences.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <div class="row">

searx/templates/pix-art/stats.html

@@ -1,4 +1,4 @@
-{% extends "default/base.html" %}
+{% extends "legacy/base.html" %}
 {% block head %} {% endblock %}
 {% block content %}
 <h2>{{ _('Engine stats') }}</h2>

searx/utils.py

@@ -252,12 +252,27 @@ def get_torrent_size(filesize, filesize_multiplier):
             filesize = int(filesize * 1024 * 1024)
         elif filesize_multiplier == 'KB':
             filesize = int(filesize * 1024)
+        elif filesize_multiplier == 'TiB':
+            filesize = int(filesize * 1000 * 1000 * 1000 * 1000)
+        elif filesize_multiplier == 'GiB':
+            filesize = int(filesize * 1000 * 1000 * 1000)
+        elif filesize_multiplier == 'MiB':
+            filesize = int(filesize * 1000 * 1000)
+        elif filesize_multiplier == 'KiB':
+            filesize = int(filesize * 1000)
     except:
         filesize = None
 
     return filesize
 
 
+def convert_str_to_int(number_str):
+    if number_str.isdigit():
+        return int(number_str)
+    else:
+        return 0
+
+
 def is_valid_lang(lang):
     is_abbr = (len(lang) == 2)
     if is_abbr:

searx/webapp.py

@@ -22,10 +22,11 @@ if __name__ == '__main__':
     from os.path import realpath, dirname
     path.append(realpath(dirname(realpath(__file__)) + '/../'))
 
-import json
 import cStringIO
-import os
 import hashlib
+import hmac
+import json
+import os
 import requests
 
 from searx import logger
@@ -245,6 +246,20 @@ def url_for_theme(endpoint, override_theme=None, **values):
     return url_for(endpoint, **values)
 
 
+def proxify(url):
+    if url.startswith('//'):
+        url = 'https:' + url
+
+    if not settings.get('result_proxy'):
+        return url
+
+    h = hmac.new(settings['result_proxy']['key'], url.encode('utf-8'), hashlib.sha256).hexdigest()
+
+    return '{0}?{1}'.format(settings['result_proxy']['url'],
+                            urlencode(dict(mortyurl=url.encode('utf-8'),
+                                           mortyhash=h)))
+
+
 def image_proxify(url):
 
     if url.startswith('//'):
@@ -253,8 +268,7 @@ def image_proxify(url):
     if not request.preferences.get_value('image_proxy'):
         return url
 
-    hash_string = url + settings['server']['secret_key']
-    h = hashlib.sha256(hash_string.encode('utf-8')).hexdigest()
+    h = hmac.new(settings['server']['secret_key'], url.encode('utf-8'), hashlib.sha256).hexdigest()
 
     return '{0}?{1}'.format(url_for('image_proxy'),
                             urlencode(dict(url=url.encode('utf-8'), h=h)))
@@ -313,6 +327,8 @@ def render(template_name, override_theme=None, **kwargs):
 
     kwargs['image_proxify'] = image_proxify
 
+    kwargs['proxify'] = proxify if settings.get('result_proxy') else None
+
     kwargs['get_result_template'] = get_result_template
 
     kwargs['theme'] = get_current_theme_name(override=override_theme)
@@ -602,7 +618,7 @@ def image_proxy():
     if not url:
         return '', 400
 
-    h = hashlib.sha256(url + settings['server']['secret_key'].encode('utf-8')).hexdigest()
+    h = hmac.new(settings['server']['secret_key'], url, hashlib.sha256).hexdigest()
 
     if h != request.args.get('h'):
         return '', 400
@@ -660,6 +676,7 @@ Allow: /
 Allow: /about
 Disallow: /stats
 Disallow: /preferences
+Disallow: /*?*q=*
 """, mimetype='text/plain')