[httpx] replace searx.poolrequests by searx.network

settings.yml:

* outgoing.networks:
   * can contains network definition
   * propertiers: enable_http, verify, http2, max_connections, max_keepalive_connections,
     keepalive_expiry, local_addresses, support_ipv4, support_ipv6, proxies, max_redirects, retries
   * retries: 0 by default, number of times searx retries to send the HTTP request (using different IP & proxy each time)
   * local_addresses can be "192.168.0.1/24" (it supports IPv6)
   * support_ipv4 & support_ipv6: both True by default
     see https://github.com/searx/searx/pull/1034
* each engine can define a "network" section:
   * either a full network description
   * either reference an existing network

* all HTTP requests of engine use the same HTTP configuration (it was not the case before, see proxy configuration in master)

searx/engines/__init__.py

@@ -27,7 +27,7 @@ from searx import settings
 from searx import logger
 from searx.data import ENGINES_LANGUAGES
 from searx.exceptions import SearxEngineResponseException
-from searx.poolrequests import get, get_proxy_cycles
+from searx.network import get, initialize as initialize_network, set_context_network_name
 from searx.utils import load_module, match_language, get_engine_from_settings, gen_useragent
 
 
@@ -89,8 +89,6 @@ def load_engine(engine_data):
                 engine.categories = []
             else:
                 engine.categories = list(map(str.strip, param_value.split(',')))
-        elif param_name == 'proxies':
-            engine.proxies = get_proxy_cycles(param_value)
         else:
             setattr(engine, param_name, param_value)
 
@@ -289,9 +287,11 @@ def load_engines(engine_list):
 
 def initialize_engines(engine_list):
     load_engines(engine_list)
+    initialize_network(engine_list, settings['outgoing'])
 
     def engine_init(engine_name, init_fn):
         try:
+            set_context_network_name(engine_name)
             init_fn(get_engine_from_settings(engine_name))
         except SearxEngineResponseException as exc:
             logger.warn('%s engine: Fail to initialize // %s', engine_name, exc)

searx/engines/duckduckgo.py

@@ -6,7 +6,7 @@
 from lxml.html import fromstring
 from json import loads
 from searx.utils import extract_text, match_language, eval_xpath, dict_subset
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {

searx/engines/duckduckgo_images.py

@@ -8,7 +8,7 @@ from urllib.parse import urlencode
 from searx.exceptions import SearxEngineAPIException
 from searx.engines.duckduckgo import get_region_code
 from searx.engines.duckduckgo import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {

searx/engines/gigablast.py

@@ -8,7 +8,7 @@ import re
 from json import loads
 from urllib.parse import urlencode
 # from searx import logger
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {

searx/engines/pubmed.py

@@ -7,7 +7,7 @@ from flask_babel import gettext
 from lxml import etree
 from datetime import datetime
 from urllib.parse import urlencode
-from searx.poolrequests import get
+from searx.network import get
 
 # about
 about = {

searx/engines/qwant.py

@@ -8,7 +8,7 @@ from json import loads
 from urllib.parse import urlencode
 from searx.utils import html_to_text, match_language
 from searx.exceptions import SearxEngineAPIException, SearxEngineCaptchaException
-from searx.raise_for_httperror import raise_for_httperror
+from searx.network import raise_for_httperror
 
 # about
 about = {

searx/engines/seznam.py

@@ -5,7 +5,7 @@
 
 from urllib.parse import urlencode
 from lxml import html
-from searx.poolrequests import get
+from searx.network import get
 from searx.exceptions import SearxEngineAccessDeniedException
 from searx.utils import (
     extract_text,

searx/engines/soundcloud.py

@@ -9,7 +9,7 @@ from lxml import html
 from dateutil import parser
 from urllib.parse import quote_plus, urlencode
 from searx import logger
-from searx.poolrequests import get as http_get
+from searx.network import get as http_get
 
 # about
 about = {

searx/engines/spotify.py

@@ -5,9 +5,10 @@
 
 from json import loads
 from urllib.parse import urlencode
-import searx.poolrequests as requests
 import base64
 
+from searx.network import post as http_post
+
 # about
 about = {
     "website": 'https://www.spotify.com',
@@ -38,7 +39,7 @@ def request(query, params):
 
     params['url'] = search_url.format(query=urlencode({'q': query}), offset=offset)
 
-    r = requests.post(
+    r = http_post(
         'https://accounts.spotify.com/api/token',
         data={'grant_type': 'client_credentials'},
         headers={'Authorization': 'Basic ' + base64.b64encode(

searx/engines/wikidata.py

@@ -12,7 +12,7 @@ from babel.dates import format_datetime, format_date, format_time, get_datetime_
 
 from searx import logger
 from searx.data import WIKIDATA_UNITS
-from searx.poolrequests import post, get
+from searx.network import post, get
 from searx.utils import match_language, searx_useragent, get_string_replaces_function
 from searx.external_urls import get_external_url, get_earth_coordinates_url, area_to_osm_zoom
 from searx.engines.wikipedia import _fetch_supported_languages, supported_languages_url  # NOQA # pylint: disable=unused-import

searx/engines/wikipedia.py

@@ -7,7 +7,7 @@ from urllib.parse import quote
 from json import loads
 from lxml.html import fromstring
 from searx.utils import match_language, searx_useragent
-from searx.raise_for_httperror import raise_for_httperror
+from searx.network import raise_for_httperror
 
 # about
 about = {

searx/engines/wolframalpha_noapi.py

@@ -7,7 +7,7 @@ from json import loads
 from time import time
 from urllib.parse import urlencode
 
-from searx.poolrequests import get as http_get
+from searx.network import get as http_get
 
 # about
 about = {

searx/engines/wordnik.py

@@ -6,7 +6,7 @@
 from lxml.html import fromstring
 from searx import logger
 from searx.utils import extract_text
-from searx.raise_for_httperror import raise_for_httperror
+from searx.network import raise_for_httperror
 
 logger = logger.getChild('Wordnik engine')
 

searx/engines/yggtorrent.py

@@ -8,7 +8,7 @@ from operator import itemgetter
 from datetime import datetime
 from urllib.parse import quote
 from searx.utils import extract_text, get_torrent_size
-from searx.poolrequests import get as http_get
+from searx.network import get as http_get
 
 # about
 about = {