[enh] Add multiple outgoing proxies

credits go to @bauruine see https://github.com/searx/searx/pull/1958
This commit is contained in:
Alexandre Flament 2020-11-16 12:44:07 +01:00
parent 2fc3b17c85
commit 3786920df9
7 changed files with 172 additions and 36 deletions

View file

@ -111,6 +111,32 @@ def get_time_for_thread():
return threadLocal.total_time
def get_proxy_cycles(proxy_settings):
if not proxy_settings:
return None
# Backwards compatibility for single proxy in settings.yml
for protocol, proxy in proxy_settings.items():
if isinstance(proxy, str):
proxy_settings[protocol] = [proxy]
for protocol in proxy_settings:
proxy_settings[protocol] = cycle(proxy_settings[protocol])
return proxy_settings
GLOBAL_PROXY_CYCLES = get_proxy_cycles(settings['outgoing'].get('proxies'))
def get_proxies(proxy_cycles):
if proxy_cycles:
return {protocol: next(proxy_cycle) for protocol, proxy_cycle in proxy_cycles.items()}
return None
def get_global_proxies():
return get_proxies(GLOBAL_PROXY_CYCLES)
def request(method, url, **kwargs):
"""same as requests/requests/api.py request(...)"""
time_before_request = time()
@ -119,8 +145,8 @@ def request(method, url, **kwargs):
session = SessionSinglePool()
# proxies
if kwargs.get('proxies') is None:
kwargs['proxies'] = settings['outgoing'].get('proxies')
if not kwargs.get('proxies'):
kwargs['proxies'] = get_global_proxies()
# timeout
if 'timeout' in kwargs: