mirror of
https://github.com/searxng/searxng.git
synced 2025-09-01 15:58:37 +02:00
[mod] multithreading only in searx.search.* packages
it prepares the new architecture change, everything about multithreading in moved in the searx.search.* packages previously the call to the "init" function of the engines was done in searx.engines: * the network was not set (request not sent using the defined proxy) * it requires to monkey patch the code to avoid HTTP requests during the tests
This commit is contained in:
parent
d36adfa59f
commit
8c1a65d32f
10 changed files with 85 additions and 65 deletions
|
@ -5,7 +5,7 @@
|
|||
|
||||
"""
|
||||
|
||||
from time import time
|
||||
from timeit import default_timer
|
||||
import asyncio
|
||||
import httpx
|
||||
|
||||
|
@ -40,6 +40,15 @@ class OnlineProcessor(EngineProcessor):
|
|||
|
||||
engine_type = 'online'
|
||||
|
||||
def initialize(self):
|
||||
# set timeout for all HTTP requests
|
||||
searx.network.set_timeout_for_thread(self.engine.timeout, start_time=default_timer())
|
||||
# reset the HTTP total time
|
||||
searx.network.reset_time_for_thread()
|
||||
# set the network
|
||||
searx.network.set_context_network_name(self.engine_name)
|
||||
super().initialize()
|
||||
|
||||
def get_params(self, search_query, engine_category):
|
||||
params = super().get_params(search_query, engine_category)
|
||||
if params is None:
|
||||
|
@ -139,7 +148,7 @@ class OnlineProcessor(EngineProcessor):
|
|||
self.handle_exception(result_container, e, suspend=True)
|
||||
logger.error("engine {0} : HTTP requests timeout"
|
||||
"(search duration : {1} s, timeout: {2} s) : {3}"
|
||||
.format(self.engine_name, time() - start_time,
|
||||
.format(self.engine_name, default_timer() - start_time,
|
||||
timeout_limit,
|
||||
e.__class__.__name__))
|
||||
except (httpx.HTTPError, httpx.StreamError) as e:
|
||||
|
@ -147,7 +156,7 @@ class OnlineProcessor(EngineProcessor):
|
|||
self.handle_exception(result_container, e, suspend=True)
|
||||
logger.exception("engine {0} : requests exception"
|
||||
"(search duration : {1} s, timeout: {2} s) : {3}"
|
||||
.format(self.engine_name, time() - start_time,
|
||||
.format(self.engine_name, default_timer() - start_time,
|
||||
timeout_limit,
|
||||
e))
|
||||
except SearxEngineCaptchaException as e:
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue