Icycoide/searxng
1
0
Fork 1
mirror of https://github.com/searxng/searxng.git synced 2025-07-23 05:09:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

Replace langdetect with fasttext

Browse source
This commit is contained in:
ArtikusHG 2022-12-11 17:45:47 +02:00
parent a6d870d5cf
commit 1f8f8c1e91
6 changed files with 151 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

16
searxng_extra/update/update_engine_descriptions.py
View file

@ -17,14 +17,11 @@ from os.path import join
from lxml.html import fromstring
from langdetect import detect_langs
from langdetect.lang_detect_exception import LangDetectException
from searx.engines import wikidata, set_loggers
from searx.utils import extract_text, match_language
from searx.locales import LOCALE_NAMES, locales_initialize
from searx import searx_dir
from searx.utils import gen_useragent
from searx.utils import gen_useragent, detect_language
import searx.search
import searx.network
@ -117,17 +114,6 @@ def get_wikipedia_summary(lang, pageid):
return None
def detect_language(text):
try:
r = detect_langs(str(text)) # pylint: disable=E1101
except LangDetectException:
return None
if len(r) > 0 and r[0].prob > 0.95:
return r[0].lang
return None
def get_website_description(url, lang1, lang2=None):
headers = {
'User-Agent': gen_useragent(),