[mod] Add searx.data module

Instead of loading the data/*.json in different location,
load these files in the new searx.data module.
This commit is contained in:
Alexandre Flament 2020-10-05 13:50:33 +02:00
parent e30dc2f0ba
commit a9dc54bebc
5 changed files with 37 additions and 41 deletions

View file

@ -1,13 +1,10 @@
# -*- coding: utf-8 -*-
import os
import sys
import re
import json
import importlib
from numbers import Number
from os.path import splitext, join
from io import open
from random import choice
from html.parser import HTMLParser
from urllib.parse import urljoin, urlparse, unquote
@ -18,6 +15,7 @@ from babel.core import get_global
from searx import settings
from searx.data import USER_AGENTS
from searx.version import VERSION_STRING
from searx.languages import language_codes
from searx import logger
@ -31,9 +29,6 @@ blocked_tags = ('script',
ecma_unescape4_re = re.compile(r'%u([0-9a-fA-F]{4})', re.UNICODE)
ecma_unescape2_re = re.compile(r'%([0-9a-fA-F]{2})', re.UNICODE)
useragents = json.loads(open(os.path.dirname(os.path.realpath(__file__))
+ "/data/useragents.json", 'r', encoding='utf-8').read())
xpath_cache = dict()
lang_to_lc_cache = dict()
@ -50,7 +45,7 @@ def gen_useragent(os=None):
See searx/data/useragents.json
"""
return str(useragents['ua'].format(os=os or choice(useragents['os']), version=choice(useragents['versions'])))
return str(USER_AGENTS['ua'].format(os=os or choice(USER_AGENTS['os']), version=choice(USER_AGENTS['versions'])))
class HTMLTextExtractorException(Exception):