人工智能-项目实践-搜索引擎-基于searx部署属于你的搜索引擎

#!/usr/bin/env python ''' searx is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. searx is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with searx. If not, see < http://www.gnu.org/licenses/ >. (C) 2013- by Adam Tauber, <[email protected]> ''' if __name__ == '__main__': from sys import path from os.path import realpath, dirname path.append(realpath(dirname(realpath(__file__)) + '/../')) import hashlib import hmac import json import os import sys import requests from searx import logger logger = logger.getChild('webapp') try: from pygments import highlight from pygments.lexers import get_lexer_by_name from pygments.formatters import HtmlFormatter except: logger.critical("cannot import dependency: pygments") from sys import exit exit(1) try: from cgi import escape except: from html import escape from six import next from datetime import datetime, timedelta from time import time from werkzeug.middleware.proxy_fix import ProxyFix from flask import ( Flask, request, render_template, url_for, Response, make_response, redirect, send_from_directory ) from babel.support import Translations import flask_babel from flask_babel import Babel, gettext, format_date, format_decimal from flask.ctx import has_request_context from flask.json import jsonify from searx import brand from searx import settings, searx_dir, searx_debug from searx.exceptions import SearxParameterException from searx.engines import ( categories, engines, engine_shortcuts, get_engines_stats, initialize_engines ) from searx.utils import ( UnicodeWriter, highlight_content, html_to_text, get_resources_directory, get_static_files, get_result_templates, get_themes, gen_useragent, dict_subset, prettify_url, match_language ) from searx.version import VERSION_STRING from searx.languages import language_codes as languages from searx.search import SearchWithPlugins, get_search_query_from_webapp from searx.query import RawTextQuery from searx.autocomplete import searx_bang, backends as autocomplete_backends from searx.plugins import plugins from searx.plugins.oa_doi_rewrite import get_doi_resolver from searx.preferences import Preferences, ValidationException, LANGUAGE_CODES from searx.answerers import answerers from searx.url_utils import urlencode, urlparse, urljoin from searx.utils import new_hmac # check if the pyopenssl package is installed. # It is needed for SSL connection without trouble, see #298 try: import OpenSSL.SSL # NOQA except ImportError: logger.critical("The pyopenssl package has to be installed.\n" "Some HTTPS connections will fail") try: from cStringIO import StringIO except: from io import StringIO if sys.version_info[0] == 3: unicode = str PY3 = True else: logger.warning('\033[1;31m Python2 is no longer supported\033[0m') exit(1) # serve pages with HTTP/1.1 from werkzeug.serving import WSGIRequestHandler WSGIRequestHandler.protocol_version = "HTTP/{}".format(settings['server'].get('http_protocol_version', '1.0')) # about static static_path = get_resources_directory(searx_dir, 'static', settings['ui']['static_path']) logger.debug('static directory is %s', static_path) static_files = get_static_files(static_path) # about templates default_theme = settings['ui']['default_theme'] templates_path = get_resources_directory(searx_dir, 'templates', settings['ui']['templates_path']) logger.debug('templates directory is %s', templates_path) themes = get_themes(templates_path) result_templates = get_result_templates(templates_path) global_favicons = [] for indice, theme in enumerate(themes): global_favicons.append([]) theme_img_path = os.path.join(static_path, 'themes', theme, 'img', 'icons') for (dirpath, dirnames, filenames) in os.walk(theme_img_path): global_favicons[indice].extend(filenames) # Flask app app = Flask( __name__, static_folder=static_path, template_folder=templates_path ) app.jinja_env.trim_blocks = True app.jinja_env.lstrip_blocks = True app.jinja_env.add_extension('jinja2.ext.loopcontrols') app.secret_key = settings['server']['secret_key'] if not searx_debug \ or os.environ.get("WERKZEUG_RUN_MAIN") == "true" \ or os.environ.get('UWSGI_ORIGINAL_PROC_NAME') is not None: initialize_engines(settings['engines']) babel = Babel(app) rtl_locales = ['ar', 'arc', 'bcc', 'bqi', 'ckb', 'dv', 'fa', 'fa_IR', 'glk', 'he', 'ku', 'mzn', 'pnb', 'ps', 'sd', 'ug', 'ur', 'yi'] # used when translating category names _category_names = (gettext('files'), gettext('general'), gettext('music'), gettext('social media'), gettext('images'), gettext('videos'), gettext('it'), gettext('news'), gettext('map'), gettext('science')) outgoing_proxies = settings['outgoing'].get('proxies') or None _flask_babel_get_translations = flask_babel.get_translations # monkey patch for flask_babel.get_translations def _get_translations(): if has_request_context() and request.form.get('use-translation') == 'oc': babel_ext = flask_babel.current_app.extensions['babel'] return Translations.load(next(babel_ext.translation_directories), 'oc') return _flask_babel_get_translations() flask_babel.get_translations = _get_translations def _get_browser_language(request, lang_list): for lang in request.headers.get("Accept-Language", "en").split(","): if ';' in lang: lang = lang.split(';')[0] locale = match_language(lang, lang_list, fallback=None) if locale is not None: return locale return settings['search']['default_lang'] or 'en' @babel.localeselector def get_locale(): locale = _get_browser_language(request, settings['locales'].keys()) logger.debug("default locale from browser info is `%s`", locale) if request.preferences.get_value('locale') != '': locale = request.preferences.get_value('locale') if 'locale' in request.form\ and request.form['locale'] in settings['locales']: locale = request.form['locale'] if locale == 'zh_TW': locale = 'zh_Hant_TW' if locale == 'oc': request.form['use-translation'] = 'oc' locale = 'fr_FR' logger.debug("selected locale is `%s`", locale) return locale # code-highlighter @app.template_filter('code_highlighter') def code_highlighter(codelines, language=None): if not language: language = 'text' try: # find lexer by programing language lexer = get_lexer_by_name(language, stripall=True) except: # if lexer is not found, using default one logger.debug('highlighter cannot find lexer for {0}'.format(language)) lexer = get_lexer_by_name('text', stripall=True) html_code = '' tmp_code = '' last_line = None # parse lines for line, code in codelines: if not last_line: line_code_start = line # new codeblock is detected if last_line is not None and\ last_line + 1 != line: # highlight last codepart formatter = HtmlFormatter(linenos='inline', linenostart=line_code_start) html_code = html_code + highlight(tmp_code, lexer, formatter) # reset conditions for next codepart tmp_code = '' line_code_start = line