Migrate to Redis (97259379) · Commits · e / infra / spot

README.rst

+4 −27

Original line number	Diff line number	Diff line
		@@ -17,32 +17,9 @@ See the `documentation <https://asciimoo.github.io/searx>`__ and the `wiki <http
		Setup MySql
		~~~~~~~~~~~

		Install MySql
		``$ sudo apt-get install mysql-server
		$ pip install pymysql``

		Start MySql
		``$ sudo service mysql start
		$ mysql -u root -p``

		Create a new database and give all rights to a new searx user
		change password!

		``mysql> create database searx;
		mysql> create user "searx"@"localhost" identified by "password";
		mysql> grant all on searx.* to "searx"@"localhost" identified by "password";``

		You can now quit the MySql console by typing ``quit`` and connect as searx user
		``$ mysql -u searx -p``

		Here are some commands to init the database
		``mysql> use searx;``

		``mysql> create table SEARCH_HISTORY(QUERY varchar(512), CATEGORY varchar(256), PAGENO int(11), PAGING tinyint(1), SAFE_SEARCH int(11), LANGUAGE varchar(8), TIME_RANGE varchar(16), ENGINES varchar(4096), RESULTS mediumtext, RESULTS_NUMBER int(11), ANSWERS varchar(2048), CORRECTIONS varchar(256), INFOBOXES varchar(8192), SUGGESTIONS varchar(1024), UNRESPONSIVE_ENGINES varchar(1024));``

		``mysql> quit``

		MySql is done !
		Install Redis
		``$ sudo apt-get install redis-server
		$ pip install redis``

		Installation
		~~~~~~~~~~~~
		@@ -52,7 +29,7 @@ Installation
		- install dependencies: ``./manage.sh update_packages``
		- edit your
		`settings.yml <https://github.com/asciimoo/searx/blob/master/searx/settings.yml>`__
		(set your ``secret_key`` and ``mysql password``!)
		(set your ``secret_key``!)
		- run ``python searx/webapp.py`` to start the application

		For all the details, follow this `step by step

requirements.txt

+1 −1

Original line number	Diff line number	Diff line
		pymysql
		redis
		certifi==2017.11.5
		flask==0.12.2
		flask-babel==0.11.2

searx/search_database.py

+54 −57

Original line number	Diff line number	Diff line
		@@ -2,7 +2,7 @@ import json
		import threading
		import urllib

		import pymysql
		import redis

		from searx.plugins import plugins
		from searx.query import SearchQuery
		@@ -32,59 +32,60 @@ class SearchData(object):
		self.unresponsive_engines = unresponsive_engines


		def _get_connection():
		return redis.StrictRedis(settings['host'], decode_responses=True)


		def read(q):
		time_range = q.time_range
		if q.time_range is None:
		q.time_range = ""
		connection = pymysql.connect(host=settings['host'], user=settings['user'], password=settings['password'],
		database=settings['database'])
		try:
		with connection.cursor() as cursor:
		sql = "SELECT RESULTS, PAGING, RESULTS_NUMBER, ANSWERS, CORRECTIONS, INFOBOXES, SUGGESTIONS, " \
		"UNRESPONSIVE_ENGINES FROM SEARCH_HISTORY WHERE QUERY='%s' AND CATEGORY='%s' AND PAGENO=%s AND " \
		"SAFE_SEARCH=%s AND LANGUAGE='%s' AND TIME_RANGE='%s' AND ENGINES='%s'"
		cursor.execute(
		sql % (e(q.query), q.categories[0], q.pageno, q.safesearch, q.lang, time_range, je(q.engines)))
		for response in cursor:
		results = jd(response[0])

		conn = _get_connection()
		key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
		e(q.query), je(q.engines), q.categories[0], q.lang, q.safesearch, q.pageno, time_range)
		response = conn.hgetall(key)
		if not response:
		return None
		results = jd(response['results'])
		for result in results:
		result['parsed_url'] = urlparse(result['url'])
		return SearchData(q, results, response[1] != 0, response[2], jds(response[3]),
		jds(response[4]), jd(response[5]), jds(response[6]), jds(response[7]))
		finally:
		connection.close()
		return None
		return SearchData(q, results, int(response['paging']) != 0, int(response['results_number']),
		jds(response['answers']), jds(response['corrections']), jd(response['infoboxes']),
		jds(response['suggestions']), jds(response['unresponsive_engines']))


		def save(d):
		connection = pymysql.connect(host=settings['host'], user=settings['user'], password=settings['password'],
		database=settings['database'])
		try:
		with connection.cursor() as cursor:
		sql = "INSERT INTO SEARCH_HISTORY(QUERY, CATEGORY, PAGENO, SAFE_SEARCH, LANGUAGE, TIME_RANGE, ENGINES, " \
		"RESULTS, PAGING, RESULTS_NUMBER, ANSWERS, CORRECTIONS, INFOBOXES, SUGGESTIONS, " \
		"UNRESPONSIVE_ENGINES) VALUES('%s', '%s', %s, %s, '%s', '%s', '%s', '%s', %s, %s, '%s', '%s', '%s'," \
		" '%s', '%s')"
		cursor.execute(sql % (e(d.query), d.categories[0], d.pageno, d.safe_search, d.language, d.time_range,
		je(d.engines), je(d.results), d.paging, d.results_number, jes(d.answers),
		jes(d.corrections), je(d.infoboxes), jes(d.suggestions), jes(d.unresponsive_engines)))
		connection.commit()
		finally:
		connection.close()
		conn = _get_connection()
		key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
		e(d.query), je(d.engines), d.categories[0], d.language, d.safe_search, d.pageno, d.time_range)
		mapping = {
		'query': e(d.query), 'category': d.categories[0], 'pageno': d.pageno, 'safe_search': d.safe_search,
		'language': d.language, 'time_range': d.time_range, 'engines': je(d.engines), 'results': je(d.results),
		'paging': d.paging, 'results_number': d.results_number, 'answers': jes(d.answers),
		'corrections': jes(d.corrections), 'infoboxes': je(d.infoboxes), 'suggestions': jes(d.suggestions),
		'unresponsive_engines': jes(d.unresponsive_engines)
		}
		conn.zadd('SEARCH_HISTORY_KEYS', conn.incr('SEARCH_HISTORY_INDEX'), key)
		conn.hmset(key, mapping)


		def get_twenty_queries(x):
		result = []
		connection = pymysql.connect(host=settings['host'], user=settings['user'], password=settings['password'],
		database=settings['database'])
		try:
		with connection.cursor() as cursor:
		cursor.execute("SELECT QUERY, ENGINES, CATEGORY, LANGUAGE , SAFE_SEARCH, PAGENO, TIME_RANGE FROM "
		"SEARCH_HISTORY LIMIT %s,20" % x)
		for row in cursor:
		result.append(SearchQuery(d(row[0]), jd(row[1]), [row[2]], row[3], row[4], row[5], row[6]))
		finally:
		connection.close()

		conn = _get_connection()
		keys = conn.zrange('SEARCH_HISTORY_KEYS', int(x), int(x) + 20)
		if not keys:
		return result

		pipe = conn.pipeline()
		for key in keys:
		pipe.hgetall(key)
		output = pipe.execute()
		for row in output:
		result.append(SearchQuery(d(row['query']), jd(row['engines']), [row['category']], row['language'],
		int(row['safe_search']), int(row['pageno']), row['time_range']))

		return result


		@@ -150,17 +151,13 @@ def search(request):


		def update(d):
		connection = pymysql.connect(host=settings['host'], user=settings['user'], password=settings['password'],
		database=settings['database'])
		try:
		with connection.cursor() as cursor:
		sql = "UPDATE SEARCH_HISTORY SET RESULTS='%s', PAGING=%s, RESULTS_NUMBER=%s, ANSWERS='%s', CORRECTIONS='%s', INFOBOXES='%s', SUGGESTIONS='%s', " \
		"UNRESPONSIVE_ENGINES='%s' WHERE QUERY='%s' AND CATEGORY='%s' AND PAGENO=%s AND " \
		"SAFE_SEARCH=%s AND LANGUAGE='%s' AND TIME_RANGE='%s' AND ENGINES='%s'"
		cursor.execute(sql % (je(d.results), d.paging, d.results_number, jes(d.answers), jes(d.corrections),
		je(d.infoboxes), jes(d.suggestions), jes(d.unresponsive_engines),
		e(d.query), d.categories[0], d.pageno, d.safe_search, d.language, d.time_range,
		je(d.engines)))
		connection.commit()
		finally:
		connection.close()
		conn = redis.StrictRedis(settings['host'])
		key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
		e(d.query), je(d.engines), d.categories[0], d.language, d.safe_search, d.pageno, d.time_range)
		current = conn.hgetall(key)
		current.update({
		'results': je(d.results), 'paging': d.paging, 'results_number': d.results_number,
		'answers': jes(d.answers), 'corrections': jes(d.corrections), 'infoboxes': je(d.infoboxes),
		'suggestions': jes(d.suggestions), 'unresponsive_engines': jes(d.unresponsive_engines)
		})
		conn.hmset(key, current)

searx/settings.yml

+1 −2

Original line number	Diff line number	Diff line
		@@ -15,11 +15,10 @@ server:
		image_proxy : False # Proxying image results through searx
		http_protocol_version : "1.0" # 1.0 and 1.1 are supported

		mysql:
		redis:
		host : "127.0.0.1"
		user : "searx"
		password : "password" # change this!
		database : "searx"
		upgrade_history: 86400 # in seconds (1day = 86400s)

		ui:

searx/webapp.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -811,7 +811,7 @@ running = threading.Event()


		def wait_updating(start_time):
		wait = settings['mysql']['upgrade_history'] - int(time.time() - start_time)
		wait = settings['redis']['upgrade_history'] - int(time.time() - start_time)
		if wait > 0:
		running.wait(wait)

		@@ -836,7 +836,7 @@ def update_results():

		def run():
		logger.debug('starting webserver on %s:%s', settings['server']['port'], settings['server']['bind_address'])
		search_database.settings = settings['mysql']
		search_database.settings = settings['redis']
		threading.Thread(target=update_results, name='results_updater').start()
		print "engine server starting"
		app.run(