Commit 17468b6e authored by Omer's avatar Omer

Migrate to Redis

parent 458c34c9
......@@ -17,32 +17,9 @@ See the `documentation <https://asciimoo.github.io/searx>`__ and the `wiki <http
Setup MySql
~~~~~~~~~~~
**Install MySql**
``$ sudo apt-get install mysql-server
$ pip install pymysql``
**Start MySql**
``$ sudo service mysql start
$ mysql -u root -p``
**Create a new database and give all rights to a new searx user**
change password!
``mysql> create database searx;
mysql> create user "searx"@"localhost" identified by "password";
mysql> grant all on searx.* to "searx"@"localhost" identified by "password";``
**You can now quit the MySql console by typing ``quit`` and connect as searx user**
``$ mysql -u searx -p``
**Here are some commands to init the database**
``mysql> use searx;``
``mysql> create table SEARCH_HISTORY(QUERY varchar(512), CATEGORY varchar(256), PAGENO int(11), PAGING tinyint(1), SAFE_SEARCH int(11), LANGUAGE varchar(8), TIME_RANGE varchar(16), ENGINES varchar(4096), RESULTS mediumtext, RESULTS_NUMBER int(11), ANSWERS varchar(2048), CORRECTIONS varchar(256), INFOBOXES varchar(8192), SUGGESTIONS varchar(1024), UNRESPONSIVE_ENGINES varchar(1024));``
``mysql> quit``
MySql is done !
**Install Redis**
``$ sudo apt-get install redis-server
$ pip install redis``
Installation
~~~~~~~~~~~~
......@@ -52,7 +29,7 @@ Installation
- install dependencies: ``./manage.sh update_packages``
- edit your
`settings.yml <https://github.com/asciimoo/searx/blob/master/searx/settings.yml>`__
(set your ``secret_key`` and ``mysql password``!)
(set your ``secret_key``!)
- run ``python searx/webapp.py`` to start the application
For all the details, follow this `step by step
......
......@@ -2,7 +2,7 @@ import json
import threading
import urllib
import pymysql
import redis
from searx.plugins import plugins
from searx.query import SearchQuery
......@@ -32,59 +32,60 @@ class SearchData(object):
self.unresponsive_engines = unresponsive_engines
def _get_connection():
return redis.StrictRedis(settings['host'], decode_responses=True)
def read(q):
time_range = q.time_range
if q.time_range is None:
q.time_range = ""
connection = pymysql.connect(host=settings['host'], user=settings['user'], password=settings['password'],
database=settings['database'])
try:
with connection.cursor() as cursor:
sql = "SELECT RESULTS, PAGING, RESULTS_NUMBER, ANSWERS, CORRECTIONS, INFOBOXES, SUGGESTIONS, " \
"UNRESPONSIVE_ENGINES FROM SEARCH_HISTORY WHERE QUERY='%s' AND CATEGORY='%s' AND PAGENO=%s AND " \
"SAFE_SEARCH=%s AND LANGUAGE='%s' AND TIME_RANGE='%s' AND ENGINES='%s'"
cursor.execute(
sql % (e(q.query), q.categories[0], q.pageno, q.safesearch, q.lang, time_range, je(q.engines)))
for response in cursor:
results = jd(response[0])
for result in results:
result['parsed_url'] = urlparse(result['url'])
return SearchData(q, results, response[1] != 0, response[2], jds(response[3]),
jds(response[4]), jd(response[5]), jds(response[6]), jds(response[7]))
finally:
connection.close()
return None
conn = _get_connection()
key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
e(q.query), je(q.engines), q.categories[0], q.lang, q.safesearch, q.pageno, time_range)
response = conn.hgetall(key)
if not response:
return None
results = jd(response['results'])
for result in results:
result['parsed_url'] = urlparse(result['url'])
return SearchData(q, results, int(response['paging']) != 0, int(response['results_number']),
jds(response['answers']), jds(response['corrections']), jd(response['infoboxes']),
jds(response['suggestions']), jds(response['unresponsive_engines']))
def save(d):
connection = pymysql.connect(host=settings['host'], user=settings['user'], password=settings['password'],
database=settings['database'])
try:
with connection.cursor() as cursor:
sql = "INSERT INTO SEARCH_HISTORY(QUERY, CATEGORY, PAGENO, SAFE_SEARCH, LANGUAGE, TIME_RANGE, ENGINES, " \
"RESULTS, PAGING, RESULTS_NUMBER, ANSWERS, CORRECTIONS, INFOBOXES, SUGGESTIONS, " \
"UNRESPONSIVE_ENGINES) VALUES('%s', '%s', %s, %s, '%s', '%s', '%s', '%s', %s, %s, '%s', '%s', '%s'," \
" '%s', '%s')"
cursor.execute(sql % (e(d.query), d.categories[0], d.pageno, d.safe_search, d.language, d.time_range,
je(d.engines), je(d.results), d.paging, d.results_number, jes(d.answers),
jes(d.corrections), je(d.infoboxes), jes(d.suggestions), jes(d.unresponsive_engines)))
connection.commit()
finally:
connection.close()
conn = _get_connection()
key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
e(d.query), je(d.engines), d.categories[0], d.language, d.safe_search, d.pageno, d.time_range)
mapping = {
'query': e(d.query), 'category': d.categories[0], 'pageno': d.pageno, 'safe_search': d.safe_search,
'language': d.language, 'time_range': d.time_range, 'engines': je(d.engines), 'results': je(d.results),
'paging': d.paging, 'results_number': d.results_number, 'answers': jes(d.answers),
'corrections': jes(d.corrections), 'infoboxes': je(d.infoboxes), 'suggestions': jes(d.suggestions),
'unresponsive_engines': jes(d.unresponsive_engines)
}
conn.zadd('SEARCH_HISTORY_KEYS', conn.incr('SEARCH_HISTORY_INDEX'), key)
conn.hmset(key, mapping)
def get_twenty_queries(x):
result = []
connection = pymysql.connect(host=settings['host'], user=settings['user'], password=settings['password'],
database=settings['database'])
try:
with connection.cursor() as cursor:
cursor.execute("SELECT QUERY, ENGINES, CATEGORY, LANGUAGE , SAFE_SEARCH, PAGENO, TIME_RANGE FROM "
"SEARCH_HISTORY LIMIT %s,20" % x)
for row in cursor:
result.append(SearchQuery(d(row[0]), jd(row[1]), [row[2]], row[3], row[4], row[5], row[6]))
finally:
connection.close()
conn = _get_connection()
keys = conn.zrange('SEARCH_HISTORY_KEYS', int(x), int(x) + 20)
if not keys:
return result
pipe = conn.pipeline()
for key in keys:
pipe.hgetall(key)
output = pipe.execute()
for row in output:
result.append(SearchQuery(d(row['query']), jd(row['engines']), [row['category']], row['language'],
int(row['safe_search']), int(row['pageno']), row['time_range']))
return result
......@@ -150,17 +151,13 @@ def search(request):
def update(d):
connection = pymysql.connect(host=settings['host'], user=settings['user'], password=settings['password'],
database=settings['database'])
try:
with connection.cursor() as cursor:
sql = "UPDATE SEARCH_HISTORY SET RESULTS='%s', PAGING=%s, RESULTS_NUMBER=%s, ANSWERS='%s', CORRECTIONS='%s', INFOBOXES='%s', SUGGESTIONS='%s', " \
"UNRESPONSIVE_ENGINES='%s' WHERE QUERY='%s' AND CATEGORY='%s' AND PAGENO=%s AND " \
"SAFE_SEARCH=%s AND LANGUAGE='%s' AND TIME_RANGE='%s' AND ENGINES='%s'"
cursor.execute(sql % (je(d.results), d.paging, d.results_number, jes(d.answers), jes(d.corrections),
je(d.infoboxes), jes(d.suggestions), jes(d.unresponsive_engines),
e(d.query), d.categories[0], d.pageno, d.safe_search, d.language, d.time_range,
je(d.engines)))
connection.commit()
finally:
connection.close()
conn = redis.StrictRedis(settings['host'])
key = "SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}".format(
e(d.query), je(d.engines), d.categories[0], d.language, d.safe_search, d.pageno, d.time_range)
current = conn.hgetall(key)
current.update({
'results': je(d.results), 'paging': d.paging, 'results_number': d.results_number,
'answers': jes(d.answers), 'corrections': jes(d.corrections), 'infoboxes': je(d.infoboxes),
'suggestions': jes(d.suggestions), 'unresponsive_engines': jes(d.unresponsive_engines)
})
conn.hmset(key, current)
......@@ -15,11 +15,10 @@ server:
image_proxy : False # Proxying image results through searx
http_protocol_version : "1.0" # 1.0 and 1.1 are supported
mysql:
redis:
host : "127.0.0.1"
user : "searx"
password : "password" # change this!
database : "searx"
upgrade_history: 86400 # in seconds (1day = 86400s)
ui:
......
......@@ -811,7 +811,7 @@ running = threading.Event()
def wait_updating(start_time):
wait = settings['mysql']['upgrade_history'] - int(time.time() - start_time)
wait = settings['redis']['upgrade_history'] - int(time.time() - start_time)
if wait > 0:
running.wait(wait)
......@@ -836,7 +836,7 @@ def update_results():
def run():
logger.debug('starting webserver on %s:%s', settings['server']['port'], settings['server']['bind_address'])
search_database.settings = settings['mysql']
search_database.settings = settings['redis']
threading.Thread(target=update_results, name='results_updater').start()
print "engine server starting"
app.run(
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment