Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
M
my-spot
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Locked Files
Merge Requests
0
Merge Requests
0
Requirements
Requirements
List
Security & Compliance
Security & Compliance
Dependency List
License Compliance
Packages & Registries
Packages & Registries
Container Registry
Analytics
Analytics
Code Review
Insights
Issue
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Commits
Open sidebar
e
cloud
my-spot
Commits
17468b6e
Commit
17468b6e
authored
Jul 01, 2018
by
Omer
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Migrate to Redis
parent
458c34c9
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
62 additions
and
89 deletions
+62
-89
README.rst
README.rst
+4
-27
requirements.txt
requirements.txt
+1
-1
searx/search_database.py
searx/search_database.py
+54
-57
searx/settings.yml
searx/settings.yml
+1
-2
searx/webapp.py
searx/webapp.py
+2
-2
No files found.
README.rst
View file @
17468b6e
...
...
@@ -17,32 +17,9 @@ See the `documentation <https://asciimoo.github.io/searx>`__ and the `wiki <http
Setup MySql
~~~~~~~~~~~
**Install MySql**
``$ sudo apt-get install mysql-server
$ pip install pymysql``
**Start MySql**
``$ sudo service mysql start
$ mysql -u root -p``
**Create a new database and give all rights to a new searx user**
change password!
``mysql> create database searx;
mysql> create user "searx"@"localhost" identified by "password";
mysql> grant all on searx.* to "searx"@"localhost" identified by "password";``
**You can now quit the MySql console by typing ``quit`` and connect as searx user**
``$ mysql -u searx -p``
**Here are some commands to init the database**
``mysql> use searx;``
``mysql> create table SEARCH_HISTORY(QUERY varchar(512), CATEGORY varchar(256), PAGENO int(11), PAGING tinyint(1), SAFE_SEARCH int(11), LANGUAGE varchar(8), TIME_RANGE varchar(16), ENGINES varchar(4096), RESULTS mediumtext, RESULTS_NUMBER int(11), ANSWERS varchar(2048), CORRECTIONS varchar(256), INFOBOXES varchar(8192), SUGGESTIONS varchar(1024), UNRESPONSIVE_ENGINES varchar(1024));``
``mysql> quit``
MySql is done !
**Install Redis**
``$ sudo apt-get install redis-server
$ pip install redis``
Installation
~~~~~~~~~~~~
...
...
@@ -52,7 +29,7 @@ Installation
- install dependencies: ``./manage.sh update_packages``
- edit your
`settings.yml <https://github.com/asciimoo/searx/blob/master/searx/settings.yml>`__
(set your ``secret_key``
and ``mysql password``
!)
(set your ``secret_key``!)
- run ``python searx/webapp.py`` to start the application
For all the details, follow this `step by step
...
...
requirements.txt
View file @
17468b6e
pymysql
redis
certifi
==2017.11.5
flask
==0.12.2
flask-babel
==0.11.2
...
...
searx/search_database.py
View file @
17468b6e
...
...
@@ -2,7 +2,7 @@ import json
import
threading
import
urllib
import
pymysql
import
redis
from
searx.plugins
import
plugins
from
searx.query
import
SearchQuery
...
...
@@ -32,59 +32,60 @@ class SearchData(object):
self
.
unresponsive_engines
=
unresponsive_engines
def
_get_connection
():
return
redis
.
StrictRedis
(
settings
[
'host'
],
decode_responses
=
True
)
def
read
(
q
):
time_range
=
q
.
time_range
if
q
.
time_range
is
None
:
q
.
time_range
=
""
connection
=
pymysql
.
connect
(
host
=
settings
[
'host'
],
user
=
settings
[
'user'
],
password
=
settings
[
'password'
],
database
=
settings
[
'database'
])
try
:
with
connection
.
cursor
()
as
cursor
:
sql
=
"SELECT RESULTS, PAGING, RESULTS_NUMBER, ANSWERS, CORRECTIONS, INFOBOXES, SUGGESTIONS, "
\
"UNRESPONSIVE_ENGINES FROM SEARCH_HISTORY WHERE QUERY='%s' AND CATEGORY='%s' AND PAGENO=%s AND "
\
"SAFE_SEARCH=%s AND LANGUAGE='%s' AND TIME_RANGE='%s' AND ENGINES='%s'"
cursor
.
execute
(
sql
%
(
e
(
q
.
query
),
q
.
categories
[
0
],
q
.
pageno
,
q
.
safesearch
,
q
.
lang
,
time_range
,
je
(
q
.
engines
)))
for
response
in
cursor
:
results
=
jd
(
response
[
0
])
for
result
in
results
:
result
[
'parsed_url'
]
=
urlparse
(
result
[
'url'
])
return
SearchData
(
q
,
results
,
response
[
1
]
!=
0
,
response
[
2
],
jds
(
response
[
3
]),
jds
(
response
[
4
]),
jd
(
response
[
5
]),
jds
(
response
[
6
]),
jds
(
response
[
7
]))
finally
:
connection
.
close
()
return
None
conn
=
_get_connection
()
key
=
"SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}"
.
format
(
e
(
q
.
query
),
je
(
q
.
engines
),
q
.
categories
[
0
],
q
.
lang
,
q
.
safesearch
,
q
.
pageno
,
time_range
)
response
=
conn
.
hgetall
(
key
)
if
not
response
:
return
None
results
=
jd
(
response
[
'results'
])
for
result
in
results
:
result
[
'parsed_url'
]
=
urlparse
(
result
[
'url'
])
return
SearchData
(
q
,
results
,
int
(
response
[
'paging'
])
!=
0
,
int
(
response
[
'results_number'
]),
jds
(
response
[
'answers'
]),
jds
(
response
[
'corrections'
]),
jd
(
response
[
'infoboxes'
]),
jds
(
response
[
'suggestions'
]),
jds
(
response
[
'unresponsive_engines'
]))
def
save
(
d
):
connection
=
pymysql
.
connect
(
host
=
settings
[
'host'
],
user
=
settings
[
'user'
],
password
=
settings
[
'password'
],
database
=
settings
[
'database'
])
try
:
with
connection
.
cursor
()
as
cursor
:
sql
=
"INSERT INTO SEARCH_HISTORY(QUERY, CATEGORY, PAGENO, SAFE_SEARCH, LANGUAGE, TIME_RANGE, ENGINES, "
\
"RESULTS, PAGING, RESULTS_NUMBER, ANSWERS, CORRECTIONS, INFOBOXES, SUGGESTIONS, "
\
"UNRESPONSIVE_ENGINES) VALUES('%s', '%s', %s, %s, '%s', '%s', '%s', '%s', %s, %s, '%s', '%s', '%s',"
\
" '%s', '%s')"
cursor
.
execute
(
sql
%
(
e
(
d
.
query
),
d
.
categories
[
0
],
d
.
pageno
,
d
.
safe_search
,
d
.
language
,
d
.
time_range
,
je
(
d
.
engines
),
je
(
d
.
results
),
d
.
paging
,
d
.
results_number
,
jes
(
d
.
answers
),
jes
(
d
.
corrections
),
je
(
d
.
infoboxes
),
jes
(
d
.
suggestions
),
jes
(
d
.
unresponsive_engines
)))
connection
.
commit
()
finally
:
connection
.
close
()
conn
=
_get_connection
()
key
=
"SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}"
.
format
(
e
(
d
.
query
),
je
(
d
.
engines
),
d
.
categories
[
0
],
d
.
language
,
d
.
safe_search
,
d
.
pageno
,
d
.
time_range
)
mapping
=
{
'query'
:
e
(
d
.
query
),
'category'
:
d
.
categories
[
0
],
'pageno'
:
d
.
pageno
,
'safe_search'
:
d
.
safe_search
,
'language'
:
d
.
language
,
'time_range'
:
d
.
time_range
,
'engines'
:
je
(
d
.
engines
),
'results'
:
je
(
d
.
results
),
'paging'
:
d
.
paging
,
'results_number'
:
d
.
results_number
,
'answers'
:
jes
(
d
.
answers
),
'corrections'
:
jes
(
d
.
corrections
),
'infoboxes'
:
je
(
d
.
infoboxes
),
'suggestions'
:
jes
(
d
.
suggestions
),
'unresponsive_engines'
:
jes
(
d
.
unresponsive_engines
)
}
conn
.
zadd
(
'SEARCH_HISTORY_KEYS'
,
conn
.
incr
(
'SEARCH_HISTORY_INDEX'
),
key
)
conn
.
hmset
(
key
,
mapping
)
def
get_twenty_queries
(
x
):
result
=
[]
connection
=
pymysql
.
connect
(
host
=
settings
[
'host'
],
user
=
settings
[
'user'
],
password
=
settings
[
'password'
],
database
=
settings
[
'database'
])
try
:
with
connection
.
cursor
()
as
cursor
:
cursor
.
execute
(
"SELECT QUERY, ENGINES, CATEGORY, LANGUAGE , SAFE_SEARCH, PAGENO, TIME_RANGE FROM "
"SEARCH_HISTORY LIMIT %s,20"
%
x
)
for
row
in
cursor
:
result
.
append
(
SearchQuery
(
d
(
row
[
0
]),
jd
(
row
[
1
]),
[
row
[
2
]],
row
[
3
],
row
[
4
],
row
[
5
],
row
[
6
]))
finally
:
connection
.
close
()
conn
=
_get_connection
()
keys
=
conn
.
zrange
(
'SEARCH_HISTORY_KEYS'
,
int
(
x
),
int
(
x
)
+
20
)
if
not
keys
:
return
result
pipe
=
conn
.
pipeline
()
for
key
in
keys
:
pipe
.
hgetall
(
key
)
output
=
pipe
.
execute
()
for
row
in
output
:
result
.
append
(
SearchQuery
(
d
(
row
[
'query'
]),
jd
(
row
[
'engines'
]),
[
row
[
'category'
]],
row
[
'language'
],
int
(
row
[
'safe_search'
]),
int
(
row
[
'pageno'
]),
row
[
'time_range'
]))
return
result
...
...
@@ -150,17 +151,13 @@ def search(request):
def
update
(
d
):
connection
=
pymysql
.
connect
(
host
=
settings
[
'host'
],
user
=
settings
[
'user'
],
password
=
settings
[
'password'
],
database
=
settings
[
'database'
])
try
:
with
connection
.
cursor
()
as
cursor
:
sql
=
"UPDATE SEARCH_HISTORY SET RESULTS='%s', PAGING=%s, RESULTS_NUMBER=%s, ANSWERS='%s', CORRECTIONS='%s', INFOBOXES='%s', SUGGESTIONS='%s', "
\
"UNRESPONSIVE_ENGINES='%s' WHERE QUERY='%s' AND CATEGORY='%s' AND PAGENO=%s AND "
\
"SAFE_SEARCH=%s AND LANGUAGE='%s' AND TIME_RANGE='%s' AND ENGINES='%s'"
cursor
.
execute
(
sql
%
(
je
(
d
.
results
),
d
.
paging
,
d
.
results_number
,
jes
(
d
.
answers
),
jes
(
d
.
corrections
),
je
(
d
.
infoboxes
),
jes
(
d
.
suggestions
),
jes
(
d
.
unresponsive_engines
),
e
(
d
.
query
),
d
.
categories
[
0
],
d
.
pageno
,
d
.
safe_search
,
d
.
language
,
d
.
time_range
,
je
(
d
.
engines
)))
connection
.
commit
()
finally
:
connection
.
close
()
conn
=
redis
.
StrictRedis
(
settings
[
'host'
])
key
=
"SEARCH_HISTORY:{}:{}:{}:{}:{}:{}:{}"
.
format
(
e
(
d
.
query
),
je
(
d
.
engines
),
d
.
categories
[
0
],
d
.
language
,
d
.
safe_search
,
d
.
pageno
,
d
.
time_range
)
current
=
conn
.
hgetall
(
key
)
current
.
update
({
'results'
:
je
(
d
.
results
),
'paging'
:
d
.
paging
,
'results_number'
:
d
.
results_number
,
'answers'
:
jes
(
d
.
answers
),
'corrections'
:
jes
(
d
.
corrections
),
'infoboxes'
:
je
(
d
.
infoboxes
),
'suggestions'
:
jes
(
d
.
suggestions
),
'unresponsive_engines'
:
jes
(
d
.
unresponsive_engines
)
})
conn
.
hmset
(
key
,
current
)
searx/settings.yml
View file @
17468b6e
...
...
@@ -15,11 +15,10 @@ server:
image_proxy
:
False
# Proxying image results through searx
http_protocol_version
:
"
1.0"
# 1.0 and 1.1 are supported
mysql
:
redis
:
host
:
"
127.0.0.1"
user
:
"
searx"
password
:
"
password"
# change this!
database
:
"
searx"
upgrade_history
:
86400
# in seconds (1day = 86400s)
ui
:
...
...
searx/webapp.py
View file @
17468b6e
...
...
@@ -811,7 +811,7 @@ running = threading.Event()
def
wait_updating
(
start_time
):
wait
=
settings
[
'
mysql
'
][
'upgrade_history'
]
-
int
(
time
.
time
()
-
start_time
)
wait
=
settings
[
'
redis
'
][
'upgrade_history'
]
-
int
(
time
.
time
()
-
start_time
)
if
wait
>
0
:
running
.
wait
(
wait
)
...
...
@@ -836,7 +836,7 @@ def update_results():
def
run
():
logger
.
debug
(
'starting webserver on %s:%s'
,
settings
[
'server'
][
'port'
],
settings
[
'server'
][
'bind_address'
])
search_database
.
settings
=
settings
[
'
mysql
'
]
search_database
.
settings
=
settings
[
'
redis
'
]
threading
.
Thread
(
target
=
update_results
,
name
=
'results_updater'
).
start
()
print
"engine server starting"
app
.
run
(
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment