diff --git a/.env b/.env new file mode 100644 index 0000000000000000000000000000000000000000..5f1eff71eac42da6b90245e05df302f7ce384e1b --- /dev/null +++ b/.env @@ -0,0 +1,5 @@ +SPOT_DOCKER_IMG=registry.gitlab.e.foundation:5000/e/cloud/my-spot +SPOT_DOCKER_TAG=latest +SPOT_NGINX_DOCKER_IMG=registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx +SPOT_NGINX_DOCKER_TAG=latest +SPOT_HOSTNAME=localhost diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 051ef445e0e23f073dbc7b1adb3863a95d2d6b4f..595504800fff135f100896088402e8df926b696c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,102 +1,14 @@ -image: $CI_REGISTRY_IMAGE/env:latest +image: $CI_REGISTRY_IMAGE/env stages: - - check - - build - - test - - report - deploy -python: - stage: check - before_script: - - ./manage.sh update_dev_packages - script: - - ./manage.sh pep8_check - -build:web: - stage: build - before_script: - - ./manage.sh npm_packages - - ./manage.sh update_dev_packages - script: - - ./manage.sh locales - - ./manage.sh styles - - ./manage.sh grunt_build - - - -.build:docker: - stage: build - image: docker:git - services: - - docker:18-dind - before_script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - script: - - docker build -t $CI_REGISTRY_IMAGE . - - docker push $CI_REGISTRY_IMAGE - -build:docker:master: - extends: .build:docker - only: - - master - -build:docker:tags: - extends: .build:docker - script: - - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - only: - - tags - -test:unit: - stage: test - before_script: - - ./manage.sh update_dev_packages - script: - - ./manage.sh unit_tests - artifacts: - paths: - - coverage - expire_in: 1 hour - -test:functional: - stage: test - image: docker:stable - services: - - docker:18-dind - variables: - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - before_script: - - docker run -id --rm -v $(pwd):/ws -e DOCKER_HOST=tcp://$(cat /etc/hosts | grep docker | cut -f1):2375/ -w /ws --name spotenv $CI_REGISTRY_IMAGE/env:latest sh - - docker exec -i spotenv ./manage.sh update_dev_packages - script: - - docker exec -i spotenv ./manage.sh functional_tests - artifacts: - paths: - - coverage - expire_in: 1 hour - -coverage: - stage: report - script: - - ./manage.sh coverage - dependencies: - - test:unit - - test:functional - coverage: '/TOTAL.*\s+(\d+%)$/' - .deploy:template: image: docker:stable stage: deploy only: - branches when: manual - dependencies: [] - variables: - DEPLOY_FOLDER: /mnt/data before_script: - 'which ssh-agent || ( apk --update add openssh-client )' - eval $(ssh-agent -s) @@ -106,9 +18,10 @@ coverage: - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts script: + - ssh -2 $PUBLISH_USER@$PUBLISH_URL "docker ps && docker info && docker-compose version" - ssh -2 $PUBLISH_USER@$PUBLISH_URL "docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}" - - ssh -2 $PUBLISH_USER@$PUBLISH_URL "mkdir -p ${DEPLOY_FOLDER} && cd ${DEPLOY_FOLDER} && if [ ! -d ${CI_PROJECT_NAME} ] ; then git clone ${CI_PROJECT_URL}.git ; fi && cd ${CI_PROJECT_NAME} && git fetch && git checkout $CI_COMMIT_SHA" - - ssh -2 $PUBLISH_USER@$PUBLISH_URL "SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG && COMPOSE_FILE=$COMPOSE_FILE && SPOT_HOSTNAME=$SPOT_HOSTNAME && export SPOT_HOSTNAME COMPOSE_FILE SPOT_DOCKER_TAG && cd ${DEPLOY_FOLDER}/${CI_PROJECT_NAME} && docker-compose pull && docker-compose up -d --build --force-recreate" + - ssh -2 $PUBLISH_USER@$PUBLISH_URL "mkdir -p ${DEPLOY_FOLDER} && cd ${DEPLOY_FOLDER} && if [ ! -d ${CI_PROJECT_NAME} ] ; then git clone ${CI_PROJECT_URL}.git ; fi && cd ${CI_PROJECT_NAME} && git clean -dffx && git fetch && git checkout $CI_COMMIT_SHA" + - ssh -2 $PUBLISH_USER@$PUBLISH_URL "SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG && COMPOSE_FILE=$COMPOSE_FILE && SPOT_HOSTNAME=$SPOT_HOSTNAME && export SPOT_HOSTNAME COMPOSE_FILE SPOT_DOCKER_TAG && cd ${DEPLOY_FOLDER}/${CI_PROJECT_NAME} && docker-compose pull && docker-compose up -d --build --force-recreate --remove-orphans" test: extends: .deploy:template @@ -119,6 +32,7 @@ test: SPOT_HOSTNAME: spot.test.ecloud.global SSH_PRIVATE_KEY: ${SSH_PRIVATE_KEY_TEST} COMPOSE_FILE: docker-compose.yml:docker-compose-dev.yml + DEPLOY_FOLDER: /mnt/data prod: extends: .deploy:template @@ -132,3 +46,4 @@ prod: SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SSH_PRIVATE_KEY: ${SSH_PRIVATE_KEY_PROD} COMPOSE_FILE: docker-compose.yml + DEPLOY_FOLDER: /mnt/data diff --git a/Dockerfile b/Dockerfile index d71dc69a70b15075c5d4423d4d38c09a2f1e33b2..ca33c13a20e6fc8f2a0d0e0f02069db1147f6791 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,7 @@ -FROM python:3.7-alpine as builder - -RUN apk add \ - git \ - build-base \ - libxml2-dev \ - libxslt-dev +FROM registry.gitlab.e.foundation:5000/e/cloud/my-spot/env as builder COPY . /src/ -RUN pip3 install --prefix /install /src +RUN pip install --force-reinstall --prefix /install /src FROM python:3.7-alpine @@ -17,10 +11,12 @@ LABEL description="A privacy-respecting, hackable metasearch engine." RUN apk add \ ca-certificates \ libxslt \ + py3-gunicorn \ && pip install coverage COPY --from=builder /install/ /usr/local/ -EXPOSE 8888 +EXPOSE 80 STOPSIGNAL SIGINT -CMD ["searx-run"] + +CMD ["gunicorn", "-w", "1", "-b", "0.0.0.0:80", "--pythonpath", "/usr/local/lib/python3.7/site-packages", "searx.webapp:app"] diff --git a/Dockerfile.env b/Dockerfile.env index 964dcbc5cac97d1af766e9949cd4d22933ff90a3..5fcee8fb5770b417234dca341ea204cec7f12ae3 100644 --- a/Dockerfile.env +++ b/Dockerfile.env @@ -1,14 +1,9 @@ -FROM fedora +FROM python:3.7-alpine COPY requirements.txt requirements-dev.txt / -RUN dnf install -y\ - wget\ - python2-pip\ - npm\ - docker \ -&& dnf groupinstall -y "Development Tools" \ -&& pip3 install ipdb ipython \ -&& pip3 install -r /requirements.txt \ -&& pip3 install -r /requirements-dev.txt \ +RUN apk add build-base git libxml2-dev libxslt-dev libffi-dev openssl-dev npm docker \ +&& pip install ipdb ipython \ +&& pip install -r /requirements.txt \ +&& pip install -r /requirements-dev.txt \ && rm -f /requirements.txt /requirements-dev.txt diff --git a/Dockerfile.nginx b/Dockerfile.nginx new file mode 100644 index 0000000000000000000000000000000000000000..a8c1e0d2b721d93fd055a36e7926695bc6a75d7b --- /dev/null +++ b/Dockerfile.nginx @@ -0,0 +1,4 @@ +FROM nginx:1.17.6-alpine + +COPY etc/nginx/conf.d/spot.conf /etc/nginx/conf.d/default.conf +COPY --chown=nginx:nginx searx/static /var/www/spot/static diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3b901e31dbf814b2c402451464a36f3ae4068dff --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +# spot for /e/ (https://e.foundation) + +A privacy-respecting, hackable [metasearch engine](https://en.wikipedia.org/wiki/Metasearch_engine). + +Spot was forked from searx: read [documentation](https://asciimoo.github.io/searx) and the [wiki](https://github.com/asciimoo/searx/wiki) for more information. + +## Changes between Spot and Searx + +* cache results in Redis database +* eelo theme +* better python packaging and docker integration for production deployement +* drop of Python 2 support +* pytest as test launcher + +The aim is to move that changes in the upstream project when it will be easier to rebase from this one. + +## Getting Started + +You can run spot with docker-compose to run the *redis* database and +the *spot* service. First of all you have to install *docker* and +*docker-compose* on your host, then follow instructions below to run spot +with one command. + +### For production + +* Run the docker-compose *up* command to start the project +``` +COMPOSE_FILE="docker-compose.yml:docker-compose-dev.yml" docker-compose up --build nginx spot redis +``` +* Getting the ip of the spot service and go to `http://`, below the docker way to get the IP of the nginx container +``` + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' my-spot_nginx_1 +``` + +### For developer + +You can directly run spot, with a python command. First, you should install all dependencies on your host `./manage.sh update_dev_packages` or you can use the `env` container. + +``` +docker run -it --rm -v $(pwd):/ws -w /ws -e PYTHONPATH=/ws registry.gitlab.e.foundation:5000/e/cloud/my-spot/env sh +python ./searx/webapp.py +``` diff --git a/README.rst b/README.rst deleted file mode 100644 index f58430b4602d0f4aaf8b6c3c284667290baae038..0000000000000000000000000000000000000000 --- a/README.rst +++ /dev/null @@ -1,61 +0,0 @@ -spot for /e/ (https://e.foundation) -=================================== - -A privacy-respecting, hackable `metasearch -engine `__. - -Spot was forked from searx: read `documentation `__ and the `wiki `__ for more information. - -Spot is based on Python3.7+ - -Getting Started -~~~~~~~~~~~~ - -You can run spot with docker-compose to run the **redis** database and -the **spot** service. First of all you have to install **docker** and -**docker-compose** on your host, then follow instructions below to run spot -with one command. - -- Run the docker-compose **up** command to start the project ``docker-compose up --build`` -- Getting the ip of the spot service and go to http://:8888 - -.. note:: Here the command to get the IP of the spot service - ``docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' my-spot_spot_1`` - -You can also install **redis** and **spot** on your host, for all the details, follow this `step by step -installation `__. - -Developer mode -~~~~~~~~~~~~ - -First run the redis database: - -- ``docker-compose up -d redis`` - -Then on spot workdir run the following commands to start spot: - -- ``export COMPOSE_FILE=docker-compose.yml:docker-compose-dev.yml`` -- ``docker-compose build spot`` -- ``docker-compose run --rm -v $(pwd):/ws -w /ws -e PYTHONPATH=/ws spot sh`` -- ``python3 -X dev searx/webapp.py`` - -Run tests: - -- ``docker run -it --rm -v $(pwd):/ws -w /ws -v /var/run/docker.sock:/var/run/docker.sock -e PYTHONPATH=/ws registry.gitlab.e.foundation:5000/e/cloud/my-spot/env bash`` -- ``COMPOSE_FILE=docker-compose.yml:docker-compose-dev.yml pytest --pdb --pdbcls IPython.terminal.debugger:TerminalPdb --dockerc-build -s tests/functional/`` - -Bugs -~~~~ - -Bugs or suggestions? Visit the `issue -tracker `__. - -`License `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -More about searx -~~~~~~~~~~~~~~~~ - -- `openhub `__ -- `twitter `__ -- IRC: #searx @ freenode diff --git a/docker-compose-coverage.yml b/docker-compose-coverage.yml index e0434e28be5f88c81a1dd486b5572572bc3b00c2..ecdafc0fa85caf9b0d3730db885c2b1e1bb66b9f 100644 --- a/docker-compose-coverage.yml +++ b/docker-compose-coverage.yml @@ -14,6 +14,11 @@ services: environment: COVERAGE_FILE: /coverage/func + nginx: + build: + dockerfile: Dockerfile.nginx + context: . + volumes: coverage: name: spot-coverage diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 139f5bf63efaa4689c17d8e1139c12f1511e070e..ef2f640c1724792c8c07767b1afd07868e1df622 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -2,4 +2,11 @@ version: '3.6' services: spot: - build: . + build: + dockerfile: Dockerfile + context: . + + nginx: + build: + dockerfile: Dockerfile.nginx + context: . diff --git a/docker-compose.yml b/docker-compose.yml index c6cd1aa3f206f4e16a67ab86040bedee6de1fef7..5e1afa3b1a50c035eff5de295b29987472584b44 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,35 +6,41 @@ services: image: redis:5-alpine spot: - image: registry.gitlab.e.foundation:5000/e/cloud/my-spot:${SPOT_DOCKER_TAG:-latest} + image: ${SPOT_DOCKER_IMG}:${SPOT_DOCKER_TAG} restart: unless-stopped environment: SEARX_REDIS_HOST: redis - VIRTUAL_HOST: ${SPOT_HOSTNAME:-spot} - LETSENCRYPT_HOST: ${SPOT_HOSTNAME:-spot} SEARX_LOGGER: INFO - proxy: - image: jwilder/nginx-proxy:alpine - restart: unless-stopped - container_name: proxy - volumes: - - /mnt/data/html:/usr/share/nginx/html - - /mnt/data/vhosts:/etc/nginx/vhost.d - - /mnt/data/certs:/etc/nginx/certs:ro - - /var/run/docker.sock:/tmp/docker.sock:ro + nginx: + image: ${SPOT_NGINX_DOCKER_IMG}:${SPOT_NGINX_DOCKER_TAG} restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.spot.rule=Host(`${SPOT_HOSTNAME}`)" + - "traefik.http.routers.spot.entrypoints=websecure" + - "traefik.http.routers.spot.tls.certresolver=spotchallenge" + - "traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`)" + - "traefik.http.routers.http-catchall.entrypoints=web" + - "traefik.http.routers.http-catchall.middlewares=redirect-to-https@docker" + - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" + + traefik: + image: "traefik:v2.0.5" + command: + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--certificatesresolvers.mytlschallenge.acme.tlschallenge=true" + - "--certificatesresolvers.mytlschallenge.acme.email=contact@e.email" + - "--certificatesresolvers.mytlschallenge.acme.storage=/letsencrypt/acme.json" ports: - "80:80" - "443:443" - - cert: - image: jrcs/letsencrypt-nginx-proxy-companion - restart: unless-stopped - environment: - NGINX_PROXY_CONTAINER: proxy volumes: - - /mnt/data/html:/usr/share/nginx/html - - /mnt/data/vhosts:/etc/nginx/vhost.d - - /mnt/data/certs:/etc/nginx/certs - - /var/run/docker.sock:/var/run/docker.sock:ro + - "letsencrypt:/letsencrypt" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + +volumes: + letsencrypt: diff --git a/etc/nginx/conf.d/spot.conf b/etc/nginx/conf.d/spot.conf new file mode 100644 index 0000000000000000000000000000000000000000..277c5e6af3666f6c6a41ea620efb6a12fec1b2f6 --- /dev/null +++ b/etc/nginx/conf.d/spot.conf @@ -0,0 +1,24 @@ +server { + listen 80; + server_name _; + + add_header Content-Security-Policy "frame-ancestors 'self'"; + add_header Referrer-Policy "no-referrer, strict-origin-when-cross-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-XSS-Protection "1; mode=block"; + add_header X-Content-Type-Options "nosniff"; + + root /var/www/spot; + + location / { + try_files $uri @searx; + } + + location @searx { + proxy_pass http://spot:80; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + } +} diff --git a/etc/nginx/vhost.d/default b/etc/nginx/vhost.d/default deleted file mode 100644 index 07c6b072ba838626ff1a22c8d577fce5df416f7b..0000000000000000000000000000000000000000 --- a/etc/nginx/vhost.d/default +++ /dev/null @@ -1,5 +0,0 @@ -add_header Content-Security-Policy "frame-ancestors 'self'"; -add_header Referrer-Policy "no-referrer, strict-origin-when-cross-origin"; -add_header X-Frame-Options "SAMEORIGIN"; -add_header X-XSS-Protection "1; mode=block"; -add_header X-Content-Type-Options "nosniff"; diff --git a/manage.sh b/manage.sh index cc96df5add8c71a275f2036b72fd1834946c31cb..f90a62353e807062d94c4d297b0669df75a72a5b 100755 --- a/manage.sh +++ b/manage.sh @@ -60,7 +60,7 @@ functional_tests() { } coverage() { - sed -i 's!/usr/local/lib/python3.7/site-packages/searx[^/]*/searx!'$SEARX_DIR'!g' "$COV_DIR"/func + sed -i 's!/usr/local/lib/python3.7/site-packages/searx!'$SEARX_DIR'!g' "$COV_DIR"/func coverage3 combine coverage/func coverage/unit coverage3 report } diff --git a/requirements-dev.txt b/requirements-dev.txt index 695f9e18333ea12f236aba72309063e2f51b121b..d53d4748e1abe6a36bf19cad6374c824611640c9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,6 +3,6 @@ mock==2.0.0 pycodestyle==2.5.0 flake8==3.7.7 mockredispy==2.9.3 -pytest==4.1.0 +pytest==5.3.0 pytest-cov==2.6.1 pytest-dockerc==1.0.5 diff --git a/searx/settings.yml b/searx/settings.yml index 417f06d39053f49d9df6ae5c04616e5f6172b788..295ff47f215f025702f19ccbc1bfe81d908d1921 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -9,7 +9,7 @@ search: max_ban_time_on_fail : 120 # max ban time in seconds after engine errors server: - port : 8888 + port : 80 bind_address : "0.0.0.0" # address to listen on secret_key : "ultrasecretkey" # change this! base_url : False # Set custom base_url. Possible values: False or "https://your.custom.host/location/" diff --git a/searx/webapp.py b/searx/webapp.py index 36f361232f27079328c827d555a5c6725f19f17e..1c22539500ec9b42b2489b5c6dd9626653cb66d3 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -827,6 +827,13 @@ def page_not_found(e): return render('404.html'), 404 +@app.before_first_request +def start_cache_refresh_task(): + if settings['redis']['enable']: + logger.info("starting cache refresh task") + threading.Thread(target=update_results, name='results_updater').start() + + running = threading.Event() @@ -856,7 +863,6 @@ def update_results(): def run(): logger.debug('starting webserver on %s:%s', settings['server']['port'], settings['server']['bind_address']) - threading.Thread(target=update_results, name='results_updater').start() print("engine server starting") app.run( debug=searx_debug, @@ -866,7 +872,6 @@ def run(): threaded=True ) print("wait for shutdown...") - running.set() class ReverseProxyPathFix(object): diff --git a/setup.py b/setup.py index ac5d115b218029943e9d04e8cb664325856f128e..4b6958236217185b9b2f2daa2a734b7e38083ba4 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,8 @@ setup( name='searx', use_scm_version={"tag_regex": r"^(?:[\w-]+-)?(?P[vV]?\d+(?:\.\d+){0,2}.*)$"}, description="A privacy-respecting, hackable metasearch engine", - long_description=open('README.rst').read(), + long_description=open('README.md').read(), + long_description_content_type="text/markdown", classifiers=[ "Development Status :: 4 - Beta", "Programming Language :: Python", diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index 22fe91e285b78c85ba3672d2a89731c297f763a5..79788d503e90a55ad67442aaaa1a5c87031f1042 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -8,11 +8,15 @@ from pytest_dockerc import Wait, Context class SpotContext(Context): @property def url(self): - addr = self.container_addr("spot") - port = self.container_port("spot") + addr = self.container_addr("nginx") + port = self.container_port("nginx") return f"http://{addr}:{port}" def wait_for_running_state(self): + spot_url = "http://{}:{}".format(self.container_addr("spot"), self.container_port("spot")) + Wait(ignored_exns=(requests.ConnectionError,), timeout=60)( + lambda: requests.get(spot_url) + ) Wait(ignored_exns=(requests.ConnectionError,), timeout=60)( lambda: requests.get(self.url) )