From e69adebc0bb01c52add34adc183a932bc27c1008 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Tue, 19 Nov 2019 23:18:35 +0100 Subject: [PATCH 01/10] update pytest --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 695f9e183..d53d4748e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,6 +3,6 @@ mock==2.0.0 pycodestyle==2.5.0 flake8==3.7.7 mockredispy==2.9.3 -pytest==4.1.0 +pytest==5.3.0 pytest-cov==2.6.1 pytest-dockerc==1.0.5 -- GitLab From 6db1c6f79395e327e70010b77c67a74ae13dfe07 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Wed, 20 Nov 2019 11:48:44 +0100 Subject: [PATCH 02/10] move env image to alpine base --- .gitlab-ci.yml | 15 +++++++++++++-- Dockerfile.env | 15 +++++---------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 051ef445e..2fb3fe7d8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: $CI_REGISTRY_IMAGE/env:latest +image: $CI_REGISTRY_IMAGE/env stages: - check @@ -50,6 +50,17 @@ build:docker:tags: only: - tags +build:docker:env: + extends: .build:docker + script: + - docker build -t $CI_REGISTRY_IMAGE/env -f Dockerfile.env + - docker push $CI_REGISTRY_IMAGE/env + only: + changes: + - Dockerfile.env + - requirements-dev.txt + - requirements.txt + test:unit: stage: test before_script: @@ -70,7 +81,7 @@ test:functional: DOCKER_HOST: tcp://docker:2375/ DOCKER_DRIVER: overlay2 before_script: - - docker run -id --rm -v $(pwd):/ws -e DOCKER_HOST=tcp://$(cat /etc/hosts | grep docker | cut -f1):2375/ -w /ws --name spotenv $CI_REGISTRY_IMAGE/env:latest sh + - docker run -id --rm -v $(pwd):/ws -e DOCKER_HOST=tcp://$(cat /etc/hosts | grep docker | cut -f1):2375/ -w /ws --name spotenv $CI_REGISTRY_IMAGE/env sh - docker exec -i spotenv ./manage.sh update_dev_packages script: - docker exec -i spotenv ./manage.sh functional_tests diff --git a/Dockerfile.env b/Dockerfile.env index 964dcbc5c..5fcee8fb5 100644 --- a/Dockerfile.env +++ b/Dockerfile.env @@ -1,14 +1,9 @@ -FROM fedora +FROM python:3.7-alpine COPY requirements.txt requirements-dev.txt / -RUN dnf install -y\ - wget\ - python2-pip\ - npm\ - docker \ -&& dnf groupinstall -y "Development Tools" \ -&& pip3 install ipdb ipython \ -&& pip3 install -r /requirements.txt \ -&& pip3 install -r /requirements-dev.txt \ +RUN apk add build-base git libxml2-dev libxslt-dev libffi-dev openssl-dev npm docker \ +&& pip install ipdb ipython \ +&& pip install -r /requirements.txt \ +&& pip install -r /requirements-dev.txt \ && rm -f /requirements.txt /requirements-dev.txt -- GitLab From 93c102ecbaa3acf942cdc2e0f89d2b7bb49f8d25 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Wed, 20 Nov 2019 18:51:50 +0100 Subject: [PATCH 03/10] reuse docker env image to build searx In that way the CI pipeline will be faster. It avoid to re-build python dependencies based on C libraries. --- Dockerfile | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index d71dc69a7..bcae6cd0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,7 @@ -FROM python:3.7-alpine as builder - -RUN apk add \ - git \ - build-base \ - libxml2-dev \ - libxslt-dev +FROM registry.gitlab.e.foundation:5000/e/cloud/my-spot/env as builder COPY . /src/ -RUN pip3 install --prefix /install /src +RUN pip install --force-reinstall --prefix /install /src FROM python:3.7-alpine -- GitLab From dbfca7c66cd7176ed4d59299c6aa3cf698ff1663 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Wed, 20 Nov 2019 18:54:03 +0100 Subject: [PATCH 04/10] run searx with gunicorn as WSGI HTTP server --- Dockerfile | 6 ++++-- searx/settings.yml | 2 +- searx/webapp.py | 9 +++++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index bcae6cd0e..ca33c13a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,10 +11,12 @@ LABEL description="A privacy-respecting, hackable metasearch engine." RUN apk add \ ca-certificates \ libxslt \ + py3-gunicorn \ && pip install coverage COPY --from=builder /install/ /usr/local/ -EXPOSE 8888 +EXPOSE 80 STOPSIGNAL SIGINT -CMD ["searx-run"] + +CMD ["gunicorn", "-w", "1", "-b", "0.0.0.0:80", "--pythonpath", "/usr/local/lib/python3.7/site-packages", "searx.webapp:app"] diff --git a/searx/settings.yml b/searx/settings.yml index 417f06d39..295ff47f2 100644 --- a/searx/settings.yml +++ b/searx/settings.yml @@ -9,7 +9,7 @@ search: max_ban_time_on_fail : 120 # max ban time in seconds after engine errors server: - port : 8888 + port : 80 bind_address : "0.0.0.0" # address to listen on secret_key : "ultrasecretkey" # change this! base_url : False # Set custom base_url. Possible values: False or "https://your.custom.host/location/" diff --git a/searx/webapp.py b/searx/webapp.py index 36f361232..1c2253950 100644 --- a/searx/webapp.py +++ b/searx/webapp.py @@ -827,6 +827,13 @@ def page_not_found(e): return render('404.html'), 404 +@app.before_first_request +def start_cache_refresh_task(): + if settings['redis']['enable']: + logger.info("starting cache refresh task") + threading.Thread(target=update_results, name='results_updater').start() + + running = threading.Event() @@ -856,7 +863,6 @@ def update_results(): def run(): logger.debug('starting webserver on %s:%s', settings['server']['port'], settings['server']['bind_address']) - threading.Thread(target=update_results, name='results_updater').start() print("engine server starting") app.run( debug=searx_debug, @@ -866,7 +872,6 @@ def run(): threaded=True ) print("wait for shutdown...") - running.set() class ReverseProxyPathFix(object): -- GitLab From a759d8abfab6a7eca0084364a3365f599c064bc4 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Thu, 21 Nov 2019 09:06:45 +0100 Subject: [PATCH 05/10] add a nginx service in front of spot --- .env | 4 ++++ .gitlab-ci.yml | 4 ++++ Dockerfile.nginx | 4 ++++ docker-compose-coverage.yml | 5 +++++ docker-compose-dev.yml | 5 +++++ docker-compose.yml | 29 +++-------------------------- etc/nginx/conf.d/spot.conf | 24 ++++++++++++++++++++++++ etc/nginx/vhost.d/default | 5 ----- tests/functional/conftest.py | 8 ++++++-- 9 files changed, 55 insertions(+), 33 deletions(-) create mode 100644 .env create mode 100644 Dockerfile.nginx create mode 100644 etc/nginx/conf.d/spot.conf delete mode 100644 etc/nginx/vhost.d/default diff --git a/.env b/.env new file mode 100644 index 000000000..6c05454a6 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +SPOT_DOCKER_IMG=registry.gitlab.e.foundation:5000/e/cloud/my-spot +SPOT_DOCKER_TAG=latest +SPOT_NGINX_DOCKER_IMG=registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx +SPOT_NGINX_DOCKER_TAG=latest diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2fb3fe7d8..d280264fd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,6 +36,8 @@ build:web: script: - docker build -t $CI_REGISTRY_IMAGE . - docker push $CI_REGISTRY_IMAGE + - docker build -t $CI_REGISTRY_IMAGE/nginx -f Dockerfile.nginx . + - docker push $CI_REGISTRY_IMAGE/nginx build:docker:master: extends: .build:docker @@ -47,6 +49,8 @@ build:docker:tags: script: - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + - docker build -t $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG -f Dockerfile.nginx . + - docker push $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG only: - tags diff --git a/Dockerfile.nginx b/Dockerfile.nginx new file mode 100644 index 000000000..a8c1e0d2b --- /dev/null +++ b/Dockerfile.nginx @@ -0,0 +1,4 @@ +FROM nginx:1.17.6-alpine + +COPY etc/nginx/conf.d/spot.conf /etc/nginx/conf.d/default.conf +COPY --chown=nginx:nginx searx/static /var/www/spot/static diff --git a/docker-compose-coverage.yml b/docker-compose-coverage.yml index e0434e28b..ecdafc0fa 100644 --- a/docker-compose-coverage.yml +++ b/docker-compose-coverage.yml @@ -14,6 +14,11 @@ services: environment: COVERAGE_FILE: /coverage/func + nginx: + build: + dockerfile: Dockerfile.nginx + context: . + volumes: coverage: name: spot-coverage diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 139f5bf63..63fad3458 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -3,3 +3,8 @@ version: '3.6' services: spot: build: . + + nginx: + build: + dockerfile: Dockerfile.nginx + context: . diff --git a/docker-compose.yml b/docker-compose.yml index c6cd1aa3f..860297b0f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,35 +6,12 @@ services: image: redis:5-alpine spot: - image: registry.gitlab.e.foundation:5000/e/cloud/my-spot:${SPOT_DOCKER_TAG:-latest} + image: ${SPOT_DOCKER_IMG}:${SPOT_DOCKER_TAG} restart: unless-stopped environment: SEARX_REDIS_HOST: redis - VIRTUAL_HOST: ${SPOT_HOSTNAME:-spot} - LETSENCRYPT_HOST: ${SPOT_HOSTNAME:-spot} SEARX_LOGGER: INFO - proxy: - image: jwilder/nginx-proxy:alpine + nginx: + image: ${SPOT_NGINX_DOCKER_IMG}:${SPOT_NGINX_DOCKER_TAG} restart: unless-stopped - container_name: proxy - volumes: - - /mnt/data/html:/usr/share/nginx/html - - /mnt/data/vhosts:/etc/nginx/vhost.d - - /mnt/data/certs:/etc/nginx/certs:ro - - /var/run/docker.sock:/tmp/docker.sock:ro - restart: unless-stopped - ports: - - "80:80" - - "443:443" - - cert: - image: jrcs/letsencrypt-nginx-proxy-companion - restart: unless-stopped - environment: - NGINX_PROXY_CONTAINER: proxy - volumes: - - /mnt/data/html:/usr/share/nginx/html - - /mnt/data/vhosts:/etc/nginx/vhost.d - - /mnt/data/certs:/etc/nginx/certs - - /var/run/docker.sock:/var/run/docker.sock:ro diff --git a/etc/nginx/conf.d/spot.conf b/etc/nginx/conf.d/spot.conf new file mode 100644 index 000000000..277c5e6af --- /dev/null +++ b/etc/nginx/conf.d/spot.conf @@ -0,0 +1,24 @@ +server { + listen 80; + server_name _; + + add_header Content-Security-Policy "frame-ancestors 'self'"; + add_header Referrer-Policy "no-referrer, strict-origin-when-cross-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-XSS-Protection "1; mode=block"; + add_header X-Content-Type-Options "nosniff"; + + root /var/www/spot; + + location / { + try_files $uri @searx; + } + + location @searx { + proxy_pass http://spot:80; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + } +} diff --git a/etc/nginx/vhost.d/default b/etc/nginx/vhost.d/default deleted file mode 100644 index 07c6b072b..000000000 --- a/etc/nginx/vhost.d/default +++ /dev/null @@ -1,5 +0,0 @@ -add_header Content-Security-Policy "frame-ancestors 'self'"; -add_header Referrer-Policy "no-referrer, strict-origin-when-cross-origin"; -add_header X-Frame-Options "SAMEORIGIN"; -add_header X-XSS-Protection "1; mode=block"; -add_header X-Content-Type-Options "nosniff"; diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index 22fe91e28..79788d503 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -8,11 +8,15 @@ from pytest_dockerc import Wait, Context class SpotContext(Context): @property def url(self): - addr = self.container_addr("spot") - port = self.container_port("spot") + addr = self.container_addr("nginx") + port = self.container_port("nginx") return f"http://{addr}:{port}" def wait_for_running_state(self): + spot_url = "http://{}:{}".format(self.container_addr("spot"), self.container_port("spot")) + Wait(ignored_exns=(requests.ConnectionError,), timeout=60)( + lambda: requests.get(spot_url) + ) Wait(ignored_exns=(requests.ConnectionError,), timeout=60)( lambda: requests.get(self.url) ) -- GitLab From 2f016cf93618a45edad951531f682a5571486c34 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Thu, 21 Nov 2019 09:52:18 +0100 Subject: [PATCH 06/10] add traefik as router --- .env | 1 + docker-compose.yml | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/.env b/.env index 6c05454a6..5f1eff71e 100644 --- a/.env +++ b/.env @@ -2,3 +2,4 @@ SPOT_DOCKER_IMG=registry.gitlab.e.foundation:5000/e/cloud/my-spot SPOT_DOCKER_TAG=latest SPOT_NGINX_DOCKER_IMG=registry.gitlab.e.foundation:5000/e/cloud/my-spot/nginx SPOT_NGINX_DOCKER_TAG=latest +SPOT_HOSTNAME=localhost diff --git a/docker-compose.yml b/docker-compose.yml index 860297b0f..5e1afa3b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,3 +15,32 @@ services: nginx: image: ${SPOT_NGINX_DOCKER_IMG}:${SPOT_NGINX_DOCKER_TAG} restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.spot.rule=Host(`${SPOT_HOSTNAME}`)" + - "traefik.http.routers.spot.entrypoints=websecure" + - "traefik.http.routers.spot.tls.certresolver=spotchallenge" + - "traefik.http.routers.http-catchall.rule=hostregexp(`{host:.+}`)" + - "traefik.http.routers.http-catchall.entrypoints=web" + - "traefik.http.routers.http-catchall.middlewares=redirect-to-https@docker" + - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" + + traefik: + image: "traefik:v2.0.5" + command: + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--certificatesresolvers.mytlschallenge.acme.tlschallenge=true" + - "--certificatesresolvers.mytlschallenge.acme.email=contact@e.email" + - "--certificatesresolvers.mytlschallenge.acme.storage=/letsencrypt/acme.json" + ports: + - "80:80" + - "443:443" + volumes: + - "letsencrypt:/letsencrypt" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + +volumes: + letsencrypt: -- GitLab From 607906cc73491fd34db7f14cd980dc8fcde14fe7 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Thu, 21 Nov 2019 11:09:15 +0100 Subject: [PATCH 07/10] rewrite readme with useful information --- README.md | 42 +++++++++++++++++++++++++++++++++++++ README.rst | 61 ------------------------------------------------------ setup.py | 3 ++- 3 files changed, 44 insertions(+), 62 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/README.md b/README.md new file mode 100644 index 000000000..3b901e31d --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +# spot for /e/ (https://e.foundation) + +A privacy-respecting, hackable [metasearch engine](https://en.wikipedia.org/wiki/Metasearch_engine). + +Spot was forked from searx: read [documentation](https://asciimoo.github.io/searx) and the [wiki](https://github.com/asciimoo/searx/wiki) for more information. + +## Changes between Spot and Searx + +* cache results in Redis database +* eelo theme +* better python packaging and docker integration for production deployement +* drop of Python 2 support +* pytest as test launcher + +The aim is to move that changes in the upstream project when it will be easier to rebase from this one. + +## Getting Started + +You can run spot with docker-compose to run the *redis* database and +the *spot* service. First of all you have to install *docker* and +*docker-compose* on your host, then follow instructions below to run spot +with one command. + +### For production + +* Run the docker-compose *up* command to start the project +``` +COMPOSE_FILE="docker-compose.yml:docker-compose-dev.yml" docker-compose up --build nginx spot redis +``` +* Getting the ip of the spot service and go to `http://`, below the docker way to get the IP of the nginx container +``` + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' my-spot_nginx_1 +``` + +### For developer + +You can directly run spot, with a python command. First, you should install all dependencies on your host `./manage.sh update_dev_packages` or you can use the `env` container. + +``` +docker run -it --rm -v $(pwd):/ws -w /ws -e PYTHONPATH=/ws registry.gitlab.e.foundation:5000/e/cloud/my-spot/env sh +python ./searx/webapp.py +``` diff --git a/README.rst b/README.rst deleted file mode 100644 index f58430b46..000000000 --- a/README.rst +++ /dev/null @@ -1,61 +0,0 @@ -spot for /e/ (https://e.foundation) -=================================== - -A privacy-respecting, hackable `metasearch -engine `__. - -Spot was forked from searx: read `documentation `__ and the `wiki `__ for more information. - -Spot is based on Python3.7+ - -Getting Started -~~~~~~~~~~~~ - -You can run spot with docker-compose to run the **redis** database and -the **spot** service. First of all you have to install **docker** and -**docker-compose** on your host, then follow instructions below to run spot -with one command. - -- Run the docker-compose **up** command to start the project ``docker-compose up --build`` -- Getting the ip of the spot service and go to http://:8888 - -.. note:: Here the command to get the IP of the spot service - ``docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' my-spot_spot_1`` - -You can also install **redis** and **spot** on your host, for all the details, follow this `step by step -installation `__. - -Developer mode -~~~~~~~~~~~~ - -First run the redis database: - -- ``docker-compose up -d redis`` - -Then on spot workdir run the following commands to start spot: - -- ``export COMPOSE_FILE=docker-compose.yml:docker-compose-dev.yml`` -- ``docker-compose build spot`` -- ``docker-compose run --rm -v $(pwd):/ws -w /ws -e PYTHONPATH=/ws spot sh`` -- ``python3 -X dev searx/webapp.py`` - -Run tests: - -- ``docker run -it --rm -v $(pwd):/ws -w /ws -v /var/run/docker.sock:/var/run/docker.sock -e PYTHONPATH=/ws registry.gitlab.e.foundation:5000/e/cloud/my-spot/env bash`` -- ``COMPOSE_FILE=docker-compose.yml:docker-compose-dev.yml pytest --pdb --pdbcls IPython.terminal.debugger:TerminalPdb --dockerc-build -s tests/functional/`` - -Bugs -~~~~ - -Bugs or suggestions? Visit the `issue -tracker `__. - -`License `__ -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -More about searx -~~~~~~~~~~~~~~~~ - -- `openhub `__ -- `twitter `__ -- IRC: #searx @ freenode diff --git a/setup.py b/setup.py index ac5d115b2..4b6958236 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,8 @@ setup( name='searx', use_scm_version={"tag_regex": r"^(?:[\w-]+-)?(?P[vV]?\d+(?:\.\d+){0,2}.*)$"}, description="A privacy-respecting, hackable metasearch engine", - long_description=open('README.rst').read(), + long_description=open('README.md').read(), + long_description_content_type="text/markdown", classifiers=[ "Development Status :: 4 - Beta", "Programming Language :: Python", -- GitLab From 63d9d20a3fef40d9779238f2547884460a7a07b7 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Thu, 21 Nov 2019 16:32:43 +0100 Subject: [PATCH 08/10] fix coverage on functional tests --- manage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manage.sh b/manage.sh index cc96df5ad..f90a62353 100755 --- a/manage.sh +++ b/manage.sh @@ -60,7 +60,7 @@ functional_tests() { } coverage() { - sed -i 's!/usr/local/lib/python3.7/site-packages/searx[^/]*/searx!'$SEARX_DIR'!g' "$COV_DIR"/func + sed -i 's!/usr/local/lib/python3.7/site-packages/searx!'$SEARX_DIR'!g' "$COV_DIR"/func coverage3 combine coverage/func coverage/unit coverage3 report } -- GitLab From 4db8f24a6bd0c06192c421b07a73e4f416c02c35 Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Wed, 4 Dec 2019 10:03:18 +0100 Subject: [PATCH 09/10] enrich docker-compose-dev --- docker-compose-dev.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 63fad3458..ef2f640c1 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -2,9 +2,11 @@ version: '3.6' services: spot: - build: . + build: + dockerfile: Dockerfile + context: . nginx: build: - dockerfile: Dockerfile.nginx - context: . + dockerfile: Dockerfile.nginx + context: . -- GitLab From 3bb67570dbcf459ef3c4e35a5fa5400c359d221f Mon Sep 17 00:00:00 2001 From: Nicolas Gelot Date: Tue, 3 Dec 2019 15:10:55 +0100 Subject: [PATCH 10/10] debug commit --- .gitlab-ci.yml | 110 +++---------------------------------------------- 1 file changed, 5 insertions(+), 105 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d280264fd..595504800 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,117 +1,14 @@ image: $CI_REGISTRY_IMAGE/env stages: - - check - - build - - test - - report - deploy -python: - stage: check - before_script: - - ./manage.sh update_dev_packages - script: - - ./manage.sh pep8_check - -build:web: - stage: build - before_script: - - ./manage.sh npm_packages - - ./manage.sh update_dev_packages - script: - - ./manage.sh locales - - ./manage.sh styles - - ./manage.sh grunt_build - - - -.build:docker: - stage: build - image: docker:git - services: - - docker:18-dind - before_script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - script: - - docker build -t $CI_REGISTRY_IMAGE . - - docker push $CI_REGISTRY_IMAGE - - docker build -t $CI_REGISTRY_IMAGE/nginx -f Dockerfile.nginx . - - docker push $CI_REGISTRY_IMAGE/nginx - -build:docker:master: - extends: .build:docker - only: - - master - -build:docker:tags: - extends: .build:docker - script: - - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG - - docker build -t $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG -f Dockerfile.nginx . - - docker push $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG - only: - - tags - -build:docker:env: - extends: .build:docker - script: - - docker build -t $CI_REGISTRY_IMAGE/env -f Dockerfile.env - - docker push $CI_REGISTRY_IMAGE/env - only: - changes: - - Dockerfile.env - - requirements-dev.txt - - requirements.txt - -test:unit: - stage: test - before_script: - - ./manage.sh update_dev_packages - script: - - ./manage.sh unit_tests - artifacts: - paths: - - coverage - expire_in: 1 hour - -test:functional: - stage: test - image: docker:stable - services: - - docker:18-dind - variables: - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - before_script: - - docker run -id --rm -v $(pwd):/ws -e DOCKER_HOST=tcp://$(cat /etc/hosts | grep docker | cut -f1):2375/ -w /ws --name spotenv $CI_REGISTRY_IMAGE/env sh - - docker exec -i spotenv ./manage.sh update_dev_packages - script: - - docker exec -i spotenv ./manage.sh functional_tests - artifacts: - paths: - - coverage - expire_in: 1 hour - -coverage: - stage: report - script: - - ./manage.sh coverage - dependencies: - - test:unit - - test:functional - coverage: '/TOTAL.*\s+(\d+%)$/' - .deploy:template: image: docker:stable stage: deploy only: - branches when: manual - dependencies: [] - variables: - DEPLOY_FOLDER: /mnt/data before_script: - 'which ssh-agent || ( apk --update add openssh-client )' - eval $(ssh-agent -s) @@ -121,9 +18,10 @@ coverage: - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts script: + - ssh -2 $PUBLISH_USER@$PUBLISH_URL "docker ps && docker info && docker-compose version" - ssh -2 $PUBLISH_USER@$PUBLISH_URL "docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}" - - ssh -2 $PUBLISH_USER@$PUBLISH_URL "mkdir -p ${DEPLOY_FOLDER} && cd ${DEPLOY_FOLDER} && if [ ! -d ${CI_PROJECT_NAME} ] ; then git clone ${CI_PROJECT_URL}.git ; fi && cd ${CI_PROJECT_NAME} && git fetch && git checkout $CI_COMMIT_SHA" - - ssh -2 $PUBLISH_USER@$PUBLISH_URL "SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG && COMPOSE_FILE=$COMPOSE_FILE && SPOT_HOSTNAME=$SPOT_HOSTNAME && export SPOT_HOSTNAME COMPOSE_FILE SPOT_DOCKER_TAG && cd ${DEPLOY_FOLDER}/${CI_PROJECT_NAME} && docker-compose pull && docker-compose up -d --build --force-recreate" + - ssh -2 $PUBLISH_USER@$PUBLISH_URL "mkdir -p ${DEPLOY_FOLDER} && cd ${DEPLOY_FOLDER} && if [ ! -d ${CI_PROJECT_NAME} ] ; then git clone ${CI_PROJECT_URL}.git ; fi && cd ${CI_PROJECT_NAME} && git clean -dffx && git fetch && git checkout $CI_COMMIT_SHA" + - ssh -2 $PUBLISH_USER@$PUBLISH_URL "SPOT_DOCKER_TAG=$SPOT_DOCKER_TAG && COMPOSE_FILE=$COMPOSE_FILE && SPOT_HOSTNAME=$SPOT_HOSTNAME && export SPOT_HOSTNAME COMPOSE_FILE SPOT_DOCKER_TAG && cd ${DEPLOY_FOLDER}/${CI_PROJECT_NAME} && docker-compose pull && docker-compose up -d --build --force-recreate --remove-orphans" test: extends: .deploy:template @@ -134,6 +32,7 @@ test: SPOT_HOSTNAME: spot.test.ecloud.global SSH_PRIVATE_KEY: ${SSH_PRIVATE_KEY_TEST} COMPOSE_FILE: docker-compose.yml:docker-compose-dev.yml + DEPLOY_FOLDER: /mnt/data prod: extends: .deploy:template @@ -147,3 +46,4 @@ prod: SPOT_DOCKER_TAG: ${CI_COMMIT_REF_SLUG} SSH_PRIVATE_KEY: ${SSH_PRIVATE_KEY_PROD} COMPOSE_FILE: docker-compose.yml + DEPLOY_FOLDER: /mnt/data -- GitLab