From c4f08bebf1b7a7079fe88b0a58e5779b261b7629 Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Fri, 25 Mar 2022 14:02:23 +0530 Subject: [PATCH 1/7] add new stage for update from upstream --- .gitlab-ci.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e200663fc..dbde1d7eb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,10 +5,39 @@ stages: - build - test - deploy + - update-from-upstream services: - docker:20.10-dind +searx: + stage: update-from-upstream + image: registry.gitlab.e.foundation/e/tools/docker-tools:latest + rules: + - if: '$CI_PIPELINE_SOURCE =~ /schedule/ && $CI_COMMIT_REF_NAME == $LOCAL_BRANCH' + variables: + GIT_STRATEGY: none + CI_PROJECT_SSH_URL: git@gitlab.e.foundation:$CI_PROJECT_PATH + before_script: + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - echo "${SSH_E_ROBOT_PRIVATE_KEY}" | tr -d '\r' | ssh-add - > /dev/null + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - echo "${SSH_KNOWN_HOSTS}" > /root/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts + - git config --global user.email $GITLAB_USER_EMAIL + - git config --global user.name "$GITLAB_USER_NAME" + - cd $CI_BUILD_DIR + - rm -rf $CI_PROJECT_DIR + - git clone $CI_PROJECT_SSH_URL $CI_PROJECT_DIR + - cd $CI_PROJECT_DIR + script: + - git fetch origin + - git checkout $CI_COMMIT_REF_NAME + - git merge origin/$UPSTREAM_BRANCH + - git push + python: stage: check script: -- GitLab From e6ef9ffcc0903a75f3eb50895d1631511fb4915c Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Fri, 25 Mar 2022 18:32:23 +0530 Subject: [PATCH 2/7] test ci for schedules --- .gitlab-ci.yml | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dbde1d7eb..f116a1594 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,12 +40,24 @@ searx: python: stage: check + rules: + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + when: never + - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + when: on_success script: - ./manage test.pep8 - ./manage test.pylint build:web: stage: build + rules: + - if: '$CI_COMMIT_TAG' + when: never + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + when: never + - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + when: on_success before_script: - ./manage node.update - ./manage node.env @@ -67,6 +79,13 @@ build:web: - docker push $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG build:docker: + rules: + - if: '$CI_COMMIT_TAG' + when: never + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + when: never + - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + when: on_success extends: .build:docker build:docker:tags: @@ -81,6 +100,13 @@ build:docker:tags: test:unit: stage: test + rules: + - if: '$CI_COMMIT_TAG' + when: never + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + when: never + - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + when: on_success script: - ./manage test.coverage coverage: '/TOTAL.*\s+(\d+%)$/' @@ -102,8 +128,13 @@ test:unit: deploy:spot.eeo.one.backend1: extends: .deploy:template when: manual - only: - - branches + rules: + - if: '$CI_COMMIT_TAG' + when: never + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + when: never + - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + when: on_success environment: name: eeo1 url: https://spot.eeo.one @@ -144,8 +175,13 @@ deploy:spot.eeo.one.backend1: deploy:spot.eeo.one.backend2: extends: .deploy:template when: manual - only: - - branches + rules: + - if: '$CI_COMMIT_TAG' + when: never + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + when: never + - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + when: on_success environment: name: eeo2 url: https://spot.eeo.one -- GitLab From 94b8a0bede034fbcf18855a92678dc386585141f Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Thu, 31 Mar 2022 15:54:36 +0530 Subject: [PATCH 3/7] fix double pipelines in MR --- .gitlab-ci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 16fc45eef..000b23d6d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,9 +41,9 @@ searx: python: stage: check rules: - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + - if: '$CI_PIPELINE_SOURCE == push' when: on_success script: - ./manage test.pep8 @@ -54,9 +54,9 @@ build:web: rules: - if: '$CI_COMMIT_TAG' when: never - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + - if: '$CI_PIPELINE_SOURCE == push' when: on_success before_script: - ./manage node.update @@ -84,9 +84,9 @@ build:docker: rules: - if: '$CI_COMMIT_TAG' when: never - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + - if: '$CI_PIPELINE_SOURCE == push' when: on_success extends: .build:docker @@ -107,9 +107,9 @@ test:unit: rules: - if: '$CI_COMMIT_TAG' when: never - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + - if: '$CI_PIPELINE_SOURCE == push' when: on_success script: - ./manage test.coverage @@ -135,9 +135,9 @@ deploy:spot.eeo.one.backend1: rules: - if: '$CI_COMMIT_TAG' when: never - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + - if: '$CI_PIPELINE_SOURCE == push' when: on_success environment: name: eeo1 @@ -183,9 +183,9 @@ deploy:spot.eeo.one.backend2: rules: - if: '$CI_COMMIT_TAG' when: never - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger/' + - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE =~ /push|merge_request_event/' + - if: '$CI_PIPELINE_SOURCE == push' when: on_success environment: name: eeo2 -- GitLab From 2ae9cd910d762df143ee4ba14ed607910668d052 Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Thu, 31 Mar 2022 15:58:05 +0530 Subject: [PATCH 4/7] fix push in CI_PIPELINE --- .gitlab-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 000b23d6d..6225e9798 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,7 +43,7 @@ python: rules: - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE == push' + - if: '$CI_PIPELINE_SOURCE == "push"' when: on_success script: - ./manage test.pep8 @@ -56,7 +56,7 @@ build:web: when: never - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE == push' + - if: '$CI_PIPELINE_SOURCE == "push"' when: on_success before_script: - ./manage node.update @@ -86,7 +86,7 @@ build:docker: when: never - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE == push' + - if: '$CI_PIPELINE_SOURCE == "push"' when: on_success extends: .build:docker @@ -109,7 +109,7 @@ test:unit: when: never - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE == push' + - if: '$CI_PIPELINE_SOURCE == "push"' when: on_success script: - ./manage test.coverage @@ -137,7 +137,7 @@ deploy:spot.eeo.one.backend1: when: never - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE == push' + - if: '$CI_PIPELINE_SOURCE == "push"' when: on_success environment: name: eeo1 @@ -185,7 +185,7 @@ deploy:spot.eeo.one.backend2: when: never - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' when: never - - if: '$CI_PIPELINE_SOURCE == push' + - if: '$CI_PIPELINE_SOURCE == "push"' when: on_success environment: name: eeo2 -- GitLab From fc7a04d304da4f12dedefa8d7888632467304014 Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Thu, 31 Mar 2022 16:47:56 +0530 Subject: [PATCH 5/7] run on schedules --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6225e9798..9618dc6a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,8 @@ services: searx: stage: update-from-upstream image: registry.gitlab.e.foundation/e/tools/docker-tools:latest - rules: - - if: '$CI_PIPELINE_SOURCE =~ /schedule/ && $CI_COMMIT_REF_NAME == $LOCAL_BRANCH' + only: + - schedules variables: GIT_STRATEGY: none CI_PROJECT_SSH_URL: git@gitlab.e.foundation:$CI_PROJECT_PATH -- GitLab From d492aefc2fdd05f9a7cd32aa6c81f41b871d915d Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Thu, 31 Mar 2022 17:04:44 +0530 Subject: [PATCH 6/7] use only and excepts --- .gitlab-ci.yml | 68 +++++++++++++++++++++----------------------------- 1 file changed, 28 insertions(+), 40 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9618dc6a3..1ff33290b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,24 +40,22 @@ searx: python: stage: check - rules: - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' - when: never - - if: '$CI_PIPELINE_SOURCE == "push"' - when: on_success + only: + - tags + - branches + except: + - schedules script: - ./manage test.pep8 - ./manage test.pylint build:web: stage: build - rules: - - if: '$CI_COMMIT_TAG' - when: never - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' - when: never - - if: '$CI_PIPELINE_SOURCE == "push"' - when: on_success + only: + - tags + - branches + except: + - schedules before_script: - ./manage node.update - ./manage node.env @@ -81,13 +79,11 @@ build:web: - docker push $CI_REGISTRY_IMAGE/nginx:$CI_COMMIT_REF_SLUG build:docker: - rules: - - if: '$CI_COMMIT_TAG' - when: never - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' - when: never - - if: '$CI_PIPELINE_SOURCE == "push"' - when: on_success + only: + - tags + - branches + except: + - schedules extends: .build:docker build:docker:tags: @@ -104,13 +100,11 @@ build:docker:tags: test:unit: stage: test - rules: - - if: '$CI_COMMIT_TAG' - when: never - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' - when: never - - if: '$CI_PIPELINE_SOURCE == "push"' - when: on_success + only: + - tags + - branches + except: + - schedules script: - ./manage test.coverage coverage: '/TOTAL.*\s+(\d+%)$/' @@ -132,13 +126,10 @@ test:unit: deploy:spot.eeo.one.backend1: extends: .deploy:template when: manual - rules: - - if: '$CI_COMMIT_TAG' - when: never - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' - when: never - - if: '$CI_PIPELINE_SOURCE == "push"' - when: on_success + only: + - branches + except: + - schedules environment: name: eeo1 url: https://spot.eeo.one @@ -180,13 +171,10 @@ deploy:spot.eeo.one.backend1: deploy:spot.eeo.one.backend2: extends: .deploy:template when: manual - rules: - - if: '$CI_COMMIT_TAG' - when: never - - if: '$CI_PIPELINE_SOURCE =~ /schedule|web|api|trigger|merge_request_event/' - when: never - - if: '$CI_PIPELINE_SOURCE == "push"' - when: on_success + only: + - branches + except: + - schedules environment: name: eeo2 url: https://spot.eeo.one -- GitLab From d42a5d06c363fbf0bd0b45185929b7ac202f2019 Mon Sep 17 00:00:00 2001 From: Nivesh Krishna Date: Fri, 8 Apr 2022 18:22:53 +0530 Subject: [PATCH 7/7] fetch from upstream --- .gitlab-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ff33290b..170b1c5c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,9 @@ searx: script: - git fetch origin - git checkout $CI_COMMIT_REF_NAME - - git merge origin/$UPSTREAM_BRANCH + - git remote add upstream $UPSTREAM_REPO + - git pull upstream/$UPSTREAM_BRANCH + - git merge upstream/$UPSTREAM_BRANCH - git push python: -- GitLab