diff --git a/.gitignore b/.gitignore
index 67956823a7615f6f9a6493ee8911a2769b9ab309..f95eabb39795d2dfd0a85885d3baf2f59f410b23 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,4 +49,5 @@ tags
# Python
build/
dist/
+venv/
*.egg-info/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 152dd89eb3d087f998acee5b147f60bf09a04c1e..5e7fda9eb6bbfffb6063a2564a9eafa408a5373d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,6 +3,7 @@
image: python:3.7-slim-stretch
stages:
+ - test
- build
- deploy
@@ -10,17 +11,106 @@ variables:
package: gitlab-job-guard
before_script:
- - env | sort | egrep -i 'CI|GIT'
+ - env | sort |
+ perl -pe 's/((?:PASSWORD|TOKEN)=)(.*)/$1 . q[X] x length($2)/e'
- pwd; pwd -P
- - python -m pip install --upgrade setuptools wheel twine
+ - python -m pip install --upgrade pip virtualenv
+ - virtualenv -p python venv
+ - . venv/bin/activate
+ - python -m pip install --upgrade pip setuptools virtualenv wheel
+ - echo '.' > requirements.txt
- pip install -e ./
- - pip freeze > requirements.txt
+ - pip freeze -l --exclude-editable > requirements.txt
- awk -F"'" '
/version/ { $0=$1"'"'"$CI_COMMIT_REF_NAME"'"'"$3 }{ print $0 }
' setup.py > setup.py.tmp; mv -v setup.py.tmp setup.py
- chmod +x ./setup.py
- - ./setup.py sdist bdist_wheel
- - find */ -type f -ls -a -exec sha256sum {} +
+ - python ./setup.py sdist bdist_wheel
+ - pip install dist/gitlab-job-guard*.tar.gz
+
+'test-py2':
+ stage: test
+ image: python:2.7-slim-stretch
+ script:
+ - python ./setup.py sdist bdist_wheel
+ - pip install dist/gitlab-job-guard*.tar.gz
+ - python2 $(which gitlab-job-guard) -c='feature/' -s 'running' -w 60;
+
+'test-py3':
+ stage: test
+ image: python:3.7-slim-stretch
+ script:
+ - python3 $(which gitlab-job-guard) -c='feature/' -s 'running' -w 60;
+
+'test-py3-exit-immediately':
+ stage: test
+ image: python:3.7-slim-stretch
+ script:
+ - ( set +e; gitlab-job-guard -c='.+' -s='.+' -x; [ $? = 7 ] )
+
+'test-py3-retry-until-timeout':
+ stage: test
+ image: python:3.7-slim-stretch
+ script:
+ - ( set +e; gitlab-job-guard -c='.+' -s='.+' -w 60; [ $? = 11 ] )
+
+'test-py3-retry-failed-until-timeout':
+ stage: test
+ image: python:3.7-slim-stretch
+ script:
+ - ( set +e;
+ gitlab-job-guard -c='.+' -s='failed|skipped' -w 60;
+ [ $? = 11 ]
+ )
+
+'test-py3-retry-feature-failed-until-timeout':
+ stage: test
+ image: python:3.7-slim-stretch
+ script:
+ - ( set +e;
+ gitlab-job-guard -c='feature/' -s='failed|skipped' -w 60;
+ [ $? = 11 ]
+ )
+
+'test-py3-unreachable-api-endpoint':
+ stage: test
+ image: python:3.7-slim-stretch
+ script:
+ - ( set +e;
+ CI_API_V4_URL='https://gitlab.local/api/v4/'
+ gitlab-job-guard -c='.+' -s='.+' -w 60;
+ [ $? = 11 ]
+ )
+
+'test-py3-invalid-api-url':
+ stage: test
+ image: python:3.7-slim-stretch
+ script:
+ - ( set +e;
+ CI_API_V4_URL="$CI_API_V4_URL/blah-blah-blah"
+ gitlab-job-guard -c='.+' -s='.+' -w 60;
+ [ $? = 11 ]
+ )
+
+'test-py3-invalid-project-id':
+ stage: test
+ image: python:3.7-slim-stretch
+ script:
+ - ( set +e;
+ CI_PROJECT_ID='123123123'
+ gitlab-job-guard -c='.+' -s='.+' -w 60;
+ [ $? = 11 ]
+ )
+
+'test-py3-invalid-token':
+ stage: test
+ image: python:3.7-slim-stretch
+ script:
+ - ( set +e;
+ PRIVATE_TOKEN='abracadabra'
+ gitlab-job-guard -c='.+' -s='.+' -w 60;
+ [ $? = 11 ]
+ )
'build-package':
stage: build
@@ -37,7 +127,9 @@ before_script:
script:
- echo "Deploying package to artifactory"
- echo "Deploying $package@$CI_COMMIT_REF_NAME to $af_repo_url as $af_username ..."
+ - python -m pip install --upgrade twine
- cat setup.py
+ - twine check dist/*
- twine upload
--repository-url "$af_url"
--username "$af_user"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cb7f6b1af6868dc9e5da009d626e24107dbeff51..8eb515dd3eb44815bca33d5f1fff026b6dc4e9fd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+v0.0.2 - 2019-04-16T18:29:44
+----------------------------
+
+* Add README.md
+* Add e2e test suite
+* Fix up CI build issues
+
v0.0.1 - 2019-04-15T20:17:13
----------------------------
diff --git a/README.md b/README.md
index 232be412a7de068982e64dbee6a9ae0e0299901b..5b02dc19f79b41c8af2422324e3905526c02b0f2 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,118 @@
+[](https://gitlab.com/s.bhooshi/gitlab-job-guard/commits/master)
+
# gitlab-job-guard
-Guard pipeline jobs from multiple simultaneous executions
\ No newline at end of file
+Guard pipeline jobs from multiple simultaneous executions
+
+```bash
+$ PRIVATE_TOKEN="$GITLAB_API_TOKEN" gitlab-job-guard -w 3600
+$ my-unguarded-deployment-task --to=production
+```
+
+`gitlab-job-guard` will block if it detects other pipelines running for
+the current project to avoid multiple pipelines from clobbering up a
+deployment/environment.
+
+While gitlab will _auto-cancel redundant, pending pipelines_ for the same branch
+by default - this is not the case for multiple pipelines from _different
+branches_ targeting a particular deployment/environment. Gitlab has no way to
+detect or control these user-defined branch-to-environment mappings and this
+means environments can easily be left in an unsafe/broken state. (e.g.
+`terraform apply` or `ansible`, etc from different pipelines running at the same
+time).
+
+`gitlab-job-guard` uses the Gitlab API to determine if existing pipelines are
+scheduled and to backoff-and-retry until it is safe to proceed. Conflicts
+are detected by user-defined matches on pipeline ref names (branch, tag, etc)
+and/or pipeline status.
+
+## Usage
+
+The simplest usage would likely be placing `gitlab-job-guard` in a
+`before_script` section in your `gitlab-ci.yml` to protect all jobs (though this
+can slow things down).
+
+```yaml
+before_script:
+ - PRIVATE_TOKEN="$GITLAB_API_TOKEN" gitlab-job-guard
+```
+
+Though often, this is only needed to guard jobs that share common state/data
+(i.e. a deployment environment, an artifact build/release, etc).
+
+```yaml
+deploy-production:
+ stage: deploy
+ script:
+ - PRIVATE_TOKEN="$GITLAB_API_TOKEN" gitlab-job-guard
+ - my-unguarded-deployment-task --to=production
+```
+
+or to guard something like a `terraform` job running for tags.
+
+```yaml
+provision-infrastructure:
+ stage: provision
+ script:
+ - export PRIVATE_TOKEN="$GITLAB_API_TOKEN"
+ - gitlab-job-guard --guard-ref-regex='^v[0-9\.]+' # Regex matches tags
+ - terraform plan ...
+ - terraform apply ...
+ only:
+ - tags
+```
+
+### Other usages
+
+To hold jobs for a collisions on pattern matches on the ref/branch name.
+
+```bash
+gitlab-job-guard -c=^master$ # Match branch names matching 'master' exactly
+
+gitlab-job-guard -c=^(master|dev(elop)?)$ # Match any of the mainline branches
+
+gitlab-job-guard -c=^(feature|release|hotfix)/ # Match any gitflow transient branch prefixes
+
+gitlab-job-guard -c=^[0-9]\- # Match branch names beginning with a number
+ # and dash ignoring all other text.
+ # e.g. a gitlab branch made from an issue
+
+gitlab-job-guard -c=^v?[\d.]+$ # Match (semver) tags like v1.0.9, 2.0
+
+gitlab-job-guard -c=^environment/ # Match any environment deployments?
+
+gitlab-job-guard -c=^environment/dc1.+ # Match environment deployments to DC1?
+
+gitlab-job-guard -c="$CI_BUILD_REF_NAME" # Match current branch name (partially).
+ # i.e. 'master' matches 'feature/master-document'
+
+gitlab-job-guard -c="^$CI_BUILD_REF_NAME$" # Match current branch name (exactly).
+ # i.e. 'master' does not match 'master-deployment'
+
+gitlab-job-guard -c='.+' -s='running|pending' # Match any pipeline in running or pending state
+```
+
+To hold a job for a collision on part of the ref name (e.g. on branch prefix
+such as `feature/` or `hotfix/` or `release/`, etc _a la_ `gitflow`).
+
+```bash
+# Assuming CI_BUILD_REF_NAME=feature/foo
+
+CI_BUILD_REF_PREFIX=$(echo "$CI_BUILD_REF_NAME" | sed -r 's@(.+/)(.+)@\1@')
+# CI_BUILD_REF_PREFIX now contains 'feature/'
+
+gitlab-job-guard -c="^$CI_BUILD_REF_PREFIX" -s='running|pending'
+```
+
+# TODO
+
+For long pipelines, this solution can have subtle consequences with growing
+queues and increased contention and unpredictability as to which pipeline is
+the first-past-the-post. An older pipeline taking precedence over newer commits
+if often not desired and newer pipelines always winning is probably desired.
+
+* Handle existing conflicting pipelines - cancel them or give-way.
+* Narrow down conflicts to jobs (`CI_JOB_NAME`) or stages (`CI_JOB_STAGE`)
+ so that other parts of the pipelines that do not share state are allowed to
+ run freely.
+
diff --git a/gitlab-job-guard/gitlab-job-guard b/gitlab-job-guard/gitlab-job-guard
new file mode 120000
index 0000000000000000000000000000000000000000..27579279f4481b29ce4001b8372d609e8df81ca5
--- /dev/null
+++ b/gitlab-job-guard/gitlab-job-guard
@@ -0,0 +1 @@
+gitlab-job-guard.py
\ No newline at end of file
diff --git a/gitlab-job-guard/guard.py b/gitlab-job-guard/gitlab-job-guard.py
similarity index 93%
rename from gitlab-job-guard/guard.py
rename to gitlab-job-guard/gitlab-job-guard.py
index c96d96178f229f07c86c6277ea71de0b359a14a0..b5e4f497d5bd5f8680b8e81643762eb36605f477 100755
--- a/gitlab-job-guard/guard.py
+++ b/gitlab-job-guard/gitlab-job-guard.py
@@ -2,7 +2,8 @@
# -*- coding: utf-8 -*-
-# guard.py - guard pipeline jobs from multiple simultaneous executions
+# gitlab-job-guard
+# guard pipeline jobs from multiple simultaneous executions
from __future__ import absolute_import, division, print_function
@@ -15,6 +16,7 @@ except ImportError:
import simplejson as json
import logging
from os import environ, path
+from os.path import basename
from posixpath import join as urljoin
from random import randint, random
import re
@@ -87,7 +89,7 @@ def setup_logger(*args, **kwargs):
'''
Setup and return the root logger ojbect for the application
'''
- root = logging.getLogger(__file__)
+ root = logging.getLogger(basename(__file__))
root.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
@@ -103,6 +105,13 @@ def setup_logger(*args, **kwargs):
def print_unbuffered(*args, **kwargs):
'''
Workaround for python3 where stderr is buffered. WTH python?
+ TODO: We could use python -u or set PYTHONUNBUFFERED in the environment but
+ those are both settings made outside of this script and also requires
+ users to be explicit in how they call it. We need an elegant way that
+ does not impose on the user and always does the right thing.
+ Fix this once python2 is fully deprecated and we only have to support
+ python3 - we may be able to set the shebang to something like
+ #!/usr/local/bin/python3 -u
'''
if PY3:
kwargs['flush'] = True
@@ -261,7 +270,7 @@ def main():
conflicts = [ p for p in runs if
re.search( args.guard_ref_regex, p.ref ) and
re.match( args.guard_status_regex, p.status ) and
- p.id != args.ci_pipeline_id ]
+ int(p.id) != int(args.ci_pipeline_id) ]
except Exception as e:
log.error('{}("{}")'.format(e.__class__.__name__, str(e)))
diff --git a/setup.py b/setup.py
index 12710500342e057dc70962e851eafff333633ece..3c5be66173fc562e0aee984df7a7d8ce9bec3124 100755
--- a/setup.py
+++ b/setup.py
@@ -14,18 +14,30 @@ requests
six
'''.split('\n')
+
+# read the contents of your README file
+from os import path
+this_directory = path.abspath(path.dirname(__file__))
+with open(path.join(this_directory, 'README.md')) as f:
+ long_description = f.read()
+
setup( name='gitlab-job-guard',
version='v0.0.1',
description="Guard gitlab jobs from multiple simultaneous executions",
+ long_description=long_description,
+ long_description_content_type="text/markdown",
url='https://gitlab.com/s.bhooshi/gitlab-job-guard',
author='Shalom Bhooshi',
author_email='s.bhooshi@gmail.com',
license='Apache License 2.0',
packages=['gitlab-job-guard'],
zip_safe=False,
- scripts=['gitlab-job-guard/guard.py'],
+ scripts=[
+ 'gitlab-job-guard/gitlab-job-guard.py',
+ 'gitlab-job-guard/gitlab-job-guard'
+ ],
install_requires=requirements,
keywords='gitlab-ci pipeline job guard',
- python_requires='>=2.7, >=3.5, !=3.0, !=3.0.*, !=3.1, !=3.1.*, !=3.2, !=3.2.*, !=3.3, !=3.3.*, !=3.4, !=3.4.*',
+ python_requires='>=2.7, >=2.7.1, !=3.0, !=3.0.*, !=3.1, !=3.1.*, !=3.2, !=3.2.*, !=3.3, !=3.3.*, !=3.4, !=3.4.*',
)