feat: implement #194: handle 'matrix' nodes as job nodes variants (853ea696) · Commits · RadianDevCore / Tools / gcil

gitlabci_local/parsers/gitlab.py

+80 −9

Changes for gitlabci_local/parsers/gitlab.py: 80 added lines, 9 removed lines.
Original line number	Diff line number	Diff line
		@@ -2,6 +2,7 @@

		# Standard libraries
		from collections import OrderedDict
		from itertools import product
		from os import environ
		from os.path import expandvars
		from pathlib import Path
		@@ -375,6 +376,54 @@ class GitLab:
		local['configurations'])
		global_values['variables'].update(configured_variables)

		# Variants
		@staticmethod
		def __variants(data, node):

		# Variables
		variants = []

		# Handle matrix variants list
		if 'parallel' in data[node] and 'matrix' in data[node]['parallel']:

		# Iterate through matrix items
		for matrix_item in data[node]['parallel']['matrix']:

		# Prepare matrix environments
		matrix_item_environments = []
		matrix_item_environment = {}

		# Iterate through matrix item variables
		for matrix_variable, matrix_values in matrix_item.items():

		# Already defined environment variable
		if matrix_variable in environ:
		matrix_item_environment[matrix_variable] = [
		environ[matrix_variable]
		]

		# Matrix defined environment variable
		else:
		matrix_item_environment[matrix_variable] = []
		for matrix_value in matrix_values:
		matrix_item_environment[matrix_variable] += [matrix_value]

		# Register all item combinations
		keys, values = zip(*matrix_item_environment.items())
		matrix_item_environments = [dict(zip(keys, v)) for v in product(*values)]
		for matrix_item_environment in matrix_item_environments:
		variants += [{
		'name': f"{node}: [{', '.join(list(matrix_item_environment.values()))}]",
		'environment': matrix_item_environment
		}]

		# Prepare default variants list
		else:
		variants = [{'name': node, 'environment': {}}]

		# Result
		return variants

		# Parse, pylint: disable=too-many-branches
		def parse(self, data, environment):

		@@ -454,26 +503,48 @@ class GitLab:
		if node[0:1] == '.':
		continue

		# Nodes variants
		variants = GitLab.__variants(data, node)

		# Iterate through node
		for variant in variants:

		# Prepare variant environment
		if variant['environment']:
		node_environ = dict(environ)
		environ.update(variant['environment'])

		# Register job
		jobs[node] = self.job(node, data, global_values)
		jobs[variant['name']] = self.job(node, variant['name'], data,
		global_values)

		# Prepare variant variables
		if variant['environment']:
		jobs[variant['name']]['variables'].update(variant['environment'])

		# Validate job script
		if not jobs[node]['options']['disabled'] and not jobs[node]['script']:
		if not jobs[variant['name']]['options']['disabled'] and not jobs[
		variant['name']]['script']:
		raise ValueError(
		f"Missing \"script\" key for \"{jobs[node]['stage']} / {jobs[node]['name']}\""
		f"Missing \"script\" key for \"{jobs[variant['name']]['stage']} / {jobs[variant['name']]['name']}\""
		)

		# Append unknown stage if required
		if jobs[node]['options']['disabled'] and jobs[node][
		if jobs[variant['name']]['options']['disabled'] and jobs[variant['name']][
		'stage'] == GitLab.JOB_STAGE_DEFAULT and GitLab.JOB_STAGE_DEFAULT not in stages:
		stages[GitLab.JOB_STAGE_DEFAULT] = list(stages.values())[-1] + 1

		# Validate job stage
		if jobs[node]['stage'] not in stages:
		if jobs[variant['name']]['stage'] not in stages:
		raise ValueError(
		f"Unknown stage \"{jobs[node]['stage']}\" for \"{jobs[node]['name']}\""
		f"Unknown stage \"{jobs[variant['name']]['stage']}\" for \"{jobs[variant['name']]['name']}\""
		)

		# Restore variant environment
		if variant['environment']:
		environ.clear()
		environ.update(node_environ)

		# Sort jobs based on stages
		jobs = OrderedDict(sorted(jobs.items(), key=lambda x: stages[x[1]['stage']]))

		@@ -485,11 +556,11 @@ class GitLab:
		return jobs

		# Job, pylint: disable=too-many-branches,too-many-statements
		def job(self, job_name, data, global_values):
		def job(self, job_node, job_name, data, global_values):

		# Variables
		job = {}
		job_data = data[job_name]
		job_data = data[job_node]

		# Prepare stage
		job['name'] = job_name
		@@ -531,7 +602,7 @@ class GitLab:
		if job['stage'] is None:
		job['stage'] = GitLab.JOB_STAGE_DEFAULT
		break
		job_extended = self.job(job_extend, data, None)
		job_extended = self.job(job_extend, job_extend, data, None)

		# Extract extended job
		if job['stage'] is None:

tests/parallel/.gitlab-ci.yml

0 → 100644

+46 −0

Changes for tests/parallel/.gitlab-ci.yml: 46 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		stages:
		- parallel

		'Job 1':
		image: python:${PYTHON_VERSION}
		stage: parallel
		parallel:
		matrix:
		- PYTHON_VERSION: ['3.6', '3.9']
		script:
		- echo 'Job 1'
		- echo "${PYTHON_VERSION}"

		'Job 2':
		image: python:${PYTHON_VERSION}
		stage: parallel
		parallel:
		matrix:
		- PYTHON_VERSION: ['3.6', '3.9']
		VALUE1:
		- '1'
		- '2'
		- '3'
		VALUE2:
		- '1'
		- '2'
		- PYTHON_VERSION: ['3.6', '3.9']
		VALUE1:
		- '3'
		- '2'
		- '1'
		VALUE2:
		- '2'
		- '1'
		VALUE3:
		- '1'
		- '2'
		script:
		- env \| grep 'VALUE.*'
		- echo 'Job 1'
		- echo "${PYTHON_VERSION}"
		- test "${VALUE1:-0}" -gt 0
		- test "${VALUE1:-0}" -lt 4
		- test "${VALUE2:-0}" -gt 0
		- test "${VALUE2:-0}" -lt 3
		- echo "${VALUE3}"

tests/parallel/test.sh

0 → 100644

+21 −0

Changes for tests/parallel/test.sh: 21 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		#!/bin/sh

		# Access folder
		script_path=$(readlink -f "${0}")
		test_path=$(readlink -f "${script_path%/*}")
		cd "${test_path}/"

		# Configure tests
		set -ex

		# Run tests
		gitlabci-local 'Job 1'
		gitlabci-local -e PYTHON_VERSION=3.6 'Job 1'
		gitlabci-local -e PYTHON_VERSION=0.0 'Job 1' && exit \|\| true
		gitlabci-local -H 'Job 2'
		gitlabci-local -H -e VALUE1= 'Job 2' && exit \|\| true
		gitlabci-local -H -e VALUE1=3 'Job 2'
		gitlabci-local -H -e VALUE1=4 'Job 2' && exit \|\| true
		gitlabci-local -H -e VALUE2= 'Job 2' && exit \|\| true
		gitlabci-local -H -e VALUE2=2 'Job 2'
		gitlabci-local -H -e VALUE2=3 'Job 2' && exit \|\| true