Commit dd335740 authored by Mitar's avatar Mitar

Compute coverage of pipelines.

parent 9048557d
Pipeline #98843344 failed with stages
in 4 minutes and 30 seconds
......@@ -72,6 +72,21 @@ validate_all_devel:
allow_failure: true
validate_coverage:
stage: test
before_script:
- git lfs fetch --all
script:
- python3 ./run_coverage.py v*
only:
refs:
- master@datadrivendiscovery/primitives
allow_failure: true
git_check:
stage: build
......
......@@ -6,6 +6,8 @@ import gzip
import itertools
import json
import os.path
import sys
import traceback
import yaml
......@@ -14,6 +16,10 @@ D3M_PIPELINES = {
}
class PipelineNotFoundError(Exception):
pass
def primitives_used_in_pipeline(pipeline):
primitives = set()
......@@ -59,23 +65,28 @@ def resolve_pipeline(pipelines_dir, pipeline_id):
if pipeline_id in D3M_PIPELINES:
return None
print("Could not resolve '{pipeline_id}' in '{pipelines_dir}'.".format(
raise PipelineNotFoundError("Could not resolve '{pipeline_id}' in '{pipelines_dir}'.".format(
pipeline_id=pipeline_id,
pipelines_dir=pipelines_dir,
))
return None
def process_interface_version(interface_version):
known_primitives = {}
primitives_used_in_pipelines = set()
error = False
for primitive_annotation_path in glob.iglob('{interface_version}/*/*/*/primitive.json'.format(interface_version=interface_version)):
with open(primitive_annotation_path, 'r', encoding='utf8') as primitive_annotation_file:
primitive_annotation = json.load(primitive_annotation_file)
try:
with open(primitive_annotation_path, 'r', encoding='utf8') as primitive_annotation_file:
primitive_annotation = json.load(primitive_annotation_file)
known_primitives[primitive_annotation['id']] = primitive_annotation
known_primitives[primitive_annotation['id']] = primitive_annotation
except Exception:
print("Error at primitive '{primitive_annotation_path}'.".format(primitive_annotation_path=primitive_annotation_path), flush=True)
traceback.print_exc()
sys.stdout.flush()
error = True
for pipeline_run_path in itertools.chain(
glob.iglob('{interface_version}/*/*/*/pipeline_runs/*.yml.gz'.format(interface_version=interface_version)),
......@@ -83,28 +94,47 @@ def process_interface_version(interface_version):
):
pipelines_dir = os.path.sep.join(pipeline_run_path.split(os.path.sep)[:-2] + ['pipelines'])
with gzip.open(pipeline_run_path, 'rt', encoding='utf8') as pipeline_run_file:
pipeline_runs = yaml.safe_load_all(pipeline_run_file)
for pipeline_run in pipeline_runs:
for pipeline_id in pipelines_used_in_pipeline_run(pipeline_run):
pipeline = resolve_pipeline(pipelines_dir, pipeline_id)
if pipeline:
primitives_used_in_pipelines.update(primitives_used_in_pipeline(pipeline))
try:
with gzip.open(pipeline_run_path, 'rt', encoding='utf8') as pipeline_run_file:
pipeline_runs = yaml.safe_load_all(pipeline_run_file)
for pipeline_run in pipeline_runs:
for pipeline_id in pipelines_used_in_pipeline_run(pipeline_run):
try:
pipeline = resolve_pipeline(pipelines_dir, pipeline_id)
except PipelineNotFoundError as e:
error = True
print(str(e), flush=True)
continue
if pipeline:
primitives_used_in_pipelines.update(primitives_used_in_pipeline(pipeline))
except Exception:
print("Error at pipeline run '{pipeline_run_path}'.".format(pipeline_run_path=pipeline_run_path), flush=True)
traceback.print_exc()
sys.stdout.flush()
error = True
for primitive_id, primitive in known_primitives.items():
if primitive_id not in primitives_used_in_pipelines:
print(primitive['source']['name'], primitive['id'], primitive['python_path'])
error = True
print(primitive['source']['name'], primitive['id'], primitive['python_path'], flush=True)
return error
def main():
parser = argparse.ArgumentParser(description="Compute pipeline coverage.")
parser = argparse.ArgumentParser(description="Compute pipelines coverage.")
parser.add_argument('interface_versions', metavar='INTERFACE', nargs='*', help="interface version(s) to compute coverage for", default=())
arguments = parser.parse_args()
error = False
for interface_version in arguments.interface_versions:
process_interface_version(interface_version)
error = process_interface_version(interface_version) or error
if error:
sys.exit(1)
if __name__ == '__main__':
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment