chore: remove job template

Signed-off-by: GridexX <arsene582@gmail.com>

.gitlab-ci.yml

@@ -63,6 +63,26 @@ ci_linter:
     - changes:
       - "jobs/**/*${JOB_FILES_EXTENSION}"
 
+job_structure:
+  image: python:${IMAGE_TAG_PYTHON_ALPINE}
+  stage: static_tests
+  variables:
+    PIPENV_PIPFILE: tools/job_structure/Pipfile
+    JOB_LOGFILE: "job_structure.log"
+    PYTHONPATH: "./:$PYTHONPATH"
+  before_script:
+    - pip install --ignore-installed distlib pipenv
+    - pipenv install
+  script:
+    - pipenv run python3 tools/job_structure/job_structure.py
+  artifacts:
+    expose_as: "job_structure"
+    paths:
+      - ${JOB_LOGFILE}
+    when: always
+  rules:
+    - changes:
+      - jobs/**/*
 
 job_customs:
   image: python:${IMAGE_TAG_PYTHON_ALPINE}

tools/job_structure/Pipfile

+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[requires]
+python_version = "3"
+
+[packages]
+pyyaml = "==5.3.1"

tools/job_structure/Pipfile.lock

+{
+    "_meta": {
+        "hash": {
+            "sha256": "ac0ce3b857220356672b701f08dc6413e0373a41f9442ca3479b74651dd8f21d"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "pyyaml": {
+            "hashes": [
+                "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",
+                "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76",
+                "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",
+                "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e",
+                "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648",
+                "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",
+                "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f",
+                "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",
+                "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee",
+                "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a",
+                "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",
+                "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",
+                "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"
+            ],
+            "index": "pypi",
+            "version": "==5.3.1"
+        }
+    },
+    "develop": {}
+}

tools/job_structure/job_structure.py

+#!/usr/bin/env python3
+
+import os
+import logging
+import sys
+import yaml
+from yaml import full_load, YAMLError
+
+# Import the Config module and set the path to run the script from root project
+# /!\ This instruction is only working if you run this script from the root of the project
+sys.path.insert(0, "./")
+from tools.utils.utils import Config
+
+utils = Config()
+
+# List of available labels for jobs
+labels_list = ["GitLab", "Build", "Container", "Docker", "PHP", "Testing", "Utilities",
+               "Yarn", "Dependency scan", "Security", "Python", "API", "Documentation",
+               "Quality", "SAST", "Linter", "Helm", "DAST", "Kubernetes",
+               "NPM"]
+# A list of paths where the file / directory is considered as not mandatory
+optional_paths=["r2_jobname/screenshots", "r2_jobname/screenshots/.gitkeep"]
+set_labels_list = set(labels_list)
+
+
+
+def check_directory_structure(template_structure, job):
+    """Verify every file and directories in template to be sure they are present in the job
+
+    Parameters
+    ----------
+    template_structure
+        Structure of the template
+    job
+        Name of the job
+
+    Returns
+    -------
+    0
+        On success
+    1
+        If at least a file/directory is missing
+    """
+
+    # Change "r2_jobname" in template for the actual job name for comparison
+    template_structure_tmp = set(template_structure).symmetric_difference(optional_paths)
+    template_structure_tmp = [obj.replace("r2_jobname", f"{job}") for obj in template_structure_tmp]
+
+    job_structure = [os.path.join(parent, name) for (parent, subdirs, files)
+                     in os.walk(f"{utils.JOBS_DIR}/{job}")
+                     for name in files + subdirs]
+    # Adding the job directory
+    job_structure.append(f"{job}")
+
+    # Clear the first directory
+    job_structure = [obj[obj.find('/') + 1:] for obj in job_structure]
+
+    # Check if file/directory is empty
+    logging.info("Checking empty file/directory in job structure of job %s", job)
+    for item in job_structure:
+        if os.path.isfile(item):
+            if os.path.getsize(item) == 0:
+                logging.error("File %s for job %s is empty", item, job)
+        elif os.path.isdir(item):
+            if len(os.listdir(item)) == 0:
+                logging.error("Directory %s for job %s is empty", item, job)
+
+    ret = utils.EXIT_SUCCESS
+    current_len = len(set(template_structure_tmp).intersection(job_structure))
+
+    if current_len != len(template_structure_tmp):
+        # Not every file and directories in template_structure_tmp matched the job structure
+        logging.error("Job structure of %s does not match the template:", job)
+        for item in set(template_structure_tmp) - set(template_structure_tmp).intersection(job_structure):
+            logging.error("\tFile/directory missing: %s", item)
+        ret = utils.EXIT_FAILURE
+    else:
+        logging.info("Job structure of %s matches the template", job)
+    return ret
+
+
+if __name__ == "__main__":
+    """Main function, iterating over job structures to compare to the template one
+
+    Returns
+    -------
+    0
+        In case nothing was mismatched
+    Number
+        Number of jobs that doesn't match the template
+    """
+    # Setup logging
+    logging.basicConfig(
+        encoding="utf-8",
+        level=logging.INFO,
+        format="%(asctime)s [%(levelname)s] %(message)s",
+        handlers=[
+            logging.FileHandler(utils.LOGFILE_NAME),
+            logging.StreamHandler()
+        ]
+    )
+
+    # Iterate over every directories in jobs directory to create their job.md for the documentation
+    jobs = os.listdir(utils.JOBS_DIR)
+    template_structure = [os.path.join(parent, name) for (parent, subdirs, files) in
+                          os.walk(f"{utils.TOOLS_DIR}/{utils.JOB_TEMPLATE_DIR}") for name in files + subdirs]
+    # Clear the first 2 directories
+    template_structure = [obj[obj.find('/') + 1:] for obj in template_structure]
+    template_structure = [obj[obj.find('/') + 1:] for obj in template_structure]
+
+    ret = utils.EXIT_SUCCESS
+    for job in jobs:
+        if check_directory_structure(template_structure, job) != utils.EXIT_SUCCESS:
+            ret = utils.EXIT_FAILURE
+    sys.exit(ret)