Commit 9b140fa0 authored by James Ennis's avatar James Ennis
Browse files

element.py: Lift ArtifactCache.get_artifact_fullname() to here 

This commit removes the method ArtifactCache.get_artifact_fullname()
and replaces it with Element.get_artifact_name()

Given a key, we are now able to construct the full name of any of an
element's artifacts.
parent 6de65306

buildstream/_artifactcache.py

+13 −47
Original line number Diff line number Diff line
@@ -19,7 +19,6 @@ 


import multiprocessing

import os

import string

from collections.abc import Mapping



from .types import _KeyStrength
@@ -112,37 +111,6 @@ class ArtifactCache(): 


        self._calculate_cache_quota()



    # get_artifact_fullname()

    #

    # Generate a full name for an artifact, including the

    # project namespace, element name and cache key.

    #

    # This can also be used as a relative path safely, and

    # will normalize parts of the element name such that only

    # digits, letters and some select characters are allowed.

    #

    # Args:

    #    element (Element): The Element object

    #    key (str): The element's cache key

    #

    # Returns:

    #    (str): The relative path for the artifact

    #

    def get_artifact_fullname(self, element, key):

        project = element._get_project()



        # Normalize ostree ref unsupported chars

        valid_chars = string.digits + string.ascii_letters + '-._'

        element_name = ''.join([

            x if x in valid_chars else '_'

            for x in element.normal_name

        ])



        assert key is not None



        # assume project and element names are not allowed to contain slashes

        return '{0}/{1}/{2}'.format(project.name, element_name, key)



    # setup_remotes():

    #

    # Sets up which remotes to use
@@ -241,7 +209,7 @@ class ArtifactCache(): 
            for key in (strong_key, weak_key):

                if key:

                    try:

                        ref = self.get_artifact_fullname(element, key)

                        ref = element.get_artifact_name(key)



                        self.cas.update_mtime(ref)

                    except CASError:
@@ -521,7 +489,7 @@ class ArtifactCache(): 
    # Returns: True if the artifact is in the cache, False otherwise

    #

    def contains(self, element, key):

        ref = self.get_artifact_fullname(element, key)

        ref = element.get_artifact_name(key)



        return self.cas.contains(ref)
@@ -538,7 +506,7 @@ class ArtifactCache(): 
    # Returns: True if the subdir exists & is populated in the cache, False otherwise

    #

    def contains_subdir_artifact(self, element, key, subdir):

        ref = self.get_artifact_fullname(element, key)

        ref = element.get_artifact_name(key)

        return self.cas.contains_subdir_artifact(ref, subdir)



    # list_artifacts():
@@ -546,8 +514,7 @@ class ArtifactCache(): 
    # List artifacts in this cache in LRU order.

    #

    # Returns:

    #     ([str]) - A list of artifact names as generated by

    #               `ArtifactCache.get_artifact_fullname` in LRU order

    #     ([str]) - A list of artifact names as generated in LRU order

    #

    def list_artifacts(self):

        return self.cas.list_refs()
@@ -559,8 +526,7 @@ class ArtifactCache(): 
    #

    # Args:

    #     ref (artifact_name): The name of the artifact to remove (as

    #                          generated by

    #                          `ArtifactCache.get_artifact_fullname`)

    #                          generated by `Element.get_artifact_name`)

    #

    # Returns:

    #    (int): The amount of space recovered in the cache, in bytes
@@ -606,7 +572,7 @@ class ArtifactCache(): 
    # Returns: path to extracted artifact

    #

    def extract(self, element, key, subdir=None):

        ref = self.get_artifact_fullname(element, key)

        ref = element.get_artifact_name(key)



        path = os.path.join(self.extractdir, element._get_project().name, element.normal_name)
@@ -622,7 +588,7 @@ class ArtifactCache(): 
    #     keys (list): The cache keys to use

    #

    def commit(self, element, content, keys):

        refs = [self.get_artifact_fullname(element, key) for key in keys]

        refs = [element.get_artifact_name(key) for key in keys]



        self.cas.commit(refs, content)
@@ -638,8 +604,8 @@ class ArtifactCache(): 
    #     subdir (str): A subdirectory to limit the comparison to

    #

    def diff(self, element, key_a, key_b, *, subdir=None):

        ref_a = self.get_artifact_fullname(element, key_a)

        ref_b = self.get_artifact_fullname(element, key_b)

        ref_a = element.get_artifact_name(key_a)

        ref_b = element.get_artifact_name(key_b)



        return self.cas.diff(ref_a, ref_b, subdir=subdir)
@@ -700,7 +666,7 @@ class ArtifactCache(): 
    #   (ArtifactError): if there was an error

    #

    def push(self, element, keys):

        refs = [self.get_artifact_fullname(element, key) for key in list(keys)]

        refs = [element.get_artifact_name(key) for key in list(keys)]



        project = element._get_project()
@@ -738,7 +704,7 @@ class ArtifactCache(): 
    #   (bool): True if pull was successful, False if artifact was not available

    #

    def pull(self, element, key, *, progress=None, subdir=None, excluded_subdirs=None):

        ref = self.get_artifact_fullname(element, key)

        ref = element.get_artifact_name(key)



        project = element._get_project()
@@ -850,8 +816,8 @@ class ArtifactCache(): 
    #     newkey (str): A new cache key for the artifact

    #

    def link_key(self, element, oldkey, newkey):

        oldref = self.get_artifact_fullname(element, oldkey)

        newref = self.get_artifact_fullname(element, newkey)

        oldref = element.get_artifact_name(oldkey)

        newref = element.get_artifact_name(newkey)



        self.cas.link_ref(oldref, newref)

buildstream/element.py

+33 −0
Original line number Diff line number Diff line
@@ -82,6 +82,7 @@ import contextlib 
from contextlib import contextmanager

import tempfile

import shutil

import string



from . import _yaml

from ._variables import Variables
@@ -577,6 +578,38 @@ class Element(Plugin): 
        self.__assert_cached()

        return self.__compute_splits(include, exclude, orphans)



    def get_artifact_name(self, key=None):

        """Compute and return this element's full artifact name



        Generate a full name for an artifact, including the project

        namespace, element name and cache key.



        This can also be used as a relative path safely, and

        will normalize parts of the element name such that only

        digits, letters and some select characters are allowed.



        Args:

           key (str): The element's cache key. Defaults to None



        Returns:

           (str): The relative path for the artifact

        """

        project = self._get_project()

        if key is None:

            key = self._get_cache_key()



        assert key is not None



        valid_chars = string.digits + string.ascii_letters + '-._'

        element_name = ''.join([

            x if x in valid_chars else '_'

            for x in self.normal_name

        ])



        # Note that project names are not allowed to contain slashes. Element names containing

        # a '/' will have this replaced with a '-' upon Element object instantiation.

        return '{0}/{1}/{2}'.format(project.name, element_name, key)



    def stage_artifact(self, sandbox, *, path=None, include=None, exclude=None, orphans=True, update_mtimes=None):

        """Stage this element's output artifact in the sandbox

tests/artifactcache/pull.py

+1 −1
Original line number Diff line number Diff line
@@ -211,7 +211,7 @@ def test_pull_tree(cli, tmpdir, datafiles): 
        assert artifactcache.contains(element, element_key)



        # Retrieve the Directory object from the cached artifact

        artifact_ref = artifactcache.get_artifact_fullname(element, element_key)

        artifact_ref = element.get_artifact_name(element_key)

        artifact_digest = cas.resolve_ref(artifact_ref)



        queue = multiprocessing.Queue()

tests/artifactcache/push.py

+1 −1
Original line number Diff line number Diff line
@@ -190,7 +190,7 @@ def test_push_directory(cli, tmpdir, datafiles): 
        assert artifactcache.has_push_remotes(element=element)



        # Recreate the CasBasedDirectory object from the cached artifact

        artifact_ref = artifactcache.get_artifact_fullname(element, element_key)

        artifact_ref = element.get_artifact_name(element_key)

        artifact_digest = cas.resolve_ref(artifact_ref)



        queue = multiprocessing.Queue()