setup.cfg

@@ -16,6 +16,7 @@ norecursedirs = tests/integration/project integration-cache tmp __pycache__ .egg
 python_files = tests/*/*.py
 env =
     D:BST_TEST_SUITE=True
+    D:HOME=./tmp
     D:XDG_CACHE_HOME=./tmp/cache
     D:XDG_CONFIG_HOME=./tmp/config
     D:XDG_DATA_HOME=./tmp/share

src/buildstream/_cas/cascache.py

@@ -823,7 +823,9 @@ class CASCache():
             # already in local repository
             return objpath
 
-        remote._fetch_blob(digest)
+        batch = _CASBatchRead(remote)
+        batch.add(digest)
+        batch.send()
 
         return objpath
 

src/buildstream/_cas/casremote.py

@@ -247,25 +247,6 @@ class CASRemote():
 
         return self.cascache.add_object(buffer=message_buffer, instance_name=self.local_cas_instance_name)
 
-    ################################################
-    #             Local Private Methods            #
-    ################################################
-    def _fetch_blob(self, digest):
-        local_cas = self.cascache._get_local_cas()
-        request = local_cas_pb2.FetchMissingBlobsRequest()
-        request.instance_name = self.local_cas_instance_name
-        request_digest = request.blob_digests.add()
-        request_digest.CopyFrom(digest)
-        response = local_cas.FetchMissingBlobs(request)
-        for blob_response in response.responses:
-            if blob_response.status.code == code_pb2.NOT_FOUND:
-                raise BlobNotFound(response.digest.hash, "Failed to download blob {}: {}".format(
-                    blob_response.digest.hash, blob_response.status.code))
-
-            if blob_response.status.code != code_pb2.OK:
-                raise CASRemoteError("Failed to download blob {}: {}".format(
-                    blob_response.digest.hash, blob_response.status.code))
-
 
 # Represents a batch of blobs queued for fetching.
 #

src/buildstream/_frontend/cli.py

@@ -4,6 +4,7 @@ import sys
 from functools import partial
 import fcntl
 
+import shutil
 import click
 from .. import _yaml
 from .._exceptions import BstError, LoadError, AppError
@@ -1219,19 +1220,45 @@ def artifact_push(app, elements, deps, remote):
 #                     Artifact Log Command                     #
 ################################################################
 @artifact.command(name='log', short_help="Show logs of artifacts")
+@click.option('--out',
+              type=click.Path(file_okay=True, writable=True),
+              help="Output logs to individual files in the specified path. If absent, logs are written to stdout.")
 @click.argument('artifacts', type=click.Path(), nargs=-1)
 @click.pass_obj
-def artifact_log(app, artifacts):
+def artifact_log(app, artifacts, out):
     """Show build logs of artifacts"""
-
     with app.initialized():
-        log_file_paths = app.stream.artifact_log(artifacts)
-
-        for log in log_file_paths:
-            with open(log) as f:
-                data = f.read()
+        artifact_logs = app.stream.artifact_log(artifacts)
+
+        if not out:
+            try:
+                for log in list(artifact_logs.values()):
+                    with open(log[0], 'r') as f:
+                        data = f.read()
+                    click.echo_via_pager(data)
+            except (OSError, FileNotFoundError):
+                click.echo("Error: file cannot be opened", err=True)
+                sys.exit(1)
 
-            click.echo_via_pager(data)
+        else:
+            try:
+                os.mkdir(out)
+            except FileExistsError:
+                click.echo("Error: {} already exists".format(out), err=True)
+                sys.exit(1)
+
+            for name, log_files in artifact_logs.items():
+                if len(log_files) > 1:
+                    os.mkdir(name)
+                    for log in log_files:
+                        dest = os.path.join(out, name, log)
+                        shutil.copy(log, dest)
+                    # make a dir and write in log files
+                else:
+                    log_name = os.path.splitext(name)[0] + '.log'
+                    dest = os.path.join(out, log_name)
+                    shutil.copy(log_files[0], dest)
+                    # write a log file
 
 
 ################################################################

src/buildstream/_stream.py

@@ -648,7 +648,7 @@ class Stream():
         # Return list of Element and/or ArtifactElement objects
         target_objects = self.load_selection(targets, selection=PipelineSelection.NONE, load_refs=True)
 
-        log_file_paths = []
+        artifact_logs = {}
         for obj in target_objects:
             ref = obj.get_artifact_name()
             if not obj._cached():
@@ -658,9 +658,9 @@ class Stream():
                 self._message(MessageType.WARN, "{} is cached without log files".format(ref))
                 continue
 
-            log_file_paths.extend(obj.get_logs())
+            artifact_logs[obj.name] = obj.get_logs()
 
-        return log_file_paths
+        return artifact_logs
 
     # artifact_list_contents()
     #

tests/frontend/artifact.py

@@ -69,8 +69,7 @@ def test_artifact_log(cli, datafiles):
     # Read the log via glob
     result = cli.run(project=project, args=['artifact', 'log', 'test/target/*'])
     assert result.exit_code == 0
-    # The artifact is cached under both a strong key and a weak key
-    assert (log + log) == result.output
+    assert log == result.output
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -140,6 +139,38 @@ def test_artifact_list_exact_contents_glob(cli, datafiles):
         assert artifact in result.output
 
 
+@pytest.mark.datafiles(DATA_DIR)
+def test_artifact_log_files(cli, datafiles):
+    project = str(datafiles)
+
+    # Ensure we have an artifact to read
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    assert result.exit_code == 0
+
+    logfiles = os.path.join(project, "logfiles")
+    target = os.path.join(project, logfiles, "target.log")
+    import_bin = os.path.join(project, logfiles, "import-bin.log")
+    # Ensure the logfile doesn't exist before the command is run
+    assert not os.path.exists(logfiles)
+    assert not os.path.exists(target)
+    assert not os.path.exists(import_bin)
+
+    # Run the command and ensure the file now exists
+    result = cli.run(project=project, args=['artifact', 'log', '--out', logfiles, 'target.bst', 'import-bin.bst'])
+    assert result.exit_code == 0
+    assert os.path.exists(logfiles)
+    assert os.path.exists(target)
+    assert os.path.exists(import_bin)
+
+    # Ensure the file contains the logs by checking for the LOG line
+    with open(target, 'r') as f:
+        data = f.read()
+        assert "LOG     target.bst" in data
+    with open(import_bin, 'r') as f:
+        data = f.read()
+        assert "LOG     import-bin.bst" in data
+
+
 # Test that we can delete the artifact of the element which corresponds
 # to the current project state
 @pytest.mark.datafiles(DATA_DIR)

tests/testutils/repo/bzr.py

@@ -14,15 +14,23 @@ class Bzr(Repo):
         super().__init__(directory, subdir)
         self.bzr = BZR
 
+        self.env = os.environ.copy()
+        self.env.update(BZR_ENV)
+
     def create(self, directory):
+        # Work around race condition in bzr's creation of ~/.bazaar in
+        # ensure_config_dir_exists() when running tests in parallel.
+        bazaar_config_dir = os.path.expanduser('~/.bazaar')
+        os.makedirs(bazaar_config_dir, exist_ok=True)
+
         branch_dir = os.path.join(self.repo, 'trunk')
 
-        subprocess.call([self.bzr, 'init-repo', self.repo], env=BZR_ENV)
-        subprocess.call([self.bzr, 'init', branch_dir], env=BZR_ENV)
+        subprocess.call([self.bzr, 'init-repo', self.repo], env=self.env)
+        subprocess.call([self.bzr, 'init', branch_dir], env=self.env)
         self.copy_directory(directory, branch_dir)
-        subprocess.call([self.bzr, 'add', '.'], env=BZR_ENV, cwd=branch_dir)
+        subprocess.call([self.bzr, 'add', '.'], env=self.env, cwd=branch_dir)
         subprocess.call([self.bzr, 'commit', '--message="Initial commit"'],
-                        env=BZR_ENV, cwd=branch_dir)
+                        env=self.env, cwd=branch_dir)
 
         return self.latest_commit()
 
@@ -42,4 +50,4 @@ class Bzr(Repo):
             self.bzr, 'version-info',
             '--custom', '--template={revno}',
             os.path.join(self.repo, 'trunk')
-        ], env=BZR_ENV, universal_newlines=True).strip()
+        ], env=self.env, universal_newlines=True).strip()

tests/testutils/repo/git.py

@@ -18,11 +18,14 @@ class Git(Repo):
 
         super().__init__(directory, subdir)
 
+        self.env = os.environ.copy()
+        self.env.update(GIT_ENV)
+
     def _run_git(self, *args, **kwargs):
         argv = [GIT]
         argv.extend(args)
         if 'env' not in kwargs:
-            kwargs['env'] = dict(GIT_ENV, PWD=self.repo)
+            kwargs['env'] = dict(self.env, PWD=self.repo)
         kwargs.setdefault('cwd', self.repo)
         kwargs.setdefault('check', True)
         return subprocess.run(argv, **kwargs)

tox.ini

@@ -55,6 +55,7 @@ passenv =
 #
 setenv =
     py{35,36,37}: COVERAGE_FILE = {envtmpdir}/.coverage
+    py{35,36,37}: HOME = {envtmpdir}
     py{35,36,37}: XDG_CACHE_HOME = {envtmpdir}/cache
     py{35,36,37}: XDG_CONFIG_HOME = {envtmpdir}/config
     py{35,36,37}: XDG_DATA_HOME = {envtmpdir}/share