buildstream/_artifactcache/cascache.py

@@ -25,6 +25,7 @@ import signal
 import stat
 import tempfile
 import uuid
+import errno
 from urllib.parse import urlparse
 
 import grpc
@@ -82,7 +83,8 @@ class CASCache(ArtifactCache):
 
         tree = self.resolve_ref(ref, update_mtime=True)
 
-        dest = os.path.join(self.extractdir, element._get_project().name, element.normal_name, tree.hash)
+        dest = os.path.join(self.extractdir, element._get_project().name,
+                            element.normal_name, tree.hash)
         if os.path.isdir(dest):
             # artifact has already been extracted
             return dest
@@ -100,7 +102,7 @@ class CASCache(ArtifactCache):
                 #
                 # If rename fails with these errors, another process beat
                 # us to it so just ignore.
-                if e.errno not in [os.errno.ENOTEMPTY, os.errno.EEXIST]:
+                if e.errno not in [errno.ENOTEMPTY, errno.EEXIST]:
                     raise ArtifactError("Failed to extract artifact for ref '{}': {}"
                                         .format(ref, e)) from e
 

doc/examples/autotools/project.conf

@@ -10,4 +10,4 @@ element-path: elements
 # Define some aliases for the tarballs we download
 aliases:
   alpine: https://gnome7.codethink.co.uk/tarballs/
-  gnu: https://ftpmirror.gnu.org/gnu/automake/
+  gnu: http://ftpmirror.gnu.org/gnu/automake/

tests/artifactcache/expiry.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Maat <tristan.maat@codethink.co.uk>
+#
+
 import os
 
 import pytest
@@ -5,7 +24,7 @@ import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 
-from tests.testutils import cli, create_element_size
+from tests.testutils import cli, create_element_size, wait_for_cache_granularity
 
 
 DATA_DIR = os.path.join(
@@ -108,6 +127,8 @@ def test_expiry_order(cli, datafiles, tmpdir):
     res = cli.run(project=project, args=['build', 'target2.bst'])
     res.assert_success()
 
+    wait_for_cache_granularity()
+
     # Now extract dep.bst
     res = cli.run(project=project, args=['checkout', 'dep.bst', checkout])
     res.assert_success()

tests/frontend/push.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#           Sam Thursfield <sam.thursfield@codethink.co.uk>
+#           Jürg Billeter <juerg.billeter@codethink.co.uk>
+#
+
 import os
 import pytest
 
 from buildstream._exceptions import ErrorDomain
 from tests.testutils import cli, create_artifact_share, create_element_size
-from tests.testutils import generate_junction
+from tests.testutils import generate_junction, wait_for_cache_granularity
 from . import configure_project
 
 
@@ -327,6 +349,8 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
         # Ensure element1 is cached locally
         assert cli.get_element_state(project, 'element1.bst') == 'cached'
 
+        wait_for_cache_granularity()
+
         # Create and build the element3 (of 5 MB)
         create_element_size('element3.bst', project, element_path, [], int(5e6))
         result = cli.run(project=project, args=['build', 'element3.bst'])

tests/frontend/workspace.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#           Tristan Maat <tristan.maat@codethink.co.uk>
+#           Chandan Singh <csingh43@bloomberg.net>
+#           Phillip Smyth <phillip.smyth@codethink.co.uk>
+#           Jonathan Maw <jonathan.maw@codethink.co.uk>
+#           Richard Maw <richard.maw@codethink.co.uk>
+#
+
 import os
 import pytest
 import shutil
 import subprocess
 from ruamel.yaml.comments import CommentedSet
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
@@ -507,6 +532,8 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
     assert cli.get_element_state(project, element_name) == 'cached'
     assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
 
+    wait_for_cache_granularity()
+
     # Modify the workspace in various different ways, ensuring we
     # properly detect the changes.
     #

tests/testutils/__init__.py

+#
+#  Copyright (C) 2018 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors: Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#           Tristan Maat <tristan.maat@codethink.co.uk>
+#           Sam Thursfield <sam.thursfield@codethink.co.uk>
+#           James Ennis <james.ennis@codethink.co.uk>
+#           Valentin David <valentin.david@codethink.co.uk>
+#           William Salmon <will.salmon@codethink.co.uk>
+#
+
 from .runcli import cli, cli_integration
 from .repo import create_repo, ALL_REPO_KINDS
 from .artifactshare import create_artifact_share
 from .element_generators import create_element_size
 from .junction import generate_junction
+from .runner_integration import wait_for_cache_granularity

tests/testutils/runner_integration.py

+#
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#         Will Salmon <will.salmon@codethink.co.uk>
+
+import time
+
+
+def wait_for_cache_granularity():
+    # This isn't called very often so has minimal impact on test runtime.
+    # If this changes it may be worth while adding a more sophisticated approach.
+    """
+    Mitigate the coarse granularity of the gitlab runners mtime
+
+    This function waits for the mtime to increment so that the cache can sort by mtime and
+    get the most recent results.
+    """
+    time.sleep(1.1)