update.py 12.2 KB
Newer Older
1
#!/usr/bin/env python
2
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 4 5
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

6 7
"""This script is used to download prebuilt clang binaries. It runs as a
"gclient hook" in Chromium checkouts.
8

9 10
It can also be run stand-alone as a convenient way of installing a well-tested
near-tip-of-tree clang version:
11

12 13 14
  $ curl -s https://raw.githubusercontent.com/chromium/chromium/master/tools/clang/scripts/update.py | python - --output-dir=/tmp/clang

(Note that the output dir may be deleted and re-created if it exists.)
15 16 17
"""

# TODO: Running stand-alone won't work on Windows due to the dia dll copying.
18

19
from __future__ import division
20
from __future__ import print_function
21
import argparse
22
import os
23
import shutil
24
import stat
25
import sys
26
import tarfile
27
import tempfile
28
import time
29 30

try:
31
  from urllib2 import HTTPError, URLError, urlopen
32
except ImportError: # For Py3 compatibility
33 34
  from urllib.error import HTTPError, URLError
  from urllib.request import urlopen
35

36
import zipfile
37

38

39
# Do NOT CHANGE this if you don't know what you're doing -- see
40
# https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md
41
# Reverting problematic clang rolls is safe, though.
42 43
CLANG_REVISION = '9284abd0040afecfd619dbcf1b244a8b533291c9'
CLANG_SVN_REVISION = 'n344329'
44
CLANG_SUB_REVISION = 2
45

46 47
PACKAGE_VERSION = '%s-%s-%s' % (CLANG_SVN_REVISION, CLANG_REVISION[:8],
                                CLANG_SUB_REVISION)
48
RELEASE_VERSION = '11.0.0'
49

50

51 52
CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
    'https://commondatastorage.googleapis.com/chromium-browser-clang')
53

54 55 56 57 58
# Path constants. (All of these should be absolute paths.)
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
                              'Release+Asserts')
59

60
STAMP_FILE = os.path.normpath(
61
    os.path.join(LLVM_BUILD_DIR, 'cr_build_revision'))
62 63
OLD_STAMP_FILE = os.path.normpath(
    os.path.join(LLVM_BUILD_DIR, '..', 'cr_build_revision'))
64 65
FORCE_HEAD_REVISION_FILE = os.path.normpath(os.path.join(LLVM_BUILD_DIR, '..',
                                                   'force_head_revision'))
66

67

68 69 70 71 72 73 74 75 76 77
def RmTree(dir):
  """Delete dir."""
  def ChmodAndRetry(func, path, _):
    # Subversion can leave read-only files around.
    if not os.access(path, os.W_OK):
      os.chmod(path, stat.S_IWUSR)
      return func(path)
    raise
  shutil.rmtree(dir, onerror=ChmodAndRetry)

78

79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
def ReadStampFile(path):
  """Return the contents of the stamp file, or '' if it doesn't exist."""
  try:
    with open(path, 'r') as f:
      return f.read().rstrip()
  except IOError:
    return ''


def WriteStampFile(s, path):
  """Write s to the stamp file."""
  EnsureDirExists(os.path.dirname(path))
  with open(path, 'w') as f:
    f.write(s)
    f.write('\n')
94

95

96 97 98 99
def DownloadUrl(url, output_file):
  """Download url into output_file."""
  CHUNK_SIZE = 4096
  TOTAL_DOTS = 10
100 101 102
  num_retries = 3
  retry_wait_s = 5  # Doubled at each retry.

103
  while True:
104 105 106
    try:
      sys.stdout.write('Downloading %s ' % url)
      sys.stdout.flush()
107 108
      response = urlopen(url)
      total_size = int(response.info().get('Content-Length').strip())
109 110 111 112 113 114 115 116
      bytes_done = 0
      dots_printed = 0
      while True:
        chunk = response.read(CHUNK_SIZE)
        if not chunk:
          break
        output_file.write(chunk)
        bytes_done += len(chunk)
117
        num_dots = TOTAL_DOTS * bytes_done // total_size
118 119 120
        sys.stdout.write('.' * (num_dots - dots_printed))
        sys.stdout.flush()
        dots_printed = num_dots
121
      if bytes_done != total_size:
122 123
        raise URLError("only got %d of %d bytes" %
                       (bytes_done, total_size))
124
      print(' Done.')
125
      return
126
    except URLError as e:
127
      sys.stdout.write('\n')
128
      print(e)
129
      if num_retries == 0 or isinstance(e, HTTPError) and e.code == 404:
130 131
        raise e
      num_retries -= 1
132
      print('Retrying in %d s ...' % retry_wait_s)
133
      sys.stdout.flush()
134 135
      time.sleep(retry_wait_s)
      retry_wait_s *= 2
136 137


Nico's avatar
Nico committed
138 139 140 141 142
def EnsureDirExists(path):
  if not os.path.exists(path):
    os.makedirs(path)


143 144 145 146
def DownloadAndUnpack(url, output_dir, path_prefixes=None):
  """Download an archive from url and extract into output_dir. If path_prefixes
     is not None, only extract files whose paths within the archive start with
     any prefix in path_prefixes."""
147 148 149
  with tempfile.TemporaryFile() as f:
    DownloadUrl(url, f)
    f.seek(0)
Nico's avatar
Nico committed
150
    EnsureDirExists(output_dir)
151
    if url.endswith('.zip'):
152
      assert path_prefixes is None
153 154
      zipfile.ZipFile(f).extractall(path=output_dir)
    else:
155 156
      t = tarfile.open(mode='r:gz', fileobj=f)
      members = None
157 158 159
      if path_prefixes is not None:
        members = [m for m in t.getmembers()
                   if any(m.name.startswith(p) for p in path_prefixes)]
160
      t.extractall(path=output_dir, members=members)
161 162


163 164 165 166 167 168 169
def GetPlatformUrlPrefix(platform):
  if platform == 'win32' or platform == 'cygwin':
    return CDS_URL + '/Win/'
  if platform == 'darwin':
    return CDS_URL + '/Mac/'
  assert platform.startswith('linux')
  return CDS_URL + '/Linux_x64/'
170

171

172 173 174 175 176 177 178 179 180 181 182 183 184 185
def DownloadAndUnpackPackage(package_file, output_dir):
  cds_file = "%s-%s.tgz" % (package_file, PACKAGE_VERSION)
  cds_full_url = GetPlatformUrlPrefix(sys.platform) + cds_file
  try:
    DownloadAndUnpack(cds_full_url, output_dir)
  except URLError:
    print('Failed to download prebuilt clang package %s' % cds_file)
    print('Use build.py if you want to build locally.')
    print('Exiting.')
    sys.exit(1)


# TODO(hans): Create a clang-win-runtime package instead.
def DownloadAndUnpackClangWinRuntime(output_dir):
186
  cds_file = "clang-%s.tgz" %  PACKAGE_VERSION
187 188 189
  cds_full_url = GetPlatformUrlPrefix('win32') + cds_file
  path_prefixes =  [ 'lib/clang/' + RELEASE_VERSION + '/lib/',
                     'bin/llvm-symbolizer.exe' ]
190
  try:
191
    DownloadAndUnpack(cds_full_url, output_dir, path_prefixes)
192
  except URLError:
193
    print('Failed to download prebuilt clang %s' % cds_file)
194
    print('Use build.py if you want to build locally.')
195
    print('Exiting.')
196
    sys.exit(1)
197 198


199
win_sdk_dir = None
200
dia_dll = None
201
def GetWinSDKDir():
202
  """Get the location of the current SDK. Sets dia_dll as a side-effect."""
203
  global win_sdk_dir
204
  global dia_dll
205 206 207
  if win_sdk_dir:
    return win_sdk_dir

208 209 210 211 212
  # Bump after VC updates.
  DIA_DLL = {
    '2013': 'msdia120.dll',
    '2015': 'msdia140.dll',
    '2017': 'msdia140.dll',
213
    '2019': 'msdia140.dll',
214 215
  }

216 217
  # Don't let vs_toolchain overwrite our environment.
  environ_bak = os.environ
218 219 220

  sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
  import vs_toolchain
221 222
  win_sdk_dir = vs_toolchain.SetEnvironmentAndGetSDKDir()
  msvs_version = vs_toolchain.GetVisualStudioVersion()
223

224 225 226
  if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))):
    dia_path = os.path.join(win_sdk_dir, '..', 'DIA SDK', 'bin', 'amd64')
  else:
227 228 229 230
    if 'GYP_MSVS_OVERRIDE_PATH' not in os.environ:
      vs_path = vs_toolchain.DetectVisualStudioPath()
    else:
      vs_path = os.environ['GYP_MSVS_OVERRIDE_PATH']
231 232 233 234
    dia_path = os.path.join(vs_path, 'DIA SDK', 'bin', 'amd64')

  dia_dll = os.path.join(dia_path, DIA_DLL[msvs_version])

235 236
  os.environ = environ_bak
  return win_sdk_dir
237 238


239 240 241 242 243 244
def CopyFile(src, dst):
  """Copy a file from src to dst."""
  print("Copying %s to %s" % (src, dst))
  shutil.copy(src, dst)


245 246
def CopyDiaDllTo(target_dir):
  # This script always wants to use the 64-bit msdia*.dll.
247
  GetWinSDKDir()
248 249 250
  CopyFile(dia_dll, target_dir)


251 252 253 254 255 256 257 258
def UpdatePackage(package_name):
  stamp_file = None
  package_file = None

  stamp_file = os.path.join(LLVM_BUILD_DIR, package_name + '_revision')
  if package_name == 'clang':
    stamp_file = STAMP_FILE
    package_file = 'clang'
259 260
  elif package_name == 'clang-tidy':
    package_file = 'clang-tidy'
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
  elif package_name == 'lld_mac':
    package_file = 'lld'
    if sys.platform != 'darwin':
      print('The lld_mac package cannot be downloaded on non-macs.')
      print('On non-mac, lld is included in the clang package.')
      return 1
  elif package_name == 'objdump':
    package_file = 'llvmobjdump'
  elif package_name == 'translation_unit':
    package_file = 'translation_unit'
  elif package_name == 'coverage_tools':
    stamp_file = os.path.join(LLVM_BUILD_DIR, 'cr_coverage_revision')
    package_file = 'llvm-code-coverage'
  elif package_name == 'libclang':
    package_file = 'libclang'
  else:
    print('Unknown package: "%s".' % package_name)
    return 1
279

280 281 282 283
  assert stamp_file is not None
  assert package_file is not None

  # TODO(hans): Create a clang-win-runtime package and use separate DEPS hook.
284
  target_os = []
285 286 287 288 289 290 291 292
  if package_name == 'clang':
    try:
      GCLIENT_CONFIG = os.path.join(os.path.dirname(CHROMIUM_DIR), '.gclient')
      env = {}
      execfile(GCLIENT_CONFIG, env, env)
      target_os = env.get('target_os', target_os)
    except:
      pass
293

294 295 296 297 298 299
  if os.path.exists(OLD_STAMP_FILE):
    # Delete the old stamp file so it doesn't look like an old version of clang
    # is available in case the user rolls back to an old version of this script
    # during a bisect for example (crbug.com/988933).
    os.remove(OLD_STAMP_FILE)

300
  expected_stamp = ','.join([PACKAGE_VERSION] + target_os)
301
  if ReadStampFile(stamp_file) == expected_stamp:
302
    return 0
303

304 305 306
  # Updating the main clang package nukes the output dir. Any other packages
  # need to be updated *after* the clang package.
  if package_name == 'clang' and os.path.exists(LLVM_BUILD_DIR):
307
    RmTree(LLVM_BUILD_DIR)
308

309 310 311 312 313 314 315 316 317
  DownloadAndUnpackPackage(package_file, LLVM_BUILD_DIR)

  if package_name == 'clang':
    if sys.platform == 'win32':
      CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
    if 'win' in target_os:
      # When doing win/cross builds on other hosts, get the Windows runtime
      # libraries, and llvm-symbolizer.exe (needed in asan builds).
      DownloadAndUnpackClangWinRuntime(LLVM_BUILD_DIR)
318

319
  WriteStampFile(expected_stamp, stamp_file)
320 321
  return 0

322 323

def main():
324
  parser = argparse.ArgumentParser(description='Update clang.')
325 326 327 328 329
  parser.add_argument('--output-dir',
                      help='Where to extract the package.')
  parser.add_argument('--package',
                      help='What package to update (default: clang)',
                      default='clang')
330
  parser.add_argument('--force-local-build', action='store_true',
331
                      help='(no longer used)')
332
  parser.add_argument('--print-revision', action='store_true',
333 334 335
                      help='Print current clang revision and exit.')
  parser.add_argument('--llvm-force-head-revision', action='store_true',
                      help='Print locally built revision with --print-revision')
336
  parser.add_argument('--print-clang-version', action='store_true',
337 338
                      help=('Print current clang release version (e.g. 9.0.0) '
                            'and exit.'))
339
  parser.add_argument('--verify-version',
340
                      help='Verify that clang has the passed-in version.')
341 342
  args = parser.parse_args()

343 344 345
  if args.force_local_build:
    print(('update.py --force-local-build is no longer used to build clang; '
           'use build.py instead.'))
krasin's avatar
krasin committed
346 347
    return 1

348 349 350
  if args.verify_version and args.verify_version != RELEASE_VERSION:
    print('RELEASE_VERSION is %s but --verify-version argument was %s.' % (
        RELEASE_VERSION, args.verify_version))
351
    print('clang_version in build/toolchain/toolchain.gni is likely outdated.')
352 353
    return 1

354 355 356
  if args.print_clang_version:
    print(RELEASE_VERSION)
    return 0
357

358
  if args.print_revision:
359
    if args.llvm_force_head_revision:
360 361 362 363 364 365 366 367
      force_head_revision = ReadStampFile(FORCE_HEAD_REVISION_FILE)
      if force_head_revision == '':
        print('No locally built version found!')
        return 1
      print(force_head_revision)
      return 0

    print(PACKAGE_VERSION)
368 369
    return 0

370
  if args.llvm_force_head_revision:
371 372
    print('--llvm-force-head-revision can only be used for --print-revision')
    return 1
373

374
  if args.output_dir:
375
    global LLVM_BUILD_DIR, STAMP_FILE
376
    LLVM_BUILD_DIR = os.path.abspath(args.output_dir)
377
    STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision')
378

379
  return UpdatePackage(args.package)
380 381 382 383


if __name__ == '__main__':
  sys.exit(main())