git-p4.py 142 KB
Newer Older
1 2 3 4
#!/usr/bin/env python
#
# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
#
5 6
# Author: Simon Hausmann <simon@lst.de>
# Copyright: 2007 Simon Hausmann <simon@lst.de>
7
#            2007 Trolltech ASA
8 9
# License: MIT <http://www.opensource.org/licenses/mit-license.php>
#
10 11 12 13 14
import sys
if sys.hexversion < 0x02040000:
    # The limiter is the subprocess module
    sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
    sys.exit(1)
15 16 17 18 19 20 21 22 23
import os
import optparse
import marshal
import subprocess
import tempfile
import time
import platform
import re
import shutil
24
import stat
25 26
import zipfile
import zlib
27
import ctypes
28
import errno
29

30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
try:
    from subprocess import CalledProcessError
except ImportError:
    # from python2.7:subprocess.py
    # Exception classes used by this module.
    class CalledProcessError(Exception):
        """This exception is raised when a process run by check_call() returns
        a non-zero exit status.  The exit status will be stored in the
        returncode attribute."""
        def __init__(self, returncode, cmd):
            self.returncode = returncode
            self.cmd = cmd
        def __str__(self):
            return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)

45
verbose = False
46

47
# Only labels/tags matching this will be imported/exported
48
defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
49

50 51 52
# Grab changes in blocks of this many revisions, unless otherwise requested
defaultBlockSize = 512

53 54 55 56 57 58 59
def p4_build_cmd(cmd):
    """Build a suitable p4 command line.

    This consolidates building and returning a p4 command line into one
    location. It means that hooking into the environment, or other configuration
    can be done more easily.
    """
60
    real_cmd = ["p4"]
61 62 63

    user = gitConfig("git-p4.user")
    if len(user) > 0:
64
        real_cmd += ["-u",user]
65 66 67

    password = gitConfig("git-p4.password")
    if len(password) > 0:
68
        real_cmd += ["-P", password]
69 70 71

    port = gitConfig("git-p4.port")
    if len(port) > 0:
72
        real_cmd += ["-p", port]
73 74 75

    host = gitConfig("git-p4.host")
    if len(host) > 0:
76
        real_cmd += ["-H", host]
77 78 79

    client = gitConfig("git-p4.client")
    if len(client) > 0:
80
        real_cmd += ["-c", client]
81

82 83 84 85
    retries = gitConfigInt("git-p4.retries")
    if retries is None:
        # Perform 3 retries by default
        retries = 3
86 87 88
    if retries > 0:
        # Provide a way to not pass this option by setting git-p4.retries to 0
        real_cmd += ["-r", str(retries)]
89 90 91 92 93

    if isinstance(cmd,basestring):
        real_cmd = ' '.join(real_cmd) + ' ' + cmd
    else:
        real_cmd += cmd
94 95
    return real_cmd

96 97 98 99 100 101 102 103 104 105
def git_dir(path):
    """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
        This won't automatically add ".git" to a directory.
    """
    d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
    if not d or len(d) == 0:
        return None
    else:
        return d

106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126
def chdir(path, is_client_path=False):
    """Do chdir to the given path, and set the PWD environment
       variable for use by P4.  It does not look at getcwd() output.
       Since we're not using the shell, it is necessary to set the
       PWD environment variable explicitly.

       Normally, expand the path to force it to be absolute.  This
       addresses the use of relative path names inside P4 settings,
       e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
       as given; it looks for .p4config using PWD.

       If is_client_path, the path was handed to us directly by p4,
       and may be a symbolic link.  Do not call os.getcwd() in this
       case, because it will cause p4 to think that PWD is not inside
       the client path.
       """

    os.chdir(path)
    if not is_client_path:
        path = os.getcwd()
    os.environ['PWD'] = path
127

128 129 130 131 132 133 134 135 136 137
def calcDiskFree():
    """Return free space in bytes on the disk of the given dirname."""
    if platform.system() == 'Windows':
        free_bytes = ctypes.c_ulonglong(0)
        ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
        return free_bytes.value
    else:
        st = os.statvfs(os.getcwd())
        return st.f_bavail * st.f_frsize

138 139 140 141 142 143 144
def die(msg):
    if verbose:
        raise Exception(msg)
    else:
        sys.stderr.write(msg + "\n")
        sys.exit(1)

145
def write_pipe(c, stdin):
146
    if verbose:
147
        sys.stderr.write('Writing pipe: %s\n' % str(c))
148

149 150 151 152 153 154 155
    expand = isinstance(c,basestring)
    p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
    pipe = p.stdin
    val = pipe.write(stdin)
    pipe.close()
    if p.wait():
        die('Command failed: %s' % str(c))
156 157 158

    return val

159
def p4_write_pipe(c, stdin):
160
    real_cmd = p4_build_cmd(c)
161
    return write_pipe(real_cmd, stdin)
162

163 164 165 166 167
def read_pipe_full(c):
    """ Read output from  command. Returns a tuple
        of the return status, stdout text and stderr
        text.
    """
168
    if verbose:
169
        sys.stderr.write('Reading pipe: %s\n' % str(c))
170

171
    expand = isinstance(c,basestring)
172 173
    p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
    (out, err) = p.communicate()
174 175 176 177 178 179 180 181 182 183 184 185 186
    return (p.returncode, out, err)

def read_pipe(c, ignore_error=False):
    """ Read output from  command. Returns the output text on
        success. On failure, terminates execution, unless
        ignore_error is True, when it returns an empty string.
    """
    (retcode, out, err) = read_pipe_full(c)
    if retcode != 0:
        if ignore_error:
            out = ""
        else:
            die('Command failed: %s\nError: %s' % (str(c), err))
187
    return out
188

189 190 191 192 193 194 195 196 197 198
def read_pipe_text(c):
    """ Read output from a command with trailing whitespace stripped.
        On error, returns None.
    """
    (retcode, out, err) = read_pipe_full(c)
    if retcode != 0:
        return None
    else:
        return out.rstrip()

199 200 201
def p4_read_pipe(c, ignore_error=False):
    real_cmd = p4_build_cmd(c)
    return read_pipe(real_cmd, ignore_error)
202

Han-Wen Nienhuys's avatar
Han-Wen Nienhuys committed
203
def read_pipe_lines(c):
204
    if verbose:
205 206 207 208 209
        sys.stderr.write('Reading pipe: %s\n' % str(c))

    expand = isinstance(c, basestring)
    p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
    pipe = p.stdout
210
    val = pipe.readlines()
211 212
    if pipe.close() or p.wait():
        die('Command failed: %s' % str(c))
213 214

    return val
215

216 217
def p4_read_pipe_lines(c):
    """Specifically invoke p4 on the command supplied. """
218
    real_cmd = p4_build_cmd(c)
219 220
    return read_pipe_lines(real_cmd)

221 222 223 224 225 226 227 228 229
def p4_has_command(cmd):
    """Ask p4 for help on this command.  If it returns an error, the
       command does not exist in this version of p4."""
    real_cmd = p4_build_cmd(["help", cmd])
    p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
    p.communicate()
    return p.returncode == 0

230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
def p4_has_move_command():
    """See if the move command exists, that it supports -k, and that
       it has not been administratively disabled.  The arguments
       must be correct, but the filenames do not have to exist.  Use
       ones with wildcards so even if they exist, it will fail."""

    if not p4_has_command("move"):
        return False
    cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    (out, err) = p.communicate()
    # return code will be 1 in either case
    if err.find("Invalid option") >= 0:
        return False
    if err.find("disabled") >= 0:
        return False
    # assume it failed because @... was invalid changelist
    return True

249
def system(cmd, ignore_error=False):
250
    expand = isinstance(cmd,basestring)
251
    if verbose:
252
        sys.stderr.write("executing %s\n" % str(cmd))
253
    retcode = subprocess.call(cmd, shell=expand)
254
    if retcode and not ignore_error:
255
        raise CalledProcessError(retcode, cmd)
Han-Wen Nienhuys's avatar
Han-Wen Nienhuys committed
256

257 258
    return retcode

259 260
def p4_system(cmd):
    """Specifically invoke p4 as the system command. """
261
    real_cmd = p4_build_cmd(cmd)
262
    expand = isinstance(real_cmd, basestring)
263 264 265
    retcode = subprocess.call(real_cmd, shell=expand)
    if retcode:
        raise CalledProcessError(retcode, real_cmd)
266

267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
_p4_version_string = None
def p4_version_string():
    """Read the version string, showing just the last line, which
       hopefully is the interesting version bit.

       $ p4 -V
       Perforce - The Fast Software Configuration Management System.
       Copyright 1995-2011 Perforce Software.  All rights reserved.
       Rev. P4/NTX86/2011.1/393975 (2011/12/16).
    """
    global _p4_version_string
    if not _p4_version_string:
        a = p4_read_pipe_lines(["-V"])
        _p4_version_string = a[-1].rstrip()
    return _p4_version_string

283
def p4_integrate(src, dest):
284
    p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
285

286
def p4_sync(f, *options):
287
    p4_system(["sync"] + list(options) + [wildcard_encode(f)])
288 289

def p4_add(f):
290 291 292 293 294
    # forcibly add file names with wildcards
    if wildcard_present(f):
        p4_system(["add", "-f", f])
    else:
        p4_system(["add", f])
295 296

def p4_delete(f):
297
    p4_system(["delete", wildcard_encode(f)])
298

299 300
def p4_edit(f, *options):
    p4_system(["edit"] + list(options) + [wildcard_encode(f)])
301 302

def p4_revert(f):
303
    p4_system(["revert", wildcard_encode(f)])
304

305 306
def p4_reopen(type, f):
    p4_system(["reopen", "-t", type, wildcard_encode(f)])
307

308 309 310 311
def p4_reopen_in_change(changelist, files):
    cmd = ["reopen", "-c", str(changelist)] + files
    p4_system(cmd)

312 313 314
def p4_move(src, dest):
    p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])

315
def p4_last_change():
316
    results = p4CmdList(["changes", "-m", "1"], skip_info=True)
317 318
    return int(results[0]['change'])

319 320 321 322 323
def p4_describe(change):
    """Make sure it returns a valid result by checking for
       the presence of field "time".  Return a dict of the
       results."""

324
    ds = p4CmdList(["describe", "-s", str(change)], skip_info=True)
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341
    if len(ds) != 1:
        die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))

    d = ds[0]

    if "p4ExitCode" in d:
        die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
                                                      str(d)))
    if "code" in d:
        if d["code"] == "error":
            die("p4 describe -s %d returned error code: %s" % (change, str(d)))

    if "time" not in d:
        die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))

    return d

342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
#
# Canonicalize the p4 type and return a tuple of the
# base type, plus any modifiers.  See "p4 help filetypes"
# for a list and explanation.
#
def split_p4_type(p4type):

    p4_filetypes_historical = {
        "ctempobj": "binary+Sw",
        "ctext": "text+C",
        "cxtext": "text+Cx",
        "ktext": "text+k",
        "kxtext": "text+kx",
        "ltext": "text+F",
        "tempobj": "binary+FSw",
        "ubinary": "binary+F",
        "uresource": "resource+F",
        "uxbinary": "binary+Fx",
        "xbinary": "binary+x",
        "xltext": "text+Fx",
        "xtempobj": "binary+Swx",
        "xtext": "text+x",
        "xunicode": "unicode+x",
        "xutf16": "utf16+x",
    }
    if p4type in p4_filetypes_historical:
        p4type = p4_filetypes_historical[p4type]
    mods = ""
    s = p4type.split("+")
    base = s[0]
    mods = ""
    if len(s) > 1:
        mods = s[1]
    return (base, mods)
376

377 378 379
#
# return the raw p4 type of a file (text, text+ko, etc)
#
380 381
def p4_type(f):
    results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399
    return results[0]['headType']

#
# Given a type base and modifier, return a regexp matching
# the keywords that can be expanded in the file
#
def p4_keywords_regexp_for_type(base, type_mods):
    if base in ("text", "unicode", "binary"):
        kwords = None
        if "ko" in type_mods:
            kwords = 'Id|Header'
        elif "k" in type_mods:
            kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
        else:
            return None
        pattern = r"""
            \$              # Starts with a dollar, followed by...
            (%s)            # one of the keywords, followed by...
400
            (:[^$\n]+)?     # possibly an old expansion, followed by...
401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417
            \$              # another dollar
            """ % kwords
        return pattern
    else:
        return None

#
# Given a file, return a regexp matching the possible
# RCS keywords that will be expanded, or None for files
# with kw expansion turned off.
#
def p4_keywords_regexp_for_file(file):
    if not os.path.exists(file):
        return None
    else:
        (type_base, type_mods) = split_p4_type(p4_type(file))
        return p4_keywords_regexp_for_type(type_base, type_mods)
418

419 420 421 422 423 424 425 426 427 428 429 430 431
def setP4ExecBit(file, mode):
    # Reopens an already open file and changes the execute bit to match
    # the execute bit setting in the passed in mode.

    p4Type = "+x"

    if not isModeExec(mode):
        p4Type = getP4OpenedType(file)
        p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
        p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
        if p4Type[-1] == "+":
            p4Type = p4Type[0:-1]

432
    p4_reopen(p4Type, file)
433 434 435 436

def getP4OpenedType(file):
    # Returns the perforce file type for the given file.

437
    result = p4_read_pipe(["opened", wildcard_encode(file)])
438
    match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
439 440 441
    if match:
        return match.group(1)
    else:
442
        die("Could not determine file type for %s (result: '%s')" % (file, result))
443

444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463
# Return the set of all p4 labels
def getP4Labels(depotPaths):
    labels = set()
    if isinstance(depotPaths,basestring):
        depotPaths = [depotPaths]

    for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
        label = l['label']
        labels.add(label)

    return labels

# Return the set of all git tags
def getGitTags():
    gitTags = set()
    for line in read_pipe_lines(["git", "tag"]):
        tag = line.strip()
        gitTags.add(tag)
    return gitTags

464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503
def diffTreePattern():
    # This is a simple generator for the diff tree regex pattern. This could be
    # a class variable if this and parseDiffTreeEntry were a part of a class.
    pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
    while True:
        yield pattern

def parseDiffTreeEntry(entry):
    """Parses a single diff tree entry into its component elements.

    See git-diff-tree(1) manpage for details about the format of the diff
    output. This method returns a dictionary with the following elements:

    src_mode - The mode of the source file
    dst_mode - The mode of the destination file
    src_sha1 - The sha1 for the source file
    dst_sha1 - The sha1 fr the destination file
    status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
    status_score - The score for the status (applicable for 'C' and 'R'
                   statuses). This is None if there is no score.
    src - The path for the source file.
    dst - The path for the destination file. This is only present for
          copy or renames. If it is not present, this is None.

    If the pattern is not matched, None is returned."""

    match = diffTreePattern().next().match(entry)
    if match:
        return {
            'src_mode': match.group(1),
            'dst_mode': match.group(2),
            'src_sha1': match.group(3),
            'dst_sha1': match.group(4),
            'status': match.group(5),
            'status_score': match.group(6),
            'src': match.group(7),
            'dst': match.group(10)
        }
    return None

504 505 506 507 508 509 510 511
def isModeExec(mode):
    # Returns True if the given git mode represents an executable file,
    # otherwise False.
    return mode[-3:] == "755"

def isModeExecChanged(src_mode, dst_mode):
    return isModeExec(src_mode) != isModeExec(dst_mode)

512
def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
513 514 515 516 517 518 519 520 521

    if isinstance(cmd,basestring):
        cmd = "-G " + cmd
        expand = True
    else:
        cmd = ["-G"] + cmd
        expand = False

    cmd = p4_build_cmd(cmd)
522
    if verbose:
523
        sys.stderr.write("Opening pipe: %s\n" % str(cmd))
524 525 526 527 528 529 530

    # Use a temporary file to avoid deadlocks without
    # subprocess.communicate(), which would put another copy
    # of stdout into memory.
    stdin_file = None
    if stdin is not None:
        stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
531 532 533 534 535
        if isinstance(stdin,basestring):
            stdin_file.write(stdin)
        else:
            for i in stdin:
                stdin_file.write(i + '\n')
536 537 538
        stdin_file.flush()
        stdin_file.seek(0)

539 540
    p4 = subprocess.Popen(cmd,
                          shell=expand,
541 542
                          stdin=stdin_file,
                          stdout=subprocess.PIPE)
543 544 545 546

    result = []
    try:
        while True:
547
            entry = marshal.load(p4.stdout)
548 549 550
            if skip_info:
                if 'code' in entry and entry['code'] == 'info':
                    continue
551 552 553 554
            if cb is not None:
                cb(entry)
            else:
                result.append(entry)
555 556
    except EOFError:
        pass
557 558
    exitCode = p4.wait()
    if exitCode != 0:
559 560 561
        entry = {}
        entry["p4ExitCode"] = exitCode
        result.append(entry)
562 563 564 565 566 567 568 569 570 571

    return result

def p4Cmd(cmd):
    list = p4CmdList(cmd)
    result = {}
    for entry in list:
        result.update(entry)
    return result;

572 573 574
def p4Where(depotPath):
    if not depotPath.endswith("/"):
        depotPath += "/"
575 576
    depotPathLong = depotPath + "..."
    outputList = p4CmdList(["where", depotPathLong])
577 578
    output = None
    for entry in outputList:
579
        if "depotFile" in entry:
580 581 582
            # Search for the base client side depot path, as long as it starts with the branch's P4 path.
            # The base path always ends with "/...".
            if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
583 584 585 586 587 588 589 590
                output = entry
                break
        elif "data" in entry:
            data = entry.get("data")
            space = data.find(" ")
            if data[:space] == depotPath:
                output = entry
                break
591 592
    if output == None:
        return ""
593 594
    if output["code"] == "error":
        return ""
595 596 597 598 599 600 601 602 603 604 605 606
    clientPath = ""
    if "path" in output:
        clientPath = output.get("path")
    elif "data" in output:
        data = output.get("data")
        lastSpace = data.rfind(" ")
        clientPath = data[lastSpace + 1:]

    if clientPath.endswith("..."):
        clientPath = clientPath[:-3]
    return clientPath

607
def currentGitBranch():
608
    return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
609

610
def isValidGitDir(path):
611
    return git_dir(path) != None
612

613
def parseRevision(ref):
614
    return read_pipe("git rev-parse %s" % ref).strip()
615

616 617 618 619 620
def branchExists(ref):
    rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
                     ignore_error=True)
    return len(rev) > 0

621 622
def extractLogMessageFromGitCommit(commit):
    logMessage = ""
623 624

    ## fixme: title is first line of commit, not 1st paragraph.
625
    foundTitle = False
626
    for log in read_pipe_lines("git cat-file commit %s" % commit):
627 628
       if not foundTitle:
           if len(log) == 1:
Simon Hausmann's avatar
Simon Hausmann committed
629
               foundTitle = True
630 631 632 633 634
           continue

       logMessage += log
    return logMessage

Han-Wen Nienhuys's avatar
Han-Wen Nienhuys committed
635
def extractSettingsGitLog(log):
636 637 638
    values = {}
    for line in log.split("\n"):
        line = line.strip()
639 640 641 642 643 644 645 646 647 648 649 650 651 652
        m = re.search (r"^ *\[git-p4: (.*)\]$", line)
        if not m:
            continue

        assignments = m.group(1).split (':')
        for a in assignments:
            vals = a.split ('=')
            key = vals[0].strip()
            val = ('='.join (vals[1:])).strip()
            if val.endswith ('\"') and val.startswith('"'):
                val = val[1:-1]

            values[key] = val

653 654 655
    paths = values.get("depot-paths")
    if not paths:
        paths = values.get("depot-path")
656 657
    if paths:
        values['depot-paths'] = paths.split(',')
Han-Wen Nienhuys's avatar
Han-Wen Nienhuys committed
658
    return values
659

660
def gitBranchExists(branch):
Han-Wen Nienhuys's avatar
Han-Wen Nienhuys committed
661 662
    proc = subprocess.Popen(["git", "rev-parse", branch],
                            stderr=subprocess.PIPE, stdout=subprocess.PIPE);
663
    return proc.wait() == 0;
664

665
_gitConfig = {}
666

667
def gitConfig(key, typeSpecifier=None):
668
    if not _gitConfig.has_key(key):
669 670 671 672
        cmd = [ "git", "config" ]
        if typeSpecifier:
            cmd += [ typeSpecifier ]
        cmd += [ key ]
673 674
        s = read_pipe(cmd, ignore_error=True)
        _gitConfig[key] = s.strip()
675
    return _gitConfig[key]
676

677 678 679 680 681
def gitConfigBool(key):
    """Return a bool, using git config --bool.  It is True only if the
       variable is set to true, and False if set to false or not present
       in the config."""

682
    if not _gitConfig.has_key(key):
683
        _gitConfig[key] = gitConfig(key, '--bool') == "true"
684
    return _gitConfig[key]
685

686 687 688
def gitConfigInt(key):
    if not _gitConfig.has_key(key):
        cmd = [ "git", "config", "--int", key ]
689 690
        s = read_pipe(cmd, ignore_error=True)
        v = s.strip()
691 692 693 694
        try:
            _gitConfig[key] = int(gitConfig(key, '--int'))
        except ValueError:
            _gitConfig[key] = None
695
    return _gitConfig[key]
696

697 698
def gitConfigList(key):
    if not _gitConfig.has_key(key):
699
        s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
700
        _gitConfig[key] = s.strip().splitlines()
701 702
        if _gitConfig[key] == ['']:
            _gitConfig[key] = []
703 704
    return _gitConfig[key]

705 706 707 708 709 710 711
def p4BranchesInGit(branchesAreInRemotes=True):
    """Find all the branches whose names start with "p4/", looking
       in remotes or heads as specified by the argument.  Return
       a dictionary of { branch: revision } for each one found.
       The branch names are the short names, without any
       "p4/" prefix."""

712 713 714 715
    branches = {}

    cmdline = "git rev-parse --symbolic "
    if branchesAreInRemotes:
716
        cmdline += "--remotes"
717
    else:
718
        cmdline += "--branches"
719 720 721 722

    for line in read_pipe_lines(cmdline):
        line = line.strip()

723 724 725 726 727
        # only import to p4/
        if not line.startswith('p4/'):
            continue
        # special symbolic ref to p4/master
        if line == "p4/HEAD":
728 729
            continue

730 731
        # strip off p4/ prefix
        branch = line[len("p4/"):]
732 733

        branches[branch] = parseRevision(line)
734

735 736
    return branches

737 738 739 740 741 742 743 744 745 746 747
def branch_exists(branch):
    """Make sure that the given ref name really exists."""

    cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, _ = p.communicate()
    if p.returncode:
        return False
    # expect exactly one line of output: the branch name
    return out.rstrip() == branch

748
def findUpstreamBranchPoint(head = "HEAD"):
749 750 751 752 753 754 755 756 757 758 759
    branches = p4BranchesInGit()
    # map from depot-path to branch name
    branchByDepotPath = {}
    for branch in branches.keys():
        tip = branches[branch]
        log = extractLogMessageFromGitCommit(tip)
        settings = extractSettingsGitLog(log)
        if settings.has_key("depot-paths"):
            paths = ",".join(settings["depot-paths"])
            branchByDepotPath[paths] = "remotes/p4/" + branch

760 761 762
    settings = None
    parent = 0
    while parent < 65535:
763
        commit = head + "~%s" % parent
764 765
        log = extractLogMessageFromGitCommit(commit)
        settings = extractSettingsGitLog(log)
766 767 768 769
        if settings.has_key("depot-paths"):
            paths = ",".join(settings["depot-paths"])
            if branchByDepotPath.has_key(paths):
                return [branchByDepotPath[paths], settings]
770

771
        parent = parent + 1
772

773
    return ["", settings]
774

775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824
def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
    if not silent:
        print ("Creating/updating branch(es) in %s based on origin branch(es)"
               % localRefPrefix)

    originPrefix = "origin/p4/"

    for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
        line = line.strip()
        if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
            continue

        headName = line[len(originPrefix):]
        remoteHead = localRefPrefix + headName
        originHead = line

        original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
        if (not original.has_key('depot-paths')
            or not original.has_key('change')):
            continue

        update = False
        if not gitBranchExists(remoteHead):
            if verbose:
                print "creating %s" % remoteHead
            update = True
        else:
            settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
            if settings.has_key('change') > 0:
                if settings['depot-paths'] == original['depot-paths']:
                    originP4Change = int(original['change'])
                    p4Change = int(settings['change'])
                    if originP4Change > p4Change:
                        print ("%s (%s) is newer than %s (%s). "
                               "Updating p4 branch from origin."
                               % (originHead, originP4Change,
                                  remoteHead, p4Change))
                        update = True
                else:
                    print ("Ignoring: %s was imported from %s while "
                           "%s was imported from %s"
                           % (originHead, ','.join(original['depot-paths']),
                              remoteHead, ','.join(settings['depot-paths'])))

        if update:
            system("git update-ref %s %s" % (remoteHead, originHead))

def originP4BranchesExist():
        return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")

825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841

def p4ParseNumericChangeRange(parts):
    changeStart = int(parts[0][1:])
    if parts[1] == '#head':
        changeEnd = p4_last_change()
    else:
        changeEnd = int(parts[1])

    return (changeStart, changeEnd)

def chooseBlockSize(blockSize):
    if blockSize:
        return blockSize
    else:
        return defaultBlockSize

def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
842
    assert depotPaths
843

844 845 846 847 848 849
    # Parse the change range into start and end. Try to find integer
    # revision ranges as these can be broken up into blocks to avoid
    # hitting server-side limits (maxrows, maxscanresults). But if
    # that doesn't work, fall back to using the raw revision specifier
    # strings, without using block mode.

850
    if changeRange is None or changeRange == '':
851 852 853
        changeStart = 1
        changeEnd = p4_last_change()
        block_size = chooseBlockSize(requestedBlockSize)
854 855 856
    else:
        parts = changeRange.split(',')
        assert len(parts) == 2
857 858 859 860 861 862 863 864 865
        try:
            (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
            block_size = chooseBlockSize(requestedBlockSize)
        except:
            changeStart = parts[0][1:]
            changeEnd = parts[1]
            if requestedBlockSize:
                die("cannot use --changes-block-size with non-numeric revisions")
            block_size = None
866

867
    changes = set()
868

869 870
    # Retrieve changes a block at a time, to prevent running
    # into a MaxResults/MaxScanRows error from the server.
871

872 873
    while True:
        cmd = ['changes']
874

875 876 877 878 879
        if block_size:
            end = min(changeEnd, changeStart + block_size)
            revisionRange = "%d,%d" % (changeStart, end)
        else:
            revisionRange = "%s,%s" % (changeStart, changeEnd)
880

881
        for p in depotPaths:
882 883
            cmd += ["%s...@%s" % (p, revisionRange)]

884
        # Insert changes in chronological order
885 886 887 888 889 890
        for entry in reversed(p4CmdList(cmd)):
            if entry.has_key('p4ExitCode'):
                die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
            if not entry.has_key('change'):
                continue
            changes.add(int(entry['change']))
891

892 893
        if not block_size:
            break
894

895 896
        if end >= changeEnd:
            break
897

898
        changeStart = end + 1
899

900 901
    changes = sorted(changes)
    return changes
902

903 904 905 906 907 908 909 910
def p4PathStartsWith(path, prefix):
    # This method tries to remedy a potential mixed-case issue:
    #
    # If UserA adds  //depot/DirA/file1
    # and UserB adds //depot/dira/file2
    #
    # we may or may not have a problem. If you have core.ignorecase=true,
    # we treat DirA and dira as the same directory
911
    if gitConfigBool("core.ignorecase"):
912 913 914
        return path.lower().startswith(prefix.lower())
    return path.startswith(prefix)

915 916 917 918 919 920 921 922 923 924 925 926
def getClientSpec():
    """Look at the p4 client spec, create a View() object that contains
       all the mappings, and return it."""

    specList = p4CmdList("client -o")
    if len(specList) != 1:
        die('Output from "client -o" is %d lines, expecting 1' %
            len(specList))

    # dictionary of all client parameters
    entry = specList[0]

927 928 929
    # the //client/ name
    client_name = entry["Client"]

930 931 932 933
    # just the keys that start with "View"
    view_keys = [ k for k in entry.keys() if k.startswith("View") ]

    # hold this new View
934
    view = View(client_name)
935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957

    # append the lines, in order, to the view
    for view_num in range(len(view_keys)):
        k = "View%d" % view_num
        if k not in view_keys:
            die("Expected view key %s missing" % k)
        view.append(entry[k])

    return view

def getClientRoot():
    """Grab the client directory."""

    output = p4CmdList("client -o")
    if len(output) != 1:
        die('Output from "client -o" is %d lines, expecting 1' % len(output))

    entry = output[0]
    if "Root" not in entry:
        die('Client has no "Root"')

    return entry["Root"]

958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983
#
# P4 wildcards are not allowed in filenames.  P4 complains
# if you simply add them, but you can force it with "-f", in
# which case it translates them into %xx encoding internally.
#
def wildcard_decode(path):
    # Search for and fix just these four characters.  Do % last so
    # that fixing it does not inadvertently create new %-escapes.
    # Cannot have * in a filename in windows; untested as to
    # what p4 would do in such a case.
    if not platform.system() == "Windows":
        path = path.replace("%2A", "*")
    path = path.replace("%23", "#") \
               .replace("%40", "@") \
               .replace("%25", "%")
    return path

def wildcard_encode(path):
    # do % first to avoid double-encoding the %s introduced here
    path = path.replace("%", "%25") \
               .replace("*", "%2A") \
               .replace("#", "%23") \
               .replace("@", "%40")
    return path

def wildcard_present(path):
984 985
    m = re.search("[*#@%]", path)
    return m is not None
986

987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
class LargeFileSystem(object):
    """Base class for large file system support."""

    def __init__(self, writeToGitStream):
        self.largeFiles = set()
        self.writeToGitStream = writeToGitStream

    def generatePointer(self, cloneDestination, contentFile):
        """Return the content of a pointer file that is stored in Git instead of
           the actual content."""
        assert False, "Method 'generatePointer' required in " + self.__class__.__name__

    def pushFile(self, localLargeFile):
        """Push the actual content which is not stored in the Git repository to
           a server."""
        assert False, "Method 'pushFile' required in " + self.__class__.__name__

    def hasLargeFileExtension(self, relPath):
        return reduce(
            lambda a, b: a or b,
            [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
            False
        )

    def generateTempFile(self, contents):
        contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
        for d in contents:
            contentFile.write(d)
        contentFile.close()
        return contentFile.name

    def exceedsLargeFileThreshold(self, relPath, contents):
        if gitConfigInt('git-p4.largeFileThreshold'):
            contentsSize = sum(len(d) for d in contents)
            if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
                return True
        if gitConfigInt('git-p4.largeFileCompressedThreshold'):
            contentsSize = sum(len(d) for d in contents)
            if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
                return False
            contentTempFile = self.generateTempFile(contents)
            compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
            zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
            zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
            zf.close()
            compressedContentsSize = zf.infolist()[0].compress_size
            os.remove(contentTempFile)
            os.remove(compressedContentFile.name)
            if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
                return True
        return False

    def addLargeFile(self, relPath):
        self.largeFiles.add(relPath)

    def removeLargeFile(self, relPath):
        self.largeFiles.remove(relPath)

    def isLargeFile(self, relPath):
        return relPath in self.largeFiles

    def processContent(self, git_mode, relPath, contents):
        """Processes the content of git fast import. This method decides if a
           file is stored in the large file system and handles all necessary
           steps."""
        if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
            contentTempFile = self.generateTempFile(contents)
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067
            (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
            if pointer_git_mode:
                git_mode = pointer_git_mode
            if localLargeFile:
                # Move temp file to final location in large file system
                largeFileDir = os.path.dirname(localLargeFile)
                if not os.path.isdir(largeFileDir):
                    os.makedirs(largeFileDir)
                shutil.move(contentTempFile, localLargeFile)
                self.addLargeFile(relPath)
                if gitConfigBool('git-p4.largeFilePush'):
                    self.pushFile(localLargeFile)
                if verbose:
                    sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
        return (git_mode, contents)

class MockLFS(LargeFileSystem):
    """Mock large file system for testing."""

    def generatePointer(self, contentFile):
        """The pointer content is the original content prefixed with "pointer-".
           The local filename of the large file storage is derived from the file content.
           """
        with open(contentFile, 'r') as f:
            content = next(f)
            gitMode = '100644'
            pointerContents = 'pointer-' + content
            localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
            return (gitMode, pointerContents, localLargeFile)

    def pushFile(self, localLargeFile):
        """The remote filename of the large file storage is the same as the local
           one but in a different directory.
           """
        remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
        if not os.path.exists(remotePath):
            os.makedirs(remotePath)
        shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))

1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106
class GitLFS(LargeFileSystem):
    """Git LFS as backend for the git-p4 large file system.
       See https://git-lfs.github.com/ for details."""

    def __init__(self, *args):
        LargeFileSystem.__init__(self, *args)
        self.baseGitAttributes = []

    def generatePointer(self, contentFile):
        """Generate a Git LFS pointer for the content. Return LFS Pointer file
           mode and content which is stored in the Git repository instead of
           the actual content. Return also the new location of the actual
           content.
           """
1107 1108 1109
        if os.path.getsize(contentFile) == 0:
            return (None, '', None)

1110 1111 1112 1113 1114 1115 1116 1117
        pointerProcess = subprocess.Popen(
            ['git', 'lfs', 'pointer', '--file=' + contentFile],
            stdout=subprocess.PIPE
        )
        pointerFile = pointerProcess.stdout.read()
        if pointerProcess.wait():
            os.remove(contentFile)
            die('git-lfs pointer command failed. Did you install the extension?')
1118 1119 1120 1121 1122 1123 1124 1125 1126

        # Git LFS removed the preamble in the output of the 'pointer' command
        # starting from version 1.2.0. Check for the preamble here to support
        # earlier versions.
        # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
        if pointerFile.startswith('Git LFS pointer for'):
            pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)

        oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1127 1128 1129 1130 1131 1132 1133
        localLargeFile = os.path.join(
            os.getcwd(),
            '.git', 'lfs', 'objects', oid[:2], oid[2:4],
            oid,
        )
        # LFS Spec states that pointer files should not have the executable bit set.
        gitMode = '100644'
1134
        return (gitMode, pointerFile, localLargeFile)
1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151

    def pushFile(self, localLargeFile):
        uploadProcess = subprocess.Popen(
            ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
        )
        if uploadProcess.wait():
            die('git-lfs push command failed. Did you define a remote?')

    def generateGitAttributes(self):
        return (
            self.baseGitAttributes +
            [
                '\n',
                '#\n',
                '# Git LFS (see https://git-lfs.github.com/)\n',
                '#\n',
            ] +
1152
            ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1153 1154
                for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
            ] +
1155
            ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
                for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
            ]
        )

    def addLargeFile(self, relPath):
        LargeFileSystem.addLargeFile(self, relPath)
        self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())

    def removeLargeFile(self, relPath):
        LargeFileSystem.removeLargeFile(self, relPath)
        self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())

    def processContent(self, git_mode, relPath, contents):
        if relPath == '.gitattributes':
            self.baseGitAttributes = contents
            return (git_mode, self.generateGitAttributes())
        else:
            return LargeFileSystem.processContent(self, git_mode, relPath, contents)

1175 1176 1177
class Command:
    def __init__(self):
        self.usage = "usage: %prog [options]"
1178
        self.needsGit = True
1179
        self.verbose = False
1180

1181 1182 1183 1184 1185 1186
    # This is required for the "append" cloneExclude action
    def ensure_value(self, attr, value):
        if not hasattr(self, attr) or getattr(self, attr) is None:
            setattr(self, attr, value)
        return getattr(self, attr)

1187 1188 1189
class P4UserMap:
    def __init__(self):
        self.userMapFromPerforceServer = False
1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209
        self.myP4UserId = None

    def p4UserId(self):
        if self.myP4UserId:
            return self.myP4UserId

        results = p4CmdList("user -o")
        for r in results:
            if r.has_key('User'):
                self.myP4UserId = r['User']
                return r['User']
        die("Could not find your p4 user id")

    def p4UserIsMe(self, p4User):
        # return True if the given p4 user is actually me
        me = self.p4UserId()
        if not p4User or p4User != me:
            return False
        else:
            return True
1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226

    def getUserCacheFilename(self):
        home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
        return home + "/.gitp4-usercache.txt"

    def getUserMapFromPerforceServer(self):
        if self.userMapFromPerforceServer:
            return
        self.users = {}
        self.emails = {}

        for output in p4CmdList("users"):
            if not output.has_key("User"):
                continue
            self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
            self.emails[output["Email"]] = output["User"]

1227 1228 1229 1230 1231 1232 1233 1234 1235
        mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
        for mapUserConfig in gitConfigList("git-p4.mapUser"):
            mapUser = mapUserConfigRegex.findall(mapUserConfig)
            if mapUser and len(mapUser[0]) == 3:
                user = mapUser[0][0]
                fullname = mapUser[0][1]
                email = mapUser[0][2]
                self.users[user] = fullname + " <" + email + ">"
                self.emails[email] = user
1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256

        s = ''
        for (key, val) in self.users.items():
            s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))

        open(self.getUserCacheFilename(), "wb").write(s)
        self.userMapFromPerforceServer = True

    def loadUserMapFromCache(self):
        self.users = {}
        self.userMapFromPerforceServer = False
        try:
            cache = open(self.getUserCacheFilename(), "rb")
            lines = cache.readlines()
            cache.close()
            for line in lines:
                entry = line.strip().split("\t")
                self.users[entry[0]] = entry[1]
        except IOError:
            self.getUserMapFromPerforceServer()

1257
class P4Debug(Command):
1258
    def __init__(self):
1259
        Command.__init__(self)
1260
        self.options = []
1261
        self.description = "A tool to debug the output of p4 -G."
1262
        self.needsGit = False
1263 1264

    def run(self, args):
1265
        j = 0
1266
        for output in p4CmdList(args):
1267 1268
            print 'Element: %d' % j
            j += 1
1269
            print output
1270
        return True
1271

1272 1273 1274 1275
class P4RollBack(Command):
    def __init__(self):
        Command.__init__(self)
        self.options = [
1276
            optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1277 1278
        ]
        self.description = "A tool to debug the multi-branch import. Don't use :)"
1279
        self.rollbackLocalBranches = False
1280 1281 1282 1283 1284

    def run(self, args):
        if len(args) != 1:
            return False
        maxChange = int(args[0])
1285

1286
        if "p4ExitCode" in p4Cmd("changes -m 1"):
1287 1288
            die("Problems executing p4");

1289 1290
        if self.rollbackLocalBranches:
            refPrefix = "refs/heads/"
1291
            lines = read_pipe_lines("git rev-parse --symbolic --branches")
1292 1293
        else:
            refPrefix = "refs/remotes/"
1294
            lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1295 1296 1297

        for line in lines:
            if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1298 1299
                line = line.strip()
                ref = refPrefix + line
1300
                log = extractLogMessageFromGitCommit(ref)
Han-Wen Nienhuys's avatar
Han-Wen Nienhuys committed
1301 1302 1303 1304 1305
                settings = extractSettingsGitLog(log)

                depotPaths = settings['depot-paths']
                change = settings['change']

1306
                changed = False
1307

1308 1309
                if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
                                                           for p in depotPaths]))) == 0:
1310 1311 1312 1313
                    print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
                    system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
                    continue

Han-Wen Nienhuys's avatar
Han-Wen Nienhuys committed
1314
                while change and int(change) > maxChange:
1315
                    changed = True
1316 1317
                    if self.verbose:
                        print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1318 1319
                    system("git update-ref %s \"%s^\"" % (ref, ref))
                    log = extractLogMessageFromGitCommit(ref)
Han-Wen Nienhuys's avatar
Han-Wen Nienhuys committed
1320 1321 1322 1323 1324
                    settings =  extractSettingsGitLog(log)


                    depotPaths = settings['depot-paths']
                    change = settings['change']
1325 1326

                if changed:
1327
                    print "%s rewound to %s" % (ref, change)
1328 1329 1330

        return True

1331
class P4Submit(Command, P4UserMap):
1332 1333 1334

    conflict_behavior_choices = ("ask", "skip", "quit")

1335
    def __init__(self):
1336
        Command.__init__(self)
1337
        P4UserMap.__init__(self)
1338 1339
        self.options = [
                optparse.make_option("--origin", dest="origin"),
1340
                optparse.make_option("-M", dest="detectRenames", action="store_true"),
1341 1342
                # preserve the user, requires relevant p4 permissions
                optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1343
                optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1344
                optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1345
                optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1346
                optparse.make_option("--conflict", dest="conflict_behavior",
1347 1348
                                     choices=self.conflict_behavior_choices),
                optparse.make_option("--branch", dest="branch"),
1349 1350 1351
                optparse.make_option("--shelve", dest="shelve", action="store_true",
                                     help="Shelve instead of submit. Shelved files are reverted, "
                                     "restoring the workspace to the state before the shelve"),
1352
                optparse.make_option("--update-shelve", dest="update_shelve", action="append", type="int",
1353
                                     metavar="CHANGELIST",
1354 1355
                                     help="update an existing shelved changelist, implies --shelve, "
                                           "repeat in-order for multiple shelved changelists")
1356 1357
        ]
        self.description = "Submit changes from git to the perforce depot."
1358
        self.usage += " [name of git branch to submit into perforce depot]"
1359
        self.origin = ""
1360
        self.detectRenames = False
1361
        self.preserveUser = gitConfigBool("git-p4.preserveUser")
1362
        self.dry_run = False
1363
        self.shelve = False
1364
        self.update_shelve = list()
1365
        self.prepare_p4_only = False
1366
        self.conflict_behavior = None
1367
        self.isWindows = (platform.system() == "Windows")
1368
        self.exportLabels = False
1369
        self.p4HasMoveCommand = p4_has_move_command()
1370
        self.branch = None
1371

1372 1373 1374
        if gitConfig('git-p4.largeFileSystem'):
            die("Large file system not supported for git-p4 submit command. Please remove it from config.")

1375 1376 1377 1378
    def check(self):
        if len(p4CmdList("opened ...")) > 0:
            die("You have files opened with perforce! Close them before starting the sync.")

1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406
    def separate_jobs_from_description(self, message):
        """Extract and return a possible Jobs field in the commit
           message.  It goes into a separate section in the p4 change
           specification.

           A jobs line starts with "Jobs:" and looks like a new field
           in a form.  Values are white-space separated on the same
           line or on following lines that start with a tab.

           This does not parse and extract the full git commit message
           like a p4 form.  It just sees the Jobs: line as a marker
           to pass everything from then on directly into the p4 form,
           but outside the description section.

           Return a tuple (stripped log message, jobs string)."""

        m = re.search(r'^Jobs:', message, re.MULTILINE)
        if m is None:
            return (message, None)

        jobtext = message[m.start():]
        stripped_message = message[:m.start()].rstrip()
        return (stripped_message, jobtext)

    def prepareLogMessage(self, template, message, jobs):
        """Edits the template returned from "p4 change -o" to insert
           the message in the Description field, and the jobs text in
           the Jobs field."""
1407 1408
        result = ""

1409 1410
        inDescriptionSection = False

1411 1412 1413 1414 1415
        for line in template.split("\n"):
            if line.startswith("#"):
                result += line + "\n"
                continue

1416
            if inDescriptionSection:
1417
                if line.startswith("Files:") or line.startswith("Jobs:"):
1418
                    inDescriptionSection = False
1419 1420 1421
                    # insert Jobs section
                    if jobs:
                        result += jobs + "\n"
1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
                else:
                    continue
            else:
                if line.startswith("Description:"):
                    inDescriptionSection = True
                    line += "\n"
                    for messageLine in message.split("\n"):
                        line += "\t" + messageLine + "\n"

            result += line + "\n"
1432 1433 1434

        return result

1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457
    def patchRCSKeywords(self, file, pattern):
        # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
        (handle, outFileName) = tempfile.mkstemp(dir='.')
        try:
            outFile = os.fdopen(handle, "w+")
            inFile = open(file, "r")
            regexp = re.compile(pattern, re.VERBOSE)
            for line in inFile.readlines():
                line = regexp.sub(r'$\1$', line)
                outFile.write(line)
            inFile.close()
            outFile.close()
            # Forcibly overwrite the original file
            os.unlink(file)
            shutil.move(outFileName, file)
        except:
            # cleanup our temporary file
            os.unlink(outFileName)
            print "Failed to strip RCS keywords in %s" % file
            raise

        print "Patched up RCS keywords in %s" % file

1458 1459 1460
    def p4UserForCommit(self,id):
        # Return the tuple (perforce user,git email) for a given git commit id
        self.getUserMapFromPerforceServer()
1461 1462
        gitEmail = read_pipe(["git", "log", "--max-count=1",
                              "--format=%ae", id])
1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474
        gitEmail = gitEmail.strip()
        if not self.emails.has_key(gitEmail):
            return (None,gitEmail)
        else:
            return (self.emails[gitEmail],gitEmail)

    def checkValidP4Users(self,commits):
        # check if any git authors cannot be mapped to p4 users
        for id in commits:
            (user,email) = self.p4UserForCommit(id)
            if not user:
                msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1475
                if gitConfigBool("git-p4.allowMissingP4Users"):
1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492
                    print "%s" % msg
                else:
                    die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)

    def lastP4Changelist(self):
        # Get back the last changelist number submitted in this client spec. This
        # then gets used to patch up the username in the change. If the same
        # client spec is being used by multiple processes then this might go
        # wrong.
        results = p4CmdList("client -o")        # find the current client
        client = None
        for r in results:
            if r.has_key('Client'):
                client = r['Client']
                break
        if not client:
            die("could not get client spec")
1493
        results = p4CmdList(["changes", "-c", client, "-m", "1"])
1494 1495 1496 1497 1498 1499 1500 1501
        for r in results:
            if r.has_key('change'):
                return r['change']
        die("Could not get changelist number for last submit - cannot patch up user details")

    def modifyChangelistUser(self, changelist, newUser):
        # fixup the user field of a changelist after it has been submitted.
        changes = p4CmdList("change -o %s" % changelist)
1502 1503 1504 1505 1506 1507 1508 1509 1510
        if len(changes) != 1:
            die("Bad output from p4 change modifying %s to user %s" %
                (changelist, newUser))

        c = changes[0]
        if c['User'] == newUser: return   # nothing to do
        c['User'] = newUser
        input = marshal.dumps(c)

1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523
        result = p4CmdList("change -f -i", stdin=input)
        for r in result:
            if r.has_key('code'):
                if r['code'] == 'error':
                    die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
            if r.has_key('data'):
                print("Updated user field for changelist %s to %s" % (changelist, newUser))
                return
        die("Could not modify user field of changelist %s to %s" % (changelist, newUser))

    def canChangeChangelists(self):
        # check to see if we have p4 admin or super-user permissions, either of
        # which are required to modify changelists.
1524
        results = p4CmdList(["protects", self.depotPath])
1525 1526 1527 1528 1529 1530 1531 1532
        for r in results:
            if r.has_key('perm'):
                if r['perm'] == 'admin':
                    return 1
                if r['perm'] == 'super':
                    return 1
        return 0

1533
    def prepareSubmitTemplate(self, changelist=None):
1534 1535 1536 1537 1538 1539 1540
        """Run "p4 change -o" to grab a change specification template.
           This does not use "p4 -G", as it is nice to keep the submission
           template in original order, since a human might edit it.

           Remove lines in the Files section that show changes to files
           outside the depot path we're committing into."""

1541 1542
        [upstream, settings] = findUpstreamBranchPoint()

1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559
        template = """\
# A Perforce Change Specification.
#
#  Change:      The change number. 'new' on a new changelist.
#  Date:        The date this specification was last modified.
#  Client:      The client on which the changelist was created.  Read-only.
#  User:        The user who created the changelist.
#  Status:      Either 'pending' or 'submitted'. Read-only.
#  Type:        Either 'public' or 'restricted'. Default is 'public'.
#  Description: Comments about the changelist.  Required.
#  Jobs:        What opened jobs are to be closed by this changelist.
#               You may delete jobs from this list.  (New changelists only.)
#  Files:       What opened files from the default changelist are to be added
#               to this changelist.  You may delete files from this list.
#               (New changelists only.)
"""
        files_list = []
1560
        inFilesSection = False
1561
        change_entry = None
1562 1563 1564
        args = ['change', '-o']
        if changelist:
            args.append(str(changelist))
1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578
        for entry in p4CmdList(args):
            if not entry.has_key('code'):
                continue
            if entry['code'] == 'stat':
                change_entry = entry
                break
        if not change_entry:
            die('Failed to decode output of p4 change -o')
        for key, value in change_entry.iteritems():
            if key.startswith('File'):
                if settings.has_key('depot-paths'):
                    if not [p for p in settings['depot-paths']
                            if p4PathStartsWith(value, p)]:
                        continue
1579
                else:
1580 1581 1582 1583 1584 1585 1586