fetch-pack.c 47 KB
Newer Older
1
#include "cache.h"
2
#include "repository.h"
3
#include "config.h"
4
#include "lockfile.h"
5 6 7 8
#include "refs.h"
#include "pkt-line.h"
#include "commit.h"
#include "tag.h"
9
#include "exec-cmd.h"
10 11 12 13 14
#include "pack.h"
#include "sideband.h"
#include "fetch-pack.h"
#include "remote.h"
#include "run-command.h"
15
#include "connect.h"
16 17
#include "transport.h"
#include "version.h"
18
#include "sha1-array.h"
19
#include "oidset.h"
20
#include "packfile.h"
21
#include "object-store.h"
22
#include "connected.h"
23
#include "fetch-negotiator.h"
24
#include "fsck.h"
25 26 27 28 29 30

static int transfer_unpack_limit = -1;
static int fetch_unpack_limit = -1;
static int unpack_limit = 100;
static int prefer_ofs_delta = 1;
static int no_done;
31
static int deepen_since_ok;
32
static int deepen_not_ok;
33 34 35
static int fetch_fsck_objects = -1;
static int transfer_fsck_objects = -1;
static int agent_supported;
36
static int server_supports_filtering;
37 38
static struct lock_file shallow_lock;
static const char *alternate_shallow_file;
39
static char *negotiation_algorithm;
40
static struct strbuf fsck_msg_types = STRBUF_INIT;
41

42
/* Remember to update object flag allocation in object.h */
43
#define COMPLETE	(1U << 0)
44
#define ALTERNATE	(1U << 1)
45 46 47 48 49 50 51

/*
 * After sending this many "have"s if we do not get any new ACK , we
 * give up traversing our history.
 */
#define MAX_IN_VAIN 256

52
static int multi_ack, use_sideband;
53 54
/* Allow specifying sha1 if it is a ref tip. */
#define ALLOW_TIP_SHA1	01
55 56
/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
#define ALLOW_REACHABLE_SHA1	02
57
static unsigned int allow_unadvertised_object_request;
58

59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
__attribute__((format (printf, 2, 3)))
static inline void print_verbose(const struct fetch_pack_args *args,
				 const char *fmt, ...)
{
	va_list params;

	if (!args->verbose)
		return;

	va_start(params, fmt);
	vfprintf(stderr, fmt, params);
	va_end(params);
	fputc('\n', stderr);
}

74 75 76 77 78
struct alternate_object_cache {
	struct object **items;
	size_t nr, alloc;
};

79
static void cache_one_alternate(const struct object_id *oid,
80 81 82
				void *vcache)
{
	struct alternate_object_cache *cache = vcache;
83
	struct object *obj = parse_object(the_repository, oid);
84 85 86 87 88 89 90 91 92

	if (!obj || (obj->flags & ALTERNATE))
		return;

	obj->flags |= ALTERNATE;
	ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
	cache->items[cache->nr++] = obj;
}

93 94
static void for_each_cached_alternate(struct fetch_negotiator *negotiator,
				      void (*cb)(struct fetch_negotiator *,
95
						 struct object *))
96 97 98 99 100 101 102 103 104 105 106
{
	static int initialized;
	static struct alternate_object_cache cache;
	size_t i;

	if (!initialized) {
		for_each_alternate_ref(cache_one_alternate, &cache);
		initialized = 1;
	}

	for (i = 0; i < cache.nr; i++)
107
		cb(negotiator, cache.items[i]);
108 109
}

110
static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
111 112
			       const char *refname,
			       const struct object_id *oid)
113
{
114 115
	struct object *o = deref_tag(the_repository,
				     parse_object(the_repository, oid),
116
				     refname, 0);
117 118

	if (o && o->type == OBJ_COMMIT)
119
		negotiator->add_tip(negotiator, (struct commit *)o);
120 121 122 123

	return 0;
}

124 125
static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
				   int flag, void *cb_data)
126
{
127
	return rev_list_insert_ref(cb_data, refname, oid);
128 129 130 131 132 133 134 135 136 137
}

enum ack_type {
	NAK = 0,
	ACK,
	ACK_continue,
	ACK_common,
	ACK_ready
};

138 139
static void consume_shallow_list(struct fetch_pack_args *args,
				 struct packet_reader *reader)
140
{
141
	if (args->stateless_rpc && args->deepen) {
142 143 144 145
		/* If we sent a depth we will get back "duplicate"
		 * shallow and unshallow commands every time there
		 * is a block of have lines exchanged.
		 */
146 147
		while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
			if (starts_with(reader->line, "shallow "))
148
				continue;
149
			if (starts_with(reader->line, "unshallow "))
150
				continue;
151
			die(_("git fetch-pack: expected shallow list"));
152
		}
153 154
		if (reader->status != PACKET_READ_FLUSH)
			die(_("git fetch-pack: expected a flush packet after shallow list"));
155 156 157
	}
}

158 159
static enum ack_type get_ack(struct packet_reader *reader,
			     struct object_id *result_oid)
160
{
161
	int len;
162
	const char *arg;
163

164
	if (packet_reader_read(reader) != PACKET_READ_NORMAL)
165
		die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
166 167 168
	len = reader->pktlen;

	if (!strcmp(reader->line, "NAK"))
169
		return NAK;
170
	if (skip_prefix(reader->line, "ACK ", &arg)) {
171
		if (!get_oid_hex(arg, result_oid)) {
172
			arg += 40;
173
			len -= arg - reader->line;
174
			if (len < 1)
175
				return ACK;
176
			if (strstr(arg, "continue"))
177
				return ACK_continue;
178
			if (strstr(arg, "common"))
179
				return ACK_common;
180
			if (strstr(arg, "ready"))
181 182 183 184
				return ACK_ready;
			return ACK;
		}
	}
185
	die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader->line);
186 187 188 189 190 191 192 193
}

static void send_request(struct fetch_pack_args *args,
			 int fd, struct strbuf *buf)
{
	if (args->stateless_rpc) {
		send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
		packet_flush(fd);
194 195 196 197
	} else {
		if (write_in_full(fd, buf->buf, buf->len) < 0)
			die_errno(_("unable to write to remote"));
	}
198 199
}

200
static void insert_one_alternate_object(struct fetch_negotiator *negotiator,
201
					struct object *obj)
202
{
203
	rev_list_insert_ref(negotiator, NULL, &obj->oid);
204 205 206 207
}

#define INITIAL_FLUSH 16
#define PIPESAFE_FLUSH 32
208
#define LARGE_FLUSH 16384
209

210
static int next_flush(int stateless_rpc, int count)
211
{
212
	if (stateless_rpc) {
213 214 215 216 217 218 219 220 221 222
		if (count < LARGE_FLUSH)
			count <<= 1;
		else
			count = count * 11 / 10;
	} else {
		if (count < PIPESAFE_FLUSH)
			count <<= 1;
		else
			count += PIPESAFE_FLUSH;
	}
223 224 225
	return count;
}

226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
static void mark_tips(struct fetch_negotiator *negotiator,
		      const struct oid_array *negotiation_tips)
{
	int i;

	if (!negotiation_tips) {
		for_each_ref(rev_list_insert_ref_oid, negotiator);
		return;
	}

	for (i = 0; i < negotiation_tips->nr; i++)
		rev_list_insert_ref(negotiator, NULL,
				    &negotiation_tips->oid[i]);
	return;
}

242
static int find_common(struct fetch_negotiator *negotiator,
243
		       struct fetch_pack_args *args,
244
		       int fd[2], struct object_id *result_oid,
245 246 247 248
		       struct ref *refs)
{
	int fetching;
	int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
249
	const struct object_id *oid;
250 251 252 253 254
	unsigned in_vain = 0;
	int got_continue = 0;
	int got_ready = 0;
	struct strbuf req_buf = STRBUF_INIT;
	size_t state_len = 0;
255
	struct packet_reader reader;
256 257

	if (args->stateless_rpc && multi_ack == 1)
258
		die(_("--stateless-rpc requires multi_ack_detailed"));
259

260
	packet_reader_init(&reader, fd[0], NULL, 0,
261 262
			   PACKET_READ_CHOMP_NEWLINE |
			   PACKET_READ_DIE_ON_ERR_PACKET);
263

264 265 266 267
	if (!args->no_dependents) {
		mark_tips(negotiator, args->negotiation_tips);
		for_each_cached_alternate(negotiator, insert_one_alternate_object);
	}
268 269 270

	fetching = 0;
	for ( ; refs ; refs = refs->next) {
271
		struct object_id *remote = &refs->old_oid;
272 273 274 275 276 277 278 279 280 281 282 283
		const char *remote_hex;
		struct object *o;

		/*
		 * If that object is complete (i.e. it is an ancestor of a
		 * local ref), we tell them we have it but do not have to
		 * tell them about its ancestors, which they already know
		 * about.
		 *
		 * We use lookup_object here because we are only
		 * interested in the case we *know* the object is
		 * reachable and we have already scanned it.
284 285 286
		 *
		 * Do this only if args->no_dependents is false (if it is true,
		 * we cannot trust the object flags).
287
		 */
288 289
		if (!args->no_dependents &&
		    ((o = lookup_object(the_repository, remote->hash)) != NULL) &&
290 291 292 293
				(o->flags & COMPLETE)) {
			continue;
		}

294
		remote_hex = oid_to_hex(remote);
295 296 297 298 299 300 301
		if (!fetching) {
			struct strbuf c = STRBUF_INIT;
			if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
			if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
			if (no_done)            strbuf_addstr(&c, " no-done");
			if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
			if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
302
			if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
303 304 305 306
			if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
			if (args->no_progress)   strbuf_addstr(&c, " no-progress");
			if (args->include_tag)   strbuf_addstr(&c, " include-tag");
			if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
307
			if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
308
			if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
309 310
			if (agent_supported)    strbuf_addf(&c, " agent=%s",
							    git_user_agent_sanitized());
311 312
			if (args->filter_options.choice)
				strbuf_addstr(&c, " filter");
313 314 315 316 317 318 319 320 321 322 323 324 325
			packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
			strbuf_release(&c);
		} else
			packet_buf_write(&req_buf, "want %s\n", remote_hex);
		fetching++;
	}

	if (!fetching) {
		strbuf_release(&req_buf);
		packet_flush(fd[1]);
		return 1;
	}

326
	if (is_repository_shallow(the_repository))
327
		write_shallow_commits(&req_buf, 1, NULL);
328 329
	if (args->depth > 0)
		packet_buf_write(&req_buf, "deepen %d", args->depth);
330
	if (args->deepen_since) {
331
		timestamp_t max_age = approxidate(args->deepen_since);
332
		packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
333
	}
334 335 336 337 338 339 340
	if (args->deepen_not) {
		int i;
		for (i = 0; i < args->deepen_not->nr; i++) {
			struct string_list_item *s = args->deepen_not->items + i;
			packet_buf_write(&req_buf, "deepen-not %s", s->string);
		}
	}
341 342 343 344
	if (server_supports_filtering && args->filter_options.choice) {
		struct strbuf expanded_filter_spec = STRBUF_INIT;
		expand_list_objects_filter_spec(&args->filter_options,
						&expanded_filter_spec);
345
		packet_buf_write(&req_buf, "filter %s",
346 347 348
				 expanded_filter_spec.buf);
		strbuf_release(&expanded_filter_spec);
	}
349 350 351
	packet_buf_flush(&req_buf);
	state_len = req_buf.len;

352
	if (args->deepen) {
353
		const char *arg;
354
		struct object_id oid;
355 356

		send_request(args, fd[1], &req_buf);
357 358
		while (packet_reader_read(&reader) == PACKET_READ_NORMAL) {
			if (skip_prefix(reader.line, "shallow ", &arg)) {
359
				if (get_oid_hex(arg, &oid))
360
					die(_("invalid shallow line: %s"), reader.line);
361
				register_shallow(the_repository, &oid);
362 363
				continue;
			}
364
			if (skip_prefix(reader.line, "unshallow ", &arg)) {
365
				if (get_oid_hex(arg, &oid))
366
					die(_("invalid unshallow line: %s"), reader.line);
367
				if (!lookup_object(the_repository, oid.hash))
368
					die(_("object not found: %s"), reader.line);
369
				/* make sure that it is parsed as shallow */
370
				if (!parse_object(the_repository, &oid))
371
					die(_("error in object: %s"), reader.line);
372
				if (unregister_shallow(&oid))
373
					die(_("no shallow found: %s"), reader.line);
374 375
				continue;
			}
376
			die(_("expected shallow/unshallow, got %s"), reader.line);
377 378 379 380 381 382 383 384 385 386 387 388 389 390
		}
	} else if (!args->stateless_rpc)
		send_request(args, fd[1], &req_buf);

	if (!args->stateless_rpc) {
		/* If we aren't using the stateless-rpc interface
		 * we don't need to retain the headers.
		 */
		strbuf_setlen(&req_buf, 0);
		state_len = 0;
	}

	flushes = 0;
	retval = -1;
391 392
	if (args->no_dependents)
		goto done;
393
	while ((oid = negotiator->next(negotiator))) {
394 395
		packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
		print_verbose(args, "have %s", oid_to_hex(oid));
396 397 398 399 400 401 402 403
		in_vain++;
		if (flush_at <= ++count) {
			int ack;

			packet_buf_flush(&req_buf);
			send_request(args, fd[1], &req_buf);
			strbuf_setlen(&req_buf, state_len);
			flushes++;
404
			flush_at = next_flush(args->stateless_rpc, count);
405 406 407 408 409 410 411 412

			/*
			 * We keep one window "ahead" of the other side, and
			 * will wait for an ACK only on the next one
			 */
			if (!args->stateless_rpc && count == INITIAL_FLUSH)
				continue;

413
			consume_shallow_list(args, &reader);
414
			do {
415
				ack = get_ack(&reader, result_oid);
416
				if (ack)
417
					print_verbose(args, _("got %s %d %s"), "ack",
418
						      ack, oid_to_hex(result_oid));
419 420 421 422 423 424 425 426 427 428
				switch (ack) {
				case ACK:
					flushes = 0;
					multi_ack = 0;
					retval = 0;
					goto done;
				case ACK_common:
				case ACK_ready:
				case ACK_continue: {
					struct commit *commit =
429 430
						lookup_commit(the_repository,
							      result_oid);
431
					int was_common;
432

433
					if (!commit)
434
						die(_("invalid commit %s"), oid_to_hex(result_oid));
435
					was_common = negotiator->ack(negotiator, commit);
436 437
					if (args->stateless_rpc
					 && ack == ACK_common
438
					 && !was_common) {
439 440 441 442
						/* We need to replay the have for this object
						 * on the next RPC request so the peer knows
						 * it is in common with us.
						 */
443
						const char *hex = oid_to_hex(result_oid);
444 445
						packet_buf_write(&req_buf, "have %s\n", hex);
						state_len = req_buf.len;
446 447 448 449 450 451 452 453 454
						/*
						 * Reset in_vain because an ack
						 * for this commit has not been
						 * seen.
						 */
						in_vain = 0;
					} else if (!args->stateless_rpc
						   || ack != ACK_common)
						in_vain = 0;
455 456
					retval = 0;
					got_continue = 1;
457
					if (ack == ACK_ready)
458 459 460 461 462 463 464
						got_ready = 1;
					break;
					}
				}
			} while (ack);
			flushes--;
			if (got_continue && MAX_IN_VAIN < in_vain) {
465
				print_verbose(args, _("giving up"));
466 467
				break; /* give up */
			}
468 469
			if (got_ready)
				break;
470 471 472 473 474 475 476
		}
	}
done:
	if (!got_ready || !no_done) {
		packet_buf_write(&req_buf, "done\n");
		send_request(args, fd[1], &req_buf);
	}
477
	print_verbose(args, _("done"));
478 479 480 481 482 483
	if (retval != 0) {
		multi_ack = 0;
		flushes++;
	}
	strbuf_release(&req_buf);

484
	if (!got_ready || !no_done)
485
		consume_shallow_list(args, &reader);
486
	while (flushes || multi_ack) {
487
		int ack = get_ack(&reader, result_oid);
488
		if (ack) {
489
			print_verbose(args, _("got %s (%d) %s"), "ack",
490
				      ack, oid_to_hex(result_oid));
491 492 493 494 495 496 497 498 499 500 501 502 503
			if (ack == ACK)
				return 0;
			multi_ack = 1;
			continue;
		}
		flushes--;
	}
	/* it is no error to fetch into a completely empty repo */
	return count ? retval : 0;
}

static struct commit_list *complete;

504
static int mark_complete(const struct object_id *oid)
505
{
506
	struct object *o = parse_object(the_repository, oid);
507 508 509 510 511 512

	while (o && o->type == OBJ_TAG) {
		struct tag *t = (struct tag *) o;
		if (!t->tagged)
			break; /* broken repository */
		o->flags |= COMPLETE;
513
		o = parse_object(the_repository, &t->tagged->oid);
514 515 516 517 518
	}
	if (o && o->type == OBJ_COMMIT) {
		struct commit *commit = (struct commit *)o;
		if (!(commit->object.flags & COMPLETE)) {
			commit->object.flags |= COMPLETE;
519
			commit_list_insert(commit, &complete);
520 521 522 523 524
		}
	}
	return 0;
}

525 526 527
static int mark_complete_oid(const char *refname, const struct object_id *oid,
			     int flag, void *cb_data)
{
528
	return mark_complete(oid);
529 530
}

531
static void mark_recent_complete_commits(struct fetch_pack_args *args,
532
					 timestamp_t cutoff)
533 534
{
	while (complete && cutoff <= complete->item->date) {
535
		print_verbose(args, _("Marking %s as complete"),
536
			      oid_to_hex(&complete->item->object.oid));
537 538 539 540
		pop_most_recent_commit(&complete, COMPLETE);
	}
}

541 542 543 544 545 546
static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
{
	for (; refs; refs = refs->next)
		oidset_insert(oids, &refs->old_oid);
}

547 548 549 550 551 552 553 554 555 556
static int is_unmatched_ref(const struct ref *ref)
{
	struct object_id oid;
	const char *p;
	return	ref->match_status == REF_NOT_MATCHED &&
		!parse_oid_hex(ref->name, &oid, &p) &&
		*p == '\0' &&
		oideq(&oid, &ref->old_oid);
}

557
static void filter_refs(struct fetch_pack_args *args,
558 559
			struct ref **refs,
			struct ref **sought, int nr_sought)
560 561 562
{
	struct ref *newlist = NULL;
	struct ref **newtail = &newlist;
563
	struct ref *unmatched = NULL;
564
	struct ref *ref, *next;
565
	struct oidset tip_oids = OIDSET_INIT;
566
	int i;
567 568
	int strict = !(allow_unadvertised_object_request &
		       (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
569

570
	i = 0;
571 572 573
	for (ref = *refs; ref; ref = next) {
		int keep = 0;
		next = ref->next;
574

575
		if (starts_with(ref->name, "refs/") &&
576 577 578 579 580 581 582 583
		    check_refname_format(ref->name, 0)) {
			/*
			 * trash or a peeled value; do not even add it to
			 * unmatched list
			 */
			free_one_ref(ref);
			continue;
		} else {
584 585
			while (i < nr_sought) {
				int cmp = strcmp(ref->name, sought[i]->name);
586 587 588 589
				if (cmp < 0)
					break; /* definitely do not have it */
				else if (cmp == 0) {
					keep = 1; /* definitely have it */
590
					sought[i]->match_status = REF_MATCHED;
591
				}
592
				i++;
593 594
			}

595 596 597 598
			if (!keep && args->fetch_all &&
			    (!args->deepen || !starts_with(ref->name, "refs/tags/")))
				keep = 1;
		}
599 600 601 602 603 604

		if (keep) {
			*newtail = ref;
			ref->next = NULL;
			newtail = &ref->next;
		} else {
605 606
			ref->next = unmatched;
			unmatched = ref;
607 608 609
		}
	}

610 611 612 613 614 615 616 617 618 619 620 621
	if (strict) {
		for (i = 0; i < nr_sought; i++) {
			ref = sought[i];
			if (!is_unmatched_ref(ref))
				continue;

			add_refs_to_oidset(&tip_oids, unmatched);
			add_refs_to_oidset(&tip_oids, newlist);
			break;
		}
	}

622
	/* Append unmatched requests to the list */
623 624
	for (i = 0; i < nr_sought; i++) {
		ref = sought[i];
625
		if (!is_unmatched_ref(ref))
626
			continue;
627

628
		if (!strict || oidset_contains(&tip_oids, &ref->old_oid)) {
629
			ref->match_status = REF_MATCHED;
630 631
			*newtail = copy_ref(ref);
			newtail = &(*newtail)->next;
632 633
		} else {
			ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
634 635
		}
	}
636 637

	oidset_clear(&tip_oids);
Jeff King's avatar
Jeff King committed
638
	free_refs(unmatched);
639

640 641 642
	*refs = newlist;
}

643
static void mark_alternate_complete(struct fetch_negotiator *unused,
644
				    struct object *obj)
645
{
646
	mark_complete(&obj->oid);
647 648
}

649 650 651 652 653
struct loose_object_iter {
	struct oidset *loose_object_set;
	struct ref *refs;
};

654 655 656 657 658 659 660 661 662 663
/*
 * Mark recent commits available locally and reachable from a local ref as
 * COMPLETE. If args->no_dependents is false, also mark COMPLETE remote refs as
 * COMMON_REF (otherwise, we are not planning to participate in negotiation, and
 * thus do not need COMMON_REF marks).
 *
 * The cutoff time for recency is determined by this heuristic: it is the
 * earliest commit time of the objects in refs that are commits and that we know
 * the commit time of.
 */
664
static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
665
					 struct fetch_pack_args *args,
666
					 struct ref **refs)
667 668
{
	struct ref *ref;
669
	int old_save_commit_buffer = save_commit_buffer;
670
	timestamp_t cutoff = 0;
671 672 673 674 675 676

	save_commit_buffer = 0;

	for (ref = *refs; ref; ref = ref->next) {
		struct object *o;

677 678
		if (!has_object_file_with_flags(&ref->old_oid,
						OBJECT_INFO_QUICK))
679
			continue;
680
		o = parse_object(the_repository, &ref->old_oid);
681 682 683 684 685 686 687 688 689 690 691 692 693 694
		if (!o)
			continue;

		/* We already have it -- which may mean that we were
		 * in sync with the other side at some time after
		 * that (it is OK if we guess wrong here).
		 */
		if (o->type == OBJ_COMMIT) {
			struct commit *commit = (struct commit *)o;
			if (!cutoff || cutoff < commit->date)
				cutoff = commit->date;
		}
	}

695 696 697 698 699 700 701
	if (!args->deepen) {
		for_each_ref(mark_complete_oid, NULL);
		for_each_cached_alternate(NULL, mark_alternate_complete);
		commit_list_sort_by_date(&complete);
		if (cutoff)
			mark_recent_complete_commits(args, cutoff);
	}
702

703 704 705 706 707 708 709 710 711
	/*
	 * Mark all complete remote refs as common refs.
	 * Don't mark them common yet; the server has to be told so first.
	 */
	for (ref = *refs; ref; ref = ref->next) {
		struct object *o = deref_tag(the_repository,
					     lookup_object(the_repository,
					     ref->old_oid.hash),
					     NULL, 0);
712

713 714
		if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
			continue;
715

716 717
		negotiator->known_common(negotiator,
					 (struct commit *)o);
718 719
	}

720 721 722 723 724 725 726 727 728 729 730 731
	save_commit_buffer = old_save_commit_buffer;
}

/*
 * Returns 1 if every object pointed to by the given remote refs is available
 * locally and reachable from a local ref, and 0 otherwise.
 */
static int everything_local(struct fetch_pack_args *args,
			    struct ref **refs)
{
	struct ref *ref;
	int retval;
732 733

	for (retval = 1, ref = *refs; ref ; ref = ref->next) {
734
		const struct object_id *remote = &ref->old_oid;
735 736
		struct object *o;

737
		o = lookup_object(the_repository, remote->hash);
738 739
		if (!o || !(o->flags & COMPLETE)) {
			retval = 0;
740
			print_verbose(args, "want %s (%s)", oid_to_hex(remote),
741
				      ref->name);
742 743
			continue;
		}
744
		print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
745
			      ref->name);
746
	}
747

748 749 750 751 752 753
	return retval;
}

static int sideband_demux(int in, int out, void *data)
{
	int *xd = data;
754
	int ret;
755

756
	ret = recv_sideband("fetch-pack", xd[0], out);
757 758 759 760 761 762 763 764 765
	close(out);
	return ret;
}

static int get_pack(struct fetch_pack_args *args,
		    int xd[2], char **pack_lockfile)
{
	struct async demux;
	int do_keep = args->keep_pack;
766 767 768
	const char *cmd_name;
	struct pack_header header;
	int pass_header = 0;
769
	struct child_process cmd = CHILD_PROCESS_INIT;
770
	int ret;
771 772 773 774 775 776 777 778 779 780

	memset(&demux, 0, sizeof(demux));
	if (use_sideband) {
		/* xd[] is talking with upload-pack; subprocess reads from
		 * xd[0], spits out band#2 to stderr, and feeds us band#1
		 * through demux->out.
		 */
		demux.proc = sideband_demux;
		demux.data = xd;
		demux.out = -1;
781
		demux.isolate_sigpipe = 1;
782
		if (start_async(&demux))
783
			die(_("fetch-pack: unable to fork off sideband demultiplexer"));
784 785 786 787 788 789 790
	}
	else
		demux.out = xd[0];

	if (!args->keep_pack && unpack_limit) {

		if (read_pack_header(demux.out, &header))
791
			die(_("protocol error: bad pack header"));
792
		pass_header = 1;
793 794 795 796 797 798
		if (ntohl(header.hdr_entries) < unpack_limit)
			do_keep = 0;
		else
			do_keep = 1;
	}

799
	if (alternate_shallow_file) {
800 801
		argv_array_push(&cmd.args, "--shallow-file");
		argv_array_push(&cmd.args, alternate_shallow_file);
802 803
	}

804
	if (do_keep || args->from_promisor) {
805 806
		if (pack_lockfile)
			cmd.out = -1;
807 808 809
		cmd_name = "index-pack";
		argv_array_push(&cmd.args, cmd_name);
		argv_array_push(&cmd.args, "--stdin");
810
		if (!args->quiet && !args->no_progress)
811
			argv_array_push(&cmd.args, "-v");
812
		if (args->use_thin_pack)
813
			argv_array_push(&cmd.args, "--fix-thin");
814
		if (do_keep && (args->lock_pack || unpack_limit)) {
815
			char hostname[HOST_NAME_MAX + 1];
816
			if (xgethostname(hostname, sizeof(hostname)))
817 818 819 820
				xsnprintf(hostname, sizeof(hostname), "localhost");
			argv_array_pushf(&cmd.args,
					"--keep=fetch-pack %"PRIuMAX " on %s",
					(uintmax_t)getpid(), hostname);
821
		}
822
		if (args->check_self_contained_and_connected)
823
			argv_array_push(&cmd.args, "--check-self-contained-and-connected");
824 825
		if (args->from_promisor)
			argv_array_push(&cmd.args, "--promisor");
826 827
	}
	else {
828 829
		cmd_name = "unpack-objects";
		argv_array_push(&cmd.args, cmd_name);
830
		if (args->quiet || args->no_progress)
831
			argv_array_push(&cmd.args, "-q");
832
		args->check_self_contained_and_connected = 0;
833
	}
834 835 836 837 838

	if (pass_header)
		argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
				 ntohl(header.hdr_version),
				 ntohl(header.hdr_entries));
839 840 841 842
	if (fetch_fsck_objects >= 0
	    ? fetch_fsck_objects
	    : transfer_fsck_objects >= 0
	    ? transfer_fsck_objects
843 844 845 846 847 848 849 850 851
	    : 0) {
		if (args->from_promisor)
			/*
			 * We cannot use --strict in index-pack because it
			 * checks both broken objects and links, but we only
			 * want to check for broken objects.
			 */
			argv_array_push(&cmd.args, "--fsck-objects");
		else
852 853
			argv_array_pushf(&cmd.args, "--strict%s",
					 fsck_msg_types.buf);
854
	}
855 856 857 858

	cmd.in = demux.out;
	cmd.git_cmd = 1;
	if (start_command(&cmd))
859
		die(_("fetch-pack: unable to fork off %s"), cmd_name);
860 861 862 863 864
	if (do_keep && pack_lockfile) {
		*pack_lockfile = index_pack_lockfile(cmd.out);
		close(cmd.out);
	}

865 866 867 868
	if (!use_sideband)
		/* Closed by start_command() */
		xd[0] = -1;

869 870 871 872 873 874
	ret = finish_command(&cmd);
	if (!ret || (args->check_self_contained_and_connected && ret == 1))
		args->self_contained_and_connected =
			args->check_self_contained_and_connected &&
			ret == 0;
	else
875
		die(_("%s failed"), cmd_name);
876
	if (use_sideband && finish_async(&demux))
877
		die(_("error in sideband demultiplexer"));
878 879 880
	return 0;
}

881 882 883 884 885 886 887
static int cmp_ref_by_name(const void *a_, const void *b_)
{
	const struct ref *a = *((const struct ref **)a_);
	const struct ref *b = *((const struct ref **)b_);
	return strcmp(a->name, b->name);
}

888 889 890
static struct ref *do_fetch_pack(struct fetch_pack_args *args,
				 int fd[2],
				 const struct ref *orig_ref,
891
				 struct ref **sought, int nr_sought,
892
				 struct shallow_info *si,
893 894 895
				 char **pack_lockfile)
{
	struct ref *ref = copy_ref_list(orig_ref);
896
	struct object_id oid;
897 898
	const char *agent_feature;
	int agent_len;
899
	struct fetch_negotiator negotiator;
900
	fetch_negotiator_init(&negotiator, negotiation_algorithm);
901 902

	sort_ref_list(&ref, ref_compare_name);
René Scharfe's avatar
René Scharfe committed
903
	QSORT(sought, nr_sought, cmp_ref_by_name);
904

905
	if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
906
		die(_("Server does not support shallow clients"));
907
	if (args->depth > 0 || args->deepen_since || args->deepen_not)
908
		args->deepen = 1;
909
	if (server_supports("multi_ack_detailed")) {
910
		print_verbose(args, _("Server supports multi_ack_detailed"));
911 912
		multi_ack = 2;
		if (server_supports("no-done")) {
913
			print_verbose(args, _("Server supports no-done"));
914 915 916 917 918
			if (args->stateless_rpc)
				no_done = 1;
		}
	}
	else if (server_supports("multi_ack")) {
919
		print_verbose(args, _("Server supports multi_ack"));
920 921 922
		multi_ack = 1;
	}
	if (server_supports("side-band-64k")) {
923
		print_verbose(args, _("Server supports side-band-64k"));
924 925 926
		use_sideband = 2;
	}
	else if (server_supports("side-band")) {
927
		print_verbose(args, _("Server supports side-band"));
928 929
		use_sideband = 1;
	}
930
	if (server_supports("allow-tip-sha1-in-want")) {
931
		print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
932
		allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
933
	}
934
	if (server_supports("allow-reachable-sha1-in-want")) {
935
		print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
936 937
		allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
	}
938 939 940 941 942 943
	if (!server_supports("thin-pack"))
		args->use_thin_pack = 0;
	if (!server_supports("no-progress"))
		args->no_progress = 0;
	if (!server_supports("include-tag"))
		args->include_tag = 0;
944
	if (server_supports("ofs-delta"))
945
		print_verbose(args, _("Server supports ofs-delta"));
946
	else
947 948
		prefer_ofs_delta = 0;

949 950 951 952 953 954 955
	if (server_supports("filter")) {
		server_supports_filtering = 1;
		print_verbose(args, _("Server supports filter"));
	} else if (args->filter_options.choice) {
		warning("filtering not recognized by server, ignoring");
	}

956 957
	if ((agent_feature = server_feature_value("agent", &agent_len))) {
		agent_supported = 1;
958
		if (agent_len)
959
			print_verbose(args, _("Server version is %.*s"),
960
				      agent_len, agent_feature);
961
	}
962 963 964 965
	if (server_supports("deepen-since"))
		deepen_since_ok = 1;
	else if (args->deepen_since)
		die(_("Server does not support --shallow-since"));
966 967 968 969
	if (server_supports("deepen-not"))
		deepen_not_ok = 1;
	else if (args->deepen_not)
		die(_("Server does not support --shallow-exclude"));
970 971
	if (!server_supports("deepen-relative") && args->deepen_relative)
		die(_("Server does not support --deepen"));
972

973 974 975 976 977 978 979 980 981
	if (!args->no_dependents) {
		mark_complete_and_common_ref(&negotiator, args, &ref);
		filter_refs(args, &ref, sought, nr_sought);
		if (everything_local(args, &ref)) {
			packet_flush(fd[1]);
			goto all_done;
		}
	} else {
		filter_refs(args, &ref, sought, nr_sought);
982
	}
983
	if (find_common(&negotiator, args, fd, &oid, ref) < 0)
984 985 986 987
		if (!args->keep_pack)
			/* When cloning, it is not unusual to have
			 * no common commit.
			 */
988
			warning(_("no common commits"));
989 990 991

	if (args->stateless_rpc)
		packet_flush(fd[1]);
992
	if (args->deepen)
993 994
		setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
					NULL);
995
	else if (si->nr_ours || si->nr_theirs)
996
		alternate_shallow_file = setup_temporary_shallow(si->shallow);
997 998
	else
		alternate_shallow_file = NULL;
999
	if (get_pack(args, fd, pack_lockfile))
1000
		die(_("git fetch-pack: fetch failed."));
1001 1002

 all_done:
1003
	negotiator.release(&negotiator);
1004 1005 1006
	return ref;
}

1007 1008 1009
static void add_shallow_requests(struct strbuf *req_buf,
				 const struct fetch_pack_args *args)
{
1010
	if (is_repository_shallow(the_repository))
1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
		write_shallow_commits(req_buf, 1, NULL);
	if (args->depth > 0)
		packet_buf_write(req_buf, "deepen %d", args->depth);
	if (args->deepen_since) {
		timestamp_t max_age = approxidate(args->deepen_since);
		packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
	}
	if (args->deepen_not) {
		int i;
		for (i = 0; i < args->deepen_not->nr; i++) {
			struct string_list_item *s = args->deepen_not->items + i;
			packet_buf_write(req_buf, "deepen-not %s", s->string);
		}
	}
1025 1026
	if (args->deepen_relative)
		packet_buf_write(req_buf, "deepen-relative\n");
1027 1028
}

1029
static void add_wants(int no_dependents, const struct ref *wants, struct strbuf *req_buf)
1030
{
1031 1032
	int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);

1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
	for ( ; wants ; wants = wants->next) {
		const struct object_id *remote = &wants->old_oid;
		struct object *o;

		/*
		 * If that object is complete (i.e. it is an ancestor of a
		 * local ref), we tell them we have it but do not have to
		 * tell them about its ancestors, which they already know
		 * about.
		 *
		 * We use lookup_object here because we are only
		 * interested in the case we *know* the object is
		 * reachable and we have already scanned it.
1046 1047 1048
		 *
		 * Do this only if args->no_dependents is false (if it is true,
		 * we cannot trust the object flags).
1049
		 */
1050 1051
		if (!no_dependents &&
		    ((o = lookup_object(the_repository, remote->hash)) != NULL) &&
1052 1053 1054 1055
		    (o->flags & COMPLETE)) {
			continue;
		}

1056 1057 1058 1059
		if (!use_ref_in_want || wants->exact_oid)
			packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
		else
			packet_buf_write(req_buf, "want-ref %s\n", wants->name);
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
	}
}

static void add_common(struct strbuf *req_buf, struct oidset *common)
{
	struct oidset_iter iter;
	const struct object_id *oid;
	oidset_iter_init(common, &iter);

	while ((oid = oidset_iter_next(&iter))) {
		packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
	}
}

1074 1075
static int add_haves(struct fetch_negotiator *negotiator,
		     struct strbuf *req_buf,
1076
		     int *haves_to_send, int *in_vain)
1077 1078 1079 1080 1081
{
	int ret = 0;
	int haves_added = 0;
	const struct object_id *oid;

1082
	while ((oid = negotiator->next(negotiator))) {
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
		packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
		if (++haves_added >= *haves_to_send)
			break;
	}

	*in_vain += haves_added;
	if (!haves_added || *in_vain >= MAX_IN_VAIN) {
		/* Send Done */
		packet_buf_write(req_buf, "done\n");
		ret = 1;
	}

	/* Increase haves to send on next round */
	*haves_to_send = next_flush(1, *haves_to_send);

	return ret;
}

1101
static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
1102
			      const struct fetch_pack_args *args,
1103
			      const struct ref *wants, struct oidset *common,
1104 1105
			      int *haves_to_send, int *in_vain,
			      int sideband_all)
1106 1107 1108 1109 1110 1111 1112 1113
{
	int ret = 0;
	struct strbuf req_buf = STRBUF_INIT;

	if (server_supports_v2("fetch", 1))
		packet_buf_write(&req_buf, "command=fetch");
	if (server_supports_v2("agent", 0))
		packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1114 1115 1116 1117 1118 1119 1120
	if (args->server_options && args->server_options->nr &&
	    server_supports_v2("server-option", 1)) {
		int i;
		for (i = 0; i < args->server_options->nr; i++)
			packet_write_fmt(fd_out, "server-option=%s",
					 args->server_options->items[i].string);
	}
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130

	packet_buf_delim(&req_buf);
	if (args->use_thin_pack)
		packet_buf_write(&req_buf, "thin-pack");
	if (args->no_progress)
		packet_buf_write(&req_buf, "no-progress");
	if (args->include_tag)
		packet_buf_write(&req_buf, "include-tag");
	if (prefer_ofs_delta)
		packet_buf_write(&req_buf, "ofs-delta");
1131 1132
	if (sideband_all)
		packet_buf_write(&req_buf, "sideband-all");
1133

1134 1135 1136
	/* Add shallow-info and deepen request */
	if (server_supports_feature("fetch", "shallow", 0))
		add_shallow_requests(&req_buf, args);
1137
	else if (is_repository_shallow(the_repository) || args->deepen)
1138 1139
		die(_("Server does not support shallow requests"));

1140 1141 1142
	/* Add filter */
	if (server_supports_feature("fetch", "filter", 0) &&
	    args->filter_options.choice) {
1143
		struct strbuf expanded_filter_spec = STRBUF_INIT;
1144
		print_verbose(args, _("Server supports filter"));
1145 1146
		expand_list_objects_filter_spec(&args->filter_options,
						&expanded_filter_spec);
1147
		packet_buf_write(&req_buf, "filter %s",
1148 1149
				 expanded_filter_spec.buf);
		strbuf_release(&expanded_filter_spec);
1150 1151 1152 1153
	} else if (args->filter_options.choice) {
		warning("filtering not recognized by server, ignoring");
	}

1154
	/* add wants */
1155
	add_wants(args->no_dependents, wants, &req_buf);
1156

1157 1158 1159 1160 1161 1162
	if (args->no_dependents) {
		packet_buf_write(&req_buf, "done");
		ret = 1;
	} else {
		/* Add all of the common commits we've found in previous rounds */
		add_common(&req_buf, common);
1163

1164
		/* Add initial haves */
1165
		ret = add_haves(negotiator, &req_buf, haves_to_send, in_vain);
1166
	}
1167 1168 1169

	/* Send request */
	packet_buf_flush(&req_buf);
1170 1171
	if (write_in_full(fd_out, req_buf.buf, req_buf.len) < 0)
		die_errno(_("unable to write request to remote"));
1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188

	strbuf_release(&req_buf);
	return ret;
}

/*
 * Processes a section header in a server's response and checks if it matches
 * `section`.  If the value of `peek` is 1, the header line will be peeked (and
 * not consumed); if 0, the line will be consumed and the function will die if
 * the section header doesn't match what was expected.
 */
static int process_section_header(struct packet_reader *reader,
				  const char *section, int peek)
{
	int ret;

	if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1189
		die(_("error reading section header '%s'"), section);
1190 1191 1192 1193 1194

	ret = !strcmp(reader->line, section);

	if (!peek) {
		if (!ret)
1195
			die(_("expected '%s', received '%s'"),
1196 1197 1198 1199 1200 1201 1202
			    section, reader->line);
		packet_reader_read(reader);
	}

	return ret;
}

1203
static int process_acks(struct fetch_negotiator *negotiator,
1204 1205
			struct packet_reader *reader,
			struct oidset *common)
1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222
{
	/* received */
	int received_ready = 0;
	int received_ack = 0;

	process_section_header(reader, "acknowledgments", 0);
	while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
		const char *arg;

		if (!strcmp(reader->line, "NAK"))
			continue;

		if (skip_prefix(reader->line, "ACK ", &arg)) {
			struct object_id oid;
			if (!get_oid_hex(arg, &oid)) {
				struct commit *commit;
				oidset_insert(common, &oid);
1223
				commit = lookup_commit(the_repository, &oid);
1224
				negotiator->ack(negotiator, commit);
1225 1226 1227 1228 1229 1230 1231 1232 1233
			}
			continue;
		}

		if (!strcmp(reader->line, "ready")) {
			received_ready = 1;
			continue;
		}

1234
		die(_("unexpected acknowledgment line: '%s'"), reader->line);
1235 1236 1237 1238
	}

	if (reader->status != PACKET_READ_FLUSH &&
	    reader->status != PACKET_READ_DELIM)
1239
		die(_("error processing acks: %d"), reader->status);
1240

1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252
	/*
	 * If an "acknowledgments" section is sent, a packfile is sent if and
	 * only if "ready" was sent in this section. The other sections
	 * ("shallow-info" and "wanted-refs") are sent only if a packfile is
	 * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH
	 * otherwise.
	 */
	if (received_ready && reader->status != PACKET_READ_DELIM)
		die(_("expected packfile to be sent after 'ready'"));
	if (!received_ready && reader->status != PACKET_READ_FLUSH)
		die(_("expected no other sections to be sent after no 'ready'"));

1253 1254 1255 1256
	/* return 0 if no common, 1 if there are common, or 2 if ready */
	return received_ready ? 2 : (received_ack ? 1 : 0);
}

1257
static void receive_shallow_info(struct fetch_pack_args *args,
1258 1259 1260
				 struct packet_reader *reader,
				 struct oid_array *shallows,
				 struct shallow_info *si)
1261
{
1262
	int unshallow_received = 0;
1263

1264 1265 1266 1267 1268 1269 1270 1271
	process_section_header(reader, "shallow-info", 0);
	while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
		const char *arg;
		struct object_id oid;

		if (skip_prefix(reader->line, "shallow ", &arg)) {
			if (get_oid_hex(arg, &oid))
				die(_("invalid shallow line: %s"), reader->line);
1272
			oid_array_append(shallows, &oid);
1273 1274 1275 1276 1277
			continue;
		}
		if (skip_prefix(reader->line, "unshallow ", &arg)) {
			if (get_oid_hex(arg, &oid))
				die(_("invalid unshallow line: %s"), reader->line);
1278
			if (!lookup_object(the_repository, oid.hash))
1279 1280
				die(_("object not found: %s"), reader->line);
			/* make sure that it is parsed as shallow */
1281
			if (!parse_object(the_repository, &oid))
1282 1283 1284
				die(_("error in object: %s"), reader->line);
			if (unregister_shallow(&oid))
				die(_("no shallow found: %s"), reader->line);
1285
			unshallow_received = 1;
1286 1287 1288 1289 1290 1291 1292
			continue;
		}
		die(_("expected shallow/unshallow, got %s"), reader->line);
	}

	if (reader->status != PACKET_READ_FLUSH &&
	    reader->status != PACKET_READ_DELIM)
1293
		die(_("error processing shallow info: %d"), reader->status);
1294

1295 1296 1297 1298 1299 1300 1301 1302 1303 1304
	if (args->