http-walker.c 15 KB
Newer Older
1
#include "cache.h"
2
#include "repository.h"
3
#include "commit.h"
4
#include "walker.h"
5
#include "http.h"
6
#include "list.h"
7
#include "transport.h"
8
#include "packfile.h"
9
#include "object-store.h"
10

11
struct alt_base {
12
	char *base;
13 14 15 16 17
	int got_indices;
	struct packed_git *packs;
	struct alt_base *next;
};

18
enum object_request_state {
19 20 21
	WAITING,
	ABORTED,
	ACTIVE,
22
	COMPLETE
23
};
24

25
struct object_request {
26
	struct walker *walker;
27
	struct object_id oid;
28
	struct alt_base *repo;
29
	enum object_request_state state;
30
	struct http_object_request *req;
31
	struct list_head node;
32 33
};

34
struct alternates_request {
35
	struct walker *walker;
36
	const char *base;
37
	struct strbuf *url;
Mike Hommey's avatar
Mike Hommey committed
38
	struct strbuf *buffer;
39 40 41 42
	struct active_request_slot *slot;
	int http_specific;
};

43 44 45 46 47 48
struct walker_data {
	const char *url;
	int got_alternates;
	struct alt_base *alt;
};

49
static LIST_HEAD(object_queue_head);
50

51
static void fetch_alternates(struct walker *walker, const char *base);
52

53
static void process_object_response(void *callback_data);
54

55 56
static void start_object_request(struct walker *walker,
				 struct object_request *obj_req)
57 58
{
	struct active_request_slot *slot;
59
	struct http_object_request *req;
60

61
	req = new_http_object_request(obj_req->repo->base, &obj_req->oid);
62
	if (req == NULL) {
63
		obj_req->state = ABORTED;
64 65
		return;
	}
66
	obj_req->req = req;
67

68
	slot = req->slot;
69
	slot->callback_func = process_object_response;
70
	slot->callback_data = obj_req;
71

72
	/* Try to get the request started, abort the request on error */
73
	obj_req->state = ACTIVE;
74
	if (!start_active_slot(slot)) {
75
		obj_req->state = ABORTED;
76
		release_http_object_request(req);
77
		return;
78 79 80
	}
}

81
static void finish_object_request(struct object_request *obj_req)
82
{
83
	if (finish_http_object_request(obj_req->req))
84 85
		return;

86
	if (obj_req->req->rename == 0)
87
		walker_say(obj_req->walker, "got %s\n", oid_to_hex(&obj_req->oid));
88 89
}

90 91
static void process_object_response(void *callback_data)
{
92 93
	struct object_request *obj_req =
		(struct object_request *)callback_data;
94 95 96
	struct walker *walker = obj_req->walker;
	struct walker_data *data = walker->data;
	struct alt_base *alt = data->alt;
97

98
	process_http_object_request(obj_req->req);
99
	obj_req->state = COMPLETE;
100

101 102 103 104 105
	normalize_curl_result(&obj_req->req->curl_result,
			      obj_req->req->http_code,
			      obj_req->req->errorstr,
			      sizeof(obj_req->req->errorstr));

106
	/* Use alternates if necessary */
107
	if (missing_target(obj_req->req)) {
108
		fetch_alternates(walker, alt->base);
109 110 111
		if (obj_req->repo->next != NULL) {
			obj_req->repo =
				obj_req->repo->next;
112
			release_http_object_request(obj_req->req);
113
			start_object_request(walker, obj_req);
114 115 116 117
			return;
		}
	}

118
	finish_object_request(obj_req);
119 120
}

121
static void release_object_request(struct object_request *obj_req)
122
{
123 124
	if (obj_req->req !=NULL && obj_req->req->localfile != -1)
		error("fd leakage in release: %d", obj_req->req->localfile);
125

126
	list_del(&obj_req->node);
127
	free(obj_req);
128 129
}

130
#ifdef USE_CURL_MULTI
131
static int fill_active_slot(struct walker *walker)
132
{
133
	struct object_request *obj_req;
134
	struct list_head *pos, *tmp, *head = &object_queue_head;
135

136 137
	list_for_each_safe(pos, tmp, head) {
		obj_req = list_entry(pos, struct object_request, node);
138
		if (obj_req->state == WAITING) {
139
			if (has_object_file(&obj_req->oid))
140
				obj_req->state = COMPLETE;
141
			else {
142
				start_object_request(walker, obj_req);
143 144
				return 1;
			}
145
		}
Junio C Hamano's avatar
Junio C Hamano committed
146
	}
147
	return 0;
148
}
149
#endif
150

151
static void prefetch(struct walker *walker, unsigned char *sha1)
152
{
153
	struct object_request *newreq;
154
	struct walker_data *data = walker->data;
155 156

	newreq = xmalloc(sizeof(*newreq));
157
	newreq->walker = walker;
158
	hashcpy(newreq->oid.hash, sha1);
159
	newreq->repo = data->alt;
160
	newreq->state = WAITING;
161
	newreq->req = NULL;
162

163
	http_is_verbose = walker->get_verbosely;
164
	list_add_tail(&newreq->node, &object_queue_head);
165

166
#ifdef USE_CURL_MULTI
167 168
	fill_active_slots();
	step_active_slots();
169
#endif
170 171
}

172 173 174 175 176 177 178
static int is_alternate_allowed(const char *url)
{
	const char *protocols[] = {
		"http", "https", "ftp", "ftps"
	};
	int i;

179 180 181 182 183
	if (http_follow_config != HTTP_FOLLOW_ALWAYS) {
		warning("alternate disabled by http.followRedirects: %s", url);
		return 0;
	}

184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202
	for (i = 0; i < ARRAY_SIZE(protocols); i++) {
		const char *end;
		if (skip_prefix(url, protocols[i], &end) &&
		    starts_with(end, "://"))
			break;
	}

	if (i >= ARRAY_SIZE(protocols)) {
		warning("ignoring alternate with unknown protocol: %s", url);
		return 0;
	}
	if (!is_transport_allowed(protocols[i], 0)) {
		warning("ignoring alternate with restricted protocol: %s", url);
		return 0;
	}

	return 1;
}

203
static void process_alternates_response(void *callback_data)
204
{
205 206
	struct alternates_request *alt_req =
		(struct alternates_request *)callback_data;
207 208
	struct walker *walker = alt_req->walker;
	struct walker_data *cdata = walker->data;
209
	struct active_request_slot *slot = alt_req->slot;
210
	struct alt_base *tail = cdata->alt;
211
	const char *base = alt_req->base;
212
	const char null_byte = '\0';
213 214
	char *data;
	int i = 0;
215

216 217 218
	normalize_curl_result(&slot->curl_result, slot->http_code,
			      curl_errorstr, sizeof(curl_errorstr));

219 220
	if (alt_req->http_specific) {
		if (slot->curl_result != CURLE_OK ||
Mike Hommey's avatar
Mike Hommey committed
221
		    !alt_req->buffer->len) {
222 223 224

			/* Try reusing the slot to get non-http alternates */
			alt_req->http_specific = 0;
225 226 227
			strbuf_reset(alt_req->url);
			strbuf_addf(alt_req->url, "%s/objects/info/alternates",
				    base);
228
			curl_easy_setopt(slot->curl, CURLOPT_URL,
229
					 alt_req->url->buf);
230 231
			active_requests++;
			slot->in_use = 1;
232 233
			if (slot->finished != NULL)
				(*slot->finished) = 0;
234
			if (!start_active_slot(slot)) {
235
				cdata->got_alternates = -1;
236
				slot->in_use = 0;
237 238
				if (slot->finished != NULL)
					(*slot->finished) = 1;
239
			}
240
			return;
241
		}
242
	} else if (slot->curl_result != CURLE_OK) {
243
		if (!missing_target(slot)) {
244
			cdata->got_alternates = -1;
245 246
			return;
		}
247 248
	}

249
	fwrite_buffer((char *)&null_byte, 1, 1, alt_req->buffer);
Mike Hommey's avatar
Mike Hommey committed
250 251
	alt_req->buffer->len--;
	data = alt_req->buffer->buf;
252

Mike Hommey's avatar
Mike Hommey committed
253
	while (i < alt_req->buffer->len) {
254
		int posn = i;
Mike Hommey's avatar
Mike Hommey committed
255
		while (posn < alt_req->buffer->len && data[posn] != '\n')
256 257
			posn++;
		if (data[posn] == '\n') {
258 259 260
			int okay = 0;
			int serverlen = 0;
			struct alt_base *newalt;
261
			if (data[i] == '/') {
262 263
				/*
				 * This counts
264 265 266 267 268 269 270 271 272 273 274
				 * http://git.host/pub/scm/linux.git/
				 * -----------here^
				 * so memcpy(dst, base, serverlen) will
				 * copy up to "...git.host".
				 */
				const char *colon_ss = strstr(base,"://");
				if (colon_ss) {
					serverlen = (strchr(colon_ss + 3, '/')
						     - base);
					okay = 1;
				}
275
			} else if (!memcmp(data + i, "../", 3)) {
276 277
				/*
				 * Relative URL; chop the corresponding
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
				 * number of subpath from base (and ../
				 * from data), and concatenate the result.
				 *
				 * The code first drops ../ from data, and
				 * then drops one ../ from data and one path
				 * from base.  IOW, one extra ../ is dropped
				 * from data than path is dropped from base.
				 *
				 * This is not wrong.  The alternate in
				 *     http://git.host/pub/scm/linux.git/
				 * to borrow from
				 *     http://git.host/pub/scm/linus.git/
				 * is ../../linus.git/objects/.  You need
				 * two ../../ to borrow from your direct
				 * neighbour.
				 */
294 295
				i += 3;
				serverlen = strlen(base);
Junio C Hamano's avatar
Junio C Hamano committed
296
				while (i + 2 < posn &&
297 298 299 300 301 302 303
				       !memcmp(data + i, "../", 3)) {
					do {
						serverlen--;
					} while (serverlen &&
						 base[serverlen - 1] != '/');
					i += 3;
				}
304
				/* If the server got removed, give up. */
Junio C Hamano's avatar
Junio C Hamano committed
305
				okay = strchr(base, ':') - base + 3 <
306
				       serverlen;
307
			} else if (alt_req->http_specific) {
308 309 310 311 312 313 314 315
				char *colon = strchr(data + i, ':');
				char *slash = strchr(data + i, '/');
				if (colon && slash && colon < data + posn &&
				    slash < data + posn && colon < slash) {
					okay = 1;
				}
			}
			if (okay) {
316 317
				struct strbuf target = STRBUF_INIT;
				strbuf_add(&target, base, serverlen);
318 319 320 321 322 323 324
				strbuf_add(&target, data + i, posn - i);
				if (!strbuf_strip_suffix(&target, "objects")) {
					warning("ignoring alternate that does"
						" not end in 'objects': %s",
						target.buf);
					strbuf_release(&target);
				} else if (is_alternate_allowed(target.buf)) {
325 326 327 328 329 330 331 332 333 334 335
					warning("adding alternate object store: %s",
						target.buf);
					newalt = xmalloc(sizeof(*newalt));
					newalt->next = NULL;
					newalt->base = strbuf_detach(&target, NULL);
					newalt->got_indices = 0;
					newalt->packs = NULL;

					while (tail->next != NULL)
						tail = tail->next;
					tail->next = newalt;
336 337
				} else {
					strbuf_release(&target);
338
				}
339 340 341 342
			}
		}
		i = posn + 1;
	}
343

344
	cdata->got_alternates = 1;
345 346
}

347
static void fetch_alternates(struct walker *walker, const char *base)
348
{
Mike Hommey's avatar
Mike Hommey committed
349
	struct strbuf buffer = STRBUF_INIT;
350
	struct strbuf url = STRBUF_INIT;
351
	struct active_request_slot *slot;
352
	struct alternates_request alt_req;
353
	struct walker_data *cdata = walker->data;
354

355 356 357 358 359
	/*
	 * If another request has already started fetching alternates,
	 * wait for them to arrive and return to processing this request's
	 * curl message
	 */
360
#ifdef USE_CURL_MULTI
361
	while (cdata->got_alternates == 0) {
362
		step_active_slots();
363
	}
364
#endif
365 366

	/* Nothing to do if they've already been fetched */
367
	if (cdata->got_alternates == 1)
368 369 370
		return;

	/* Start the fetch */
371
	cdata->got_alternates = 0;
372

373
	if (walker->get_verbosely)
374
		fprintf(stderr, "Getting alternates list for %s\n", base);
Junio C Hamano's avatar
Junio C Hamano committed
375

376
	strbuf_addf(&url, "%s/objects/info/http-alternates", base);
377

378 379 380 381
	/*
	 * Use a callback to process the result, since another request
	 * may fail and need to have alternates loaded before continuing
	 */
382
	slot = get_active_slot();
383
	slot->callback_func = process_alternates_response;
384
	alt_req.walker = walker;
385 386 387
	slot->callback_data = &alt_req;

	curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
388
	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
389
	curl_easy_setopt(slot->curl, CURLOPT_URL, url.buf);
390 391

	alt_req.base = base;
392
	alt_req.url = &url;
393 394 395 396 397 398 399
	alt_req.buffer = &buffer;
	alt_req.http_specific = 1;
	alt_req.slot = slot;

	if (start_active_slot(slot))
		run_active_slot(slot);
	else
400
		cdata->got_alternates = -1;
401

Mike Hommey's avatar
Mike Hommey committed
402
	strbuf_release(&buffer);
403
	strbuf_release(&url);
404 405
}

406
static int fetch_indices(struct walker *walker, struct alt_base *repo)
407
{
Ray's avatar
Ray committed
408
	int ret;
409

410
	if (repo->got_indices)
411 412
		return 0;

413
	if (walker->get_verbosely)
414
		fprintf(stderr, "Getting pack list for %s\n", repo->base);
Junio C Hamano's avatar
Junio C Hamano committed
415

Ray's avatar
Ray committed
416 417 418 419 420 421 422
	switch (http_get_info_packs(repo->base, &repo->packs)) {
	case HTTP_OK:
	case HTTP_MISSING_TARGET:
		repo->got_indices = 1;
		ret = 0;
		break;
	default:
423
		repo->got_indices = 0;
Ray's avatar
Ray committed
424
		ret = -1;
425
	}
426

427
	return ret;
428 429
}

430
static int http_fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
431 432
{
	struct packed_git *target;
433
	int ret;
434
	struct slot_results results;
435
	struct http_pack_request *preq;
436

437
	if (fetch_indices(walker, repo))
438
		return -1;
439
	target = find_sha1_pack(sha1, repo->packs);
440
	if (!target)
441
		return -1;
442

443
	if (walker->get_verbosely) {
444
		fprintf(stderr, "Getting pack %s\n",
445
			hash_to_hex(target->hash));
446
		fprintf(stderr, " which contains %s\n",
447
			hash_to_hex(sha1));
448 449
	}

450 451 452 453 454
	preq = new_http_pack_request(target, repo->base);
	if (preq == NULL)
		goto abort;
	preq->lst = &repo->packs;
	preq->slot->results = &results;
455

456 457
	if (start_active_slot(preq->slot)) {
		run_active_slot(preq->slot);
458
		if (results.curl_result != CURLE_OK) {
459 460 461
			error("Unable to get pack file %s\n%s", preq->url,
			      curl_errorstr);
			goto abort;
462 463
		}
	} else {
464 465
		error("Unable to start request");
		goto abort;
466 467
	}

468 469
	ret = finish_http_pack_request(preq);
	release_http_pack_request(preq);
470
	if (ret)
471
		return ret;
472

473
	return 0;
474 475 476

abort:
	return -1;
477 478
}

479 480 481 482 483
static void abort_object_request(struct object_request *obj_req)
{
	release_object_request(obj_req);
}

484
static int fetch_object(struct walker *walker, unsigned char *hash)
485
{
486
	char *hex = hash_to_hex(hash);
487
	int ret = 0;
488
	struct object_request *obj_req = NULL;
489
	struct http_object_request *req;
490
	struct list_head *pos, *head = &object_queue_head;
491

492 493
	list_for_each(pos, head) {
		obj_req = list_entry(pos, struct object_request, node);
494
		if (hasheq(obj_req->oid.hash, hash))
495 496
			break;
	}
497
	if (obj_req == NULL)
498 499
		return error("Couldn't find request for %s in the queue", hex);

500
	if (has_object_file(&obj_req->oid)) {
501 502
		if (obj_req->req != NULL)
			abort_http_object_request(obj_req->req);
503
		abort_object_request(obj_req);
504 505 506
		return 0;
	}

507
#ifdef USE_CURL_MULTI
508
	while (obj_req->state == WAITING)
509
		step_active_slots();
510
#else
511
	start_object_request(walker, obj_req);
512
#endif
513

514 515 516 517 518
	/*
	 * obj_req->req might change when fetching alternates in the callback
	 * process_object_response; therefore, the "shortcut" variable, req,
	 * is used only after we're done with slots.
	 */
519
	while (obj_req->state == ACTIVE)
520 521 522
		run_active_slot(obj_req->req->slot);

	req = obj_req->req;
523

524 525 526
	if (req->localfile != -1) {
		close(req->localfile);
		req->localfile = -1;
527
	}
528

529 530
	normalize_curl_result(&req->curl_result, req->http_code,
			      req->errorstr, sizeof(req->errorstr));
531

532
	if (obj_req->state == ABORTED) {
533
		ret = error("Request for %s aborted", hex);
534 535 536
	} else if (req->curl_result != CURLE_OK &&
		   req->http_code != 416) {
		if (missing_target(req))
537 538 539
			ret = -1; /* Be silent, it is probably in a pack. */
		else
			ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
540 541 542
				    req->errorstr, req->curl_result,
				    req->http_code, hex);
	} else if (req->zret != Z_STREAM_END) {
543
		walker->corrupt_object_found++;
544
		ret = error("File %s (%s) corrupt", hex, req->url);
545
	} else if (!oideq(&obj_req->oid, &req->real_oid)) {
546
		ret = error("File %s has bad hash", hex);
547
	} else if (req->rename < 0) {
548
		struct strbuf buf = STRBUF_INIT;
549
		loose_object_path(the_repository, &buf, &req->oid);
550 551
		ret = error("unable to write sha1 filename %s", buf.buf);
		strbuf_release(&buf);
552
	}
553

554
	release_http_object_request(req);
555
	release_object_request(obj_req);
556
	return ret;
557 558
}

559
static int fetch(struct walker *walker, unsigned char *hash)
560
{
561 562
	struct walker_data *data = walker->data;
	struct alt_base *altbase = data->alt;
563

564
	if (!fetch_object(walker, hash))
565
		return 0;
566
	while (altbase) {
567
		if (!http_fetch_pack(walker, altbase, hash))
568
			return 0;
569
		fetch_alternates(walker, data->alt->base);
570 571
		altbase = altbase->next;
	}
572
	return error("Unable to find %s under %s", hash_to_hex(hash),
573
		     data->alt->base);
574 575
}

576
static int fetch_ref(struct walker *walker, struct ref *ref)
577
{
578
	struct walker_data *data = walker->data;
579
	return http_fetch_ref(data->alt->base, ref);
580 581
}

582 583
static void cleanup(struct walker *walker)
{
584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599
	struct walker_data *data = walker->data;
	struct alt_base *alt, *alt_next;

	if (data) {
		alt = data->alt;
		while (alt) {
			alt_next = alt->next;

			free(alt->base);
			free(alt);

			alt = alt_next;
		}
		free(data);
		walker->data = NULL;
	}
600 601
}

602
struct walker *get_http_walker(const char *url)
603
{
604
	char *s;
605 606
	struct walker_data *data = xmalloc(sizeof(struct walker_data));
	struct walker *walker = xmalloc(sizeof(struct walker));
607

608
	data->alt = xmalloc(sizeof(*data->alt));
609
	data->alt->base = xstrdup(url);
610
	for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
611
		*s = 0;
612

613 614 615 616
	data->alt->got_indices = 0;
	data->alt->packs = NULL;
	data->alt->next = NULL;
	data->got_alternates = -1;
617

618 619 620 621 622 623
	walker->corrupt_object_found = 0;
	walker->fetch = fetch;
	walker->fetch_ref = fetch_ref;
	walker->prefetch = prefetch;
	walker->cleanup = cleanup;
	walker->data = data;
624

625 626 627
#ifdef USE_CURL_MULTI
	add_fill_function(walker, (int (*)(void *)) fill_active_slot);
#endif
628

629
	return walker;
630
}