http-push.c 50 KB
Newer Older
1
#include "cache.h"
2
#include "repository.h"
3 4 5
#include "commit.h"
#include "tag.h"
#include "blob.h"
6
#include "http.h"
7
#include "refs.h"
8
#include "diff.h"
9
#include "revision.h"
10
#include "exec-cmd.h"
11
#include "remote.h"
12
#include "list-objects.h"
13
#include "sigchain.h"
14
#include "argv-array.h"
15
#include "packfile.h"
16
#include "object-store.h"
17
#include "commit-reach.h"
18

19 20 21
#ifdef EXPAT_NEEDS_XMLPARSE_H
#include <xmlparse.h>
#else
22
#include <expat.h>
23
#endif
24 25

static const char http_push_usage[] =
26
"git http-push [--all] [--dry-run] [--force] [--verbose] <remote> [<head>...]\n";
27

28 29 30 31 32 33 34 35 36
#ifndef XML_STATUS_OK
enum XML_Status {
  XML_STATUS_OK = 1,
  XML_STATUS_ERROR = 0
};
#define XML_STATUS_OK    1
#define XML_STATUS_ERROR 0
#endif

37
#define PREV_BUF_SIZE 4096
38

39
/* DAV methods */
40 41 42 43 44 45
#define DAV_LOCK "LOCK"
#define DAV_MKCOL "MKCOL"
#define DAV_MOVE "MOVE"
#define DAV_PROPFIND "PROPFIND"
#define DAV_PUT "PUT"
#define DAV_UNLOCK "UNLOCK"
46
#define DAV_DELETE "DELETE"
47 48 49 50 51 52 53 54 55 56 57 58 59

/* DAV lock flags */
#define DAV_PROP_LOCKWR (1u << 0)
#define DAV_PROP_LOCKEX (1u << 1)
#define DAV_LOCK_OK (1u << 2)

/* DAV XML properties */
#define DAV_CTX_LOCKENTRY ".multistatus.response.propstat.prop.supportedlock.lockentry"
#define DAV_CTX_LOCKTYPE_WRITE ".multistatus.response.propstat.prop.supportedlock.lockentry.locktype.write"
#define DAV_CTX_LOCKTYPE_EXCLUSIVE ".multistatus.response.propstat.prop.supportedlock.lockentry.lockscope.exclusive"
#define DAV_ACTIVELOCK_OWNER ".prop.lockdiscovery.activelock.owner.href"
#define DAV_ACTIVELOCK_TIMEOUT ".prop.lockdiscovery.activelock.timeout"
#define DAV_ACTIVELOCK_TOKEN ".prop.lockdiscovery.activelock.locktoken.href"
60 61 62
#define DAV_PROPFIND_RESP ".multistatus.response"
#define DAV_PROPFIND_NAME ".multistatus.response.href"
#define DAV_PROPFIND_COLLECTION ".multistatus.response.propstat.prop.resourcetype.collection"
63 64

/* DAV request body templates */
65 66
#define PROPFIND_SUPPORTEDLOCK_REQUEST "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n<D:propfind xmlns:D=\"DAV:\">\n<D:prop xmlns:R=\"%s\">\n<D:supportedlock/>\n</D:prop>\n</D:propfind>"
#define PROPFIND_ALL_REQUEST "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n<D:propfind xmlns:D=\"DAV:\">\n<D:allprop/>\n</D:propfind>"
67 68
#define LOCK_REQUEST "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n<D:lockinfo xmlns:D=\"DAV:\">\n<D:lockscope><D:exclusive/></D:lockscope>\n<D:locktype><D:write/></D:locktype>\n<D:owner>\n<D:href>mailto:%s</D:href>\n</D:owner>\n</D:lockinfo>"

69 70 71
#define LOCK_TIME 600
#define LOCK_REFRESH 30

72
/* Remember to update object flag allocation in object.h */
73 74 75 76
#define LOCAL    (1u<<16)
#define REMOTE   (1u<<17)
#define FETCHING (1u<<18)
#define PUSHING  (1u<<19)
77

78 79 80
/* We allow "recursive" symbolic refs. Only within reason, though */
#define MAXDEPTH 5

81 82
static int pushing;
static int aborted;
83
static signed char remote_dir_exists[256];
84

85
static int push_verbosely;
86
static int push_all = MATCH_REFS_NONE;
87
static int force_all;
88
static int dry_run;
89
static int helper_status;
90

91
static struct object_list *objects;
92

93
struct repo {
94
	char *url;
95
	char *path;
96
	int path_len;
97 98 99
	int has_info_refs;
	int can_update_info_refs;
	int has_info_packs;
100
	struct packed_git *packs;
101
	struct remote_lock *locks;
102 103
};

104
static struct repo *repo;
105 106

enum transfer_state {
107 108 109
	NEED_FETCH,
	RUN_FETCH_LOOSE,
	RUN_FETCH_PACKED,
110 111 112 113 114
	NEED_PUSH,
	RUN_MKCOL,
	RUN_PUT,
	RUN_MOVE,
	ABORTED,
115
	COMPLETE
116 117
};

118
struct transfer_request {
119
	struct object *obj;
120 121
	char *url;
	char *dest;
122
	struct remote_lock *lock;
123 124 125 126 127 128
	struct curl_slist *headers;
	struct buffer buffer;
	enum transfer_state state;
	CURLcode curl_result;
	char errorstr[CURL_ERROR_SIZE];
	long http_code;
129
	void *userData;
130 131 132 133
	struct active_request_slot *slot;
	struct transfer_request *next;
};

134
static struct transfer_request *request_queue_head;
135

136
struct xml_ctx {
137 138 139 140 141 142 143
	char *name;
	int len;
	char *cdata;
	void (*userFunc)(struct xml_ctx *ctx, int tag_closed);
	void *userData;
};

144
struct remote_lock {
145
	char *url;
Nick Hengeveld's avatar
Nick Hengeveld committed
146
	char *owner;
147
	char *token;
148
	char tmpfile_suffix[41];
Nick Hengeveld's avatar
Nick Hengeveld committed
149 150
	time_t start_time;
	long timeout;
151
	int refreshing;
152 153 154
	struct remote_lock *next;
};

155 156 157 158 159 160 161 162
/* Flags that control remote_ls processing */
#define PROCESS_FILES (1u << 0)
#define PROCESS_DIRS  (1u << 1)
#define RECURSIVE     (1u << 2)

/* Flags that remote_ls passes to callback functions */
#define IS_DIR (1u << 0)

163
struct remote_ls_ctx {
164 165 166 167 168 169 170
	char *path;
	void (*userFunc)(struct remote_ls_ctx *ls);
	void *userData;
	int flags;
	char *dentry_name;
	int dentry_flags;
	struct remote_ls_ctx *parent;
Nick Hengeveld's avatar
Nick Hengeveld committed
171 172
};

173 174 175 176 177 178 179
/* get_dav_token_headers options */
enum dav_header_flag {
	DAV_HEADER_IF = (1u << 0),
	DAV_HEADER_LOCK = (1u << 1),
	DAV_HEADER_TIMEOUT = (1u << 2)
};

180
static char *xml_entities(const char *s)
181 182
{
	struct strbuf buf = STRBUF_INIT;
183
	strbuf_addstr_xml_quoted(&buf, s);
184 185 186
	return strbuf_detach(&buf, NULL);
}

187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
static void curl_setup_http_get(CURL *curl, const char *url,
		const char *custom_req)
{
	curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
	curl_easy_setopt(curl, CURLOPT_URL, url);
	curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, custom_req);
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, fwrite_null);
}

static void curl_setup_http(CURL *curl, const char *url,
		const char *custom_req, struct buffer *buffer,
		curl_write_callback write_fn)
{
	curl_easy_setopt(curl, CURLOPT_PUT, 1);
	curl_easy_setopt(curl, CURLOPT_URL, url);
	curl_easy_setopt(curl, CURLOPT_INFILE, buffer);
	curl_easy_setopt(curl, CURLOPT_INFILESIZE, buffer->buf.len);
	curl_easy_setopt(curl, CURLOPT_READFUNCTION, fread_buffer);
#ifndef NO_CURL_IOCTL
	curl_easy_setopt(curl, CURLOPT_IOCTLFUNCTION, ioctl_buffer);
207
	curl_easy_setopt(curl, CURLOPT_IOCTLDATA, buffer);
208 209 210 211 212 213 214
#endif
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_fn);
	curl_easy_setopt(curl, CURLOPT_NOBODY, 0);
	curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, custom_req);
	curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);
}

215 216
static struct curl_slist *get_dav_token_headers(struct remote_lock *lock, enum dav_header_flag options)
{
217
	struct strbuf buf = STRBUF_INIT;
218
	struct curl_slist *dav_headers = http_copy_default_headers();
219

220
	if (options & DAV_HEADER_IF) {
221 222 223 224
		strbuf_addf(&buf, "If: (<%s>)", lock->token);
		dav_headers = curl_slist_append(dav_headers, buf.buf);
		strbuf_reset(&buf);
	}
225
	if (options & DAV_HEADER_LOCK) {
226 227 228 229
		strbuf_addf(&buf, "Lock-Token: <%s>", lock->token);
		dav_headers = curl_slist_append(dav_headers, buf.buf);
		strbuf_reset(&buf);
	}
230
	if (options & DAV_HEADER_TIMEOUT) {
231 232 233 234 235 236 237 238 239
		strbuf_addf(&buf, "Timeout: Second-%ld", lock->timeout);
		dav_headers = curl_slist_append(dav_headers, buf.buf);
		strbuf_reset(&buf);
	}
	strbuf_release(&buf);

	return dav_headers;
}

240
static void finish_request(struct transfer_request *request);
241
static void release_request(struct transfer_request *request);
242

243
static void process_response(void *callback_data)
244
{
245 246
	struct transfer_request *request =
		(struct transfer_request *)callback_data;
247

248
	finish_request(request);
249 250
}

Junio C Hamano's avatar
Junio C Hamano committed
251
#ifdef USE_CURL_MULTI
252

253 254 255
static void start_fetch_loose(struct transfer_request *request)
{
	struct active_request_slot *slot;
256
	struct http_object_request *obj_req;
257

258
	obj_req = new_http_object_request(repo->url, &request->obj->oid);
259
	if (obj_req == NULL) {
260 261 262 263
		request->state = ABORTED;
		return;
	}

264
	slot = obj_req->slot;
265 266 267
	slot->callback_func = process_response;
	slot->callback_data = request;
	request->slot = slot;
268
	request->userData = obj_req;
269 270 271 272 273

	/* Try to get the request started, abort the request on error */
	request->state = RUN_FETCH_LOOSE;
	if (!start_active_slot(slot)) {
		fprintf(stderr, "Unable to start GET request\n");
274
		repo->can_update_info_refs = 0;
275
		release_http_object_request(obj_req);
276 277 278 279
		release_request(request);
	}
}

Junio C Hamano's avatar
Junio C Hamano committed
280 281
static void start_mkcol(struct transfer_request *request)
{
282
	char *hex = oid_to_hex(&request->obj->oid);
Junio C Hamano's avatar
Junio C Hamano committed
283 284
	struct active_request_slot *slot;

285
	request->url = get_remote_object_url(repo->url, hex, 1);
Junio C Hamano's avatar
Junio C Hamano committed
286 287 288 289

	slot = get_active_slot();
	slot->callback_func = process_response;
	slot->callback_data = request;
290
	curl_setup_http_get(slot->curl, request->url, DAV_MKCOL);
Junio C Hamano's avatar
Junio C Hamano committed
291 292 293 294 295 296 297
	curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, request->errorstr);

	if (start_active_slot(slot)) {
		request->slot = slot;
		request->state = RUN_MKCOL;
	} else {
		request->state = ABORTED;
298
		FREE_AND_NULL(request->url);
Junio C Hamano's avatar
Junio C Hamano committed
299 300 301 302
	}
}
#endif

303 304 305 306 307
static void start_fetch_packed(struct transfer_request *request)
{
	struct packed_git *target;

	struct transfer_request *check_request = request_queue_head;
308
	struct http_pack_request *preq;
309

310
	target = find_sha1_pack(request->obj->oid.hash, repo->packs);
311
	if (!target) {
312
		fprintf(stderr, "Unable to fetch %s, will not be able to update server info refs\n", oid_to_hex(&request->obj->oid));
313
		repo->can_update_info_refs = 0;
314 315 316 317 318
		release_request(request);
		return;
	}

	fprintf(stderr,	"Fetching pack %s\n", sha1_to_hex(target->sha1));
319
	fprintf(stderr, " which contains %s\n", oid_to_hex(&request->obj->oid));
320

321 322 323 324 325 326
	preq = new_http_pack_request(target, repo->url);
	if (preq == NULL) {
		repo->can_update_info_refs = 0;
		return;
	}
	preq->lst = &repo->packs;
327 328 329 330

	/* Make sure there isn't another open request for this pack */
	while (check_request) {
		if (check_request->state == RUN_FETCH_PACKED &&
331 332
		    !strcmp(check_request->url, preq->url)) {
			release_http_pack_request(preq);
333 334 335 336 337 338
			release_request(request);
			return;
		}
		check_request = check_request->next;
	}

339 340 341 342
	preq->slot->callback_func = process_response;
	preq->slot->callback_data = request;
	request->slot = preq->slot;
	request->userData = preq;
343 344 345

	/* Try to get the request started, abort the request on error */
	request->state = RUN_FETCH_PACKED;
346
	if (!start_active_slot(preq->slot)) {
347
		fprintf(stderr, "Unable to start GET request\n");
348
		release_http_pack_request(preq);
349
		repo->can_update_info_refs = 0;
350 351 352 353
		release_request(request);
	}
}

354 355
static void start_put(struct transfer_request *request)
{
356
	char *hex = oid_to_hex(&request->obj->oid);
357
	struct active_request_slot *slot;
358
	struct strbuf buf = STRBUF_INIT;
359
	enum object_type type;
360 361 362 363 364
	char hdr[50];
	void *unpacked;
	unsigned long len;
	int hdrlen;
	ssize_t size;
365
	git_zstream stream;
366

367
	unpacked = read_object_file(&request->obj->oid, &type, &len);
368
	hdrlen = xsnprintf(hdr, sizeof(hdr), "%s %"PRIuMAX , type_name(type), (uintmax_t)len) + 1;
369 370

	/* Set it up */
371
	git_deflate_init(&stream, zlib_compression_level);
372
	size = git_deflate_bound(&stream, len + hdrlen);
Mike Hommey's avatar
Mike Hommey committed
373 374
	strbuf_init(&request->buffer.buf, size);
	request->buffer.posn = 0;
375 376

	/* Compress it */
Mike Hommey's avatar
Mike Hommey committed
377
	stream.next_out = (unsigned char *)request->buffer.buf.buf;
378 379 380 381 382
	stream.avail_out = size;

	/* First header.. */
	stream.next_in = (void *)hdr;
	stream.avail_in = hdrlen;
383 384
	while (git_deflate(&stream, 0) == Z_OK)
		; /* nothing */
385 386 387 388

	/* Then the data itself.. */
	stream.next_in = unpacked;
	stream.avail_in = len;
389 390 391
	while (git_deflate(&stream, Z_FINISH) == Z_OK)
		; /* nothing */
	git_deflate_end(&stream);
392 393
	free(unpacked);

Mike Hommey's avatar
Mike Hommey committed
394
	request->buffer.buf.len = stream.total_out;
395

396
	strbuf_addstr(&buf, "Destination: ");
397
	append_remote_object_url(&buf, repo->url, hex, 0);
398 399
	request->dest = strbuf_detach(&buf, NULL);

400
	append_remote_object_url(&buf, repo->url, hex, 0);
401
	strbuf_add(&buf, request->lock->tmpfile_suffix, 41);
402
	request->url = strbuf_detach(&buf, NULL);
403 404

	slot = get_active_slot();
405 406
	slot->callback_func = process_response;
	slot->callback_data = request;
407 408
	curl_setup_http(slot->curl, request->url, DAV_PUT,
			&request->buffer, fwrite_null);
409 410 411 412 413 414

	if (start_active_slot(slot)) {
		request->slot = slot;
		request->state = RUN_PUT;
	} else {
		request->state = ABORTED;
415
		FREE_AND_NULL(request->url);
416 417 418 419 420 421
	}
}

static void start_move(struct transfer_request *request)
{
	struct active_request_slot *slot;
422
	struct curl_slist *dav_headers = http_copy_default_headers();
423 424

	slot = get_active_slot();
425 426
	slot->callback_func = process_response;
	slot->callback_data = request;
427
	curl_setup_http_get(slot->curl, request->url, DAV_MOVE);
428 429 430 431 432 433 434 435 436
	dav_headers = curl_slist_append(dav_headers, request->dest);
	dav_headers = curl_slist_append(dav_headers, "Overwrite: T");
	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);

	if (start_active_slot(slot)) {
		request->slot = slot;
		request->state = RUN_MOVE;
	} else {
		request->state = ABORTED;
437
		FREE_AND_NULL(request->url);
438 439 440
	}
}

441
static int refresh_lock(struct remote_lock *lock)
442 443
{
	struct active_request_slot *slot;
Nick Hengeveld's avatar
Nick Hengeveld committed
444
	struct slot_results results;
445
	struct curl_slist *dav_headers;
446
	int rc = 0;
447

448
	lock->refreshing = 1;
449

450
	dav_headers = get_dav_token_headers(lock, DAV_HEADER_IF | DAV_HEADER_TIMEOUT);
451

452 453
	slot = get_active_slot();
	slot->results = &results;
454
	curl_setup_http_get(slot->curl, lock->url, DAV_LOCK);
455
	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
456

457 458 459 460 461 462 463 464 465 466
	if (start_active_slot(slot)) {
		run_active_slot(slot);
		if (results.curl_result != CURLE_OK) {
			fprintf(stderr, "LOCK HTTP error %ld\n",
				results.http_code);
		} else {
			lock->start_time = time(NULL);
			rc = 1;
		}
	}
467

468 469
	lock->refreshing = 0;
	curl_slist_free_all(dav_headers);
470

471 472 473
	return rc;
}

474
static void check_locks(void)
475
{
476
	struct remote_lock *lock = repo->locks;
477 478 479 480 481 482 483 484 485 486 487 488 489
	time_t current_time = time(NULL);
	int time_remaining;

	while (lock) {
		time_remaining = lock->start_time + lock->timeout -
			current_time;
		if (!lock->refreshing && time_remaining < LOCK_REFRESH) {
			if (!refresh_lock(lock)) {
				fprintf(stderr,
					"Unable to refresh lock for %s\n",
					lock->url);
				aborted = 1;
				return;
490
			}
491
		}
492
		lock = lock->next;
493
	}
494
}
495

496 497 498 499 500 501 502 503 504 505 506 507 508
static void release_request(struct transfer_request *request)
{
	struct transfer_request *entry = request_queue_head;

	if (request == request_queue_head) {
		request_queue_head = request->next;
	} else {
		while (entry->next != NULL && entry->next != request)
			entry = entry->next;
		if (entry->next == request)
			entry->next = entry->next->next;
	}

509
	free(request->url);
510
	free(request);
511 512
}

513 514
static void finish_request(struct transfer_request *request)
{
515
	struct http_pack_request *preq;
516
	struct http_object_request *obj_req;
517 518

	request->curl_result = request->slot->curl_result;
519 520
	request->http_code = request->slot->http_code;
	request->slot = NULL;
521

522
	/* Keep locks active */
523
	check_locks();
524

525 526
	if (request->headers != NULL)
		curl_slist_free_all(request->headers);
527 528 529

	/* URL is reused for MOVE after PUT */
	if (request->state != RUN_PUT) {
530
		FREE_AND_NULL(request->url);
531
	}
532

533
	if (request->state == RUN_MKCOL) {
534 535
		if (request->curl_result == CURLE_OK ||
		    request->http_code == 405) {
536
			remote_dir_exists[request->obj->oid.hash[0]] = 1;
537 538 539
			start_put(request);
		} else {
			fprintf(stderr, "MKCOL %s failed, aborting (%d/%ld)\n",
540
				oid_to_hex(&request->obj->oid),
541 542 543 544 545 546 547 548 549
				request->curl_result, request->http_code);
			request->state = ABORTED;
			aborted = 1;
		}
	} else if (request->state == RUN_PUT) {
		if (request->curl_result == CURLE_OK) {
			start_move(request);
		} else {
			fprintf(stderr,	"PUT %s failed, aborting (%d/%ld)\n",
550
				oid_to_hex(&request->obj->oid),
551 552 553 554 555 556
				request->curl_result, request->http_code);
			request->state = ABORTED;
			aborted = 1;
		}
	} else if (request->state == RUN_MOVE) {
		if (request->curl_result == CURLE_OK) {
Nick Hengeveld's avatar
Nick Hengeveld committed
557 558
			if (push_verbosely)
				fprintf(stderr, "    sent %s\n",
559
					oid_to_hex(&request->obj->oid));
560 561
			request->obj->flags |= REMOTE;
			release_request(request);
562 563
		} else {
			fprintf(stderr, "MOVE %s failed, aborting (%d/%ld)\n",
564
				oid_to_hex(&request->obj->oid),
565 566 567 568
				request->curl_result, request->http_code);
			request->state = ABORTED;
			aborted = 1;
		}
569
	} else if (request->state == RUN_FETCH_LOOSE) {
570
		obj_req = (struct http_object_request *)request->userData;
571

572 573 574
		if (finish_http_object_request(obj_req) == 0)
			if (obj_req->rename == 0)
				request->obj->flags |= (LOCAL | REMOTE);
575 576

		/* Try fetching packed if necessary */
577 578
		if (request->obj->flags & LOCAL) {
			release_http_object_request(obj_req);
579
			release_request(request);
580
		} else
581 582 583
			start_fetch_packed(request);

	} else if (request->state == RUN_FETCH_PACKED) {
584
		int fail = 1;
585 586 587 588
		if (request->curl_result != CURLE_OK) {
			fprintf(stderr, "Unable to get pack file %s\n%s",
				request->url, curl_errorstr);
		} else {
589 590 591
			preq = (struct http_pack_request *)request->userData;

			if (preq) {
592
				if (finish_http_pack_request(preq) == 0)
593 594
					fail = 0;
				release_http_pack_request(preq);
595 596
			}
		}
597 598
		if (fail)
			repo->can_update_info_refs = 0;
599
		release_request(request);
600 601 602
	}
}

Nick Hengeveld's avatar
Nick Hengeveld committed
603
#ifdef USE_CURL_MULTI
604
static int is_running_queue;
605
static int fill_active_slot(void *unused)
606
{
607
	struct transfer_request *request;
608

609
	if (aborted || !is_running_queue)
610
		return 0;
611

612
	for (request = request_queue_head; request; request = request->next) {
613 614
		if (request->state == NEED_FETCH) {
			start_fetch_loose(request);
615
			return 1;
616
		} else if (pushing && request->state == NEED_PUSH) {
617
			if (remote_dir_exists[request->obj->oid.hash[0]] == 1) {
618
				start_put(request);
619
			} else {
620
				start_mkcol(request);
621
			}
622
			return 1;
623
		}
624
	}
625
	return 0;
626
}
Nick Hengeveld's avatar
Nick Hengeveld committed
627
#endif
628

629 630
static void get_remote_object_list(unsigned char parent);

631 632 633 634 635 636 637 638 639 640
static void add_fetch_request(struct object *obj)
{
	struct transfer_request *request;

	check_locks();

	/*
	 * Don't fetch the object if it's known to exist locally
	 * or is already in the request queue
	 */
641 642
	if (remote_dir_exists[obj->oid.hash[0]] == -1)
		get_remote_object_list(obj->oid.hash[0]);
643 644 645 646 647 648 649 650 651 652 653 654 655
	if (obj->flags & (LOCAL | FETCHING))
		return;

	obj->flags |= FETCHING;
	request = xmalloc(sizeof(*request));
	request->obj = obj;
	request->url = NULL;
	request->lock = NULL;
	request->headers = NULL;
	request->state = NEED_FETCH;
	request->next = request_queue_head;
	request_queue_head = request;

Nick Hengeveld's avatar
Nick Hengeveld committed
656
#ifdef USE_CURL_MULTI
657 658
	fill_active_slots();
	step_active_slots();
Nick Hengeveld's avatar
Nick Hengeveld committed
659
#endif
660 661
}

Nick Hengeveld's avatar
Nick Hengeveld committed
662
static int add_send_request(struct object *obj, struct remote_lock *lock)
663
{
664
	struct transfer_request *request;
665 666
	struct packed_git *target;

667 668 669
	/* Keep locks active */
	check_locks();

670 671 672 673
	/*
	 * Don't push the object if it's known to exist on the remote
	 * or is already in the request queue
	 */
674 675
	if (remote_dir_exists[obj->oid.hash[0]] == -1)
		get_remote_object_list(obj->oid.hash[0]);
676
	if (obj->flags & (REMOTE | PUSHING))
Nick Hengeveld's avatar
Nick Hengeveld committed
677
		return 0;
678
	target = find_sha1_pack(obj->oid.hash, repo->packs);
679 680
	if (target) {
		obj->flags |= REMOTE;
Nick Hengeveld's avatar
Nick Hengeveld committed
681
		return 0;
682
	}
683

684
	obj->flags |= PUSHING;
685
	request = xmalloc(sizeof(*request));
686
	request->obj = obj;
687
	request->url = NULL;
Nick Hengeveld's avatar
Nick Hengeveld committed
688
	request->lock = lock;
689
	request->headers = NULL;
690
	request->state = NEED_PUSH;
691 692
	request->next = request_queue_head;
	request_queue_head = request;
693

Nick Hengeveld's avatar
Nick Hengeveld committed
694
#ifdef USE_CURL_MULTI
695 696
	fill_active_slots();
	step_active_slots();
Nick Hengeveld's avatar
Nick Hengeveld committed
697
#endif
Nick Hengeveld's avatar
Nick Hengeveld committed
698 699

	return 1;
700 701
}

702
static int fetch_indices(void)
703
{
Ray's avatar
Ray committed
704
	int ret;
705 706 707

	if (push_verbosely)
		fprintf(stderr, "Getting pack list\n");
Nick Hengeveld's avatar
Nick Hengeveld committed
708

Ray's avatar
Ray committed
709 710 711 712 713 714 715
	switch (http_get_info_packs(repo->url, &repo->packs)) {
	case HTTP_OK:
	case HTTP_MISSING_TARGET:
		ret = 0;
		break;
	default:
		ret = -1;
716 717
	}

Ray's avatar
Ray committed
718
	return ret;
719 720
}

721
static void one_remote_object(const struct object_id *oid)
722 723 724
{
	struct object *obj;

725
	obj = lookup_object(the_repository, oid->hash);
726
	if (!obj)
727
		obj = parse_object(the_repository, oid);
728 729 730 731 732 733 734

	/* Ignore remote objects that don't exist locally */
	if (!obj)
		return;

	obj->flags |= REMOTE;
	if (!object_list_contains(objects, obj))
735
		object_list_insert(obj, &objects);
736 737
}

738
static void handle_lockprop_ctx(struct xml_ctx *ctx, int tag_closed)
Nick Hengeveld's avatar
Nick Hengeveld committed
739
{
740 741 742 743 744 745 746 747 748 749 750 751 752 753 754
	int *lock_flags = (int *)ctx->userData;

	if (tag_closed) {
		if (!strcmp(ctx->name, DAV_CTX_LOCKENTRY)) {
			if ((*lock_flags & DAV_PROP_LOCKEX) &&
			    (*lock_flags & DAV_PROP_LOCKWR)) {
				*lock_flags |= DAV_LOCK_OK;
			}
			*lock_flags &= DAV_LOCK_OK;
		} else if (!strcmp(ctx->name, DAV_CTX_LOCKTYPE_WRITE)) {
			*lock_flags |= DAV_PROP_LOCKWR;
		} else if (!strcmp(ctx->name, DAV_CTX_LOCKTYPE_EXCLUSIVE)) {
			*lock_flags |= DAV_PROP_LOCKEX;
		}
	}
Nick Hengeveld's avatar
Nick Hengeveld committed
755 756
}

757
static void handle_new_lock_ctx(struct xml_ctx *ctx, int tag_closed)
Nick Hengeveld's avatar
Nick Hengeveld committed
758
{
759
	struct remote_lock *lock = (struct remote_lock *)ctx->userData;
760 761
	git_SHA_CTX sha_ctx;
	unsigned char lock_token_sha1[20];
762 763 764

	if (tag_closed && ctx->cdata) {
		if (!strcmp(ctx->name, DAV_ACTIVELOCK_OWNER)) {
765
			lock->owner = xstrdup(ctx->cdata);
766
		} else if (!strcmp(ctx->name, DAV_ACTIVELOCK_TIMEOUT)) {
767 768 769
			const char *arg;
			if (skip_prefix(ctx->cdata, "Second-", &arg))
				lock->timeout = strtol(arg, NULL, 10);
770
		} else if (!strcmp(ctx->name, DAV_ACTIVELOCK_TOKEN)) {
771
			lock->token = xstrdup(ctx->cdata);
772 773 774 775 776 777 778

			git_SHA1_Init(&sha_ctx);
			git_SHA1_Update(&sha_ctx, lock->token, strlen(lock->token));
			git_SHA1_Final(lock_token_sha1, &sha_ctx);

			lock->tmpfile_suffix[0] = '_';
			memcpy(lock->tmpfile_suffix + 1, sha1_to_hex(lock_token_sha1), 40);
779
		}
Nick Hengeveld's avatar
Nick Hengeveld committed
780 781 782
	}
}

783
static void one_remote_ref(const char *refname);
784

Nick Hengeveld's avatar
Nick Hengeveld committed
785
static void
786
xml_start_tag(void *userData, const char *name, const char **atts)
Nick Hengeveld's avatar
Nick Hengeveld committed
787
{
788
	struct xml_ctx *ctx = (struct xml_ctx *)userData;
789
	const char *c = strchr(name, ':');
790
	int old_namelen, new_len;
791 792 793 794 795 796

	if (c == NULL)
		c = name;
	else
		c++;

797 798
	old_namelen = strlen(ctx->name);
	new_len = old_namelen + strlen(c) + 2;
799 800 801 802

	if (new_len > ctx->len) {
		ctx->name = xrealloc(ctx->name, new_len);
		ctx->len = new_len;
Nick Hengeveld's avatar
Nick Hengeveld committed
803
	}
804
	xsnprintf(ctx->name + old_namelen, ctx->len - old_namelen, ".%s", c);
Nick Hengeveld's avatar
Nick Hengeveld committed
805

806
	FREE_AND_NULL(ctx->cdata);
807 808

	ctx->userFunc(ctx, 0);
Nick Hengeveld's avatar
Nick Hengeveld committed
809 810
}

811
static void
812
xml_end_tag(void *userData, const char *name)
813
{
814
	struct xml_ctx *ctx = (struct xml_ctx *)userData;
815
	const char *c = strchr(name, ':');
816
	char *ep;
817

818 819 820 821 822 823 824 825 826
	ctx->userFunc(ctx, 1);

	if (c == NULL)
		c = name;
	else
		c++;

	ep = ctx->name + strlen(ctx->name) - strlen(c) - 1;
	*ep = 0;
827 828 829
}

static void
830
xml_cdata(void *userData, const XML_Char *s, int len)
831
{
832
	struct xml_ctx *ctx = (struct xml_ctx *)userData;
833
	free(ctx->cdata);
834
	ctx->cdata = xmemdupz(s, len);
835 836
}

Timo Hirvonen's avatar
Timo Hirvonen committed
837
static struct remote_lock *lock_remote(const char *path, long timeout)
838 839
{
	struct active_request_slot *slot;
Nick Hengeveld's avatar
Nick Hengeveld committed
840
	struct slot_results results;
Mike Hommey's avatar
Mike Hommey committed
841 842
	struct buffer out_buffer = { STRBUF_INIT, 0 };
	struct strbuf in_buffer = STRBUF_INIT;
843
	char *url;
844
	char *ep;
845
	char timeout_header[25];
846
	struct remote_lock *lock = NULL;
847
	struct curl_slist *dav_headers = http_copy_default_headers();
848
	struct xml_ctx ctx;
849
	char *escaped;
850

851
	url = xstrfmt("%s%s", repo->url, path);
852

853
	/* Make sure leading directories exist for the remote ref */
854
	ep = strchr(url + strlen(repo->url) + 1, '/');
855
	while (ep) {
856 857
		char saved_character = ep[1];
		ep[1] = '\0';
858
		slot = get_active_slot();
Nick Hengeveld's avatar
Nick Hengeveld committed
859
		slot->results = &results;
860
		curl_setup_http_get(slot->curl, url, DAV_MKCOL);
861 862
		if (start_active_slot(slot)) {
			run_active_slot(slot);
Nick Hengeveld's avatar
Nick Hengeveld committed
863 864
			if (results.curl_result != CURLE_OK &&
			    results.http_code != 405) {
865 866 867 868 869 870 871
				fprintf(stderr,
					"Unable to create branch path %s\n",
					url);
				free(url);
				return NULL;
			}
		} else {
Nick Hengeveld's avatar
Nick Hengeveld committed
872
			fprintf(stderr, "Unable to start MKCOL request\n");
873 874 875
			free(url);
			return NULL;
		}
876
		ep[1] = saved_character;
877 878 879
		ep = strchr(ep + 1, '/');
	}

880
	escaped = xml_entities(ident_default_email());
881 882
	strbuf_addf(&out_buffer.buf, LOCK_REQUEST, escaped);
	free(escaped);
Nick Hengeveld's avatar
Nick Hengeveld committed
883

884
	xsnprintf(timeout_header, sizeof(timeout_header), "Timeout: Second-%ld", timeout);
885 886 887 888
	dav_headers = curl_slist_append(dav_headers, timeout_header);
	dav_headers = curl_slist_append(dav_headers, "Content-Type: text/xml");

	slot = get_active_slot();
Nick Hengeveld's avatar
Nick Hengeveld committed
889
	slot->results = &results;
890
	curl_setup_http(slot->curl, url, DAV_LOCK, &out_buffer, fwrite_buffer);
891
	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
892
	curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer);
893

894 895
	lock = xcalloc(1, sizeof(*lock));
	lock->timeout = -1;
896

897 898
	if (start_active_slot(slot)) {
		run_active_slot(slot);
Nick Hengeveld's avatar
Nick Hengeveld committed
899
		if (results.curl_result == CURLE_OK) {
900 901
			XML_Parser parser = XML_ParserCreate(NULL);
			enum XML_Status result;
902 903 904 905
			ctx.name = xcalloc(10, 1);
			ctx.len = 0;
			ctx.cdata = NULL;
			ctx.userFunc = handle_new_lock_ctx;
906
			ctx.userData = lock;
907 908 909 910
			XML_SetUserData(parser, &ctx);
			XML_SetElementHandler(parser, xml_start_tag,
					      xml_end_tag);
			XML_SetCharacterDataHandler(parser, xml_cdata);
Mike Hommey's avatar
Mike Hommey committed
911 912
			result = XML_Parse(parser, in_buffer.buf,
					   in_buffer.len, 1);
913 914 915 916 917
			free(ctx.name);
			if (result != XML_STATUS_OK) {
				fprintf(stderr, "XML error: %s\n",
					XML_ErrorString(
						XML_GetErrorCode(parser)));
918
				lock->timeout = -1;
919
			}
920
			XML_ParserFree(parser);
921 922 923 924
		} else {
			fprintf(stderr,
				"error: curl result=%d, HTTP code=%ld\n",
				results.curl_result, results.http_code);
925 926
		}
	} else {
Nick Hengeveld's avatar
Nick Hengeveld committed
927
		fprintf(stderr, "Unable to start LOCK request\n");
928 929
	}

930
	curl_slist_free_all(dav_headers);
Mike Hommey's avatar
Mike Hommey committed
931 932
	strbuf_release(&out_buffer.buf);
	strbuf_release(&in_buffer);
Nick Hengeveld's avatar
Nick Hengeveld committed
933

934
	if (lock->token == NULL || lock->timeout <= 0) {
935 936
		free(lock->token);
		free(lock->owner);
937
		free(url);
938
		FREE_AND_NULL(lock);
939
	} else {
940 941
		lock->url = url;
		lock->start_time = time(NULL);
942 943
		lock->next = repo->locks;
		repo->locks = lock;
Nick Hengeveld's avatar
Nick Hengeveld committed
944 945
	}

946
	return lock;
947 948
}

949
static int unlock_remote(struct remote_lock *lock)
950 951
{
	struct active_request_slot *slot;
Nick Hengeveld's avatar
Nick Hengeveld committed
952
	struct slot_results results;
953
	struct remote_lock *prev = repo->locks;
954
	struct curl_slist *dav_headers;
955 956
	int rc = 0;

957
	dav_headers = get_dav_token_headers(lock, DAV_HEADER_LOCK);
958 959

	slot = get_active_slot();
Nick Hengeveld's avatar
Nick Hengeveld committed
960
	slot->results = &results;
961
	curl_setup_http_get(slot->curl, lock->url, DAV_UNLOCK);
962 963 964 965
	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);

	if (start_active_slot(slot)) {
		run_active_slot(slot);
Nick Hengeveld's avatar
Nick Hengeveld committed
966
		if (results.curl_result == CURLE_OK)
967 968
			rc = 1;
		else
969
			fprintf(stderr, "UNLOCK HTTP error %ld\n",
Nick Hengeveld's avatar
Nick Hengeveld committed
970
				results.http_code);
971
	} else {
972
		fprintf(stderr, "Unable to start UNLOCK request\n");
973 974 975
	}

	curl_slist_free_all(dav_headers);
976

977 978
	if (repo->locks == lock) {
		repo->locks = lock->next;
979 980 981 982 983 984 985
	} else {
		while (prev && prev->next != lock)
			prev = prev->next;
		if (prev)
			prev->next = prev->next->next;
	}

986
	free(lock->owner);
987 988 989
	free(lock->url);
	free(lock->token);
	free(lock);
990 991 992 993

	return rc;
}

994 995
static void remove_locks(void)
{
996
	struct remote_lock *lock = repo->locks;
997 998 999

	fprintf(stderr, "Removing remote locks...\n");
	while (lock) {
1000
		struct remote_lock *next = lock->next;
1001
		unlock_remote(lock);
1002
		lock = next;
1003 1004 1005 1006 1007 1008
	}
}

static void remove_locks_on_signal(int signo)
{
	remove_locks();
1009
	sigchain_pop(signo);
1010 1011 1012
	raise(signo);
}

1013 1014 1015
static void remote_ls(const char *path, int flags,
		      void (*userFunc)(struct remote_ls_ctx *ls),
		      void *userData);
1016

1017
/* extract hex from sharded "xx/x{38}" filename */
1018
static int get_oid_hex_from_objpath(const char *path, struct object_id *oid)
1019
{
1020
	if (strlen(path) != GIT_SHA1_HEXSZ + 1)
1021 1022
		return -1;

1023 1024
	if (hex_to_bytes(oid->hash, path, 1))
		return -1;
1025 1026 1027
	path += 2;
	path++; /* skip '/' */

1028
	return hex_to_bytes(oid->hash + 1, path, GIT_SHA1_RAWSZ - 1);
1029 1030
}

1031 1032 1033
static void process_ls_object(struct remote_ls_ctx *ls)
{
	unsigned int *parent = (unsigned int *)ls->userData;
1034
	const char *path = ls->dentry_name;
1035
	struct object_id oid;
1036

1037 1038 1039 1040
	if (!strcmp(ls->path, ls->dentry_name) && (ls->flags & IS_DIR)) {
		remote_dir_exists[*parent] = 1;
		return;
	}
1041

1042
	if (!skip_prefix(path, "objects/", &path) ||
1043
	    get_oid_hex_from_objpath(path, &oid))
1044
		return;
1045

1046
	one_remote_object(&oid);
1047
}
1048

1049 1050 1051 1052 1053 1054
static void process_ls_ref(struct remote_ls_ctx *ls)
{
	if (!strcmp(ls->path, ls->dentry_name) && (ls->dentry_flags & IS_DIR)) {
		fprintf(stderr, "  %s\n", ls->dentry_name);
		return;
	}
1055

1056 1057 1058
	if (!(ls->dentry_flags & IS_DIR))
		one_remote_ref(ls->dentry_name);
}
1059

1060 1061 1062
static void handle_remote_ls_ctx(struct xml_ctx *ctx, int tag_closed)
{
	struct remote_ls_ctx *ls = (struct remote_ls_ctx *)ctx->userData;
1063

1064 1065 1066
	if (tag_closed) {
		if (!strcmp(ctx->name, DAV_PROPFIND_RESP) && ls->dentry_name) {
			if (ls->dentry_flags & IS_DIR) {
1067 1068 1069 1070

				/* ensure collection names end with slash */
				str_end_url_with_slash(ls->dentry_name, &ls->dentry_name);

1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
				if (ls->flags & PROCESS_DIRS) {
					ls->userFunc(ls);
				}
				if (strcmp(ls->dentry_name, ls->path) &&
				    ls->flags & RECURSIVE) {
					remote_ls(ls->dentry_name,
						  ls->flags,
						  ls->userFunc,
						  ls->userData);
				}
			} else if (ls->flags & PROCESS_FILES) {
				ls->userFunc(ls);
1083
			}
1084
		} else if (!strcmp(ctx->name, DAV_PROPFIND_NAME) && ctx->cdata) {
1085 1086 1087 1088 1089 1090 1091 1092
			char *path = ctx->cdata;
			if (*ctx->cdata == 'h') {
				path = strstr(path, "//");
				if (path) {
					path = strchr(path+2, '/');
				}
			}
			if (path) {
1093 1094 1095 1096
				const char *url = repo->url;
				if (repo->path)
					url = repo->path;
				if (strncmp(path, url, repo->path_len))
1097
					error("Parsed path '%s' does not match url: '%s'",
1098 1099 1100 1101 1102
					      path, url);
				else {
					path += repo->path_len;
					ls->dentry_name = xstrdup(path);
				}
1103
			}
1104 1105
		} else if (!strcmp(ctx->name, DAV_PROPFIND_COLLECTION)) {
			ls->dentry_flags |= IS_DIR;
1106
		}
1107
	} else if (!strcmp(ctx->name, DAV_PROPFIND_RESP)) {
1108
		FREE_AND_NULL(ls->dentry_name);
1109
		ls->dentry_flags = 0;
1110 1111 1112
	}
}

1113 1114 1115 1116 1117 1118
/*
 * NEEDSWORK: remote_ls() ignores info/refs on the remote side.  But it
 * should _only_ heed the information from that file, instead of trying to
 * determine the refs from the remote file system (badly: it does not even
 * know about packed-refs).
 */
1119 1120 1121
static void remote_ls(const char *path, int flags,
		      void (*userFunc)(struct remote_ls_ctx *ls),
		      void *userData)
1122
{
1123
	char *url = xstrfmt("%s%s", repo->url, path);
1124
	struct active_request_slot *slot;
Nick Hengeveld's avatar
Nick Hengeveld committed
1125
	struct slot_results results;
Mike Hommey's avatar
Mike Hommey committed
1126 1127
	struct strbuf in_buffer = STRBUF_INIT;
	struct buffer out_buffer = { STRBUF_INIT, 0 };
1128
	struct curl_slist *dav_headers = http_copy_default_headers();
1129
	struct xml_ctx ctx;
1130 1131 1132
	struct remote_ls_ctx ls;

	ls.flags = flags;