http.c 34.5 KB
Newer Older
1
#include "http.h"
2
#include "pack.h"
3
#include "sideband.h"
4
#include "run-command.h"
5
#include "url.h"
6
#include "credential.h"
7
#include "version.h"
8
#include "pkt-line.h"
9

Junio C Hamano's avatar
Junio C Hamano committed
10
int active_requests;
11
int http_is_verbose;
12
size_t http_post_buffer = 16 * LARGE_PACKET_MAX;
13

14 15 16 17
#if LIBCURL_VERSION_NUM >= 0x070a06
#define LIBCURL_CAN_HANDLE_AUTH_ANY
#endif

Ray's avatar
Ray committed
18 19
static int min_curl_sessions = 1;
static int curl_session_count;
20
#ifdef USE_CURL_MULTI
Mike Hommey's avatar
Mike Hommey committed
21 22
static int max_requests = -1;
static CURLM *curlm;
23 24
#endif
#ifndef NO_CURL_EASY_DUPHANDLE
Mike Hommey's avatar
Mike Hommey committed
25
static CURL *curl_default;
26
#endif
27 28 29 30

#define PREV_BUF_SIZE 4096
#define RANGE_HEADER_SIZE 30

31 32
char curl_errorstr[CURL_ERROR_SIZE];

Mike Hommey's avatar
Mike Hommey committed
33
static int curl_ssl_verify = -1;
34
static int curl_ssl_try;
Junio C Hamano's avatar
Junio C Hamano committed
35
static const char *ssl_cert;
36
#if LIBCURL_VERSION_NUM >= 0x070903
Junio C Hamano's avatar
Junio C Hamano committed
37
static const char *ssl_key;
38 39
#endif
#if LIBCURL_VERSION_NUM >= 0x070908
Junio C Hamano's avatar
Junio C Hamano committed
40
static const char *ssl_capath;
41
#endif
Junio C Hamano's avatar
Junio C Hamano committed
42
static const char *ssl_cainfo;
Mike Hommey's avatar
Mike Hommey committed
43 44
static long curl_low_speed_limit = -1;
static long curl_low_speed_time = -1;
Junio C Hamano's avatar
Junio C Hamano committed
45 46
static int curl_ftp_no_epsv;
static const char *curl_http_proxy;
47
static const char *curl_cookie_file;
48
static struct credential http_auth = CREDENTIAL_INIT;
49
static int http_proactive_auth;
50
static const char *user_agent;
51

52 53 54 55 56 57 58 59
#if LIBCURL_VERSION_NUM >= 0x071700
/* Use CURLOPT_KEYPASSWD as is */
#elif LIBCURL_VERSION_NUM >= 0x070903
#define CURLOPT_KEYPASSWD CURLOPT_SSLKEYPASSWD
#else
#define CURLOPT_KEYPASSWD CURLOPT_SSLCERTPASSWD
#endif

60
static struct credential cert_auth = CREDENTIAL_INIT;
61 62
static int ssl_cert_password_required;

Mike Hommey's avatar
Mike Hommey committed
63
static struct curl_slist *pragma_header;
64
static struct curl_slist *no_pragma_header;
65

Junio C Hamano's avatar
Junio C Hamano committed
66
static struct active_request_slot *active_queue_head;
67

68
size_t fread_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
69 70
{
	size_t size = eltsize * nmemb;
71 72
	struct buffer *buffer = buffer_;

Mike Hommey's avatar
Mike Hommey committed
73 74 75
	if (size > buffer->buf.len - buffer->posn)
		size = buffer->buf.len - buffer->posn;
	memcpy(ptr, buffer->buf.buf + buffer->posn, size);
76
	buffer->posn += size;
Mike Hommey's avatar
Mike Hommey committed
77

78 79 80
	return size;
}

81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
#ifndef NO_CURL_IOCTL
curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp)
{
	struct buffer *buffer = clientp;

	switch (cmd) {
	case CURLIOCMD_NOP:
		return CURLIOE_OK;

	case CURLIOCMD_RESTARTREAD:
		buffer->posn = 0;
		return CURLIOE_OK;

	default:
		return CURLIOE_UNKNOWNCMD;
	}
}
#endif

100
size_t fwrite_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
101 102
{
	size_t size = eltsize * nmemb;
103 104
	struct strbuf *buffer = buffer_;

Mike Hommey's avatar
Mike Hommey committed
105
	strbuf_add(buffer, ptr, size);
106 107 108
	return size;
}

109
size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf)
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
{
	return eltsize * nmemb;
}

#ifdef USE_CURL_MULTI
static void process_curl_messages(void)
{
	int num_messages;
	struct active_request_slot *slot;
	CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);

	while (curl_message != NULL) {
		if (curl_message->msg == CURLMSG_DONE) {
			int curl_result = curl_message->data.result;
			slot = active_queue_head;
			while (slot != NULL &&
			       slot->curl != curl_message->easy_handle)
				slot = slot->next;
			if (slot != NULL) {
				curl_multi_remove_handle(curlm, slot->curl);
				slot->curl_result = curl_result;
				finish_active_slot(slot);
			} else {
				fprintf(stderr, "Received DONE message for unknown request!\n");
			}
		} else {
			fprintf(stderr, "Unknown CURL message received: %d\n",
				(int)curl_message->msg);
		}
		curl_message = curl_multi_info_read(curlm, &num_messages);
	}
}
#endif

144
static int http_options(const char *var, const char *value, void *cb)
145 146
{
	if (!strcmp("http.sslverify", var)) {
147
		curl_ssl_verify = git_config_bool(var, value);
148 149
		return 0;
	}
150 151
	if (!strcmp("http.sslcert", var))
		return git_config_string(&ssl_cert, var, value);
152
#if LIBCURL_VERSION_NUM >= 0x070903
153 154
	if (!strcmp("http.sslkey", var))
		return git_config_string(&ssl_key, var, value);
155 156
#endif
#if LIBCURL_VERSION_NUM >= 0x070908
157 158
	if (!strcmp("http.sslcapath", var))
		return git_config_string(&ssl_capath, var, value);
159
#endif
160 161
	if (!strcmp("http.sslcainfo", var))
		return git_config_string(&ssl_cainfo, var, value);
162 163 164 165 166
	if (!strcmp("http.sslcertpasswordprotected", var)) {
		if (git_config_bool(var, value))
			ssl_cert_password_required = 1;
		return 0;
	}
167 168 169 170
	if (!strcmp("http.ssltry", var)) {
		curl_ssl_try = git_config_bool(var, value);
		return 0;
	}
Ray's avatar
Ray committed
171 172 173 174 175 176 177 178
	if (!strcmp("http.minsessions", var)) {
		min_curl_sessions = git_config_int(var, value);
#ifndef USE_CURL_MULTI
		if (min_curl_sessions > 1)
			min_curl_sessions = 1;
#endif
		return 0;
	}
Junio C Hamano's avatar
Junio C Hamano committed
179
#ifdef USE_CURL_MULTI
180
	if (!strcmp("http.maxrequests", var)) {
181
		max_requests = git_config_int(var, value);
182 183 184 185
		return 0;
	}
#endif
	if (!strcmp("http.lowspeedlimit", var)) {
186
		curl_low_speed_limit = (long)git_config_int(var, value);
187 188 189
		return 0;
	}
	if (!strcmp("http.lowspeedtime", var)) {
190
		curl_low_speed_time = (long)git_config_int(var, value);
191 192 193
		return 0;
	}

194 195 196 197
	if (!strcmp("http.noepsv", var)) {
		curl_ftp_no_epsv = git_config_bool(var, value);
		return 0;
	}
198 199
	if (!strcmp("http.proxy", var))
		return git_config_string(&curl_http_proxy, var, value);
200

201 202 203
	if (!strcmp("http.cookiefile", var))
		return git_config_string(&curl_cookie_file, var, value);

204 205 206 207 208 209 210
	if (!strcmp("http.postbuffer", var)) {
		http_post_buffer = git_config_int(var, value);
		if (http_post_buffer < LARGE_PACKET_MAX)
			http_post_buffer = LARGE_PACKET_MAX;
		return 0;
	}

211 212 213
	if (!strcmp("http.useragent", var))
		return git_config_string(&user_agent, var, value);

214
	/* Fall back on the default ones */
215
	return git_default_config(var, value, cb);
216 217
}

218 219
static void init_curl_http_auth(CURL *result)
{
220 221 222 223 224 225 226 227 228 229
	if (!http_auth.username)
		return;

	credential_fill(&http_auth);

#if LIBCURL_VERSION_NUM >= 0x071301
	curl_easy_setopt(result, CURLOPT_USERNAME, http_auth.username);
	curl_easy_setopt(result, CURLOPT_PASSWORD, http_auth.password);
#else
	{
230
		static struct strbuf up = STRBUF_INIT;
231 232 233 234 235 236 237 238 239
		/*
		 * Note that we assume we only ever have a single set of
		 * credentials in a given program run, so we do not have
		 * to worry about updating this buffer, only setting its
		 * initial value.
		 */
		if (!up.len)
			strbuf_addf(&up, "%s:%s",
				http_auth.username, http_auth.password);
240
		curl_easy_setopt(result, CURLOPT_USERPWD, up.buf);
241
	}
242
#endif
243 244
}

245 246 247 248
static int has_cert_password(void)
{
	if (ssl_cert == NULL || ssl_cert_password_required != 1)
		return 0;
249 250
	if (!cert_auth.password) {
		cert_auth.protocol = xstrdup("cert");
251
		cert_auth.username = xstrdup("");
252 253 254 255
		cert_auth.path = xstrdup(ssl_cert);
		credential_fill(&cert_auth);
	}
	return 1;
256 257
}

Junio C Hamano's avatar
Junio C Hamano committed
258
static CURL *get_curl_handle(void)
259
{
Junio C Hamano's avatar
Junio C Hamano committed
260
	CURL *result = curl_easy_init();
261

262 263 264 265 266 267 268 269 270 271
	if (!curl_ssl_verify) {
		curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
		curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
	} else {
		/* Verify authenticity of the peer's certificate */
		curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1);
		/* The name in the cert must match whom we tried to connect */
		curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2);
	}

272 273 274
#if LIBCURL_VERSION_NUM >= 0x070907
	curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
#endif
275
#ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
Junio C Hamano's avatar
Junio C Hamano committed
276
	curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
277
#endif
278

279 280 281
	if (http_proactive_auth)
		init_curl_http_auth(result);

282 283
	if (ssl_cert != NULL)
		curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
284
	if (has_cert_password())
285
		curl_easy_setopt(result, CURLOPT_KEYPASSWD, cert_auth.password);
286
#if LIBCURL_VERSION_NUM >= 0x070903
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
	if (ssl_key != NULL)
		curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
#endif
#if LIBCURL_VERSION_NUM >= 0x070908
	if (ssl_capath != NULL)
		curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
#endif
	if (ssl_cainfo != NULL)
		curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);

	if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
		curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
				 curl_low_speed_limit);
		curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
				 curl_low_speed_time);
	}

	curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
305 306 307 308 309
#if LIBCURL_VERSION_NUM >= 0x071301
	curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
#elif LIBCURL_VERSION_NUM >= 0x071101
	curl_easy_setopt(result, CURLOPT_POST301, 1);
#endif
310

311 312 313
	if (getenv("GIT_CURL_VERBOSE"))
		curl_easy_setopt(result, CURLOPT_VERBOSE, 1);

314
	curl_easy_setopt(result, CURLOPT_USERAGENT,
315
		user_agent ? user_agent : git_user_agent());
316

317 318 319
	if (curl_ftp_no_epsv)
		curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0);

320 321 322 323 324
#ifdef CURLOPT_USE_SSL
	if (curl_ssl_try)
		curl_easy_setopt(result, CURLOPT_USE_SSL, CURLUSESSL_TRY);
#endif

325
	if (curl_http_proxy) {
326
		curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy);
327 328
		curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
	}
329

330 331 332
	return result;
}

333 334 335 336 337 338 339
static void set_from_env(const char **var, const char *envname)
{
	const char *val = getenv(envname);
	if (val)
		*var = val;
}

340
void http_init(struct remote *remote, const char *url, int proactive_auth)
341 342 343 344
{
	char *low_speed_limit;
	char *low_speed_time;

345 346
	http_is_verbose = 0;

347 348
	git_config(http_options, NULL);

349 350
	curl_global_init(CURL_GLOBAL_ALL);

351 352
	http_proactive_auth = proactive_auth;

353 354 355
	if (remote && remote->http_proxy)
		curl_http_proxy = xstrdup(remote->http_proxy);

356
	pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
357
	no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375

#ifdef USE_CURL_MULTI
	{
		char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
		if (http_max_requests != NULL)
			max_requests = atoi(http_max_requests);
	}

	curlm = curl_multi_init();
	if (curlm == NULL) {
		fprintf(stderr, "Error creating curl multi handle.\n");
		exit(1);
	}
#endif

	if (getenv("GIT_SSL_NO_VERIFY"))
		curl_ssl_verify = 0;

376
	set_from_env(&ssl_cert, "GIT_SSL_CERT");
377
#if LIBCURL_VERSION_NUM >= 0x070903
378
	set_from_env(&ssl_key, "GIT_SSL_KEY");
379 380
#endif
#if LIBCURL_VERSION_NUM >= 0x070908
381
	set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
382
#endif
383
	set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
384

385 386
	set_from_env(&user_agent, "GIT_HTTP_USER_AGENT");

387 388 389 390 391 392 393 394 395 396
	low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
	if (low_speed_limit != NULL)
		curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
	low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
	if (low_speed_time != NULL)
		curl_low_speed_time = strtol(low_speed_time, NULL, 10);

	if (curl_ssl_verify == -1)
		curl_ssl_verify = 1;

Ray's avatar
Ray committed
397
	curl_session_count = 0;
398 399 400 401 402
#ifdef USE_CURL_MULTI
	if (max_requests < 1)
		max_requests = DEFAULT_MAX_REQUESTS;
#endif

403 404 405
	if (getenv("GIT_CURL_FTP_NO_EPSV"))
		curl_ftp_no_epsv = 1;

406
	if (url) {
407
		credential_from_url(&http_auth, url);
408 409
		if (!ssl_cert_password_required &&
		    getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
410
		    !prefixcmp(url, "https://"))
411 412
			ssl_cert_password_required = 1;
	}
413

414 415 416 417 418 419 420 421 422 423
#ifndef NO_CURL_EASY_DUPHANDLE
	curl_default = get_curl_handle();
#endif
}

void http_cleanup(void)
{
	struct active_request_slot *slot = active_queue_head;

	while (slot != NULL) {
424
		struct active_request_slot *next = slot->next;
425
		if (slot->curl != NULL) {
426
#ifdef USE_CURL_MULTI
427
			curl_multi_remove_handle(curlm, slot->curl);
428 429
#endif
			curl_easy_cleanup(slot->curl);
430
		}
431 432
		free(slot);
		slot = next;
433
	}
434
	active_queue_head = NULL;
435 436 437 438 439 440 441 442 443

#ifndef NO_CURL_EASY_DUPHANDLE
	curl_easy_cleanup(curl_default);
#endif

#ifdef USE_CURL_MULTI
	curl_multi_cleanup(curlm);
#endif
	curl_global_cleanup();
Nick Hengeveld's avatar
Nick Hengeveld committed
444 445

	curl_slist_free_all(pragma_header);
446
	pragma_header = NULL;
447

448 449 450
	curl_slist_free_all(no_pragma_header);
	no_pragma_header = NULL;

451
	if (curl_http_proxy) {
452
		free((void *)curl_http_proxy);
453 454
		curl_http_proxy = NULL;
	}
455

456 457 458 459
	if (cert_auth.password != NULL) {
		memset(cert_auth.password, 0, strlen(cert_auth.password));
		free(cert_auth.password);
		cert_auth.password = NULL;
460 461
	}
	ssl_cert_password_required = 0;
462 463 464 465 466 467 468 469 470 471 472 473 474
}

struct active_request_slot *get_active_slot(void)
{
	struct active_request_slot *slot = active_queue_head;
	struct active_request_slot *newslot;

#ifdef USE_CURL_MULTI
	int num_transfers;

	/* Wait for a slot to open up if the queue is full */
	while (active_requests >= max_requests) {
		curl_multi_perform(curlm, &num_transfers);
Junio C Hamano's avatar
Junio C Hamano committed
475
		if (num_transfers < active_requests)
476 477 478 479
			process_curl_messages();
	}
#endif

Junio C Hamano's avatar
Junio C Hamano committed
480
	while (slot != NULL && slot->in_use)
481
		slot = slot->next;
Junio C Hamano's avatar
Junio C Hamano committed
482

483 484 485 486 487 488 489 490 491 492
	if (slot == NULL) {
		newslot = xmalloc(sizeof(*newslot));
		newslot->curl = NULL;
		newslot->in_use = 0;
		newslot->next = NULL;

		slot = active_queue_head;
		if (slot == NULL) {
			active_queue_head = newslot;
		} else {
Junio C Hamano's avatar
Junio C Hamano committed
493
			while (slot->next != NULL)
494 495 496 497 498 499 500 501 502 503 504 505
				slot = slot->next;
			slot->next = newslot;
		}
		slot = newslot;
	}

	if (slot->curl == NULL) {
#ifdef NO_CURL_EASY_DUPHANDLE
		slot->curl = get_curl_handle();
#else
		slot->curl = curl_easy_duphandle(curl_default);
#endif
Ray's avatar
Ray committed
506
		curl_session_count++;
507 508 509 510
	}

	active_requests++;
	slot->in_use = 1;
511
	slot->results = NULL;
Nick Hengeveld's avatar
Nick Hengeveld committed
512
	slot->finished = NULL;
513 514
	slot->callback_data = NULL;
	slot->callback_func = NULL;
515
	curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file);
516 517
	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
	curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
518 519 520
	curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
	curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
	curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
521
	curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL);
522 523
	curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
524
	curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 1);
525 526
	if (http_auth.password)
		init_curl_http_auth(slot->curl);
527 528 529 530 531 532 533 534

	return slot;
}

int start_active_slot(struct active_request_slot *slot)
{
#ifdef USE_CURL_MULTI
	CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
535
	int num_transfers;
536 537 538 539 540 541 542

	if (curlm_result != CURLM_OK &&
	    curlm_result != CURLM_CALL_MULTI_PERFORM) {
		active_requests--;
		slot->in_use = 0;
		return 0;
	}
543 544 545 546 547 548

	/*
	 * We know there must be something to do, since we just added
	 * something.
	 */
	curl_multi_perform(curlm, &num_transfers);
549 550 551 552 553
#endif
	return 1;
}

#ifdef USE_CURL_MULTI
554 555 556 557 558 559
struct fill_chain {
	void *data;
	int (*fill)(void *);
	struct fill_chain *next;
};

Junio C Hamano's avatar
Junio C Hamano committed
560
static struct fill_chain *fill_cfg;
561 562 563

void add_fill_function(void *data, int (*fill)(void *))
{
564
	struct fill_chain *new = xmalloc(sizeof(*new));
565 566 567 568 569 570 571 572 573
	struct fill_chain **linkp = &fill_cfg;
	new->data = data;
	new->fill = fill;
	new->next = NULL;
	while (*linkp)
		linkp = &(*linkp)->next;
	*linkp = new;
}

574 575 576 577
void fill_active_slots(void)
{
	struct active_request_slot *slot = active_queue_head;

578 579 580 581 582 583 584
	while (active_requests < max_requests) {
		struct fill_chain *fill;
		for (fill = fill_cfg; fill; fill = fill->next)
			if (fill->fill(fill->data))
				break;

		if (!fill)
585
			break;
586
	}
587 588

	while (slot != NULL) {
Ray's avatar
Ray committed
589 590
		if (!slot->in_use && slot->curl != NULL
			&& curl_session_count > min_curl_sessions) {
591 592
			curl_easy_cleanup(slot->curl);
			slot->curl = NULL;
Ray's avatar
Ray committed
593
			curl_session_count--;
594 595 596 597 598
		}
		slot = slot->next;
	}
}

599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
void step_active_slots(void)
{
	int num_transfers;
	CURLMcode curlm_result;

	do {
		curlm_result = curl_multi_perform(curlm, &num_transfers);
	} while (curlm_result == CURLM_CALL_MULTI_PERFORM);
	if (num_transfers < active_requests) {
		process_curl_messages();
		fill_active_slots();
	}
}
#endif

void run_active_slot(struct active_request_slot *slot)
{
#ifdef USE_CURL_MULTI
	fd_set readfds;
	fd_set writefds;
	fd_set excfds;
	int max_fd;
	struct timeval select_timeout;
Nick Hengeveld's avatar
Nick Hengeveld committed
622
	int finished = 0;
623

Nick Hengeveld's avatar
Nick Hengeveld committed
624 625
	slot->finished = &finished;
	while (!finished) {
626 627
		step_active_slots();

628
		if (slot->in_use) {
629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644
#if LIBCURL_VERSION_NUM >= 0x070f04
			long curl_timeout;
			curl_multi_timeout(curlm, &curl_timeout);
			if (curl_timeout == 0) {
				continue;
			} else if (curl_timeout == -1) {
				select_timeout.tv_sec  = 0;
				select_timeout.tv_usec = 50000;
			} else {
				select_timeout.tv_sec  =  curl_timeout / 1000;
				select_timeout.tv_usec = (curl_timeout % 1000) * 1000;
			}
#else
			select_timeout.tv_sec  = 0;
			select_timeout.tv_usec = 50000;
#endif
645

646
			max_fd = -1;
647 648 649
			FD_ZERO(&readfds);
			FD_ZERO(&writefds);
			FD_ZERO(&excfds);
650
			curl_multi_fdset(curlm, &readfds, &writefds, &excfds, &max_fd);
651

652 653 654 655 656 657 658 659 660 661 662 663
			/*
			 * It can happen that curl_multi_timeout returns a pathologically
			 * long timeout when curl_multi_fdset returns no file descriptors
			 * to read.  See commit message for more details.
			 */
			if (max_fd < 0 &&
			    (select_timeout.tv_sec > 0 ||
			     select_timeout.tv_usec > 50000)) {
				select_timeout.tv_sec  = 0;
				select_timeout.tv_usec = 50000;
			}

664
			select(max_fd+1, &readfds, &writefds, &excfds, &select_timeout);
665 666 667 668 669 670 671 672 673 674
		}
	}
#else
	while (slot->in_use) {
		slot->curl_result = curl_easy_perform(slot->curl);
		finish_active_slot(slot);
	}
#endif
}

675
static void closedown_active_slot(struct active_request_slot *slot)
676
{
Mike Hommey's avatar
Mike Hommey committed
677 678
	active_requests--;
	slot->in_use = 0;
679 680
}

681
static void release_active_slot(struct active_request_slot *slot)
682 683
{
	closedown_active_slot(slot);
Ray's avatar
Ray committed
684
	if (slot->curl && curl_session_count > min_curl_sessions) {
Nick Hengeveld's avatar
Nick Hengeveld committed
685
#ifdef USE_CURL_MULTI
686
		curl_multi_remove_handle(curlm, slot->curl);
Nick Hengeveld's avatar
Nick Hengeveld committed
687
#endif
688 689
		curl_easy_cleanup(slot->curl);
		slot->curl = NULL;
Ray's avatar
Ray committed
690
		curl_session_count--;
691
	}
Nick Hengeveld's avatar
Nick Hengeveld committed
692
#ifdef USE_CURL_MULTI
693
	fill_active_slots();
Nick Hengeveld's avatar
Nick Hengeveld committed
694
#endif
695 696
}

697
void finish_active_slot(struct active_request_slot *slot)
698 699
{
	closedown_active_slot(slot);
Mike Hommey's avatar
Mike Hommey committed
700
	curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
701

Nick Hengeveld's avatar
Nick Hengeveld committed
702 703 704
	if (slot->finished != NULL)
		(*slot->finished) = 1;

705 706 707 708 709 710
	/* Store slot results so they can be read after the slot is reused */
	if (slot->results != NULL) {
		slot->results->curl_result = slot->curl_result;
		slot->results->http_code = slot->http_code;
	}

Mike Hommey's avatar
Mike Hommey committed
711
	/* Run callback if appropriate */
Junio C Hamano's avatar
Junio C Hamano committed
712
	if (slot->callback_func != NULL)
Mike Hommey's avatar
Mike Hommey committed
713
		slot->callback_func(slot->callback_data);
714 715 716 717 718 719 720 721 722 723 724 725 726 727
}

void finish_all_active_slots(void)
{
	struct active_request_slot *slot = active_queue_head;

	while (slot != NULL)
		if (slot->in_use) {
			run_active_slot(slot);
			slot = active_queue_head;
		} else {
			slot = slot->next;
		}
}
728

729
/* Helpers for modifying and creating URLs */
730 731 732 733 734 735 736 737 738 739 740 741 742 743
static inline int needs_quote(int ch)
{
	if (((ch >= 'A') && (ch <= 'Z'))
			|| ((ch >= 'a') && (ch <= 'z'))
			|| ((ch >= '0') && (ch <= '9'))
			|| (ch == '/')
			|| (ch == '-')
			|| (ch == '.'))
		return 0;
	return 1;
}

static char *quote_ref_url(const char *base, const char *ref)
{
744
	struct strbuf buf = STRBUF_INIT;
745
	const char *cp;
746
	int ch;
747

748
	end_url_with_slash(&buf, base);
749 750

	for (cp = ref; (ch = *cp) != 0; cp++)
751
		if (needs_quote(ch))
752
			strbuf_addf(&buf, "%%%02x", ch);
753
		else
754
			strbuf_addch(&buf, *cp);
755

756
	return strbuf_detach(&buf, NULL);
757 758
}

759 760 761 762
void append_remote_object_url(struct strbuf *buf, const char *url,
			      const char *hex,
			      int only_two_digit_prefix)
{
763 764 765
	end_url_with_slash(buf, url);

	strbuf_addf(buf, "objects/%.*s/", 2, hex);
766 767 768 769 770 771 772 773 774 775 776 777
	if (!only_two_digit_prefix)
		strbuf_addf(buf, "%s", hex+2);
}

char *get_remote_object_url(const char *url, const char *hex,
			    int only_two_digit_prefix)
{
	struct strbuf buf = STRBUF_INIT;
	append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
	return strbuf_detach(&buf, NULL);
}

778
int handle_curl_result(struct slot_results *results)
779
{
Jeff King's avatar
Jeff King committed
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798
	/*
	 * If we see a failing http code with CURLE_OK, we have turned off
	 * FAILONERROR (to keep the server's custom error response), and should
	 * translate the code into failure here.
	 */
	if (results->curl_result == CURLE_OK &&
	    results->http_code >= 400) {
		results->curl_result = CURLE_HTTP_RETURNED_ERROR;
		/*
		 * Normally curl will already have put the "reason phrase"
		 * from the server into curl_errorstr; unfortunately without
		 * FAILONERROR it is lost, so we can give only the numeric
		 * status code.
		 */
		snprintf(curl_errorstr, sizeof(curl_errorstr),
			 "The requested URL returned error: %ld",
			 results->http_code);
	}

799 800 801 802 803 804 805 806 807 808 809 810 811 812
	if (results->curl_result == CURLE_OK) {
		credential_approve(&http_auth);
		return HTTP_OK;
	} else if (missing_target(results))
		return HTTP_MISSING_TARGET;
	else if (results->http_code == 401) {
		if (http_auth.username && http_auth.password) {
			credential_reject(&http_auth);
			return HTTP_NOAUTH;
		} else {
			credential_fill(&http_auth);
			return HTTP_REAUTH;
		}
	} else {
813
#if LIBCURL_VERSION_NUM >= 0x070c00
814 815 816 817
		if (!curl_errorstr[0])
			strlcpy(curl_errorstr,
				curl_easy_strerror(results->curl_result),
				sizeof(curl_errorstr));
818
#endif
819 820 821 822
		return HTTP_ERROR;
	}
}

823 824 825 826
/* http_request() targets */
#define HTTP_REQUEST_STRBUF	0
#define HTTP_REQUEST_FILE	1

827 828
static int http_request(const char *url, struct strbuf *type,
			void *result, int target, int options)
829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862
{
	struct active_request_slot *slot;
	struct slot_results results;
	struct curl_slist *headers = NULL;
	struct strbuf buf = STRBUF_INIT;
	int ret;

	slot = get_active_slot();
	slot->results = &results;
	curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);

	if (result == NULL) {
		curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
	} else {
		curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
		curl_easy_setopt(slot->curl, CURLOPT_FILE, result);

		if (target == HTTP_REQUEST_FILE) {
			long posn = ftell(result);
			curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
					 fwrite);
			if (posn > 0) {
				strbuf_addf(&buf, "Range: bytes=%ld-", posn);
				headers = curl_slist_append(headers, buf.buf);
				strbuf_reset(&buf);
			}
		} else
			curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
					 fwrite_buffer);
	}

	strbuf_addstr(&buf, "Pragma:");
	if (options & HTTP_NO_CACHE)
		strbuf_addstr(&buf, " no-cache");
Jeff King's avatar
Jeff King committed
863 864
	if (options & HTTP_KEEP_ERROR)
		curl_easy_setopt(slot->curl, CURLOPT_FAILONERROR, 0);
865 866 867 868 869

	headers = curl_slist_append(headers, buf.buf);

	curl_easy_setopt(slot->curl, CURLOPT_URL, url);
	curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
870
	curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "gzip");
871 872 873

	if (start_active_slot(slot)) {
		run_active_slot(slot);
874
		ret = handle_curl_result(&results);
875
	} else {
876 877
		snprintf(curl_errorstr, sizeof(curl_errorstr),
			 "failed to start HTTP request");
878 879 880
		ret = HTTP_START_FAILED;
	}

881 882
	if (type) {
		char *t;
883
		strbuf_reset(type);
884 885 886 887 888
		curl_easy_getinfo(slot->curl, CURLINFO_CONTENT_TYPE, &t);
		if (t)
			strbuf_addstr(type, t);
	}

889 890 891 892 893 894
	curl_slist_free_all(headers);
	strbuf_release(&buf);

	return ret;
}

895 896 897
static int http_request_reauth(const char *url,
			       struct strbuf *type,
			       void *result, int target,
898 899
			       int options)
{
900
	int ret = http_request(url, type, result, target, options);
901 902
	if (ret != HTTP_REAUTH)
		return ret;
Jeff King's avatar
Jeff King committed
903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918

	/*
	 * If we are using KEEP_ERROR, the previous request may have
	 * put cruft into our output stream; we should clear it out before
	 * making our next request. We only know how to do this for
	 * the strbuf case, but that is enough to satisfy current callers.
	 */
	if (options & HTTP_KEEP_ERROR) {
		switch (target) {
		case HTTP_REQUEST_STRBUF:
			strbuf_reset(result);
			break;
		default:
			die("BUG: HTTP_KEEP_ERROR is only supported with strbufs");
		}
	}
919
	return http_request(url, type, result, target, options);
920 921
}

922 923 924
int http_get_strbuf(const char *url,
		    struct strbuf *type,
		    struct strbuf *result, int options)
925
{
926 927
	return http_request_reauth(url, type, result,
				   HTTP_REQUEST_STRBUF, options);
928 929
}

930
/*
931
 * Downloads a URL and stores the result in the given file.
932 933 934 935 936
 *
 * If a previous interrupted download is detected (i.e. a previous temporary
 * file is still around) the download is resumed.
 */
static int http_get_file(const char *url, const char *filename, int options)
937 938 939 940 941 942 943 944 945 946 947 948 949
{
	int ret;
	struct strbuf tmpfile = STRBUF_INIT;
	FILE *result;

	strbuf_addf(&tmpfile, "%s.temp", filename);
	result = fopen(tmpfile.buf, "a");
	if (! result) {
		error("Unable to open local file %s", tmpfile.buf);
		ret = HTTP_ERROR;
		goto cleanup;
	}

950
	ret = http_request_reauth(url, NULL, result, HTTP_REQUEST_FILE, options);
951 952 953 954 955 956 957 958 959
	fclose(result);

	if ((ret == HTTP_OK) && move_temp_to_file(tmpfile.buf, filename))
		ret = HTTP_ERROR;
cleanup:
	strbuf_release(&tmpfile);
	return ret;
}

960
int http_fetch_ref(const char *base, struct ref *ref)
961 962 963
{
	char *url;
	struct strbuf buffer = STRBUF_INIT;
964
	int ret = -1;
965

966
	url = quote_ref_url(base, ref->name);
967
	if (http_get_strbuf(url, NULL, &buffer, HTTP_NO_CACHE) == HTTP_OK) {
968 969 970 971 972 973
		strbuf_rtrim(&buffer);
		if (buffer.len == 40)
			ret = get_sha1_hex(buffer.buf, ref->old_sha1);
		else if (!prefixcmp(buffer.buf, "ref: ")) {
			ref->symref = xstrdup(buffer.buf + 5);
			ret = 0;
974 975 976 977 978 979 980
		}
	}

	strbuf_release(&buffer);
	free(url);
	return ret;
}
Ray's avatar
Ray committed
981 982

/* Helpers for fetching packs */
983
static char *fetch_pack_index(unsigned char *sha1, const char *base_url)
Ray's avatar
Ray committed
984
{
985
	char *url, *tmp;
Ray's avatar
Ray committed
986 987 988
	struct strbuf buf = STRBUF_INIT;

	if (http_is_verbose)
989
		fprintf(stderr, "Getting index for pack %s\n", sha1_to_hex(sha1));
Ray's avatar
Ray committed
990 991

	end_url_with_slash(&buf, base_url);
992
	strbuf_addf(&buf, "objects/pack/pack-%s.idx", sha1_to_hex(sha1));
Ray's avatar
Ray committed
993 994
	url = strbuf_detach(&buf, NULL);

995 996 997 998
	strbuf_addf(&buf, "%s.temp", sha1_pack_index_name(sha1));
	tmp = strbuf_detach(&buf, NULL);

	if (http_get_file(url, tmp, 0) != HTTP_OK) {
999
		error("Unable to get pack index %s", url);
1000 1001 1002
		free(tmp);
		tmp = NULL;
	}
Ray's avatar
Ray committed
1003 1004

	free(url);
1005
	return tmp;
Ray's avatar
Ray committed
1006 1007 1008 1009 1010 1011
}

static int fetch_and_setup_pack_index(struct packed_git **packs_head,
	unsigned char *sha1, const char *base_url)
{
	struct packed_git *new_pack;
1012 1013
	char *tmp_idx = NULL;
	int ret;
Ray's avatar
Ray committed
1014

1015 1016 1017 1018 1019 1020 1021 1022 1023
	if (has_pack_index(sha1)) {
		new_pack = parse_pack_index(sha1, NULL);
		if (!new_pack)
			return -1; /* parse_pack_index() already issued error message */
		goto add_pack;
	}

	tmp_idx = fetch_pack_index(sha1, base_url);
	if (!tmp_idx)
Ray's avatar
Ray committed
1024 1025
		return -1;

1026 1027 1028 1029 1030
	new_pack = parse_pack_index(sha1, tmp_idx);
	if (!new_pack) {
		unlink(tmp_idx);
		free(tmp_idx);

Ray's avatar
Ray committed
1031
		return -1; /* parse_pack_index() already issued error message */
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
	}

	ret = verify_pack_index(new_pack);
	if (!ret) {
		close_pack_index(new_pack);
		ret = move_temp_to_file(tmp_idx, sha1_pack_index_name(sha1));
	}
	free(tmp_idx);
	if (ret)
		return -1;

add_pack:
Ray's avatar
Ray committed
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
	new_pack->next = *packs_head;
	*packs_head = new_pack;
	return 0;
}

int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
{
	int ret = 0, i = 0;
	char *url, *data;
	struct strbuf buf = STRBUF_INIT;
	unsigned char sha1[20];

	end_url_with_slash(&buf, base_url);
	strbuf_addstr(&buf, "objects/info/packs");
	url = strbuf_detach(&buf, NULL);

1060
	ret = http_get_strbuf(url, NULL, &buf, HTTP_NO_CACHE);
Ray's avatar
Ray committed
1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
	if (ret != HTTP_OK)
		goto cleanup;

	data = buf.buf;
	while (i < buf.len) {
		switch (data[i]) {
		case 'P':
			i++;
			if (i + 52 <= buf.len &&
			    !prefixcmp(data + i, " pack-") &&
			    !prefixcmp(data + i + 46, ".pack\n")) {
				get_sha1_hex(data + i + 6, sha1);
				fetch_and_setup_pack_index(packs_head, sha1,
						      base_url);
				i += 51;
				break;
			}
		default:
			while (i < buf.len && data[i] != '\n')
				i++;
		}
		i++;
	}

cleanup:
	free(url);
	return ret;
}
1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106

void release_http_pack_request(struct http_pack_request *preq)
{
	if (preq->packfile != NULL) {
		fclose(preq->packfile);
		preq->packfile = NULL;
	}
	if (preq->range_header != NULL) {
		curl_slist_free_all(preq->range_header);
		preq->range_header = NULL;
	}
	preq->slot = NULL;
	free(preq->url);
}

int finish_http_pack_request(struct http_pack_request *preq)
{
	struct packed_git **lst;
1107
	struct packed_git *p = preq->target;
1108 1109 1110
	char *tmp_idx;
	struct child_process ip;
	const char *ip_argv[8];
1111

1112
	close_pack_index(p);
1113

1114 1115
	fclose(preq->packfile);
	preq->packfile = NULL;
1116 1117

	lst = preq->lst;
1118
	while (*lst != p)
1119 1120 1121
		lst = &((*lst)->next);
	*lst = (*lst)->next;

1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141
	tmp_idx = xstrdup(preq->tmpfile);
	strcpy(tmp_idx + strlen(tmp_idx) - strlen(".pack.temp"),
	       ".idx.temp");

	ip_argv[0] = "index-pack";
	ip_argv[1] = "-o";
	ip_argv[2] = tmp_idx;
	ip_argv[3] = preq->tmpfile;
	ip_argv[4] = NULL;

	memset(&ip, 0, sizeof(ip));
	ip.argv = ip_argv;
	ip.git_cmd = 1;
	ip.no_stdin = 1;
	ip.no_stdout = 1;

	if (run_command(&ip)) {
		unlink(preq->tmpfile);
		unlink(tmp_idx);
		free(tmp_idx);
1142
		return -1;
1143 1144 1145
	}

	unlink(sha1_pack_index_name(p->sha1));
1146

1147 1148 1149
	if (move_temp_to_file(preq->tmpfile, sha1_pack_name(p->sha1))
	 || move_temp_to_file(tmp_idx, sha1_pack_index_name(p->sha1))) {
		free(tmp_idx);
1150
		return -1;
1151
	}
1152

1153 1154
	install_packed_git(p);
	free(tmp_idx);
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165
	return 0;
}

struct http_pack_request *new_http_pack_request(
	struct packed_git *target, const char *base_url)
{
	long prev_posn = 0;
	char range[RANGE_HEADER_SIZE];
	struct strbuf buf = STRBUF_INIT;
	struct http_pack_request *preq;

Ray's avatar
Ray committed
1166
	preq = xcalloc(1, sizeof(*preq));
1167 1168 1169 1170 1171
	preq->target = target;

	end_url_with_slash(&buf, base_url);
	strbuf_addf(&buf, "objects/pack/pack-%s.pack",
		sha1_to_hex(target->sha1));
1172
	preq->url = strbuf_detach(&buf, NULL);
1173

1174 1175
	snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp",
		sha1_pack_name(target->sha1));
1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
	preq->packfile = fopen(preq->tmpfile, "a");
	if (!preq->packfile) {
		error("Unable to open local file %s for pack",
		      preq->tmpfile);
		goto abort;
	}

	preq->slot = get_active_slot();
	curl_easy_setopt(preq->slot->curl, CURLOPT_FILE, preq->packfile);
	curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
1186
	curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208
	curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
		no_pragma_header);

	/*
	 * If there is data present from a previous transfer attempt,
	 * resume where it left off
	 */
	prev_posn = ftell(preq->packfile);
	if (prev_posn>0) {
		if (http_is_verbose)
			fprintf(stderr,
				"Resuming fetch of pack %s at byte %ld\n",
				sha1_to_hex(target->sha1), prev_posn);
		sprintf(range, "Range: bytes=%ld-", prev_posn);
		preq->range_header = curl_slist_append(NULL, range);
		curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
			preq->range_header);
	}

	return preq;

abort:
1209
	free(preq->url);
Ray's avatar
Ray committed
1210
	free(preq);
1211 1212
	return NULL;
}
1213 1214

/* Helpers for fetching objects (loose) */
1215
static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb,
1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231
			       void *data)
{
	unsigned char expn[4096];
	size_t size = eltsize * nmemb;
	int posn = 0;
	struct http_object_request *freq =
		(struct http_object_request *)data;
	do {
		ssize_t retval = xwrite(freq->localfile,
					(char *) ptr + posn, size - posn);
		if (retval < 0)
			return posn;
		posn += retval;
	} while (posn < size);

	freq->stream.avail_in = size;
1232
	freq->stream.next_in = (void *)ptr;
1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249
	do {
		freq->stream.next_out = expn;
		freq->stream.avail_out = sizeof(expn);
		freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
		git_SHA1_Update(&freq->c, expn,
				sizeof(expn) - freq->stream.avail_out);
	} while (freq->stream.avail_in && freq->zret == Z_OK);
	return size;
}

struct http_object_request *new_http_object_request(const char *base_url,
	unsigned char *sha1)
{
	char *hex = sha1_to_hex(sha1);
	char *filename;
	char prevfile[PATH_MAX];
	int prevlocal;
1250
	char prev_buf[PREV_BUF_SIZE];
1251 1252 1253 1254 1255 1256
	ssize_t prev_read = 0;
	long prev_posn = 0;
	char range[RANGE_HEADER_SIZE];
	struct curl_slist *range_header = NULL;
	struct http_object_request *freq;

Ray's avatar
Ray committed
1257
	freq = xcalloc(1, sizeof(*freq));
1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289
	hashcpy(freq->sha1, sha1);
	freq->localfile = -1;

	filename = sha1_file_name(sha1);
	snprintf(freq->tmpfile, sizeof(freq->tmpfile),
		 "%s.temp", filename);

	snprintf(prevfile, sizeof(prevfile), "%s.prev", filename);
	unlink_or_warn(prevfile);
	rename(freq->tmpfile, prevfile);
	unlink_or_warn(freq->tmpfile);

	if (freq->localfile != -1)
		error("fd leakage in start: %d", freq->localfile);
	freq->localfile = open(freq->tmpfile,
			       O_WRONLY | O_CREAT | O_EXCL, 0666);
	/*
	 * This could have failed due to the "lazy directory creation";
	 * try to mkdir the last path component.
	 */
	if (freq->localfile