ip_tunnel.c 28.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
/*
 * Copyright (c) 2013 Nicira, Inc.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of version 2 of the GNU General Public
 * License as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/capability.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/in.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/if_arp.h>
#include <linux/init.h>
#include <linux/in6.h>
#include <linux/inetdevice.h>
#include <linux/igmp.h>
#include <linux/netfilter_ipv4.h>
#include <linux/etherdevice.h>
#include <linux/if_ether.h>
#include <linux/if_vlan.h>
#include <linux/rculist.h>
Sachin Kamat's avatar
Sachin Kamat committed
42
#include <linux/err.h>
43 44 45 46 47 48 49 50 51 52 53 54 55 56

#include <net/sock.h>
#include <net/ip.h>
#include <net/icmp.h>
#include <net/protocol.h>
#include <net/ip_tunnels.h>
#include <net/arp.h>
#include <net/checksum.h>
#include <net/dsfield.h>
#include <net/inet_ecn.h>
#include <net/xfrm.h>
#include <net/net_namespace.h>
#include <net/netns/generic.h>
#include <net/rtnetlink.h>
57
#include <net/udp.h>
58
#include <net/dst_metadata.h>
59

60 61 62 63 64 65
#if IS_ENABLED(CONFIG_IPV6)
#include <net/ipv6.h>
#include <net/ip6_fib.h>
#include <net/ip6_route.h>
#endif

66
static unsigned int ip_tunnel_hash(__be32 key, __be32 remote)
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
{
	return hash_32((__force u32)key ^ (__force u32)remote,
			 IP_TNL_HASH_BITS);
}

static bool ip_tunnel_key_match(const struct ip_tunnel_parm *p,
				__be16 flags, __be32 key)
{
	if (p->i_flags & TUNNEL_KEY) {
		if (flags & TUNNEL_KEY)
			return key == p->i_key;
		else
			/* key expected, none present */
			return false;
	} else
		return !(flags & TUNNEL_KEY);
}

/* Fallback tunnel: no source, no destination, no key, no options

   Tunnel hash table:
   We require exact key match i.e. if a key is present in packet
   it will match only tunnel with the same key; if it is not present,
   it will match only keyless tunnel.

   All keysless packets, if not matched configured keyless tunnels
   will match fallback tunnel.
   Given src, dst and key, find appropriate for input tunnel.
*/
struct ip_tunnel *ip_tunnel_lookup(struct ip_tunnel_net *itn,
				   int link, __be16 flags,
				   __be32 remote, __be32 local,
				   __be32 key)
{
	unsigned int hash;
	struct ip_tunnel *t, *cand = NULL;
	struct hlist_head *head;

105
	hash = ip_tunnel_hash(key, remote);
106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124
	head = &itn->tunnels[hash];

	hlist_for_each_entry_rcu(t, head, hash_node) {
		if (local != t->parms.iph.saddr ||
		    remote != t->parms.iph.daddr ||
		    !(t->dev->flags & IFF_UP))
			continue;

		if (!ip_tunnel_key_match(&t->parms, flags, key))
			continue;

		if (t->parms.link == link)
			return t;
		else
			cand = t;
	}

	hlist_for_each_entry_rcu(t, head, hash_node) {
		if (remote != t->parms.iph.daddr ||
Dmitry Popov's avatar
Dmitry Popov committed
125
		    t->parms.iph.saddr != 0 ||
126 127 128 129 130 131 132 133 134 135 136 137
		    !(t->dev->flags & IFF_UP))
			continue;

		if (!ip_tunnel_key_match(&t->parms, flags, key))
			continue;

		if (t->parms.link == link)
			return t;
		else if (!cand)
			cand = t;
	}

138
	hash = ip_tunnel_hash(key, 0);
139 140 141
	head = &itn->tunnels[hash];

	hlist_for_each_entry_rcu(t, head, hash_node) {
Dmitry Popov's avatar
Dmitry Popov committed
142 143 144 145 146
		if ((local != t->parms.iph.saddr || t->parms.iph.daddr != 0) &&
		    (local != t->parms.iph.daddr || !ipv4_is_multicast(local)))
			continue;

		if (!(t->dev->flags & IFF_UP))
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
			continue;

		if (!ip_tunnel_key_match(&t->parms, flags, key))
			continue;

		if (t->parms.link == link)
			return t;
		else if (!cand)
			cand = t;
	}

	if (flags & TUNNEL_NO_KEY)
		goto skip_key_lookup;

	hlist_for_each_entry_rcu(t, head, hash_node) {
		if (t->parms.i_key != key ||
Dmitry Popov's avatar
Dmitry Popov committed
163 164
		    t->parms.iph.saddr != 0 ||
		    t->parms.iph.daddr != 0 ||
165 166 167 168 169 170 171 172 173 174 175 176 177
		    !(t->dev->flags & IFF_UP))
			continue;

		if (t->parms.link == link)
			return t;
		else if (!cand)
			cand = t;
	}

skip_key_lookup:
	if (cand)
		return cand;

178
	t = rcu_dereference(itn->collect_md_tun);
179
	if (t && t->dev->flags & IFF_UP)
180 181
		return t;

182 183 184 185 186 187 188 189 190 191 192 193
	if (itn->fb_tunnel_dev && itn->fb_tunnel_dev->flags & IFF_UP)
		return netdev_priv(itn->fb_tunnel_dev);

	return NULL;
}
EXPORT_SYMBOL_GPL(ip_tunnel_lookup);

static struct hlist_head *ip_bucket(struct ip_tunnel_net *itn,
				    struct ip_tunnel_parm *parms)
{
	unsigned int h;
	__be32 remote;
194
	__be32 i_key = parms->i_key;
195 196 197 198 199 200

	if (parms->iph.daddr && !ipv4_is_multicast(parms->iph.daddr))
		remote = parms->iph.daddr;
	else
		remote = 0;

201 202 203 204
	if (!(parms->i_flags & TUNNEL_KEY) && (parms->i_flags & VTI_ISVTI))
		i_key = 0;

	h = ip_tunnel_hash(i_key, remote);
205 206 207 208 209 210 211
	return &itn->tunnels[h];
}

static void ip_tunnel_add(struct ip_tunnel_net *itn, struct ip_tunnel *t)
{
	struct hlist_head *head = ip_bucket(itn, &t->parms);

212 213
	if (t->collect_md)
		rcu_assign_pointer(itn->collect_md_tun, t);
214 215 216
	hlist_add_head_rcu(&t->hash_node, head);
}

217
static void ip_tunnel_del(struct ip_tunnel_net *itn, struct ip_tunnel *t)
218
{
219 220
	if (t->collect_md)
		rcu_assign_pointer(itn->collect_md_tun, NULL);
221 222 223 224 225 226 227 228 229 230
	hlist_del_init_rcu(&t->hash_node);
}

static struct ip_tunnel *ip_tunnel_find(struct ip_tunnel_net *itn,
					struct ip_tunnel_parm *parms,
					int type)
{
	__be32 remote = parms->iph.daddr;
	__be32 local = parms->iph.saddr;
	__be32 key = parms->i_key;
231
	__be16 flags = parms->i_flags;
232 233 234 235 236 237 238 239
	int link = parms->link;
	struct ip_tunnel *t = NULL;
	struct hlist_head *head = ip_bucket(itn, parms);

	hlist_for_each_entry_rcu(t, head, hash_node) {
		if (local == t->parms.iph.saddr &&
		    remote == t->parms.iph.daddr &&
		    link == t->parms.link &&
240 241
		    type == t->dev->type &&
		    ip_tunnel_key_match(&t->parms, flags, key))
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
			break;
	}
	return t;
}

static struct net_device *__ip_tunnel_create(struct net *net,
					     const struct rtnl_link_ops *ops,
					     struct ip_tunnel_parm *parms)
{
	int err;
	struct ip_tunnel *tunnel;
	struct net_device *dev;
	char name[IFNAMSIZ];

	if (parms->name[0])
		strlcpy(name, parms->name, IFNAMSIZ);
	else {
259
		if (strlen(ops->kind) > (IFNAMSIZ - 3)) {
260 261 262 263 264 265 266 267
			err = -E2BIG;
			goto failed;
		}
		strlcpy(name, ops->kind, IFNAMSIZ);
		strncat(name, "%d", 2);
	}

	ASSERT_RTNL();
268
	dev = alloc_netdev(ops->priv_size, name, NET_NAME_UNKNOWN, ops->setup);
269 270 271 272 273 274 275 276 277 278
	if (!dev) {
		err = -ENOMEM;
		goto failed;
	}
	dev_net_set(dev, net);

	dev->rtnl_link_ops = ops;

	tunnel = netdev_priv(dev);
	tunnel->parms = *parms;
Nicolas Dichtel's avatar
Nicolas Dichtel committed
279
	tunnel->net = net;
280 281 282 283 284 285 286 287 288 289 290 291 292

	err = register_netdevice(dev);
	if (err)
		goto failed_free;

	return dev;

failed_free:
	free_netdev(dev);
failed:
	return ERR_PTR(err);
}

Tom Herbert's avatar
Tom Herbert committed
293 294 295
static inline void init_tunnel_flow(struct flowi4 *fl4,
				    int proto,
				    __be32 daddr, __be32 saddr,
296 297
				    __be32 key, __u8 tos, int oif,
				    __u32 mark)
298 299 300 301 302 303 304 305
{
	memset(fl4, 0, sizeof(*fl4));
	fl4->flowi4_oif = oif;
	fl4->daddr = daddr;
	fl4->saddr = saddr;
	fl4->flowi4_tos = tos;
	fl4->flowi4_proto = proto;
	fl4->fl4_gre_key = key;
306
	fl4->flowi4_mark = mark;
307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
}

static int ip_tunnel_bind_dev(struct net_device *dev)
{
	struct net_device *tdev = NULL;
	struct ip_tunnel *tunnel = netdev_priv(dev);
	const struct iphdr *iph;
	int hlen = LL_MAX_HEADER;
	int mtu = ETH_DATA_LEN;
	int t_hlen = tunnel->hlen + sizeof(struct iphdr);

	iph = &tunnel->parms.iph;

	/* Guess output device to choose reasonable mtu and needed_headroom */
	if (iph->daddr) {
		struct flowi4 fl4;
		struct rtable *rt;

Tom Herbert's avatar
Tom Herbert committed
325 326
		init_tunnel_flow(&fl4, iph->protocol, iph->daddr,
				 iph->saddr, tunnel->parms.o_key,
327 328
				 RT_TOS(iph->tos), tunnel->parms.link,
				 tunnel->fwmark);
Tom Herbert's avatar
Tom Herbert committed
329 330
		rt = ip_route_output_key(tunnel->net, &fl4);

331 332 333 334 335 336
		if (!IS_ERR(rt)) {
			tdev = rt->dst.dev;
			ip_rt_put(rt);
		}
		if (dev->type != ARPHRD_ETHER)
			dev->flags |= IFF_POINTOPOINT;
337 338

		dst_cache_reset(&tunnel->dst_cache);
339 340 341
	}

	if (!tdev && tunnel->parms.link)
Nicolas Dichtel's avatar
Nicolas Dichtel committed
342
		tdev = __dev_get_by_index(tunnel->net, tunnel->parms.link);
343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361

	if (tdev) {
		hlen = tdev->hard_header_len + tdev->needed_headroom;
		mtu = tdev->mtu;
	}

	dev->needed_headroom = t_hlen + hlen;
	mtu -= (dev->hard_header_len + t_hlen);

	if (mtu < 68)
		mtu = 68;

	return mtu;
}

static struct ip_tunnel *ip_tunnel_create(struct net *net,
					  struct ip_tunnel_net *itn,
					  struct ip_tunnel_parm *parms)
{
362
	struct ip_tunnel *nt;
363
	struct net_device *dev;
364
	int t_hlen;
365 366 367 368

	BUG_ON(!itn->fb_tunnel_dev);
	dev = __ip_tunnel_create(net, itn->fb_tunnel_dev->rtnl_link_ops, parms);
	if (IS_ERR(dev))
369
		return ERR_CAST(dev);
370 371 372 373

	dev->mtu = ip_tunnel_bind_dev(dev);

	nt = netdev_priv(dev);
374 375 376
	t_hlen = nt->hlen + sizeof(struct iphdr);
	dev->min_mtu = ETH_MIN_MTU;
	dev->max_mtu = 0xFFF8 - dev->hard_header_len - t_hlen;
377 378 379 380 381
	ip_tunnel_add(itn, nt);
	return nt;
}

int ip_tunnel_rcv(struct ip_tunnel *tunnel, struct sk_buff *skb,
382 383
		  const struct tnl_ptk_info *tpi, struct metadata_dst *tun_dst,
		  bool log_ecn_error)
384
{
385
	struct pcpu_sw_netstats *tstats;
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412
	const struct iphdr *iph = ip_hdr(skb);
	int err;

#ifdef CONFIG_NET_IPGRE_BROADCAST
	if (ipv4_is_multicast(iph->daddr)) {
		tunnel->dev->stats.multicast++;
		skb->pkt_type = PACKET_BROADCAST;
	}
#endif

	if ((!(tpi->flags&TUNNEL_CSUM) &&  (tunnel->parms.i_flags&TUNNEL_CSUM)) ||
	     ((tpi->flags&TUNNEL_CSUM) && !(tunnel->parms.i_flags&TUNNEL_CSUM))) {
		tunnel->dev->stats.rx_crc_errors++;
		tunnel->dev->stats.rx_errors++;
		goto drop;
	}

	if (tunnel->parms.i_flags&TUNNEL_SEQ) {
		if (!(tpi->flags&TUNNEL_SEQ) ||
		    (tunnel->i_seqno && (s32)(ntohl(tpi->seq) - tunnel->i_seqno) < 0)) {
			tunnel->dev->stats.rx_fifo_errors++;
			tunnel->dev->stats.rx_errors++;
			goto drop;
		}
		tunnel->i_seqno = ntohl(tpi->seq) + 1;
	}

413 414
	skb_reset_network_header(skb);

415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
	err = IP_ECN_decapsulate(iph, skb);
	if (unlikely(err)) {
		if (log_ecn_error)
			net_info_ratelimited("non-ECT from %pI4 with TOS=%#x\n",
					&iph->saddr, iph->tos);
		if (err > 1) {
			++tunnel->dev->stats.rx_frame_errors;
			++tunnel->dev->stats.rx_errors;
			goto drop;
		}
	}

	tstats = this_cpu_ptr(tunnel->dev->tstats);
	u64_stats_update_begin(&tstats->syncp);
	tstats->rx_packets++;
	tstats->rx_bytes += skb->len;
	u64_stats_update_end(&tstats->syncp);

433 434
	skb_scrub_packet(skb, !net_eq(tunnel->net, dev_net(tunnel->dev)));

435 436 437 438 439 440
	if (tunnel->dev->type == ARPHRD_ETHER) {
		skb->protocol = eth_type_trans(skb, tunnel->dev);
		skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
	} else {
		skb->dev = tunnel->dev;
	}
441

442 443 444
	if (tun_dst)
		skb_dst_set(skb, (struct dst_entry *)tun_dst);

445 446 447 448
	gro_cells_receive(&tunnel->gro_cells, skb);
	return 0;

drop:
449 450
	if (tun_dst)
		dst_release((struct dst_entry *)tun_dst);
451 452 453 454 455
	kfree_skb(skb);
	return 0;
}
EXPORT_SYMBOL_GPL(ip_tunnel_rcv);

456 457 458
int ip_tunnel_encap_add_ops(const struct ip_tunnel_encap_ops *ops,
			    unsigned int num)
{
459 460 461
	if (num >= MAX_IPTUN_ENCAP_OPS)
		return -ERANGE;

462 463 464
	return !cmpxchg((const struct ip_tunnel_encap_ops **)
			&iptun_encaps[num],
			NULL, ops) ? 0 : -1;
465
}
466 467 468 469 470 471 472
EXPORT_SYMBOL(ip_tunnel_encap_add_ops);

int ip_tunnel_encap_del_ops(const struct ip_tunnel_encap_ops *ops,
			    unsigned int num)
{
	int ret;

473 474 475
	if (num >= MAX_IPTUN_ENCAP_OPS)
		return -ERANGE;

476 477 478 479 480 481 482 483 484
	ret = (cmpxchg((const struct ip_tunnel_encap_ops **)
		       &iptun_encaps[num],
		       ops, NULL) == ops) ? 0 : -1;

	synchronize_net();

	return ret;
}
EXPORT_SYMBOL(ip_tunnel_encap_del_ops);
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508

int ip_tunnel_encap_setup(struct ip_tunnel *t,
			  struct ip_tunnel_encap *ipencap)
{
	int hlen;

	memset(&t->encap, 0, sizeof(t->encap));

	hlen = ip_encap_hlen(ipencap);
	if (hlen < 0)
		return hlen;

	t->encap.type = ipencap->type;
	t->encap.sport = ipencap->sport;
	t->encap.dport = ipencap->dport;
	t->encap.flags = ipencap->flags;

	t->encap_hlen = hlen;
	t->hlen = t->encap_hlen + t->tun_hlen;

	return 0;
}
EXPORT_SYMBOL_GPL(ip_tunnel_encap_setup);

509
static int tnl_update_pmtu(struct net_device *dev, struct sk_buff *skb,
510 511
			    struct rtable *rt, __be16 df,
			    const struct iphdr *inner_iph)
512 513
{
	struct ip_tunnel *tunnel = netdev_priv(dev);
514
	int pkt_size = skb->len - tunnel->hlen - dev->hard_header_len;
515 516 517 518 519 520 521 522 523 524 525 526 527
	int mtu;

	if (df)
		mtu = dst_mtu(&rt->dst) - dev->hard_header_len
					- sizeof(struct iphdr) - tunnel->hlen;
	else
		mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;

	if (skb_dst(skb))
		skb_dst(skb)->ops->update_pmtu(skb_dst(skb), NULL, skb, mtu);

	if (skb->protocol == htons(ETH_P_IP)) {
		if (!skb_is_gso(skb) &&
528 529
		    (inner_iph->frag_off & htons(IP_DF)) &&
		    mtu < pkt_size) {
530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
			memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
			icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu));
			return -E2BIG;
		}
	}
#if IS_ENABLED(CONFIG_IPV6)
	else if (skb->protocol == htons(ETH_P_IPV6)) {
		struct rt6_info *rt6 = (struct rt6_info *)skb_dst(skb);

		if (rt6 && mtu < dst_mtu(skb_dst(skb)) &&
			   mtu >= IPV6_MIN_MTU) {
			if ((tunnel->parms.iph.daddr &&
			    !ipv4_is_multicast(tunnel->parms.iph.daddr)) ||
			    rt6->rt6i_dst.plen == 128) {
				rt6->rt6i_flags |= RTF_MODIFIED;
				dst_metric_set(skb_dst(skb), RTAX_MTU, mtu);
			}
		}

		if (!skb_is_gso(skb) && mtu >= IPV6_MIN_MTU &&
					mtu < pkt_size) {
			icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
			return -E2BIG;
		}
	}
#endif
	return 0;
}

559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
void ip_md_tunnel_xmit(struct sk_buff *skb, struct net_device *dev, u8 proto)
{
	struct ip_tunnel *tunnel = netdev_priv(dev);
	u32 headroom = sizeof(struct iphdr);
	struct ip_tunnel_info *tun_info;
	const struct ip_tunnel_key *key;
	const struct iphdr *inner_iph;
	struct rtable *rt;
	struct flowi4 fl4;
	__be16 df = 0;
	u8 tos, ttl;

	tun_info = skb_tunnel_info(skb);
	if (unlikely(!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX) ||
		     ip_tunnel_info_af(tun_info) != AF_INET))
		goto tx_error;
	key = &tun_info->key;
	memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
	inner_iph = (const struct iphdr *)skb_inner_network_header(skb);
	tos = key->tos;
	if (tos == 1) {
		if (skb->protocol == htons(ETH_P_IP))
			tos = inner_iph->tos;
		else if (skb->protocol == htons(ETH_P_IPV6))
			tos = ipv6_get_dsfield((const struct ipv6hdr *)inner_iph);
	}
	init_tunnel_flow(&fl4, proto, key->u.ipv4.dst, key->u.ipv4.src, 0,
586
			 RT_TOS(tos), tunnel->parms.link, tunnel->fwmark);
587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620
	if (tunnel->encap.type != TUNNEL_ENCAP_NONE)
		goto tx_error;
	rt = ip_route_output_key(tunnel->net, &fl4);
	if (IS_ERR(rt)) {
		dev->stats.tx_carrier_errors++;
		goto tx_error;
	}
	if (rt->dst.dev == dev) {
		ip_rt_put(rt);
		dev->stats.collisions++;
		goto tx_error;
	}
	tos = ip_tunnel_ecn_encap(tos, inner_iph, skb);
	ttl = key->ttl;
	if (ttl == 0) {
		if (skb->protocol == htons(ETH_P_IP))
			ttl = inner_iph->ttl;
		else if (skb->protocol == htons(ETH_P_IPV6))
			ttl = ((const struct ipv6hdr *)inner_iph)->hop_limit;
		else
			ttl = ip4_dst_hoplimit(&rt->dst);
	}
	if (key->tun_flags & TUNNEL_DONT_FRAGMENT)
		df = htons(IP_DF);
	else if (skb->protocol == htons(ETH_P_IP))
		df = inner_iph->frag_off & htons(IP_DF);
	headroom += LL_RESERVED_SPACE(rt->dst.dev) + rt->dst.header_len;
	if (headroom > dev->needed_headroom)
		dev->needed_headroom = headroom;

	if (skb_cow_head(skb, dev->needed_headroom)) {
		ip_rt_put(rt);
		goto tx_dropped;
	}
621 622
	iptunnel_xmit(NULL, rt, skb, fl4.saddr, fl4.daddr, proto, tos, ttl,
		      df, !net_eq(tunnel->net, dev_net(dev)));
623 624 625 626 627 628 629 630 631 632 633
	return;
tx_error:
	dev->stats.tx_errors++;
	goto kfree;
tx_dropped:
	dev->stats.tx_dropped++;
kfree:
	kfree_skb(skb);
}
EXPORT_SYMBOL_GPL(ip_md_tunnel_xmit);

634
void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev,
635
		    const struct iphdr *tnl_params, u8 protocol)
636 637 638 639 640 641
{
	struct ip_tunnel *tunnel = netdev_priv(dev);
	const struct iphdr *inner_iph;
	struct flowi4 fl4;
	u8     tos, ttl;
	__be16 df;
642
	struct rtable *rt;		/* Route to the other host */
643 644
	unsigned int max_headroom;	/* The extra header space needed */
	__be32 dst;
645
	bool connected;
646 647

	inner_iph = (const struct iphdr *)skb_inner_network_header(skb);
648
	connected = (tunnel->parms.iph.daddr != 0);
649

650 651
	memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));

652 653 654 655
	dst = tnl_params->daddr;
	if (dst == 0) {
		/* NBMA tunnel */

656
		if (!skb_dst(skb)) {
657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673
			dev->stats.tx_fifo_errors++;
			goto tx_error;
		}

		if (skb->protocol == htons(ETH_P_IP)) {
			rt = skb_rtable(skb);
			dst = rt_nexthop(rt, inner_iph->daddr);
		}
#if IS_ENABLED(CONFIG_IPV6)
		else if (skb->protocol == htons(ETH_P_IPV6)) {
			const struct in6_addr *addr6;
			struct neighbour *neigh;
			bool do_tx_error_icmp;
			int addr_type;

			neigh = dst_neigh_lookup(skb_dst(skb),
						 &ipv6_hdr(skb)->daddr);
674
			if (!neigh)
675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697
				goto tx_error;

			addr6 = (const struct in6_addr *)&neigh->primary_key;
			addr_type = ipv6_addr_type(addr6);

			if (addr_type == IPV6_ADDR_ANY) {
				addr6 = &ipv6_hdr(skb)->daddr;
				addr_type = ipv6_addr_type(addr6);
			}

			if ((addr_type & IPV6_ADDR_COMPATv4) == 0)
				do_tx_error_icmp = true;
			else {
				do_tx_error_icmp = false;
				dst = addr6->s6_addr32[3];
			}
			neigh_release(neigh);
			if (do_tx_error_icmp)
				goto tx_error_icmp;
		}
#endif
		else
			goto tx_error;
Tom Herbert's avatar
Tom Herbert committed
698 699

		connected = false;
700 701 702 703 704
	}

	tos = tnl_params->tos;
	if (tos & 0x1) {
		tos &= ~0x1;
Tom Herbert's avatar
Tom Herbert committed
705
		if (skb->protocol == htons(ETH_P_IP)) {
706
			tos = inner_iph->tos;
Tom Herbert's avatar
Tom Herbert committed
707 708
			connected = false;
		} else if (skb->protocol == htons(ETH_P_IPV6)) {
709
			tos = ipv6_get_dsfield((const struct ipv6hdr *)inner_iph);
Tom Herbert's avatar
Tom Herbert committed
710 711
			connected = false;
		}
712 713
	}

Tom Herbert's avatar
Tom Herbert committed
714
	init_tunnel_flow(&fl4, protocol, dst, tnl_params->saddr,
715 716
			 tunnel->parms.o_key, RT_TOS(tos), tunnel->parms.link,
			 tunnel->fwmark);
Tom Herbert's avatar
Tom Herbert committed
717

718 719 720
	if (ip_tunnel_encap(skb, tunnel, &protocol, &fl4) < 0)
		goto tx_error;

721 722
	rt = connected ? dst_cache_get_ip4(&tunnel->dst_cache, &fl4.saddr) :
			 NULL;
Tom Herbert's avatar
Tom Herbert committed
723 724 725 726 727 728 729 730 731

	if (!rt) {
		rt = ip_route_output_key(tunnel->net, &fl4);

		if (IS_ERR(rt)) {
			dev->stats.tx_carrier_errors++;
			goto tx_error;
		}
		if (connected)
732 733
			dst_cache_set_ip4(&tunnel->dst_cache, &rt->dst,
					  fl4.saddr);
734
	}
Tom Herbert's avatar
Tom Herbert committed
735

736
	if (rt->dst.dev == dev) {
737 738 739 740 741
		ip_rt_put(rt);
		dev->stats.collisions++;
		goto tx_error;
	}

742
	if (tnl_update_pmtu(dev, skb, rt, tnl_params->frag_off, inner_iph)) {
743 744
		ip_rt_put(rt);
		goto tx_error;
745 746 747 748 749 750 751 752 753 754 755 756
	}

	if (tunnel->err_count > 0) {
		if (time_before(jiffies,
				tunnel->err_time + IPTUNNEL_ERR_TIMEO)) {
			tunnel->err_count--;

			dst_link_failure(skb);
		} else
			tunnel->err_count = 0;
	}

757
	tos = ip_tunnel_ecn_encap(tos, inner_iph, skb);
758 759 760 761 762 763 764 765 766 767 768 769
	ttl = tnl_params->ttl;
	if (ttl == 0) {
		if (skb->protocol == htons(ETH_P_IP))
			ttl = inner_iph->ttl;
#if IS_ENABLED(CONFIG_IPV6)
		else if (skb->protocol == htons(ETH_P_IPV6))
			ttl = ((const struct ipv6hdr *)inner_iph)->hop_limit;
#endif
		else
			ttl = ip4_dst_hoplimit(&rt->dst);
	}

770
	df = tnl_params->frag_off;
771
	if (skb->protocol == htons(ETH_P_IP) && !tunnel->ignore_df)
772 773
		df |= (inner_iph->frag_off&htons(IP_DF));

774
	max_headroom = LL_RESERVED_SPACE(rt->dst.dev) + sizeof(struct iphdr)
775
			+ rt->dst.header_len + ip_encap_hlen(&tunnel->encap);
776
	if (max_headroom > dev->needed_headroom)
777
		dev->needed_headroom = max_headroom;
778 779

	if (skb_cow_head(skb, dev->needed_headroom)) {
780
		ip_rt_put(rt);
781
		dev->stats.tx_dropped++;
782
		kfree_skb(skb);
783
		return;
784 785
	}

786 787
	iptunnel_xmit(NULL, rt, skb, fl4.saddr, fl4.daddr, protocol, tos, ttl,
		      df, !net_eq(tunnel->net, dev_net(dev)));
788 789 790 791 792 793 794 795
	return;

#if IS_ENABLED(CONFIG_IPV6)
tx_error_icmp:
	dst_link_failure(skb);
#endif
tx_error:
	dev->stats.tx_errors++;
796
	kfree_skb(skb);
797 798 799 800 801 802 803
}
EXPORT_SYMBOL_GPL(ip_tunnel_xmit);

static void ip_tunnel_update(struct ip_tunnel_net *itn,
			     struct ip_tunnel *t,
			     struct net_device *dev,
			     struct ip_tunnel_parm *p,
804 805
			     bool set_mtu,
			     __u32 fwmark)
806
{
807
	ip_tunnel_del(itn, t);
808 809 810 811 812 813 814 815 816 817 818 819 820 821
	t->parms.iph.saddr = p->iph.saddr;
	t->parms.iph.daddr = p->iph.daddr;
	t->parms.i_key = p->i_key;
	t->parms.o_key = p->o_key;
	if (dev->type != ARPHRD_ETHER) {
		memcpy(dev->dev_addr, &p->iph.saddr, 4);
		memcpy(dev->broadcast, &p->iph.daddr, 4);
	}
	ip_tunnel_add(itn, t);

	t->parms.iph.ttl = p->iph.ttl;
	t->parms.iph.tos = p->iph.tos;
	t->parms.iph.frag_off = p->iph.frag_off;

822
	if (t->parms.link != p->link || t->fwmark != fwmark) {
823 824 825
		int mtu;

		t->parms.link = p->link;
826
		t->fwmark = fwmark;
827 828 829 830
		mtu = ip_tunnel_bind_dev(dev);
		if (set_mtu)
			dev->mtu = mtu;
	}
831
	dst_cache_reset(&t->dst_cache);
832 833 834 835 836 837
	netdev_state_change(dev);
}

int ip_tunnel_ioctl(struct net_device *dev, struct ip_tunnel_parm *p, int cmd)
{
	int err = 0;
838 839 840
	struct ip_tunnel *t = netdev_priv(dev);
	struct net *net = t->net;
	struct ip_tunnel_net *itn = net_generic(net, t->ip_tnl_net_id);
841 842 843 844

	BUG_ON(!itn->fb_tunnel_dev);
	switch (cmd) {
	case SIOCGETTUNNEL:
845
		if (dev == itn->fb_tunnel_dev) {
846
			t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
847
			if (!t)
848 849
				t = netdev_priv(dev);
		}
850 851 852 853 854 855 856 857 858 859
		memcpy(p, &t->parms, sizeof(*p));
		break;

	case SIOCADDTUNNEL:
	case SIOCCHGTUNNEL:
		err = -EPERM;
		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
			goto done;
		if (p->iph.ttl)
			p->iph.frag_off |= htons(IP_DF);
860 861 862 863 864 865
		if (!(p->i_flags & VTI_ISVTI)) {
			if (!(p->i_flags & TUNNEL_KEY))
				p->i_key = 0;
			if (!(p->o_flags & TUNNEL_KEY))
				p->o_key = 0;
		}
866 867 868

		t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);

869 870 871 872 873 874 875 876
		if (cmd == SIOCADDTUNNEL) {
			if (!t) {
				t = ip_tunnel_create(net, itn, p);
				err = PTR_ERR_OR_ZERO(t);
				break;
			}

			err = -EEXIST;
Duan Jiong's avatar
Duan Jiong committed
877
			break;
878
		}
879
		if (dev != itn->fb_tunnel_dev && cmd == SIOCCHGTUNNEL) {
880
			if (t) {
881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903
				if (t->dev != dev) {
					err = -EEXIST;
					break;
				}
			} else {
				unsigned int nflags = 0;

				if (ipv4_is_multicast(p->iph.daddr))
					nflags = IFF_BROADCAST;
				else if (p->iph.daddr)
					nflags = IFF_POINTOPOINT;

				if ((dev->flags^nflags)&(IFF_POINTOPOINT|IFF_BROADCAST)) {
					err = -EINVAL;
					break;
				}

				t = netdev_priv(dev);
			}
		}

		if (t) {
			err = 0;
904
			ip_tunnel_update(itn, t, dev, p, true, 0);
905 906 907
		} else {
			err = -ENOENT;
		}
908 909 910 911 912 913 914 915 916 917
		break;

	case SIOCDELTUNNEL:
		err = -EPERM;
		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
			goto done;

		if (dev == itn->fb_tunnel_dev) {
			err = -ENOENT;
			t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
918
			if (!t)
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937
				goto done;
			err = -EPERM;
			if (t == netdev_priv(itn->fb_tunnel_dev))
				goto done;
			dev = t->dev;
		}
		unregister_netdevice(dev);
		err = 0;
		break;

	default:
		err = -EINVAL;
	}

done:
	return err;
}
EXPORT_SYMBOL_GPL(ip_tunnel_ioctl);

938
int __ip_tunnel_change_mtu(struct net_device *dev, int new_mtu, bool strict)
939 940 941
{
	struct ip_tunnel *tunnel = netdev_priv(dev);
	int t_hlen = tunnel->hlen + sizeof(struct iphdr);
942
	int max_mtu = 0xFFF8 - dev->hard_header_len - t_hlen;
943

944
	if (new_mtu < ETH_MIN_MTU)
945
		return -EINVAL;
946 947 948 949 950 951 952 953

	if (new_mtu > max_mtu) {
		if (strict)
			return -EINVAL;

		new_mtu = max_mtu;
	}

954 955 956
	dev->mtu = new_mtu;
	return 0;
}
957 958 959 960 961 962
EXPORT_SYMBOL_GPL(__ip_tunnel_change_mtu);

int ip_tunnel_change_mtu(struct net_device *dev, int new_mtu)
{
	return __ip_tunnel_change_mtu(dev, new_mtu, true);
}
963 964 965 966 967 968 969
EXPORT_SYMBOL_GPL(ip_tunnel_change_mtu);

static void ip_tunnel_dev_free(struct net_device *dev)
{
	struct ip_tunnel *tunnel = netdev_priv(dev);

	gro_cells_destroy(&tunnel->gro_cells);
970
	dst_cache_destroy(&tunnel->dst_cache);
971 972 973 974 975 976 977 978
	free_percpu(dev->tstats);
}

void ip_tunnel_dellink(struct net_device *dev, struct list_head *head)
{
	struct ip_tunnel *tunnel = netdev_priv(dev);
	struct ip_tunnel_net *itn;

Nicolas Dichtel's avatar
Nicolas Dichtel committed
979
	itn = net_generic(tunnel->net, tunnel->ip_tnl_net_id);
980 981

	if (itn->fb_tunnel_dev != dev) {
982
		ip_tunnel_del(itn, netdev_priv(dev));
983 984 985 986 987
		unregister_netdevice_queue(dev, head);
	}
}
EXPORT_SYMBOL_GPL(ip_tunnel_dellink);

988 989 990 991 992 993 994 995
struct net *ip_tunnel_get_link_net(const struct net_device *dev)
{
	struct ip_tunnel *tunnel = netdev_priv(dev);

	return tunnel->net;
}
EXPORT_SYMBOL(ip_tunnel_get_link_net);

996 997 998 999 1000 1001 1002 1003
int ip_tunnel_get_iflink(const struct net_device *dev)
{
	struct ip_tunnel *tunnel = netdev_priv(dev);

	return tunnel->parms.link;
}
EXPORT_SYMBOL(ip_tunnel_get_iflink);

1004
int ip_tunnel_init_net(struct net *net, unsigned int ip_tnl_net_id,
1005 1006 1007 1008
				  struct rtnl_link_ops *ops, char *devname)
{
	struct ip_tunnel_net *itn = net_generic(net, ip_tnl_net_id);
	struct ip_tunnel_parm parms;
1009
	unsigned int i;
1010

1011 1012
	for (i = 0; i < IP_TNL_HASH_SIZE; i++)
		INIT_HLIST_HEAD(&itn->tunnels[i]);
1013 1014 1015 1016 1017

	if (!ops) {
		itn->fb_tunnel_dev = NULL;
		return 0;
	}
1018

1019 1020 1021 1022 1023 1024
	memset(&parms, 0, sizeof(parms));
	if (devname)
		strlcpy(parms.name, devname, IFNAMSIZ);

	rtnl_lock();
	itn->fb_tunnel_dev = __ip_tunnel_create(net, ops, &parms);
1025 1026 1027
	/* FB netdevice is special: we have one, and only one per netns.
	 * Allowing to move it to another netns is clearly unsafe.
	 */
1028
	if (!IS_ERR(itn->fb_tunnel_dev)) {
1029
		itn->fb_tunnel_dev->features |= NETIF_F_NETNS_LOCAL;
1030
		itn->fb_tunnel_dev->mtu = ip_tunnel_bind_dev(itn->fb_tunnel_dev);
1031 1032
		ip_tunnel_add(itn, netdev_priv(itn->fb_tunnel_dev));
	}
1033
	rtnl_unlock();
1034

Sachin Kamat's avatar
Sachin Kamat committed
1035
	return PTR_ERR_OR_ZERO(itn->fb_tunnel_dev);
1036 1037 1038
}
EXPORT_SYMBOL_GPL(ip_tunnel_init_net);

Nicolas Dichtel's avatar
Nicolas Dichtel committed
1039 1040
static void ip_tunnel_destroy(struct ip_tunnel_net *itn, struct list_head *head,
			      struct rtnl_link_ops *ops)
1041
{
Nicolas Dichtel's avatar
Nicolas Dichtel committed
1042 1043
	struct net *net = dev_net(itn->fb_tunnel_dev);
	struct net_device *dev, *aux;
1044 1045
	int h;

Nicolas Dichtel's avatar
Nicolas Dichtel committed
1046 1047 1048 1049
	for_each_netdev_safe(net, dev, aux)
		if (dev->rtnl_link_ops == ops)
			unregister_netdevice_queue(dev, head);

1050 1051 1052 1053 1054 1055
	for (h = 0; h < IP_TNL_HASH_SIZE; h++) {
		struct ip_tunnel *t;
		struct hlist_node *n;
		struct hlist_head *thead = &itn->tunnels[h];

		hlist_for_each_entry_safe(t, n, thead, hash_node)
Nicolas Dichtel's avatar
Nicolas Dichtel committed
1056 1057 1058 1059 1060
			/* If dev is in the same netns, it has already
			 * been added to the list by the previous loop.
			 */
			if (!net_eq(dev_net(t->dev), net))
				unregister_netdevice_queue(t->dev, head);
1061 1062 1063
	}
}

Nicolas Dichtel's avatar
Nicolas Dichtel committed
1064
void ip_tunnel_delete_net(struct ip_tunnel_net *itn, struct rtnl_link_ops *ops)
1065 1066 1067 1068
{
	LIST_HEAD(list);

	rtnl_lock();
Nicolas Dichtel's avatar
Nicolas Dichtel committed
1069
	ip_tunnel_destroy(itn, &list, ops);
1070 1071 1072 1073 1074 1075
	unregister_netdevice_many(&list);
	rtnl_unlock();
}
EXPORT_SYMBOL_GPL(ip_tunnel_delete_net);

int ip_tunnel_newlink(struct net_device *dev, struct nlattr *tb[],
1076
		      struct ip_tunnel_parm *p, __u32 fwmark)
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086
{
	struct ip_tunnel *nt;
	struct net *net = dev_net(dev);
	struct ip_tunnel_net *itn;
	int mtu;
	int err;

	nt = netdev_priv(dev);
	itn = net_generic(net, nt->ip_tnl_net_id);

1087 1088 1089 1090 1091 1092 1093
	if (nt->collect_md) {
		if (rtnl_dereference(itn->collect_md_tun))
			return -EEXIST;
	} else {
		if (ip_tunnel_find(itn, p, dev->type))
			return -EEXIST;
	}
1094

Nicolas Dichtel's avatar
Nicolas Dichtel committed
1095
	nt->net = net;
1096
	nt->parms = *p;
1097
	nt->fwmark = fwmark;
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115
	err = register_netdevice(dev);
	if (err)
		goto out;

	if (dev->type == ARPHRD_ETHER && !tb[IFLA_ADDRESS])
		eth_hw_addr_random(dev);

	mtu = ip_tunnel_bind_dev(dev);
	if (!tb[IFLA_MTU])
		dev->mtu = mtu;

	ip_tunnel_add(itn, nt);
out:
	return err;
}
EXPORT_SYMBOL_GPL(ip_tunnel_newlink);

int ip_tunnel_changelink(struct net_device *dev, struct nlattr *tb[],
1116
			 struct ip_tunnel_parm *p, __u32 fwmark)
1117
{
Nicolas Dichtel's avatar
Nicolas Dichtel committed
1118
	struct ip_tunnel *t;
1119
	struct ip_tunnel *tunnel = netdev_priv(dev);
Nicolas Dichtel's avatar
Nicolas Dichtel committed
1120
	struct net *net = tunnel->net;
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
	struct ip_tunnel_net *itn = net_generic(net, tunnel->ip_tnl_net_id);

	if (dev == itn->fb_tunnel_dev)
		return -EINVAL;

	t = ip_tunnel_find(itn, p, dev->type);

	if (t) {
		if (t->dev != dev)
			return -EEXIST;
	} else {
Nicolas Dichtel's avatar
Nicolas Dichtel committed
1132
		t = tunnel;
1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147

		if (dev->type != ARPHRD_ETHER) {
			unsigned int nflags = 0;

			if (ipv4_is_multicast(p->iph.daddr))
				nflags = IFF_BROADCAST;
			else if (p->iph.daddr)
				nflags = IFF_POINTOPOINT;

			if ((dev->flags ^ nflags) &
			    (IFF_POINTOPOINT | IFF_BROADCAST))
				return -EINVAL;
		}
	}

1148
	ip_tunnel_update(itn, t, dev, p, !tb[IFLA_MTU], fwmark);
1149 1150 1151 1152 1153 1154 1155 1156
	return 0;
}
EXPORT_SYMBOL_GPL(ip_tunnel_changelink);

int ip_tunnel_init(struct net_device *dev)
{
	struct ip_tunnel *tunnel = netdev_priv(dev);
	struct iphdr *iph = &tunnel->parms.iph;
1157
	int err;
1158

1159 1160
	dev->needs_free_netdev = true;
	dev->priv_destructor = ip_tunnel_dev_free;
1161
	dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
1162 1163 1164
	if (!dev->tstats)
		return -ENOMEM;

1165 1166
	err = dst_cache_init(&tunnel->dst_cache, GFP_KERNEL);
	if (err) {
1167
		free_percpu(dev->tstats);
1168
		return err;
1169 1170
	}

1171 1172
	err = gro_cells_init(&tunnel->gro_cells, dev);
	if (err) {
1173
		dst_cache_destroy(&tunnel->dst_cache);
1174 1175 1176 1177 1178
		free_percpu(dev->tstats);
		return err;
	}

	tunnel->dev = dev;
Nicolas Dichtel's avatar
Nicolas Dichtel committed
1179
	tunnel->net = dev_net(dev);
1180 1181 1182 1183
	strcpy(tunnel->parms.name, dev->name);
	iph->version		= 4;
	iph->ihl		= 5;

1184 1185 1186 1187
	if (tunnel->collect_md) {
		dev->features |= NETIF_F_NETNS_LOCAL;
		netif_keep_dst(dev);
	}
1188 1189 1190 1191 1192 1193 1194
	return 0;
}
EXPORT_SYMBOL_GPL(ip_tunnel_init);

void ip_tunnel_uninit(struct net_device *dev)
{
	struct ip_tunnel *tunnel = netdev_priv(dev);
Nicolas Dichtel's avatar
Nicolas Dichtel committed
1195
	struct net *net = tunnel->net;
1196 1197 1198 1199 1200
	struct ip_tunnel_net *itn;

	itn = net_generic(net, tunnel->ip_tnl_net_id);
	/* fb_tunnel_dev will be unregisted in net-exit call. */
	if (itn->fb_tunnel_dev != dev)
1201
		ip_tunnel_del(itn, netdev_priv(dev));
Tom Herbert's avatar
Tom Herbert committed
1202

1203
	dst_cache_reset(&tunnel->dst_cache);
1204 1205 1206 1207
}
EXPORT_SYMBOL_GPL(ip_tunnel_uninit);

/* Do least required initialization, rest of init is done in tunnel_init call */
1208
void ip_tunnel_setup(struct net_device *dev, unsigned int net_id)
1209 1210 1211 1212 1213 1214 1215
{
	struct ip_tunnel *tunnel = netdev_priv(dev);
	tunnel->ip_tnl_net_id = net_id;
}
EXPORT_SYMBOL_GPL(ip_tunnel_setup);

MODULE_LICENSE("GPL");