| // SPDX-License-Identifier: GPL-2.0-or-later |
| /* |
| * Routines having to do with the 'struct sk_buff' memory handlers. |
| * |
| * Authors: Alan Cox <alan@lxorguk.ukuu.org.uk> |
| * Florian La Roche <rzsfl@rz.uni-sb.de> |
| * |
| * Fixes: |
| * Alan Cox : Fixed the worst of the load |
| * balancer bugs. |
| * Dave Platt : Interrupt stacking fix. |
| * Richard Kooijman : Timestamp fixes. |
| * Alan Cox : Changed buffer format. |
| * Alan Cox : destructor hook for AF_UNIX etc. |
| * Linus Torvalds : Better skb_clone. |
| * Alan Cox : Added skb_copy. |
| * Alan Cox : Added all the changed routines Linus |
| * only put in the headers |
| * Ray VanTassle : Fixed --skb->lock in free |
| * Alan Cox : skb_copy copy arp field |
| * Andi Kleen : slabified it. |
| * Robert Olsson : Removed skb_head_pool |
| * |
| * NOTE: |
| * The __skb_ routines should be called with interrupts |
| * disabled, or you better be *real* sure that the operation is atomic |
| * with respect to whatever list is being frobbed (e.g. via lock_sock() |
| * or via disabling bottom half handlers, etc). |
| */ |
| |
| /* |
| * The functions in this file will not compile correctly with gcc 2.4.x |
| */ |
| |
| #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
| |
| #include <linux/module.h> |
| #include <linux/types.h> |
| #include <linux/kernel.h> |
| #include <linux/mm.h> |
| #include <linux/interrupt.h> |
| #include <linux/in.h> |
| #include <linux/inet.h> |
| #include <linux/slab.h> |
| #include <linux/tcp.h> |
| #include <linux/udp.h> |
| #include <linux/sctp.h> |
| #include <linux/netdevice.h> |
| #ifdef CONFIG_NET_CLS_ACT |
| #include <net/pkt_sched.h> |
| #endif |
| #include <linux/string.h> |
| #include <linux/skbuff.h> |
| #include <linux/splice.h> |
| #include <linux/cache.h> |
| #include <linux/rtnetlink.h> |
| #include <linux/init.h> |
| #include <linux/scatterlist.h> |
| #include <linux/errqueue.h> |
| #include <linux/prefetch.h> |
| #include <linux/if_vlan.h> |
| #include <linux/mpls.h> |
| #include <linux/kcov.h> |
| |
| #include <net/protocol.h> |
| #include <net/dst.h> |
| #include <net/sock.h> |
| #include <net/checksum.h> |
| #include <net/ip6_checksum.h> |
| #include <net/xfrm.h> |
| #include <net/mpls.h> |
| #include <net/mptcp.h> |
| #include <net/page_pool.h> |
| |
| #include <linux/uaccess.h> |
| #include <trace/events/skb.h> |
| #include <linux/highmem.h> |
| #include <linux/capability.h> |
| #include <linux/user_namespace.h> |
| #include <linux/indirect_call_wrapper.h> |
| #include <trace/hooks/net.h> |
| |
| #include "datagram.h" |
| #include "sock_destructor.h" |
| |
| struct kmem_cache *skbuff_head_cache __ro_after_init; |
| static struct kmem_cache *skbuff_fclone_cache __ro_after_init; |
| #ifdef CONFIG_SKB_EXTENSIONS |
| static struct kmem_cache *skbuff_ext_cache __ro_after_init; |
| #endif |
| int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS; |
| EXPORT_SYMBOL(sysctl_max_skb_frags); |
| |
| /** |
| * skb_panic - private function for out-of-line support |
| * @skb: buffer |
| * @sz: size |
| * @addr: address |
| * @msg: skb_over_panic or skb_under_panic |
| * |
| * Out-of-line support for skb_put() and skb_push(). |
| * Called via the wrapper skb_over_panic() or skb_under_panic(). |
| * Keep out of line to prevent kernel bloat. |
| * __builtin_return_address is not used because it is not always reliable. |
| */ |
| static void skb_panic(struct sk_buff *skb, unsigned int sz, void *addr, |
| const char msg[]) |
| { |
| pr_emerg("%s: text:%px len:%d put:%d head:%px data:%px tail:%#lx end:%#lx dev:%s\n", |
| msg, addr, skb->len, sz, skb->head, skb->data, |
| (unsigned long)skb->tail, (unsigned long)skb->end, |
| skb->dev ? skb->dev->name : "<NULL>"); |
| BUG(); |
| } |
| |
| static void skb_over_panic(struct sk_buff *skb, unsigned int sz, void *addr) |
| { |
| skb_panic(skb, sz, addr, __func__); |
| } |
| |
| static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr) |
| { |
| skb_panic(skb, sz, addr, __func__); |
| } |
| |
| #define NAPI_SKB_CACHE_SIZE 64 |
| #define NAPI_SKB_CACHE_BULK 16 |
| #define NAPI_SKB_CACHE_HALF (NAPI_SKB_CACHE_SIZE / 2) |
| |
| struct napi_alloc_cache { |
| struct page_frag_cache page; |
| unsigned int skb_count; |
| void *skb_cache[NAPI_SKB_CACHE_SIZE]; |
| }; |
| |
| static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache); |
| static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache); |
| |
| static void *__alloc_frag_align(unsigned int fragsz, gfp_t gfp_mask, |
| unsigned int align_mask) |
| { |
| struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); |
| |
| return page_frag_alloc_align(&nc->page, fragsz, gfp_mask, align_mask); |
| } |
| |
| void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) |
| { |
| fragsz = SKB_DATA_ALIGN(fragsz); |
| |
| return __alloc_frag_align(fragsz, GFP_ATOMIC, align_mask); |
| } |
| EXPORT_SYMBOL(__napi_alloc_frag_align); |
| |
| void *__netdev_alloc_frag_align(unsigned int fragsz, unsigned int align_mask) |
| { |
| struct page_frag_cache *nc; |
| void *data; |
| |
| fragsz = SKB_DATA_ALIGN(fragsz); |
| if (in_hardirq() || irqs_disabled()) { |
| nc = this_cpu_ptr(&netdev_alloc_cache); |
| data = page_frag_alloc_align(nc, fragsz, GFP_ATOMIC, align_mask); |
| } else { |
| local_bh_disable(); |
| data = __alloc_frag_align(fragsz, GFP_ATOMIC, align_mask); |
| local_bh_enable(); |
| } |
| return data; |
| } |
| EXPORT_SYMBOL(__netdev_alloc_frag_align); |
| |
| static struct sk_buff *napi_skb_cache_get(void) |
| { |
| struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); |
| struct sk_buff *skb; |
| |
| if (unlikely(!nc->skb_count)) |
| nc->skb_count = kmem_cache_alloc_bulk(skbuff_head_cache, |
| GFP_ATOMIC, |
| NAPI_SKB_CACHE_BULK, |
| nc->skb_cache); |
| if (unlikely(!nc->skb_count)) |
| return NULL; |
| |
| skb = nc->skb_cache[--nc->skb_count]; |
| kasan_unpoison_object_data(skbuff_head_cache, skb); |
| |
| return skb; |
| } |
| |
| /* Caller must provide SKB that is memset cleared */ |
| static void __build_skb_around(struct sk_buff *skb, void *data, |
| unsigned int frag_size) |
| { |
| struct skb_shared_info *shinfo; |
| unsigned int size = frag_size ? : ksize(data); |
| |
| size -= SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); |
| |
| /* Assumes caller memset cleared SKB */ |
| skb->truesize = SKB_TRUESIZE(size); |
| refcount_set(&skb->users, 1); |
| skb->head = data; |
| skb->data = data; |
| skb_reset_tail_pointer(skb); |
| skb_set_end_offset(skb, size); |
| skb->mac_header = (typeof(skb->mac_header))~0U; |
| skb->transport_header = (typeof(skb->transport_header))~0U; |
| |
| /* make sure we initialize shinfo sequentially */ |
| shinfo = skb_shinfo(skb); |
| memset(shinfo, 0, offsetof(struct skb_shared_info, dataref)); |
| atomic_set(&shinfo->dataref, 1); |
| |
| skb_set_kcov_handle(skb, kcov_common_handle()); |
| } |
| |
| /** |
| * __build_skb - build a network buffer |
| * @data: data buffer provided by caller |
| * @frag_size: size of data, or 0 if head was kmalloced |
| * |
| * Allocate a new &sk_buff. Caller provides space holding head and |
| * skb_shared_info. @data must have been allocated by kmalloc() only if |
| * @frag_size is 0, otherwise data should come from the page allocator |
| * or vmalloc() |
| * The return is the new skb buffer. |
| * On a failure the return is %NULL, and @data is not freed. |
| * Notes : |
| * Before IO, driver allocates only data buffer where NIC put incoming frame |
| * Driver should add room at head (NET_SKB_PAD) and |
| * MUST add room at tail (SKB_DATA_ALIGN(skb_shared_info)) |
| * After IO, driver calls build_skb(), to allocate sk_buff and populate it |
| * before giving packet to stack. |
| * RX rings only contains data buffers, not full skbs. |
| */ |
| struct sk_buff *__build_skb(void *data, unsigned int frag_size) |
| { |
| struct sk_buff *skb; |
| |
| skb = kmem_cache_alloc(skbuff_head_cache, GFP_ATOMIC); |
| if (unlikely(!skb)) |
| return NULL; |
| |
| memset(skb, 0, offsetof(struct sk_buff, tail)); |
| __build_skb_around(skb, data, frag_size); |
| |
| return skb; |
| } |
| |
| /* build_skb() is wrapper over __build_skb(), that specifically |
| * takes care of skb->head and skb->pfmemalloc |
| * This means that if @frag_size is not zero, then @data must be backed |
| * by a page fragment, not kmalloc() or vmalloc() |
| */ |
| struct sk_buff *build_skb(void *data, unsigned int frag_size) |
| { |
| struct sk_buff *skb = __build_skb(data, frag_size); |
| |
| if (skb && frag_size) { |
| skb->head_frag = 1; |
| if (page_is_pfmemalloc(virt_to_head_page(data))) |
| skb->pfmemalloc = 1; |
| } |
| return skb; |
| } |
| EXPORT_SYMBOL(build_skb); |
| |
| /** |
| * build_skb_around - build a network buffer around provided skb |
| * @skb: sk_buff provide by caller, must be memset cleared |
| * @data: data buffer provided by caller |
| * @frag_size: size of data, or 0 if head was kmalloced |
| */ |
| struct sk_buff *build_skb_around(struct sk_buff *skb, |
| void *data, unsigned int frag_size) |
| { |
| if (unlikely(!skb)) |
| return NULL; |
| |
| __build_skb_around(skb, data, frag_size); |
| |
| if (frag_size) { |
| skb->head_frag = 1; |
| if (page_is_pfmemalloc(virt_to_head_page(data))) |
| skb->pfmemalloc = 1; |
| } |
| return skb; |
| } |
| EXPORT_SYMBOL(build_skb_around); |
| |
| /** |
| * __napi_build_skb - build a network buffer |
| * @data: data buffer provided by caller |
| * @frag_size: size of data, or 0 if head was kmalloced |
| * |
| * Version of __build_skb() that uses NAPI percpu caches to obtain |
| * skbuff_head instead of inplace allocation. |
| * |
| * Returns a new &sk_buff on success, %NULL on allocation failure. |
| */ |
| static struct sk_buff *__napi_build_skb(void *data, unsigned int frag_size) |
| { |
| struct sk_buff *skb; |
| |
| skb = napi_skb_cache_get(); |
| if (unlikely(!skb)) |
| return NULL; |
| |
| memset(skb, 0, offsetof(struct sk_buff, tail)); |
| __build_skb_around(skb, data, frag_size); |
| |
| return skb; |
| } |
| |
| /** |
| * napi_build_skb - build a network buffer |
| * @data: data buffer provided by caller |
| * @frag_size: size of data, or 0 if head was kmalloced |
| * |
| * Version of __napi_build_skb() that takes care of skb->head_frag |
| * and skb->pfmemalloc when the data is a page or page fragment. |
| * |
| * Returns a new &sk_buff on success, %NULL on allocation failure. |
| */ |
| struct sk_buff *napi_build_skb(void *data, unsigned int frag_size) |
| { |
| struct sk_buff *skb = __napi_build_skb(data, frag_size); |
| |
| if (likely(skb) && frag_size) { |
| skb->head_frag = 1; |
| skb_propagate_pfmemalloc(virt_to_head_page(data), skb); |
| } |
| |
| return skb; |
| } |
| EXPORT_SYMBOL(napi_build_skb); |
| |
| /* |
| * kmalloc_reserve is a wrapper around kmalloc_node_track_caller that tells |
| * the caller if emergency pfmemalloc reserves are being used. If it is and |
| * the socket is later found to be SOCK_MEMALLOC then PFMEMALLOC reserves |
| * may be used. Otherwise, the packet data may be discarded until enough |
| * memory is free |
| */ |
| static void *kmalloc_reserve(size_t size, gfp_t flags, int node, |
| bool *pfmemalloc) |
| { |
| void *obj; |
| bool ret_pfmemalloc = false; |
| |
| /* |
| * Try a regular allocation, when that fails and we're not entitled |
| * to the reserves, fail. |
| */ |
| obj = kmalloc_node_track_caller(size, |
| flags | __GFP_NOMEMALLOC | __GFP_NOWARN, |
| node); |
| if (obj || !(gfp_pfmemalloc_allowed(flags))) |
| goto out; |
| |
| /* Try again but now we are using pfmemalloc reserves */ |
| ret_pfmemalloc = true; |
| obj = kmalloc_node_track_caller(size, flags, node); |
| |
| out: |
| if (pfmemalloc) |
| *pfmemalloc = ret_pfmemalloc; |
| |
| return obj; |
| } |
| |
| /* Allocate a new skbuff. We do this ourselves so we can fill in a few |
| * 'private' fields and also do memory statistics to find all the |
| * [BEEP] leaks. |
| * |
| */ |
| |
| /** |
| * __alloc_skb - allocate a network buffer |
| * @size: size to allocate |
| * @gfp_mask: allocation mask |
| * @flags: If SKB_ALLOC_FCLONE is set, allocate from fclone cache |
| * instead of head cache and allocate a cloned (child) skb. |
| * If SKB_ALLOC_RX is set, __GFP_MEMALLOC will be used for |
| * allocations in case the data is required for writeback |
| * @node: numa node to allocate memory on |
| * |
| * Allocate a new &sk_buff. The returned buffer has no headroom and a |
| * tail room of at least size bytes. The object has a reference count |
| * of one. The return is the buffer. On a failure the return is %NULL. |
| * |
| * Buffers may only be allocated from interrupts using a @gfp_mask of |
| * %GFP_ATOMIC. |
| */ |
| struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask, |
| int flags, int node) |
| { |
| struct kmem_cache *cache; |
| struct sk_buff *skb; |
| u8 *data; |
| bool pfmemalloc; |
| |
| cache = (flags & SKB_ALLOC_FCLONE) |
| ? skbuff_fclone_cache : skbuff_head_cache; |
| |
| if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) |
| gfp_mask |= __GFP_MEMALLOC; |
| |
| /* Get the HEAD */ |
| if ((flags & (SKB_ALLOC_FCLONE | SKB_ALLOC_NAPI)) == SKB_ALLOC_NAPI && |
| likely(node == NUMA_NO_NODE || node == numa_mem_id())) |
| skb = napi_skb_cache_get(); |
| else |
| skb = kmem_cache_alloc_node(cache, gfp_mask & ~GFP_DMA, node); |
| if (unlikely(!skb)) |
| return NULL; |
| prefetchw(skb); |
| |
| /* We do our best to align skb_shared_info on a separate cache |
| * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives |
| * aligned memory blocks, unless SLUB/SLAB debug is enabled. |
| * Both skb->head and skb_shared_info are cache line aligned. |
| */ |
| size = SKB_DATA_ALIGN(size); |
| size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); |
| data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc); |
| if (unlikely(!data)) |
| goto nodata; |
| /* kmalloc(size) might give us more room than requested. |
| * Put skb_shared_info exactly at the end of allocated zone, |
| * to allow max possible filling before reallocation. |
| */ |
| size = SKB_WITH_OVERHEAD(ksize(data)); |
| prefetchw(data + size); |
| |
| /* |
| * Only clear those fields we need to clear, not those that we will |
| * actually initialise below. Hence, don't put any more fields after |
| * the tail pointer in struct sk_buff! |
| */ |
| memset(skb, 0, offsetof(struct sk_buff, tail)); |
| __build_skb_around(skb, data, 0); |
| skb->pfmemalloc = pfmemalloc; |
| |
| if (flags & SKB_ALLOC_FCLONE) { |
| struct sk_buff_fclones *fclones; |
| |
| fclones = container_of(skb, struct sk_buff_fclones, skb1); |
| |
| skb->fclone = SKB_FCLONE_ORIG; |
| refcount_set(&fclones->fclone_ref, 1); |
| |
| fclones->skb2.fclone = SKB_FCLONE_CLONE; |
| } |
| |
| return skb; |
| |
| nodata: |
| kmem_cache_free(cache, skb); |
| return NULL; |
| } |
| EXPORT_SYMBOL(__alloc_skb); |
| |
| /** |
| * __netdev_alloc_skb - allocate an skbuff for rx on a specific device |
| * @dev: network device to receive on |
| * @len: length to allocate |
| * @gfp_mask: get_free_pages mask, passed to alloc_skb |
| * |
| * Allocate a new &sk_buff and assign it a usage count of one. The |
| * buffer has NET_SKB_PAD headroom built in. Users should allocate |
| * the headroom they think they need without accounting for the |
| * built in space. The built in space is used for optimisations. |
| * |
| * %NULL is returned if there is no free memory. |
| */ |
| struct sk_buff *__netdev_alloc_skb(struct net_device *dev, unsigned int len, |
| gfp_t gfp_mask) |
| { |
| struct page_frag_cache *nc; |
| struct sk_buff *skb; |
| bool pfmemalloc; |
| void *data; |
| |
| len += NET_SKB_PAD; |
| |
| /* If requested length is either too small or too big, |
| * we use kmalloc() for skb->head allocation. |
| */ |
| if (len <= SKB_WITH_OVERHEAD(1024) || |
| len > SKB_WITH_OVERHEAD(PAGE_SIZE) || |
| (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { |
| skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX, NUMA_NO_NODE); |
| if (!skb) |
| goto skb_fail; |
| goto skb_success; |
| } |
| |
| len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); |
| len = SKB_DATA_ALIGN(len); |
| |
| if (sk_memalloc_socks()) |
| gfp_mask |= __GFP_MEMALLOC; |
| |
| if (in_hardirq() || irqs_disabled()) { |
| nc = this_cpu_ptr(&netdev_alloc_cache); |
| data = page_frag_alloc(nc, len, gfp_mask); |
| pfmemalloc = nc->pfmemalloc; |
| } else { |
| local_bh_disable(); |
| nc = this_cpu_ptr(&napi_alloc_cache.page); |
| data = page_frag_alloc(nc, len, gfp_mask); |
| pfmemalloc = nc->pfmemalloc; |
| local_bh_enable(); |
| } |
| |
| if (unlikely(!data)) |
| return NULL; |
| |
| skb = __build_skb(data, len); |
| if (unlikely(!skb)) { |
| skb_free_frag(data); |
| return NULL; |
| } |
| |
| if (pfmemalloc) |
| skb->pfmemalloc = 1; |
| skb->head_frag = 1; |
| |
| skb_success: |
| skb_reserve(skb, NET_SKB_PAD); |
| skb->dev = dev; |
| |
| skb_fail: |
| return skb; |
| } |
| EXPORT_SYMBOL(__netdev_alloc_skb); |
| |
| /** |
| * __napi_alloc_skb - allocate skbuff for rx in a specific NAPI instance |
| * @napi: napi instance this buffer was allocated for |
| * @len: length to allocate |
| * @gfp_mask: get_free_pages mask, passed to alloc_skb and alloc_pages |
| * |
| * Allocate a new sk_buff for use in NAPI receive. This buffer will |
| * attempt to allocate the head from a special reserved region used |
| * only for NAPI Rx allocation. By doing this we can save several |
| * CPU cycles by avoiding having to disable and re-enable IRQs. |
| * |
| * %NULL is returned if there is no free memory. |
| */ |
| struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len, |
| gfp_t gfp_mask) |
| { |
| struct napi_alloc_cache *nc; |
| struct sk_buff *skb; |
| void *data; |
| |
| len += NET_SKB_PAD + NET_IP_ALIGN; |
| |
| /* If requested length is either too small or too big, |
| * we use kmalloc() for skb->head allocation. |
| */ |
| if (len <= SKB_WITH_OVERHEAD(1024) || |
| len > SKB_WITH_OVERHEAD(PAGE_SIZE) || |
| (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) { |
| skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI, |
| NUMA_NO_NODE); |
| if (!skb) |
| goto skb_fail; |
| goto skb_success; |
| } |
| |
| nc = this_cpu_ptr(&napi_alloc_cache); |
| len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); |
| len = SKB_DATA_ALIGN(len); |
| |
| if (sk_memalloc_socks()) |
| gfp_mask |= __GFP_MEMALLOC; |
| |
| data = page_frag_alloc(&nc->page, len, gfp_mask); |
| if (unlikely(!data)) |
| return NULL; |
| |
| skb = __napi_build_skb(data, len); |
| if (unlikely(!skb)) { |
| skb_free_frag(data); |
| return NULL; |
| } |
| |
| if (nc->page.pfmemalloc) |
| skb->pfmemalloc = 1; |
| skb->head_frag = 1; |
| |
| skb_success: |
| skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); |
| skb->dev = napi->dev; |
| |
| skb_fail: |
| return skb; |
| } |
| EXPORT_SYMBOL(__napi_alloc_skb); |
| |
| void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, |
| int size, unsigned int truesize) |
| { |
| skb_fill_page_desc(skb, i, page, off, size); |
| skb->len += size; |
| skb->data_len += size; |
| skb->truesize += truesize; |
| } |
| EXPORT_SYMBOL(skb_add_rx_frag); |
| |
| void skb_coalesce_rx_frag(struct sk_buff *skb, int i, int size, |
| unsigned int truesize) |
| { |
| skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
| |
| skb_frag_size_add(frag, size); |
| skb->len += size; |
| skb->data_len += size; |
| skb->truesize += truesize; |
| } |
| EXPORT_SYMBOL(skb_coalesce_rx_frag); |
| |
| static void skb_drop_list(struct sk_buff **listp) |
| { |
| kfree_skb_list(*listp); |
| *listp = NULL; |
| } |
| |
| static inline void skb_drop_fraglist(struct sk_buff *skb) |
| { |
| skb_drop_list(&skb_shinfo(skb)->frag_list); |
| } |
| |
| static void skb_clone_fraglist(struct sk_buff *skb) |
| { |
| struct sk_buff *list; |
| |
| skb_walk_frags(skb, list) |
| skb_get(list); |
| } |
| |
| static void skb_free_head(struct sk_buff *skb) |
| { |
| unsigned char *head = skb->head; |
| |
| if (skb->head_frag) { |
| if (skb_pp_recycle(skb, head)) |
| return; |
| skb_free_frag(head); |
| } else { |
| kfree(head); |
| } |
| } |
| |
| static void skb_release_data(struct sk_buff *skb) |
| { |
| struct skb_shared_info *shinfo = skb_shinfo(skb); |
| int i; |
| |
| if (skb->cloned && |
| atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, |
| &shinfo->dataref)) |
| goto exit; |
| |
| skb_zcopy_clear(skb, true); |
| |
| for (i = 0; i < shinfo->nr_frags; i++) |
| __skb_frag_unref(&shinfo->frags[i], skb->pp_recycle); |
| |
| if (shinfo->frag_list) |
| kfree_skb_list(shinfo->frag_list); |
| |
| skb_free_head(skb); |
| exit: |
| /* When we clone an SKB we copy the reycling bit. The pp_recycle |
| * bit is only set on the head though, so in order to avoid races |
| * while trying to recycle fragments on __skb_frag_unref() we need |
| * to make one SKB responsible for triggering the recycle path. |
| * So disable the recycling bit if an SKB is cloned and we have |
| * additional references to to the fragmented part of the SKB. |
| * Eventually the last SKB will have the recycling bit set and it's |
| * dataref set to 0, which will trigger the recycling |
| */ |
| skb->pp_recycle = 0; |
| } |
| |
| /* |
| * Free an skbuff by memory without cleaning the state. |
| */ |
| static void kfree_skbmem(struct sk_buff *skb) |
| { |
| struct sk_buff_fclones *fclones; |
| |
| switch (skb->fclone) { |
| case SKB_FCLONE_UNAVAILABLE: |
| kmem_cache_free(skbuff_head_cache, skb); |
| return; |
| |
| case SKB_FCLONE_ORIG: |
| fclones = container_of(skb, struct sk_buff_fclones, skb1); |
| |
| /* We usually free the clone (TX completion) before original skb |
| * This test would have no chance to be true for the clone, |
| * while here, branch prediction will be good. |
| */ |
| if (refcount_read(&fclones->fclone_ref) == 1) |
| goto fastpath; |
| break; |
| |
| default: /* SKB_FCLONE_CLONE */ |
| fclones = container_of(skb, struct sk_buff_fclones, skb2); |
| break; |
| } |
| if (!refcount_dec_and_test(&fclones->fclone_ref)) |
| return; |
| fastpath: |
| kmem_cache_free(skbuff_fclone_cache, fclones); |
| } |
| |
| void skb_release_head_state(struct sk_buff *skb) |
| { |
| skb_dst_drop(skb); |
| if (skb->destructor) { |
| WARN_ON(in_hardirq()); |
| skb->destructor(skb); |
| } |
| #if IS_ENABLED(CONFIG_NF_CONNTRACK) |
| nf_conntrack_put(skb_nfct(skb)); |
| #endif |
| skb_ext_put(skb); |
| } |
| |
| /* Free everything but the sk_buff shell. */ |
| static void skb_release_all(struct sk_buff *skb) |
| { |
| skb_release_head_state(skb); |
| if (likely(skb->head)) |
| skb_release_data(skb); |
| } |
| |
| /** |
| * __kfree_skb - private function |
| * @skb: buffer |
| * |
| * Free an sk_buff. Release anything attached to the buffer. |
| * Clean the state. This is an internal helper function. Users should |
| * always call kfree_skb |
| */ |
| |
| void __kfree_skb(struct sk_buff *skb) |
| { |
| skb_release_all(skb); |
| kfree_skbmem(skb); |
| } |
| EXPORT_SYMBOL(__kfree_skb); |
| |
| /** |
| * kfree_skb_reason - free an sk_buff with special reason |
| * @skb: buffer to free |
| * @reason: reason why this skb is dropped |
| * |
| * Drop a reference to the buffer and free it if the usage count has |
| * hit zero. Meanwhile, pass the drop reason to 'kfree_skb' |
| * tracepoint. |
| */ |
| void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason) |
| { |
| if (!skb_unref(skb)) |
| return; |
| |
| trace_android_vh_kfree_skb(skb); |
| trace_kfree_skb(skb, __builtin_return_address(0), reason); |
| __kfree_skb(skb); |
| } |
| EXPORT_SYMBOL(kfree_skb_reason); |
| |
| /** |
| * kfree_skb - free an sk_buff with 'NOT_SPECIFIED' reason |
| * @skb: buffer to free |
| */ |
| void kfree_skb(struct sk_buff *skb) |
| { |
| /* ANDROID: only still present to preserve the ABI, this went away in |
| * mainline and LTS releases, to be replaced with an inline version |
| * instead. |
| */ |
| kfree_skb_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED); |
| } |
| EXPORT_SYMBOL(kfree_skb); |
| |
| void kfree_skb_list(struct sk_buff *segs) |
| { |
| while (segs) { |
| struct sk_buff *next = segs->next; |
| |
| kfree_skb(segs); |
| segs = next; |
| } |
| } |
| EXPORT_SYMBOL(kfree_skb_list); |
| |
| /* Dump skb information and contents. |
| * |
| * Must only be called from net_ratelimit()-ed paths. |
| * |
| * Dumps whole packets if full_pkt, only headers otherwise. |
| */ |
| void skb_dump(const char *level, const struct sk_buff *skb, bool full_pkt) |
| { |
| struct skb_shared_info *sh = skb_shinfo(skb); |
| struct net_device *dev = skb->dev; |
| struct sock *sk = skb->sk; |
| struct sk_buff *list_skb; |
| bool has_mac, has_trans; |
| int headroom, tailroom; |
| int i, len, seg_len; |
| |
| if (full_pkt) |
| len = skb->len; |
| else |
| len = min_t(int, skb->len, MAX_HEADER + 128); |
| |
| headroom = skb_headroom(skb); |
| tailroom = skb_tailroom(skb); |
| |
| has_mac = skb_mac_header_was_set(skb); |
| has_trans = skb_transport_header_was_set(skb); |
| |
| printk("%sskb len=%u headroom=%u headlen=%u tailroom=%u\n" |
| "mac=(%d,%d) net=(%d,%d) trans=%d\n" |
| "shinfo(txflags=%u nr_frags=%u gso(size=%hu type=%u segs=%hu))\n" |
| "csum(0x%x ip_summed=%u complete_sw=%u valid=%u level=%u)\n" |
| "hash(0x%x sw=%u l4=%u) proto=0x%04x pkttype=%u iif=%d\n", |
| level, skb->len, headroom, skb_headlen(skb), tailroom, |
| has_mac ? skb->mac_header : -1, |
| has_mac ? skb_mac_header_len(skb) : -1, |
| skb->network_header, |
| has_trans ? skb_network_header_len(skb) : -1, |
| has_trans ? skb->transport_header : -1, |
| sh->tx_flags, sh->nr_frags, |
| sh->gso_size, sh->gso_type, sh->gso_segs, |
| skb->csum, skb->ip_summed, skb->csum_complete_sw, |
| skb->csum_valid, skb->csum_level, |
| skb->hash, skb->sw_hash, skb->l4_hash, |
| ntohs(skb->protocol), skb->pkt_type, skb->skb_iif); |
| |
| if (dev) |
| printk("%sdev name=%s feat=%pNF\n", |
| level, dev->name, &dev->features); |
| if (sk) |
| printk("%ssk family=%hu type=%u proto=%u\n", |
| level, sk->sk_family, sk->sk_type, sk->sk_protocol); |
| |
| if (full_pkt && headroom) |
| print_hex_dump(level, "skb headroom: ", DUMP_PREFIX_OFFSET, |
| 16, 1, skb->head, headroom, false); |
| |
| seg_len = min_t(int, skb_headlen(skb), len); |
| if (seg_len) |
| print_hex_dump(level, "skb linear: ", DUMP_PREFIX_OFFSET, |
| 16, 1, skb->data, seg_len, false); |
| len -= seg_len; |
| |
| if (full_pkt && tailroom) |
| print_hex_dump(level, "skb tailroom: ", DUMP_PREFIX_OFFSET, |
| 16, 1, skb_tail_pointer(skb), tailroom, false); |
| |
| for (i = 0; len && i < skb_shinfo(skb)->nr_frags; i++) { |
| skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
| u32 p_off, p_len, copied; |
| struct page *p; |
| u8 *vaddr; |
| |
| skb_frag_foreach_page(frag, skb_frag_off(frag), |
| skb_frag_size(frag), p, p_off, p_len, |
| copied) { |
| seg_len = min_t(int, p_len, len); |
| vaddr = kmap_atomic(p); |
| print_hex_dump(level, "skb frag: ", |
| DUMP_PREFIX_OFFSET, |
| 16, 1, vaddr + p_off, seg_len, false); |
| kunmap_atomic(vaddr); |
| len -= seg_len; |
| if (!len) |
| break; |
| } |
| } |
| |
| if (full_pkt && skb_has_frag_list(skb)) { |
| printk("skb fraglist:\n"); |
| skb_walk_frags(skb, list_skb) |
| skb_dump(level, list_skb, true); |
| } |
| } |
| EXPORT_SYMBOL(skb_dump); |
| |
| /** |
| * skb_tx_error - report an sk_buff xmit error |
| * @skb: buffer that triggered an error |
| * |
| * Report xmit error if a device callback is tracking this skb. |
| * skb must be freed afterwards. |
| */ |
| void skb_tx_error(struct sk_buff *skb) |
| { |
| skb_zcopy_clear(skb, true); |
| } |
| EXPORT_SYMBOL(skb_tx_error); |
| |
| #ifdef CONFIG_TRACEPOINTS |
| /** |
| * consume_skb - free an skbuff |
| * @skb: buffer to free |
| * |
| * Drop a ref to the buffer and free it if the usage count has hit zero |
| * Functions identically to kfree_skb, but kfree_skb assumes that the frame |
| * is being dropped after a failure and notes that |
| */ |
| void consume_skb(struct sk_buff *skb) |
| { |
| if (!skb_unref(skb)) |
| return; |
| |
| trace_consume_skb(skb); |
| __kfree_skb(skb); |
| } |
| EXPORT_SYMBOL(consume_skb); |
| #endif |
| |
| /** |
| * __consume_stateless_skb - free an skbuff, assuming it is stateless |
| * @skb: buffer to free |
| * |
| * Alike consume_skb(), but this variant assumes that this is the last |
| * skb reference and all the head states have been already dropped |
| */ |
| void __consume_stateless_skb(struct sk_buff *skb) |
| { |
| trace_consume_skb(skb); |
| skb_release_data(skb); |
| kfree_skbmem(skb); |
| } |
| |
| static void napi_skb_cache_put(struct sk_buff *skb) |
| { |
| struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache); |
| u32 i; |
| |
| kasan_poison_object_data(skbuff_head_cache, skb); |
| nc->skb_cache[nc->skb_count++] = skb; |
| |
| if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { |
| for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++) |
| kasan_unpoison_object_data(skbuff_head_cache, |
| nc->skb_cache[i]); |
| |
| kmem_cache_free_bulk(skbuff_head_cache, NAPI_SKB_CACHE_HALF, |
| nc->skb_cache + NAPI_SKB_CACHE_HALF); |
| nc->skb_count = NAPI_SKB_CACHE_HALF; |
| } |
| } |
| |
| void __kfree_skb_defer(struct sk_buff *skb) |
| { |
| skb_release_all(skb); |
| napi_skb_cache_put(skb); |
| } |
| |
| void napi_skb_free_stolen_head(struct sk_buff *skb) |
| { |
| if (unlikely(skb->slow_gro)) { |
| nf_reset_ct(skb); |
| skb_dst_drop(skb); |
| skb_ext_put(skb); |
| skb_orphan(skb); |
| skb->slow_gro = 0; |
| } |
| napi_skb_cache_put(skb); |
| } |
| |
| void napi_consume_skb(struct sk_buff *skb, int budget) |
| { |
| /* Zero budget indicate non-NAPI context called us, like netpoll */ |
| if (unlikely(!budget)) { |
| dev_consume_skb_any(skb); |
| return; |
| } |
| |
| lockdep_assert_in_softirq(); |
| |
| if (!skb_unref(skb)) |
| return; |
| |
| /* if reaching here SKB is ready to free */ |
| trace_consume_skb(skb); |
| |
| /* if SKB is a clone, don't handle this case */ |
| if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { |
| __kfree_skb(skb); |
| return; |
| } |
| |
| skb_release_all(skb); |
| napi_skb_cache_put(skb); |
| } |
| EXPORT_SYMBOL(napi_consume_skb); |
| |
| /* Make sure a field is enclosed inside headers_start/headers_end section */ |
| #define CHECK_SKB_FIELD(field) \ |
| BUILD_BUG_ON(offsetof(struct sk_buff, field) < \ |
| offsetof(struct sk_buff, headers_start)); \ |
| BUILD_BUG_ON(offsetof(struct sk_buff, field) > \ |
| offsetof(struct sk_buff, headers_end)); \ |
| |
| static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) |
| { |
| new->tstamp = old->tstamp; |
| /* We do not copy old->sk */ |
| new->dev = old->dev; |
| memcpy(new->cb, old->cb, sizeof(old->cb)); |
| skb_dst_copy(new, old); |
| __skb_ext_copy(new, old); |
| __nf_copy(new, old, false); |
| |
| /* Note : this field could be in headers_start/headers_end section |
| * It is not yet because we do not want to have a 16 bit hole |
| */ |
| new->queue_mapping = old->queue_mapping; |
| |
| memcpy(&new->headers_start, &old->headers_start, |
| offsetof(struct sk_buff, headers_end) - |
| offsetof(struct sk_buff, headers_start)); |
| CHECK_SKB_FIELD(protocol); |
| CHECK_SKB_FIELD(csum); |
| CHECK_SKB_FIELD(hash); |
| CHECK_SKB_FIELD(priority); |
| CHECK_SKB_FIELD(skb_iif); |
| CHECK_SKB_FIELD(vlan_proto); |
| CHECK_SKB_FIELD(vlan_tci); |
| CHECK_SKB_FIELD(transport_header); |
| CHECK_SKB_FIELD(network_header); |
| CHECK_SKB_FIELD(mac_header); |
| CHECK_SKB_FIELD(inner_protocol); |
| CHECK_SKB_FIELD(inner_transport_header); |
| CHECK_SKB_FIELD(inner_network_header); |
| CHECK_SKB_FIELD(inner_mac_header); |
| CHECK_SKB_FIELD(mark); |
| #ifdef CONFIG_NETWORK_SECMARK |
| CHECK_SKB_FIELD(secmark); |
| #endif |
| #ifdef CONFIG_NET_RX_BUSY_POLL |
| CHECK_SKB_FIELD(napi_id); |
| #endif |
| #ifdef CONFIG_XPS |
| CHECK_SKB_FIELD(sender_cpu); |
| #endif |
| #ifdef CONFIG_NET_SCHED |
| CHECK_SKB_FIELD(tc_index); |
| #endif |
| /* ANDROID: |
| * Due to attempts to keep the ABI stable for struct sk_buff, the new |
| * fields were incorrectly added _AFTER_ the headers_end field, which |
| * requires that we manually copy the fields here from the old to the |
| * new one. |
| * Be sure to add any new field that is added in the |
| * ANDROID_KABI_REPLACE() macros below here as well. |
| */ |
| new->scm_io_uring = old->scm_io_uring; |
| } |
| |
| /* |
| * You should not add any new code to this function. Add it to |
| * __copy_skb_header above instead. |
| */ |
| static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb) |
| { |
| #define C(x) n->x = skb->x |
| |
| n->next = n->prev = NULL; |
| n->sk = NULL; |
| __copy_skb_header(n, skb); |
| |
| C(len); |
| C(data_len); |
| C(mac_len); |
| n->hdr_len = skb->nohdr ? skb_headroom(skb) : skb->hdr_len; |
| n->cloned = 1; |
| n->nohdr = 0; |
| n->peeked = 0; |
| C(pfmemalloc); |
| C(pp_recycle); |
| n->destructor = NULL; |
| C(tail); |
| C(end); |
| C(head); |
| C(head_frag); |
| C(data); |
| C(truesize); |
| refcount_set(&n->users, 1); |
| |
| atomic_inc(&(skb_shinfo(skb)->dataref)); |
| skb->cloned = 1; |
| |
| return n; |
| #undef C |
| } |
| |
| /** |
| * alloc_skb_for_msg() - allocate sk_buff to wrap frag list forming a msg |
| * @first: first sk_buff of the msg |
| */ |
| struct sk_buff *alloc_skb_for_msg(struct sk_buff *first) |
| { |
| struct sk_buff *n; |
| |
| n = alloc_skb(0, GFP_ATOMIC); |
| if (!n) |
| return NULL; |
| |
| n->len = first->len; |
| n->data_len = first->len; |
| n->truesize = first->truesize; |
| |
| skb_shinfo(n)->frag_list = first; |
| |
| __copy_skb_header(n, first); |
| n->destructor = NULL; |
| |
| return n; |
| } |
| EXPORT_SYMBOL_GPL(alloc_skb_for_msg); |
| |
| /** |
| * skb_morph - morph one skb into another |
| * @dst: the skb to receive the contents |
| * @src: the skb to supply the contents |
| * |
| * This is identical to skb_clone except that the target skb is |
| * supplied by the user. |
| * |
| * The target skb is returned upon exit. |
| */ |
| struct sk_buff *skb_morph(struct sk_buff *dst, struct sk_buff *src) |
| { |
| skb_release_all(dst); |
| return __skb_clone(dst, src); |
| } |
| EXPORT_SYMBOL_GPL(skb_morph); |
| |
| int mm_account_pinned_pages(struct mmpin *mmp, size_t size) |
| { |
| unsigned long max_pg, num_pg, new_pg, old_pg; |
| struct user_struct *user; |
| |
| if (capable(CAP_IPC_LOCK) || !size) |
| return 0; |
| |
| num_pg = (size >> PAGE_SHIFT) + 2; /* worst case */ |
| max_pg = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; |
| user = mmp->user ? : current_user(); |
| |
| do { |
| old_pg = atomic_long_read(&user->locked_vm); |
| new_pg = old_pg + num_pg; |
| if (new_pg > max_pg) |
| return -ENOBUFS; |
| } while (atomic_long_cmpxchg(&user->locked_vm, old_pg, new_pg) != |
| old_pg); |
| |
| if (!mmp->user) { |
| mmp->user = get_uid(user); |
| mmp->num_pg = num_pg; |
| } else { |
| mmp->num_pg += num_pg; |
| } |
| |
| return 0; |
| } |
| EXPORT_SYMBOL_GPL(mm_account_pinned_pages); |
| |
| void mm_unaccount_pinned_pages(struct mmpin *mmp) |
| { |
| if (mmp->user) { |
| atomic_long_sub(mmp->num_pg, &mmp->user->locked_vm); |
| free_uid(mmp->user); |
| } |
| } |
| EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages); |
| |
| struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size) |
| { |
| struct ubuf_info *uarg; |
| struct sk_buff *skb; |
| |
| WARN_ON_ONCE(!in_task()); |
| |
| skb = sock_omalloc(sk, 0, GFP_KERNEL); |
| if (!skb) |
| return NULL; |
| |
| BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); |
| uarg = (void *)skb->cb; |
| uarg->mmp.user = NULL; |
| |
| if (mm_account_pinned_pages(&uarg->mmp, size)) { |
| kfree_skb(skb); |
| return NULL; |
| } |
| |
| uarg->callback = msg_zerocopy_callback; |
| uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; |
| uarg->len = 1; |
| uarg->bytelen = size; |
| uarg->zerocopy = 1; |
| uarg->flags = SKBFL_ZEROCOPY_FRAG; |
| refcount_set(&uarg->refcnt, 1); |
| sock_hold(sk); |
| |
| return uarg; |
| } |
| EXPORT_SYMBOL_GPL(msg_zerocopy_alloc); |
| |
| static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg) |
| { |
| return container_of((void *)uarg, struct sk_buff, cb); |
| } |
| |
| struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size, |
| struct ubuf_info *uarg) |
| { |
| if (uarg) { |
| const u32 byte_limit = 1 << 19; /* limit to a few TSO */ |
| u32 bytelen, next; |
| |
| /* realloc only when socket is locked (TCP, UDP cork), |
| * so uarg->len and sk_zckey access is serialized |
| */ |
| if (!sock_owned_by_user(sk)) { |
| WARN_ON_ONCE(1); |
| return NULL; |
| } |
| |
| bytelen = uarg->bytelen + size; |
| if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) { |
| /* TCP can create new skb to attach new uarg */ |
| if (sk->sk_type == SOCK_STREAM) |
| goto new_alloc; |
| return NULL; |
| } |
| |
| next = (u32)atomic_read(&sk->sk_zckey); |
| if ((u32)(uarg->id + uarg->len) == next) { |
| if (mm_account_pinned_pages(&uarg->mmp, size)) |
| return NULL; |
| uarg->len++; |
| uarg->bytelen = bytelen; |
| atomic_set(&sk->sk_zckey, ++next); |
| |
| /* no extra ref when appending to datagram (MSG_MORE) */ |
| if (sk->sk_type == SOCK_STREAM) |
| net_zcopy_get(uarg); |
| |
| return uarg; |
| } |
| } |
| |
| new_alloc: |
| return msg_zerocopy_alloc(sk, size); |
| } |
| EXPORT_SYMBOL_GPL(msg_zerocopy_realloc); |
| |
| static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len) |
| { |
| struct sock_exterr_skb *serr = SKB_EXT_ERR(skb); |
| u32 old_lo, old_hi; |
| u64 sum_len; |
| |
| old_lo = serr->ee.ee_info; |
| old_hi = serr->ee.ee_data; |
| sum_len = old_hi - old_lo + 1ULL + len; |
| |
| if (sum_len >= (1ULL << 32)) |
| return false; |
| |
| if (lo != old_hi + 1) |
| return false; |
| |
| serr->ee.ee_data += len; |
| return true; |
| } |
| |
| static void __msg_zerocopy_callback(struct ubuf_info *uarg) |
| { |
| struct sk_buff *tail, *skb = skb_from_uarg(uarg); |
| struct sock_exterr_skb *serr; |
| struct sock *sk = skb->sk; |
| struct sk_buff_head *q; |
| unsigned long flags; |
| bool is_zerocopy; |
| u32 lo, hi; |
| u16 len; |
| |
| mm_unaccount_pinned_pages(&uarg->mmp); |
| |
| /* if !len, there was only 1 call, and it was aborted |
| * so do not queue a completion notification |
| */ |
| if (!uarg->len || sock_flag(sk, SOCK_DEAD)) |
| goto release; |
| |
| len = uarg->len; |
| lo = uarg->id; |
| hi = uarg->id + len - 1; |
| is_zerocopy = uarg->zerocopy; |
| |
| serr = SKB_EXT_ERR(skb); |
| memset(serr, 0, sizeof(*serr)); |
| serr->ee.ee_errno = 0; |
| serr->ee.ee_origin = SO_EE_ORIGIN_ZEROCOPY; |
| serr->ee.ee_data = hi; |
| serr->ee.ee_info = lo; |
| if (!is_zerocopy) |
| serr->ee.ee_code |= SO_EE_CODE_ZEROCOPY_COPIED; |
| |
| q = &sk->sk_error_queue; |
| spin_lock_irqsave(&q->lock, flags); |
| tail = skb_peek_tail(q); |
| if (!tail || SKB_EXT_ERR(tail)->ee.ee_origin != SO_EE_ORIGIN_ZEROCOPY || |
| !skb_zerocopy_notify_extend(tail, lo, len)) { |
| __skb_queue_tail(q, skb); |
| skb = NULL; |
| } |
| spin_unlock_irqrestore(&q->lock, flags); |
| |
| sk_error_report(sk); |
| |
| release: |
| consume_skb(skb); |
| sock_put(sk); |
| } |
| |
| void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, |
| bool success) |
| { |
| uarg->zerocopy = uarg->zerocopy & success; |
| |
| if (refcount_dec_and_test(&uarg->refcnt)) |
| __msg_zerocopy_callback(uarg); |
| } |
| EXPORT_SYMBOL_GPL(msg_zerocopy_callback); |
| |
| void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) |
| { |
| struct sock *sk = skb_from_uarg(uarg)->sk; |
| |
| atomic_dec(&sk->sk_zckey); |
| uarg->len--; |
| |
| if (have_uref) |
| msg_zerocopy_callback(NULL, uarg, true); |
| } |
| EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort); |
| |
| int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len) |
| { |
| return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len); |
| } |
| EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram); |
| |
| int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, |
| struct msghdr *msg, int len, |
| struct ubuf_info *uarg) |
| { |
| struct ubuf_info *orig_uarg = skb_zcopy(skb); |
| struct iov_iter orig_iter = msg->msg_iter; |
| int err, orig_len = skb->len; |
| |
| /* An skb can only point to one uarg. This edge case happens when |
| * TCP appends to an skb, but zerocopy_realloc triggered a new alloc. |
| */ |
| if (orig_uarg && uarg != orig_uarg) |
| return -EEXIST; |
| |
| err = __zerocopy_sg_from_iter(sk, skb, &msg->msg_iter, len); |
| if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { |
| struct sock *save_sk = skb->sk; |
| |
| /* Streams do not free skb on error. Reset to prev state. */ |
| msg->msg_iter = orig_iter; |
| skb->sk = sk; |
| ___pskb_trim(skb, orig_len); |
| skb->sk = save_sk; |
| return err; |
| } |
| |
| skb_zcopy_set(skb, uarg, NULL); |
| return skb->len - orig_len; |
| } |
| EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream); |
| |
| static int skb_zerocopy_clone(struct sk_buff *nskb, struct sk_buff *orig, |
| gfp_t gfp_mask) |
| { |
| if (skb_zcopy(orig)) { |
| if (skb_zcopy(nskb)) { |
| /* !gfp_mask callers are verified to !skb_zcopy(nskb) */ |
| if (!gfp_mask) { |
| WARN_ON_ONCE(1); |
| return -ENOMEM; |
| } |
| if (skb_uarg(nskb) == skb_uarg(orig)) |
| return 0; |
| if (skb_copy_ubufs(nskb, GFP_ATOMIC)) |
| return -EIO; |
| } |
| skb_zcopy_set(nskb, skb_uarg(orig), NULL); |
| } |
| return 0; |
| } |
| |
| /** |
| * skb_copy_ubufs - copy userspace skb frags buffers to kernel |
| * @skb: the skb to modify |
| * @gfp_mask: allocation priority |
| * |
| * This must be called on skb with SKBFL_ZEROCOPY_ENABLE. |
| * It will copy all frags into kernel and drop the reference |
| * to userspace pages. |
| * |
| * If this function is called from an interrupt gfp_mask() must be |
| * %GFP_ATOMIC. |
| * |
| * Returns 0 on success or a negative error code on failure |
| * to allocate kernel memory to copy to. |
| */ |
| int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) |
| { |
| int num_frags = skb_shinfo(skb)->nr_frags; |
| struct page *page, *head = NULL; |
| int i, new_frags; |
| u32 d_off; |
| |
| if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) |
| return -EINVAL; |
| |
| if (!num_frags) |
| goto release; |
| |
| new_frags = (__skb_pagelen(skb) + PAGE_SIZE - 1) >> PAGE_SHIFT; |
| for (i = 0; i < new_frags; i++) { |
| page = alloc_page(gfp_mask); |
| if (!page) { |
| while (head) { |
| struct page *next = (struct page *)page_private(head); |
| put_page(head); |
| head = next; |
| } |
| return -ENOMEM; |
| } |
| set_page_private(page, (unsigned long)head); |
| head = page; |
| } |
| |
| page = head; |
| d_off = 0; |
| for (i = 0; i < num_frags; i++) { |
| skb_frag_t *f = &skb_shinfo(skb)->frags[i]; |
| u32 p_off, p_len, copied; |
| struct page *p; |
| u8 *vaddr; |
| |
| skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f), |
| p, p_off, p_len, copied) { |
| u32 copy, done = 0; |
| vaddr = kmap_atomic(p); |
| |
| while (done < p_len) { |
| if (d_off == PAGE_SIZE) { |
| d_off = 0; |
| page = (struct page *)page_private(page); |
| } |
| copy = min_t(u32, PAGE_SIZE - d_off, p_len - done); |
| memcpy(page_address(page) + d_off, |
| vaddr + p_off + done, copy); |
| done += copy; |
| d_off += copy; |
| } |
| kunmap_atomic(vaddr); |
| } |
| } |
| |
| /* skb frags release userspace buffers */ |
| for (i = 0; i < num_frags; i++) |
| skb_frag_unref(skb, i); |
| |
| /* skb frags point to kernel buffers */ |
| for (i = 0; i < new_frags - 1; i++) { |
| __skb_fill_page_desc(skb, i, head, 0, PAGE_SIZE); |
| head = (struct page *)page_private(head); |
| } |
| __skb_fill_page_desc(skb, new_frags - 1, head, 0, d_off); |
| skb_shinfo(skb)->nr_frags = new_frags; |
| |
| release: |
| skb_zcopy_clear(skb, false); |
| return 0; |
| } |
| EXPORT_SYMBOL_GPL(skb_copy_ubufs); |
| |
| /** |
| * skb_clone - duplicate an sk_buff |
| * @skb: buffer to clone |
| * @gfp_mask: allocation priority |
| * |
| * Duplicate an &sk_buff. The new one is not owned by a socket. Both |
| * copies share the same packet data but not structure. The new |
| * buffer has a reference count of 1. If the allocation fails the |
| * function returns %NULL otherwise the new buffer is returned. |
| * |
| * If this function is called from an interrupt gfp_mask() must be |
| * %GFP_ATOMIC. |
| */ |
| |
| struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask) |
| { |
| struct sk_buff_fclones *fclones = container_of(skb, |
| struct sk_buff_fclones, |
| skb1); |
| struct sk_buff *n; |
| |
| if (skb_orphan_frags(skb, gfp_mask)) |
| return NULL; |
| |
| if (skb->fclone == SKB_FCLONE_ORIG && |
| refcount_read(&fclones->fclone_ref) == 1) { |
| n = &fclones->skb2; |
| refcount_set(&fclones->fclone_ref, 2); |
| } else { |
| if (skb_pfmemalloc(skb)) |
| gfp_mask |= __GFP_MEMALLOC; |
| |
| n = kmem_cache_alloc(skbuff_head_cache, gfp_mask); |
| if (!n) |
| return NULL; |
| |
| n->fclone = SKB_FCLONE_UNAVAILABLE; |
| } |
| |
| return __skb_clone(n, skb); |
| } |
| EXPORT_SYMBOL(skb_clone); |
| |
| void skb_headers_offset_update(struct sk_buff *skb, int off) |
| { |
| /* Only adjust this if it actually is csum_start rather than csum */ |
| if (skb->ip_summed == CHECKSUM_PARTIAL) |
| skb->csum_start += off; |
| /* {transport,network,mac}_header and tail are relative to skb->head */ |
| skb->transport_header += off; |
| skb->network_header += off; |
| if (skb_mac_header_was_set(skb)) |
| skb->mac_header += off; |
| skb->inner_transport_header += off; |
| skb->inner_network_header += off; |
| skb->inner_mac_header += off; |
| } |
| EXPORT_SYMBOL(skb_headers_offset_update); |
| |
| void skb_copy_header(struct sk_buff *new, const struct sk_buff *old) |
| { |
| __copy_skb_header(new, old); |
| |
| skb_shinfo(new)->gso_size = skb_shinfo(old)->gso_size; |
| skb_shinfo(new)->gso_segs = skb_shinfo(old)->gso_segs; |
| skb_shinfo(new)->gso_type = skb_shinfo(old)->gso_type; |
| } |
| EXPORT_SYMBOL(skb_copy_header); |
| |
| static inline int skb_alloc_rx_flag(const struct sk_buff *skb) |
| { |
| if (skb_pfmemalloc(skb)) |
| return SKB_ALLOC_RX; |
| return 0; |
| } |
| |
| /** |
| * skb_copy - create private copy of an sk_buff |
| * @skb: buffer to copy |
| * @gfp_mask: allocation priority |
| * |
| * Make a copy of both an &sk_buff and its data. This is used when the |
| * caller wishes to modify the data and needs a private copy of the |
| * data to alter. Returns %NULL on failure or the pointer to the buffer |
| * on success. The returned buffer has a reference count of 1. |
| * |
| * As by-product this function converts non-linear &sk_buff to linear |
| * one, so that &sk_buff becomes completely private and caller is allowed |
| * to modify all the data of returned buffer. This means that this |
| * function is not recommended for use in circumstances when only |
| * header is going to be modified. Use pskb_copy() instead. |
| */ |
| |
| struct sk_buff *skb_copy(const struct sk_buff *skb, gfp_t gfp_mask) |
| { |
| int headerlen = skb_headroom(skb); |
| unsigned int size = skb_end_offset(skb) + skb->data_len; |
| struct sk_buff *n = __alloc_skb(size, gfp_mask, |
| skb_alloc_rx_flag(skb), NUMA_NO_NODE); |
| |
| if (!n) |
| return NULL; |
| |
| /* Set the data pointer */ |
| skb_reserve(n, headerlen); |
| /* Set the tail pointer and length */ |
| skb_put(n, skb->len); |
| |
| BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); |
| |
| skb_copy_header(n, skb); |
| return n; |
| } |
| EXPORT_SYMBOL(skb_copy); |
| |
| /** |
| * __pskb_copy_fclone - create copy of an sk_buff with private head. |
| * @skb: buffer to copy |
| * @headroom: headroom of new skb |
| * @gfp_mask: allocation priority |
| * @fclone: if true allocate the copy of the skb from the fclone |
| * cache instead of the head cache; it is recommended to set this |
| * to true for the cases where the copy will likely be cloned |
| * |
| * Make a copy of both an &sk_buff and part of its data, located |
| * in header. Fragmented data remain shared. This is used when |
| * the caller wishes to modify only header of &sk_buff and needs |
| * private copy of the header to alter. Returns %NULL on failure |
| * or the pointer to the buffer on success. |
| * The returned buffer has a reference count of 1. |
| */ |
| |
| struct sk_buff *__pskb_copy_fclone(struct sk_buff *skb, int headroom, |
| gfp_t gfp_mask, bool fclone) |
| { |
| unsigned int size = skb_headlen(skb) + headroom; |
| int flags = skb_alloc_rx_flag(skb) | (fclone ? SKB_ALLOC_FCLONE : 0); |
| struct sk_buff *n = __alloc_skb(size, gfp_mask, flags, NUMA_NO_NODE); |
| |
| if (!n) |
| goto out; |
| |
| /* Set the data pointer */ |
| skb_reserve(n, headroom); |
| /* Set the tail pointer and length */ |
| skb_put(n, skb_headlen(skb)); |
| /* Copy the bytes */ |
| skb_copy_from_linear_data(skb, n->data, n->len); |
| |
| n->truesize += skb->data_len; |
| n->data_len = skb->data_len; |
| n->len = skb->len; |
| |
| if (skb_shinfo(skb)->nr_frags) { |
| int i; |
| |
| if (skb_orphan_frags(skb, gfp_mask) || |
| skb_zerocopy_clone(n, skb, gfp_mask)) { |
| kfree_skb(n); |
| n = NULL; |
| goto out; |
| } |
| for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
| skb_shinfo(n)->frags[i] = skb_shinfo(skb)->frags[i]; |
| skb_frag_ref(skb, i); |
| } |
| skb_shinfo(n)->nr_frags = i; |
| } |
| |
| if (skb_has_frag_list(skb)) { |
| skb_shinfo(n)->frag_list = skb_shinfo(skb)->frag_list; |
| skb_clone_fraglist(n); |
| } |
| |
| skb_copy_header(n, skb); |
| out: |
| return n; |
| } |
| EXPORT_SYMBOL(__pskb_copy_fclone); |
| |
| /** |
| * pskb_expand_head - reallocate header of &sk_buff |
| * @skb: buffer to reallocate |
| * @nhead: room to add at head |
| * @ntail: room to add at tail |
| * @gfp_mask: allocation priority |
| * |
| * Expands (or creates identical copy, if @nhead and @ntail are zero) |
| * header of @skb. &sk_buff itself is not changed. &sk_buff MUST have |
| * reference count of 1. Returns zero in the case of success or error, |
| * if expansion failed. In the last case, &sk_buff is not changed. |
| * |
| * All the pointers pointing into skb header may change and must be |
| * reloaded after call to this function. |
| */ |
| |
| int pskb_expand_head(struct sk_buff *skb, int nhead, int ntail, |
| gfp_t gfp_mask) |
| { |
| int i, osize = skb_end_offset(skb); |
| int size = osize + nhead + ntail; |
| long off; |
| u8 *data; |
| |
| BUG_ON(nhead < 0); |
| |
| BUG_ON(skb_shared(skb)); |
| |
| size = SKB_DATA_ALIGN(size); |
| |
| if (skb_pfmemalloc(skb)) |
| gfp_mask |= __GFP_MEMALLOC; |
| data = kmalloc_reserve(size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)), |
| gfp_mask, NUMA_NO_NODE, NULL); |
| if (!data) |
| goto nodata; |
| size = SKB_WITH_OVERHEAD(ksize(data)); |
| |
| /* Copy only real data... and, alas, header. This should be |
| * optimized for the cases when header is void. |
| */ |
| memcpy(data + nhead, skb->head, skb_tail_pointer(skb) - skb->head); |
| |
| memcpy((struct skb_shared_info *)(data + size), |
| skb_shinfo(skb), |
| offsetof(struct skb_shared_info, frags[skb_shinfo(skb)->nr_frags])); |
| |
| /* |
| * if shinfo is shared we must drop the old head gracefully, but if it |
| * is not we can just drop the old head and let the existing refcount |
| * be since all we did is relocate the values |
| */ |
| if (skb_cloned(skb)) { |
| if (skb_orphan_frags(skb, gfp_mask)) |
| goto nofrags; |
| if (skb_zcopy(skb)) |
| refcount_inc(&skb_uarg(skb)->refcnt); |
| for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) |
| skb_frag_ref(skb, i); |
| |
| if (skb_has_frag_list(skb)) |
| skb_clone_fraglist(skb); |
| |
| skb_release_data(skb); |
| } else { |
| skb_free_head(skb); |
| } |
| off = (data + nhead) - skb->head; |
| |
| skb->head = data; |
| skb->head_frag = 0; |
| skb->data += off; |
| |
| skb_set_end_offset(skb, size); |
| #ifdef NET_SKBUFF_DATA_USES_OFFSET |
| off = nhead; |
| #endif |
| skb->tail += off; |
| skb_headers_offset_update(skb, nhead); |
| skb->cloned = 0; |
| skb->hdr_len = 0; |
| skb->nohdr = 0; |
| atomic_set(&skb_shinfo(skb)->dataref, 1); |
| |
| skb_metadata_clear(skb); |
| |
| /* It is not generally safe to change skb->truesize. |
| * For the moment, we really care of rx path, or |
| * when skb is orphaned (not attached to a socket). |
| */ |
| if (!skb->sk || skb->destructor == sock_edemux) |
| skb->truesize += size - osize; |
| |
| return 0; |
| |
| nofrags: |
| kfree(data); |
| nodata: |
| return -ENOMEM; |
| } |
| EXPORT_SYMBOL(pskb_expand_head); |
| |
| /* Make private copy of skb with writable head and some headroom */ |
| |
| struct sk_buff *skb_realloc_headroom(struct sk_buff *skb, unsigned int headroom) |
| { |
| struct sk_buff *skb2; |
| int delta = headroom - skb_headroom(skb); |
| |
| if (delta <= 0) |
| skb2 = pskb_copy(skb, GFP_ATOMIC); |
| else { |
| skb2 = skb_clone(skb, GFP_ATOMIC); |
| if (skb2 && pskb_expand_head(skb2, SKB_DATA_ALIGN(delta), 0, |
| GFP_ATOMIC)) { |
| kfree_skb(skb2); |
| skb2 = NULL; |
| } |
| } |
| return skb2; |
| } |
| EXPORT_SYMBOL(skb_realloc_headroom); |
| |
| int __skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) |
| { |
| unsigned int saved_end_offset, saved_truesize; |
| struct skb_shared_info *shinfo; |
| int res; |
| |
| saved_end_offset = skb_end_offset(skb); |
| saved_truesize = skb->truesize; |
| |
| res = pskb_expand_head(skb, 0, 0, pri); |
| if (res) |
| return res; |
| |
| skb->truesize = saved_truesize; |
| |
| if (likely(skb_end_offset(skb) == saved_end_offset)) |
| return 0; |
| |
| shinfo = skb_shinfo(skb); |
| |
| /* We are about to change back skb->end, |
| * we need to move skb_shinfo() to its new location. |
| */ |
| memmove(skb->head + saved_end_offset, |
| shinfo, |
| offsetof(struct skb_shared_info, frags[shinfo->nr_frags])); |
| |
| skb_set_end_offset(skb, saved_end_offset); |
| |
| return 0; |
| } |
| |
| /** |
| * skb_expand_head - reallocate header of &sk_buff |
| * @skb: buffer to reallocate |
| * @headroom: needed headroom |
| * |
| * Unlike skb_realloc_headroom, this one does not allocate a new skb |
| * if possible; copies skb->sk to new skb as needed |
| * and frees original skb in case of failures. |
| * |
| * It expect increased headroom and generates warning otherwise. |
| */ |
| |
| struct sk_buff *skb_expand_head(struct sk_buff *skb, unsigned int headroom) |
| { |
| int delta = headroom - skb_headroom(skb); |
| int osize = skb_end_offset(skb); |
| struct sock *sk = skb->sk; |
| |
| if (WARN_ONCE(delta <= 0, |
| "%s is expecting an increase in the headroom", __func__)) |
| return skb; |
| |
| delta = SKB_DATA_ALIGN(delta); |
| /* pskb_expand_head() might crash, if skb is shared. */ |
| if (skb_shared(skb) || !is_skb_wmem(skb)) { |
| struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC); |
| |
| if (unlikely(!nskb)) |
| goto fail; |
| |
| if (sk) |
| skb_set_owner_w(nskb, sk); |
| consume_skb(skb); |
| skb = nskb; |
| } |
| if (pskb_expand_head(skb, delta, 0, GFP_ATOMIC)) |
| goto fail; |
| |
| if (sk && is_skb_wmem(skb)) { |
| delta = skb_end_offset(skb) - osize; |
| refcount_add(delta, &sk->sk_wmem_alloc); |
| skb->truesize += delta; |
| } |
| return skb; |
| |
| fail: |
| kfree_skb(skb); |
| return NULL; |
| } |
| EXPORT_SYMBOL(skb_expand_head); |
| |
| /** |
| * skb_copy_expand - copy and expand sk_buff |
| * @skb: buffer to copy |
| * @newheadroom: new free bytes at head |
| * @newtailroom: new free bytes at tail |
| * @gfp_mask: allocation priority |
| * |
| * Make a copy of both an &sk_buff and its data and while doing so |
| * allocate additional space. |
| * |
| * This is used when the caller wishes to modify the data and needs a |
| * private copy of the data to alter as well as more space for new fields. |
| * Returns %NULL on failure or the pointer to the buffer |
| * on success. The returned buffer has a reference count of 1. |
| * |
| * You must pass %GFP_ATOMIC as the allocation priority if this function |
| * is called from an interrupt. |
| */ |
| struct sk_buff *skb_copy_expand(const struct sk_buff *skb, |
| int newheadroom, int newtailroom, |
| gfp_t gfp_mask) |
| { |
| /* |
| * Allocate the copy buffer |
| */ |
| struct sk_buff *n = __alloc_skb(newheadroom + skb->len + newtailroom, |
| gfp_mask, skb_alloc_rx_flag(skb), |
| NUMA_NO_NODE); |
| int oldheadroom = skb_headroom(skb); |
| int head_copy_len, head_copy_off; |
| |
| if (!n) |
| return NULL; |
| |
| skb_reserve(n, newheadroom); |
| |
| /* Set the tail pointer and length */ |
| skb_put(n, skb->len); |
| |
| head_copy_len = oldheadroom; |
| head_copy_off = 0; |
| if (newheadroom <= head_copy_len) |
| head_copy_len = newheadroom; |
| else |
| head_copy_off = newheadroom - head_copy_len; |
| |
| /* Copy the linear header and data. */ |
| BUG_ON(skb_copy_bits(skb, -head_copy_len, n->head + head_copy_off, |
| skb->len + head_copy_len)); |
| |
| skb_copy_header(n, skb); |
| |
| skb_headers_offset_update(n, newheadroom - oldheadroom); |
| |
| return n; |
| } |
| EXPORT_SYMBOL(skb_copy_expand); |
| |
| /** |
| * __skb_pad - zero pad the tail of an skb |
| * @skb: buffer to pad |
| * @pad: space to pad |
| * @free_on_error: free buffer on error |
| * |
| * Ensure that a buffer is followed by a padding area that is zero |
| * filled. Used by network drivers which may DMA or transfer data |
| * beyond the buffer end onto the wire. |
| * |
| * May return error in out of memory cases. The skb is freed on error |
| * if @free_on_error is true. |
| */ |
| |
| int __skb_pad(struct sk_buff *skb, int pad, bool free_on_error) |
| { |
| int err; |
| int ntail; |
| |
| /* If the skbuff is non linear tailroom is always zero.. */ |
| if (!skb_cloned(skb) && skb_tailroom(skb) >= pad) { |
| memset(skb->data+skb->len, 0, pad); |
| return 0; |
| } |
| |
| ntail = skb->data_len + pad - (skb->end - skb->tail); |
| if (likely(skb_cloned(skb) || ntail > 0)) { |
| err = pskb_expand_head(skb, 0, ntail, GFP_ATOMIC); |
| if (unlikely(err)) |
| goto free_skb; |
| } |
| |
| /* FIXME: The use of this function with non-linear skb's really needs |
| * to be audited. |
| */ |
| err = skb_linearize(skb); |
| if (unlikely(err)) |
| goto free_skb; |
| |
| memset(skb->data + skb->len, 0, pad); |
| return 0; |
| |
| free_skb: |
| if (free_on_error) |
| kfree_skb(skb); |
| return err; |
| } |
| EXPORT_SYMBOL(__skb_pad); |
| |
| /** |
| * pskb_put - add data to the tail of a potentially fragmented buffer |
| * @skb: start of the buffer to use |
| * @tail: tail fragment of the buffer to use |
| * @len: amount of data to add |
| * |
| * This function extends the used data area of the potentially |
| * fragmented buffer. @tail must be the last fragment of @skb -- or |
| * @skb itself. If this would exceed the total buffer size the kernel |
| * will panic. A pointer to the first byte of the extra data is |
| * returned. |
| */ |
| |
| void *pskb_put(struct sk_buff *skb, struct sk_buff *tail, int len) |
| { |
| if (tail != skb) { |
| skb->data_len += len; |
| skb->len += len; |
| } |
| return skb_put(tail, len); |
| } |
| EXPORT_SYMBOL_GPL(pskb_put); |
| |
| /** |
| * skb_put - add data to a buffer |
| * @skb: buffer to use |
| * @len: amount of data to add |
| * |
| * This function extends the used data area of the buffer. If this would |
| * exceed the total buffer size the kernel will panic. A pointer to the |
| * first byte of the extra data is returned. |
| */ |
| void *skb_put(struct sk_buff *skb, unsigned int len) |
| { |
| void *tmp = skb_tail_pointer(skb); |
| SKB_LINEAR_ASSERT(skb); |
| skb->tail += len; |
| skb->len += len; |
| if (unlikely(skb->tail > skb->end)) |
| skb_over_panic(skb, len, __builtin_return_address(0)); |
| return tmp; |
| } |
| EXPORT_SYMBOL(skb_put); |
| |
| /** |
| * skb_push - add data to the start of a buffer |
| * @skb: buffer to use |
| * @len: amount of data to add |
| * |
| * This function extends the used data area of the buffer at the buffer |
| * start. If this would exceed the total buffer headroom the kernel will |
| * panic. A pointer to the first byte of the extra data is returned. |
| */ |
| void *skb_push(struct sk_buff *skb, unsigned int len) |
| { |
| skb->data -= len; |
| skb->len += len; |
| if (unlikely(skb->data < skb->head)) |
| skb_under_panic(skb, len, __builtin_return_address(0)); |
| return skb->data; |
| } |
| EXPORT_SYMBOL(skb_push); |
| |
| /** |
| * skb_pull - remove data from the start of a buffer |
| * @skb: buffer to use |
| * @len: amount of data to remove |
| * |
| * This function removes data from the start of a buffer, returning |
| * the memory to the headroom. A pointer to the next data in the buffer |
| * is returned. Once the data has been pulled future pushes will overwrite |
| * the old data. |
| */ |
| void *skb_pull(struct sk_buff *skb, unsigned int len) |
| { |
| return skb_pull_inline(skb, len); |
| } |
| EXPORT_SYMBOL(skb_pull); |
| |
| /** |
| * skb_trim - remove end from a buffer |
| * @skb: buffer to alter |
| * @len: new length |
| * |
| * Cut the length of a buffer down by removing data from the tail. If |
| * the buffer is already under the length specified it is not modified. |
| * The skb must be linear. |
| */ |
| void skb_trim(struct sk_buff *skb, unsigned int len) |
| { |
| if (skb->len > len) |
| __skb_trim(skb, len); |
| } |
| EXPORT_SYMBOL(skb_trim); |
| |
| /* Trims skb to length len. It can change skb pointers. |
| */ |
| |
| int ___pskb_trim(struct sk_buff *skb, unsigned int len) |
| { |
| struct sk_buff **fragp; |
| struct sk_buff *frag; |
| int offset = skb_headlen(skb); |
| int nfrags = skb_shinfo(skb)->nr_frags; |
| int i; |
| int err; |
| |
| if (skb_cloned(skb) && |
| unlikely((err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))) |
| return err; |
| |
| i = 0; |
| if (offset >= len) |
| goto drop_pages; |
| |
| for (; i < nfrags; i++) { |
| int end = offset + skb_frag_size(&skb_shinfo(skb)->frags[i]); |
| |
| if (end < len) { |
| offset = end; |
| continue; |
| } |
| |
| skb_frag_size_set(&skb_shinfo(skb)->frags[i++], len - offset); |
| |
| drop_pages: |
| skb_shinfo(skb)->nr_frags = i; |
| |
| for (; i < nfrags; i++) |
| skb_frag_unref(skb, i); |
| |
| if (skb_has_frag_list(skb)) |
| skb_drop_fraglist(skb); |
| goto done; |
| } |
| |
| for (fragp = &skb_shinfo(skb)->frag_list; (frag = *fragp); |
| fragp = &frag->next) { |
| int end = offset + frag->len; |
| |
| if (skb_shared(frag)) { |
| struct sk_buff *nfrag; |
| |
| nfrag = skb_clone(frag, GFP_ATOMIC); |
| if (unlikely(!nfrag)) |
| return -ENOMEM; |
| |
| nfrag->next = frag->next; |
| consume_skb(frag); |
| frag = nfrag; |
| *fragp = frag; |
| } |
| |
| if (end < len) { |
| offset = end; |
| continue; |
| } |
| |
| if (end > len && |
| unlikely((err = pskb_trim(frag, len - offset)))) |
| return err; |
| |
| if (frag->next) |
| skb_drop_list(&frag->next); |
| break; |
| } |
| |
| done: |
| if (len > skb_headlen(skb)) { |
| skb->data_len -= skb->len - len; |
| skb->len = len; |
| } else { |
| skb->len = len; |
| skb->data_len = 0; |
| skb_set_tail_pointer(skb, len); |
| } |
| |
| if (!skb->sk || skb->destructor == sock_edemux) |
| skb_condense(skb); |
| return 0; |
| } |
| EXPORT_SYMBOL(___pskb_trim); |
| |
| /* Note : use pskb_trim_rcsum() instead of calling this directly |
| */ |
| int pskb_trim_rcsum_slow(struct sk_buff *skb, unsigned int len) |
| { |
| if (skb->ip_summed == CHECKSUM_COMPLETE) { |
| int delta = skb->len - len; |
| |
| skb->csum = csum_block_sub(skb->csum, |
| skb_checksum(skb, len, delta, 0), |
| len); |
| } else if (skb->ip_summed == CHECKSUM_PARTIAL) { |
| int hdlen = (len > skb_headlen(skb)) ? skb_headlen(skb) : len; |
| int offset = skb_checksum_start_offset(skb) + skb->csum_offset; |
| |
| if (offset + sizeof(__sum16) > hdlen) |
| return -EINVAL; |
| } |
| return __pskb_trim(skb, len); |
| } |
| EXPORT_SYMBOL(pskb_trim_rcsum_slow); |
| |
| /** |
| * __pskb_pull_tail - advance tail of skb header |
| * @skb: buffer to reallocate |
| * @delta: number of bytes to advance tail |
| * |
| * The function makes a sense only on a fragmented &sk_buff, |
| * it expands header moving its tail forward and copying necessary |
| * data from fragmented part. |
| * |
| * &sk_buff MUST have reference count of 1. |
| * |
| * Returns %NULL (and &sk_buff does not change) if pull failed |
| * or value of new tail of skb in the case of success. |
| * |
| * All the pointers pointing into skb header may change and must be |
| * reloaded after call to this function. |
| */ |
| |
| /* Moves tail of skb head forward, copying data from fragmented part, |
| * when it is necessary. |
| * 1. It may fail due to malloc failure. |
| * 2. It may change skb pointers. |
| * |
| * It is pretty complicated. Luckily, it is called only in exceptional cases. |
| */ |
| void *__pskb_pull_tail(struct sk_buff *skb, int delta) |
| { |
| /* If skb has not enough free space at tail, get new one |
| * plus 128 bytes for future expansions. If we have enough |
| * room at tail, reallocate without expansion only if skb is cloned. |
| */ |
| int i, k, eat = (skb->tail + delta) - skb->end; |
| |
| if (eat > 0 || skb_cloned(skb)) { |
| if (pskb_expand_head(skb, 0, eat > 0 ? eat + 128 : 0, |
| GFP_ATOMIC)) |
| return NULL; |
| } |
| |
| BUG_ON(skb_copy_bits(skb, skb_headlen(skb), |
| skb_tail_pointer(skb), delta)); |
| |
| /* Optimization: no fragments, no reasons to preestimate |
| * size of pulled pages. Superb. |
| */ |
| if (!skb_has_frag_list(skb)) |
| goto pull_pages; |
| |
| /* Estimate size of pulled pages. */ |
| eat = delta; |
| for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
| int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); |
| |
| if (size >= eat) |
| goto pull_pages; |
| eat -= size; |
| } |
| |
| /* If we need update frag list, we are in troubles. |
| * Certainly, it is possible to add an offset to skb data, |
| * but taking into account that pulling is expected to |
| * be very rare operation, it is worth to fight against |
| * further bloating skb head and crucify ourselves here instead. |
| * Pure masohism, indeed. 8)8) |
| */ |
| if (eat) { |
| struct sk_buff *list = skb_shinfo(skb)->frag_list; |
| struct sk_buff *clone = NULL; |
| struct sk_buff *insp = NULL; |
| |
| do { |
| if (list->len <= eat) { |
| /* Eaten as whole. */ |
| eat -= list->len; |
| list = list->next; |
| insp = list; |
| } else { |
| /* Eaten partially. */ |
| if (skb_is_gso(skb) && !list->head_frag && |
| skb_headlen(list)) |
| skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; |
| |
| if (skb_shared(list)) { |
| /* Sucks! We need to fork list. :-( */ |
| clone = skb_clone(list, GFP_ATOMIC); |
| if (!clone) |
| return NULL; |
| insp = list->next; |
| list = clone; |
| } else { |
| /* This may be pulled without |
| * problems. */ |
| insp = list; |
| } |
| if (!pskb_pull(list, eat)) { |
| kfree_skb(clone); |
| return NULL; |
| } |
| break; |
| } |
| } while (eat); |
| |
| /* Free pulled out fragments. */ |
| while ((list = skb_shinfo(skb)->frag_list) != insp) { |
| skb_shinfo(skb)->frag_list = list->next; |
| consume_skb(list); |
| } |
| /* And insert new clone at head. */ |
| if (clone) { |
| clone->next = list; |
| skb_shinfo(skb)->frag_list = clone; |
| } |
| } |
| /* Success! Now we may commit changes to skb data. */ |
| |
| pull_pages: |
| eat = delta; |
| k = 0; |
| for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
| int size = skb_frag_size(&skb_shinfo(skb)->frags[i]); |
| |
| if (size <= eat) { |
| skb_frag_unref(skb, i); |
| eat -= size; |
| } else { |
| skb_frag_t *frag = &skb_shinfo(skb)->frags[k]; |
| |
| *frag = skb_shinfo(skb)->frags[i]; |
| if (eat) { |
| skb_frag_off_add(frag, eat); |
| skb_frag_size_sub(frag, eat); |
| if (!i) |
| goto end; |
| eat = 0; |
| } |
| k++; |
| } |
| } |
| skb_shinfo(skb)->nr_frags = k; |
| |
| end: |
| skb->tail += delta; |
| skb->data_len -= delta; |
| |
| if (!skb->data_len) |
| skb_zcopy_clear(skb, false); |
| |
| return skb_tail_pointer(skb); |
| } |
| EXPORT_SYMBOL(__pskb_pull_tail); |
| |
| /** |
| * skb_copy_bits - copy bits from skb to kernel buffer |
| * @skb: source skb |
| * @offset: offset in source |
| * @to: destination buffer |
| * @len: number of bytes to copy |
| * |
| * Copy the specified number of bytes from the source skb to the |
| * destination buffer. |
| * |
| * CAUTION ! : |
| * If its prototype is ever changed, |
| * check arch/{*}/net/{*}.S files, |
| * since it is called from BPF assembly code. |
| */ |
| int skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) |
| { |
| int start = skb_headlen(skb); |
| struct sk_buff *frag_iter; |
| int i, copy; |
| |
| if (offset > (int)skb->len - len) |
| goto fault; |
| |
| /* Copy header. */ |
| if ((copy = start - offset) > 0) { |
| if (copy > len) |
| copy = len; |
| skb_copy_from_linear_data_offset(skb, offset, to, copy); |
| if ((len -= copy) == 0) |
| return 0; |
| offset += copy; |
| to += copy; |
| } |
| |
| for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
| int end; |
| skb_frag_t *f = &skb_shinfo(skb)->frags[i]; |
| |
| WARN_ON(start > offset + len); |
| |
| end = start + skb_frag_size(f); |
| if ((copy = end - offset) > 0) { |
| u32 p_off, p_len, copied; |
| struct page *p; |
| u8 *vaddr; |
| |
| if (copy > len) |
| copy = len; |
| |
| skb_frag_foreach_page(f, |
| skb_frag_off(f) + offset - start, |
| copy, p, p_off, p_len, copied) { |
| vaddr = kmap_atomic(p); |
| memcpy(to + copied, vaddr + p_off, p_len); |
| kunmap_atomic(vaddr); |
| } |
| |
| if ((len -= copy) == 0) |
| return 0; |
| offset += copy; |
| to += copy; |
| } |
| start = end; |
| } |
| |
| skb_walk_frags(skb, frag_iter) { |
| int end; |
| |
| WARN_ON(start > offset + len); |
| |
| end = start + frag_iter->len; |
| if ((copy = end - offset) > 0) { |
| if (copy > len) |
| copy = len; |
| if (skb_copy_bits(frag_iter, offset - start, to, copy)) |
| goto fault; |
| if ((len -= copy) == 0) |
| return 0; |
| offset += copy; |
| to += copy; |
| } |
| start = end; |
| } |
| |
| if (!len) |
| return 0; |
| |
| fault: |
| return -EFAULT; |
| } |
| EXPORT_SYMBOL(skb_copy_bits); |
| |
| /* |
| * Callback from splice_to_pipe(), if we need to release some pages |
| * at the end of the spd in case we error'ed out in filling the pipe. |
| */ |
| static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) |
| { |
| put_page(spd->pages[i]); |
| } |
| |
| static struct page *linear_to_page(struct page *page, unsigned int *len, |
| unsigned int *offset, |
| struct sock *sk) |
| { |
| struct page_frag *pfrag = sk_page_frag(sk); |
| |
| if (!sk_page_frag_refill(sk, pfrag)) |
| return NULL; |
| |
| *len = min_t(unsigned int, *len, pfrag->size - pfrag->offset); |
| |
| memcpy(page_address(pfrag->page) + pfrag->offset, |
| page_address(page) + *offset, *len); |
| *offset = pfrag->offset; |
| pfrag->offset += *len; |
| |
| return pfrag->page; |
| } |
| |
| static bool spd_can_coalesce(const struct splice_pipe_desc *spd, |
| struct page *page, |
| unsigned int offset) |
| { |
| return spd->nr_pages && |
| spd->pages[spd->nr_pages - 1] == page && |
| (spd->partial[spd->nr_pages - 1].offset + |
| spd->partial[spd->nr_pages - 1].len == offset); |
| } |
| |
| /* |
| * Fill page/offset/length into spd, if it can hold more pages. |
| */ |
| static bool spd_fill_page(struct splice_pipe_desc *spd, |
| struct pipe_inode_info *pipe, struct page *page, |
| unsigned int *len, unsigned int offset, |
| bool linear, |
| struct sock *sk) |
| { |
| if (unlikely(spd->nr_pages == MAX_SKB_FRAGS)) |
| return true; |
| |
| if (linear) { |
| page = linear_to_page(page, len, &offset, sk); |
| if (!page) |
| return true; |
| } |
| if (spd_can_coalesce(spd, page, offset)) { |
| spd->partial[spd->nr_pages - 1].len += *len; |
| return false; |
| } |
| get_page(page); |
| spd->pages[spd->nr_pages] = page; |
| spd->partial[spd->nr_pages].len = *len; |
| spd->partial[spd->nr_pages].offset = offset; |
| spd->nr_pages++; |
| |
| return false; |
| } |
| |
| static bool __splice_segment(struct page *page, unsigned int poff, |
| unsigned int plen, unsigned int *off, |
| unsigned int *len, |
| struct splice_pipe_desc *spd, bool linear, |
| struct sock *sk, |
| struct pipe_inode_info *pipe) |
| { |
| if (!*len) |
| return true; |
| |
| /* skip this segment if already processed */ |
| if (*off >= plen) { |
| *off -= plen; |
| return false; |
| } |
| |
| /* ignore any bits we already processed */ |
| poff += *off; |
| plen -= *off; |
| *off = 0; |
| |
| do { |
| unsigned int flen = min(*len, plen); |
| |
| if (spd_fill_page(spd, pipe, page, &flen, poff, |
| linear, sk)) |
| return true; |
| poff += flen; |
| plen -= flen; |
| *len -= flen; |
| } while (*len && plen); |
| |
| return false; |
| } |
| |
| /* |
| * Map linear and fragment data from the skb to spd. It reports true if the |
| * pipe is full or if we already spliced the requested length. |
| */ |
| static bool __skb_splice_bits(struct sk_buff *skb, struct pipe_inode_info *pipe, |
| unsigned int *offset, unsigned int *len, |
| struct splice_pipe_desc *spd, struct sock *sk) |
| { |
| int seg; |
| struct sk_buff *iter; |
| |
| /* map the linear part : |
| * If skb->head_frag is set, this 'linear' part is backed by a |
| * fragment, and if the head is not shared with any clones then |
| * we can avoid a copy since we own the head portion of this page. |
| */ |
| if (__splice_segment(virt_to_page(skb->data), |
| (unsigned long) skb->data & (PAGE_SIZE - 1), |
| skb_headlen(skb), |
| offset, len, spd, |
| skb_head_is_locked(skb), |
| sk, pipe)) |
| return true; |
| |
| /* |
| * then map the fragments |
| */ |
| for (seg = 0; seg < skb_shinfo(skb)->nr_frags; seg++) { |
| const skb_frag_t *f = &skb_shinfo(skb)->frags[seg]; |
| |
| if (__splice_segment(skb_frag_page(f), |
| skb_frag_off(f), skb_frag_size(f), |
| offset, len, spd, false, sk, pipe)) |
| return true; |
| } |
| |
| skb_walk_frags(skb, iter) { |
| if (*offset >= iter->len) { |
| *offset -= iter->len; |
| continue; |
| } |
| /* __skb_splice_bits() only fails if the output has no room |
| * left, so no point in going over the frag_list for the error |
| * case. |
| */ |
| if (__skb_splice_bits(iter, pipe, offset, len, spd, sk)) |
| return true; |
| } |
| |
| return false; |
| } |
| |
| /* |
| * Map data from the skb to a pipe. Should handle both the linear part, |
| * the fragments, and the frag list. |
| */ |
| int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, |
| struct pipe_inode_info *pipe, unsigned int tlen, |
| unsigned int flags) |
| { |
| struct partial_page partial[MAX_SKB_FRAGS]; |
| struct page *pages[MAX_SKB_FRAGS]; |
| struct splice_pipe_desc spd = { |
| .pages = pages, |
| .partial = partial, |
| .nr_pages_max = MAX_SKB_FRAGS, |
| .ops = &nosteal_pipe_buf_ops, |
| .spd_release = sock_spd_release, |
| }; |
| int ret = 0; |
| |
| __skb_splice_bits(skb, pipe, &offset, &tlen, &spd, sk); |
| |
| if (spd.nr_pages) |
| ret = splice_to_pipe(pipe, &spd); |
| |
| return ret; |
| } |
| EXPORT_SYMBOL_GPL(skb_splice_bits); |
| |
| static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg, |
| struct kvec *vec, size_t num, size_t size) |
| { |
| struct socket *sock = sk->sk_socket; |
| |
| if (!sock) |
| return -EINVAL; |
| return kernel_sendmsg(sock, msg, vec, num, size); |
| } |
| |
| static int sendpage_unlocked(struct sock *sk, struct page *page, int offset, |
| size_t size, int flags) |
| { |
| struct socket *sock = sk->sk_socket; |
| |
| if (!sock) |
| return -EINVAL; |
| return kernel_sendpage(sock, page, offset, size, flags); |
| } |
| |
| typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg, |
| struct kvec *vec, size_t num, size_t size); |
| typedef int (*sendpage_func)(struct sock *sk, struct page *page, int offset, |
| size_t size, int flags); |
| static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, |
| int len, sendmsg_func sendmsg, sendpage_func sendpage) |
| { |
| unsigned int orig_len = len; |
| struct sk_buff *head = skb; |
| unsigned short fragidx; |
| int slen, ret; |
| |
| do_frag_list: |
| |
| /* Deal with head data */ |
| while (offset < skb_headlen(skb) && len) { |
| struct kvec kv; |
| struct msghdr msg; |
| |
| slen = min_t(int, len, skb_headlen(skb) - offset); |
| kv.iov_base = skb->data + offset; |
| kv.iov_len = slen; |
| memset(&msg, 0, sizeof(msg)); |
| msg.msg_flags = MSG_DONTWAIT; |
| |
| ret = INDIRECT_CALL_2(sendmsg, kernel_sendmsg_locked, |
| sendmsg_unlocked, sk, &msg, &kv, 1, slen); |
| if (ret <= 0) |
| goto error; |
| |
| offset += ret; |
| len -= ret; |
| } |
| |
| /* All the data was skb head? */ |
| if (!len) |
| goto out; |
| |
| /* Make offset relative to start of frags */ |
| offset -= skb_headlen(skb); |
| |
| /* Find where we are in frag list */ |
| for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { |
| skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; |
| |
| if (offset < skb_frag_size(frag)) |
| break; |
| |
| offset -= skb_frag_size(frag); |
| } |
| |
| for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) { |
| skb_frag_t *frag = &skb_shinfo(skb)->frags[fragidx]; |
| |
| slen = min_t(size_t, len, skb_frag_size(frag) - offset); |
| |
| while (slen) { |
| ret = INDIRECT_CALL_2(sendpage, kernel_sendpage_locked, |
| sendpage_unlocked, sk, |
| skb_frag_page(frag), |
| skb_frag_off(frag) + offset, |
| slen, MSG_DONTWAIT); |
| if (ret <= 0) |
| goto error; |
| |
| len -= ret; |
| offset += ret; |
| slen -= ret; |
| } |
| |
| offset = 0; |
| } |
| |
| if (len) { |
| /* Process any frag lists */ |
| |
| if (skb == head) { |
| if (skb_has_frag_list(skb)) { |
| skb = skb_shinfo(skb)->frag_list; |
| goto do_frag_list; |
| } |
| } else if (skb->next) { |
| skb = skb->next; |
| goto do_frag_list; |
| } |
| } |
| |
| out: |
| return orig_len - len; |
| |
| error: |
| return orig_len == len ? ret : orig_len - len; |
| } |
| |
| /* Send skb data on a socket. Socket must be locked. */ |
| int skb_send_sock_locked(struct sock *sk, struct sk_buff *skb, int offset, |
| int len) |
| { |
| return __skb_send_sock(sk, skb, offset, len, kernel_sendmsg_locked, |
| kernel_sendpage_locked); |
| } |
| EXPORT_SYMBOL_GPL(skb_send_sock_locked); |
| |
| /* Send skb data on a socket. Socket must be unlocked. */ |
| int skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset, int len) |
| { |
| return __skb_send_sock(sk, skb, offset, len, sendmsg_unlocked, |
| sendpage_unlocked); |
| } |
| |
| /** |
| * skb_store_bits - store bits from kernel buffer to skb |
| * @skb: destination buffer |
| * @offset: offset in destination |
| * @from: source buffer |
| * @len: number of bytes to copy |
| * |
| * Copy the specified number of bytes from the source buffer to the |
| * destination skb. This function handles all the messy bits of |
| * traversing fragment lists and such. |
| */ |
| |
| int skb_store_bits(struct sk_buff *skb, int offset, const void *from, int len) |
| { |
| int start = skb_headlen(skb); |
| struct sk_buff *frag_iter; |
| int i, copy; |
| |
| if (offset > (int)skb->len - len) |
| goto fault; |
| |
| if ((copy = start - offset) > 0) { |
| if (copy > len) |
| copy = len; |
| skb_copy_to_linear_data_offset(skb, offset, from, copy); |
| if ((len -= copy) == 0) |
| return 0; |
| offset += copy; |
| from += copy; |
| } |
| |
| for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
| skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
| int end; |
| |
| WARN_ON(start > offset + len); |
| |
| end = start + skb_frag_size(frag); |
| if ((copy = end - offset) > 0) { |
| u32 p_off, p_len, copied; |
| struct page *p; |
| u8 *vaddr; |
| |
| if (copy > len) |
| copy = len; |
| |
| skb_frag_foreach_page(frag, |
| skb_frag_off(frag) + offset - start, |
| copy, p, p_off, p_len, copied) { |
| vaddr = kmap_atomic(p); |
| memcpy(vaddr + p_off, from + copied, p_len); |
| kunmap_atomic(vaddr); |
| } |
| |
| if ((len -= copy) == 0) |
| return 0; |
| offset += copy; |
| from += copy; |
| } |
| start = end; |
| } |
| |
| skb_walk_frags(skb, frag_iter) { |
| int end; |
| |
| WARN_ON(start > offset + len); |
| |
| end = start + frag_iter->len; |
| if ((copy = end - offset) > 0) { |
| if (copy > len) |
| copy = len; |
| if (skb_store_bits(frag_iter, offset - start, |
| from, copy)) |
| goto fault; |
| if ((len -= copy) == 0) |
| return 0; |
| offset += copy; |
| from += copy; |
| } |
| start = end; |
| } |
| if (!len) |
| return 0; |
| |
| fault: |
| return -EFAULT; |
| } |
| EXPORT_SYMBOL(skb_store_bits); |
| |
| /* Checksum skb data. */ |
| __wsum __skb_checksum(const struct sk_buff *skb, int offset, int len, |
| __wsum csum, const struct skb_checksum_ops *ops) |
| { |
| int start = skb_headlen(skb); |
| int i, copy = start - offset; |
| struct sk_buff *frag_iter; |
| int pos = 0; |
| |
| /* Checksum header. */ |
| if (copy > 0) { |
| if (copy > len) |
| copy = len; |
| csum = INDIRECT_CALL_1(ops->update, csum_partial_ext, |
| skb->data + offset, copy, csum); |
| if ((len -= copy) == 0) |
| return csum; |
| offset += copy; |
| pos = copy; |
| } |
| |
| for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
| int end; |
| skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
| |
| WARN_ON(start > offset + len); |
| |
| end = start + skb_frag_size(frag); |
| if ((copy = end - offset) > 0) { |
| u32 p_off, p_len, copied; |
| struct page *p; |
| __wsum csum2; |
| u8 *vaddr; |
| |
| if (copy > len) |
| copy = len; |
| |
| skb_frag_foreach_page(frag, |
| skb_frag_off(frag) + offset - start, |
| copy, p, p_off, p_len, copied) { |
| vaddr = kmap_atomic(p); |
| csum2 = INDIRECT_CALL_1(ops->update, |
| csum_partial_ext, |
| vaddr + p_off, p_len, 0); |
| kunmap_atomic(vaddr); |
| csum = INDIRECT_CALL_1(ops->combine, |
| csum_block_add_ext, csum, |
| csum2, pos, p_len); |
| pos += p_len; |
| } |
| |
| if (!(len -= copy)) |
| return csum; |
| offset += copy; |
| } |
| start = end; |
| } |
| |
| skb_walk_frags(skb, frag_iter) { |
| int end; |
| |
| WARN_ON(start > offset + len); |
| |
| end = start + frag_iter->len; |
| if ((copy = end - offset) > 0) { |
| __wsum csum2; |
| if (copy > len) |
| copy = len; |
| csum2 = __skb_checksum(frag_iter, offset - start, |
| copy, 0, ops); |
| csum = INDIRECT_CALL_1(ops->combine, csum_block_add_ext, |
| csum, csum2, pos, copy); |
| if ((len -= copy) == 0) |
| return csum; |
| offset += copy; |
| pos += copy; |
| } |
| start = end; |
| } |
| BUG_ON(len); |
| |
| return csum; |
| } |
| EXPORT_SYMBOL(__skb_checksum); |
| |
| __wsum skb_checksum(const struct sk_buff *skb, int offset, |
| int len, __wsum csum) |
| { |
| const struct skb_checksum_ops ops = { |
| .update = csum_partial_ext, |
| .combine = csum_block_add_ext, |
| }; |
| |
| return __skb_checksum(skb, offset, len, csum, &ops); |
| } |
| EXPORT_SYMBOL(skb_checksum); |
| |
| /* Both of above in one bottle. */ |
| |
| __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, |
| u8 *to, int len) |
| { |
| int start = skb_headlen(skb); |
| int i, copy = start - offset; |
| struct sk_buff *frag_iter; |
| int pos = 0; |
| __wsum csum = 0; |
| |
| /* Copy header. */ |
| if (copy > 0) { |
| if (copy > len) |
| copy = len; |
| csum = csum_partial_copy_nocheck(skb->data + offset, to, |
| copy); |
| if ((len -= copy) == 0) |
| return csum; |
| offset += copy; |
| to += copy; |
| pos = copy; |
| } |
| |
| for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { |
| int end; |
| |
| WARN_ON(start > offset + len); |
| |
| end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]); |
| if ((copy = end - offset) > 0) { |
| skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; |
| u32 p_off, p_len, copied; |
| struct page *p; |
| __wsum csum2; |
| u8 *vaddr; |
| |
| if (copy > len) |
| copy = len; |
| |
| skb_frag_foreach_page(frag, |
| skb_frag_off(frag) + offset - start, |
| copy, p, p_off, p_len, copied) { |
| vaddr = kmap_atomic(p); |
| csum2 = csum_partial_copy_nocheck(vaddr + p_off, |
| to + copied, |
| p_len); |
| kunmap_atomic(vaddr); |
| csum = csum_block_add(csum, csum2, pos); |
| pos += p_len; |
| } |
| |
| if (!(len -= copy)) |
| return csum; |
| offset += copy; |
| to += copy; |
| } |
| start = end; |
| } |
| |
| skb_walk_frags(skb, frag_iter) { |
| __wsum csum2; |
| int end; |
| |
| WARN_ON(start > offset + len); |
| |
| end = start + frag_iter->len; |
| if ((copy = end - offset) > 0) { |
| if (copy > len) |
| copy = len; |
| csum2 = skb_copy_and_csum_bits(frag_iter, |
| offset - start, |
| to, copy); |
| csum = csum_block_add(csum, csum2, pos); |
| if ((len -= copy) == 0) |
| return csum; |
| offset += copy; |
| to += copy; |
| pos += copy; |
| } |
| start = end; |
| } |
| BUG_ON(len); |
| return csum; |
| } |
| EXPORT_SYMBOL(skb_copy_and_csum_bits); |
| |
| __sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len) |
| { |
| __sum16 sum; |
| |
| sum = csum_fold(skb_checksum(skb, 0, len, skb->csum)); |
| /* See comments in __skb_checksum_complete(). */ |
| if (likely(!sum)) { |
| if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && |
| !skb->csum_complete_sw) |
| netdev_rx_csum_fault(skb->dev, skb); |
| } |
| if (!skb_shared(skb)) |
| skb->csum_valid = !sum; |
| return sum; |
| } |
| EXPORT_SYMBOL(__skb_checksum_complete_head); |
| |
| /* This function assumes skb->csum already holds pseudo header's checksum, |
| * which has been changed from the hardware checksum, for example, by |
| * __skb_checksum_validate_complete(). And, the original skb->csum must |
| * have been validated unsuccessfully for CHECKSUM_COMPLETE case. |
| * |
| * It returns non-zero if the recomputed checksum is still invalid, otherwise |
| * zero. The new checksum is stored back into skb->csum unless the skb is |
| * shared. |
| */ |
| __sum16 __skb_checksum_complete(struct sk_buff *skb) |
| { |
| __wsum csum; |
| __sum16 sum; |
| |
| csum = skb_checksum(skb, 0, skb->len, 0); |
| |
| sum = csum_fold(csum_add(skb->csum, csum)); |
| /* This check is inverted, because we already knew the hardware |
| * checksum is invalid before calling this function. So, if the |
| * re-computed checksum is valid instead, then we have a mismatch |
| * between the original skb->csum and skb_checksum(). This means either |
| * the original hardware checksum is incorrect or we screw up skb->csum |
| * when moving skb->data around. |
| */ |
| if (likely(!sum)) { |
| if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && |
| !skb->csum_complete_sw) |
| netdev_rx_csum_fault(skb->dev, skb); |
| } |
| |
| if (!skb_shared(skb)) { |
| /* Save full packet checksum */ |
| skb->csum = csum; |
| skb->ip_summed = CHECKSUM_COMPLETE; |
| skb->csum_complete_sw = 1; |
| skb->csum_valid = !sum; |
| } |
| |
| return sum; |
| } |
| EXPORT_SYMBOL(__skb_checksum_complete); |
| |
| static __wsum warn_crc32c_csum_update(const void *buff, int len, __wsum sum) |
| { |
| net_warn_ratelimited( |
| "%s: attempt to compute crc32c without libcrc32c.ko\n", |
| __func__); |
| return 0; |
| } |
| |
| static __wsum warn_crc32c_csum_combine(__wsum csum, __wsum csum2, |
| int offset, int len) |
| { |
| net_warn_ratelimited( |
| "%s: attempt to compute crc32c without libcrc32c.ko\n", |
| __func__); |
| return 0; |
| } |
| |
| static const struct skb_checksum_ops default_crc32c_ops = { |
| .update = warn_crc32c_csum_update, |
| .combine = warn_crc32c_csum_combine, |
| }; |
| |
| const struct skb_checksum_ops *crc32c_csum_stub __read_mostly = |
| &default_crc32c_ops; |
| EXPORT_SYMBOL(crc32c_csum_stub); |
| |
| /** |
| * skb_zerocopy_headlen - Calculate headroom needed for skb_zerocopy() |
| * @from: source buffer |
| * |
| * Calculates the amount of linear headroom needed in the 'to' skb passed |
| * into skb_zerocopy(). |
| */ |
| unsigned int |
| skb_zerocopy_headlen(const struct sk_buff *from) |
| { |
| unsigned int hlen = 0; |
| |
| if (!from->head_frag || |
| skb_headlen(from) < L1_CACHE_BYTES || |
| skb_shinfo(from)->nr_frags >= MAX_SKB_FRAGS) { |
| hlen = skb_headlen(from); |
| if (!hlen) |
| hlen = from->len; |
| } |
| |
| if (skb_has_frag_list(from)) |
| hlen = from->len; |
| |
| return hlen; |
| } |
| EXPORT_SYMBOL_GPL(skb_zerocopy_headlen); |
| |
| /** |
| * skb_zerocopy - Zero copy skb to skb |
| * @to: destination buffer |
| * @from: source buffer |
| * @len: number of bytes to copy from source buffer |
| * @hlen: size of linear headroom in destination buffer |
| * |
| * Copies up to `len` bytes from `from` to `to` by creating references |
| * to the frags in the source buffer. |
| * |
| * The `hlen` as calculated by skb_zerocopy_headlen() specifies the |
| * headroom in the `to` buffer. |
| * |
| * Return value: |
| * 0: everything is OK |
| * -ENOMEM: couldn't orphan frags of @from due to lack of memory |
| * -EFAULT: skb_copy_bits() found some problem with skb geometry |
| */ |
| int |
| skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) |
| { |
| int i, j = 0; |
| int plen = 0; /* length of skb->head fragment */ |
| int ret; |
| struct page *page; |
| unsigned int offset; |
| |
| BUG_ON(!from->head_frag && !hlen); |
| |
| /* dont bother with small payloads */ |
| if (len <= skb_tailroom(to)) |
| return skb_copy_bits(from, 0, skb_put(to, len), len); |
| |
| if (hlen) { |
| ret = skb_copy_bits(from, 0, skb_put(to, hlen), hlen |