drivers/rmnet/shs/rmnet_shs_wq.c

/* Copyright (c) 2018-2020 The Linux Foundation. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 and
 * only version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * RMNET Data Smart Hash Workqueue solution
 *
 */

#include "rmnet_shs.h"
#include "rmnet_shs_wq_genl.h"
#include "rmnet_shs_wq_mem.h"
#include <linux/workqueue.h>
#include <linux/list_sort.h>
#include <net/sock.h>
#include <linux/skbuff.h>

MODULE_LICENSE("GPL v2");
/* Local Macros */
#define RMNET_SHS_RX_BPNSEC_TO_BPSEC(x) ((x)*1000000000)
#define RMNET_SHS_SEC_TO_NSEC(x) ((x)*1000000000)
#define RMNET_SHS_NSEC_TO_SEC(x) ((x)/1000000000)
#define RMNET_SHS_BYTE_TO_BIT(x) ((x)*8)
#define RMNET_SHS_MIN_HSTAT_NODES_REQD 16
#define RMNET_SHS_FILTER_PKT_LIMIT 200
#define RMNET_SHS_FILTER_FLOW_RATE 100

#define PERIODIC_CLEAN 0
/* FORCE_CLEAN should only used during module de-init.*/
#define FORCE_CLEAN 1

/* Local Definitions and Declarations */
unsigned int rmnet_shs_cpu_prio_dur __read_mostly = 3;
module_param(rmnet_shs_cpu_prio_dur, uint, 0644);
MODULE_PARM_DESC(rmnet_shs_cpu_prio_dur, "Priority ignore duration (wq intervals)");

#define PRIO_BACKOFF ((!rmnet_shs_cpu_prio_dur) ? 2 : rmnet_shs_cpu_prio_dur)

unsigned int rmnet_shs_wq_interval_ms __read_mostly = RMNET_SHS_WQ_INTERVAL_MS;
module_param(rmnet_shs_wq_interval_ms, uint, 0644);
MODULE_PARM_DESC(rmnet_shs_wq_interval_ms, "Interval between wq runs (ms)");

unsigned long rmnet_shs_max_flow_inactivity_sec __read_mostly =
						RMNET_SHS_MAX_SKB_INACTIVE_TSEC;
module_param(rmnet_shs_max_flow_inactivity_sec, ulong, 0644);
MODULE_PARM_DESC(rmnet_shs_max_flow_inactivity_sec,
		 "Max flow inactive time before clean up");

unsigned int rmnet_shs_wq_tuning __read_mostly = 80;
module_param(rmnet_shs_wq_tuning, uint, 0644);
MODULE_PARM_DESC(rmnet_shs_wq_tuning, "moving average weightage");

unsigned long long rmnet_shs_cpu_rx_max_pps_thresh[MAX_CPUS]__read_mostly = {
			 RMNET_SHS_UDP_PPS_LPWR_CPU_UTHRESH,
			 RMNET_SHS_UDP_PPS_LPWR_CPU_UTHRESH,
			 RMNET_SHS_UDP_PPS_LPWR_CPU_UTHRESH,
			 RMNET_SHS_UDP_PPS_LPWR_CPU_UTHRESH,
			 RMNET_SHS_UDP_PPS_PERF_CPU_UTHRESH,
			 RMNET_SHS_UDP_PPS_PERF_CPU_UTHRESH,
			 RMNET_SHS_UDP_PPS_PERF_CPU_UTHRESH,
			 RMNET_SHS_UDP_PPS_PERF_CPU_UTHRESH
			};
module_param_array(rmnet_shs_cpu_rx_max_pps_thresh, ullong, 0, 0644);
MODULE_PARM_DESC(rmnet_shs_cpu_rx_max_pps_thresh, "Max pkts core can handle");

unsigned long long rmnet_shs_cpu_rx_min_pps_thresh[MAX_CPUS]__read_mostly = {
			 RMNET_SHS_UDP_PPS_LPWR_CPU_LTHRESH,
			 RMNET_SHS_UDP_PPS_LPWR_CPU_LTHRESH,
			 RMNET_SHS_UDP_PPS_LPWR_CPU_LTHRESH,
			 RMNET_SHS_UDP_PPS_LPWR_CPU_LTHRESH,
			 RMNET_SHS_UDP_PPS_PERF_CPU_LTHRESH,
			 RMNET_SHS_UDP_PPS_PERF_CPU_LTHRESH,
			 RMNET_SHS_UDP_PPS_PERF_CPU_LTHRESH,
			 RMNET_SHS_UDP_PPS_PERF_CPU_LTHRESH
			};
module_param_array(rmnet_shs_cpu_rx_min_pps_thresh, ullong, 0, 0644);
MODULE_PARM_DESC(rmnet_shs_cpu_rx_min_pps_thresh, "Min pkts core can handle");

unsigned int rmnet_shs_cpu_rx_flows[MAX_CPUS];
module_param_array(rmnet_shs_cpu_rx_flows, uint, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_cpu_rx_flows, "Num flows processed per core");

unsigned int rmnet_shs_cpu_rx_filter_flows[MAX_CPUS];
module_param_array(rmnet_shs_cpu_rx_filter_flows, uint, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_cpu_rx_filter_flows, "Num filtered flows per core");

unsigned long long rmnet_shs_cpu_rx_bytes[MAX_CPUS];
module_param_array(rmnet_shs_cpu_rx_bytes, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_cpu_rx_bytes, "SHS stamp bytes per CPU");

unsigned long long rmnet_shs_cpu_rx_pkts[MAX_CPUS];
module_param_array(rmnet_shs_cpu_rx_pkts, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_cpu_rx_pkts, "SHS stamp total pkts per CPU");

unsigned long long rmnet_shs_cpu_rx_bps[MAX_CPUS];
module_param_array(rmnet_shs_cpu_rx_bps, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_cpu_rx_bps, "SHS stamp enq rate per CPU");

unsigned long long rmnet_shs_cpu_rx_pps[MAX_CPUS];
module_param_array(rmnet_shs_cpu_rx_pps, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_cpu_rx_pps, "SHS stamp pkt enq rate per CPU");

unsigned long long rmnet_shs_cpu_qhead_diff[MAX_CPUS];
module_param_array(rmnet_shs_cpu_qhead_diff, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_cpu_qhead_diff, "SHS nw stack queue processed diff");

unsigned long long rmnet_shs_cpu_qhead_total[MAX_CPUS];
module_param_array(rmnet_shs_cpu_qhead_total, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_cpu_qhead_total, "SHS nw stack queue processed total");

unsigned long rmnet_shs_flow_hash[MAX_SUPPORTED_FLOWS_DEBUG];
module_param_array(rmnet_shs_flow_hash, ulong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_hash, "SHS stamp hash flow");

unsigned long rmnet_shs_flow_proto[MAX_SUPPORTED_FLOWS_DEBUG];
module_param_array(rmnet_shs_flow_proto, ulong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_proto, "SHS stamp hash transport protocol");

unsigned long long rmnet_shs_flow_inactive_tsec[MAX_SUPPORTED_FLOWS_DEBUG];
module_param_array(rmnet_shs_flow_inactive_tsec, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_inactive_tsec, "SHS stamp inactive flow time");

int rmnet_shs_flow_cpu[MAX_SUPPORTED_FLOWS_DEBUG] = {
			-1, -1, -1, -1, -1, -1, -1, -1,
			-1, -1, -1, -1, -1, -1, -1, -1};
module_param_array(rmnet_shs_flow_cpu, int, NULL, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_cpu, "SHS stamp flow processing CPU");

int rmnet_shs_flow_cpu_recommended[MAX_SUPPORTED_FLOWS_DEBUG] = {
			 -1, -1, -1, -1, -1, -1, -1, -1,
			 -1, -1, -1, -1, -1, -1, -1, -1
			 };
module_param_array(rmnet_shs_flow_cpu_recommended, int, NULL, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_cpu_recommended, "SHS stamp flow proc CPU");

unsigned long long rmnet_shs_flow_rx_bytes[MAX_SUPPORTED_FLOWS_DEBUG];
module_param_array(rmnet_shs_flow_rx_bytes, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_rx_bytes, "SHS stamp bytes per flow");

unsigned long long rmnet_shs_flow_rx_pkts[MAX_SUPPORTED_FLOWS_DEBUG];
module_param_array(rmnet_shs_flow_rx_pkts, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_rx_pkts, "SHS stamp total pkts per flow");

unsigned long long rmnet_shs_flow_rx_bps[MAX_SUPPORTED_FLOWS_DEBUG];
module_param_array(rmnet_shs_flow_rx_bps, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_rx_bps, "SHS stamp enq rate per flow");

unsigned long long rmnet_shs_flow_rx_pps[MAX_SUPPORTED_FLOWS_DEBUG];
module_param_array(rmnet_shs_flow_rx_pps, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_rx_pps, "SHS stamp pkt enq rate per flow");

/* Counters for suggestions made by wq */
unsigned long long rmnet_shs_flow_silver_to_gold[MAX_SUPPORTED_FLOWS_DEBUG];
module_param_array(rmnet_shs_flow_silver_to_gold, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_silver_to_gold, "SHS Suggest Silver to Gold");

unsigned long long rmnet_shs_flow_gold_to_silver[MAX_SUPPORTED_FLOWS_DEBUG];
module_param_array(rmnet_shs_flow_gold_to_silver, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_gold_to_silver, "SHS Suggest Gold to Silver");

unsigned long long rmnet_shs_flow_gold_balance[MAX_SUPPORTED_FLOWS_DEBUG];
module_param_array(rmnet_shs_flow_gold_balance, ullong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_flow_gold_balance, "SHS Suggest Gold Balance");

static DEFINE_SPINLOCK(rmnet_shs_hstat_tbl_lock);
static DEFINE_SPINLOCK(rmnet_shs_ep_lock);

static time_t rmnet_shs_wq_tnsec;
static struct workqueue_struct *rmnet_shs_wq;
static struct rmnet_shs_delay_wq_s *rmnet_shs_delayed_wq;
static struct rmnet_shs_wq_rx_flow_s rmnet_shs_rx_flow_tbl;

static struct list_head rmnet_shs_wq_hstat_tbl =
				LIST_HEAD_INIT(rmnet_shs_wq_hstat_tbl);
static int rmnet_shs_flow_dbg_stats_idx_cnt;
struct list_head rmnet_shs_wq_ep_tbl = LIST_HEAD_INIT(rmnet_shs_wq_ep_tbl);

/* Helper functions to add and remove entries to the table
 * that maintains a list of all endpoints (vnd's) available on this device.
 */
void rmnet_shs_wq_ep_tbl_add(struct rmnet_shs_wq_ep_s *ep)
{
	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_EP_TBL, RMNET_SHS_WQ_EP_TBL_ADD,
				0xDEF, 0xDEF, 0xDEF, 0xDEF, ep, NULL);
	list_add(&ep->ep_list_id, &rmnet_shs_wq_ep_tbl);
}

void rmnet_shs_wq_ep_tbl_remove(struct rmnet_shs_wq_ep_s *ep)
{
	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_EP_TBL, RMNET_SHS_WQ_EP_TBL_DEL,
				0xDEF, 0xDEF, 0xDEF, 0xDEF, ep, NULL);
	list_del_init(&ep->ep_list_id);
}

/* Helper functions to add and remove entries to the table
 * that maintains a list of all nodes that maintain statistics per flow
 */
void rmnet_shs_wq_hstat_tbl_add(struct rmnet_shs_wq_hstat_s *hnode)
{
	unsigned long flags;

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL,
			       RMNET_SHS_WQ_HSTAT_TBL_ADD,
				0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL);
	spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
	list_add(&hnode->hstat_node_id, &rmnet_shs_wq_hstat_tbl);
	spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);
}

void rmnet_shs_wq_hstat_tbl_remove(struct rmnet_shs_wq_hstat_s *hnode)
{
	unsigned long flags;

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL,
			       RMNET_SHS_WQ_HSTAT_TBL_DEL,
				0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL);

	spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
	list_del_init(&hnode->hstat_node_id);
	spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);

}

/* We maintain a list of all flow nodes processed by a cpu.
 * Below helper functions are used to maintain flow<=>cpu
 * association.*
 */
void rmnet_shs_wq_cpu_list_remove(struct rmnet_shs_wq_hstat_s *hnode)
{
	unsigned long flags;

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL,
			    RMNET_SHS_WQ_CPU_HSTAT_TBL_DEL,
			    0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL);
	spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
	list_del_init(&hnode->cpu_node_id);
	spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);

}

void rmnet_shs_wq_cpu_list_add(struct rmnet_shs_wq_hstat_s *hnode,
			    struct list_head *head)
{
	unsigned long flags;

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL,
			    RMNET_SHS_WQ_CPU_HSTAT_TBL_ADD,
			    0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL);

	spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
	list_add(&hnode->cpu_node_id, head);
	spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);
}

void rmnet_shs_wq_cpu_list_move(struct rmnet_shs_wq_hstat_s *hnode,
			     struct list_head *head)
{
	unsigned long flags;

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL,
			    RMNET_SHS_WQ_CPU_HSTAT_TBL_MOVE,
			    hnode->current_cpu,
			    0xDEF, 0xDEF, 0xDEF, hnode, NULL);
	spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
	list_move(&hnode->cpu_node_id, head);
	spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);

}

/* Resets all the parameters used to maintain hash statistics */
void rmnet_shs_wq_hstat_reset_node(struct rmnet_shs_wq_hstat_s *hnode)
{
	hnode->c_epoch = 0;
	hnode->l_epoch = 0;
	hnode->node = NULL;
	hnode->inactive_duration = 0;
	hnode->rx_skb = 0;
	hnode->rx_bytes = 0;
	hnode->rx_pps = 0;
	hnode->rx_bps = 0;
	hnode->last_rx_skb = 0;
	hnode->last_rx_bytes = 0;
	hnode->rps_config_msk = 0;
	hnode->current_core_msk = 0;
	hnode->def_core_msk = 0;
	hnode->pri_core_msk = 0;
	hnode->available_core_msk = 0;
	hnode->hash = 0;
	hnode->suggested_cpu = 0;
	hnode->current_cpu = 0;
	hnode->skb_tport_proto = 0;
	hnode->stat_idx = -1;
	INIT_LIST_HEAD(&hnode->cpu_node_id);
	hnode->is_new_flow = 0;
	/* clear in use flag as a last action. This is required to ensure
	 * the same node does not get allocated until all the paramaeters
	 * are cleared.
	 */
	hnode->in_use = 0;
	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL,
			    RMNET_SHS_WQ_HSTAT_TBL_NODE_RESET,
			    hnode->is_perm, 0xDEF, 0xDEF, 0xDEF, hnode, NULL);
}

/* Preallocates a set of flow nodes that maintain flow level statistics*/
void rmnet_shs_wq_hstat_alloc_nodes(u8 num_nodes_to_allocate, u8 is_store_perm)
{
	struct rmnet_shs_wq_hstat_s *hnode = NULL;

	while (num_nodes_to_allocate > 0) {
		hnode = kzalloc(sizeof(*hnode), GFP_ATOMIC);
		if (hnode) {
			hnode->is_perm = is_store_perm;
			rmnet_shs_wq_hstat_reset_node(hnode);
			INIT_LIST_HEAD(&hnode->hstat_node_id);
			INIT_LIST_HEAD(&hnode->cpu_node_id);
			rmnet_shs_wq_hstat_tbl_add(hnode);
		} else {
			rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_HSTAT_ERR]++;
		}
		hnode = NULL;
		num_nodes_to_allocate--;
	}

}

/* If there is an already pre-allocated node available and not in use,
 * we will try to re-use them.
 */
struct rmnet_shs_wq_hstat_s *rmnet_shs_wq_get_new_hstat_node(void)
{
	struct rmnet_shs_wq_hstat_s *hnode = NULL;
	struct rmnet_shs_wq_hstat_s *ret_node = NULL;
	unsigned long flags;

	spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
	list_for_each_entry(hnode, &rmnet_shs_wq_hstat_tbl, hstat_node_id) {
		if (hnode == NULL)
			continue;

		if (hnode->in_use == 0) {
			ret_node = hnode;
			ret_node->in_use = 1;
			ret_node->is_new_flow = 1;
			break;
		}
	}
	spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);

	if (ret_node) {
		trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL,
				    RMNET_SHS_WQ_HSTAT_TBL_NODE_REUSE,
				    hnode->is_perm, 0xDEF, 0xDEF, 0xDEF,
				    hnode, NULL);
		return ret_node;
	}

	/* We have reached a point where all pre-allocated nodes are in use
	 * Allocating memory to maintain the flow level stats for new flow.
	 * However, this newly allocated memory will be released as soon as we
	 * realize that this flow is inactive
	 */
	ret_node = kzalloc(sizeof(*hnode), GFP_ATOMIC);

	if (!ret_node) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_HSTAT_ERR]++;
		return NULL;
	}

	rmnet_shs_wq_hstat_reset_node(ret_node);
	ret_node->is_perm = 0;
	ret_node->in_use = 1;
	ret_node->is_new_flow = 1;
	INIT_LIST_HEAD(&ret_node->hstat_node_id);
	INIT_LIST_HEAD(&ret_node->cpu_node_id);

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL,
			    RMNET_SHS_WQ_HSTAT_TBL_NODE_DYN_ALLOCATE,
			    ret_node->is_perm, 0xDEF, 0xDEF, 0xDEF,
			    ret_node, NULL);

	rmnet_shs_wq_hstat_tbl_add(ret_node);

	return ret_node;
}

void rmnet_shs_wq_create_new_flow(struct rmnet_shs_skbn_s *node_p)
{
	struct timespec time;

	if (!node_p) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++;
		return;
	}

	node_p->hstats = rmnet_shs_wq_get_new_hstat_node();
	if (node_p->hstats != NULL) {
		(void)getnstimeofday(&time);

		node_p->hstats->hash = node_p->hash;
		node_p->hstats->skb_tport_proto = node_p->skb_tport_proto;
		node_p->hstats->current_cpu = node_p->map_cpu;
		node_p->hstats->suggested_cpu = node_p->map_cpu;

		/* Start TCP flows with segmentation if userspace connected */
		if (rmnet_shs_userspace_connected &&
		    node_p->hstats->skb_tport_proto == IPPROTO_TCP)
			node_p->hstats->segment_enable = 1;

		node_p->hstats->node = node_p;
		node_p->hstats->c_epoch = RMNET_SHS_SEC_TO_NSEC(time.tv_sec) +
		   time.tv_nsec;
		node_p->hstats->l_epoch = RMNET_SHS_SEC_TO_NSEC(time.tv_sec) +
		   time.tv_nsec;
	}

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_HSTAT_TBL,
				RMNET_SHS_WQ_HSTAT_TBL_NODE_NEW_REQ,
				0xDEF, 0xDEF, 0xDEF, 0xDEF,
				node_p, node_p->hstats);
}


/* Compute the average pps for a flow based on tuning param
 * Often when we decide to switch from a small cluster core,
 * it is because of the heavy traffic on that core. In such
 * circumstances, we want to switch to a big cluster
 * core as soon as possible. Therefore, we will provide a
 * greater weightage to the most recent sample compared to
 * the previous samples.
 *
 * On the other hand, when a flow which is on a big cluster
 * cpu suddenly starts to receive low traffic we move to a
 * small cluster core after observing low traffic for some
 * more samples. This approach avoids switching back and forth
 * to small cluster cpus due to momentary decrease in data
 * traffic.
 */
static u64 rmnet_shs_wq_get_flow_avg_pps(struct rmnet_shs_wq_hstat_s *hnode)
{
	u64 avg_pps, mov_avg_pps;
	u16 new_weight, old_weight;

	if (!hnode) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++;
		return 0;
	}

	if (rmnet_shs_is_lpwr_cpu(hnode->current_cpu)) {
		/* More weight to current value */
		new_weight = rmnet_shs_wq_tuning;
		old_weight = 100 - rmnet_shs_wq_tuning;
	} else {
		old_weight = rmnet_shs_wq_tuning;
		new_weight = 100 - rmnet_shs_wq_tuning;
	}

	/* computing weighted average per flow, if the flow has just started,
	 * there is no past values, so we use the current pps as the avg
	 */
	if (hnode->last_pps == 0) {
		avg_pps = hnode->rx_pps;
	} else {
		mov_avg_pps = (hnode->last_pps + hnode->avg_pps) / 2;
		avg_pps = (((new_weight * hnode->rx_pps) +
			    (old_weight * mov_avg_pps)) /
			    (new_weight + old_weight));
	}

	return avg_pps;
}

static u64 rmnet_shs_wq_get_cpu_avg_pps(u16 cpu_num)
{
	u64 avg_pps, mov_avg_pps;
	u16 new_weight, old_weight;
	struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_node;
	struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl;

	if (cpu_num >= MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++;
		return 0;
	}

	cpu_node = &rx_flow_tbl_p->cpu_list[cpu_num];

	if (rmnet_shs_is_lpwr_cpu(cpu_num)) {
		/* More weight to current value */
		new_weight = rmnet_shs_wq_tuning;
		old_weight = 100 - rmnet_shs_wq_tuning;
	} else {
		old_weight = rmnet_shs_wq_tuning;
		new_weight = 100 - rmnet_shs_wq_tuning;
	}

	/* computing weighted average per flow, if the cpu has not past values
	 * for pps, we use the current value as the average
	 */
	if (cpu_node->last_rx_pps == 0) {
		avg_pps = cpu_node->avg_pps;
	} else {
		mov_avg_pps = (cpu_node->last_rx_pps + cpu_node->avg_pps) / 2;
		avg_pps = (((new_weight * cpu_node->rx_pps) +
			    (old_weight * mov_avg_pps)) /
			    (new_weight + old_weight));
	}

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS,
			   RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_EVAL_CPU,
			   cpu_num, cpu_node->rx_pps, cpu_node->last_rx_pps,
			   avg_pps, NULL, NULL);

	return avg_pps;
}

/* Refresh the RPS mask associated with this flow */
void rmnet_shs_wq_update_hstat_rps_msk(struct rmnet_shs_wq_hstat_s *hstat_p)
{
	struct rmnet_shs_skbn_s *node_p = NULL;
	struct rmnet_shs_wq_ep_s *ep = NULL;

	if (!hstat_p) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++;
		return;
	}

	node_p = hstat_p->node;

	/*Map RPS mask from the endpoint associated with this flow*/
	list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) {

		if (ep && (node_p->dev == ep->ep)) {
			hstat_p->rps_config_msk = ep->rps_config_msk;
			hstat_p->def_core_msk = ep->default_core_msk;
			hstat_p->pri_core_msk = ep->pri_core_msk;

			/* Update ep tput stats while we're here */
			if (hstat_p->skb_tport_proto == IPPROTO_TCP) {
				rm_err("SHS_UDP: adding TCP bps %lu to ep_total %lu ep name %s",
				       hstat_p->rx_bps, ep->tcp_rx_bps, node_p->dev->name);
				ep->tcp_rx_bps += hstat_p->rx_bps;
			} else if (hstat_p->skb_tport_proto == IPPROTO_UDP) {
				rm_err("SHS_UDP: adding UDP rx_bps %lu to ep_total %lu ep name %s",
				       hstat_p->rx_bps, ep->udp_rx_bps, node_p->dev->name);
				ep->udp_rx_bps += hstat_p->rx_bps;
			}
			break;
		}
	}
	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_FLOW_STATS,
				RMNET_SHS_WQ_FLOW_STATS_UPDATE_MSK,
				hstat_p->rps_config_msk,
				hstat_p->def_core_msk,
				hstat_p->pri_core_msk,
				0xDEF, hstat_p, node_p);
}

void rmnet_shs_wq_update_hash_stats_debug(struct rmnet_shs_wq_hstat_s *hstats_p,
					  struct rmnet_shs_skbn_s *node_p)
{
	int idx = rmnet_shs_flow_dbg_stats_idx_cnt;

	if (!rmnet_shs_stats_enabled)
		return;

	if (!hstats_p || !node_p) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++;
		return;
	}

	if (hstats_p->stat_idx < 0) {
		idx = idx % MAX_SUPPORTED_FLOWS_DEBUG;
		hstats_p->stat_idx = idx;
		rmnet_shs_flow_dbg_stats_idx_cnt++;
	}

	rmnet_shs_flow_hash[hstats_p->stat_idx] = hstats_p->hash;
	rmnet_shs_flow_proto[hstats_p->stat_idx] = node_p->skb_tport_proto;
	rmnet_shs_flow_inactive_tsec[hstats_p->stat_idx] =
			RMNET_SHS_NSEC_TO_SEC(hstats_p->inactive_duration);
	rmnet_shs_flow_rx_bps[hstats_p->stat_idx] = hstats_p->rx_bps;
	rmnet_shs_flow_rx_pps[hstats_p->stat_idx] = hstats_p->rx_pps;
	rmnet_shs_flow_rx_bytes[hstats_p->stat_idx] = hstats_p->rx_bytes;
	rmnet_shs_flow_rx_pkts[hstats_p->stat_idx] = hstats_p->rx_skb;
	rmnet_shs_flow_cpu[hstats_p->stat_idx] = hstats_p->current_cpu;
	rmnet_shs_flow_cpu_recommended[hstats_p->stat_idx] =
						hstats_p->suggested_cpu;
	rmnet_shs_flow_silver_to_gold[hstats_p->stat_idx] =
		hstats_p->rmnet_shs_wq_suggs[RMNET_SHS_WQ_SUGG_SILVER_TO_GOLD];
	rmnet_shs_flow_gold_to_silver[hstats_p->stat_idx] =
		hstats_p->rmnet_shs_wq_suggs[RMNET_SHS_WQ_SUGG_GOLD_TO_SILVER];
	rmnet_shs_flow_gold_balance[hstats_p->stat_idx] =
		hstats_p->rmnet_shs_wq_suggs[RMNET_SHS_WQ_SUGG_GOLD_BALANCE];

}

/* Returns TRUE if this flow received a new packet
 *         FALSE otherwise
 */
u8 rmnet_shs_wq_is_hash_rx_new_pkt(struct rmnet_shs_wq_hstat_s *hstats_p,
				   struct rmnet_shs_skbn_s *node_p)
{
	if (!hstats_p || !node_p) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++;
		return 0;
	}

	if (node_p->num_skb == hstats_p->rx_skb)
		return 0;

	return 1;
}

void rmnet_shs_wq_update_hash_tinactive(struct rmnet_shs_wq_hstat_s *hstats_p,
					struct rmnet_shs_skbn_s *node_p)
{
	time_t tdiff;

	if (!hstats_p || !node_p) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++;
		return;
	}

	tdiff = rmnet_shs_wq_tnsec - hstats_p->c_epoch;
	hstats_p->inactive_duration = tdiff;

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_FLOW_STATS,
				RMNET_SHS_WQ_FLOW_STATS_FLOW_INACTIVE,
				hstats_p->hash, tdiff, 0xDEF, 0xDEF,
				hstats_p, NULL);
}

void rmnet_shs_wq_update_hash_stats(struct rmnet_shs_wq_hstat_s *hstats_p)
{
	time_t tdiff;
	u64 skb_diff, bytes_diff;
	struct rmnet_shs_skbn_s *node_p;

	if (!hstats_p) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++;
		return;
	}

	node_p = hstats_p->node;

	if (!rmnet_shs_wq_is_hash_rx_new_pkt(hstats_p, node_p)) {
		hstats_p->rx_pps = 0;
		hstats_p->avg_pps = 0;
		hstats_p->rx_bps = 0;
		rmnet_shs_wq_update_hash_tinactive(hstats_p, node_p);
		rmnet_shs_wq_update_hash_stats_debug(hstats_p, node_p);
		return;
	}

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_FLOW_STATS,
				RMNET_SHS_WQ_FLOW_STATS_START,
				hstats_p->hash, 0xDEF, hstats_p->rx_pps,
				hstats_p->rx_bps, hstats_p, NULL);

	rmnet_shs_wq_update_hstat_rps_msk(hstats_p);
	hstats_p->inactive_duration = 0;
	hstats_p->l_epoch = node_p->hstats->c_epoch;
	hstats_p->last_rx_skb = node_p->hstats->rx_skb;
	hstats_p->last_rx_bytes = node_p->hstats->rx_bytes;

	hstats_p->c_epoch = rmnet_shs_wq_tnsec;
	hstats_p->rx_skb = node_p->num_skb;
	hstats_p->rx_bytes = node_p->num_skb_bytes;

	tdiff = (hstats_p->c_epoch - hstats_p->l_epoch);
	skb_diff = hstats_p->rx_skb - hstats_p->last_rx_skb;
	bytes_diff = hstats_p->rx_bytes - hstats_p->last_rx_bytes;

	hstats_p->rx_pps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(skb_diff)/(tdiff);
	hstats_p->rx_bps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(bytes_diff)/(tdiff);
	hstats_p->rx_bps = RMNET_SHS_BYTE_TO_BIT(hstats_p->rx_bps);
	hstats_p->avg_pps = rmnet_shs_wq_get_flow_avg_pps(hstats_p);
	hstats_p->last_pps = hstats_p->rx_pps;
	rmnet_shs_wq_update_hash_stats_debug(hstats_p, node_p);

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_FLOW_STATS,
				RMNET_SHS_WQ_FLOW_STATS_END,
				hstats_p->hash, hstats_p->rx_pps,
				hstats_p->rx_bps, (tdiff/1000000),
				hstats_p, NULL);
}

static void rmnet_shs_wq_refresh_cpu_rates_debug(u16 cpu,
				struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_p)
{
	if (!rmnet_shs_stats_enabled)
		return;

	if (cpu >= MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++;
		return;
	}

	if (!cpu_p) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++;
		return;
	}

	rmnet_shs_cpu_rx_bps[cpu] = cpu_p->rx_bps;
	rmnet_shs_cpu_rx_pps[cpu] = cpu_p->rx_pps;
	rmnet_shs_cpu_rx_flows[cpu] = cpu_p->flows;
	rmnet_shs_cpu_rx_bytes[cpu] = cpu_p->rx_bytes;
	rmnet_shs_cpu_rx_pkts[cpu] = cpu_p->rx_skbs;
	rmnet_shs_cpu_qhead_diff[cpu] = cpu_p->qhead_diff;
	rmnet_shs_cpu_qhead_total[cpu] = cpu_p->qhead_total;
}

static void rmnet_shs_wq_refresh_dl_mrkr_stats(void)
{
	struct rmnet_shs_wq_rx_flow_s *tbl_p = &rmnet_shs_rx_flow_tbl;
	struct rmnet_port *port;
	u64 pkt_diff, byte_diff;
	time_t tdiff;

	tbl_p->dl_mrk_last_rx_bytes = tbl_p->dl_mrk_rx_bytes;
	tbl_p->dl_mrk_last_rx_pkts = tbl_p->dl_mrk_rx_pkts;

	port = rmnet_shs_cfg.port;
	if (!port) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_GET_RMNET_PORT_ERR]++;
		return;
	}
	tbl_p->dl_mrk_rx_pkts = port->stats.dl_hdr_total_pkts;
	tbl_p->dl_mrk_rx_bytes = port->stats.dl_hdr_total_bytes;
	tdiff = rmnet_shs_wq_tnsec - tbl_p->l_epoch;
	pkt_diff = tbl_p->dl_mrk_rx_pkts - tbl_p->dl_mrk_last_rx_pkts;
	byte_diff = tbl_p->dl_mrk_rx_bytes - tbl_p->dl_mrk_last_rx_bytes;
	tbl_p->dl_mrk_rx_pps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(pkt_diff)/tdiff;
	tbl_p->dl_mrk_rx_bps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(byte_diff)/tdiff;
	tbl_p->dl_mrk_rx_bps = RMNET_SHS_BYTE_TO_BIT(tbl_p->dl_mrk_rx_bps);

}

static void rmnet_shs_wq_refresh_total_stats(void)
{
	struct rmnet_shs_wq_rx_flow_s *tbl_p = &rmnet_shs_rx_flow_tbl;
	u64 pkt_diff, byte_diff, pps, bps;
	time_t tdiff;

	tdiff = rmnet_shs_wq_tnsec - tbl_p->l_epoch;
	pkt_diff = (tbl_p->rx_skbs -  tbl_p->last_rx_skbs);
	byte_diff = tbl_p->rx_bytes -  tbl_p->last_rx_bytes;
	pps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(pkt_diff)/tdiff;
	bps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(byte_diff)/tdiff;
	tbl_p->last_rx_bps = tbl_p->rx_bps;
	tbl_p->last_rx_pps = tbl_p->rx_pps;
	tbl_p->rx_bps = RMNET_SHS_BYTE_TO_BIT(bps);
	tbl_p->rx_pps = pps;
	tbl_p->l_epoch  = rmnet_shs_wq_tnsec;
	tbl_p->last_rx_bytes = tbl_p->rx_bytes;
	tbl_p->last_rx_skbs = tbl_p->rx_skbs;

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_TOTAL_STATS,
				RMNET_SHS_WQ_TOTAL_STATS_UPDATE,
				tbl_p->rx_pps,
				tbl_p->dl_mrk_rx_pps,
				tbl_p->rx_bps,
				tbl_p->dl_mrk_rx_bps, NULL, NULL);

}

static void rmnet_shs_wq_refresh_cpu_stats(u16 cpu)
{
	struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_p;
	time_t tdiff;
	u64 new_skbs, new_bytes;
	u64 last_rx_bps, last_rx_pps;
	u32 new_qhead;

	if (cpu >= MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++;
		return;
	}

	cpu_p = &rmnet_shs_rx_flow_tbl.cpu_list[cpu];
	new_skbs = cpu_p->rx_skbs - cpu_p->last_rx_skbs;

	new_qhead = rmnet_shs_get_cpu_qhead(cpu);
	if (cpu_p->qhead_start == 0)
		cpu_p->qhead_start = new_qhead;

	cpu_p->last_qhead = cpu_p->qhead;
	cpu_p->qhead = new_qhead;
	cpu_p->qhead_diff = cpu_p->qhead - cpu_p->last_qhead;
	cpu_p->qhead_total = cpu_p->qhead - cpu_p->qhead_start;

	if (rmnet_shs_cpu_node_tbl[cpu].wqprio)
		rmnet_shs_cpu_node_tbl[cpu].wqprio = (rmnet_shs_cpu_node_tbl[cpu].wqprio + 1)
						     % (PRIO_BACKOFF);
	if (new_skbs == 0) {
		cpu_p->l_epoch =  rmnet_shs_wq_tnsec;
		cpu_p->rx_bps = 0;
		cpu_p->rx_pps = 0;
		cpu_p->avg_pps = 0;
		if (rmnet_shs_userspace_connected) {
			rmnet_shs_wq_cpu_caps_list_add(&rmnet_shs_rx_flow_tbl,
						       cpu_p, &cpu_caps);
		}
		rmnet_shs_wq_refresh_cpu_rates_debug(cpu, cpu_p);
		return;
	}

	tdiff = rmnet_shs_wq_tnsec - cpu_p->l_epoch;
	new_bytes = cpu_p->rx_bytes - cpu_p->last_rx_bytes;

	last_rx_bps = cpu_p->rx_bps;
	last_rx_pps = cpu_p->rx_pps;
	cpu_p->rx_pps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(new_skbs)/tdiff;
	cpu_p->rx_bps = RMNET_SHS_RX_BPNSEC_TO_BPSEC(new_bytes)/tdiff;
	cpu_p->rx_bps = RMNET_SHS_BYTE_TO_BIT(cpu_p->rx_bps);
	cpu_p->avg_pps = rmnet_shs_wq_get_cpu_avg_pps(cpu);
	cpu_p->last_rx_bps = last_rx_bps;
	cpu_p->last_rx_pps = last_rx_pps;

	cpu_p->l_epoch =  rmnet_shs_wq_tnsec;
	cpu_p->last_rx_skbs = cpu_p->rx_skbs;
	cpu_p->last_rx_bytes = cpu_p->rx_bytes;
	cpu_p->rx_bps_est = cpu_p->rx_bps;

	if (rmnet_shs_userspace_connected) {
		rmnet_shs_wq_cpu_caps_list_add(&rmnet_shs_rx_flow_tbl,
					       cpu_p, &cpu_caps);
	}

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS,
				RMNET_SHS_WQ_CPU_STATS_UPDATE, cpu,
				cpu_p->flows, cpu_p->rx_pps,
				cpu_p->rx_bps, NULL, NULL);
	rmnet_shs_wq_refresh_cpu_rates_debug(cpu, cpu_p);

}

static void rmnet_shs_wq_refresh_all_cpu_stats(void)
{
	u16 cpu;

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS,
				RMNET_SHS_WQ_CPU_STATS_START,
				0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);

	for (cpu = 0; cpu < MAX_CPUS; cpu++)
		rmnet_shs_wq_refresh_cpu_stats(cpu);

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS,
				RMNET_SHS_WQ_CPU_STATS_END,
				0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);
}

void rmnet_shs_wq_update_cpu_rx_tbl(struct rmnet_shs_wq_hstat_s *hstat_p)
{
	struct rmnet_shs_wq_rx_flow_s *tbl_p = &rmnet_shs_rx_flow_tbl;
	struct rmnet_shs_skbn_s *node_p;
	u64 skb_diff, byte_diff;
	u16 cpu_num;

	if (!hstat_p) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++;
		return;
	}

	node_p = hstat_p->node;

	if (hstat_p->inactive_duration > 0)
		return;

	cpu_num = node_p->map_cpu;

	if (cpu_num >= MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_INVALID_CPU_ERR]++;
		return;
	}
	skb_diff = hstat_p->rx_skb - hstat_p->last_rx_skb;
	byte_diff = hstat_p->rx_bytes - hstat_p->last_rx_bytes;

	if (hstat_p->is_new_flow) {
		rmnet_shs_wq_cpu_list_add(hstat_p,
				       &tbl_p->cpu_list[cpu_num].hstat_id);
		rm_err("SHS_FLOW: adding flow 0x%x on cpu[%d] "
		       "pps: %llu | avg_pps %llu",
		       hstat_p->hash, hstat_p->current_cpu,
		       hstat_p->rx_pps, hstat_p->avg_pps);
		hstat_p->is_new_flow = 0;
	}
	/* check if the flow has switched to another CPU*/
	if (cpu_num != hstat_p->current_cpu) {
		rm_err("SHS_FLOW: moving flow 0x%x on cpu[%d] to cpu[%d] "
		       "pps: %llu | avg_pps %llu",
		       hstat_p->hash, hstat_p->current_cpu, cpu_num,
		       hstat_p->rx_pps, hstat_p->avg_pps);
		trace_rmnet_shs_wq_high(RMNET_SHS_WQ_FLOW_STATS,
					RMNET_SHS_WQ_FLOW_STATS_UPDATE_NEW_CPU,
					hstat_p->hash, hstat_p->current_cpu,
					cpu_num, 0xDEF, hstat_p, NULL);

		rmnet_shs_wq_cpu_list_move(hstat_p,
				   &tbl_p->cpu_list[cpu_num].hstat_id);

		rmnet_shs_wq_inc_cpu_flow(cpu_num);
		rmnet_shs_wq_dec_cpu_flow(hstat_p->current_cpu);
		hstat_p->current_cpu = cpu_num;
	}

	/* Assuming that the data transfers after the last refresh
	 * interval have happened with the newer CPU
	 */
	tbl_p->cpu_list[cpu_num].rx_skbs += skb_diff;
	tbl_p->cpu_list[cpu_num].rx_bytes += byte_diff;
	tbl_p->rx_skbs += skb_diff;
	tbl_p->rx_bytes += byte_diff;

}

void rmnet_shs_wq_chng_suggested_cpu(u16 old_cpu, u16 new_cpu,
					      struct rmnet_shs_wq_ep_s *ep)
{
	struct rmnet_shs_skbn_s *node_p;
	struct rmnet_shs_wq_hstat_s *hstat_p;
	u16 bkt;

	hash_for_each(RMNET_SHS_HT, bkt, node_p, list) {
		if (!node_p)
			continue;

		if (!node_p->hstats)
			continue;

		hstat_p = node_p->hstats;

		if ((hstat_p->suggested_cpu == old_cpu) &&
		    (node_p->dev == ep->ep)) {

			trace_rmnet_shs_wq_high(RMNET_SHS_WQ_FLOW_STATS,
				RMNET_SHS_WQ_FLOW_STATS_SUGGEST_NEW_CPU,
				hstat_p->hash, hstat_p->suggested_cpu,
				new_cpu, 0xDEF, hstat_p, NULL);

			node_p->hstats->suggested_cpu = new_cpu;
		}
	}
}

/* Increment the per-flow counter for suggestion type */
static void rmnet_shs_wq_inc_sugg_type(u32 sugg_type,
				       struct rmnet_shs_wq_hstat_s *hstat_p)
{
	if (sugg_type >= RMNET_SHS_WQ_SUGG_MAX || hstat_p == NULL)
		return;

	hstat_p->rmnet_shs_wq_suggs[sugg_type] += 1;
}

/* Change suggested cpu, return 1 if suggestion was made, 0 otherwise */
static int rmnet_shs_wq_chng_flow_cpu(u16 old_cpu, u16 new_cpu,
				      struct rmnet_shs_wq_ep_s *ep,
				      u32 hash_to_move, u32 sugg_type)
{
	struct rmnet_shs_skbn_s *node_p;
	struct rmnet_shs_wq_hstat_s *hstat_p;
	int rc = 0;
	u16 bkt;

	if (!ep) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++;
		return 0;
	}

	if (old_cpu >= MAX_CPUS || new_cpu >= MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++;
		return 0;
	}

	hash_for_each(RMNET_SHS_HT, bkt, node_p, list) {
		if (!node_p)
			continue;

		if (!node_p->hstats)
			continue;

		hstat_p = node_p->hstats;

		if (hash_to_move != 0) {
			/* If hash_to_move is given, only move that flow,
			 * otherwise move all the flows on that cpu
			 */
			if (hstat_p->hash != hash_to_move)
				continue;
		}

		rm_err("SHS_HT: >>  sugg cpu %d | old cpu %d | new_cpu %d | "
		       "map_cpu = %d | flow 0x%x",
		       hstat_p->suggested_cpu, old_cpu, new_cpu,
		       node_p->map_cpu, hash_to_move);

		if ((hstat_p->suggested_cpu == old_cpu) &&
		    (node_p->dev == ep->ep)) {

			trace_rmnet_shs_wq_high(RMNET_SHS_WQ_FLOW_STATS,
				RMNET_SHS_WQ_FLOW_STATS_SUGGEST_NEW_CPU,
				hstat_p->hash, hstat_p->suggested_cpu,
				new_cpu, 0xDEF, hstat_p, NULL);

			node_p->hstats->suggested_cpu = new_cpu;
			rmnet_shs_wq_inc_sugg_type(sugg_type, hstat_p);
			if (hash_to_move) { /* Stop after moving one flow */
				rm_err("SHS_CHNG: moving single flow: flow 0x%x "
				       "sugg_cpu changed from %d to %d",
				       hstat_p->hash, old_cpu,
				       node_p->hstats->suggested_cpu);
				return 1;
			}
			rm_err("SHS_CHNG: moving all flows: flow 0x%x "
			       "sugg_cpu changed from %d to %d",
			       hstat_p->hash, old_cpu,
			       node_p->hstats->suggested_cpu);
			rc |= 1;
		}
	}
	return rc;
}

u64 rmnet_shs_wq_get_max_pps_among_cores(u32 core_msk)
{
	int cpu_num;
	u64 max_pps = 0;
	struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl;

	for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) {
		if (((1 << cpu_num) & core_msk) &&
		     (rx_flow_tbl_p->cpu_list[cpu_num].rx_pps > max_pps)) {
			max_pps = rx_flow_tbl_p->cpu_list[cpu_num].rx_pps;
		}
	}
	return max_pps;
}

/* Returns the least utilized core from a core mask
 * In order of priority
 *    1) Returns leftmost core with no flows (Fully Idle)
 *    2) Returns the core with least flows with no pps (Semi Idle)
 *    3) Returns the core with the least pps (Non-Idle)
 */
int rmnet_shs_wq_get_least_utilized_core(u16 core_msk)
{
	struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl;
	struct rmnet_shs_wq_cpu_rx_pkt_q_s *list_p;
	u64 min_pps = U64_MAX;
	u32 min_flows = U32_MAX;
	int ret_val = -1;
	int semi_idle_ret = -1;
	int full_idle_ret = -1;
	int cpu_num = 0;
	u16 is_cpu_in_msk;

	for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) {

		is_cpu_in_msk = (1 << cpu_num) & core_msk;
		if (!is_cpu_in_msk)
			continue;

		list_p = &rx_flow_tbl_p->cpu_list[cpu_num];
		trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS,
				       RMNET_SHS_WQ_CPU_STATS_CURRENT_UTIL,
				       cpu_num, list_p->rx_pps, min_pps,
				       0, NULL, NULL);

		/* When there are multiple free CPUs the first free CPU will
		 * be returned
		 */
		if (list_p->flows == 0) {
			full_idle_ret = cpu_num;
			break;
		}
		/* When there are semi-idle CPUs the CPU w/ least flows will
		 * be returned
		 */
		if (list_p->rx_pps == 0 && list_p->flows < min_flows) {
			min_flows = list_p->flows;
			semi_idle_ret = cpu_num;
		}

		/* Found a core that is processing even lower packets */
		if (list_p->rx_pps <= min_pps) {
			min_pps = list_p->rx_pps;
			ret_val = cpu_num;
		}
	}

	if (full_idle_ret >= 0)
		ret_val = full_idle_ret;
	else if (semi_idle_ret >= 0)
		ret_val = semi_idle_ret;

	return ret_val;
}

u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu,
					struct rmnet_shs_wq_ep_s *ep)
{
	struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl;
	struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_list_p, *cur_cpu_list_p;
	u64 cpu_rx_pps, reqd_pps, cur_cpu_rx_pps;
	u64 pps_uthresh, pps_lthresh = 0;
	u16 cpu_to_move = current_cpu;
	u16 cpu_num;
	u8 is_core_in_msk;
	u32 cpu_to_move_util = 0;

	if (!ep) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++;
		return cpu_to_move;
	}

	cur_cpu_list_p = &rx_flow_tbl_p->cpu_list[current_cpu];
	cur_cpu_rx_pps = cur_cpu_list_p->rx_pps;
	pps_uthresh = rmnet_shs_cpu_rx_max_pps_thresh[current_cpu];
	pps_lthresh = rmnet_shs_cpu_rx_min_pps_thresh[current_cpu];

	/* If we are already on a perf core and required pps is beyond
	 * beyond the capacity that even perf cores aren't sufficient
	 * there is nothing much we can do. So we will continue to let flows
	 * process packets on same perf core
	 */
	if (!rmnet_shs_is_lpwr_cpu(current_cpu) &&
	    (cur_cpu_rx_pps > pps_lthresh)) {
		return cpu_to_move;
	}
	/* If a core (should only be lpwr was marked prio we don't touch it
	 * for a few ticks and reset it afterwards
	 */

	if (rmnet_shs_cpu_node_tbl[current_cpu].wqprio)
		return current_cpu;

	for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) {

		is_core_in_msk = ((1 << cpu_num) & ep->rps_config_msk);

		/* We are looking for a core that is configured and that
		 * can handle traffic better than the current core
		 */
		if ((cpu_num == current_cpu) || (!is_core_in_msk) ||
		    !cpu_online(current_cpu))
			continue;

		pps_uthresh = rmnet_shs_cpu_rx_max_pps_thresh[cpu_num];
		pps_lthresh = rmnet_shs_cpu_rx_min_pps_thresh[cpu_num];

		cpu_list_p = &rx_flow_tbl_p->cpu_list[cpu_num];
		cpu_rx_pps = cpu_list_p->rx_pps;
		reqd_pps = cpu_rx_pps + cur_cpu_rx_pps;

		trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS,
				       RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_FIND,
				       current_cpu, cpu_num, reqd_pps,
				       cpu_rx_pps, NULL, NULL);

		/* Return the most available valid CPU */
		if ((reqd_pps > pps_lthresh) && (reqd_pps < pps_uthresh) &&
			cpu_rx_pps <= cpu_to_move_util) {
			cpu_to_move = cpu_num;
			cpu_to_move_util = cpu_rx_pps;
		}
	}

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS,
			     RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_FIND,
			     current_cpu, cpu_to_move, cur_cpu_rx_pps,
			     rx_flow_tbl_p->cpu_list[cpu_to_move].rx_pps,
			     NULL, NULL);
	return cpu_to_move;
}

void rmnet_shs_wq_find_cpu_and_move_flows(u16 cur_cpu)
{
	struct rmnet_shs_wq_ep_s *ep = NULL;
	u16 new_cpu;

	list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) {
		if (!ep)
			continue;

		if (!ep->is_ep_active)
			continue;

		new_cpu = rmnet_shs_wq_find_cpu_to_move_flows(cur_cpu, ep);

		if (new_cpu != cur_cpu)
			rmnet_shs_wq_chng_suggested_cpu(cur_cpu, new_cpu, ep);
	}
}

/* Return 1 if we can move a flow to dest_cpu for this endpoint,
 * otherwise return 0. Basically check rps mask and cpu is online
 * Also check that dest cpu is not isolated
 */
int rmnet_shs_wq_check_cpu_move_for_ep(u16 current_cpu, u16 dest_cpu,
				       struct rmnet_shs_wq_ep_s *ep)
{
	u16 cpu_in_rps_mask = 0;

	if (!ep) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++;
		return 0;
	}

	if (current_cpu >= MAX_CPUS || dest_cpu >= MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++;
		return 0;
	}

	cpu_in_rps_mask = (1 << dest_cpu) & ep->rps_config_msk;

	rm_err("SHS_MASK:  cur cpu [%d] | dest_cpu [%d] | "
	       "cpu isolation_mask = 0x%x | ep_rps_mask = 0x%x | "
	       "cpu_online(dest) = %d cpu_in_rps_mask = %d | "
	       "cpu isolated(dest) = %d",
	       current_cpu, dest_cpu, __cpu_isolated_mask, ep->rps_config_msk,
	       cpu_online(dest_cpu), cpu_in_rps_mask, cpu_isolated(dest_cpu));

	/* We cannot move to dest cpu if the cur cpu is the same,
	 * the dest cpu is offline, dest cpu is not in the rps mask,
	 * or if the dest cpu is isolated
	 */
	if (current_cpu == dest_cpu || !cpu_online(dest_cpu) ||
	    !cpu_in_rps_mask || cpu_isolated(dest_cpu)) {
		return 0;
	}

	return 1;
}

/* rmnet_shs_wq_try_to_move_flow - try to make a flow suggestion
 * return 1 if flow move was suggested, otherwise return 0
 */
int rmnet_shs_wq_try_to_move_flow(u16 cur_cpu, u16 dest_cpu, u32 hash_to_move,
				  u32 sugg_type)
{
	unsigned long flags;
	struct rmnet_shs_wq_ep_s *ep;

	if (cur_cpu >= MAX_CPUS || dest_cpu >= MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++;
		return 0;
	}

	/* Traverse end-point list, check if cpu can be used, based
	 * on it if is online, rps mask, isolation, etc. then make
	 * suggestion to change the cpu for the flow by passing its hash
	 */
	spin_lock_irqsave(&rmnet_shs_ep_lock, flags);
	list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) {
		if (!ep)
			continue;

		if (!ep->is_ep_active)
			continue;

		if (!rmnet_shs_wq_check_cpu_move_for_ep(cur_cpu,
							dest_cpu,
							ep)) {
			rm_err("SHS_FDESC: >> Cannot move flow 0x%x on ep"
			       " from cpu[%d] to cpu[%d]",
			       hash_to_move, cur_cpu, dest_cpu);
			continue;
		}

		if (rmnet_shs_wq_chng_flow_cpu(cur_cpu, dest_cpu, ep,
					       hash_to_move, sugg_type)) {
			rm_err("SHS_FDESC: >> flow 0x%x was suggested to"
			       " move from cpu[%d] to cpu[%d] sugg_type [%d]",
			       hash_to_move, cur_cpu, dest_cpu, sugg_type);

			spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags);
			return 1;
		}
	}

	spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags);
	return 0;
}

/* Change flow segmentation, return 1 if set, 0 otherwise */
int rmnet_shs_wq_set_flow_segmentation(u32 hash_to_set, u8 seg_enable)
{
	struct rmnet_shs_skbn_s *node_p;
	struct rmnet_shs_wq_hstat_s *hstat_p;
	unsigned long ht_flags;
	u16 bkt;

	spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags);
	hash_for_each(RMNET_SHS_HT, bkt, node_p, list) {
		if (!node_p)
			continue;

		if (!node_p->hstats)
			continue;

		hstat_p = node_p->hstats;

		if (hstat_p->hash != hash_to_set)
			continue;

		rm_err("SHS_HT: >> segmentation on hash 0x%x enable %u",
		       hash_to_set, seg_enable);

		trace_rmnet_shs_wq_high(RMNET_SHS_WQ_FLOW_STATS,
				RMNET_SHS_WQ_FLOW_STATS_SET_FLOW_SEGMENTATION,
				hstat_p->hash, seg_enable,
				0xDEF, 0xDEF, hstat_p, NULL);

		node_p->hstats->segment_enable = seg_enable;
		spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags);
		return 1;
	}
	spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags);

	rm_err("SHS_HT: >> segmentation on hash 0x%x enable %u not set - hash not found",
	       hash_to_set, seg_enable);
	return 0;
}


/* Comparison function to sort gold flow loads - based on flow avg_pps
 * return -1 if a is before b, 1 if a is after b, 0 if equal
 */
int cmp_fn_flow_pps(void *priv, struct list_head *a, struct list_head *b)
{
	struct rmnet_shs_wq_gold_flow_s *flow_a;
	struct rmnet_shs_wq_gold_flow_s *flow_b;

	if (!a || !b)
		return 0;

	flow_a = list_entry(a, struct rmnet_shs_wq_gold_flow_s, gflow_list);
	flow_b = list_entry(b, struct rmnet_shs_wq_gold_flow_s, gflow_list);

	if (flow_a->avg_pps > flow_b->avg_pps)
		return -1;
	else if (flow_a->avg_pps < flow_b->avg_pps)
		return 1;

	return 0;
}

/* Comparison function to sort cpu capacities - based on cpu avg_pps capacity
 * return -1 if a is before b, 1 if a is after b, 0 if equal
 */
int cmp_fn_cpu_pps(void *priv, struct list_head *a, struct list_head *b)
{
	struct rmnet_shs_wq_cpu_cap_s *cpu_a;
	struct rmnet_shs_wq_cpu_cap_s *cpu_b;

	if (!a || !b)
		return 0;

	cpu_a = list_entry(a, struct rmnet_shs_wq_cpu_cap_s, cpu_cap_list);
	cpu_b = list_entry(b, struct rmnet_shs_wq_cpu_cap_s, cpu_cap_list);

	if (cpu_a->avg_pps_capacity > cpu_b->avg_pps_capacity)
		return -1;
	else if (cpu_a->avg_pps_capacity < cpu_b->avg_pps_capacity)
		return 1;

	return 0;
}


/* Prints cpu stats and flows to dmesg for debugging */
void rmnet_shs_wq_debug_print_flows(void)
{
	struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl;
	struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_node;
	struct rmnet_shs_wq_hstat_s *hnode;
	int flows, i;
	u16 cpu_num = 0;

	if (!RMNET_SHS_DEBUG)
		return;

	for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) {
		cpu_node = &rx_flow_tbl_p->cpu_list[cpu_num];
		flows = rx_flow_tbl_p->cpu_list[cpu_num].flows;

		rm_err("SHS_CPU: cpu[%d]: flows=%d pps=%llu bps=%llu "
		       "qhead_diff %u qhead_total = %u qhead_start = %u "
		       "qhead = %u qhead_last = %u isolated = %d ",
		       cpu_num, flows, cpu_node->rx_pps, cpu_node->rx_bps,
		       cpu_node->qhead_diff, cpu_node->qhead_total,
		       cpu_node->qhead_start,
		       cpu_node->qhead, cpu_node->last_qhead,
		       cpu_isolated(cpu_num));

		list_for_each_entry(hnode,
				    &rmnet_shs_wq_hstat_tbl,
				    hstat_node_id) {
			if (!hnode)
				continue;

			if (hnode->in_use == 0)
				continue;

			if (hnode->node) {
				if (hnode->current_cpu == cpu_num)
					rm_err("SHS_CPU:         > flow 0x%x "
					       "with pps %llu avg_pps %llu rx_bps %llu ",
					       hnode->hash, hnode->rx_pps,
					       hnode->avg_pps, hnode->rx_bps);
			}
		} /* loop per flow */

		for (i = 0; i < 3 - flows; i++) {
			rm_err("%s", "SHS_CPU:         > ");
		}
	} /* loop per cpu */
}

/* Prints the sorted gold flow list to dmesg */
void rmnet_shs_wq_debug_print_sorted_gold_flows(struct list_head *gold_flows)
{
	struct rmnet_shs_wq_gold_flow_s *gflow_node;

	if (!RMNET_SHS_DEBUG)
		return;

	if (!gold_flows) {
		rm_err("%s", "SHS_GDMA: Gold Flows List is NULL");
		return;
	}

	rm_err("%s", "SHS_GDMA: List of sorted gold flows:");
	list_for_each_entry(gflow_node, gold_flows, gflow_list) {
		if (!gflow_node)
			continue;

		rm_err("SHS_GDMA: > flow 0x%x with pps %llu on cpu[%d]",
		       gflow_node->hash, gflow_node->rx_pps,
		       gflow_node->cpu_num);
	}
}

/* Userspace evaluation. we send userspace the response to the sync message
 * after we update shared memory. shsusr will send a netlink message if
 * flows should be moved around.
 */
void rmnet_shs_wq_eval_cpus_caps_and_flows(struct list_head *cpu_caps,
					   struct list_head *gold_flows,
					   struct list_head *ss_flows)
{
	if (!cpu_caps || !gold_flows || !ss_flows) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_PTR_ERR]++;
		return;
	}

	list_sort(NULL, cpu_caps, &cmp_fn_cpu_pps);
	list_sort(NULL, gold_flows, &cmp_fn_flow_pps);

	rmnet_shs_wq_mem_update_cached_cpu_caps(cpu_caps);
	rmnet_shs_wq_mem_update_cached_sorted_gold_flows(gold_flows);
	rmnet_shs_wq_mem_update_cached_sorted_ss_flows(ss_flows);
	rmnet_shs_wq_mem_update_cached_netdevs();

	rmnet_shs_genl_send_int_to_userspace_no_info(RMNET_SHS_SYNC_RESP_INT);

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_SHSUSR, RMNET_SHS_WQ_SHSUSR_SYNC_END,
				0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);
}

/* Default wq evaluation logic, use this if rmnet_shs_userspace_connected is 0 */
void rmnet_shs_wq_eval_suggested_cpu(void)

{
	struct rmnet_shs_wq_rx_flow_s *rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl;
	struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_list_p;
	u64 cpu_curr_pps, cpu_last_pps, last_avg_pps;
	u64 moving_avg_pps, avg_pps;
	u64 pps_uthresh, pps_lthresh = 0;
	u16 cpu_num, new_weight, old_weight;
	int flows;

	for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) {
		flows = rx_flow_tbl_p->cpu_list[cpu_num].flows;

		/* Nothing to evaluate if there is no traffic on this cpu */
		if (flows <= 0)
			continue;

		cpu_list_p = &rx_flow_tbl_p->cpu_list[cpu_num];
		cpu_curr_pps = cpu_list_p->rx_pps;
		cpu_last_pps = cpu_list_p->last_rx_pps;
		last_avg_pps = cpu_list_p->avg_pps;
		pps_uthresh = rmnet_shs_cpu_rx_max_pps_thresh[cpu_num];
		pps_lthresh = rmnet_shs_cpu_rx_min_pps_thresh[cpu_num];

		/* Often when we decide to switch from a small cluster core,
		 * it is because of the heavy traffic on that core. In such
		 * circumstances, we want to switch to a big cluster
		 * core as soon as possible. Therefore, we will provide a
		 * greater weightage to the most recent sample compared to
		 * the previous samples.
		 *
		 * On the other hand, when a flow which is on a big cluster
		 * cpu suddenly starts to receive low traffic we move to a
		 * small cluster core after observing low traffic for some
		 * more samples. This approach avoids switching back and forth
		 * to small cluster cpus due to momentary decrease in data
		 * traffic.
		 */
		if (rmnet_shs_is_lpwr_cpu(cpu_num)) {
			new_weight = rmnet_shs_wq_tuning;
			old_weight = 100 - rmnet_shs_wq_tuning;

		} else	{
			old_weight = rmnet_shs_wq_tuning;
			new_weight = 100 - rmnet_shs_wq_tuning;

		}

		/*computing weighted average*/
		moving_avg_pps = (cpu_last_pps + last_avg_pps) / 2;
		avg_pps = ((new_weight * cpu_curr_pps) +
			   (old_weight * moving_avg_pps)) /
			   (new_weight + old_weight);

		cpu_list_p->avg_pps = avg_pps;

		trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_STATS,
				   RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_EVAL_CPU,
				   cpu_num, cpu_curr_pps, cpu_last_pps,
				   avg_pps, NULL, NULL);

		if ((avg_pps > pps_uthresh) ||
		    ((avg_pps < pps_lthresh) && (cpu_curr_pps < pps_lthresh)))
			rmnet_shs_wq_find_cpu_and_move_flows(cpu_num);
	}

}

void rmnet_shs_wq_refresh_new_flow_list_per_ep(struct rmnet_shs_wq_ep_s *ep)
{
	int lo_core;
	int hi_core;
	u16 rps_msk;
	u16 lo_msk;
	u16 hi_msk;
	u8 lo_core_idx = 0;
	u8 hi_core_idx = 0;

	if (!ep) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++;
		return;
	}

	rps_msk = ep->rps_config_msk;
	lo_msk = ep->default_core_msk;
	hi_msk = ep->pri_core_msk;
	memset(ep->new_lo_core, -1, sizeof(*ep->new_lo_core) * MAX_CPUS);
	memset(ep->new_hi_core, -1, sizeof(*ep->new_hi_core) * MAX_CPUS);
	do {
		lo_core = rmnet_shs_wq_get_least_utilized_core(lo_msk);
		if (lo_core >= 0) {
			ep->new_lo_core[lo_core_idx] = lo_core;
			lo_msk = lo_msk & ~(1 << lo_core);
			lo_core_idx++;
		} else {
			break;
		}

	} while (lo_msk != 0);

		trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS,
			    RMNET_SHS_WQ_CPU_STATS_NEW_FLOW_LIST_LO,
			    ep->new_lo_core[0], ep->new_lo_core[1],
			    ep->new_lo_core[2], ep->new_lo_max,
			    ep, NULL);

	do {
		hi_core = rmnet_shs_wq_get_least_utilized_core(hi_msk);
		if (hi_core >= 0) {
			ep->new_hi_core[hi_core_idx] = hi_core;
			hi_msk = hi_msk & ~(1 << hi_core);
			hi_core_idx++;
		} else
			break;

	} while (hi_msk != 0);

	ep->new_lo_max = lo_core_idx;
	ep->new_hi_max = hi_core_idx;
	ep->new_lo_idx = 0;
	ep->new_hi_idx = 0;

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS,
			    RMNET_SHS_WQ_CPU_STATS_NEW_FLOW_LIST_HI,
			    ep->new_hi_core[0], ep->new_hi_core[1],
			    ep->new_hi_core[2], ep->new_hi_max,
			    ep, NULL);

	return;

}
void rmnet_shs_wq_refresh_new_flow_list(void)
{
	struct rmnet_shs_wq_ep_s *ep = NULL;

	list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) {
		if (!ep)
			continue;
		if (!ep->is_ep_active)
			continue;

		rmnet_shs_wq_refresh_new_flow_list_per_ep(ep);
	}
}

/* Return Invalid core if only pri core available*/
int rmnet_shs_wq_get_lpwr_cpu_new_flow(struct net_device *dev)
{
	u8 lo_idx;
	u8 lo_max;
	int cpu_assigned = -1;
	u8 is_match_found = 0;
	struct rmnet_shs_wq_ep_s *ep = NULL;
	unsigned long flags;

	if (!dev) {
		rmnet_shs_crit_err[RMNET_SHS_NETDEV_ERR]++;
		return cpu_assigned;
	}

	spin_lock_irqsave(&rmnet_shs_ep_lock, flags);
	list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) {
		if (!ep)
			continue;
		if (!ep->is_ep_active)
			continue;

		if (ep->ep == dev) {
			is_match_found = 1;
			break;
		}

	}

	if (!is_match_found) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++;
		spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags);
		return cpu_assigned;
	}

	lo_idx = ep->new_lo_idx;
	lo_max = ep->new_lo_max;

	while (lo_idx < lo_max) {
		if (ep->new_lo_core[lo_idx] >= 0) {
			cpu_assigned = ep->new_lo_core[lo_idx];
			break;
		}
		lo_idx++;
	}

	/* Increment CPU assignment idx to be ready for next flow assignment*/
	if ((cpu_assigned >= 0) || ((ep->new_lo_idx + 1) >= ep->new_lo_max))
		ep->new_lo_idx = ((ep->new_lo_idx + 1) % ep->new_lo_max);
	spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags);

	return cpu_assigned;
}

int rmnet_shs_wq_get_perf_cpu_new_flow(struct net_device *dev)
{
	struct rmnet_shs_wq_ep_s *ep = NULL;
	int cpu_assigned = -1;
	u8 hi_idx;
	u8 hi_max;
	u8 is_match_found = 0;
	unsigned long flags;

	if (!dev) {
		rmnet_shs_crit_err[RMNET_SHS_NETDEV_ERR]++;
		return cpu_assigned;
	}

	spin_lock_irqsave(&rmnet_shs_ep_lock, flags);
	list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) {
		if (!ep)
			continue;

		if (!ep->is_ep_active)
			continue;

		if (ep->ep == dev) {
			is_match_found = 1;
			break;
		}
	}

	if (!is_match_found) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++;
		spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags);
		return cpu_assigned;
	}

	hi_idx = ep->new_hi_idx;
	hi_max = ep->new_hi_max;

	while (hi_idx < hi_max) {
		if (ep->new_hi_core[hi_idx] >= 0) {
			cpu_assigned = ep->new_hi_core[hi_idx];
			break;
		}
		hi_idx++;
	}
	/* Increment CPU assignment idx to be ready for next flow assignment*/
	if (cpu_assigned >= 0)
		ep->new_hi_idx = ((hi_idx + 1) % hi_max);
	spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags);

	return cpu_assigned;
}


static int rmnet_shs_wq_time_check(time_t time, int num_flows)
{

	int ret = false;

	if (time > rmnet_shs_max_flow_inactivity_sec)
		ret = true;
	else if (num_flows > FLOW_LIMIT2 && time > INACTIVE_TSEC2)
		ret = true;
	else if (num_flows > FLOW_LIMIT1 && time > INACTIVE_TSEC1)
		ret = true;

	return ret;
}

void rmnet_shs_wq_cleanup_hash_tbl(u8 force_clean)
{
	struct rmnet_shs_skbn_s *node_p = NULL;
	time_t tns2s;
	unsigned long ht_flags;
	struct rmnet_shs_wq_hstat_s *hnode = NULL;
	struct list_head *ptr = NULL, *next = NULL;

	list_for_each_safe(ptr, next, &rmnet_shs_wq_hstat_tbl) {
		hnode = list_entry(ptr,
				   struct rmnet_shs_wq_hstat_s, hstat_node_id);
		if (hnode == NULL)
			continue;

		if (hnode->node == NULL)
			continue;

		node_p = hnode->node;
		tns2s = RMNET_SHS_NSEC_TO_SEC(hnode->inactive_duration);

		/* Flows are cleanup from book keeping faster if
		 * there are a lot of active flows already in memory
		 */

		if (rmnet_shs_wq_time_check(tns2s, rmnet_shs_cfg.num_flows) ||
		    force_clean) {

			trace_rmnet_shs_wq_low(RMNET_SHS_WQ_FLOW_STATS,
			    RMNET_SHS_WQ_FLOW_STATS_FLOW_INACTIVE_TIMEOUT,
			    node_p->hash, tns2s, 0xDEF, 0xDEF, node_p, hnode);

			spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags);
			rmnet_shs_clear_node(node_p, RMNET_WQ_CTXT);
			rmnet_shs_wq_dec_cpu_flow(hnode->current_cpu);
			if (node_p) {
				rmnet_shs_cpu_node_remove(node_p);
				hash_del_rcu(&node_p->list);
				kfree(node_p);
			}
			rm_err("SHS_FLOW: removing flow 0x%x on cpu[%d] "
			       "pps: %llu avg_pps: %llu",
			       hnode->hash, hnode->current_cpu,
			       hnode->rx_pps, hnode->avg_pps);
			rmnet_shs_wq_cpu_list_remove(hnode);
			if (hnode->is_perm == 0 || force_clean) {
				rmnet_shs_wq_hstat_tbl_remove(hnode);
				kfree(hnode);
			} else {
				rmnet_shs_wq_hstat_reset_node(hnode);
			}
			rmnet_shs_cfg.num_flows--;
			spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags);
		}
	}

}

void rmnet_shs_wq_update_ep_rps_msk(struct rmnet_shs_wq_ep_s *ep)
{
	struct rps_map *map;
	u8 len = 0;

	if (!ep || !ep->ep ) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++;
		return;
	}

	rcu_read_lock();
	if (!ep->ep) {
		pr_info(" rmnet_shs invalid state %p", ep->ep);
		rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++;
		return;
	}
	map = rcu_dereference(ep->ep->_rx->rps_map);

	ep->rps_config_msk = 0;
	if (map != NULL) {
		for (len = 0; len < map->len; len++)
			ep->rps_config_msk |= (1 << map->cpus[len]);
	}
	rcu_read_unlock();

	ep->default_core_msk = ep->rps_config_msk & 0x0F;
	ep->pri_core_msk = ep->rps_config_msk & 0xF0;
}

void rmnet_shs_wq_reset_ep_active(struct net_device *dev)
{
	struct rmnet_shs_wq_ep_s *ep = NULL;
	struct rmnet_shs_wq_ep_s *tmp = NULL;
	unsigned long flags;

	if (!dev) {
		rmnet_shs_crit_err[RMNET_SHS_NETDEV_ERR]++;
		return;
	}

	spin_lock_irqsave(&rmnet_shs_ep_lock, flags);
	list_for_each_entry_safe(ep, tmp, &rmnet_shs_wq_ep_tbl, ep_list_id) {
		if (!ep)
			continue;

		if (ep->ep == dev){
			ep->is_ep_active = 0;
			rmnet_shs_wq_ep_tbl_remove(ep);
			kfree(ep);
			break;
		}
	}

	spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags);
}

void rmnet_shs_wq_set_ep_active(struct net_device *dev)
{
	struct rmnet_shs_wq_ep_s *ep = NULL;
	unsigned long flags;

	if (!dev) {
		rmnet_shs_crit_err[RMNET_SHS_NETDEV_ERR]++;
		return;
	}

	spin_lock_irqsave(&rmnet_shs_ep_lock, flags);

	ep = kzalloc(sizeof(*ep), GFP_ATOMIC);

	if (!ep) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_EP_TBL_ERR]++;
		spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags);
		return;
	}
	ep->ep = dev;
	ep->is_ep_active = 1;

	INIT_LIST_HEAD(&ep->ep_list_id);
	rmnet_shs_wq_update_ep_rps_msk(ep);
	rmnet_shs_wq_ep_tbl_add(ep);

	spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags);
}

void rmnet_shs_wq_refresh_ep_masks(void)
{
	struct rmnet_shs_wq_ep_s *ep = NULL;

	list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) {

		if (!ep)
			continue;

		if (!ep->is_ep_active)
			continue;
		rmnet_shs_wq_update_ep_rps_msk(ep);

		/* These tput totals get re-added as we go through each flow */
		ep->udp_rx_bps = 0;
		ep->tcp_rx_bps = 0;

	}
}

void rmnet_shs_update_cfg_mask(void)
{
	/* Start with most avaible mask all eps could share*/
	u8 mask = UPDATE_MASK;
	u8 rps_enabled = 0;
	struct rmnet_shs_wq_ep_s *ep;

	list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) {

		if (!ep->is_ep_active)
			continue;
		/* Bitwise and to get common mask from non-null masks.
		 * VNDs with different mask  will have UNDEFINED behavior
		 */
		if (ep->rps_config_msk) {
			mask &= ep->rps_config_msk;
			rps_enabled = 1;
		}
	}

	if (!rps_enabled) {
		rmnet_shs_cfg.map_mask = 0;
		rmnet_shs_cfg.map_len = 0;
		return;
        } else if (rmnet_shs_cfg.map_mask != mask) {
		rmnet_shs_cfg.map_mask = mask;
		rmnet_shs_cfg.map_len = rmnet_shs_get_mask_len(mask);
	}
}

void rmnet_shs_wq_filter(void)
{
	int cpu, cur_cpu;
	int temp;
	struct rmnet_shs_wq_hstat_s *hnode = NULL;

	for (cpu = 0; cpu < MAX_CPUS; cpu++) {
		rmnet_shs_cpu_rx_filter_flows[cpu] = 0;
		rmnet_shs_cpu_node_tbl[cpu].seg = 0;
	}

	/* Filter out flows with low pkt count and
	 * mark CPUS with slowstart flows
	 */
	list_for_each_entry(hnode, &rmnet_shs_wq_hstat_tbl, hstat_node_id) {

		if (hnode->in_use == 0)
			continue;
		if (hnode->avg_pps > RMNET_SHS_FILTER_FLOW_RATE &&
		    hnode->rx_skb > RMNET_SHS_FILTER_PKT_LIMIT)
			if (hnode->current_cpu < MAX_CPUS){
				temp = hnode->current_cpu;
				rmnet_shs_cpu_rx_filter_flows[temp]++;
			}
		cur_cpu = hnode->current_cpu;
		if (cur_cpu >= MAX_CPUS) {
			continue;
		}

		if (hnode->node->hstats->segment_enable) {
			rmnet_shs_cpu_node_tbl[cur_cpu].seg++;
		}
	}
}

void rmnet_shs_wq_update_stats(void)
{
	struct timespec time;
	struct rmnet_shs_wq_hstat_s *hnode = NULL;

	(void) getnstimeofday(&time);
	rmnet_shs_wq_tnsec = RMNET_SHS_SEC_TO_NSEC(time.tv_sec) + time.tv_nsec;
	rmnet_shs_wq_refresh_ep_masks();
	rmnet_shs_update_cfg_mask();

	list_for_each_entry(hnode, &rmnet_shs_wq_hstat_tbl, hstat_node_id) {
		if (!hnode)
			continue;

		if (hnode->in_use == 0)
			continue;

		if (hnode->node) {
			rmnet_shs_wq_update_hash_stats(hnode);
			rmnet_shs_wq_update_cpu_rx_tbl(hnode);

			if (rmnet_shs_userspace_connected) {
				if (!rmnet_shs_is_lpwr_cpu(hnode->current_cpu)) {
					/* Add golds flows to list */
					rmnet_shs_wq_gflow_list_add(hnode, &gflows);
				}
				if (hnode->skb_tport_proto == IPPROTO_TCP) {
					rmnet_shs_wq_ssflow_list_add(hnode, &ssflows);
				}
			} else {
				/* Disable segmentation if userspace gets disconnected connected */
				hnode->node->hstats->segment_enable = 0;
			}
		}
	}
	rmnet_shs_wq_refresh_all_cpu_stats();
	rmnet_shs_wq_refresh_total_stats();
	rmnet_shs_wq_refresh_dl_mrkr_stats();

	if (rmnet_shs_userspace_connected) {
		rm_err("%s", "SHS_UPDATE: Userspace connected, relying on userspace evaluation");
		rmnet_shs_wq_eval_cpus_caps_and_flows(&cpu_caps, &gflows, &ssflows);
		rmnet_shs_wq_cleanup_gold_flow_list(&gflows);
		rmnet_shs_wq_cleanup_ss_flow_list(&ssflows);
		rmnet_shs_wq_cleanup_cpu_caps_list(&cpu_caps);
	} else {
		rm_err("%s", "SHS_UPDATE: shs userspace not connected, using default logic");
		rmnet_shs_wq_eval_suggested_cpu();
	}

	rmnet_shs_wq_refresh_new_flow_list();
	rmnet_shs_wq_filter();
}

void rmnet_shs_wq_process_wq(struct work_struct *work)
{
	unsigned long flags;
	unsigned long jiffies;

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_PROCESS_WQ,
				RMNET_SHS_WQ_PROCESS_WQ_START,
				0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);

	spin_lock_irqsave(&rmnet_shs_ep_lock, flags);
	rmnet_shs_wq_update_stats();
	spin_unlock_irqrestore(&rmnet_shs_ep_lock, flags);

        /*Invoke after both the locks are released*/
        rmnet_shs_wq_cleanup_hash_tbl(PERIODIC_CLEAN);
        rmnet_shs_wq_debug_print_flows();

<<<<<<< HEAD
=======
	jiffies = msecs_to_jiffies(rmnet_shs_wq_interval_ms);

>>>>>>> LA.UM.9.1.R1.10.00.00.604.038
	queue_delayed_work(rmnet_shs_wq, &rmnet_shs_delayed_wq->wq,
			   jiffies);

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_PROCESS_WQ,
				RMNET_SHS_WQ_PROCESS_WQ_END,
				0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);
}

void rmnet_shs_wq_clean_ep_tbl(void)
{
	struct rmnet_shs_wq_ep_s *ep = NULL;
	struct list_head *ptr = NULL, *next = NULL;

	list_for_each_safe(ptr, next, &rmnet_shs_wq_ep_tbl) {
		ep = list_entry(ptr, struct rmnet_shs_wq_ep_s, ep_list_id);
		if (!ep)
			continue;

		trace_rmnet_shs_wq_high(RMNET_SHS_WQ_EP_TBL,
					RMNET_SHS_WQ_EP_TBL_CLEANUP,
					0xDEF, 0xDEF, 0xDEF, 0xDEF, ep, NULL);

		rmnet_shs_wq_ep_tbl_remove(ep);
		kfree(ep);
	}
}

void rmnet_shs_wq_exit(void)
{

	/*If Wq is not initialized, nothing to cleanup */
	if (!rmnet_shs_wq || !rmnet_shs_delayed_wq)
		return;

	rmnet_shs_wq_mem_deinit();
	rmnet_shs_genl_send_int_to_userspace_no_info(RMNET_SHS_SYNC_WQ_EXIT);

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_EXIT, RMNET_SHS_WQ_EXIT_START,
				   0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);

	cancel_delayed_work_sync(&rmnet_shs_delayed_wq->wq);
	drain_workqueue(rmnet_shs_wq);
	destroy_workqueue(rmnet_shs_wq);
	kfree(rmnet_shs_delayed_wq);

	rmnet_shs_delayed_wq = NULL;
	rmnet_shs_wq = NULL;
	rmnet_shs_wq_cleanup_hash_tbl(FORCE_CLEAN);
	rmnet_shs_wq_clean_ep_tbl();
	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_EXIT, RMNET_SHS_WQ_EXIT_END,
				   0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);
}

void rmnet_shs_wq_init_cpu_rx_flow_tbl(void)
{
	u8 cpu_num;
	struct rmnet_shs_wq_cpu_rx_pkt_q_s *rx_flow_tbl_p;

	for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) {

		trace_rmnet_shs_wq_high(RMNET_SHS_WQ_CPU_HSTAT_TBL,
					RMNET_SHS_WQ_CPU_HSTAT_TBL_INIT,
					cpu_num, 0xDEF, 0xDEF, 0xDEF,
					NULL, NULL);

		rx_flow_tbl_p = &rmnet_shs_rx_flow_tbl.cpu_list[cpu_num];
		INIT_LIST_HEAD(&rx_flow_tbl_p->hstat_id);
		rx_flow_tbl_p->cpu_num = cpu_num;
	}

}

void rmnet_shs_wq_pause(void)
{
	int cpu;

	if (rmnet_shs_wq && rmnet_shs_delayed_wq)
		cancel_delayed_work_sync(&rmnet_shs_delayed_wq->wq);

	for (cpu = 0; cpu < MAX_CPUS; cpu++)
		rmnet_shs_cpu_rx_filter_flows[cpu] = 0;
}

void rmnet_shs_wq_restart(void)
{
	if (rmnet_shs_wq && rmnet_shs_delayed_wq)
		queue_delayed_work(rmnet_shs_wq, &rmnet_shs_delayed_wq->wq, 0);
}

void rmnet_shs_wq_init(struct net_device *dev)
{
	/*If the workqueue is already initialized we should not be
	 *initializing again
	 */
	if (rmnet_shs_wq)
		return;

	if (!dev) {
		rmnet_shs_crit_err[RMNET_SHS_NETDEV_ERR]++;
		return;
	}

	rmnet_shs_wq_mem_init();

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_INIT, RMNET_SHS_WQ_INIT_START,
				0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);
	rmnet_shs_wq = alloc_workqueue("rmnet_shs_wq",
					WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1);
	if (!rmnet_shs_wq) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_WQ_ERR]++;
		return;
	}

	rmnet_shs_delayed_wq = kmalloc(sizeof(struct rmnet_shs_delay_wq_s),
				       GFP_ATOMIC);

	if (!rmnet_shs_delayed_wq) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_DEL_WQ_ERR]++;
		rmnet_shs_wq_exit();
		return;
	}

	/*All hstat nodes allocated during Wq init will be held for ever*/
	rmnet_shs_wq_hstat_alloc_nodes(RMNET_SHS_MIN_HSTAT_NODES_REQD, 1);
	rmnet_shs_wq_init_cpu_rx_flow_tbl();
	INIT_DEFERRABLE_WORK(&rmnet_shs_delayed_wq->wq,
			     rmnet_shs_wq_process_wq);

	trace_rmnet_shs_wq_high(RMNET_SHS_WQ_INIT, RMNET_SHS_WQ_INIT_END,
				0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);
}

int rmnet_shs_wq_get_num_cpu_flows(u16 cpu)
{
	int flows = -1;

	if (cpu >= MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_INVALID_CPU_ERR]++;
		return flows;
	}
	flows = rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows;

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS,
			       RMNET_SHS_WQ_CPU_STATS_GET_CPU_FLOW,
				cpu, flows, 0xDEF, 0xDEF, NULL, NULL);

	return flows;
}

int rmnet_shs_wq_get_max_flows_per_core(void)
{
	u16 cpu;
	int max_flows = -1;
	int cpu_flows;

	for (cpu = 0; cpu < MAX_CPUS; cpu++) {
		cpu_flows = rmnet_shs_wq_get_num_cpu_flows(cpu);
		if (cpu_flows > max_flows)
			max_flows = cpu_flows;

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS,
			       RMNET_SHS_WQ_CPU_STATS_GET_MAX_CPU_FLOW,
				cpu, cpu_flows, max_flows, 0xDEF, NULL, NULL);
	}

	return max_flows;
}

int rmnet_shs_wq_get_max_flows_per_cluster(u16 cpu)
{
	u32 big_cluster_mask = 1<<4;
	u32 core_mask = 1;
	u16 start_core = 0;
	u16 end_core = 4;
	int max_flows = -1;
	int cpu_flows;

	if (cpu > MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_INVALID_CPU_ERR]++;
		return max_flows;
	}

	core_mask <<= cpu;
	if (core_mask >= big_cluster_mask) {
		start_core = 4;
		end_core = MAX_CPUS;
	}

	for (start_core; start_core < end_core; start_core++) {
		cpu_flows = rmnet_shs_wq_get_num_cpu_flows(start_core);
		if (cpu_flows > max_flows)
			max_flows = cpu_flows;
	}

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS,
			       RMNET_SHS_WQ_CPU_STATS_MAX_FLOW_IN_CLUSTER,
			       start_core, end_core, cpu, max_flows,
			       NULL, NULL);
	return max_flows;
}

void rmnet_shs_wq_inc_cpu_flow(u16 cpu)
{
	if (cpu >= MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++;
		return;
	}

	rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows++;

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS,
			       RMNET_SHS_WQ_CPU_STATS_INC_CPU_FLOW,
			       cpu, rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows,
			       0xDEF, 0xDEF, NULL, NULL);
}

void rmnet_shs_wq_dec_cpu_flow(u16 cpu)
{
	if (cpu >= MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++;
		return;
	}

	if (rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows > 0)
		rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows--;

	trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS,
			       RMNET_SHS_WQ_CPU_STATS_DEC_CPU_FLOW,
			       cpu, rmnet_shs_rx_flow_tbl.cpu_list[cpu].flows,
			       0xDEF, 0xDEF, NULL, NULL);
}

u64 rmnet_shs_wq_get_max_allowed_pps(u16 cpu)
{

	if (cpu >= MAX_CPUS) {
		rmnet_shs_crit_err[RMNET_SHS_WQ_INVALID_CPU_ERR]++;
		return 0;
	}

	return rmnet_shs_cpu_rx_max_pps_thresh[cpu];
}

void rmnet_shs_wq_ep_lock_bh(void)
{
	spin_lock_bh(&rmnet_shs_ep_lock);
}

void rmnet_shs_wq_ep_unlock_bh(void)
{
	spin_unlock_bh(&rmnet_shs_ep_lock);
}