diff options
| -rw-r--r-- | MAINTAINERS | 5 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/Makefile | 2 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt.c | 50 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt.h | 34 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c | 32 | ||||
| -rw-r--r-- | drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 12 | ||||
| -rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/Makefile | 2 | ||||
| -rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/en.h | 46 | ||||
| -rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/en_dim.c | 48 | ||||
| -rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c | 6 | ||||
| -rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 40 | ||||
| -rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/en_rep.c | 2 | ||||
| -rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c | 341 | ||||
| -rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c | 10 | ||||
| -rw-r--r-- | include/linux/net_dim.h | 373 |
15 files changed, 592 insertions, 411 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index f9af20ccee73..e22ca0ae995d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4942,6 +4942,11 @@ S: Maintained F: lib/dynamic_debug.c F: include/linux/dynamic_debug.h +DYNAMIC INTERRUPT MODERATION +M: Tal Gilboa <talgi@mellanox.com> +S: Maintained +F: include/linux/net_dim.h + DZ DECSTATION DZ11 SERIAL DRIVER M: "Maciej W. Rozycki" <macro@linux-mips.org> S: Maintained diff --git a/drivers/net/ethernet/broadcom/bnxt/Makefile b/drivers/net/ethernet/broadcom/bnxt/Makefile index 59c8ec9c1cad..7c560d545c03 100644 --- a/drivers/net/ethernet/broadcom/bnxt/Makefile +++ b/drivers/net/ethernet/broadcom/bnxt/Makefile @@ -1,4 +1,4 @@ obj-$(CONFIG_BNXT) += bnxt_en.o -bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o +bnxt_en-y := bnxt.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_vfr.o bnxt_devlink.o bnxt_dim.o bnxt_en-$(CONFIG_BNXT_FLOWER_OFFLOAD) += bnxt_tc.o diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 89c3c8760a78..cf6ebf1e324b 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -1645,6 +1645,8 @@ next_rx: rxr->rx_next_cons = NEXT_RX(cons); next_rx_no_prod: + cpr->rx_packets += 1; + cpr->rx_bytes += len; *raw_cons = tmp_raw_cons; return rc; @@ -1802,6 +1804,7 @@ static irqreturn_t bnxt_msix(int irq, void *dev_instance) struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; u32 cons = RING_CMP(cpr->cp_raw_cons); + cpr->event_ctr++; prefetch(&cpr->cp_desc_ring[CP_RING(cons)][CP_IDX(cons)]); napi_schedule(&bnapi->napi); return IRQ_HANDLED; @@ -2025,6 +2028,15 @@ static int bnxt_poll(struct napi_struct *napi, int budget) break; } } + if (bp->flags & BNXT_FLAG_DIM) { + struct net_dim_sample dim_sample; + + net_dim_sample(cpr->event_ctr, + cpr->rx_packets, + cpr->rx_bytes, + &dim_sample); + net_dim(&cpr->dim, dim_sample); + } mmiowb(); return work_done; } @@ -2617,6 +2629,8 @@ static void bnxt_init_cp_rings(struct bnxt *bp) struct bnxt_ring_struct *ring = &cpr->cp_ring_struct; ring->fw_ring_id = INVALID_HW_RING_ID; + cpr->rx_ring_coal.coal_ticks = bp->rx_coal.coal_ticks; + cpr->rx_ring_coal.coal_bufs = bp->rx_coal.coal_bufs; } } @@ -4593,6 +4607,36 @@ static void bnxt_hwrm_set_coal_params(struct bnxt_coal *hw_coal, req->flags = cpu_to_le16(flags); } +int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi) +{ + struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}; + struct bnxt_cp_ring_info *cpr = &bnapi->cp_ring; + struct bnxt_coal coal; + unsigned int grp_idx; + + /* Tick values in micro seconds. + * 1 coal_buf x bufs_per_record = 1 completion record. + */ + memcpy(&coal, &bp->rx_coal, sizeof(struct bnxt_coal)); + + coal.coal_ticks = cpr->rx_ring_coal.coal_ticks; + coal.coal_bufs = cpr->rx_ring_coal.coal_bufs; + + if (!bnapi->rx_ring) + return -ENODEV; + + bnxt_hwrm_cmd_hdr_init(bp, &req_rx, + HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS, -1, -1); + + bnxt_hwrm_set_coal_params(&coal, &req_rx); + + grp_idx = bnapi->index; + req_rx.ring_id = cpu_to_le16(bp->grp_info[grp_idx].cp_fw_ring_id); + + return hwrm_send_message(bp, &req_rx, sizeof(req_rx), + HWRM_CMD_TIMEOUT); +} + int bnxt_hwrm_set_coal(struct bnxt *bp) { int i, rc = 0; @@ -5715,7 +5759,13 @@ static void bnxt_enable_napi(struct bnxt *bp) int i; for (i = 0; i < bp->cp_nr_rings; i++) { + struct bnxt_cp_ring_info *cpr = &bp->bnapi[i]->cp_ring; bp->bnapi[i]->in_reset = false; + + if (bp->bnapi[i]->rx_ring) { + INIT_WORK(&cpr->dim.work, bnxt_dim_work); + cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; + } napi_enable(&bp->bnapi[i]->napi); } } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 2d268fc26f5e..89887a88b1bd 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -24,6 +24,7 @@ #include <net/dst_metadata.h> #include <net/switchdev.h> #include <net/xdp.h> +#include <linux/net_dim.h> struct tx_bd { __le32 tx_bd_len_flags_type; @@ -608,6 +609,17 @@ struct bnxt_tx_ring_info { struct bnxt_ring_struct tx_ring_struct; }; +struct bnxt_coal { + u16 coal_ticks; + u16 coal_ticks_irq; + u16 coal_bufs; + u16 coal_bufs_irq; + /* RING_IDLE enabled when coal ticks < idle_thresh */ + u16 idle_thresh; + u8 bufs_per_record; + u8 budget; +}; + struct bnxt_tpa_info { void *data; u8 *data_ptr; @@ -672,6 +684,13 @@ struct bnxt_cp_ring_info { u32 cp_raw_cons; void __iomem *cp_doorbell; + struct bnxt_coal rx_ring_coal; + u64 rx_packets; + u64 rx_bytes; + u64 event_ctr; + + struct net_dim dim; + struct tx_cmp *cp_desc_ring[MAX_CP_PAGES]; dma_addr_t cp_desc_mapping[MAX_CP_PAGES]; @@ -946,17 +965,6 @@ struct bnxt_test_info { #define BNXT_CAG_REG_LEGACY_INT_STATUS 0x4014 #define BNXT_CAG_REG_BASE 0x300000 -struct bnxt_coal { - u16 coal_ticks; - u16 coal_ticks_irq; - u16 coal_bufs; - u16 coal_bufs_irq; - /* RING_IDLE enabled when coal ticks < idle_thresh */ - u16 idle_thresh; - u8 bufs_per_record; - u8 budget; -}; - struct bnxt_tc_flow_stats { u64 packets; u64 bytes; @@ -1128,6 +1136,7 @@ struct bnxt { #define BNXT_FLAG_DOUBLE_DB 0x400000 #define BNXT_FLAG_FW_DCBX_AGENT 0x800000 #define BNXT_FLAG_CHIP_NITRO_A0 0x1000000 + #define BNXT_FLAG_DIM 0x2000000 #define BNXT_FLAG_ALL_CONFIG_FEATS (BNXT_FLAG_TPA | \ BNXT_FLAG_RFS | \ @@ -1425,4 +1434,7 @@ int bnxt_setup_mq_tc(struct net_device *dev, u8 tc); int bnxt_get_max_rings(struct bnxt *, int *, int *, bool); void bnxt_restore_pf_fw_resources(struct bnxt *bp); int bnxt_port_attr_get(struct bnxt *bp, struct switchdev_attr *attr); +void bnxt_dim_work(struct work_struct *work); +int bnxt_hwrm_set_ring_coal(struct bnxt *bp, struct bnxt_napi *bnapi); + #endif diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c new file mode 100644 index 000000000000..408dd190331e --- /dev/null +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c @@ -0,0 +1,32 @@ +/* Broadcom NetXtreme-C/E network driver. + * + * Copyright (c) 2017-2018 Broadcom Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + */ + +#include <linux/net_dim.h> +#include "bnxt_hsi.h" +#include "bnxt.h" + +void bnxt_dim_work(struct work_struct *work) +{ + struct net_dim *dim = container_of(work, struct net_dim, + work); + struct bnxt_cp_ring_info *cpr = container_of(dim, + struct bnxt_cp_ring_info, + dim); + struct bnxt_napi *bnapi = container_of(cpr, + struct bnxt_napi, + cp_ring); + struct net_dim_cq_moder cur_profile = net_dim_get_profile(dim->mode, + dim->profile_ix); + + cpr->rx_ring_coal.coal_ticks = cur_profile.usec; + cpr->rx_ring_coal.coal_bufs = cur_profile.pkts; + + bnxt_hwrm_set_ring_coal(bnapi->bp, bnapi); + dim->state = NET_DIM_START_MEASURE; +} diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index fe7599f404bf..1801582076be 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -49,6 +49,8 @@ static int bnxt_get_coalesce(struct net_device *dev, memset(coal, 0, sizeof(*coal)); + coal->use_adaptive_rx_coalesce = bp->flags & BNXT_FLAG_DIM; + hw_coal = &bp->rx_coal; mult = hw_coal->bufs_per_record; coal->rx_coalesce_usecs = hw_coal->coal_ticks; @@ -77,6 +79,15 @@ static int bnxt_set_coalesce(struct net_device *dev, int rc = 0; u16 mult; + if (coal->use_adaptive_rx_coalesce) { + bp->flags |= BNXT_FLAG_DIM; + } else { + if (bp->flags & BNXT_FLAG_DIM) { + bp->flags &= ~(BNXT_FLAG_DIM); + goto reset_coalesce; + } + } + hw_coal = &bp->rx_coal; mult = hw_coal->bufs_per_record; hw_coal->coal_ticks = coal->rx_coalesce_usecs; @@ -104,6 +115,7 @@ static int bnxt_set_coalesce(struct net_device *dev, update_stats = true; } +reset_coalesce: if (netif_running(dev)) { if (update_stats) { rc = bnxt_close_nic(bp, true, false); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 19b21b40ab07..c805769d92a9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -14,7 +14,7 @@ mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o \ fpga/ipsec.o mlx5_core-$(CONFIG_MLX5_CORE_EN) += en_main.o en_common.o en_fs.o en_ethtool.o \ - en_tx.o en_rx.o en_rx_am.o en_txrx.o en_stats.o vxlan.o \ + en_tx.o en_rx.o en_dim.o en_txrx.o en_stats.o vxlan.o \ en_arfs.o en_fs_ethtool.o en_selftest.o mlx5_core-$(CONFIG_MLX5_MPFS) += lib/mpfs.o diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 72bab8d3f4b0..d629da213511 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -47,6 +47,7 @@ #include <linux/rhashtable.h> #include <net/switchdev.h> #include <net/xdp.h> +#include <linux/net_dim.h> #include "wq.h" #include "mlx5_core.h" #include "en_stats.h" @@ -227,12 +228,6 @@ enum mlx5e_priv_flag { #define MLX5E_MAX_BW_ALLOC 100 /* Max percentage of BW allocation */ #endif -struct mlx5e_cq_moder { - u16 usec; - u16 pkts; - u8 cq_period_mode; -}; - struct mlx5e_params { u8 log_sq_size; u8 rq_wq_type; @@ -243,8 +238,8 @@ struct mlx5e_params { u16 num_channels; u8 num_tc; bool rx_cqe_compress_def; - struct mlx5e_cq_moder rx_cq_moderation; - struct mlx5e_cq_moder tx_cq_moderation; + struct net_dim_cq_moder rx_cq_moderation; + struct net_dim_cq_moder tx_cq_moderation; bool lro_en; u32 lro_wqe_sz; u16 tx_max_inline; @@ -254,7 +249,7 @@ struct mlx5e_params { u32 indirection_rqt[MLX5E_INDIR_RQT_SIZE]; bool vlan_strip_disable; bool scatter_fcs_en; - bool rx_am_enabled; + bool rx_dim_enabled; u32 lro_timeout; u32 pflags; struct bpf_prog *xdp_prog; @@ -473,32 +468,6 @@ struct mlx5e_mpw_info { u16 skbs_frags[MLX5_MPWRQ_PAGES_PER_WQE]; }; -struct mlx5e_rx_am_stats { - int ppms; /* packets per msec */ - int bpms; /* bytes per msec */ - int epms; /* events per msec */ -}; - -struct mlx5e_rx_am_sample { - ktime_t time; - u32 pkt_ctr; - u32 byte_ctr; - u16 event_ctr; -}; - -struct mlx5e_rx_am { /* Adaptive Moderation */ - u8 state; - struct mlx5e_rx_am_stats prev_stats; - struct mlx5e_rx_am_sample start_sample; - struct work_struct work; - u8 profile_ix; - u8 mode; - u8 tune_state; - u8 steps_right; - u8 steps_left; - u8 tired; -}; - /* a single cache unit is capable to serve one napi call (for non-striding rq) * or a MPWQE (for striding rq). */ @@ -559,7 +528,7 @@ struct mlx5e_rq { unsigned long state; int ix; - struct mlx5e_rx_am am; /* Adaptive Moderation */ + struct net_dim dim; /* Dynamic Interrupt Moderation */ /* XDP */ struct bpf_prog *xdp_prog; @@ -865,10 +834,6 @@ void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix); void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix); void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi); -void mlx5e_rx_am(struct mlx5e_rq *rq); -void mlx5e_rx_am_work(struct work_struct *work); -struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode); - void mlx5e_update_stats(struct mlx5e_priv *priv, bool full); int mlx5e_create_flow_steering(struct mlx5e_priv *priv); @@ -1115,4 +1080,5 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params, u16 max_channels); u8 mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev); +void mlx5e_rx_dim_work(struct work_struct *work); #endif /* __MLX5_EN_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c new file mode 100644 index 000000000000..602851ab5b14 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dim.c @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2016, Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/net_dim.h> +#include "en.h" + +void mlx5e_rx_dim_work(struct work_struct *work) +{ + struct net_dim *dim = container_of(work, struct net_dim, + work); + struct mlx5e_rq *rq = container_of(dim, struct mlx5e_rq, dim); + struct net_dim_cq_moder cur_profile = net_dim_get_profile(dim->mode, + dim->profile_ix); + + mlx5_core_modify_cq_moderation(rq->mdev, &rq->cq.mcq, + cur_profile.usec, cur_profile.pkts); + + dim->state = NET_DIM_START_MEASURE; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 1554780d1810..bd5af7f37198 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -465,7 +465,7 @@ int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv, coal->rx_max_coalesced_frames = priv->channels.params.rx_cq_moderation.pkts; coal->tx_coalesce_usecs = priv->channels.params.tx_cq_moderation.usec; coal->tx_max_coalesced_frames = priv->channels.params.tx_cq_moderation.pkts; - coal->use_adaptive_rx_coalesce = priv->channels.params.rx_am_enabled; + coal->use_adaptive_rx_coalesce = priv->channels.params.rx_dim_enabled; return 0; } @@ -519,7 +519,7 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv, new_channels.params.tx_cq_moderation.pkts = coal->tx_max_coalesced_frames; new_channels.params.rx_cq_moderation.usec = coal->rx_coalesce_usecs; new_channels.params.rx_cq_moderation.pkts = coal->rx_max_coalesced_frames; - new_channels.params.rx_am_enabled = !!coal->use_adaptive_rx_coalesce; + new_channels.params.rx_dim_enabled = !!coal->use_adaptive_rx_coalesce; if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { priv->channels.params = new_channels.params; @@ -527,7 +527,7 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv, } /* we are opened */ - reset = !!coal->use_adaptive_rx_coalesce != priv->channels.params.rx_am_enabled; + reset = !!coal->use_adaptive_rx_coalesce != priv->channels.params.rx_dim_enabled; if (!reset) { mlx5e_set_priv_channels_coalesce(priv, coal); priv->channels.params = new_channels.params; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 539bd1d24396..a5f82bbfa7dc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -677,8 +677,17 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, wqe->data.lkey = rq->mkey_be; } - INIT_WORK(&rq->am.work, mlx5e_rx_am_work); - rq->am.mode = params->rx_cq_moderation.cq_period_mode; + INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work); + + switch (params->rx_cq_moderation.cq_period_mode) { + case MLX5_CQ_PERIOD_MODE_START_FROM_CQE: + rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE; + break; + case MLX5_CQ_PERIOD_MODE_START_FROM_EQE: + default: + rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; + } + rq->page_cache.head = 0; rq->page_cache.tail = 0; @@ -925,7 +934,7 @@ static int mlx5e_open_rq(struct mlx5e_channel *c, if (err) goto err_destroy_rq; - if (params->rx_am_enabled) + if (params->rx_dim_enabled) c->rq.state |= BIT(MLX5E_RQ_STATE_AM); return 0; @@ -958,7 +967,7 @@ static void mlx5e_deactivate_rq(struct mlx5e_rq *rq) static void mlx5e_close_rq(struct mlx5e_rq *rq) { - cancel_work_sync(&rq->am.work); + cancel_work_sync(&rq->dim.work); mlx5e_destroy_rq(rq); mlx5e_free_rx_descs(rq); mlx5e_free_rq(rq); @@ -1571,7 +1580,7 @@ static void mlx5e_destroy_cq(struct mlx5e_cq *cq) } static int mlx5e_open_cq(struct mlx5e_channel *c, - struct mlx5e_cq_moder moder, + struct net_dim_cq_moder moder, struct mlx5e_cq_param *param, struct mlx5e_cq *cq) { @@ -1753,7 +1762,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, struct mlx5e_channel_param *cparam, struct mlx5e_channel **cp) { - struct mlx5e_cq_moder icocq_moder = {0, 0}; + struct net_dim_cq_moder icocq_moder = {0, 0}; struct net_device *netdev = priv->netdev; int cpu = mlx5e_get_cpu(priv, ix); struct mlx5e_channel *c; @@ -2005,7 +2014,7 @@ static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv, mlx5e_build_common_cq_param(priv, param); - param->cq_period_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE; + param->cq_period_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; } static void mlx5e_build_icosq_param(struct mlx5e_priv *priv, @@ -4047,9 +4056,18 @@ void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode) params->rx_cq_moderation.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE; - if (params->rx_am_enabled) - params->rx_cq_moderation = - mlx5e_am_get_def_profile(cq_period_mode); + if (params->rx_dim_enabled) { + switch (cq_period_mode) { + case MLX5_CQ_PERIOD_MODE_START_FROM_CQE: + params->rx_cq_moderation = + net_dim_get_def_profile(NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE); + break; + case MLX5_CQ_PERIOD_MODE_START_FROM_EQE: + default: + params->rx_cq_moderation = + net_dim_get_def_profile(NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE); + } + } MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER, params->rx_cq_moderation.cq_period_mode == @@ -4111,7 +4129,7 @@ void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? MLX5_CQ_PERIOD_MODE_START_FROM_CQE : MLX5_CQ_PERIOD_MODE_START_FROM_EQE; - params->rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation); + params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); mlx5e_set_rx_cq_mode_params(params, cq_period_mode); mlx5e_set_tx_cq_mode_params(params, cq_period_mode); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index c6a77f8e99a4..4d1b0ff4b6e4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -884,7 +884,7 @@ static void mlx5e_build_rep_params(struct mlx5_core_dev *mdev, params->rq_wq_type = MLX5_WQ_TYPE_LINKED_LIST; params->log_rq_size = MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE; - params->rx_am_enabled = MLX5_CAP_GEN(mdev, cq_moderation); + params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); mlx5e_set_rx_cq_mode_params(params, cq_period_mode); params->tx_max_inline = mlx5e_get_max_inline_cap(mdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c deleted file mode 100644 index e401d9d245f3..000000000000 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx_am.c +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Copyright (c) 2016, Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include "en.h" - -/* Adaptive moderation profiles */ -#define MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 -#define MLX5E_RX_AM_DEF_PROFILE_CQE 1 -#define MLX5E_RX_AM_DEF_PROFILE_EQE 1 -#define MLX5E_PARAMS_AM_NUM_PROFILES 5 - -/* All profiles sizes must be MLX5E_PARAMS_AM_NUM_PROFILES */ -#define MLX5_AM_EQE_PROFILES { \ - {1, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {8, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {64, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {128, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ - {256, MLX5E_AM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ -} - -#define MLX5_AM_CQE_PROFILES { \ - {2, 256}, \ - {8, 128}, \ - {16, 64}, \ - {32, 64}, \ - {64, 64} \ -} - -static const struct mlx5e_cq_moder -profile[MLX5_CQ_PERIOD_NUM_MODES][MLX5E_PARAMS_AM_NUM_PROFILES] = { - MLX5_AM_EQE_PROFILES, - MLX5_AM_CQE_PROFILES, -}; - -static inline struct mlx5e_cq_moder mlx5e_am_get_profile(u8 cq_period_mode, int ix) -{ - struct mlx5e_cq_moder cq_moder; - - cq_moder = profile[cq_period_mode][ix]; - cq_moder.cq_period_mode = cq_period_mode; - return cq_moder; -} - -struct mlx5e_cq_moder mlx5e_am_get_def_profile(u8 rx_cq_period_mode) -{ - int default_profile_ix; - - if (rx_cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE) - default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_CQE; - else /* MLX5_CQ_PERIOD_MODE_START_FROM_EQE */ - default_profile_ix = MLX5E_RX_AM_DEF_PROFILE_EQE; - - return mlx5e_am_get_profile(rx_cq_period_mode, default_profile_ix); -} - -/* Adaptive moderation logic */ -enum { - MLX5E_AM_START_MEASURE, - MLX5E_AM_MEASURE_IN_PROGRESS, - MLX5E_AM_APPLY_NEW_PROFILE, -}; - -enum { - MLX5E_AM_PARKING_ON_TOP, - MLX5E_AM_PARKING_TIRED, - MLX5E_AM_GOING_RIGHT, - MLX5E_AM_GOING_LEFT, -}; - -enum { - MLX5E_AM_STATS_WORSE, - MLX5E_AM_STATS_SAME, - MLX5E_AM_STATS_BETTER, -}; - -enum { - MLX5E_AM_STEPPED, - MLX5E_AM_TOO_TIRED, - MLX5E_AM_ON_EDGE, -}; - -static bool mlx5e_am_on_top(struct mlx5e_rx_am *am) -{ - switch (am->tune_state) { - case MLX5E_AM_PARKING_ON_TOP: - case MLX5E_AM_PARKING_TIRED: - return true; - case MLX5E_AM_GOING_RIGHT: - return (am->steps_left > 1) && (am->steps_right == 1); - default: /* MLX5E_AM_GOING_LEFT */ - return (am->steps_right > 1) && (am->steps_left == 1); - } -} - -static void mlx5e_am_turn(struct mlx5e_rx_am *am) -{ - switch (am->tune_state) { - case MLX5E_AM_PARKING_ON_TOP: - case MLX5E_AM_PARKING_TIRED: - break; - case MLX5E_AM_GOING_RIGHT: - am->tune_state = MLX5E_AM_GOING_LEFT; - am->steps_left = 0; - break; - case MLX5E_AM_GOING_LEFT: - am->tune_state = MLX5E_AM_GOING_RIGHT; - am->steps_right = 0; - break; - } -} - -static int mlx5e_am_step(struct mlx5e_rx_am *am) -{ - if (am->tired == (MLX5E_PARAMS_AM_NUM_PROFILES * 2)) - return MLX5E_AM_TOO_TIRED; - - switch (am->tune_state) { - case MLX5E_AM_PARKING_ON_TOP: - case MLX5E_AM_PARKING_TIRED: - break; - case MLX5E_AM_GOING_RIGHT: - if (am->profile_ix == (MLX5E_PARAMS_AM_NUM_PROFILES - 1)) - return MLX5E_AM_ON_EDGE; - am->profile_ix++; - am->steps_right++; - break; - case MLX5E_AM_GOING_LEFT: - if (am->profile_ix == 0) - return MLX5E_AM_ON_EDGE; - am->profile_ix--; - am->steps_left++; - break; - } - - am->tired++; - return MLX5E_AM_STEPPED; -} - -static void mlx5e_am_park_on_top(struct mlx5e_rx_am *am) -{ - am->steps_right = 0; - am->steps_left = 0; - am->tired = 0; - am->tune_state = MLX5E_AM_PARKING_ON_TOP; -} - -static void mlx5e_am_park_tired(struct mlx5e_rx_am *am) -{ - am->steps_right = 0; - am->steps_left = 0; - am->tune_state = MLX5E_AM_PARKING_TIRED; -} - -static void mlx5e_am_exit_parking(struct mlx5e_rx_am *am) -{ - am->tune_state = am->profile_ix ? MLX5E_AM_GOING_LEFT : - MLX5E_AM_GOING_RIGHT; - mlx5e_am_step(am); -} - -#define IS_SIGNIFICANT_DIFF(val, ref) \ - (((100 * abs((val) - (ref))) / (ref)) > 10) /* more than 10% difference */ - -static int mlx5e_am_stats_compare(struct mlx5e_rx_am_stats *curr, - struct mlx5e_rx_am_stats *prev) -{ - if (!prev->bpms) - return curr->bpms ? MLX5E_AM_STATS_BETTER : - MLX5E_AM_STATS_SAME; - - if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms)) - return (curr->bpms > prev->bpms) ? MLX5E_AM_STATS_BETTER : - MLX5E_AM_STATS_WORSE; - - if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms)) - return (curr->ppms > prev->ppms) ? MLX5E_AM_STATS_BETTER : - MLX5E_AM_STATS_WORSE; - - if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms)) - return (curr->epms < prev->epms) ? MLX5E_AM_STATS_BETTER : - MLX5E_AM_STATS_WORSE; - - return MLX5E_AM_STATS_SAME; -} - -static bool mlx5e_am_decision(struct mlx5e_rx_am_stats *curr_stats, - struct mlx5e_rx_am *am) -{ - int prev_state = am->tune_state; - int prev_ix = am->profile_ix; - int stats_res; - int step_res; - - switch (am->tune_state) { - case MLX5E_AM_PARKING_ON_TOP: - stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats); - if (stats_res != MLX5E_AM_STATS_SAME) - mlx5e_am_exit_parking(am); - break; - - case MLX5E_AM_PARKING_TIRED: - am->tired--; - if (!am->tired) - mlx5e_am_exit_parking(am); - break; - - case MLX5E_AM_GOING_RIGHT: - case MLX5E_AM_GOING_LEFT: - stats_res = mlx5e_am_stats_compare(curr_stats, &am->prev_stats); - if (stats_res != MLX5E_AM_STATS_BETTER) - mlx5e_am_turn(am); - - if (mlx5e_am_on_top(am)) { - mlx5e_am_park_on_top(am); - break; - } - - step_res = mlx5e_am_step(am); - switch (step_res) { - case MLX5E_AM_ON_EDGE: - mlx5e_am_park_on_top(am); - break; - case MLX5E_AM_TOO_TIRED: - mlx5e_am_park_tired(am); - break; - } - - break; - } - - if ((prev_state != MLX5E_AM_PARKING_ON_TOP) || - (am->tune_state != MLX5E_AM_PARKING_ON_TOP)) - am->prev_stats = *curr_stats; - - return am->profile_ix != prev_ix; -} - -static void mlx5e_am_sample(struct mlx5e_rq *rq, - struct mlx5e_rx_am_sample *s) -{ - s->time = ktime_get(); - s->pkt_ctr = rq->stats.packets; - s->byte_ctr = rq->stats.bytes; - s->event_ctr = rq->cq.event_ctr; -} - -#define MLX5E_AM_NEVENTS 64 -#define BITS_PER_TYPE(type) (sizeof(type) * BITS_PER_BYTE) -#define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1)) - -static void mlx5e_am_calc_stats(struct mlx5e_rx_am_sample *start, - struct mlx5e_rx_am_sample *end, - struct mlx5e_rx_am_stats *curr_stats) -{ - /* u32 holds up to 71 minutes, should be enough */ - u32 delta_us = ktime_us_delta(end->time, start->time); - u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr); - u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr, - start->byte_ctr); - - if (!delta_us) - return; - - curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us); - curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us); - curr_stats->epms = DIV_ROUND_UP(MLX5E_AM_NEVENTS * USEC_PER_MSEC, - delta_us); -} - -void mlx5e_rx_am_work(struct work_struct *work) -{ - struct mlx5e_rx_am *am = container_of(work, struct mlx5e_rx_am, - work); - struct mlx5e_rq *rq = container_of(am, struct mlx5e_rq, am); - struct mlx5e_cq_moder cur_profile = profile[am->mode][am->profile_ix]; - - mlx5_core_modify_cq_moderation(rq->mdev, &rq->cq.mcq, - cur_profile.usec, cur_profile.pkts); - - am->state = MLX5E_AM_START_MEASURE; -} - -void mlx5e_rx_am(struct mlx5e_rq *rq) -{ - struct mlx5e_rx_am *am = &rq->am; - struct mlx5e_rx_am_sample end_sample; - struct mlx5e_rx_am_stats curr_stats; - u16 nevents; - - switch (am->state) { - case MLX5E_AM_MEASURE_IN_PROGRESS: - nevents = BIT_GAP(BITS_PER_TYPE(u16), rq->cq.event_ctr, - am->start_sample.event_ctr); - if (nevents < MLX5E_AM_NEVENTS) - break; - mlx5e_am_sample(rq, &end_sample); - mlx5e_am_calc_stats(&am->start_sample, &end_sample, - &curr_stats); - if (mlx5e_am_decision(&curr_stats, am)) { - am->state = MLX5E_AM_APPLY_NEW_PROFILE; - schedule_work(&am->work); - break; - } - /* fall through */ - case MLX5E_AM_START_MEASURE: - mlx5e_am_sample(rq, &am->start_sample); - am->state = MLX5E_AM_MEASURE_IN_PROGRESS; - break; - case MLX5E_AM_APPLY_NEW_PROFILE: - break; - } -} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c index ab92298eafc3..f292bb346985 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c @@ -78,8 +78,14 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) for (i = 0; i < c->num_tc; i++) mlx5e_cq_arm(&c->sq[i].cq); - if (MLX5E_TEST_BIT(c->rq.state, MLX5E_RQ_STATE_AM)) - mlx5e_rx_am(&c->rq); + if (MLX5E_TEST_BIT(c->rq.state, MLX5E_RQ_STATE_AM)) { + struct net_dim_sample dim_sample; + net_dim_sample(c->rq.cq.event_ctr, + c->rq.stats.packets, + c->rq.stats.bytes, + &dim_sample); + net_dim(&c->rq.dim, dim_sample); + } mlx5e_cq_arm(&c->rq.cq); mlx5e_cq_arm(&c->icosq.cq); diff --git a/include/linux/net_dim.h b/include/linux/net_dim.h new file mode 100644 index 000000000000..1c7e45016120 --- /dev/null +++ b/include/linux/net_dim.h @@ -0,0 +1,373 @@ +/* + * Copyright (c) 2016, Mellanox Technologies. All rights reserved. + * Copyright (c) 2017-2018, Broadcom Limited. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef NET_DIM_H +#define NET_DIM_H + +#include <linux/module.h> + +struct net_dim_cq_moder { + u16 usec; + u16 pkts; + u8 cq_period_mode; +}; + +struct net_dim_sample { + ktime_t time; + u32 pkt_ctr; + u32 byte_ctr; + u16 event_ctr; +}; + +struct net_dim_stats { + int ppms; /* packets per msec */ + int bpms; /* bytes per msec */ + int epms; /* events per msec */ +}; + +struct net_dim { /* Adaptive Moderation */ + u8 state; + struct net_dim_stats prev_stats; + struct net_dim_sample start_sample; + struct work_struct work; + u8 profile_ix; + u8 mode; + u8 tune_state; + u8 steps_right; + u8 steps_left; + u8 tired; +}; + +enum { + NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE = 0x0, + NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE = 0x1, + NET_DIM_CQ_PERIOD_NUM_MODES +}; + +/* Adaptive moderation logic */ +enum { + NET_DIM_START_MEASURE, + NET_DIM_MEASURE_IN_PROGRESS, + NET_DIM_APPLY_NEW_PROFILE, +}; + +enum { + NET_DIM_PARKING_ON_TOP, + NET_DIM_PARKING_TIRED, + NET_DIM_GOING_RIGHT, + NET_DIM_GOING_LEFT, +}; + +enum { + NET_DIM_STATS_WORSE, + NET_DIM_STATS_SAME, + NET_DIM_STATS_BETTER, +}; + +enum { + NET_DIM_STEPPED, + NET_DIM_TOO_TIRED, + NET_DIM_ON_EDGE, +}; + +#define NET_DIM_PARAMS_NUM_PROFILES 5 +/* Adaptive moderation profiles */ +#define NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE 256 +#define NET_DIM_DEF_PROFILE_CQE 1 +#define NET_DIM_DEF_PROFILE_EQE 1 + +/* All profiles sizes must be NET_PARAMS_DIM_NUM_PROFILES */ +#define NET_DIM_EQE_PROFILES { \ + {1, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {8, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {64, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {128, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ + {256, NET_DIM_DEFAULT_RX_CQ_MODERATION_PKTS_FROM_EQE}, \ +} + +#define NET_DIM_CQE_PROFILES { \ + {2, 256}, \ + {8, 128}, \ + {16, 64}, \ + {32, 64}, \ + {64, 64} \ +} + +static const struct net_dim_cq_moder +profile[NET_DIM_CQ_PERIOD_NUM_MODES][NET_DIM_PARAMS_NUM_PROFILES] = { + NET_DIM_EQE_PROFILES, + NET_DIM_CQE_PROFILES, +}; + +static inline struct net_dim_cq_moder net_dim_get_profile(u8 cq_period_mode, + int ix) +{ + struct net_dim_cq_moder cq_moder; + + cq_moder = profile[cq_period_mode][ix]; + cq_moder.cq_period_mode = cq_period_mode; + return cq_moder; +} + +static inline struct net_dim_cq_moder net_dim_get_def_profile(u8 rx_cq_period_mode) +{ + int default_profile_ix; + + if (rx_cq_period_mode == NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE) + default_profile_ix = NET_DIM_DEF_PROFILE_CQE; + else /* NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE */ + default_profile_ix = NET_DIM_DEF_PROFILE_EQE; + + return net_dim_get_profile(rx_cq_period_mode, default_profile_ix); +} + +static inline bool net_dim_on_top(struct net_dim *dim) +{ + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + case NET_DIM_PARKING_TIRED: + return true; + case NET_DIM_GOING_RIGHT: + return (dim->steps_left > 1) && (dim->steps_right == 1); + default: /* NET_DIM_GOING_LEFT */ + return (dim->steps_right > 1) && (dim->steps_left == 1); + } +} + +static inline void net_dim_turn(struct net_dim *dim) +{ + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + case NET_DIM_PARKING_TIRED: + break; + case NET_DIM_GOING_RIGHT: + dim->tune_state = NET_DIM_GOING_LEFT; + dim->steps_left = 0; + break; + case NET_DIM_GOING_LEFT: + dim->tune_state = NET_DIM_GOING_RIGHT; + dim->steps_right = 0; + break; + } +} + +static inline int net_dim_step(struct net_dim *dim) +{ + if (dim->tired == (NET_DIM_PARAMS_NUM_PROFILES * 2)) + return NET_DIM_TOO_TIRED; + + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + case NET_DIM_PARKING_TIRED: + break; + case NET_DIM_GOING_RIGHT: + if (dim->profile_ix == (NET_DIM_PARAMS_NUM_PROFILES - 1)) + return NET_DIM_ON_EDGE; + dim->profile_ix++; + dim->steps_right++; + break; + case NET_DIM_GOING_LEFT: + if (dim->profile_ix == 0) + return NET_DIM_ON_EDGE; + dim->profile_ix--; + dim->steps_left++; + break; + } + + dim->tired++; + return NET_DIM_STEPPED; +} + +static inline void net_dim_park_on_top(struct net_dim *dim) +{ + dim->steps_right = 0; + dim->steps_left = 0; + dim->tired = 0; + dim->tune_state = NET_DIM_PARKING_ON_TOP; +} + +static inline void net_dim_park_tired(struct net_dim *dim) +{ + dim->steps_right = 0; + dim->steps_left = 0; + dim->tune_state = NET_DIM_PARKING_TIRED; +} + +static inline void net_dim_exit_parking(struct net_dim *dim) +{ + dim->tune_state = dim->profile_ix ? NET_DIM_GOING_LEFT : + NET_DIM_GOING_RIGHT; + net_dim_step(dim); +} + +#define IS_SIGNIFICANT_DIFF(val, ref) \ + (((100 * abs((val) - (ref))) / (ref)) > 10) /* more than 10% difference */ + +static inline int net_dim_stats_compare(struct net_dim_stats *curr, + struct net_dim_stats *prev) +{ + if (!prev->bpms) + return curr->bpms ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_SAME; + + if (IS_SIGNIFICANT_DIFF(curr->bpms, prev->bpms)) + return (curr->bpms > prev->bpms) ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_WORSE; + + if (IS_SIGNIFICANT_DIFF(curr->ppms, prev->ppms)) + return (curr->ppms > prev->ppms) ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_WORSE; + + if (IS_SIGNIFICANT_DIFF(curr->epms, prev->epms)) + return (curr->epms < prev->epms) ? NET_DIM_STATS_BETTER : + NET_DIM_STATS_WORSE; + + return NET_DIM_STATS_SAME; +} + +static inline bool net_dim_decision(struct net_dim_stats *curr_stats, + struct net_dim *dim) +{ + int prev_state = dim->tune_state; + int prev_ix = dim->profile_ix; + int stats_res; + int step_res; + + switch (dim->tune_state) { + case NET_DIM_PARKING_ON_TOP: + stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); + if (stats_res != NET_DIM_STATS_SAME) + net_dim_exit_parking(dim); + break; + + case NET_DIM_PARKING_TIRED: + dim->tired--; + if (!dim->tired) + net_dim_exit_parking(dim); + break; + + case NET_DIM_GOING_RIGHT: + case NET_DIM_GOING_LEFT: + stats_res = net_dim_stats_compare(curr_stats, &dim->prev_stats); + if (stats_res != NET_DIM_STATS_BETTER) + net_dim_turn(dim); + + if (net_dim_on_top(dim)) { + net_dim_park_on_top(dim); + break; + } + + step_res = net_dim_step(dim); + switch (step_res) { + case NET_DIM_ON_EDGE: + net_dim_park_on_top(dim); + break; + case NET_DIM_TOO_TIRED: + net_dim_park_tired(dim); + break; + } + + break; + } + + if ((prev_state != NET_DIM_PARKING_ON_TOP) || + (dim->tune_state != NET_DIM_PARKING_ON_TOP)) + dim->prev_stats = *curr_stats; + + return dim->profile_ix != prev_ix; +} + +static inline void net_dim_sample(u16 event_ctr, + u64 packets, + u64 bytes, + struct net_dim_sample *s) +{ + s->time = ktime_get(); + s->pkt_ctr = packets; + s->byte_ctr = bytes; + s->event_ctr = event_ctr; +} + +#define NET_DIM_NEVENTS 64 +#define BITS_PER_TYPE(type) (sizeof(type) * BITS_PER_BYTE) +#define BIT_GAP(bits, end, start) ((((end) - (start)) + BIT_ULL(bits)) & (BIT_ULL(bits) - 1)) + +static inline void net_dim_calc_stats(struct net_dim_sample *start, + struct net_dim_sample *end, + struct net_dim_stats *curr_stats) +{ + /* u32 holds up to 71 minutes, should be enough */ + u32 delta_us = ktime_us_delta(end->time, start->time); + u32 npkts = BIT_GAP(BITS_PER_TYPE(u32), end->pkt_ctr, start->pkt_ctr); + u32 nbytes = BIT_GAP(BITS_PER_TYPE(u32), end->byte_ctr, + start->byte_ctr); + + if (!delta_us) + return; + + curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us); + curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us); + curr_stats->epms = DIV_ROUND_UP(NET_DIM_NEVENTS * USEC_PER_MSEC, + delta_us); +} + +static inline void net_dim(struct net_dim *dim, + struct net_dim_sample end_sample) +{ + struct net_dim_stats curr_stats; + u16 nevents; + + switch (dim->state) { + case NET_DIM_MEASURE_IN_PROGRESS: + nevents = BIT_GAP(BITS_PER_TYPE(u16), + end_sample.event_ctr, + dim->start_sample.event_ctr); + if (nevents < NET_DIM_NEVENTS) + break; + net_dim_calc_stats(&dim->start_sample, &end_sample, + &curr_stats); + if (net_dim_decision(&curr_stats, dim)) { + dim->state = NET_DIM_APPLY_NEW_PROFILE; + schedule_work(&dim->work); + break; + } + /* fall through */ + case NET_DIM_START_MEASURE: + dim->state = NET_DIM_MEASURE_IN_PROGRESS; + break; + case NET_DIM_APPLY_NEW_PROFILE: + break; + } +} + +#endif /* NET_DIM_H */ |