diff --git a/drivers/net/e1000/e1000.h b/drivers/net/e1000/e1000.h index 896ea8a72386..f091042b146e 100644 --- a/drivers/net/e1000/e1000.h +++ b/drivers/net/e1000/e1000.h @@ -257,6 +257,17 @@ struct e1000_adapter { spinlock_t tx_queue_lock; #endif atomic_t irq_sem; + unsigned int detect_link; + unsigned int total_tx_bytes; + unsigned int total_tx_packets; + unsigned int total_rx_bytes; + unsigned int total_rx_packets; + /* Interrupt Throttle Rate */ + uint32_t itr; + uint32_t itr_setting; + uint16_t tx_itr; + uint16_t rx_itr; + struct work_struct reset_task; uint8_t fc_autoneg; @@ -314,8 +325,6 @@ struct e1000_adapter { uint64_t gorcl_old; uint16_t rx_ps_bsize0; - /* Interrupt Throttle Rate */ - uint32_t itr; /* OS defined structs */ struct net_device *netdev; diff --git a/drivers/net/e1000/e1000_main.c b/drivers/net/e1000/e1000_main.c index 35e4e32c7702..56be5c89363c 100644 --- a/drivers/net/e1000/e1000_main.c +++ b/drivers/net/e1000/e1000_main.c @@ -1897,7 +1897,7 @@ e1000_configure_rx(struct e1000_adapter *adapter) if (hw->mac_type >= e1000_82540) { E1000_WRITE_REG(hw, RADV, adapter->rx_abs_int_delay); - if (adapter->itr > 1) + if (adapter->itr_setting != 0) E1000_WRITE_REG(hw, ITR, 1000000000 / (adapter->itr * 256)); } @@ -1907,11 +1907,11 @@ e1000_configure_rx(struct e1000_adapter *adapter) /* Reset delay timers after every interrupt */ ctrl_ext |= E1000_CTRL_EXT_INT_TIMER_CLR; #ifdef CONFIG_E1000_NAPI - /* Auto-Mask interrupts upon ICR read. */ + /* Auto-Mask interrupts upon ICR access */ ctrl_ext |= E1000_CTRL_EXT_IAME; + E1000_WRITE_REG(hw, IAM, 0xffffffff); #endif E1000_WRITE_REG(hw, CTRL_EXT, ctrl_ext); - E1000_WRITE_REG(hw, IAM, ~0); E1000_WRITE_FLUSH(hw); } @@ -2576,19 +2576,6 @@ e1000_watchdog(unsigned long data) } } - /* Dynamic mode for Interrupt Throttle Rate (ITR) */ - if (adapter->hw.mac_type >= e1000_82540 && adapter->itr == 1) { - /* Symmetric Tx/Rx gets a reduced ITR=2000; Total - * asymmetrical Tx or Rx gets ITR=8000; everyone - * else is between 2000-8000. */ - uint32_t goc = (adapter->gotcl + adapter->gorcl) / 10000; - uint32_t dif = (adapter->gotcl > adapter->gorcl ? - adapter->gotcl - adapter->gorcl : - adapter->gorcl - adapter->gotcl) / 10000; - uint32_t itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000; - E1000_WRITE_REG(&adapter->hw, ITR, 1000000000 / (itr * 256)); - } - /* Cause software interrupt to ensure rx ring is cleaned */ E1000_WRITE_REG(&adapter->hw, ICS, E1000_ICS_RXDMT0); @@ -2604,6 +2591,135 @@ e1000_watchdog(unsigned long data) mod_timer(&adapter->watchdog_timer, jiffies + 2 * HZ); } +enum latency_range { + lowest_latency = 0, + low_latency = 1, + bulk_latency = 2, + latency_invalid = 255 +}; + +/** + * e1000_update_itr - update the dynamic ITR value based on statistics + * Stores a new ITR value based on packets and byte + * counts during the last interrupt. The advantage of per interrupt + * computation is faster updates and more accurate ITR for the current + * traffic pattern. Constants in this function were computed + * based on theoretical maximum wire speed and thresholds were set based + * on testing data as well as attempting to minimize response time + * while increasing bulk throughput. + * this functionality is controlled by the InterruptThrottleRate module + * parameter (see e1000_param.c) + * @adapter: pointer to adapter + * @itr_setting: current adapter->itr + * @packets: the number of packets during this measurement interval + * @bytes: the number of bytes during this measurement interval + **/ +static unsigned int e1000_update_itr(struct e1000_adapter *adapter, + uint16_t itr_setting, + int packets, + int bytes) +{ + unsigned int retval = itr_setting; + struct e1000_hw *hw = &adapter->hw; + + if (unlikely(hw->mac_type < e1000_82540)) + goto update_itr_done; + + if (packets == 0) + goto update_itr_done; + + + switch (itr_setting) { + case lowest_latency: + if ((packets < 5) && (bytes > 512)) + retval = low_latency; + break; + case low_latency: /* 50 usec aka 20000 ints/s */ + if (bytes > 10000) { + if ((packets < 10) || + ((bytes/packets) > 1200)) + retval = bulk_latency; + else if ((packets > 35)) + retval = lowest_latency; + } else if (packets <= 2 && bytes < 512) + retval = lowest_latency; + break; + case bulk_latency: /* 250 usec aka 4000 ints/s */ + if (bytes > 25000) { + if (packets > 35) + retval = low_latency; + } else { + if (bytes < 6000) + retval = low_latency; + } + break; + } + +update_itr_done: + return retval; +} + +static void e1000_set_itr(struct e1000_adapter *adapter) +{ + struct e1000_hw *hw = &adapter->hw; + uint16_t current_itr; + uint32_t new_itr = adapter->itr; + + if (unlikely(hw->mac_type < e1000_82540)) + return; + + /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ + if (unlikely(adapter->link_speed != SPEED_1000)) { + current_itr = 0; + new_itr = 4000; + goto set_itr_now; + } + + adapter->tx_itr = e1000_update_itr(adapter, + adapter->tx_itr, + adapter->total_tx_packets, + adapter->total_tx_bytes); + adapter->rx_itr = e1000_update_itr(adapter, + adapter->rx_itr, + adapter->total_rx_packets, + adapter->total_rx_bytes); + + current_itr = max(adapter->rx_itr, adapter->tx_itr); + + /* conservative mode eliminates the lowest_latency setting */ + if (current_itr == lowest_latency && (adapter->itr_setting == 3)) + current_itr = low_latency; + + switch (current_itr) { + /* counts and packets in update_itr are dependent on these numbers */ + case lowest_latency: + new_itr = 70000; + break; + case low_latency: + new_itr = 20000; /* aka hwitr = ~200 */ + break; + case bulk_latency: + new_itr = 4000; + break; + default: + break; + } + +set_itr_now: + if (new_itr != adapter->itr) { + /* this attempts to bias the interrupt rate towards Bulk + * by adding intermediate steps when interrupt rate is + * increasing */ + new_itr = new_itr > adapter->itr ? + min(adapter->itr + (new_itr >> 2), new_itr) : + new_itr; + adapter->itr = new_itr; + E1000_WRITE_REG(hw, ITR, 1000000000 / (new_itr * 256)); + } + + return; +} + #define E1000_TX_FLAGS_CSUM 0x00000001 #define E1000_TX_FLAGS_VLAN 0x00000002 #define E1000_TX_FLAGS_TSO 0x00000004 @@ -3538,15 +3654,27 @@ irqreturn_t e1000_intr_msi(int irq, void *data) } #ifdef CONFIG_E1000_NAPI - if (likely(netif_rx_schedule_prep(netdev))) + if (likely(netif_rx_schedule_prep(netdev))) { + adapter->total_tx_bytes = 0; + adapter->total_tx_packets = 0; + adapter->total_rx_bytes = 0; + adapter->total_rx_packets = 0; __netif_rx_schedule(netdev); - else + } else e1000_irq_enable(adapter); #else + adapter->total_tx_bytes = 0; + adapter->total_rx_bytes = 0; + adapter->total_tx_packets = 0; + adapter->total_rx_packets = 0; + for (i = 0; i < E1000_MAX_INTR; i++) if (unlikely(!adapter->clean_rx(adapter, adapter->rx_ring) & !e1000_clean_tx_irq(adapter, adapter->tx_ring))) break; + + if (likely(adapter->itr_setting & 3)) + e1000_set_itr(adapter); #endif return IRQ_HANDLED; @@ -3568,7 +3696,17 @@ e1000_intr(int irq, void *data) uint32_t rctl, icr = E1000_READ_REG(hw, ICR); #ifndef CONFIG_E1000_NAPI int i; -#else +#endif + if (unlikely(!icr)) + return IRQ_NONE; /* Not our interrupt */ + +#ifdef CONFIG_E1000_NAPI + /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is + * not set, then the adapter didn't send an interrupt */ + if (unlikely(hw->mac_type >= e1000_82571 && + !(icr & E1000_ICR_INT_ASSERTED))) + return IRQ_NONE; + /* Interrupt Auto-Mask...upon reading ICR, * interrupts are masked. No need for the * IMC write, but it does mean we should @@ -3577,14 +3715,6 @@ e1000_intr(int irq, void *data) atomic_inc(&adapter->irq_sem); #endif - if (unlikely(!icr)) { -#ifdef CONFIG_E1000_NAPI - if (hw->mac_type >= e1000_82571) - e1000_irq_enable(adapter); -#endif - return IRQ_NONE; /* Not our interrupt */ - } - if (unlikely(icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))) { hw->get_link_status = 1; /* 80003ES2LAN workaround-- @@ -3605,13 +3735,18 @@ e1000_intr(int irq, void *data) #ifdef CONFIG_E1000_NAPI if (unlikely(hw->mac_type < e1000_82571)) { + /* disable interrupts, without the synchronize_irq bit */ atomic_inc(&adapter->irq_sem); E1000_WRITE_REG(hw, IMC, ~0); E1000_WRITE_FLUSH(hw); } - if (likely(netif_rx_schedule_prep(netdev))) + if (likely(netif_rx_schedule_prep(netdev))) { + adapter->total_tx_bytes = 0; + adapter->total_tx_packets = 0; + adapter->total_rx_bytes = 0; + adapter->total_rx_packets = 0; __netif_rx_schedule(netdev); - else + } else /* this really should not happen! if it does it is basically a * bug, but not a hard error, so enable ints and continue */ e1000_irq_enable(adapter); @@ -3631,11 +3766,19 @@ e1000_intr(int irq, void *data) E1000_WRITE_REG(hw, IMC, ~0); } + adapter->total_tx_bytes = 0; + adapter->total_rx_bytes = 0; + adapter->total_tx_packets = 0; + adapter->total_rx_packets = 0; + for (i = 0; i < E1000_MAX_INTR; i++) if (unlikely(!adapter->clean_rx(adapter, adapter->rx_ring) & !e1000_clean_tx_irq(adapter, adapter->tx_ring))) break; + if (likely(adapter->itr_setting & 3)) + e1000_set_itr(adapter); + if (hw->mac_type == e1000_82547 || hw->mac_type == e1000_82547_rev_2) e1000_irq_enable(adapter); @@ -3683,6 +3826,8 @@ e1000_clean(struct net_device *poll_dev, int *budget) if ((!tx_cleaned && (work_done == 0)) || !netif_running(poll_dev)) { quit_polling: + if (likely(adapter->itr_setting & 3)) + e1000_set_itr(adapter); netif_rx_complete(poll_dev); e1000_irq_enable(adapter); return 0; @@ -3709,6 +3854,7 @@ e1000_clean_tx_irq(struct e1000_adapter *adapter, unsigned int count = 0; #endif boolean_t cleaned = FALSE; + unsigned int total_tx_bytes=0, total_tx_packets=0; i = tx_ring->next_to_clean; eop = tx_ring->buffer_info[i].next_to_watch; @@ -3720,6 +3866,13 @@ e1000_clean_tx_irq(struct e1000_adapter *adapter, buffer_info = &tx_ring->buffer_info[i]; cleaned = (i == eop); + if (cleaned) { + /* this packet count is wrong for TSO but has a + * tendency to make dynamic ITR change more + * towards bulk */ + total_tx_packets++; + total_tx_bytes += buffer_info->skb->len; + } e1000_unmap_and_free_tx_resource(adapter, buffer_info); tx_desc->upper.data = 0; @@ -3785,6 +3938,8 @@ e1000_clean_tx_irq(struct e1000_adapter *adapter, netif_stop_queue(netdev); } } + adapter->total_tx_bytes += total_tx_bytes; + adapter->total_tx_packets += total_tx_packets; return cleaned; } @@ -3864,6 +4019,7 @@ e1000_clean_rx_irq(struct e1000_adapter *adapter, unsigned int i; int cleaned_count = 0; boolean_t cleaned = FALSE; + unsigned int total_rx_bytes=0, total_rx_packets=0; i = rx_ring->next_to_clean; rx_desc = E1000_RX_DESC(*rx_ring, i); @@ -3930,6 +4086,10 @@ e1000_clean_rx_irq(struct e1000_adapter *adapter, * done after the TBI_ACCEPT workaround above */ length -= 4; + /* probably a little skewed due to removing CRC */ + total_rx_bytes += length; + total_rx_packets++; + /* code added for copybreak, this should improve * performance for small packets with large amounts * of reassembly being done in the stack */ @@ -3998,6 +4158,8 @@ e1000_clean_rx_irq(struct e1000_adapter *adapter, if (cleaned_count) adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count); + adapter->total_rx_packets += total_rx_packets; + adapter->total_rx_bytes += total_rx_bytes; return cleaned; } @@ -4027,6 +4189,7 @@ e1000_clean_rx_irq_ps(struct e1000_adapter *adapter, uint32_t length, staterr; int cleaned_count = 0; boolean_t cleaned = FALSE; + unsigned int total_rx_bytes=0, total_rx_packets=0; i = rx_ring->next_to_clean; rx_desc = E1000_RX_DESC_PS(*rx_ring, i); @@ -4131,6 +4294,9 @@ e1000_clean_rx_irq_ps(struct e1000_adapter *adapter, pskb_trim(skb, skb->len - 4); copydone: + total_rx_bytes += skb->len; + total_rx_packets++; + e1000_rx_checksum(adapter, staterr, le16_to_cpu(rx_desc->wb.lower.hi_dword.csum_ip.csum), skb); skb->protocol = eth_type_trans(skb, netdev); @@ -4179,6 +4345,8 @@ e1000_clean_rx_irq_ps(struct e1000_adapter *adapter, if (cleaned_count) adapter->alloc_rx_buf(adapter, rx_ring, cleaned_count); + adapter->total_rx_packets += total_rx_packets; + adapter->total_rx_bytes += total_rx_bytes; return cleaned; } diff --git a/drivers/net/e1000/e1000_param.c b/drivers/net/e1000/e1000_param.c index e4f88920b46b..cbfcd7f2889f 100644 --- a/drivers/net/e1000/e1000_param.c +++ b/drivers/net/e1000/e1000_param.c @@ -139,7 +139,7 @@ E1000_PARAM(XsumRX, "Disable or enable Receive Checksum offload"); * Valid Range: 0-65535 */ E1000_PARAM(TxIntDelay, "Transmit Interrupt Delay"); -#define DEFAULT_TIDV 64 +#define DEFAULT_TIDV 8 #define MAX_TXDELAY 0xFFFF #define MIN_TXDELAY 0 @@ -148,7 +148,7 @@ E1000_PARAM(TxIntDelay, "Transmit Interrupt Delay"); * Valid Range: 0-65535 */ E1000_PARAM(TxAbsIntDelay, "Transmit Absolute Interrupt Delay"); -#define DEFAULT_TADV 64 +#define DEFAULT_TADV 32 #define MAX_TXABSDELAY 0xFFFF #define MIN_TXABSDELAY 0 @@ -167,16 +167,16 @@ E1000_PARAM(RxIntDelay, "Receive Interrupt Delay"); * Valid Range: 0-65535 */ E1000_PARAM(RxAbsIntDelay, "Receive Absolute Interrupt Delay"); -#define DEFAULT_RADV 128 +#define DEFAULT_RADV 8 #define MAX_RXABSDELAY 0xFFFF #define MIN_RXABSDELAY 0 /* Interrupt Throttle Rate (interrupts/sec) * - * Valid Range: 100-100000 (0=off, 1=dynamic) + * Valid Range: 100-100000 (0=off, 1=dynamic, 3=dynamic conservative) */ E1000_PARAM(InterruptThrottleRate, "Interrupt Throttling Rate"); -#define DEFAULT_ITR 8000 +#define DEFAULT_ITR 3 #define MAX_ITR 100000 #define MIN_ITR 100 @@ -472,15 +472,27 @@ e1000_check_options(struct e1000_adapter *adapter) break; case 1: DPRINTK(PROBE, INFO, "%s set to dynamic mode\n", - opt.name); + opt.name); + adapter->itr_setting = adapter->itr; + adapter->itr = 20000; + break; + case 3: + DPRINTK(PROBE, INFO, + "%s set to dynamic conservative mode\n", + opt.name); + adapter->itr_setting = adapter->itr; + adapter->itr = 20000; break; default: e1000_validate_option(&adapter->itr, &opt, - adapter); + adapter); + /* save the setting, because the dynamic bits change itr */ + adapter->itr_setting = adapter->itr; break; } } else { - adapter->itr = opt.def; + adapter->itr_setting = opt.def; + adapter->itr = 20000; } } { /* Smart Power Down */