// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019 Intel Corporation */
#include "igc.h"
#include "igc_hw.h"
#include "igc_tsn.h"
static bool is_any_launchtime(struct igc_adapter *adapter)
{
int i;
for (i = 0; i < adapter->num_tx_queues; i++) {
struct igc_ring *ring = adapter->tx_ring[i];
if (ring->launchtime_enable)
return true;
}
return false;
}
static bool is_cbs_enabled(struct igc_adapter *adapter)
{
int i;
for (i = 0; i < adapter->num_tx_queues; i++) {
struct igc_ring *ring = adapter->tx_ring[i];
if (ring->cbs_enable)
return true;
}
return false;
}
static unsigned int igc_tsn_new_flags(struct igc_adapter *adapter)
{
unsigned int new_flags = adapter->flags & ~IGC_FLAG_TSN_ANY_ENABLED;
if (adapter->taprio_offload_enable)
new_flags |= IGC_FLAG_TSN_QBV_ENABLED;
if (is_any_launchtime(adapter))
new_flags |= IGC_FLAG_TSN_QBV_ENABLED;
if (is_cbs_enabled(adapter))
new_flags |= IGC_FLAG_TSN_QAV_ENABLED;
if (adapter->strict_priority_enable)
new_flags |= IGC_FLAG_TSN_LEGACY_ENABLED;
return new_flags;
}
static bool igc_tsn_is_tx_mode_in_tsn(struct igc_adapter *adapter)
{
struct igc_hw *hw = &adapter->hw;
return !!(rd32(IGC_TQAVCTRL) & IGC_TQAVCTRL_TRANSMIT_MODE_TSN);
}
void igc_tsn_adjust_txtime_offset(struct igc_adapter *adapter)
{
struct igc_hw *hw = &adapter->hw;
u16 txoffset;
if (!igc_tsn_is_tx_mode_in_tsn(adapter))
return;
switch (adapter->link_speed) {
case SPEED_10:
txoffset = IGC_TXOFFSET_SPEED_10;
break;
case SPEED_100:
txoffset = IGC_TXOFFSET_SPEED_100;
break;
case SPEED_1000:
txoffset = IGC_TXOFFSET_SPEED_1000;
break;
case SPEED_2500:
txoffset = IGC_TXOFFSET_SPEED_2500;
break;
default:
txoffset = 0;
break;
}
wr32(IGC_GTXOFFSET, txoffset);
}
static void igc_tsn_restore_retx_default(struct igc_adapter *adapter)
{
struct igc_hw *hw = &adapter->hw;
u32 retxctl;
retxctl = rd32(IGC_RETX_CTL) & IGC_RETX_CTL_WATERMARK_MASK;
wr32(IGC_RETX_CTL, retxctl);
}
bool igc_tsn_is_taprio_activated_by_user(struct igc_adapter *adapter)
{
struct igc_hw *hw = &adapter->hw;
return (rd32(IGC_BASET_H) || rd32(IGC_BASET_L)) &&
adapter->taprio_offload_enable;
}
static void igc_tsn_tx_arb(struct igc_adapter *adapter, u16 *queue_per_tc)
{
struct igc_hw *hw = &adapter->hw;
u32 txarb;
txarb = rd32(IGC_TXARB);
txarb &= ~(IGC_TXARB_TXQ_PRIO_0_MASK |
IGC_TXARB_TXQ_PRIO_1_MASK |
IGC_TXARB_TXQ_PRIO_2_MASK |
IGC_TXARB_TXQ_PRIO_3_MASK);
txarb |= IGC_TXARB_TXQ_PRIO_0(queue_per_tc[3]);
txarb |= IGC_TXARB_TXQ_PRIO_1(queue_per_tc[2]);
txarb |= IGC_TXARB_TXQ_PRIO_2(queue_per_tc[1]);
txarb |= IGC_TXARB_TXQ_PRIO_3(queue_per_tc[0]);
wr32(IGC_TXARB, txarb);
}
/* Returns the TSN specific registers to their default values after
* the adapter is reset.
*/
static int igc_tsn_disable_offload(struct igc_adapter *adapter)
{
u16 queue_per_tc[4] = { 3, 2, 1, 0 };
struct igc_hw *hw = &adapter->hw;
u32 tqavctrl;
int i;
wr32(IGC_GTXOFFSET, 0);
wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_DEFAULT);
if (igc_is_device_id_i226(hw))
igc_tsn_restore_retx_default(adapter);
tqavctrl = rd32(IGC_TQAVCTRL);
tqavctrl &= ~(IGC_TQAVCTRL_TRANSMIT_MODE_TSN |
IGC_TQAVCTRL_ENHANCED_QAV | IGC_TQAVCTRL_FUTSCDDIS);
wr32(IGC_TQAVCTRL, tqavctrl);
for (i = 0; i < adapter->num_tx_queues; i++) {
wr32(IGC_TXQCTL(i), 0);
wr32(IGC_STQT(i), 0);
wr32(IGC_ENDQT(i), NSEC_PER_SEC);
}
wr32(IGC_QBVCYCLET_S, 0);
wr32(IGC_QBVCYCLET, NSEC_PER_SEC);
/* Reset mqprio TC configuration. */
netdev_reset_tc(adapter->netdev);
/* Restore the default Tx arbitration: Priority 0 has the highest
* priority and is assigned to queue 0 and so on and so forth.
*/
igc_tsn_tx_arb(adapter, queue_per_tc);
adapter->flags &= ~IGC_FLAG_TSN_QBV_ENABLED;
adapter->flags &= ~IGC_FLAG_TSN_LEGACY_ENABLED;
return 0;
}
/* To partially fix i226 HW errata, reduce MAC internal buffering from 192 Bytes
* to 88 Bytes by setting RETX_CTL register using the recommendation from:
* a) Ethernet Controller I225/I226 Specification Update Rev 2.1
* Item 9: TSN: Packet Transmission Might Cross the Qbv Window
* b) I225/6 SW User Manual Rev 1.2.4: Section 8.11.5 Retry Buffer Control
*/
static void igc_tsn_set_retx_qbvfullthreshold(struct igc_adapter *adapter)
{
struct igc_hw *hw = &adapter->hw;
u32 retxctl, watermark;
retxctl = rd32(IGC_RETX_CTL);
watermark = retxctl & IGC_RETX_CTL_WATERMARK_MASK;
/* Set QBVFULLTH value using watermark and set QBVFULLEN */
retxctl |= (watermark << IGC_RETX_CTL_QBVFULLTH_SHIFT) |
IGC_RETX_CTL_QBVFULLEN;
wr32(IGC_RETX_CTL, retxctl);
}
static int igc_tsn_enable_offload(struct igc_adapter *adapter)
{
struct igc_hw *hw = &adapter->hw;
u32 tqavctrl, baset_l, baset_h;
u32 sec, nsec, cycle;
ktime_t base_time, systim;
int i;
wr32(IGC_TSAUXC, 0);
wr32(IGC_DTXMXPKTSZ, IGC_DTXMXPKTSZ_TSN);
wr32(IGC_TXPBS, IGC_TXPBSIZE_TSN);
if (igc_is_device_id_i226(hw))
igc_tsn_set_retx_qbvfullthreshold(adapter);
if (adapter->strict_priority_enable) {
int err;
err = netdev_set_num_tc(adapter->netdev, adapter->num_tc);
if (err)
return err;
for (i = 0; i < adapter->num_tc; i++) {
err = netdev_set_tc_queue(adapter->netdev, i, 1,
adapter->queue_per_tc[i]);
if (err)
return err;
}
/* In case the card is configured with less than four queues. */
for (; i < IGC_MAX_TX_QUEUES; i++)
adapter->queue_per_tc[i] = i;
/* Configure queue priorities according to the user provided
* mapping.
*/
igc_tsn_tx_arb(adapter, adapter->queue_per_tc);
/* Enable legacy TSN mode which will do strict priority without
* any other TSN features.
*/
tqavctrl = rd32(IGC_TQAVCTRL);
tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN;
tqavctrl &= ~IGC_TQAVCTRL_ENHANCED_QAV;
wr32(IGC_TQAVCTRL, tqavctrl);
return 0;
}
for (i = 0; i < adapter->num_tx_queues; i++) {
struct igc_ring *ring = adapter->tx_ring[i];
u32 txqctl = 0;
u16 cbs_value;
u32 tqavcc;
wr32(IGC_STQT(i), ring->start_time);
wr32(IGC_ENDQT(i), ring->end_time);
if (adapter->taprio_offload_enable) {
/* If taprio_offload_enable is set we are in "taprio"
* mode and we need to be strict about the
* cycles: only transmit a packet if it can be
* completed during that cycle.
*
* If taprio_offload_enable is NOT true when
* enabling TSN offload, the cycle should have
* no external effects, but is only used internally
* to adapt the base time register after a second
* has passed.
*
* Enabling strict mode in this case would
* unnecessarily prevent the transmission of
* certain packets (i.e. at the boundary of a
* second) and thus interfere with the launchtime
* feature that promises transmission at a
* certain point in time.
*/
txqctl |= IGC_TXQCTL_STRICT_CYCLE |
IGC_TXQCTL_STRICT_END;
}
if (ring->launchtime_enable)
txqctl |= IGC_TXQCTL_QUEUE_MODE_LAUNCHT;
/* Skip configuring CBS for Q2 and Q3 */
if (i > 1)
goto skip_cbs;
if (ring->cbs_enable) {
if (i == 0)
txqctl |= IGC_TXQCTL_QAV_SEL_CBS0;
else
txqctl |= IGC_TXQCTL_QAV_SEL_CBS1;
/* According to i225 datasheet section 7.5.2.7, we
* should set the 'idleSlope' field from TQAVCC
* register following the equation:
*
* value = link-speed 0x7736 * BW * 0.2
* ---------- * ----------------- (E1)
* 100Mbps 2.5
*
* Note that 'link-speed' is in Mbps.
*
* 'BW' is the percentage bandwidth out of full
* link speed which can be found with the
* following equation. Note that idleSlope here
* is the parameter from this function
* which is in kbps.
*
* BW = idleSlope
* ----------------- (E2)
* link-speed * 1000
*
* That said, we can come up with a generic
* equation to calculate the value we should set
* it TQAVCC register by replacing 'BW' in E1 by E2.
* The resulting equation is:
*
* value = link-speed * 0x7736 * idleSlope * 0.2
* ------------------------------------- (E3)
* 100 * 2.5 * link-speed * 1000
*
* 'link-speed' is present in both sides of the
* fraction so it is canceled out. The final
* equation is the following:
*
* value = idleSlope * 61036
* ----------------- (E4)
* 2500000
*
* NOTE: For i225, given the above, we can see
* that idleslope is represented in
* 40.959433 kbps units by the value at
* the TQAVCC register (2.5Gbps / 61036),
* which reduces the granularity for
* idleslope increments.
*
* In i225 controller, the sendSlope and loCredit
* parameters from CBS are not configurable
* by software so we don't do any
* 'controller configuration' in respect to
* these parameters.
*/
cbs_value = DIV_ROUND_UP_ULL(ring->idleslope
* 61036ULL, 2500000);
tqavcc = rd32(IGC_TQAVCC(i));
tqavcc &= ~IGC_TQAVCC_IDLESLOPE_MASK;
tqavcc |= cbs_value | IGC_TQAVCC_KEEP_CREDITS;
wr32(IGC_TQAVCC(i), tqavcc);
wr32(IGC_TQAVHC(i),
0x80000000 + ring->hicredit * 0x7736);
} else {
/* Disable any CBS for the queue */
txqctl &= ~(IGC_TXQCTL_QAV_SEL_MASK);
/* Set idleSlope to zero. */
tqavcc = rd32(IGC_TQAVCC(i));
tqavcc &= ~(IGC_TQAVCC_IDLESLOPE_MASK |
IGC_TQAVCC_KEEP_CREDITS);
wr32(IGC_TQAVCC(i), tqavcc);
/* Set hiCredit to zero. */
wr32(IGC_TQAVHC(i), 0);
}
skip_cbs:
wr32(IGC_TXQCTL(i), txqctl);
}
tqavctrl = rd32(IGC_TQAVCTRL) & ~IGC_TQAVCTRL_FUTSCDDIS;
tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN | IGC_TQAVCTRL_ENHANCED_QAV;
adapter->qbv_count++;
cycle = adapter->cycle_time;
base_time = adapter->base_time;
nsec = rd32(IGC_SYSTIML);
sec = rd32(IGC_SYSTIMH);
systim = ktime_set(sec, nsec);
if (ktime_compare(systim, base_time) > 0) {
s64 n = div64_s64(ktime_sub_ns(systim, base_time), cycle);
base_time = ktime_add_ns(base_time, (n + 1) * cycle);
} else {
if (igc_is_device_id_i226(hw)) {
ktime_t adjust_time, expires_time;
/* According to datasheet section 7.5.2.9.3.3, FutScdDis bit
* has to be configured before the cycle time and base time.
* Tx won't hang if a GCL is already running,
* so in this case we don't need to set FutScdDis.
*/
if (!(rd32(IGC_BASET_H) || rd32(IGC_BASET_L)))
tqavctrl |= IGC_TQAVCTRL_FUTSCDDIS;
nsec = rd32(IGC_SYSTIML);
sec = rd32(IGC_SYSTIMH);
systim = ktime_set(sec, nsec);
adjust_time = adapter->base_time;
expires_time = ktime_sub_ns(adjust_time, systim);
hrtimer_start(&adapter->hrtimer, expires_time, HRTIMER_MODE_REL);
}
}
wr32(IGC_TQAVCTRL, tqavctrl);
wr32(IGC_QBVCYCLET_S, cycle);
wr32(IGC_QBVCYCLET, cycle);
baset_h = div_s64_rem(base_time, NSEC_PER_SEC, &baset_l);
wr32(IGC_BASET_H, baset_h);
/* In i226, Future base time is only supported when FutScdDis bit
* is enabled and only active for re-configuration.
* In this case, initialize the base time with zero to create
* "re-configuration" scenario then only set the desired base time.
*/
if (tqavctrl & IGC_TQAVCTRL_FUTSCDDIS)
wr32(IGC_BASET_L, 0);
wr32(IGC_BASET_L, baset_l);
return 0;
}
int igc_tsn_reset(struct igc_adapter *adapter)
{
unsigned int new_flags;
int err = 0;
new_flags = igc_tsn_new_flags(adapter);
if (!(new_flags & IGC_FLAG_TSN_ANY_ENABLED))
return igc_tsn_disable_offload(adapter);
err = igc_tsn_enable_offload(adapter);
if (err < 0)
return err;
adapter->flags = new_flags;
return err;
}
static bool igc_tsn_will_tx_mode_change(struct igc_adapter *adapter)
{
bool any_tsn_enabled = !!(igc_tsn_new_flags(adapter) &
IGC_FLAG_TSN_ANY_ENABLED);
return (any_tsn_enabled && !igc_tsn_is_tx_mode_in_tsn(adapter)) ||
(!any_tsn_enabled && igc_tsn_is_tx_mode_in_tsn(adapter));
}
int igc_tsn_offload_apply(struct igc_adapter *adapter)
{
/* Per I225/6 HW Design Section 7.5.2.1 guideline, if tx mode change
* from legacy->tsn or tsn->legacy, then reset adapter is needed.
*/
if (netif_running(adapter->netdev) &&
igc_tsn_will_tx_mode_change(adapter)) {
schedule_work(&adapter->reset_task);
return 0;
}
igc_tsn_reset(adapter);
return 0;
}
Codebase Browser
linux