Google Git
Sign in
fuchsia / third_party / qemu / refs/heads/upstream/master / . / hw / net / igb_core.c
blob: bcd5f6cd9cddd56c6ad9df847bf27352f9a6564b [file] [log] [blame]
/*
* Core code for QEMU igb emulation
*
* Datasheet:
* https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82576eg-gbe-datasheet.pdf
*
* Copyright (c) 2020-2023 Red Hat, Inc.
* Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com)
* Developed by Daynix Computing LTD (http://www.daynix.com)
*
* Authors:
* Akihiko Odaki <akihiko.odaki@daynix.com>
* Gal Hammmer <gal.hammer@sap.com>
* Marcel Apfelbaum <marcel.apfelbaum@gmail.com>
* Dmitry Fleytman <dmitry@daynix.com>
* Leonid Bloch <leonid@daynix.com>
* Yan Vugenfirer <yan@daynix.com>
*
* Based on work done by:
* Nir Peleg, Tutis Systems Ltd. for Qumranet Inc.
* Copyright (c) 2008 Qumranet
* Based on work done by:
* Copyright (c) 2007 Dan Aloni
* Copyright (c) 2004 Antony T Curtis
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
#include "qemu/osdep.h"
#include "qemu/log.h"
#include "net/net.h"
#include "net/tap.h"
#include "hw/net/mii.h"
#include "hw/pci/msi.h"
#include "hw/pci/msix.h"
#include "sysemu/runstate.h"
#include "net_tx_pkt.h"
#include "net_rx_pkt.h"
#include "igb_common.h"
#include "e1000x_common.h"
#include "igb_core.h"
#include "trace.h"
#define E1000E_MAX_TX_FRAGS (64)
union e1000_rx_desc_union {
struct e1000_rx_desc legacy;
union e1000_adv_rx_desc adv;
};
typedef struct IGBTxPktVmdqCallbackContext {
IGBCore *core;
NetClientState *nc;
} IGBTxPktVmdqCallbackContext;
typedef struct L2Header {
struct eth_header eth;
struct vlan_header vlan[2];
} L2Header;
typedef struct PTP2 {
uint8_t message_id_transport_specific;
uint8_t version_ptp;
uint16_t message_length;
uint8_t subdomain_number;
uint8_t reserved0;
uint16_t flags;
uint64_t correction;
uint8_t reserved1[5];
uint8_t source_communication_technology;
uint32_t source_uuid_lo;
uint16_t source_uuid_hi;
uint16_t source_port_id;
uint16_t sequence_id;
uint8_t control;
uint8_t log_message_period;
} PTP2;
static ssize_t
igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt,
bool has_vnet, bool *external_tx);
static void igb_raise_interrupts(IGBCore *core, size_t index, uint32_t causes);
static void igb_reset(IGBCore *core, bool sw);
static inline void
igb_raise_legacy_irq(IGBCore *core)
{
trace_e1000e_irq_legacy_notify(true);
e1000x_inc_reg_if_not_full(core->mac, IAC);
pci_set_irq(core->owner, 1);
}
static inline void
igb_lower_legacy_irq(IGBCore *core)
{
trace_e1000e_irq_legacy_notify(false);
pci_set_irq(core->owner, 0);
}
static void igb_msix_notify(IGBCore *core, unsigned int cause)
{
PCIDevice *dev = core->owner;
uint16_t vfn;
uint32_t effective_eiac;
unsigned int vector;
vfn = 8 - (cause + 2) / IGBVF_MSIX_VEC_NUM;
if (vfn < pcie_sriov_num_vfs(core->owner)) {
dev = pcie_sriov_get_vf_at_index(core->owner, vfn);
assert(dev);
vector = (cause + 2) % IGBVF_MSIX_VEC_NUM;
} else if (cause >= IGB_MSIX_VEC_NUM) {
qemu_log_mask(LOG_GUEST_ERROR,
"igb: Tried to use vector unavailable for PF");
return;
} else {
vector = cause;
}
msix_notify(dev, vector);
trace_e1000e_irq_icr_clear_eiac(core->mac[EICR], core->mac[EIAC]);
effective_eiac = core->mac[EIAC] & BIT(cause);
core->mac[EICR] &= ~effective_eiac;
}
static inline void
igb_intrmgr_rearm_timer(IGBIntrDelayTimer *timer)
{
int64_t delay_ns = (int64_t) timer->core->mac[timer->delay_reg] *
timer->delay_resolution_ns;
trace_e1000e_irq_rearm_timer(timer->delay_reg << 2, delay_ns);
timer_mod(timer->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + delay_ns);
timer->running = true;
}
static void
igb_intmgr_timer_resume(IGBIntrDelayTimer *timer)
{
if (timer->running) {
igb_intrmgr_rearm_timer(timer);
}
}
static void
igb_intrmgr_on_msix_throttling_timer(void *opaque)
{
IGBIntrDelayTimer *timer = opaque;
int idx = timer - &timer->core->eitr[0];
timer->running = false;
trace_e1000e_irq_msix_notify_postponed_vec(idx);
igb_msix_notify(timer->core, idx);
}
static void
igb_intrmgr_initialize_all_timers(IGBCore *core, bool create)
{
int i;
for (i = 0; i < IGB_INTR_NUM; i++) {
core->eitr[i].core = core;
core->eitr[i].delay_reg = EITR0 + i;
core->eitr[i].delay_resolution_ns = E1000_INTR_DELAY_NS_RES;
}
if (!create) {
return;
}
for (i = 0; i < IGB_INTR_NUM; i++) {
core->eitr[i].timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
igb_intrmgr_on_msix_throttling_timer,
&core->eitr[i]);
}
}
static void
igb_intrmgr_resume(IGBCore *core)
{
int i;
for (i = 0; i < IGB_INTR_NUM; i++) {
igb_intmgr_timer_resume(&core->eitr[i]);
}
}
static void
igb_intrmgr_reset(IGBCore *core)
{
int i;
for (i = 0; i < IGB_INTR_NUM; i++) {
if (core->eitr[i].running) {
timer_del(core->eitr[i].timer);
igb_intrmgr_on_msix_throttling_timer(&core->eitr[i]);
}
}
}
static void
igb_intrmgr_pci_unint(IGBCore *core)
{
int i;
for (i = 0; i < IGB_INTR_NUM; i++) {
timer_free(core->eitr[i].timer);
}
}
static void
igb_intrmgr_pci_realize(IGBCore *core)
{
igb_intrmgr_initialize_all_timers(core, true);
}
static inline bool
igb_rx_csum_enabled(IGBCore *core)
{
return (core->mac[RXCSUM] & E1000_RXCSUM_PCSD) ? false : true;
}
static inline bool
igb_rx_use_legacy_descriptor(IGBCore *core)
{
/*
* TODO: If SRRCTL[n],DESCTYPE = 000b, the 82576 uses the legacy Rx
* descriptor.
*/
return false;
}
typedef struct E1000ERingInfo {
int dbah;
int dbal;
int dlen;
int dh;
int dt;
int idx;
} E1000ERingInfo;
static uint32_t
igb_rx_queue_desctyp_get(IGBCore *core, const E1000ERingInfo *r)
{
return core->mac[E1000_SRRCTL(r->idx) >> 2] & E1000_SRRCTL_DESCTYPE_MASK;
}
static bool
igb_rx_use_ps_descriptor(IGBCore *core, const E1000ERingInfo *r)
{
uint32_t desctyp = igb_rx_queue_desctyp_get(core, r);
return desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT ||
desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
}
static inline bool
igb_rss_enabled(IGBCore *core)
{
return (core->mac[MRQC] & 3) == E1000_MRQC_ENABLE_RSS_MQ &&
!igb_rx_csum_enabled(core) &&
!igb_rx_use_legacy_descriptor(core);
}
typedef struct E1000E_RSSInfo_st {
bool enabled;
uint32_t hash;
uint32_t queue;
uint32_t type;
} E1000E_RSSInfo;
static uint32_t
igb_rss_get_hash_type(IGBCore *core, struct NetRxPkt *pkt)
{
bool hasip4, hasip6;
EthL4HdrProto l4hdr_proto;
assert(igb_rss_enabled(core));
net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
if (hasip4) {
trace_e1000e_rx_rss_ip4(l4hdr_proto, core->mac[MRQC],
E1000_MRQC_EN_TCPIPV4(core->mac[MRQC]),
E1000_MRQC_EN_IPV4(core->mac[MRQC]));
if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP &&
E1000_MRQC_EN_TCPIPV4(core->mac[MRQC])) {
return E1000_MRQ_RSS_TYPE_IPV4TCP;
}
if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP &&
(core->mac[MRQC] & E1000_MRQC_RSS_FIELD_IPV4_UDP)) {
return E1000_MRQ_RSS_TYPE_IPV4UDP;
}
if (E1000_MRQC_EN_IPV4(core->mac[MRQC])) {
return E1000_MRQ_RSS_TYPE_IPV4;
}
} else if (hasip6) {
eth_ip6_hdr_info *ip6info = net_rx_pkt_get_ip6_info(pkt);
bool ex_dis = core->mac[RFCTL] & E1000_RFCTL_IPV6_EX_DIS;
bool new_ex_dis = core->mac[RFCTL] & E1000_RFCTL_NEW_IPV6_EXT_DIS;
/*
* Following two traces must not be combined because resulting
* event will have 11 arguments totally and some trace backends
* (at least "ust") have limitation of maximum 10 arguments per
* event. Events with more arguments fail to compile for
* backends like these.
*/
trace_e1000e_rx_rss_ip6_rfctl(core->mac[RFCTL]);
trace_e1000e_rx_rss_ip6(ex_dis, new_ex_dis, l4hdr_proto,
ip6info->has_ext_hdrs,
ip6info->rss_ex_dst_valid,
ip6info->rss_ex_src_valid,
core->mac[MRQC],
E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC]),
E1000_MRQC_EN_IPV6EX(core->mac[MRQC]),
E1000_MRQC_EN_IPV6(core->mac[MRQC]));
if ((!ex_dis || !ip6info->has_ext_hdrs) &&
(!new_ex_dis || !(ip6info->rss_ex_dst_valid ||
ip6info->rss_ex_src_valid))) {
if (l4hdr_proto == ETH_L4_HDR_PROTO_TCP &&
E1000_MRQC_EN_TCPIPV6EX(core->mac[MRQC])) {
return E1000_MRQ_RSS_TYPE_IPV6TCPEX;
}
if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP &&
(core->mac[MRQC] & E1000_MRQC_RSS_FIELD_IPV6_UDP)) {
return E1000_MRQ_RSS_TYPE_IPV6UDP;
}
if (E1000_MRQC_EN_IPV6EX(core->mac[MRQC])) {
return E1000_MRQ_RSS_TYPE_IPV6EX;
}
}
if (E1000_MRQC_EN_IPV6(core->mac[MRQC])) {
return E1000_MRQ_RSS_TYPE_IPV6;
}
}
return E1000_MRQ_RSS_TYPE_NONE;
}
static uint32_t
igb_rss_calc_hash(IGBCore *core, struct NetRxPkt *pkt, E1000E_RSSInfo *info)
{
NetRxPktRssType type;
assert(igb_rss_enabled(core));
switch (info->type) {
case E1000_MRQ_RSS_TYPE_IPV4:
type = NetPktRssIpV4;
break;
case E1000_MRQ_RSS_TYPE_IPV4TCP:
type = NetPktRssIpV4Tcp;
break;
case E1000_MRQ_RSS_TYPE_IPV6TCPEX:
type = NetPktRssIpV6TcpEx;
break;
case E1000_MRQ_RSS_TYPE_IPV6:
type = NetPktRssIpV6;
break;
case E1000_MRQ_RSS_TYPE_IPV6EX:
type = NetPktRssIpV6Ex;
break;
case E1000_MRQ_RSS_TYPE_IPV4UDP:
type = NetPktRssIpV4Udp;
break;
case E1000_MRQ_RSS_TYPE_IPV6UDP:
type = NetPktRssIpV6Udp;
break;
default:
assert(false);
return 0;
}
return net_rx_pkt_calc_rss_hash(pkt, type, (uint8_t *) &core->mac[RSSRK]);
}
static void
igb_rss_parse_packet(IGBCore *core, struct NetRxPkt *pkt, bool tx,
E1000E_RSSInfo *info)
{
trace_e1000e_rx_rss_started();
if (tx || !igb_rss_enabled(core)) {
info->enabled = false;
info->hash = 0;
info->queue = 0;
info->type = 0;
trace_e1000e_rx_rss_disabled();
return;
}
info->enabled = true;
info->type = igb_rss_get_hash_type(core, pkt);
trace_e1000e_rx_rss_type(info->type);
if (info->type == E1000_MRQ_RSS_TYPE_NONE) {
info->hash = 0;
info->queue = 0;
return;
}
info->hash = igb_rss_calc_hash(core, pkt, info);
info->queue = E1000_RSS_QUEUE(&core->mac[RETA], info->hash);
}
static void
igb_tx_insert_vlan(IGBCore *core, uint16_t qn, struct igb_tx *tx,
uint16_t vlan, bool insert_vlan)
{
if (core->mac[MRQC] & 1) {
uint16_t pool = qn % IGB_NUM_VM_POOLS;
if (core->mac[VMVIR0 + pool] & E1000_VMVIR_VLANA_DEFAULT) {
/* always insert default VLAN */
insert_vlan = true;
vlan = core->mac[VMVIR0 + pool] & 0xffff;
} else if (core->mac[VMVIR0 + pool] & E1000_VMVIR_VLANA_NEVER) {
insert_vlan = false;
}
}
if (insert_vlan) {
net_tx_pkt_setup_vlan_header_ex(tx->tx_pkt, vlan,
core->mac[VET] & 0xffff);
}
}
static bool
igb_setup_tx_offloads(IGBCore *core, struct igb_tx *tx)
{
uint32_t idx = (tx->first_olinfo_status >> 4) & 1;
if (tx->first_cmd_type_len & E1000_ADVTXD_DCMD_TSE) {
uint32_t mss = tx->ctx[idx].mss_l4len_idx >> E1000_ADVTXD_MSS_SHIFT;
if (!net_tx_pkt_build_vheader(tx->tx_pkt, true, true, mss)) {
return false;
}
net_tx_pkt_update_ip_checksums(tx->tx_pkt);
e1000x_inc_reg_if_not_full(core->mac, TSCTC);
return true;
}
if ((tx->first_olinfo_status & E1000_ADVTXD_POTS_TXSM) &&
!((tx->ctx[idx].type_tucmd_mlhl & E1000_ADVTXD_TUCMD_L4T_SCTP) ?
net_tx_pkt_update_sctp_checksum(tx->tx_pkt) :
net_tx_pkt_build_vheader(tx->tx_pkt, false, true, 0))) {
return false;
}
if (tx->first_olinfo_status & E1000_ADVTXD_POTS_IXSM) {
net_tx_pkt_update_ip_hdr_checksum(tx->tx_pkt);
}
return true;
}
static void igb_tx_pkt_mac_callback(void *core,
const struct iovec *iov,
int iovcnt,
const struct iovec *virt_iov,
int virt_iovcnt)
{
igb_receive_internal(core, virt_iov, virt_iovcnt, true, NULL);
}
static void igb_tx_pkt_vmdq_callback(void *opaque,
const struct iovec *iov,
int iovcnt,
const struct iovec *virt_iov,
int virt_iovcnt)
{
IGBTxPktVmdqCallbackContext *context = opaque;
bool external_tx;
igb_receive_internal(context->core, virt_iov, virt_iovcnt, true,
&external_tx);
if (external_tx) {
if (context->core->has_vnet) {
qemu_sendv_packet(context->nc, virt_iov, virt_iovcnt);
} else {
qemu_sendv_packet(context->nc, iov, iovcnt);
}
}
}
/* TX Packets Switching (7.10.3.6) */
static bool igb_tx_pkt_switch(IGBCore *core, struct igb_tx *tx,
NetClientState *nc)
{
IGBTxPktVmdqCallbackContext context;
/* TX switching is only used to serve VM to VM traffic. */
if (!(core->mac[MRQC] & 1)) {
goto send_out;
}
/* TX switching requires DTXSWC.Loopback_en bit enabled. */
if (!(core->mac[DTXSWC] & E1000_DTXSWC_VMDQ_LOOPBACK_EN)) {
goto send_out;
}
context.core = core;
context.nc = nc;
return net_tx_pkt_send_custom(tx->tx_pkt, false,
igb_tx_pkt_vmdq_callback, &context);
send_out:
return net_tx_pkt_send(tx->tx_pkt, nc);
}
static bool
igb_tx_pkt_send(IGBCore *core, struct igb_tx *tx, int queue_index)
{
int target_queue = MIN(core->max_queue_num, queue_index);
NetClientState *queue = qemu_get_subqueue(core->owner_nic, target_queue);
if (!igb_setup_tx_offloads(core, tx)) {
return false;
}
net_tx_pkt_dump(tx->tx_pkt);
if ((core->phy[MII_BMCR] & MII_BMCR_LOOPBACK) ||
((core->mac[RCTL] & E1000_RCTL_LBM_MAC) == E1000_RCTL_LBM_MAC)) {
return net_tx_pkt_send_custom(tx->tx_pkt, false,
igb_tx_pkt_mac_callback, core);
} else {
return igb_tx_pkt_switch(core, tx, queue);
}
}
static void
igb_on_tx_done_update_stats(IGBCore *core, struct NetTxPkt *tx_pkt, int qn)
{
static const int PTCregs[6] = { PTC64, PTC127, PTC255, PTC511,
PTC1023, PTC1522 };
size_t tot_len = net_tx_pkt_get_total_len(tx_pkt) + 4;
e1000x_increase_size_stats(core->mac, PTCregs, tot_len);
e1000x_inc_reg_if_not_full(core->mac, TPT);
e1000x_grow_8reg_if_not_full(core->mac, TOTL, tot_len);
switch (net_tx_pkt_get_packet_type(tx_pkt)) {
case ETH_PKT_BCAST:
e1000x_inc_reg_if_not_full(core->mac, BPTC);
break;
case ETH_PKT_MCAST:
e1000x_inc_reg_if_not_full(core->mac, MPTC);
break;
case ETH_PKT_UCAST:
break;
default:
g_assert_not_reached();
}
e1000x_inc_reg_if_not_full(core->mac, GPTC);
e1000x_grow_8reg_if_not_full(core->mac, GOTCL, tot_len);
if (core->mac[MRQC] & 1) {
uint16_t pool = qn % IGB_NUM_VM_POOLS;
core->mac[PVFGOTC0 + (pool * 64)] += tot_len;
core->mac[PVFGPTC0 + (pool * 64)]++;
}
}
static void
igb_process_tx_desc(IGBCore *core,
PCIDevice *dev,
struct igb_tx *tx,
union e1000_adv_tx_desc *tx_desc,
int queue_index)
{
struct e1000_adv_tx_context_desc *tx_ctx_desc;
uint32_t cmd_type_len;
uint32_t idx;
uint64_t buffer_addr;
uint16_t length;
cmd_type_len = le32_to_cpu(tx_desc->read.cmd_type_len);
if (cmd_type_len & E1000_ADVTXD_DCMD_DEXT) {
if ((cmd_type_len & E1000_ADVTXD_DTYP_DATA) ==
E1000_ADVTXD_DTYP_DATA) {
/* advanced transmit data descriptor */
if (tx->first) {
tx->first_cmd_type_len = cmd_type_len;
tx->first_olinfo_status = le32_to_cpu(tx_desc->read.olinfo_status);
tx->first = false;
}
} else if ((cmd_type_len & E1000_ADVTXD_DTYP_CTXT) ==
E1000_ADVTXD_DTYP_CTXT) {
/* advanced transmit context descriptor */
tx_ctx_desc = (struct e1000_adv_tx_context_desc *)tx_desc;
idx = (le32_to_cpu(tx_ctx_desc->mss_l4len_idx) >> 4) & 1;
tx->ctx[idx].vlan_macip_lens = le32_to_cpu(tx_ctx_desc->vlan_macip_lens);
tx->ctx[idx].seqnum_seed = le32_to_cpu(tx_ctx_desc->seqnum_seed);
tx->ctx[idx].type_tucmd_mlhl = le32_to_cpu(tx_ctx_desc->type_tucmd_mlhl);
tx->ctx[idx].mss_l4len_idx = le32_to_cpu(tx_ctx_desc->mss_l4len_idx);
return;
} else {
/* unknown descriptor type */
return;
}
} else {
/* legacy descriptor */
/* TODO: Implement a support for legacy descriptors (7.2.2.1). */
}
buffer_addr = le64_to_cpu(tx_desc->read.buffer_addr);
length = cmd_type_len & 0xFFFF;
if (!tx->skip_cp) {
if (!net_tx_pkt_add_raw_fragment_pci(tx->tx_pkt, dev,
buffer_addr, length)) {
tx->skip_cp = true;
}
}
if (cmd_type_len & E1000_TXD_CMD_EOP) {
if (!tx->skip_cp && net_tx_pkt_parse(tx->tx_pkt)) {
idx = (tx->first_olinfo_status >> 4) & 1;
igb_tx_insert_vlan(core, queue_index, tx,
tx->ctx[idx].vlan_macip_lens >> IGB_TX_FLAGS_VLAN_SHIFT,
!!(tx->first_cmd_type_len & E1000_TXD_CMD_VLE));
if ((tx->first_cmd_type_len & E1000_ADVTXD_MAC_TSTAMP) &&
(core->mac[TSYNCTXCTL] & E1000_TSYNCTXCTL_ENABLED) &&
!(core->mac[TSYNCTXCTL] & E1000_TSYNCTXCTL_VALID)) {
core->mac[TSYNCTXCTL] |= E1000_TSYNCTXCTL_VALID;
e1000x_timestamp(core->mac, core->timadj, TXSTMPL, TXSTMPH);
}
if (igb_tx_pkt_send(core, tx, queue_index)) {
igb_on_tx_done_update_stats(core, tx->tx_pkt, queue_index);
}
}
tx->first = true;
tx->skip_cp = false;
net_tx_pkt_reset(tx->tx_pkt, net_tx_pkt_unmap_frag_pci, dev);
}
}
static uint32_t igb_tx_wb_eic(IGBCore *core, int queue_idx)
{
uint32_t n, ent = 0;
n = igb_ivar_entry_tx(queue_idx);
ent = (core->mac[IVAR0 + n / 4] >> (8 * (n % 4))) & 0xff;
return (ent & E1000_IVAR_VALID) ? BIT(ent & 0x1f) : 0;
}
static uint32_t igb_rx_wb_eic(IGBCore *core, int queue_idx)
{
uint32_t n, ent = 0;
n = igb_ivar_entry_rx(queue_idx);
ent = (core->mac[IVAR0 + n / 4] >> (8 * (n % 4))) & 0xff;
return (ent & E1000_IVAR_VALID) ? BIT(ent & 0x1f) : 0;
}
static inline bool
igb_ring_empty(IGBCore *core, const E1000ERingInfo *r)
{
return core->mac[r->dh] == core->mac[r->dt] ||
core->mac[r->dt] >= core->mac[r->dlen] / E1000_RING_DESC_LEN;
}
static inline uint64_t
igb_ring_base(IGBCore *core, const E1000ERingInfo *r)
{
uint64_t bah = core->mac[r->dbah];
uint64_t bal = core->mac[r->dbal];
return (bah << 32) + bal;
}
static inline uint64_t
igb_ring_head_descr(IGBCore *core, const E1000ERingInfo *r)
{
return igb_ring_base(core, r) + E1000_RING_DESC_LEN * core->mac[r->dh];
}
static inline void
igb_ring_advance(IGBCore *core, const E1000ERingInfo *r, uint32_t count)
{
core->mac[r->dh] += count;
if (core->mac[r->dh] * E1000_RING_DESC_LEN >= core->mac[r->dlen]) {
core->mac[r->dh] = 0;
}
}
static inline uint32_t
igb_ring_free_descr_num(IGBCore *core, const E1000ERingInfo *r)
{
trace_e1000e_ring_free_space(r->idx, core->mac[r->dlen],
core->mac[r->dh], core->mac[r->dt]);
if (core->mac[r->dh] <= core->mac[r->dt]) {
return core->mac[r->dt] - core->mac[r->dh];
}
if (core->mac[r->dh] > core->mac[r->dt]) {
return core->mac[r->dlen] / E1000_RING_DESC_LEN +
core->mac[r->dt] - core->mac[r->dh];
}
g_assert_not_reached();
return 0;
}
static inline bool
igb_ring_enabled(IGBCore *core, const E1000ERingInfo *r)
{
return core->mac[r->dlen] > 0;
}
typedef struct IGB_TxRing_st {
const E1000ERingInfo *i;
struct igb_tx *tx;
} IGB_TxRing;
static inline int
igb_mq_queue_idx(int base_reg_idx, int reg_idx)
{
return (reg_idx - base_reg_idx) / 16;
}
static inline void
igb_tx_ring_init(IGBCore *core, IGB_TxRing *txr, int idx)
{
static const E1000ERingInfo i[IGB_NUM_QUEUES] = {
{ TDBAH0, TDBAL0, TDLEN0, TDH0, TDT0, 0 },
{ TDBAH1, TDBAL1, TDLEN1, TDH1, TDT1, 1 },
{ TDBAH2, TDBAL2, TDLEN2, TDH2, TDT2, 2 },
{ TDBAH3, TDBAL3, TDLEN3, TDH3, TDT3, 3 },
{ TDBAH4, TDBAL4, TDLEN4, TDH4, TDT4, 4 },
{ TDBAH5, TDBAL5, TDLEN5, TDH5, TDT5, 5 },
{ TDBAH6, TDBAL6, TDLEN6, TDH6, TDT6, 6 },
{ TDBAH7, TDBAL7, TDLEN7, TDH7, TDT7, 7 },
{ TDBAH8, TDBAL8, TDLEN8, TDH8, TDT8, 8 },
{ TDBAH9, TDBAL9, TDLEN9, TDH9, TDT9, 9 },
{ TDBAH10, TDBAL10, TDLEN10, TDH10, TDT10, 10 },
{ TDBAH11, TDBAL11, TDLEN11, TDH11, TDT11, 11 },
{ TDBAH12, TDBAL12, TDLEN12, TDH12, TDT12, 12 },
{ TDBAH13, TDBAL13, TDLEN13, TDH13, TDT13, 13 },
{ TDBAH14, TDBAL14, TDLEN14, TDH14, TDT14, 14 },
{ TDBAH15, TDBAL15, TDLEN15, TDH15, TDT15, 15 }
};
assert(idx < ARRAY_SIZE(i));
txr->i = &i[idx];
txr->tx = &core->tx[idx];
}
typedef struct E1000E_RxRing_st {
const E1000ERingInfo *i;
} E1000E_RxRing;
static inline void
igb_rx_ring_init(IGBCore *core, E1000E_RxRing *rxr, int idx)
{
static const E1000ERingInfo i[IGB_NUM_QUEUES] = {
{ RDBAH0, RDBAL0, RDLEN0, RDH0, RDT0, 0 },
{ RDBAH1, RDBAL1, RDLEN1, RDH1, RDT1, 1 },
{ RDBAH2, RDBAL2, RDLEN2, RDH2, RDT2, 2 },
{ RDBAH3, RDBAL3, RDLEN3, RDH3, RDT3, 3 },
{ RDBAH4, RDBAL4, RDLEN4, RDH4, RDT4, 4 },
{ RDBAH5, RDBAL5, RDLEN5, RDH5, RDT5, 5 },
{ RDBAH6, RDBAL6, RDLEN6, RDH6, RDT6, 6 },
{ RDBAH7, RDBAL7, RDLEN7, RDH7, RDT7, 7 },
{ RDBAH8, RDBAL8, RDLEN8, RDH8, RDT8, 8 },
{ RDBAH9, RDBAL9, RDLEN9, RDH9, RDT9, 9 },
{ RDBAH10, RDBAL10, RDLEN10, RDH10, RDT10, 10 },
{ RDBAH11, RDBAL11, RDLEN11, RDH11, RDT11, 11 },
{ RDBAH12, RDBAL12, RDLEN12, RDH12, RDT12, 12 },
{ RDBAH13, RDBAL13, RDLEN13, RDH13, RDT13, 13 },
{ RDBAH14, RDBAL14, RDLEN14, RDH14, RDT14, 14 },
{ RDBAH15, RDBAL15, RDLEN15, RDH15, RDT15, 15 }
};
assert(idx < ARRAY_SIZE(i));
rxr->i = &i[idx];
}
static uint32_t
igb_txdesc_writeback(IGBCore *core, dma_addr_t base,
union e1000_adv_tx_desc *tx_desc,
const E1000ERingInfo *txi)
{
PCIDevice *d;
uint32_t cmd_type_len = le32_to_cpu(tx_desc->read.cmd_type_len);
uint64_t tdwba;
tdwba = core->mac[E1000_TDWBAL(txi->idx) >> 2];
tdwba |= (uint64_t)core->mac[E1000_TDWBAH(txi->idx) >> 2] << 32;
if (!(cmd_type_len & E1000_TXD_CMD_RS)) {
return 0;
}
d = pcie_sriov_get_vf_at_index(core->owner, txi->idx % 8);
if (!d) {
d = core->owner;
}
if (tdwba & 1) {
uint32_t buffer = cpu_to_le32(core->mac[txi->dh]);
pci_dma_write(d, tdwba & ~3, &buffer, sizeof(buffer));
} else {
uint32_t status = le32_to_cpu(tx_desc->wb.status) | E1000_TXD_STAT_DD;
tx_desc->wb.status = cpu_to_le32(status);
pci_dma_write(d, base + offsetof(union e1000_adv_tx_desc, wb),
&tx_desc->wb, sizeof(tx_desc->wb));
}
return igb_tx_wb_eic(core, txi->idx);
}
static inline bool
igb_tx_enabled(IGBCore *core, const E1000ERingInfo *txi)
{
bool vmdq = core->mac[MRQC] & 1;
uint16_t qn = txi->idx;
uint16_t pool = qn % IGB_NUM_VM_POOLS;
return (core->mac[TCTL] & E1000_TCTL_EN) &&
(!vmdq || core->mac[VFTE] & BIT(pool)) &&
(core->mac[TXDCTL0 + (qn * 16)] & E1000_TXDCTL_QUEUE_ENABLE);
}
static void
igb_start_xmit(IGBCore *core, const IGB_TxRing *txr)
{
PCIDevice *d;
dma_addr_t base;
union e1000_adv_tx_desc desc;
const E1000ERingInfo *txi = txr->i;
uint32_t eic = 0;
if (!igb_tx_enabled(core, txi)) {
trace_e1000e_tx_disabled();
return;
}
d = pcie_sriov_get_vf_at_index(core->owner, txi->idx % 8);
if (!d) {
d = core->owner;
}
while (!igb_ring_empty(core, txi)) {
base = igb_ring_head_descr(core, txi);
pci_dma_read(d, base, &desc, sizeof(desc));
trace_e1000e_tx_descr((void *)(intptr_t)desc.read.buffer_addr,
desc.read.cmd_type_len, desc.wb.status);
igb_process_tx_desc(core, d, txr->tx, &desc, txi->idx);
igb_ring_advance(core, txi, 1);
eic |= igb_txdesc_writeback(core, base, &desc, txi);
}
if (eic) {
igb_raise_interrupts(core, EICR, eic);
igb_raise_interrupts(core, ICR, E1000_ICR_TXDW);
}
net_tx_pkt_reset(txr->tx->tx_pkt, net_tx_pkt_unmap_frag_pci, d);
}
static uint32_t
igb_rxbufsize(IGBCore *core, const E1000ERingInfo *r)
{
uint32_t srrctl = core->mac[E1000_SRRCTL(r->idx) >> 2];
uint32_t bsizepkt = srrctl & E1000_SRRCTL_BSIZEPKT_MASK;
if (bsizepkt) {
return bsizepkt << E1000_SRRCTL_BSIZEPKT_SHIFT;
}
return e1000x_rxbufsize(core->mac[RCTL]);
}
static bool
igb_has_rxbufs(IGBCore *core, const E1000ERingInfo *r, size_t total_size)
{
uint32_t bufs = igb_ring_free_descr_num(core, r);
uint32_t bufsize = igb_rxbufsize(core, r);
trace_e1000e_rx_has_buffers(r->idx, bufs, total_size, bufsize);
return total_size <= bufs / (core->rx_desc_len / E1000_MIN_RX_DESC_LEN) *
bufsize;
}
static uint32_t
igb_rxhdrbufsize(IGBCore *core, const E1000ERingInfo *r)
{
uint32_t srrctl = core->mac[E1000_SRRCTL(r->idx) >> 2];
return (srrctl & E1000_SRRCTL_BSIZEHDRSIZE_MASK) >>
E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
}
void
igb_start_recv(IGBCore *core)
{
int i;
trace_e1000e_rx_start_recv();
for (i = 0; i <= core->max_queue_num; i++) {
qemu_flush_queued_packets(qemu_get_subqueue(core->owner_nic, i));
}
}
bool
igb_can_receive(IGBCore *core)
{
int i;
if (!e1000x_rx_ready(core->owner, core->mac)) {
return false;
}
for (i = 0; i < IGB_NUM_QUEUES; i++) {
E1000E_RxRing rxr;
if (!(core->mac[RXDCTL0 + (i * 16)] & E1000_RXDCTL_QUEUE_ENABLE)) {
continue;
}
igb_rx_ring_init(core, &rxr, i);
if (igb_ring_enabled(core, rxr.i) && igb_has_rxbufs(core, rxr.i, 1)) {
trace_e1000e_rx_can_recv();
return true;
}
}
trace_e1000e_rx_can_recv_rings_full();
return false;
}
ssize_t
igb_receive(IGBCore *core, const uint8_t *buf, size_t size)
{
const struct iovec iov = {
.iov_base = (uint8_t *)buf,
.iov_len = size
};
return igb_receive_iov(core, &iov, 1);
}
static inline bool
igb_rx_l3_cso_enabled(IGBCore *core)
{
return !!(core->mac[RXCSUM] & E1000_RXCSUM_IPOFLD);
}
static inline bool
igb_rx_l4_cso_enabled(IGBCore *core)
{
return !!(core->mac[RXCSUM] & E1000_RXCSUM_TUOFLD);
}
static bool igb_rx_is_oversized(IGBCore *core, const struct eth_header *ehdr,
size_t size, size_t vlan_num,
bool lpe, uint16_t rlpml)
{
size_t vlan_header_size = sizeof(struct vlan_header) * vlan_num;
size_t header_size = sizeof(struct eth_header) + vlan_header_size;
return lpe ? size + ETH_FCS_LEN > rlpml : size > header_size + ETH_MTU;
}
static uint16_t igb_receive_assign(IGBCore *core, const struct iovec *iov,
size_t iovcnt, size_t iov_ofs,
const L2Header *l2_header, size_t size,
E1000E_RSSInfo *rss_info,
uint16_t *etqf, bool *ts, bool *external_tx)
{
static const int ta_shift[] = { 4, 3, 2, 0 };
const struct eth_header *ehdr = &l2_header->eth;
uint32_t f, ra[2], *macp, rctl = core->mac[RCTL];
uint16_t queues = 0;
uint16_t oversized = 0;
size_t vlan_num = 0;
PTP2 ptp2;
bool lpe;
uint16_t rlpml;
int i;
memset(rss_info, 0, sizeof(E1000E_RSSInfo));
*ts = false;
if (external_tx) {
*external_tx = true;
}
if (core->mac[CTRL_EXT] & BIT(26)) {
if (be16_to_cpu(ehdr->h_proto) == core->mac[VET] >> 16 &&
be16_to_cpu(l2_header->vlan[0].h_proto) == (core->mac[VET] & 0xffff)) {
vlan_num = 2;
}
} else {
if (be16_to_cpu(ehdr->h_proto) == (core->mac[VET] & 0xffff)) {
vlan_num = 1;
}
}
lpe = !!(core->mac[RCTL] & E1000_RCTL_LPE);
rlpml = core->mac[RLPML];
if (!(core->mac[RCTL] & E1000_RCTL_SBP) &&
igb_rx_is_oversized(core, ehdr, size, vlan_num, lpe, rlpml)) {
trace_e1000x_rx_oversized(size);
return queues;
}
for (*etqf = 0; *etqf < 8; (*etqf)++) {
if ((core->mac[ETQF0 + *etqf] & E1000_ETQF_FILTER_ENABLE) &&
be16_to_cpu(ehdr->h_proto) == (core->mac[ETQF0 + *etqf] & E1000_ETQF_ETYPE_MASK)) {
if ((core->mac[ETQF0 + *etqf] & E1000_ETQF_1588) &&
(core->mac[TSYNCRXCTL] & E1000_TSYNCRXCTL_ENABLED) &&
!(core->mac[TSYNCRXCTL] & E1000_TSYNCRXCTL_VALID) &&
iov_to_buf(iov, iovcnt, iov_ofs + ETH_HLEN, &ptp2, sizeof(ptp2)) >= sizeof(ptp2) &&
(ptp2.version_ptp & 15) == 2 &&
ptp2.message_id_transport_specific == ((core->mac[TSYNCRXCFG] >> 8) & 255)) {
e1000x_timestamp(core->mac, core->timadj, RXSTMPL, RXSTMPH);
*ts = true;
core->mac[TSYNCRXCTL] |= E1000_TSYNCRXCTL_VALID;
core->mac[RXSATRL] = le32_to_cpu(ptp2.source_uuid_lo);
core->mac[RXSATRH] = le16_to_cpu(ptp2.source_uuid_hi) |
(le16_to_cpu(ptp2.sequence_id) << 16);
}
break;
}
}
if (vlan_num &&
!e1000x_rx_vlan_filter(core->mac, l2_header->vlan + vlan_num - 1)) {
return queues;
}
if (core->mac[MRQC] & 1) {
if (is_broadcast_ether_addr(ehdr->h_dest)) {
for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
if (core->mac[VMOLR0 + i] & E1000_VMOLR_BAM) {
queues |= BIT(i);
}
}
} else {
for (macp = core->mac + RA; macp < core->mac + RA + 32; macp += 2) {
if (!(macp[1] & E1000_RAH_AV)) {
continue;
}
ra[0] = cpu_to_le32(macp[0]);
ra[1] = cpu_to_le32(macp[1]);
if (!memcmp(ehdr->h_dest, (uint8_t *)ra, ETH_ALEN)) {
queues |= (macp[1] & E1000_RAH_POOL_MASK) / E1000_RAH_POOL_1;
}
}
for (macp = core->mac + RA2; macp < core->mac + RA2 + 16; macp += 2) {
if (!(macp[1] & E1000_RAH_AV)) {
continue;
}
ra[0] = cpu_to_le32(macp[0]);
ra[1] = cpu_to_le32(macp[1]);
if (!memcmp(ehdr->h_dest, (uint8_t *)ra, ETH_ALEN)) {
queues |= (macp[1] & E1000_RAH_POOL_MASK) / E1000_RAH_POOL_1;
}
}
if (!queues) {
macp = core->mac + (is_multicast_ether_addr(ehdr->h_dest) ? MTA : UTA);
f = ta_shift[(rctl >> E1000_RCTL_MO_SHIFT) & 3];
f = (((ehdr->h_dest[5] << 8) | ehdr->h_dest[4]) >> f) & 0xfff;
if (macp[f >> 5] & (1 << (f & 0x1f))) {
for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
if (core->mac[VMOLR0 + i] & E1000_VMOLR_ROMPE) {
queues |= BIT(i);
}
}
}
} else if (is_unicast_ether_addr(ehdr->h_dest) && external_tx) {
*external_tx = false;
}
}
if (e1000x_vlan_rx_filter_enabled(core->mac)) {
uint16_t mask = 0;
if (vlan_num) {
uint16_t vid = be16_to_cpu(l2_header->vlan[vlan_num - 1].h_tci) & VLAN_VID_MASK;
for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
if ((core->mac[VLVF0 + i] & E1000_VLVF_VLANID_MASK) == vid &&
(core->mac[VLVF0 + i] & E1000_VLVF_VLANID_ENABLE)) {
uint32_t poolsel = core->mac[VLVF0 + i] & E1000_VLVF_POOLSEL_MASK;
mask |= poolsel >> E1000_VLVF_POOLSEL_SHIFT;
}
}
} else {
for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
if (core->mac[VMOLR0 + i] & E1000_VMOLR_AUPE) {
mask |= BIT(i);
}
}
}
queues &= mask;
}
if (is_unicast_ether_addr(ehdr->h_dest) && !queues && !external_tx &&
!(core->mac[VT_CTL] & E1000_VT_CTL_DISABLE_DEF_POOL)) {
uint32_t def_pl = core->mac[VT_CTL] & E1000_VT_CTL_DEFAULT_POOL_MASK;
queues = BIT(def_pl >> E1000_VT_CTL_DEFAULT_POOL_SHIFT);
}
queues &= core->mac[VFRE];
if (queues) {
for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
lpe = !!(core->mac[VMOLR0 + i] & E1000_VMOLR_LPE);
rlpml = core->mac[VMOLR0 + i] & E1000_VMOLR_RLPML_MASK;
if ((queues & BIT(i)) &&
igb_rx_is_oversized(core, ehdr, size, vlan_num,
lpe, rlpml)) {
oversized |= BIT(i);
}
}
/* 8.19.37 increment ROC if packet is oversized for all queues */
if (oversized == queues) {
trace_e1000x_rx_oversized(size);
e1000x_inc_reg_if_not_full(core->mac, ROC);
}
queues &= ~oversized;
}
if (queues) {
igb_rss_parse_packet(core, core->rx_pkt,
external_tx != NULL, rss_info);
/* Sec 8.26.1: PQn = VFn + VQn*8 */
if (rss_info->queue & 1) {
for (i = 0; i < IGB_NUM_VM_POOLS; i++) {
if ((queues & BIT(i)) &&
(core->mac[VMOLR0 + i] & E1000_VMOLR_RSSE)) {
queues |= BIT(i + IGB_NUM_VM_POOLS);
queues &= ~BIT(i);
}
}
}
}
} else {
bool accepted = e1000x_rx_group_filter(core->mac, ehdr);
if (!accepted) {
for (macp = core->mac + RA2; macp < core->mac + RA2 + 16; macp += 2) {
if (!(macp[1] & E1000_RAH_AV)) {
continue;
}
ra[0] = cpu_to_le32(macp[0]);
ra[1] = cpu_to_le32(macp[1]);
if (!memcmp(ehdr->h_dest, (uint8_t *)ra, ETH_ALEN)) {
trace_e1000x_rx_flt_ucast_match((int)(macp - core->mac - RA2) / 2,
MAC_ARG(ehdr->h_dest));
accepted = true;
break;
}
}
}
if (accepted) {
igb_rss_parse_packet(core, core->rx_pkt, false, rss_info);
queues = BIT(rss_info->queue);
}
}
return queues;
}
static inline void
igb_read_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc,
hwaddr *buff_addr)
{
*buff_addr = le64_to_cpu(desc->buffer_addr);
}
static inline void
igb_read_adv_rx_single_buf_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
hwaddr *buff_addr)
{
*buff_addr = le64_to_cpu(desc->read.pkt_addr);
}
static inline void
igb_read_adv_rx_split_buf_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
hwaddr *buff_addr)
{
buff_addr[0] = le64_to_cpu(desc->read.hdr_addr);
buff_addr[1] = le64_to_cpu(desc->read.pkt_addr);
}
typedef struct IGBBAState {
uint16_t written[IGB_MAX_PS_BUFFERS];
uint8_t cur_idx;
} IGBBAState;
typedef struct IGBSplitDescriptorData {
bool sph;
bool hbo;
size_t hdr_len;
} IGBSplitDescriptorData;
typedef struct IGBPacketRxDMAState {
size_t size;
size_t total_size;
size_t ps_hdr_len;
size_t desc_size;
size_t desc_offset;
uint32_t rx_desc_packet_buf_size;
uint32_t rx_desc_header_buf_size;
struct iovec *iov;
size_t iov_ofs;
bool do_ps;
bool is_first;
IGBBAState bastate;
hwaddr ba[IGB_MAX_PS_BUFFERS];
IGBSplitDescriptorData ps_desc_data;
} IGBPacketRxDMAState;
static inline void
igb_read_rx_descr(IGBCore *core,
union e1000_rx_desc_union *desc,
IGBPacketRxDMAState *pdma_st,
const E1000ERingInfo *r)
{
uint32_t desc_type;
if (igb_rx_use_legacy_descriptor(core)) {
igb_read_lgcy_rx_descr(core, &desc->legacy, &pdma_st->ba[1]);
pdma_st->ba[0] = 0;
return;
}
/* advanced header split descriptor */
if (igb_rx_use_ps_descriptor(core, r)) {
igb_read_adv_rx_split_buf_descr(core, &desc->adv, &pdma_st->ba[0]);
return;
}
/* descriptor replication modes not supported */
desc_type = igb_rx_queue_desctyp_get(core, r);
if (desc_type != E1000_SRRCTL_DESCTYPE_ADV_ONEBUF) {
trace_igb_wrn_rx_desc_modes_not_supp(desc_type);
}
/* advanced single buffer descriptor */
igb_read_adv_rx_single_buf_descr(core, &desc->adv, &pdma_st->ba[1]);
pdma_st->ba[0] = 0;
}
static void
igb_verify_csum_in_sw(IGBCore *core,
struct NetRxPkt *pkt,
uint32_t *status_flags,
EthL4HdrProto l4hdr_proto)
{
bool csum_valid;
uint32_t csum_error;
if (igb_rx_l3_cso_enabled(core)) {
if (!net_rx_pkt_validate_l3_csum(pkt, &csum_valid)) {
trace_e1000e_rx_metadata_l3_csum_validation_failed();
} else {
csum_error = csum_valid ? 0 : E1000_RXDEXT_STATERR_IPE;
*status_flags |= E1000_RXD_STAT_IPCS | csum_error;
}
} else {
trace_e1000e_rx_metadata_l3_cso_disabled();
}
if (!igb_rx_l4_cso_enabled(core)) {
trace_e1000e_rx_metadata_l4_cso_disabled();
return;
}
if (!net_rx_pkt_validate_l4_csum(pkt, &csum_valid)) {
trace_e1000e_rx_metadata_l4_csum_validation_failed();
return;
}
csum_error = csum_valid ? 0 : E1000_RXDEXT_STATERR_TCPE;
*status_flags |= E1000_RXD_STAT_TCPCS | csum_error;
if (l4hdr_proto == ETH_L4_HDR_PROTO_UDP) {
*status_flags |= E1000_RXD_STAT_UDPCS;
}
}
static void
igb_build_rx_metadata_common(IGBCore *core,
struct NetRxPkt *pkt,
bool is_eop,
uint32_t *status_flags,
uint16_t *vlan_tag)
{
struct virtio_net_hdr *vhdr;
bool hasip4, hasip6, csum_valid;
EthL4HdrProto l4hdr_proto;
*status_flags = E1000_RXD_STAT_DD;
/* No additional metadata needed for non-EOP descriptors */
if (!is_eop) {
goto func_exit;
}
*status_flags |= E1000_RXD_STAT_EOP;
net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
trace_e1000e_rx_metadata_protocols(hasip4, hasip6, l4hdr_proto);
/* VLAN state */
if (net_rx_pkt_is_vlan_stripped(pkt)) {
*status_flags |= E1000_RXD_STAT_VP;
*vlan_tag = cpu_to_le16(net_rx_pkt_get_vlan_tag(pkt));
trace_e1000e_rx_metadata_vlan(*vlan_tag);
}
/* RX CSO information */
if (hasip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) {
trace_e1000e_rx_metadata_ipv6_sum_disabled();
goto func_exit;
}
vhdr = net_rx_pkt_get_vhdr(pkt);
if (!(vhdr->flags & VIRTIO_NET_HDR_F_DATA_VALID) &&
!(vhdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) {
trace_e1000e_rx_metadata_virthdr_no_csum_info();
igb_verify_csum_in_sw(core, pkt, status_flags, l4hdr_proto);
goto func_exit;
}
if (igb_rx_l3_cso_enabled(core)) {
*status_flags |= hasip4 ? E1000_RXD_STAT_IPCS : 0;
} else {
trace_e1000e_rx_metadata_l3_cso_disabled();
}
if (igb_rx_l4_cso_enabled(core)) {
switch (l4hdr_proto) {
case ETH_L4_HDR_PROTO_SCTP:
if (!net_rx_pkt_validate_l4_csum(pkt, &csum_valid)) {
trace_e1000e_rx_metadata_l4_csum_validation_failed();
goto func_exit;
}
if (!csum_valid) {
*status_flags |= E1000_RXDEXT_STATERR_TCPE;
}
/* fall through */
case ETH_L4_HDR_PROTO_TCP:
*status_flags |= E1000_RXD_STAT_TCPCS;
break;
case ETH_L4_HDR_PROTO_UDP:
*status_flags |= E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS;
break;
default:
break;
}
} else {
trace_e1000e_rx_metadata_l4_cso_disabled();
}
func_exit:
trace_e1000e_rx_metadata_status_flags(*status_flags);
*status_flags = cpu_to_le32(*status_flags);
}
static inline void
igb_write_lgcy_rx_descr(IGBCore *core, struct e1000_rx_desc *desc,
struct NetRxPkt *pkt,
const E1000E_RSSInfo *rss_info,
uint16_t length)
{
uint32_t status_flags;
assert(!rss_info->enabled);
memset(desc, 0, sizeof(*desc));
desc->length = cpu_to_le16(length);
igb_build_rx_metadata_common(core, pkt, pkt != NULL,
&status_flags,
&desc->special);
desc->errors = (uint8_t) (le32_to_cpu(status_flags) >> 24);
desc->status = (uint8_t) le32_to_cpu(status_flags);
}
static bool
igb_rx_ps_descriptor_split_always(IGBCore *core, const E1000ERingInfo *r)
{
uint32_t desctyp = igb_rx_queue_desctyp_get(core, r);
return desctyp == E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
}
static uint16_t
igb_rx_desc_get_packet_type(IGBCore *core, struct NetRxPkt *pkt, uint16_t etqf)
{
uint16_t pkt_type;
bool hasip4, hasip6;
EthL4HdrProto l4hdr_proto;
if (etqf < 8) {
pkt_type = BIT(11) | etqf;
return pkt_type;
}
net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
if (hasip6 && !(core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) {
eth_ip6_hdr_info *ip6hdr_info = net_rx_pkt_get_ip6_info(pkt);
pkt_type = ip6hdr_info->has_ext_hdrs ? E1000_ADVRXD_PKT_IP6E :
E1000_ADVRXD_PKT_IP6;
} else if (hasip4) {
pkt_type = E1000_ADVRXD_PKT_IP4;
} else {
pkt_type = 0;
}
switch (l4hdr_proto) {
case ETH_L4_HDR_PROTO_TCP:
pkt_type |= E1000_ADVRXD_PKT_TCP;
break;
case ETH_L4_HDR_PROTO_UDP:
pkt_type |= E1000_ADVRXD_PKT_UDP;
break;
case ETH_L4_HDR_PROTO_SCTP:
pkt_type |= E1000_ADVRXD_PKT_SCTP;
break;
default:
break;
}
return pkt_type;
}
static inline void
igb_write_adv_rx_descr(IGBCore *core, union e1000_adv_rx_desc *desc,
struct NetRxPkt *pkt,
const E1000E_RSSInfo *rss_info, uint16_t etqf, bool ts,
uint16_t length)
{
bool hasip4, hasip6;
EthL4HdrProto l4hdr_proto;
uint16_t rss_type = 0, pkt_type;
bool eop = (pkt != NULL);
uint32_t adv_desc_status_error = 0;
memset(&desc->wb, 0, sizeof(desc->wb));
desc->wb.upper.length = cpu_to_le16(length);
igb_build_rx_metadata_common(core, pkt, eop,
&desc->wb.upper.status_error,
&desc->wb.upper.vlan);
if (!eop) {
return;
}
net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) != 0) {
if (rss_info->enabled) {
desc->wb.lower.hi_dword.rss = cpu_to_le32(rss_info->hash);
rss_type = rss_info->type;
trace_igb_rx_metadata_rss(desc->wb.lower.hi_dword.rss, rss_type);
}
} else if (hasip4) {
adv_desc_status_error |= E1000_RXD_STAT_IPIDV;
desc->wb.lower.hi_dword.csum_ip.ip_id =
cpu_to_le16(net_rx_pkt_get_ip_id(pkt));
trace_e1000e_rx_metadata_ip_id(
desc->wb.lower.hi_dword.csum_ip.ip_id);
}
if (ts) {
adv_desc_status_error |= BIT(16);
}
pkt_type = igb_rx_desc_get_packet_type(core, pkt, etqf);
trace_e1000e_rx_metadata_pkt_type(pkt_type);
desc->wb.lower.lo_dword.pkt_info = cpu_to_le16(rss_type | (pkt_type << 4));
desc->wb.upper.status_error |= cpu_to_le32(adv_desc_status_error);
}
static inline void
igb_write_adv_ps_rx_descr(IGBCore *core,
union e1000_adv_rx_desc *desc,
struct NetRxPkt *pkt,
const E1000E_RSSInfo *rss_info,
const E1000ERingInfo *r,
uint16_t etqf,
bool ts,
IGBPacketRxDMAState *pdma_st)
{
size_t pkt_len;
uint16_t hdr_info = 0;
if (pdma_st->do_ps) {
pkt_len = pdma_st->bastate.written[1];
} else {
pkt_len = pdma_st->bastate.written[0] + pdma_st->bastate.written[1];
}
igb_write_adv_rx_descr(core, desc, pkt, rss_info, etqf, ts, pkt_len);
hdr_info = (pdma_st->ps_desc_data.hdr_len << E1000_ADVRXD_HDR_LEN_OFFSET) &
E1000_ADVRXD_ADV_HDR_LEN_MASK;
hdr_info |= pdma_st->ps_desc_data.sph ? E1000_ADVRXD_HDR_SPH : 0;
desc->wb.lower.lo_dword.hdr_info = cpu_to_le16(hdr_info);
desc->wb.upper.status_error |= cpu_to_le32(
pdma_st->ps_desc_data.hbo ? E1000_ADVRXD_ST_ERR_HBO_OFFSET : 0);
}
static inline void
igb_write_rx_descr(IGBCore *core,
union e1000_rx_desc_union *desc,
struct NetRxPkt *pkt,
const E1000E_RSSInfo *rss_info,
uint16_t etqf,
bool ts,
IGBPacketRxDMAState *pdma_st,
const E1000ERingInfo *r)
{
if (igb_rx_use_legacy_descriptor(core)) {
igb_write_lgcy_rx_descr(core, &desc->legacy, pkt, rss_info,
pdma_st->bastate.written[1]);
} else if (igb_rx_use_ps_descriptor(core, r)) {
igb_write_adv_ps_rx_descr(core, &desc->adv, pkt, rss_info, r, etqf, ts,
pdma_st);
} else {
igb_write_adv_rx_descr(core, &desc->adv, pkt, rss_info,
etqf, ts, pdma_st->bastate.written[1]);
}
}
static inline void
igb_pci_dma_write_rx_desc(IGBCore *core, PCIDevice *dev, dma_addr_t addr,
union e1000_rx_desc_union *desc, dma_addr_t len)
{
if (igb_rx_use_legacy_descriptor(core)) {
struct e1000_rx_desc *d = &desc->legacy;
size_t offset = offsetof(struct e1000_rx_desc, status);
uint8_t status = d->status;
d->status &= ~E1000_RXD_STAT_DD;
pci_dma_write(dev, addr, desc, len);
if (status & E1000_RXD_STAT_DD) {
d->status = status;
pci_dma_write(dev, addr + offset, &status, sizeof(status));
}
} else {
union e1000_adv_rx_desc *d = &desc->adv;
size_t offset =
offsetof(union e1000_adv_rx_desc, wb.upper.status_error);
uint32_t status = d->wb.upper.status_error;
d->wb.upper.status_error &= ~E1000_RXD_STAT_DD;
pci_dma_write(dev, addr, desc, len);
if (status & E1000_RXD_STAT_DD) {
d->wb.upper.status_error = status;
pci_dma_write(dev, addr + offset, &status, sizeof(status));
}
}
}
static void
igb_update_rx_stats(IGBCore *core, const E1000ERingInfo *rxi,
size_t pkt_size, size_t pkt_fcs_size)
{
eth_pkt_types_e pkt_type = net_rx_pkt_get_packet_type(core->rx_pkt);
e1000x_update_rx_total_stats(core->mac, pkt_type, pkt_size, pkt_fcs_size);
if (core->mac[MRQC] & 1) {
uint16_t pool = rxi->idx % IGB_NUM_VM_POOLS;
core->mac[PVFGORC0 + (pool * 64)] += pkt_size + 4;
core->mac[PVFGPRC0 + (pool * 64)]++;
if (pkt_type == ETH_PKT_MCAST) {
core->mac[PVFMPRC0 + (pool * 64)]++;
}
}
}
static inline bool
igb_rx_descr_threshold_hit(IGBCore *core, const E1000ERingInfo *rxi)
{
return igb_ring_free_descr_num(core, rxi) ==
((core->mac[E1000_SRRCTL(rxi->idx) >> 2] >> 20) & 31) * 16;
}
static bool
igb_do_ps(IGBCore *core,
const E1000ERingInfo *r,
struct NetRxPkt *pkt,
IGBPacketRxDMAState *pdma_st)
{
bool hasip4, hasip6;
EthL4HdrProto l4hdr_proto;
bool fragment;
bool split_always;
size_t bheader_size;
size_t total_pkt_len;
if (!igb_rx_use_ps_descriptor(core, r)) {
return false;
}
total_pkt_len = net_rx_pkt_get_total_len(pkt);
bheader_size = igb_rxhdrbufsize(core, r);
split_always = igb_rx_ps_descriptor_split_always(core, r);
if (split_always && total_pkt_len <= bheader_size) {
pdma_st->ps_hdr_len = total_pkt_len;
pdma_st->ps_desc_data.hdr_len = total_pkt_len;
return true;
}
net_rx_pkt_get_protocols(pkt, &hasip4, &hasip6, &l4hdr_proto);
if (hasip4) {
fragment = net_rx_pkt_get_ip4_info(pkt)->fragment;
} else if (hasip6) {
fragment = net_rx_pkt_get_ip6_info(pkt)->fragment;
} else {
pdma_st->ps_desc_data.hdr_len = bheader_size;
goto header_not_handled;
}
if (fragment && (core->mac[RFCTL] & E1000_RFCTL_IPFRSP_DIS)) {
pdma_st->ps_desc_data.hdr_len = bheader_size;
goto header_not_handled;
}
/* no header splitting for SCTP */
if (!fragment && (l4hdr_proto == ETH_L4_HDR_PROTO_UDP ||
l4hdr_proto == ETH_L4_HDR_PROTO_TCP)) {
pdma_st->ps_hdr_len = net_rx_pkt_get_l5_hdr_offset(pkt);
} else {
pdma_st->ps_hdr_len = net_rx_pkt_get_l4_hdr_offset(pkt);
}
pdma_st->ps_desc_data.sph = true;
pdma_st->ps_desc_data.hdr_len = pdma_st->ps_hdr_len;
if (pdma_st->ps_hdr_len > bheader_size) {
pdma_st->ps_desc_data.hbo = true;
goto header_not_handled;
}
return true;
header_not_handled:
if (split_always) {
pdma_st->ps_hdr_len = bheader_size;
return true;
}
return false;
}
static void
igb_truncate_to_descriptor_size(IGBPacketRxDMAState *pdma_st, size_t *size)
{
if (pdma_st->do_ps && pdma_st->is_first) {
if (*size > pdma_st->rx_desc_packet_buf_size + pdma_st->ps_hdr_len) {
*size = pdma_st->rx_desc_packet_buf_size + pdma_st->ps_hdr_len;
}
} else {
if (*size > pdma_st->rx_desc_packet_buf_size) {
*size = pdma_st->rx_desc_packet_buf_size;
}
}
}
static inline void
igb_write_hdr_frag_to_rx_buffers(IGBCore *core,
PCIDevice *d,
IGBPacketRxDMAState *pdma_st,
const char *data,
dma_addr_t data_len)
{
assert(data_len <= pdma_st->rx_desc_header_buf_size -
pdma_st->bastate.written[0]);
pci_dma_write(d,
pdma_st->ba[0] + pdma_st->bastate.written[0],
data, data_len);
pdma_st->bastate.written[0] += data_len;
pdma_st->bastate.cur_idx = 1;
}
static void
igb_write_header_to_rx_buffers(IGBCore *core,
struct NetRxPkt *pkt,
PCIDevice *d,
IGBPacketRxDMAState *pdma_st,
size_t *copy_size)
{
size_t iov_copy;
size_t ps_hdr_copied = 0;
if (!pdma_st->is_first) {
/* Leave buffer 0 of each descriptor except first */
/* empty */
pdma_st->bastate.cur_idx = 1;
return;
}
do {
iov_copy = MIN(pdma_st->ps_hdr_len - ps_hdr_copied,
pdma_st->iov->iov_len - pdma_st->iov_ofs);
igb_write_hdr_frag_to_rx_buffers(core, d, pdma_st,
pdma_st->iov->iov_base,
iov_copy);
*copy_size -= iov_copy;
ps_hdr_copied += iov_copy;
pdma_st->iov_ofs += iov_copy;
if (pdma_st->iov_ofs == pdma_st->iov->iov_len) {
pdma_st->iov++;
pdma_st->iov_ofs = 0;
}
} while (ps_hdr_copied < pdma_st->ps_hdr_len);
pdma_st->is_first = false;
}
static void
igb_write_payload_frag_to_rx_buffers(IGBCore *core,
PCIDevice *d,
IGBPacketRxDMAState *pdma_st,
const char *data,
dma_addr_t data_len)
{
while (data_len > 0) {
assert(pdma_st->bastate.cur_idx < IGB_MAX_PS_BUFFERS);
uint32_t cur_buf_bytes_left =
pdma_st->rx_desc_packet_buf_size -
pdma_st->bastate.written[pdma_st->bastate.cur_idx];
uint32_t bytes_to_write = MIN(data_len, cur_buf_bytes_left);
trace_igb_rx_desc_buff_write(
pdma_st->bastate.cur_idx,
pdma_st->ba[pdma_st->bastate.cur_idx],
pdma_st->bastate.written[pdma_st->bastate.cur_idx],
data,
bytes_to_write);
pci_dma_write(d,
pdma_st->ba[pdma_st->bastate.cur_idx] +
pdma_st->bastate.written[pdma_st->bastate.cur_idx],
data, bytes_to_write);
pdma_st->bastate.written[pdma_st->bastate.cur_idx] += bytes_to_write;
data += bytes_to_write;
data_len -= bytes_to_write;
if (pdma_st->bastate.written[pdma_st->bastate.cur_idx] ==
pdma_st->rx_desc_packet_buf_size) {
pdma_st->bastate.cur_idx++;
}
}
}
static void
igb_write_payload_to_rx_buffers(IGBCore *core,
struct NetRxPkt *pkt,
PCIDevice *d,
IGBPacketRxDMAState *pdma_st,
size_t *copy_size)
{
static const uint32_t fcs_pad;
size_t iov_copy;
/* Copy packet payload */
while (*copy_size) {
iov_copy = MIN(*copy_size, pdma_st->iov->iov_len - pdma_st->iov_ofs);
igb_write_payload_frag_to_rx_buffers(core, d,
pdma_st,
pdma_st->iov->iov_base +
pdma_st->iov_ofs,
iov_copy);
*copy_size -= iov_copy;
pdma_st->iov_ofs += iov_copy;
if (pdma_st->iov_ofs == pdma_st->iov->iov_len) {
pdma_st->iov++;
pdma_st->iov_ofs = 0;
}
}
if (pdma_st->desc_offset + pdma_st->desc_size >= pdma_st->total_size) {
/* Simulate FCS checksum presence in the last descriptor */
igb_write_payload_frag_to_rx_buffers(core, d,
pdma_st,
(const char *) &fcs_pad,
e1000x_fcs_len(core->mac));
}
}
static void
igb_write_to_rx_buffers(IGBCore *core,
struct NetRxPkt *pkt,
PCIDevice *d,
IGBPacketRxDMAState *pdma_st)
{
size_t copy_size;
if (!(pdma_st->ba)[1] || (pdma_st->do_ps && !(pdma_st->ba[0]))) {
/* as per intel docs; skip descriptors with null buf addr */
trace_e1000e_rx_null_descriptor();
return;
}
if (pdma_st->desc_offset >= pdma_st->size) {
return;
}
pdma_st->desc_size = pdma_st->total_size - pdma_st->desc_offset;
igb_truncate_to_descriptor_size(pdma_st, &pdma_st->desc_size);
copy_size = pdma_st->size - pdma_st->desc_offset;
igb_truncate_to_descriptor_size(pdma_st, &copy_size);
/* For PS mode copy the packet header first */
if (pdma_st->do_ps) {
igb_write_header_to_rx_buffers(core, pkt, d, pdma_st, &copy_size);
} else {
pdma_st->bastate.cur_idx = 1;
}
igb_write_payload_to_rx_buffers(core, pkt, d, pdma_st, &copy_size);
}
static void
igb_write_packet_to_guest(IGBCore *core, struct NetRxPkt *pkt,
const E1000E_RxRing *rxr,
const E1000E_RSSInfo *rss_info,
uint16_t etqf, bool ts)
{
PCIDevice *d;
dma_addr_t base;
union e1000_rx_desc_union desc;
const E1000ERingInfo *rxi;
size_t rx_desc_len;
IGBPacketRxDMAState pdma_st = {0};
pdma_st.is_first = true;
pdma_st.size = net_rx_pkt_get_total_len(pkt);
pdma_st.total_size = pdma_st.size + e1000x_fcs_len(core->mac);
rxi = rxr->i;
rx_desc_len = core->rx_desc_len;
pdma_st.rx_desc_packet_buf_size = igb_rxbufsize(core, rxi);
pdma_st.rx_desc_header_buf_size = igb_rxhdrbufsize(core, rxi);
pdma_st.iov = net_rx_pkt_get_iovec(pkt);
d = pcie_sriov_get_vf_at_index(core->owner, rxi->idx % 8);
if (!d) {
d = core->owner;
}
pdma_st.do_ps = igb_do_ps(core, rxi, pkt, &pdma_st);
do {
memset(&pdma_st.bastate, 0, sizeof(IGBBAState));
bool is_last = false;
if (igb_ring_empty(core, rxi)) {
return;
}
base = igb_ring_head_descr(core, rxi);
pci_dma_read(d, base, &desc, rx_desc_len);
trace_e1000e_rx_descr(rxi->idx, base, rx_desc_len);
igb_read_rx_descr(core, &desc, &pdma_st, rxi);
igb_write_to_rx_buffers(core, pkt, d, &pdma_st);
pdma_st.desc_offset += pdma_st.desc_size;
if (pdma_st.desc_offset >= pdma_st.total_size) {
is_last = true;
}
igb_write_rx_descr(core, &desc,
is_last ? pkt : NULL,
rss_info,
etqf, ts,
&pdma_st,
rxi);
igb_pci_dma_write_rx_desc(core, d, base, &desc, rx_desc_len);
igb_ring_advance(core, rxi, rx_desc_len / E1000_MIN_RX_DESC_LEN);
} while (pdma_st.desc_offset < pdma_st.total_size);
igb_update_rx_stats(core, rxi, pdma_st.size, pdma_st.total_size);
}
static bool
igb_rx_strip_vlan(IGBCore *core, const E1000ERingInfo *rxi)
{
if (core->mac[MRQC] & 1) {
uint16_t pool = rxi->idx % IGB_NUM_VM_POOLS;
/* Sec 7.10.3.8: CTRL.VME is ignored, only VMOLR/RPLOLR is used */
return (net_rx_pkt_get_packet_type(core->rx_pkt) == ETH_PKT_MCAST) ?
core->mac[RPLOLR] & E1000_RPLOLR_STRVLAN :
core->mac[VMOLR0 + pool] & E1000_VMOLR_STRVLAN;
}
return e1000x_vlan_enabled(core->mac);
}
static inline void
igb_rx_fix_l4_csum(IGBCore *core, struct NetRxPkt *pkt)
{
struct virtio_net_hdr *vhdr = net_rx_pkt_get_vhdr(pkt);
if (vhdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
net_rx_pkt_fix_l4_csum(pkt);
}
}
ssize_t
igb_receive_iov(IGBCore *core, const struct iovec *iov, int iovcnt)
{
return igb_receive_internal(core, iov, iovcnt, core->has_vnet, NULL);
}
static ssize_t
igb_receive_internal(IGBCore *core, const struct iovec *iov, int iovcnt,
bool has_vnet, bool *external_tx)
{
uint16_t queues = 0;
uint32_t causes = 0;
uint32_t ecauses = 0;
union {
L2Header l2_header;
uint8_t octets[ETH_ZLEN];
} buf;
struct iovec min_iov;
size_t size, orig_size;
size_t iov_ofs = 0;
E1000E_RxRing rxr;
E1000E_RSSInfo rss_info;
uint16_t etqf;
bool ts;
size_t total_size;
int strip_vlan_index;
int i;
trace_e1000e_rx_receive_iov(iovcnt);
if (external_tx) {
*external_tx = true;
}
if (!e1000x_hw_rx_enabled(core->mac)) {
return -1;
}
/* Pull virtio header in */
if (has_vnet) {
net_rx_pkt_set_vhdr_iovec(core->rx_pkt, iov, iovcnt);
iov_ofs = sizeof(struct virtio_net_hdr);
} else {
net_rx_pkt_unset_vhdr(core->rx_pkt);
}
orig_size = iov_size(iov, iovcnt);
size = orig_size - iov_ofs;
/* Pad to minimum Ethernet frame length */
if (size < sizeof(buf)) {
iov_to_buf(iov, iovcnt, iov_ofs, &buf, size);
memset(&buf.octets[size], 0, sizeof(buf) - size);
e1000x_inc_reg_if_not_full(core->mac, RUC);
min_iov.iov_base = &buf;
min_iov.iov_len = size = sizeof(buf);
iovcnt = 1;
iov = &min_iov;
iov_ofs = 0;
} else {
iov_to_buf(iov, iovcnt, iov_ofs, &buf, sizeof(buf.l2_header));
}
net_rx_pkt_set_packet_type(core->rx_pkt,
get_eth_packet_type(&buf.l2_header.eth));
net_rx_pkt_set_protocols(core->rx_pkt, iov, iovcnt, iov_ofs);
queues = igb_receive_assign(core, iov, iovcnt, iov_ofs,
&buf.l2_header, size,
&rss_info, &etqf, &ts, external_tx);
if (!queues) {
trace_e1000e_rx_flt_dropped();
return orig_size;
}
for (i = 0; i < IGB_NUM_QUEUES; i++) {
if (!(queues & BIT(i)) ||
!(core->mac[RXDCTL0 + (i * 16)] & E1000_RXDCTL_QUEUE_ENABLE)) {
continue;
}
igb_rx_ring_init(core, &rxr, i);
if (!igb_rx_strip_vlan(core, rxr.i)) {
strip_vlan_index = -1;
} else if (core->mac[CTRL_EXT] & BIT(26)) {
strip_vlan_index = 1;
} else {
strip_vlan_index = 0;
}
net_rx_pkt_attach_iovec_ex(core->rx_pkt, iov, iovcnt, iov_ofs,
strip_vlan_index,
core->mac[VET] & 0xffff,
core->mac[VET] >> 16);
total_size = net_rx_pkt_get_total_len(core->rx_pkt) +
e1000x_fcs_len(core->mac);
if (!igb_has_rxbufs(core, rxr.i, total_size)) {
causes |= E1000_ICS_RXO;
trace_e1000e_rx_not_written_to_guest(rxr.i->idx);
continue;
}
causes |= E1000_ICR_RXDW;
igb_rx_fix_l4_csum(core, core->rx_pkt);
igb_write_packet_to_guest(core, core->rx_pkt, &rxr, &rss_info, etqf, ts);
/* Check if receive descriptor minimum threshold hit */
if (igb_rx_descr_threshold_hit(core, rxr.i)) {
causes |= E1000_ICS_RXDMT0;
}
ecauses |= igb_rx_wb_eic(core, rxr.i->idx);
trace_e1000e_rx_written_to_guest(rxr.i->idx);
}
trace_e1000e_rx_interrupt_set(causes);
igb_raise_interrupts(core, EICR, ecauses);
igb_raise_interrupts(core, ICR, causes);
return orig_size;
}
static inline bool
igb_have_autoneg(IGBCore *core)
{
return core->phy[MII_BMCR] & MII_BMCR_AUTOEN;
}
static void igb_update_flowctl_status(IGBCore *core)
{
if (igb_have_autoneg(core) && core->phy[MII_BMSR] & MII_BMSR_AN_COMP) {
trace_e1000e_link_autoneg_flowctl(true);
core->mac[CTRL] |= E1000_CTRL_TFCE | E1000_CTRL_RFCE;
} else {
trace_e1000e_link_autoneg_flowctl(false);
}
}
static inline void
igb_link_down(IGBCore *core)
{
e1000x_update_regs_on_link_down(core->mac, core->phy);
igb_update_flowctl_status(core);
}
static inline void
igb_set_phy_ctrl(IGBCore *core, uint16_t val)
{
/* bits 0-5 reserved; MII_BMCR_[ANRESTART,RESET] are self clearing */
core->phy[MII_BMCR] = val & ~(0x3f | MII_BMCR_RESET | MII_BMCR_ANRESTART);
if ((val & MII_BMCR_ANRESTART) && igb_have_autoneg(core)) {
e1000x_restart_autoneg(core->mac, core->phy, core->autoneg_timer);
}
}
void igb_core_set_link_status(IGBCore *core)
{
NetClientState *nc = qemu_get_queue(core->owner_nic);
uint32_t old_status = core->mac[STATUS];
trace_e1000e_link_status_changed(nc->link_down ? false : true);
if (nc->link_down) {
e1000x_update_regs_on_link_down(core->mac, core->phy);
} else {
if (igb_have_autoneg(core) &&
!(core->phy[MII_BMSR] & MII_BMSR_AN_COMP)) {
e1000x_restart_autoneg(core->mac, core->phy,
core->autoneg_timer);
} else {
e1000x_update_regs_on_link_up(core->mac, core->phy);
igb_start_recv(core);
}
}
if (core->mac[STATUS] != old_status) {
igb_raise_interrupts(core, ICR, E1000_ICR_LSC);
}
}
static void
igb_set_ctrl(IGBCore *core, int index, uint32_t val)
{
trace_e1000e_core_ctrl_write(index, val);
/* RST is self clearing */
core->mac[CTRL] = val & ~E1000_CTRL_RST;
core->mac[CTRL_DUP] = core->mac[CTRL];
trace_e1000e_link_set_params(
!!(val & E1000_CTRL_ASDE),
(val & E1000_CTRL_SPD_SEL) >> E1000_CTRL_SPD_SHIFT,
!!(val & E1000_CTRL_FRCSPD),
!!(val & E1000_CTRL_FRCDPX),
!!(val & E1000_CTRL_RFCE),
!!(val & E1000_CTRL_TFCE));
if (val & E1000_CTRL_RST) {
trace_e1000e_core_ctrl_sw_reset();
igb_reset(core, true);
}
if (val & E1000_CTRL_PHY_RST) {
trace_e1000e_core_ctrl_phy_reset();
core->mac[STATUS] |= E1000_STATUS_PHYRA;
}
}
static void
igb_set_rfctl(IGBCore *core, int index, uint32_t val)
{
trace_e1000e_rx_set_rfctl(val);
if (!(val & E1000_RFCTL_ISCSI_DIS)) {
trace_e1000e_wrn_iscsi_filtering_not_supported();
}
if (!(val & E1000_RFCTL_NFSW_DIS)) {
trace_e1000e_wrn_nfsw_filtering_not_supported();
}
if (!(val & E1000_RFCTL_NFSR_DIS)) {
trace_e1000e_wrn_nfsr_filtering_not_supported();
}
core->mac[RFCTL] = val;
}
static void
igb_calc_rxdesclen(IGBCore *core)
{
if (igb_rx_use_legacy_descriptor(core)) {
core->rx_desc_len = sizeof(struct e1000_rx_desc);
} else {
core->rx_desc_len = sizeof(union e1000_adv_rx_desc);
}
trace_e1000e_rx_desc_len(core->rx_desc_len);
}
static void
igb_set_rx_control(IGBCore *core, int index, uint32_t val)
{
core->mac[RCTL] = val;
trace_e1000e_rx_set_rctl(core->mac[RCTL]);
if (val & E1000_RCTL_DTYP_MASK) {
qemu_log_mask(LOG_GUEST_ERROR,
"igb: RCTL.DTYP must be zero for compatibility");
}
if (val & E1000_RCTL_EN) {
igb_calc_rxdesclen(core);
igb_start_recv(core);
}
}
static inline bool
igb_postpone_interrupt(IGBIntrDelayTimer *timer)
{
if (timer->running) {
trace_e1000e_irq_postponed_by_xitr(timer->delay_reg << 2);
return true;
}
if (timer->core->mac[timer->delay_reg] != 0) {
igb_intrmgr_rearm_timer(timer);
}
return false;
}
static inline bool
igb_eitr_should_postpone(IGBCore *core, int idx)
{
return igb_postpone_interrupt(&core->eitr[idx]);
}
static void igb_send_msix(IGBCore *core, uint32_t causes)
{
int vector;
for (vector = 0; vector < IGB_INTR_NUM; ++vector) {
if ((causes & BIT(vector)) && !igb_eitr_should_postpone(core, vector)) {
trace_e1000e_irq_msix_notify_vec(vector);
igb_msix_notify(core, vector);
}
}
}
static inline void
igb_fix_icr_asserted(IGBCore *core)
{
core->mac[ICR] &= ~E1000_ICR_ASSERTED;
if (core->mac[ICR]) {
core->mac[ICR] |= E1000_ICR_ASSERTED;
}
trace_e1000e_irq_fix_icr_asserted(core->mac[ICR]);
}
static void igb_raise_interrupts(IGBCore *core, size_t index, uint32_t causes)
{
uint32_t old_causes = core->mac[ICR] & core->mac[IMS];
uint32_t old_ecauses = core->mac[EICR] & core->mac[EIMS];
uint32_t raised_causes;
uint32_t raised_ecauses;
uint32_t int_alloc;
trace_e1000e_irq_set(index << 2,
core->mac[index], core->mac[index] | causes);
core->mac[index] |= causes;
if (core->mac[GPIE] & E1000_GPIE_MSIX_MODE) {
raised_causes = core->mac[ICR] & core->mac[IMS] & ~old_causes;
if (raised_causes & E1000_ICR_DRSTA) {
int_alloc = core->mac[IVAR_MISC] & 0xff;
if (int_alloc & E1000_IVAR_VALID) {
core->mac[EICR] |= BIT(int_alloc & 0x1f);
}
}
/* Check if other bits (excluding the TCP Timer) are enabled. */
if (raised_causes & ~E1000_ICR_DRSTA) {
int_alloc = (core->mac[IVAR_MISC] >> 8) & 0xff;
if (int_alloc & E1000_IVAR_VALID) {
core->mac[EICR] |= BIT(int_alloc & 0x1f);
}
}
raised_ecauses = core->mac[EICR] & core->mac[EIMS] & ~old_ecauses;
if (!raised_ecauses) {
return;
}
igb_send_msix(core, raised_ecauses);
} else {
igb_fix_icr_asserted(core);
raised_causes = core->mac[ICR] & core->mac[IMS] & ~old_causes;
if (!raised_causes) {
return;
}
core->mac[EICR] |= (raised_causes & E1000_ICR_DRSTA) | E1000_EICR_OTHER;
if (msix_enabled(core->owner)) {
trace_e1000e_irq_msix_notify_vec(0);
msix_notify(core->owner, 0);
} else if (msi_enabled(core->owner)) {
trace_e1000e_irq_msi_notify(raised_causes);
msi_notify(core->owner, 0);
} else {
igb_raise_legacy_irq(core);
}
}
}
static void igb_lower_interrupts(IGBCore *core, size_t index, uint32_t causes)
{
trace_e1000e_irq_clear(index << 2,
core->mac[index], core->mac[index] & ~causes);
core->mac[index] &= ~causes;
trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS],
core->mac[ICR], core->mac[IMS]);
if (!(core->mac[ICR] & core->mac[IMS]) &&
!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE)) {
core->mac[EICR] &= ~E1000_EICR_OTHER;
if (!msix_enabled(core->owner) && !msi_enabled(core->owner)) {
igb_lower_legacy_irq(core);
}
}
}
static void igb_set_eics(IGBCore *core, int index, uint32_t val)
{
bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
trace_igb_irq_write_eics(val, msix);
igb_raise_interrupts(core, EICR, val & mask);
}
static void igb_set_eims(IGBCore *core, int index, uint32_t val)
{
bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
trace_igb_irq_write_eims(val, msix);
igb_raise_interrupts(core, EIMS, val & mask);
}
static void mailbox_interrupt_to_vf(IGBCore *core, uint16_t vfn)
{
uint32_t ent = core->mac[VTIVAR_MISC + vfn];
uint32_t causes;
if ((ent & E1000_IVAR_VALID)) {
causes = (ent & 0x3) << (22 - vfn * IGBVF_MSIX_VEC_NUM);
igb_raise_interrupts(core, EICR, causes);
}
}
static void mailbox_interrupt_to_pf(IGBCore *core)
{
igb_raise_interrupts(core, ICR, E1000_ICR_VMMB);
}
static void igb_set_pfmailbox(IGBCore *core, int index, uint32_t val)
{
uint16_t vfn = index - P2VMAILBOX0;
trace_igb_set_pfmailbox(vfn, val);
if (val & E1000_P2VMAILBOX_STS) {
core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_PFSTS;
mailbox_interrupt_to_vf(core, vfn);
}
if (val & E1000_P2VMAILBOX_ACK) {
core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_PFACK;
mailbox_interrupt_to_vf(core, vfn);
}
/* Buffer Taken by PF (can be set only if the VFU is cleared). */
if (val & E1000_P2VMAILBOX_PFU) {
if (!(core->mac[index] & E1000_P2VMAILBOX_VFU)) {
core->mac[index] |= E1000_P2VMAILBOX_PFU;
core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_PFU;
}
} else {
core->mac[index] &= ~E1000_P2VMAILBOX_PFU;
core->mac[V2PMAILBOX0 + vfn] &= ~E1000_V2PMAILBOX_PFU;
}
if (val & E1000_P2VMAILBOX_RVFU) {
core->mac[V2PMAILBOX0 + vfn] &= ~E1000_V2PMAILBOX_VFU;
core->mac[MBVFICR] &= ~((E1000_MBVFICR_VFACK_VF1 << vfn) |
(E1000_MBVFICR_VFREQ_VF1 << vfn));
}
}
static void igb_set_vfmailbox(IGBCore *core, int index, uint32_t val)
{
uint16_t vfn = index - V2PMAILBOX0;
trace_igb_set_vfmailbox(vfn, val);
if (val & E1000_V2PMAILBOX_REQ) {
core->mac[MBVFICR] |= E1000_MBVFICR_VFREQ_VF1 << vfn;
mailbox_interrupt_to_pf(core);
}
if (val & E1000_V2PMAILBOX_ACK) {
core->mac[MBVFICR] |= E1000_MBVFICR_VFACK_VF1 << vfn;
mailbox_interrupt_to_pf(core);
}
/* Buffer Taken by VF (can be set only if the PFU is cleared). */
if (val & E1000_V2PMAILBOX_VFU) {
if (!(core->mac[index] & E1000_V2PMAILBOX_PFU)) {
core->mac[index] |= E1000_V2PMAILBOX_VFU;
core->mac[P2VMAILBOX0 + vfn] |= E1000_P2VMAILBOX_VFU;
}
} else {
core->mac[index] &= ~E1000_V2PMAILBOX_VFU;
core->mac[P2VMAILBOX0 + vfn] &= ~E1000_P2VMAILBOX_VFU;
}
}
void igb_core_vf_reset(IGBCore *core, uint16_t vfn)
{
uint16_t qn0 = vfn;
uint16_t qn1 = vfn + IGB_NUM_VM_POOLS;
trace_igb_core_vf_reset(vfn);
/* disable Rx and Tx for the VF*/
core->mac[RXDCTL0 + (qn0 * 16)] &= ~E1000_RXDCTL_QUEUE_ENABLE;
core->mac[RXDCTL0 + (qn1 * 16)] &= ~E1000_RXDCTL_QUEUE_ENABLE;
core->mac[TXDCTL0 + (qn0 * 16)] &= ~E1000_TXDCTL_QUEUE_ENABLE;
core->mac[TXDCTL0 + (qn1 * 16)] &= ~E1000_TXDCTL_QUEUE_ENABLE;
core->mac[VFRE] &= ~BIT(vfn);
core->mac[VFTE] &= ~BIT(vfn);
/* indicate VF reset to PF */
core->mac[VFLRE] |= BIT(vfn);
/* VFLRE and mailbox use the same interrupt cause */
mailbox_interrupt_to_pf(core);
}
static void igb_w1c(IGBCore *core, int index, uint32_t val)
{
core->mac[index] &= ~val;
}
static void igb_set_eimc(IGBCore *core, int index, uint32_t val)
{
bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
trace_igb_irq_write_eimc(val, msix);
/* Interrupts are disabled via a write to EIMC and reflected in EIMS. */
igb_lower_interrupts(core, EIMS, val & mask);
}
static void igb_set_eiac(IGBCore *core, int index, uint32_t val)
{
bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
if (msix) {
trace_igb_irq_write_eiac(val);
/*
* TODO: When using IOV, the bits that correspond to MSI-X vectors
* that are assigned to a VF are read-only.
*/
core->mac[EIAC] |= (val & E1000_EICR_MSIX_MASK);
}
}
static void igb_set_eiam(IGBCore *core, int index, uint32_t val)
{
bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
/*
* TODO: When using IOV, the bits that correspond to MSI-X vectors that
* are assigned to a VF are read-only.
*/
core->mac[EIAM] |=
~(val & (msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK));
trace_igb_irq_write_eiam(val, msix);
}
static void igb_set_eicr(IGBCore *core, int index, uint32_t val)
{
bool msix = !!(core->mac[GPIE] & E1000_GPIE_MSIX_MODE);
/*
* TODO: In IOV mode, only bit zero of this vector is available for the PF
* function.
*/
uint32_t mask = msix ? E1000_EICR_MSIX_MASK : E1000_EICR_LEGACY_MASK;
trace_igb_irq_write_eicr(val, msix);
igb_lower_interrupts(core, EICR, val & mask);
}
static void igb_set_vtctrl(IGBCore *core, int index, uint32_t val)
{
uint16_t vfn;
if (val & E1000_CTRL_RST) {
vfn = (index - PVTCTRL0) / 0x40;
igb_core_vf_reset(core, vfn);
}
}
static void igb_set_vteics(IGBCore *core, int index, uint32_t val)
{
uint16_t vfn = (index - PVTEICS0) / 0x40;
core->mac[index] = val;
igb_set_eics(core, EICS, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
}
static void igb_set_vteims(IGBCore *core, int index, uint32_t val)
{
uint16_t vfn = (index - PVTEIMS0) / 0x40;
core->mac[index] = val;
igb_set_eims(core, EIMS, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
}
static void igb_set_vteimc(IGBCore *core, int index, uint32_t val)
{
uint16_t vfn = (index - PVTEIMC0) / 0x40;
core->mac[index] = val;
igb_set_eimc(core, EIMC, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
}
static void igb_set_vteiac(IGBCore *core, int index, uint32_t val)
{
uint16_t vfn = (index - PVTEIAC0) / 0x40;
core->mac[index] = val;
igb_set_eiac(core, EIAC, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
}
static void igb_set_vteiam(IGBCore *core, int index, uint32_t val)
{
uint16_t vfn = (index - PVTEIAM0) / 0x40;
core->mac[index] = val;
igb_set_eiam(core, EIAM, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
}
static void igb_set_vteicr(IGBCore *core, int index, uint32_t val)
{
uint16_t vfn = (index - PVTEICR0) / 0x40;
core->mac[index] = val;
igb_set_eicr(core, EICR, (val & 0x7) << (22 - vfn * IGBVF_MSIX_VEC_NUM));
}
static void igb_set_vtivar(IGBCore *core, int index, uint32_t val)
{
uint16_t vfn = (index - VTIVAR);
uint16_t qn = vfn;
uint8_t ent;
int n;
core->mac[index] = val;
/* Get assigned vector associated with queue Rx#0. */
if ((val & E1000_IVAR_VALID)) {
n = igb_ivar_entry_rx(qn);
ent = E1000_IVAR_VALID | (24 - vfn * IGBVF_MSIX_VEC_NUM - (2 - (val & 0x7)));
core->mac[IVAR0 + n / 4] |= ent << 8 * (n % 4);
}
/* Get assigned vector associated with queue Tx#0 */
ent = val >> 8;
if ((ent & E1000_IVAR_VALID)) {
n = igb_ivar_entry_tx(qn);
ent = E1000_IVAR_VALID | (24 - vfn * IGBVF_MSIX_VEC_NUM - (2 - (ent & 0x7)));
core->mac[IVAR0 + n / 4] |= ent << 8 * (n % 4);
}
/*
* Ignoring assigned vectors associated with queues Rx#1 and Tx#1 for now.
*/
}
static inline void
igb_autoneg_timer(void *opaque)
{
IGBCore *core = opaque;
if (!qemu_get_queue(core->owner_nic)->link_down) {
e1000x_update_regs_on_autoneg_done(core->mac, core->phy);
igb_start_recv(core);
igb_update_flowctl_status(core);
/* signal link status change to the guest */
igb_raise_interrupts(core, ICR, E1000_ICR_LSC);
}
}
static inline uint16_t
igb_get_reg_index_with_offset(const uint16_t *mac_reg_access, hwaddr addr)
{
uint16_t index = (addr & 0x1ffff) >> 2;
return index + (mac_reg_access[index] & 0xfffe);
}
static const char igb_phy_regcap[MAX_PHY_REG_ADDRESS + 1] = {
[MII_BMCR] = PHY_RW,
[MII_BMSR] = PHY_R,
[MII_PHYID1] = PHY_R,
[MII_PHYID2] = PHY_R,
[MII_ANAR] = PHY_RW,
[MII_ANLPAR] = PHY_R,
[MII_ANER] = PHY_R,
[MII_ANNP] = PHY_RW,
[MII_ANLPRNP] = PHY_R,
[MII_CTRL1000] = PHY_RW,
[MII_STAT1000] = PHY_R,
[MII_EXTSTAT] = PHY_R,
[IGP01E1000_PHY_PORT_CONFIG] = PHY_RW,
[IGP01E1000_PHY_PORT_STATUS] = PHY_R,
[IGP01E1000_PHY_PORT_CTRL] = PHY_RW,
[IGP01E1000_PHY_LINK_HEALTH] = PHY_R,
[IGP02E1000_PHY_POWER_MGMT] = PHY_RW,
[IGP01E1000_PHY_PAGE_SELECT] = PHY_W
};
static void
igb_phy_reg_write(IGBCore *core, uint32_t addr, uint16_t data)
{
assert(addr <= MAX_PHY_REG_ADDRESS);
if (addr == MII_BMCR) {
igb_set_phy_ctrl(core, data);
} else {
core->phy[addr] = data;
}
}
static void
igb_set_mdic(IGBCore *core, int index, uint32_t val)
{
uint32_t data = val & E1000_MDIC_DATA_MASK;
uint32_t addr = ((val & E1000_MDIC_REG_MASK) >> E1000_MDIC_REG_SHIFT);
if ((val & E1000_MDIC_PHY_MASK) >> E1000_MDIC_PHY_SHIFT != 1) { /* phy # */
val = core->mac[MDIC] | E1000_MDIC_ERROR;
} else if (val & E1000_MDIC_OP_READ) {
if (!(igb_phy_regcap[addr] & PHY_R)) {
trace_igb_core_mdic_read_unhandled(addr);
val |= E1000_MDIC_ERROR;
} else {
val = (val ^ data) | core->phy[addr];
trace_igb_core_mdic_read(addr, val);
}
} else if (val & E1000_MDIC_OP_WRITE) {
if (!(igb_phy_regcap[addr] & PHY_W)) {
trace_igb_core_mdic_write_unhandled(addr);
val |= E1000_MDIC_ERROR;
} else {
trace_igb_core_mdic_write(addr, data);
igb_phy_reg_write(core, addr, data);
}
}
core->mac[MDIC] = val | E1000_MDIC_READY;
if (val & E1000_MDIC_INT_EN) {
igb_raise_interrupts(core, ICR, E1000_ICR_MDAC);
}
}
static void
igb_set_rdt(IGBCore *core, int index, uint32_t val)
{
core->mac[index] = val & 0xffff;
trace_e1000e_rx_set_rdt(igb_mq_queue_idx(RDT0, index), val);
igb_start_recv(core);
}
static void
igb_set_status(IGBCore *core, int index, uint32_t val)
{
if ((val & E1000_STATUS_PHYRA) == 0) {
core->mac[index] &= ~E1000_STATUS_PHYRA;
}
}
static void
igb_set_ctrlext(IGBCore *core, int index, uint32_t val)
{
trace_igb_link_set_ext_params(!!(val & E1000_CTRL_EXT_ASDCHK),
!!(val & E1000_CTRL_EXT_SPD_BYPS),
!!(val & E1000_CTRL_EXT_PFRSTD));
/* Zero self-clearing bits */
val &= ~(E1000_CTRL_EXT_ASDCHK | E1000_CTRL_EXT_EE_RST);
core->mac[CTRL_EXT] = val;
if (core->mac[CTRL_EXT] & E1000_CTRL_EXT_PFRSTD) {
for (int vfn = 0; vfn < IGB_MAX_VF_FUNCTIONS; vfn++) {
core->mac[V2PMAILBOX0 + vfn] &= ~E1000_V2PMAILBOX_RSTI;
core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_RSTD;
}
}
}
static void
igb_set_pbaclr(IGBCore *core, int index, uint32_t val)
{
int i;
core->mac[PBACLR] = val & E1000_PBACLR_VALID_MASK;
if (!msix_enabled(core->owner)) {
return;
}
for (i = 0; i < IGB_INTR_NUM; i++) {
if (core->mac[PBACLR] & BIT(i)) {
msix_clr_pending(core->owner, i);
}
}
}
static void
igb_set_fcrth(IGBCore *core, int index, uint32_t val)
{
core->mac[FCRTH] = val & 0xFFF8;
}
static void
igb_set_fcrtl(IGBCore *core, int index, uint32_t val)
{
core->mac[FCRTL] = val & 0x8000FFF8;
}
#define IGB_LOW_BITS_SET_FUNC(num) \
static void \
igb_set_##num##bit(IGBCore *core, int index, uint32_t val) \
{ \
core->mac[index] = val & (BIT(num) - 1); \
}
IGB_LOW_BITS_SET_FUNC(4)
IGB_LOW_BITS_SET_FUNC(13)
IGB_LOW_BITS_SET_FUNC(16)
static void
igb_set_dlen(IGBCore *core, int index, uint32_t val)
{
core->mac[index] = val & 0xffff0;
}
static void
igb_set_dbal(IGBCore *core, int index, uint32_t val)
{
core->mac[index] = val & E1000_XDBAL_MASK;
}
static void
igb_set_tdt(IGBCore *core, int index, uint32_t val)
{
IGB_TxRing txr;
int qn = igb_mq_queue_idx(TDT0, index);
core->mac[index] = val & 0xffff;
igb_tx_ring_init(core, &txr, qn);
igb_start_xmit(core, &txr);
}
static void
igb_set_ics(IGBCore *core, int index, uint32_t val)
{
trace_e1000e_irq_write_ics(val);
igb_raise_interrupts(core, ICR, val);
}
static void
igb_set_imc(IGBCore *core, int index, uint32_t val)
{
trace_e1000e_irq_ims_clear_set_imc(val);
igb_lower_interrupts(core, IMS, val);
}
static void
igb_set_ims(IGBCore *core, int index, uint32_t val)
{
igb_raise_interrupts(core, IMS, val & 0x77D4FBFD);
}
static void igb_nsicr(IGBCore *core)
{
/*
* If GPIE.NSICR = 0, then the clear of IMS will occur only if at
* least one bit is set in the IMS and there is a true interrupt as
* reflected in ICR.INTA.
*/
if ((core->mac[GPIE] & E1000_GPIE_NSICR) ||
(core->mac[IMS] && (core->mac[ICR] & E1000_ICR_INT_ASSERTED))) {
igb_lower_interrupts(core, IMS, core->mac[IAM]);
}
}
static void igb_set_icr(IGBCore *core, int index, uint32_t val)
{
igb_nsicr(core);
igb_lower_interrupts(core, ICR, val);
}
static uint32_t
igb_mac_readreg(IGBCore *core, int index)
{
return core->mac[index];
}
static uint32_t
igb_mac_ics_read(IGBCore *core, int index)
{
trace_e1000e_irq_read_ics(core->mac[ICS]);
return core->mac[ICS];
}
static uint32_t
igb_mac_ims_read(IGBCore *core, int index)
{
trace_e1000e_irq_read_ims(core->mac[IMS]);
return core->mac[IMS];
}
static uint32_t
igb_mac_swsm_read(IGBCore *core, int index)
{
uint32_t val = core->mac[SWSM];
core->mac[SWSM] = val | E1000_SWSM_SMBI;
return val;
}
static uint32_t
igb_mac_eitr_read(IGBCore *core, int index)
{
return core->eitr_guest_value[index - EITR0];
}
static uint32_t igb_mac_vfmailbox_read(IGBCore *core, int index)
{
uint32_t val = core->mac[index];
core->mac[index] &= ~(E1000_V2PMAILBOX_PFSTS | E1000_V2PMAILBOX_PFACK |
E1000_V2PMAILBOX_RSTD);
return val;
}
static uint32_t
igb_mac_icr_read(IGBCore *core, int index)
{
uint32_t ret = core->mac[ICR];
if (core->mac[GPIE] & E1000_GPIE_NSICR) {
trace_igb_irq_icr_clear_gpie_nsicr();
igb_lower_interrupts(core, ICR, 0xffffffff);
} else if (core->mac[IMS] == 0) {
trace_e1000e_irq_icr_clear_zero_ims();
igb_lower_interrupts(core, ICR, 0xffffffff);
} else if (core->mac[ICR] & E1000_ICR_INT_ASSERTED) {
igb_lower_interrupts(core, ICR, 0xffffffff);
} else if (!msix_enabled(core->owner)) {
trace_e1000e_irq_icr_clear_nonmsix_icr_read();
igb_lower_interrupts(core, ICR, 0xffffffff);
}
igb_nsicr(core);
return ret;
}
static uint32_t
igb_mac_read_clr4(IGBCore *core, int index)
{
uint32_t ret = core->mac[index];
core->mac[index] = 0;
return ret;
}
static uint32_t
igb_mac_read_clr8(IGBCore *core, int index)
{
uint32_t ret = core->mac[index];
core->mac[index] = 0;
core->mac[index - 1] = 0;
return ret;
}
static uint32_t
igb_get_ctrl(IGBCore *core, int index)
{
uint32_t val = core->mac[CTRL];
trace_e1000e_link_read_params(
!!(val & E1000_CTRL_ASDE),
(val & E1000_CTRL_SPD_SEL) >> E1000_CTRL_SPD_SHIFT,
!!(val & E1000_CTRL_FRCSPD),
!!(val & E1000_CTRL_FRCDPX),
!!(val & E1000_CTRL_RFCE),
!!(val & E1000_CTRL_TFCE));
return val;
}
static uint32_t igb_get_status(IGBCore *core, int index)
{
uint32_t res = core->mac[STATUS];
uint16_t num_vfs = pcie_sriov_num_vfs(core->owner);
if (core->mac[CTRL] & E1000_CTRL_FRCDPX) {
res |= (core->mac[CTRL] & E1000_CTRL_FD) ? E1000_STATUS_FD : 0;
} else {
res |= E1000_STATUS_FD;
}
if ((core->mac[CTRL] & E1000_CTRL_FRCSPD) ||
(core->mac[CTRL_EXT] & E1000_CTRL_EXT_SPD_BYPS)) {
switch (core->mac[CTRL] & E1000_CTRL_SPD_SEL) {
case E1000_CTRL_SPD_10:
res |= E1000_STATUS_SPEED_10;
break;
case E1000_CTRL_SPD_100:
res |= E1000_STATUS_SPEED_100;
break;
case E1000_CTRL_SPD_1000:
default:
res |= E1000_STATUS_SPEED_1000;
break;
}
} else {
res |= E1000_STATUS_SPEED_1000;
}
if (num_vfs) {
res |= num_vfs << E1000_STATUS_NUM_VFS_SHIFT;
res |= E1000_STATUS_IOV_MODE;
}
if (!(core->mac[CTRL] & E1000_CTRL_GIO_MASTER_DISABLE)) {
res |= E1000_STATUS_GIO_MASTER_ENABLE;
}
return res;
}
static void
igb_mac_writereg(IGBCore *core, int index, uint32_t val)
{
core->mac[index] = val;
}
static void
igb_mac_setmacaddr(IGBCore *core, int index, uint32_t val)
{
uint32_t macaddr[2];
core->mac[index] = val;
macaddr[0] = cpu_to_le32(core->mac[RA]);
macaddr[1] = cpu_to_le32(core->mac[RA + 1]);
qemu_format_nic_info_str(qemu_get_queue(core->owner_nic),
(uint8_t *) macaddr);
trace_e1000e_mac_set_sw(MAC_ARG(macaddr));
}
static void
igb_set_eecd(IGBCore *core, int index, uint32_t val)
{
static const uint32_t ro_bits = E1000_EECD_PRES |
E1000_EECD_AUTO_RD |
E1000_EECD_SIZE_EX_MASK;
core->mac[EECD] = (core->mac[EECD] & ro_bits) | (val & ~ro_bits);
}
static void
igb_set_eerd(IGBCore *core, int index, uint32_t val)
{
uint32_t addr = (val >> E1000_EERW_ADDR_SHIFT) & E1000_EERW_ADDR_MASK;
uint32_t flags = 0;
uint32_t data = 0;
if ((addr < IGB_EEPROM_SIZE) && (val & E1000_EERW_START)) {
data = core->eeprom[addr];
flags = E1000_EERW_DONE;
}
core->mac[EERD] = flags |
(addr << E1000_EERW_ADDR_SHIFT) |
(data << E1000_EERW_DATA_SHIFT);
}
static void
igb_set_eitr(IGBCore *core, int index, uint32_t val)
{
uint32_t eitr_num = index - EITR0;
trace_igb_irq_eitr_set(eitr_num, val);
core->eitr_guest_value[eitr_num] = val & ~E1000_EITR_CNT_IGNR;
core->mac[index] = val & 0x7FFE;
}
static void
igb_update_rx_offloads(IGBCore *core)
{
int cso_state = igb_rx_l4_cso_enabled(core);
trace_e1000e_rx_set_cso(cso_state);
if (core->has_vnet) {
qemu_set_offload(qemu_get_queue(core->owner_nic)->peer,
cso_state, 0, 0, 0, 0, 0, 0);
}
}
static void
igb_set_rxcsum(IGBCore *core, int index, uint32_t val)
{
core->mac[RXCSUM] = val;
igb_update_rx_offloads(core);
}
static void
igb_set_gcr(IGBCore *core, int index, uint32_t val)
{
uint32_t ro_bits = core->mac[GCR] & E1000_GCR_RO_BITS;
core->mac[GCR] = (val & ~E1000_GCR_RO_BITS) | ro_bits;
}
static uint32_t igb_get_systiml(IGBCore *core, int index)
{
e1000x_timestamp(core->mac, core->timadj, SYSTIML, SYSTIMH);
return core->mac[SYSTIML];
}
static uint32_t igb_get_rxsatrh(IGBCore *core, int index)
{
core->mac[TSYNCRXCTL] &= ~E1000_TSYNCRXCTL_VALID;
return core->mac[RXSATRH];
}
static uint32_t igb_get_txstmph(IGBCore *core, int index)
{
core->mac[TSYNCTXCTL] &= ~E1000_TSYNCTXCTL_VALID;
return core->mac[TXSTMPH];
}
static void igb_set_timinca(IGBCore *core, int index, uint32_t val)
{
e1000x_set_timinca(core->mac, &core->timadj, val);
}
static void igb_set_timadjh(IGBCore *core, int index, uint32_t val)
{
core->mac[TIMADJH] = val;
core->timadj += core->mac[TIMADJL] | ((int64_t)core->mac[TIMADJH] << 32);
}
#define igb_getreg(x) [x] = igb_mac_readreg
typedef uint32_t (*readops)(IGBCore *, int);
static const readops igb_macreg_readops[] = {
igb_getreg(WUFC),
igb_getreg(MANC),
igb_getreg(TOTL),
igb_getreg(RDT0),
igb_getreg(RDT1),
igb_getreg(RDT2),
igb_getreg(RDT3),
igb_getreg(RDT4),
igb_getreg(RDT5),
igb_getreg(RDT6),
igb_getreg(RDT7),
igb_getreg(RDT8),
igb_getreg(RDT9),
igb_getreg(RDT10),
igb_getreg(RDT11),
igb_getreg(RDT12),
igb_getreg(RDT13),
igb_getreg(RDT14),
igb_getreg(RDT15),
igb_getreg(RDBAH0),
igb_getreg(RDBAH1),
igb_getreg(RDBAH2),
igb_getreg(RDBAH3),
igb_getreg(RDBAH4),
igb_getreg(RDBAH5),
igb_getreg(RDBAH6),
igb_getreg(RDBAH7),
igb_getreg(RDBAH8),
igb_getreg(RDBAH9),
igb_getreg(RDBAH10),
igb_getreg(RDBAH11),
igb_getreg(RDBAH12),
igb_getreg(RDBAH13),
igb_getreg(RDBAH14),
igb_getreg(RDBAH15),
igb_getreg(TDBAL0),
igb_getreg(TDBAL1),
igb_getreg(TDBAL2),
igb_getreg(TDBAL3),
igb_getreg(TDBAL4),
igb_getreg(TDBAL5),
igb_getreg(TDBAL6),
igb_getreg(TDBAL7),
igb_getreg(TDBAL8),
igb_getreg(TDBAL9),
igb_getreg(TDBAL10),
igb_getreg(TDBAL11),
igb_getreg(TDBAL12),
igb_getreg(TDBAL13),
igb_getreg(TDBAL14),
igb_getreg(TDBAL15),
igb_getreg(RDLEN0),
igb_getreg(RDLEN1),
igb_getreg(RDLEN2),
igb_getreg(RDLEN3),
igb_getreg(RDLEN4),
igb_getreg(RDLEN5),
igb_getreg(RDLEN6),
igb_getreg(RDLEN7),
igb_getreg(RDLEN8),
igb_getreg(RDLEN9),
igb_getreg(RDLEN10),
igb_getreg(RDLEN11),
igb_getreg(RDLEN12),
igb_getreg(RDLEN13),
igb_getreg(RDLEN14),
igb_getreg(RDLEN15),
igb_getreg(SRRCTL0),
igb_getreg(SRRCTL1),
igb_getreg(SRRCTL2),
igb_getreg(SRRCTL3),
igb_getreg(SRRCTL4),
igb_getreg(SRRCTL5),
igb_getreg(SRRCTL6),
igb_getreg(SRRCTL7),
igb_getreg(SRRCTL8),
igb_getreg(SRRCTL9),
igb_getreg(SRRCTL10),
igb_getreg(SRRCTL11),
igb_getreg(SRRCTL12),
igb_getreg(SRRCTL13),
igb_getreg(SRRCTL14),
igb_getreg(SRRCTL15),
igb_getreg(LATECOL),
igb_getreg(XONTXC),
igb_getreg(TDFH),
igb_getreg(TDFT),
igb_getreg(TDFHS),
igb_getreg(TDFTS),
igb_getreg(TDFPC),
igb_getreg(WUS),
igb_getreg(RDFH),
igb_getreg(RDFT),
igb_getreg(RDFHS),
igb_getreg(RDFTS),
igb_getreg(RDFPC),
igb_getreg(GORCL),
igb_getreg(MGTPRC),
igb_getreg(EERD),
igb_getreg(EIAC),
igb_getreg(MANC2H),
igb_getreg(RXCSUM),
igb_getreg(GSCL_3),
igb_getreg(GSCN_2),
igb_getreg(FCAH),
igb_getreg(FCRTH),
igb_getreg(FLOP),
igb_getreg(RXSTMPH),
igb_getreg(TXSTMPL),
igb_getreg(TIMADJL),
igb_getreg(RDH0),
igb_getreg(RDH1),
igb_getreg(RDH2),
igb_getreg(RDH3),
igb_getreg(RDH4),
igb_getreg(RDH5),
igb_getreg(RDH6),
igb_getreg(RDH7),
igb_getreg(RDH8),
igb_getreg(RDH9),
igb_getreg(RDH10),
igb_getreg(RDH11),
igb_getreg(RDH12),
igb_getreg(RDH13),
igb_getreg(RDH14),
igb_getreg(RDH15),
igb_getreg(TDT0),
igb_getreg(TDT1),
igb_getreg(TDT2),
igb_getreg(TDT3),
igb_getreg(TDT4),
igb_getreg(TDT5),
igb_getreg(TDT6),
igb_getreg(TDT7),
igb_getreg(TDT8),
igb_getreg(TDT9),
igb_getreg(TDT10),
igb_getreg(TDT11),
igb_getreg(TDT12),
igb_getreg(TDT13),
igb_getreg(TDT14),
igb_getreg(TDT15),
igb_getreg(TNCRS),
igb_getreg(RJC),
igb_getreg(IAM),
igb_getreg(GSCL_2),
igb_getreg(TIPG),
igb_getreg(FLMNGCTL),
igb_getreg(FLMNGCNT),
igb_getreg(TSYNCTXCTL),
igb_getreg(EEMNGDATA),
igb_getreg(CTRL_EXT),
igb_getreg(SYSTIMH),
igb_getreg(EEMNGCTL),
igb_getreg(FLMNGDATA),
igb_getreg(TSYNCRXCTL),
igb_getreg(LEDCTL),
igb_getreg(TCTL),
igb_getreg(TCTL_EXT),
igb_getreg(DTXCTL),
igb_getreg(RXPBS),
igb_getreg(TDH0),
igb_getreg(TDH1),
igb_getreg(TDH2),
igb_getreg(TDH3),
igb_getreg(TDH4),
igb_getreg(TDH5),
igb_getreg(TDH6),
igb_getreg(TDH7),
igb_getreg(TDH8),
igb_getreg(TDH9),
igb_getreg(TDH10),
igb_getreg(TDH11),
igb_getreg(TDH12),
igb_getreg(TDH13),
igb_getreg(TDH14),
igb_getreg(TDH15),
igb_getreg(ECOL),
igb_getreg(DC),
igb_getreg(RLEC),
igb_getreg(XOFFTXC),
igb_getreg(RFC),
igb_getreg(RNBC),
igb_getreg(MGTPTC),
igb_getreg(TIMINCA),
igb_getreg(FACTPS),
igb_getreg(GSCL_1),
igb_getreg(GSCN_0),
igb_getreg(PBACLR),
igb_getreg(FCTTV),
igb_getreg(RXSATRL),
igb_getreg(TORL),
igb_getreg(TDLEN0),
igb_getreg(TDLEN1),
igb_getreg(TDLEN2),
igb_getreg(TDLEN3),
igb_getreg(TDLEN4),
igb_getreg(TDLEN5),
igb_getreg(TDLEN6),
igb_getreg(TDLEN7),
igb_getreg(TDLEN8),
igb_getreg(TDLEN9),
igb_getreg(TDLEN10),
igb_getreg(TDLEN11),
igb_getreg(TDLEN12),
igb_getreg(TDLEN13),
igb_getreg(TDLEN14),
igb_getreg(TDLEN15),
igb_getreg(MCC),
igb_getreg(WUC),
igb_getreg(EECD),
igb_getreg(FCRTV),
igb_getreg(TXDCTL0),
igb_getreg(TXDCTL1),
igb_getreg(TXDCTL2),
igb_getreg(TXDCTL3),
igb_getreg(TXDCTL4),
igb_getreg(TXDCTL5),
igb_getreg(TXDCTL6),
igb_getreg(TXDCTL7),
igb_getreg(TXDCTL8),
igb_getreg(TXDCTL9),
igb_getreg(TXDCTL10),
igb_getreg(TXDCTL11),
igb_getreg(TXDCTL12),
igb_getreg(TXDCTL13),
igb_getreg(TXDCTL14),
igb_getreg(TXDCTL15),
igb_getreg(TXCTL0),
igb_getreg(TXCTL1),
igb_getreg(TXCTL2),
igb_getreg(TXCTL3),
igb_getreg(TXCTL4),
igb_getreg(TXCTL5),
igb_getreg(TXCTL6),
igb_getreg(TXCTL7),
igb_getreg(TXCTL8),
igb_getreg(TXCTL9),
igb_getreg(TXCTL10),
igb_getreg(TXCTL11),
igb_getreg(TXCTL12),
igb_getreg(TXCTL13),
igb_getreg(TXCTL14),
igb_getreg(TXCTL15),
igb_getreg(TDWBAL0),
igb_getreg(TDWBAL1),
igb_getreg(TDWBAL2),
igb_getreg(TDWBAL3),
igb_getreg(TDWBAL4),
igb_getreg(TDWBAL5),
igb_getreg(TDWBAL6),
igb_getreg(TDWBAL7),
igb_getreg(TDWBAL8),
igb_getreg(TDWBAL9),
igb_getreg(TDWBAL10),
igb_getreg(TDWBAL11),
igb_getreg(TDWBAL12),
igb_getreg(TDWBAL13),
igb_getreg(TDWBAL14),
igb_getreg(TDWBAL15),
igb_getreg(TDWBAH0),
igb_getreg(TDWBAH1),
igb_getreg(TDWBAH2),
igb_getreg(TDWBAH3),
igb_getreg(TDWBAH4),
igb_getreg(TDWBAH5),
igb_getreg(TDWBAH6),
igb_getreg(TDWBAH7),
igb_getreg(TDWBAH8),
igb_getreg(TDWBAH9),
igb_getreg(TDWBAH10),
igb_getreg(TDWBAH11),
igb_getreg(TDWBAH12),
igb_getreg(TDWBAH13),
igb_getreg(TDWBAH14),
igb_getreg(TDWBAH15),
igb_getreg(PVTCTRL0),
igb_getreg(PVTCTRL1),
igb_getreg(PVTCTRL2),
igb_getreg(PVTCTRL3),
igb_getreg(PVTCTRL4),
igb_getreg(PVTCTRL5),
igb_getreg(PVTCTRL6),
igb_getreg(PVTCTRL7),
igb_getreg(PVTEIMS0),
igb_getreg(PVTEIMS1),
igb_getreg(PVTEIMS2),
igb_getreg(PVTEIMS3),
igb_getreg(PVTEIMS4),
igb_getreg(PVTEIMS5),
igb_getreg(PVTEIMS6),
igb_getreg(PVTEIMS7),
igb_getreg(PVTEIAC0),
igb_getreg(PVTEIAC1),
igb_getreg(PVTEIAC2),
igb_getreg(PVTEIAC3),
igb_getreg(PVTEIAC4),
igb_getreg(PVTEIAC5),
igb_getreg(PVTEIAC6),
igb_getreg(PVTEIAC7),
igb_getreg(PVTEIAM0),
igb_getreg(PVTEIAM1),
igb_getreg(PVTEIAM2),
igb_getreg(PVTEIAM3),
igb_getreg(PVTEIAM4),
igb_getreg(PVTEIAM5),
igb_getreg(PVTEIAM6),
igb_getreg(PVTEIAM7),
igb_getreg(PVFGPRC0),
igb_getreg(PVFGPRC1),
igb_getreg(PVFGPRC2),
igb_getreg(PVFGPRC3),
igb_getreg(PVFGPRC4),
igb_getreg(PVFGPRC5),
igb_getreg(PVFGPRC6),
igb_getreg(PVFGPRC7),
igb_getreg(PVFGPTC0),
igb_getreg(PVFGPTC1),
igb_getreg(PVFGPTC2),
igb_getreg(PVFGPTC3),
igb_getreg(PVFGPTC4),
igb_getreg(PVFGPTC5),
igb_getreg(PVFGPTC6),
igb_getreg(PVFGPTC7),
igb_getreg(PVFGORC0),
igb_getreg(PVFGORC1),
igb_getreg(PVFGORC2),
igb_getreg(PVFGORC3),
igb_getreg(PVFGORC4),
igb_getreg(PVFGORC5),
igb_getreg(PVFGORC6),
igb_getreg(PVFGORC7),
igb_getreg(PVFGOTC0),
igb_getreg(PVFGOTC1),
igb_getreg(PVFGOTC2),
igb_getreg(PVFGOTC3),
igb_getreg(PVFGOTC4),
igb_getreg(PVFGOTC5),
igb_getreg(PVFGOTC6),
igb_getreg(PVFGOTC7),
igb_getreg(PVFMPRC0),
igb_getreg(PVFMPRC1),
igb_getreg(PVFMPRC2),
igb_getreg(PVFMPRC3),
igb_getreg(PVFMPRC4),
igb_getreg(PVFMPRC5),
igb_getreg(PVFMPRC6),
igb_getreg(PVFMPRC7),
igb_getreg(PVFGPRLBC0),
igb_getreg(PVFGPRLBC1),
igb_getreg(PVFGPRLBC2),
igb_getreg(PVFGPRLBC3),
igb_getreg(PVFGPRLBC4),
igb_getreg(PVFGPRLBC5),
igb_getreg(PVFGPRLBC6),
igb_getreg(PVFGPRLBC7),
igb_getreg(PVFGPTLBC0),
igb_getreg(PVFGPTLBC1),
igb_getreg(PVFGPTLBC2),
igb_getreg(PVFGPTLBC3),
igb_getreg(PVFGPTLBC4),
igb_getreg(PVFGPTLBC5),
igb_getreg(PVFGPTLBC6),
igb_getreg(PVFGPTLBC7),
igb_getreg(PVFGORLBC0),
igb_getreg(PVFGORLBC1),
igb_getreg(PVFGORLBC2),
igb_getreg(PVFGORLBC3),
igb_getreg(PVFGORLBC4),
igb_getreg(PVFGORLBC5),
igb_getreg(PVFGORLBC6),
igb_getreg(PVFGORLBC7),
igb_getreg(PVFGOTLBC0),
igb_getreg(PVFGOTLBC1),
igb_getreg(PVFGOTLBC2),
igb_getreg(PVFGOTLBC3),
igb_getreg(PVFGOTLBC4),
igb_getreg(PVFGOTLBC5),
igb_getreg(PVFGOTLBC6),
igb_getreg(PVFGOTLBC7),
igb_getreg(RCTL),
igb_getreg(MDIC),
igb_getreg(FCRUC),
igb_getreg(VET),
igb_getreg(RDBAL0),
igb_getreg(RDBAL1),
igb_getreg(RDBAL2),
igb_getreg(RDBAL3),
igb_getreg(RDBAL4),
igb_getreg(RDBAL5),
igb_getreg(RDBAL6),
igb_getreg(RDBAL7),
igb_getreg(RDBAL8),
igb_getreg(RDBAL9),
igb_getreg(RDBAL10),
igb_getreg(RDBAL11),
igb_getreg(RDBAL12),
igb_getreg(RDBAL13),
igb_getreg(RDBAL14),
igb_getreg(RDBAL15),
igb_getreg(TDBAH0),
igb_getreg(TDBAH1),
igb_getreg(TDBAH2),
igb_getreg(TDBAH3),
igb_getreg(TDBAH4),
igb_getreg(TDBAH5),
igb_getreg(TDBAH6),
igb_getreg(TDBAH7),
igb_getreg(TDBAH8),
igb_getreg(TDBAH9),
igb_getreg(TDBAH10),
igb_getreg(TDBAH11),
igb_getreg(TDBAH12),
igb_getreg(TDBAH13),
igb_getreg(TDBAH14),
igb_getreg(TDBAH15),
igb_getreg(SCC),
igb_getreg(COLC),
igb_getreg(XOFFRXC),
igb_getreg(IPAV),
igb_getreg(GOTCL),
igb_getreg(MGTPDC),
igb_getreg(GCR),
igb_getreg(MFVAL),
igb_getreg(FUNCTAG),
igb_getreg(GSCL_4),
igb_getreg(GSCN_3),
igb_getreg(MRQC),
igb_getreg(FCT),
igb_getreg(FLA),
igb_getreg(RXDCTL0),
igb_getreg(RXDCTL1),
igb_getreg(RXDCTL2),
igb_getreg(RXDCTL3),
igb_getreg(RXDCTL4),
igb_getreg(RXDCTL5),
igb_getreg(RXDCTL6),
igb_getreg(RXDCTL7),
igb_getreg(RXDCTL8),
igb_getreg(RXDCTL9),
igb_getreg(RXDCTL10),
igb_getreg(RXDCTL11),
igb_getreg(RXDCTL12),
igb_getreg(RXDCTL13),
igb_getreg(RXDCTL14),
igb_getreg(RXDCTL15),
igb_getreg(RXSTMPL),
igb_getreg(TIMADJH),
igb_getreg(FCRTL),
igb_getreg(XONRXC),
igb_getreg(RFCTL),
igb_getreg(GSCN_1),
igb_getreg(FCAL),
igb_getreg(GPIE),
igb_getreg(TXPBS),
igb_getreg(RLPML),
[TOTH] = igb_mac_read_clr8,
[GOTCH] = igb_mac_read_clr8,
[PRC64] = igb_mac_read_clr4,
[PRC255] = igb_mac_read_clr4,
[PRC1023] = igb_mac_read_clr4,
[PTC64] = igb_mac_read_clr4,
[PTC255] = igb_mac_read_clr4,
[PTC1023] = igb_mac_read_clr4,
[GPRC] = igb_mac_read_clr4,
[TPT] = igb_mac_read_clr4,
[RUC] = igb_mac_read_clr4,
[BPRC] = igb_mac_read_clr4,
[MPTC] = igb_mac_read_clr4,
[IAC] = igb_mac_read_clr4,
[ICR] = igb_mac_icr_read,
[STATUS] = igb_get_status,
[ICS] = igb_mac_ics_read,
/*
* 8.8.10: Reading the IMC register returns the value of the IMS register.
*/
[IMC] = igb_mac_ims_read,
[TORH] = igb_mac_read_clr8,
[GORCH] = igb_mac_read_clr8,
[PRC127] = igb_mac_read_clr4,
[PRC511] = igb_mac_read_clr4,
[PRC1522] = igb_mac_read_clr4,
[PTC127] = igb_mac_read_clr4,
[PTC511] = igb_mac_read_clr4,
[PTC1522] = igb_mac_read_clr4,
[GPTC] = igb_mac_read_clr4,
[TPR] = igb_mac_read_clr4,
[ROC] = igb_mac_read_clr4,
[MPRC] = igb_mac_read_clr4,
[BPTC] = igb_mac_read_clr4,
[TSCTC] = igb_mac_read_clr4,
[CTRL] = igb_get_ctrl,
[SWSM] = igb_mac_swsm_read,
[IMS] = igb_mac_ims_read,
[SYSTIML] = igb_get_systiml,
[RXSATRH] = igb_get_rxsatrh,
[TXSTMPH] = igb_get_txstmph,
[CRCERRS ... MPC] = igb_mac_readreg,
[IP6AT ... IP6AT + 3] = igb_mac_readreg,
[IP4AT ... IP4AT + 6] = igb_mac_readreg,
[RA ... RA + 31] = igb_mac_readreg,
[RA2 ... RA2 + 31] = igb_mac_readreg,
[WUPM ... WUPM + 31] = igb_mac_readreg,
[MTA ... MTA + E1000_MC_TBL_SIZE - 1] = igb_mac_readreg,
[VFTA ... VFTA + E1000_VLAN_FILTER_TBL_SIZE - 1] = igb_mac_readreg,
[FFMT ... FFMT + 254] = igb_mac_readreg,
[MDEF ... MDEF + 7] = igb_mac_readreg,
[FTFT ... FTFT + 254] = igb_mac_readreg,
[RETA ... RETA + 31] = igb_mac_readreg,
[RSSRK ... RSSRK + 9] = igb_mac_readreg,
[MAVTV0 ... MAVTV3] = igb_mac_readreg,
[EITR0 ... EITR0 + IGB_INTR_NUM - 1] = igb_mac_eitr_read,
[PVTEICR0] = igb_mac_read_clr4,
[PVTEICR1] = igb_mac_read_clr4,
[PVTEICR2] = igb_mac_read_clr4,
[PVTEICR3] = igb_mac_read_clr4,
[PVTEICR4] = igb_mac_read_clr4,
[PVTEICR5] = igb_mac_read_clr4,
[PVTEICR6] = igb_mac_read_clr4,
[PVTEICR7] = igb_mac_read_clr4,
/* IGB specific: */
[FWSM] = igb_mac_readreg,
[SW_FW_SYNC] = igb_mac_readreg,
[HTCBDPC] = igb_mac_read_clr4,
[EICR] = igb_mac_read_clr4,
[EIMS] = igb_mac_readreg,
[EIAM] = igb_mac_readreg,
[IVAR0 ... IVAR0 + 7] = igb_mac_readreg,
igb_getreg(IVAR_MISC),
igb_getreg(TSYNCRXCFG),
[ETQF0 ... ETQF0 + 7] = igb_mac_readreg,
igb_getreg(VT_CTL),
[P2VMAILBOX0 ... P2VMAILBOX7] = igb_mac_readreg,
[V2PMAILBOX0 ... V2PMAILBOX7] = igb_mac_vfmailbox_read,
igb_getreg(MBVFICR),
[VMBMEM0 ... VMBMEM0 + 127] = igb_mac_readreg,
igb_getreg(MBVFIMR),
igb_getreg(VFLRE),
igb_getreg(VFRE),
igb_getreg(VFTE),
igb_getreg(QDE),
igb_getreg(DTXSWC),
igb_getreg(RPLOLR),
[VLVF0 ... VLVF0 + E1000_VLVF_ARRAY_SIZE - 1] = igb_mac_readreg,
[VMVIR0 ... VMVIR7] = igb_mac_readreg,
[VMOLR0 ... VMOLR7] = igb_mac_readreg,
[WVBR] = igb_mac_read_clr4,
[RQDPC0] = igb_mac_read_clr4,
[RQDPC1] = igb_mac_read_clr4,
[RQDPC2] = igb_mac_read_clr4,
[RQDPC3] = igb_mac_read_clr4,
[RQDPC4] = igb_mac_read_clr4,
[RQDPC5] = igb_mac_read_clr4,
[RQDPC6] = igb_mac_read_clr4,
[RQDPC7] = igb_mac_read_clr4,
[RQDPC8] = igb_mac_read_clr4,
[RQDPC9] = igb_mac_read_clr4,
[RQDPC10] = igb_mac_read_clr4,
[RQDPC11] = igb_mac_read_clr4,
[RQDPC12] = igb_mac_read_clr4,
[RQDPC13] = igb_mac_read_clr4,
[RQDPC14] = igb_mac_read_clr4,
[RQDPC15] = igb_mac_read_clr4,
[VTIVAR ... VTIVAR + 7] = igb_mac_readreg,
[VTIVAR_MISC ... VTIVAR_MISC + 7] = igb_mac_readreg,
};
enum { IGB_NREADOPS = ARRAY_SIZE(igb_macreg_readops) };
#define igb_putreg(x) [x] = igb_mac_writereg
typedef void (*writeops)(IGBCore *, int, uint32_t);
static const writeops igb_macreg_writeops[] = {
igb_putreg(SWSM),
igb_putreg(WUFC),
igb_putreg(RDBAH0),
igb_putreg(RDBAH1),
igb_putreg(RDBAH2),
igb_putreg(RDBAH3),
igb_putreg(RDBAH4),
igb_putreg(RDBAH5),
igb_putreg(RDBAH6),
igb_putreg(RDBAH7),
igb_putreg(RDBAH8),
igb_putreg(RDBAH9),
igb_putreg(RDBAH10),
igb_putreg(RDBAH11),
igb_putreg(RDBAH12),
igb_putreg(RDBAH13),
igb_putreg(RDBAH14),
igb_putreg(RDBAH15),
igb_putreg(SRRCTL0),
igb_putreg(SRRCTL1),
igb_putreg(SRRCTL2),
igb_putreg(SRRCTL3),
igb_putreg(SRRCTL4),
igb_putreg(SRRCTL5),
igb_putreg(SRRCTL6),
igb_putreg(SRRCTL7),
igb_putreg(SRRCTL8),
igb_putreg(SRRCTL9),
igb_putreg(SRRCTL10),
igb_putreg(SRRCTL11),
igb_putreg(SRRCTL12),
igb_putreg(SRRCTL13),
igb_putreg(SRRCTL14),
igb_putreg(SRRCTL15),
igb_putreg(RXDCTL0),
igb_putreg(RXDCTL1),
igb_putreg(RXDCTL2),
igb_putreg(RXDCTL3),
igb_putreg(RXDCTL4),
igb_putreg(RXDCTL5),
igb_putreg(RXDCTL6),
igb_putreg(RXDCTL7),
igb_putreg(RXDCTL8),
igb_putreg(RXDCTL9),
igb_putreg(RXDCTL10),
igb_putreg(RXDCTL11),
igb_putreg(RXDCTL12),
igb_putreg(RXDCTL13),
igb_putreg(RXDCTL14),
igb_putreg(RXDCTL15),
igb_putreg(LEDCTL),
igb_putreg(TCTL),
igb_putreg(TCTL_EXT),
igb_putreg(DTXCTL),
igb_putreg(RXPBS),
igb_putreg(RQDPC0),
igb_putreg(FCAL),
igb_putreg(FCRUC),
igb_putreg(WUC),
igb_putreg(WUS),
igb_putreg(IPAV),
igb_putreg(TDBAH0),
igb_putreg(TDBAH1),
igb_putreg(TDBAH2),
igb_putreg(TDBAH3),
igb_putreg(TDBAH4),
igb_putreg(TDBAH5),
igb_putreg(TDBAH6),
igb_putreg(TDBAH7),
igb_putreg(TDBAH8),
igb_putreg(TDBAH9),
igb_putreg(TDBAH10),
igb_putreg(TDBAH11),
igb_putreg(TDBAH12),
igb_putreg(TDBAH13),
igb_putreg(TDBAH14),
igb_putreg(TDBAH15),
igb_putreg(IAM),
igb_putreg(MANC),
igb_putreg(MANC2H),
igb_putreg(MFVAL),
igb_putreg(FACTPS),
igb_putreg(FUNCTAG),
igb_putreg(GSCL_1),
igb_putreg(GSCL_2),
igb_putreg(GSCL_3),
igb_putreg(GSCL_4),
igb_putreg(GSCN_0),
igb_putreg(GSCN_1),
igb_putreg(GSCN_2),
igb_putreg(GSCN_3),
igb_putreg(MRQC),
igb_putreg(FLOP),
igb_putreg(FLA),
igb_putreg(TXDCTL0),
igb_putreg(TXDCTL1),
igb_putreg(TXDCTL2),
igb_putreg(TXDCTL3),
igb_putreg(TXDCTL4),
igb_putreg(TXDCTL5),
igb_putreg(TXDCTL6),
igb_putreg(TXDCTL7),
igb_putreg(TXDCTL8),
igb_putreg(TXDCTL9),
igb_putreg(TXDCTL10),
igb_putreg(TXDCTL11),
igb_putreg(TXDCTL12),
igb_putreg(TXDCTL13),
igb_putreg(TXDCTL14),
igb_putreg(TXDCTL15),
igb_putreg(TXCTL0),
igb_putreg(TXCTL1),
igb_putreg(TXCTL2),
igb_putreg(TXCTL3),
igb_putreg(TXCTL4),
igb_putreg(TXCTL5),
igb_putreg(TXCTL6),
igb_putreg(TXCTL7),
igb_putreg(TXCTL8),
igb_putreg(TXCTL9),
igb_putreg(TXCTL10),
igb_putreg(TXCTL11),
igb_putreg(TXCTL12),
igb_putreg(TXCTL13),
igb_putreg(TXCTL14),
igb_putreg(TXCTL15),
igb_putreg(TDWBAL0),
igb_putreg(TDWBAL1),
igb_putreg(TDWBAL2),
igb_putreg(TDWBAL3),
igb_putreg(TDWBAL4),
igb_putreg(TDWBAL5),
igb_putreg(TDWBAL6),
igb_putreg(TDWBAL7),
igb_putreg(TDWBAL8),
igb_putreg(TDWBAL9),
igb_putreg(TDWBAL10),
igb_putreg(TDWBAL11),
igb_putreg(TDWBAL12),
igb_putreg(TDWBAL13),
igb_putreg(TDWBAL14),
igb_putreg(TDWBAL15),
igb_putreg(TDWBAH0),
igb_putreg(TDWBAH1),
igb_putreg(TDWBAH2),
igb_putreg(TDWBAH3),
igb_putreg(TDWBAH4),
igb_putreg(TDWBAH5),
igb_putreg(TDWBAH6),
igb_putreg(TDWBAH7),
igb_putreg(TDWBAH8),
igb_putreg(TDWBAH9),
igb_putreg(TDWBAH10),
igb_putreg(TDWBAH11),
igb_putreg(TDWBAH12),
igb_putreg(TDWBAH13),
igb_putreg(TDWBAH14),
igb_putreg(TDWBAH15),
igb_putreg(TIPG),
igb_putreg(RXSTMPH),
igb_putreg(RXSTMPL),
igb_putreg(RXSATRL),
igb_putreg(RXSATRH),
igb_putreg(TXSTMPL),
igb_putreg(TXSTMPH),
igb_putreg(SYSTIML),
igb_putreg(SYSTIMH),
igb_putreg(TIMADJL),
igb_putreg(TSYNCRXCTL),
igb_putreg(TSYNCTXCTL),
igb_putreg(EEMNGCTL),
igb_putreg(GPIE),
igb_putreg(TXPBS),
igb_putreg(RLPML),
igb_putreg(VET),
[TDH0] = igb_set_16bit,
[TDH1] = igb_set_16bit,
[TDH2] = igb_set_16bit,
[TDH3] = igb_set_16bit,
[TDH4] = igb_set_16bit,
[TDH5] = igb_set_16bit,
[TDH6] = igb_set_16bit,
[TDH7] = igb_set_16bit,
[TDH8] = igb_set_16bit,
[TDH9] = igb_set_16bit,
[TDH10] = igb_set_16bit,
[TDH11] = igb_set_16bit,
[TDH12] = igb_set_16bit,
[TDH13] = igb_set_16bit,
[TDH14] = igb_set_16bit,
[TDH15] = igb_set_16bit,
[TDT0] = igb_set_tdt,
[TDT1] = igb_set_tdt,
[TDT2] = igb_set_tdt,
[TDT3] = igb_set_tdt,
[TDT4] = igb_set_tdt,
[TDT5] = igb_set_tdt,
[TDT6] = igb_set_tdt,
[TDT7] = igb_set_tdt,
[TDT8] = igb_set_tdt,
[TDT9] = igb_set_tdt,
[TDT10] = igb_set_tdt,
[TDT11] = igb_set_tdt,
[TDT12] = igb_set_tdt,
[TDT13] = igb_set_tdt,
[TDT14] = igb_set_tdt,
[TDT15] = igb_set_tdt,
[MDIC] = igb_set_mdic,
[ICS] = igb_set_ics,
[RDH0] = igb_set_16bit,
[RDH1] = igb_set_16bit,
[RDH2] = igb_set_16bit,
[RDH3] = igb_set_16bit,
[RDH4] = igb_set_16bit,
[RDH5] = igb_set_16bit,
[RDH6] = igb_set_16bit,
[RDH7] = igb_set_16bit,
[RDH8] = igb_set_16bit,
[RDH9] = igb_set_16bit,
[RDH10] = igb_set_16bit,
[RDH11] = igb_set_16bit,
[RDH12] = igb_set_16bit,
[RDH13] = igb_set_16bit,
[RDH14] = igb_set_16bit,
[RDH15] = igb_set_16bit,
[RDT0] = igb_set_rdt,
[RDT1] = igb_set_rdt,
[RDT2] = igb_set_rdt,
[RDT3] = igb_set_rdt,
[RDT4] = igb_set_rdt,
[RDT5] = igb_set_rdt,
[RDT6] = igb_set_rdt,
[RDT7] = igb_set_rdt,
[RDT8] = igb_set_rdt,
[RDT9] = igb_set_rdt,
[RDT10] = igb_set_rdt,
[RDT11] = igb_set_rdt,
[RDT12] = igb_set_rdt,
[RDT13] = igb_set_rdt,
[RDT14] = igb_set_rdt,
[RDT15] = igb_set_rdt,
[IMC] = igb_set_imc,
[IMS] = igb_set_ims,
[ICR] = igb_set_icr,
[EECD] = igb_set_eecd,
[RCTL] = igb_set_rx_control,
[CTRL] = igb_set_ctrl,
[EERD] = igb_set_eerd,
[TDFH] = igb_set_13bit,
[TDFT] = igb_set_13bit,
[TDFHS] = igb_set_13bit,
[TDFTS] = igb_set_13bit,
[TDFPC] = igb_set_13bit,
[RDFH] = igb_set_13bit,
[RDFT] = igb_set_13bit,
[RDFHS] = igb_set_13bit,
[RDFTS] = igb_set_13bit,
[RDFPC] = igb_set_13bit,
[GCR] = igb_set_gcr,
[RXCSUM] = igb_set_rxcsum,
[TDLEN0] = igb_set_dlen,
[TDLEN1] = igb_set_dlen,
[TDLEN2] = igb_set_dlen,
[TDLEN3] = igb_set_dlen,
[TDLEN4] = igb_set_dlen,
[TDLEN5] = igb_set_dlen,
[TDLEN6] = igb_set_dlen,
[TDLEN7] = igb_set_dlen,
[TDLEN8] = igb_set_dlen,
[TDLEN9] = igb_set_dlen,
[TDLEN10] = igb_set_dlen,
[TDLEN11] = igb_set_dlen,
[TDLEN12] = igb_set_dlen,
[TDLEN13] = igb_set_dlen,
[TDLEN14] = igb_set_dlen,
[TDLEN15] = igb_set_dlen,
[RDLEN0] = igb_set_dlen,
[RDLEN1] = igb_set_dlen,
[RDLEN2] = igb_set_dlen,
[RDLEN3] = igb_set_dlen,
[RDLEN4] = igb_set_dlen,
[RDLEN5] = igb_set_dlen,
[RDLEN6] = igb_set_dlen,
[RDLEN7] = igb_set_dlen,
[RDLEN8] = igb_set_dlen,
[RDLEN9] = igb_set_dlen,
[RDLEN10] = igb_set_dlen,
[RDLEN11] = igb_set_dlen,
[RDLEN12] = igb_set_dlen,
[RDLEN13] = igb_set_dlen,
[RDLEN14] = igb_set_dlen,
[RDLEN15] = igb_set_dlen,
[TDBAL0] = igb_set_dbal,
[TDBAL1] = igb_set_dbal,
[TDBAL2] = igb_set_dbal,
[TDBAL3] = igb_set_dbal,
[TDBAL4] = igb_set_dbal,
[TDBAL5] = igb_set_dbal,
[TDBAL6] = igb_set_dbal,
[TDBAL7] = igb_set_dbal,
[TDBAL8] = igb_set_dbal,
[TDBAL9] = igb_set_dbal,
[TDBAL10] = igb_set_dbal,
[TDBAL11] = igb_set_dbal,
[TDBAL12] = igb_set_dbal,
[TDBAL13] = igb_set_dbal,
[TDBAL14] = igb_set_dbal,
[TDBAL15] = igb_set_dbal,
[RDBAL0] = igb_set_dbal,
[RDBAL1] = igb_set_dbal,
[RDBAL2] = igb_set_dbal,
[RDBAL3] = igb_set_dbal,
[RDBAL4] = igb_set_dbal,
[RDBAL5] = igb_set_dbal,
[RDBAL6] = igb_set_dbal,
[RDBAL7] = igb_set_dbal,
[RDBAL8] = igb_set_dbal,
[RDBAL9] = igb_set_dbal,
[RDBAL10] = igb_set_dbal,
[RDBAL11] = igb_set_dbal,
[RDBAL12] = igb_set_dbal,
[RDBAL13] = igb_set_dbal,
[RDBAL14] = igb_set_dbal,
[RDBAL15] = igb_set_dbal,
[STATUS] = igb_set_status,
[PBACLR] = igb_set_pbaclr,
[CTRL_EXT] = igb_set_ctrlext,
[FCAH] = igb_set_16bit,
[FCT] = igb_set_16bit,
[FCTTV] = igb_set_16bit,
[FCRTV] = igb_set_16bit,
[FCRTH] = igb_set_fcrth,
[FCRTL] = igb_set_fcrtl,
[CTRL_DUP] = igb_set_ctrl,
[RFCTL] = igb_set_rfctl,
[TIMINCA] = igb_set_timinca,
[TIMADJH] = igb_set_timadjh,
[IP6AT ... IP6AT + 3] = igb_mac_writereg,
[IP4AT ... IP4AT + 6] = igb_mac_writereg,
[RA] = igb_mac_writereg,
[RA + 1] = igb_mac_setmacaddr,
[RA + 2 ... RA + 31] = igb_mac_writereg,
[RA2 ... RA2 + 31] = igb_mac_writereg,
[WUPM ... WUPM + 31] = igb_mac_writereg,
[MTA ... MTA + E1000_MC_TBL_SIZE - 1] = igb_mac_writereg,
[VFTA ... VFTA + E1000_VLAN_FILTER_TBL_SIZE - 1] = igb_mac_writereg,
[FFMT ... FFMT + 254] = igb_set_4bit,
[MDEF ... MDEF + 7] = igb_mac_writereg,
[FTFT ... FTFT + 254] = igb_mac_writereg,
[RETA ... RETA + 31] = igb_mac_writereg,
[RSSRK ... RSSRK + 9] = igb_mac_writereg,
[MAVTV0 ... MAVTV3] = igb_mac_writereg,
[EITR0 ... EITR0 + IGB_INTR_NUM - 1] = igb_set_eitr,
/* IGB specific: */
[FWSM] = igb_mac_writereg,
[SW_FW_SYNC] = igb_mac_writereg,
[EICR] = igb_set_eicr,
[EICS] = igb_set_eics,
[EIAC] = igb_set_eiac,
[EIAM] = igb_set_eiam,
[EIMC] = igb_set_eimc,
[EIMS] = igb_set_eims,
[IVAR0 ... IVAR0 + 7] = igb_mac_writereg,
igb_putreg(IVAR_MISC),
igb_putreg(TSYNCRXCFG),
[ETQF0 ... ETQF0 + 7] = igb_mac_writereg,
igb_putreg(VT_CTL),
[P2VMAILBOX0 ... P2VMAILBOX7] = igb_set_pfmailbox,
[V2PMAILBOX0 ... V2PMAILBOX7] = igb_set_vfmailbox,
[MBVFICR] = igb_w1c,
[VMBMEM0 ... VMBMEM0 + 127] = igb_mac_writereg,
igb_putreg(MBVFIMR),
[VFLRE] = igb_w1c,
igb_putreg(VFRE),
igb_putreg(VFTE),
igb_putreg(QDE),
igb_putreg(DTXSWC),
igb_putreg(RPLOLR),
[VLVF0 ... VLVF0 + E1000_VLVF_ARRAY_SIZE - 1] = igb_mac_writereg,
[VMVIR0 ... VMVIR7] = igb_mac_writereg,
[VMOLR0 ... VMOLR7] = igb_mac_writereg,
[UTA ... UTA + E1000_MC_TBL_SIZE - 1] = igb_mac_writereg,
[PVTCTRL0] = igb_set_vtctrl,
[PVTCTRL1] = igb_set_vtctrl,
[PVTCTRL2] = igb_set_vtctrl,
[PVTCTRL3] = igb_set_vtctrl,
[PVTCTRL4] = igb_set_vtctrl,
[PVTCTRL5] = igb_set_vtctrl,
[PVTCTRL6] = igb_set_vtctrl,
[PVTCTRL7] = igb_set_vtctrl,
[PVTEICS0] = igb_set_vteics,
[PVTEICS1] = igb_set_vteics,
[PVTEICS2] = igb_set_vteics,
[PVTEICS3] = igb_set_vteics,
[PVTEICS4] = igb_set_vteics,
[PVTEICS5] = igb_set_vteics,
[PVTEICS6] = igb_set_vteics,
[PVTEICS7] = igb_set_vteics,
[PVTEIMS0] = igb_set_vteims,
[PVTEIMS1] = igb_set_vteims,
[PVTEIMS2] = igb_set_vteims,
[PVTEIMS3] = igb_set_vteims,
[PVTEIMS4] = igb_set_vteims,
[PVTEIMS5] = igb_set_vteims,
[PVTEIMS6] = igb_set_vteims,
[PVTEIMS7] = igb_set_vteims,
[PVTEIMC0] = igb_set_vteimc,
[PVTEIMC1] = igb_set_vteimc,
[PVTEIMC2] = igb_set_vteimc,
[PVTEIMC3] = igb_set_vteimc,
[PVTEIMC4] = igb_set_vteimc,
[PVTEIMC5] = igb_set_vteimc,
[PVTEIMC6] = igb_set_vteimc,
[PVTEIMC7] = igb_set_vteimc,
[PVTEIAC0] = igb_set_vteiac,
[PVTEIAC1] = igb_set_vteiac,
[PVTEIAC2] = igb_set_vteiac,
[PVTEIAC3] = igb_set_vteiac,
[PVTEIAC4] = igb_set_vteiac,
[PVTEIAC5] = igb_set_vteiac,
[PVTEIAC6] = igb_set_vteiac,
[PVTEIAC7] = igb_set_vteiac,
[PVTEIAM0] = igb_set_vteiam,
[PVTEIAM1] = igb_set_vteiam,
[PVTEIAM2] = igb_set_vteiam,
[PVTEIAM3] = igb_set_vteiam,
[PVTEIAM4] = igb_set_vteiam,
[PVTEIAM5] = igb_set_vteiam,
[PVTEIAM6] = igb_set_vteiam,
[PVTEIAM7] = igb_set_vteiam,
[PVTEICR0] = igb_set_vteicr,
[PVTEICR1] = igb_set_vteicr,
[PVTEICR2] = igb_set_vteicr,
[PVTEICR3] = igb_set_vteicr,
[PVTEICR4] = igb_set_vteicr,
[PVTEICR5] = igb_set_vteicr,
[PVTEICR6] = igb_set_vteicr,
[PVTEICR7] = igb_set_vteicr,
[VTIVAR ... VTIVAR + 7] = igb_set_vtivar,
[VTIVAR_MISC ... VTIVAR_MISC + 7] = igb_mac_writereg
};
enum { IGB_NWRITEOPS = ARRAY_SIZE(igb_macreg_writeops) };
enum { MAC_ACCESS_PARTIAL = 1 };
/*
* The array below combines alias offsets of the index values for the
* MAC registers that have aliases, with the indication of not fully
* implemented registers (lowest bit). This combination is possible
* because all of the offsets are even.
*/
static const uint16_t mac_reg_access[E1000E_MAC_SIZE] = {
/* Alias index offsets */
[FCRTL_A] = 0x07fe,
[RDFH_A] = 0xe904, [RDFT_A] = 0xe904,
[TDFH_A] = 0xed00, [TDFT_A] = 0xed00,
[RA_A ... RA_A + 31] = 0x14f0,
[VFTA_A ... VFTA_A + E1000_VLAN_FILTER_TBL_SIZE - 1] = 0x1400,
[RDBAL0_A] = 0x2600,
[RDBAH0_A] = 0x2600,
[RDLEN0_A] = 0x2600,
[SRRCTL0_A] = 0x2600,
[RDH0_A] = 0x2600,
[RDT0_A] = 0x2600,
[RXDCTL0_A] = 0x2600,
[RXCTL0_A] = 0x2600,
[RQDPC0_A] = 0x2600,
[RDBAL1_A] = 0x25D0,
[RDBAL2_A] = 0x25A0,
[RDBAL3_A] = 0x2570,
[RDBAH1_A] = 0x25D0,
[RDBAH2_A] = 0x25A0,
[RDBAH3_A] = 0x2570,
[RDLEN1_A] = 0x25D0,
[RDLEN2_A] = 0x25A0,
[RDLEN3_A] = 0x2570,
[SRRCTL1_A] = 0x25D0,
[SRRCTL2_A] = 0x25A0,
[SRRCTL3_A] = 0x2570,
[RDH1_A] = 0x25D0,
[RDH2_A] = 0x25A0,
[RDH3_A] = 0x2570,
[RDT1_A] = 0x25D0,
[RDT2_A] = 0x25A0,
[RDT3_A] = 0x2570,
[RXDCTL1_A] = 0x25D0,
[RXDCTL2_A] = 0x25A0,
[RXDCTL3_A] = 0x2570,
[RXCTL1_A] = 0x25D0,
[RXCTL2_A] = 0x25A0,
[RXCTL3_A] = 0x2570,
[RQDPC1_A] = 0x25D0,
[RQDPC2_A] = 0x25A0,
[RQDPC3_A] = 0x2570,
[TDBAL0_A] = 0x2A00,
[TDBAH0_A] = 0x2A00,
[TDLEN0_A] = 0x2A00,
[TDH0_A] = 0x2A00,
[TDT0_A] = 0x2A00,
[TXCTL0_A] = 0x2A00,
[TDWBAL0_A] = 0x2A00,
[TDWBAH0_A] = 0x2A00,
[TDBAL1_A] = 0x29D0,
[TDBAL2_A] = 0x29A0,
[TDBAL3_A] = 0x2970,
[TDBAH1_A] = 0x29D0,
[TDBAH2_A] = 0x29A0,
[TDBAH3_A] = 0x2970,
[TDLEN1_A] = 0x29D0,
[TDLEN2_A] = 0x29A0,
[TDLEN3_A] = 0x2970,
[TDH1_A] = 0x29D0,
[TDH2_A] = 0x29A0,
[TDH3_A] = 0x2970,
[TDT1_A] = 0x29D0,
[TDT2_A] = 0x29A0,
[TDT3_A] = 0x2970,
[TXDCTL0_A] = 0x2A00,
[TXDCTL1_A] = 0x29D0,
[TXDCTL2_A] = 0x29A0,
[TXDCTL3_A] = 0x2970,
[TXCTL1_A] = 0x29D0,
[TXCTL2_A] = 0x29A0,
[TXCTL3_A] = 0x29D0,
[TDWBAL1_A] = 0x29D0,
[TDWBAL2_A] = 0x29A0,
[TDWBAL3_A] = 0x2970,
[TDWBAH1_A] = 0x29D0,
[TDWBAH2_A] = 0x29A0,
[TDWBAH3_A] = 0x2970,
/* Access options */
[RDFH] = MAC_ACCESS_PARTIAL, [RDFT] = MAC_ACCESS_PARTIAL,
[RDFHS] = MAC_ACCESS_PARTIAL, [RDFTS] = MAC_ACCESS_PARTIAL,
[RDFPC] = MAC_ACCESS_PARTIAL,
[TDFH] = MAC_ACCESS_PARTIAL, [TDFT] = MAC_ACCESS_PARTIAL,
[TDFHS] = MAC_ACCESS_PARTIAL, [TDFTS] = MAC_ACCESS_PARTIAL,
[TDFPC] = MAC_ACCESS_PARTIAL, [EECD] = MAC_ACCESS_PARTIAL,
[FLA] = MAC_ACCESS_PARTIAL,
[FCAL] = MAC_ACCESS_PARTIAL, [FCAH] = MAC_ACCESS_PARTIAL,
[FCT] = MAC_ACCESS_PARTIAL, [FCTTV] = MAC_ACCESS_PARTIAL,
[FCRTV] = MAC_ACCESS_PARTIAL, [FCRTL] = MAC_ACCESS_PARTIAL,
[FCRTH] = MAC_ACCESS_PARTIAL,
[MAVTV0 ... MAVTV3] = MAC_ACCESS_PARTIAL
};
void
igb_core_write(IGBCore *core, hwaddr addr, uint64_t val, unsigned size)
{
uint16_t index = igb_get_reg_index_with_offset(mac_reg_access, addr);
if (index < IGB_NWRITEOPS && igb_macreg_writeops[index]) {
if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
trace_e1000e_wrn_regs_write_trivial(index << 2);
}
trace_e1000e_core_write(index << 2, size, val);
igb_macreg_writeops[index](core, index, val);
} else if (index < IGB_NREADOPS && igb_macreg_readops[index]) {
trace_e1000e_wrn_regs_write_ro(index << 2, size, val);
} else {
trace_e1000e_wrn_regs_write_unknown(index << 2, size, val);
}
}
uint64_t
igb_core_read(IGBCore *core, hwaddr addr, unsigned size)
{
uint64_t val;
uint16_t index = igb_get_reg_index_with_offset(mac_reg_access, addr);
if (index < IGB_NREADOPS && igb_macreg_readops[index]) {
if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
trace_e1000e_wrn_regs_read_trivial(index << 2);
}
val = igb_macreg_readops[index](core, index);
trace_e1000e_core_read(index << 2, size, val);
return val;
} else {
trace_e1000e_wrn_regs_read_unknown(index << 2, size);
}
return 0;
}
static void
igb_autoneg_resume(IGBCore *core)
{
if (igb_have_autoneg(core) &&
!(core->phy[MII_BMSR] & MII_BMSR_AN_COMP)) {
qemu_get_queue(core->owner_nic)->link_down = false;
timer_mod(core->autoneg_timer,
qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 500);
}
}
void
igb_core_pci_realize(IGBCore *core,
const uint16_t *eeprom_templ,
uint32_t eeprom_size,
const uint8_t *macaddr)
{
int i;
core->autoneg_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
igb_autoneg_timer, core);
igb_intrmgr_pci_realize(core);
for (i = 0; i < IGB_NUM_QUEUES; i++) {
net_tx_pkt_init(&core->tx[i].tx_pkt, E1000E_MAX_TX_FRAGS);
}
net_rx_pkt_init(&core->rx_pkt);
e1000x_core_prepare_eeprom(core->eeprom,
eeprom_templ,
eeprom_size,
PCI_DEVICE_GET_CLASS(core->owner)->device_id,
macaddr);
igb_update_rx_offloads(core);
}
void
igb_core_pci_uninit(IGBCore *core)
{
int i;
timer_free(core->autoneg_timer);
igb_intrmgr_pci_unint(core);
for (i = 0; i < IGB_NUM_QUEUES; i++) {
net_tx_pkt_uninit(core->tx[i].tx_pkt);
}
net_rx_pkt_uninit(core->rx_pkt);
}
static const uint16_t
igb_phy_reg_init[] = {
[MII_BMCR] = MII_BMCR_SPEED1000 |
MII_BMCR_FD |
MII_BMCR_AUTOEN,
[MII_BMSR] = MII_BMSR_EXTCAP |
MII_BMSR_LINK_ST |
MII_BMSR_AUTONEG |
MII_BMSR_MFPS |
MII_BMSR_EXTSTAT |
MII_BMSR_10T_HD |
MII_BMSR_10T_FD |
MII_BMSR_100TX_HD |
MII_BMSR_100TX_FD,
[MII_PHYID1] = IGP03E1000_E_PHY_ID >> 16,
[MII_PHYID2] = (IGP03E1000_E_PHY_ID & 0xfff0) | 1,
[MII_ANAR] = MII_ANAR_CSMACD | MII_ANAR_10 |
MII_ANAR_10FD | MII_ANAR_TX |
MII_ANAR_TXFD | MII_ANAR_PAUSE |
MII_ANAR_PAUSE_ASYM,
[MII_ANLPAR] = MII_ANLPAR_10 | MII_ANLPAR_10FD |
MII_ANLPAR_TX | MII_ANLPAR_TXFD |
MII_ANLPAR_T4 | MII_ANLPAR_PAUSE,
[MII_ANER] = MII_ANER_NP | MII_ANER_NWAY,
[MII_ANNP] = 0x1 | MII_ANNP_MP,
[MII_CTRL1000] = MII_CTRL1000_HALF | MII_CTRL1000_FULL |
MII_CTRL1000_PORT | MII_CTRL1000_MASTER,
[MII_STAT1000] = MII_STAT1000_HALF | MII_STAT1000_FULL |
MII_STAT1000_ROK | MII_STAT1000_LOK,
[MII_EXTSTAT] = MII_EXTSTAT_1000T_HD | MII_EXTSTAT_1000T_FD,
[IGP01E1000_PHY_PORT_CONFIG] = BIT(5) | BIT(8),
[IGP01E1000_PHY_PORT_STATUS] = IGP01E1000_PSSR_SPEED_1000MBPS,
[IGP02E1000_PHY_POWER_MGMT] = BIT(0) | BIT(3) | IGP02E1000_PM_D3_LPLU |
IGP01E1000_PSCFR_SMART_SPEED
};
static const uint32_t igb_mac_reg_init[] = {
[LEDCTL] = 2 | (3 << 8) | BIT(15) | (6 << 16) | (7 << 24),
[EEMNGCTL] = BIT(31),
[TXDCTL0] = E1000_TXDCTL_QUEUE_ENABLE,
[RXDCTL0] = E1000_RXDCTL_QUEUE_ENABLE | (1 << 16),
[RXDCTL1] = 1 << 16,
[RXDCTL2] = 1 << 16,
[RXDCTL3] = 1 << 16,
[RXDCTL4] = 1 << 16,
[RXDCTL5] = 1 << 16,
[RXDCTL6] = 1 << 16,
[RXDCTL7] = 1 << 16,
[RXDCTL8] = 1 << 16,
[RXDCTL9] = 1 << 16,
[RXDCTL10] = 1 << 16,
[RXDCTL11] = 1 << 16,
[RXDCTL12] = 1 << 16,
[RXDCTL13] = 1 << 16,
[RXDCTL14] = 1 << 16,
[RXDCTL15] = 1 << 16,
[TIPG] = 0x08 | (0x04 << 10) | (0x06 << 20),
[CTRL] = E1000_CTRL_FD | E1000_CTRL_LRST | E1000_CTRL_SPD_1000 |
E1000_CTRL_ADVD3WUC,
[STATUS] = E1000_STATUS_PHYRA | BIT(31),
[EECD] = E1000_EECD_FWE_DIS | E1000_EECD_PRES |
(2 << E1000_EECD_SIZE_EX_SHIFT),
[GCR] = E1000_L0S_ADJUST |
E1000_GCR_CMPL_TMOUT_RESEND |
E1000_GCR_CAP_VER2 |
E1000_L1_ENTRY_LATENCY_MSB |
E1000_L1_ENTRY_LATENCY_LSB,
[RXCSUM] = E1000_RXCSUM_IPOFLD | E1000_RXCSUM_TUOFLD,
[TXPBS] = 0x28,
[RXPBS] = 0x40,
[TCTL] = E1000_TCTL_PSP | (0xF << E1000_CT_SHIFT) |
(0x40 << E1000_COLD_SHIFT) | (0x1 << 26) | (0xA << 28),
[TCTL_EXT] = 0x40 | (0x42 << 10),
[DTXCTL] = E1000_DTXCTL_8023LL | E1000_DTXCTL_SPOOF_INT,
[VET] = ETH_P_VLAN | (ETH_P_VLAN << 16),
[V2PMAILBOX0 ... V2PMAILBOX0 + IGB_MAX_VF_FUNCTIONS - 1] = E1000_V2PMAILBOX_RSTI,
[MBVFIMR] = 0xFF,
[VFRE] = 0xFF,
[VFTE] = 0xFF,
[VMOLR0 ... VMOLR0 + 7] = 0x2600 | E1000_VMOLR_STRCRC,
[RPLOLR] = E1000_RPLOLR_STRCRC,
[RLPML] = 0x2600,
[TXCTL0] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL1] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL2] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL3] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL4] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL5] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL6] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL7] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL8] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL9] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL10] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL11] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL12] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL13] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL14] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
[TXCTL15] = E1000_DCA_TXCTRL_DATA_RRO_EN |
E1000_DCA_TXCTRL_TX_WB_RO_EN |
E1000_DCA_TXCTRL_DESC_RRO_EN,
};
static void igb_reset(IGBCore *core, bool sw)
{
struct igb_tx *tx;
int i;
timer_del(core->autoneg_timer);
igb_intrmgr_reset(core);
memset(core->phy, 0, sizeof core->phy);
memcpy(core->phy, igb_phy_reg_init, sizeof igb_phy_reg_init);
for (i = 0; i < E1000E_MAC_SIZE; i++) {
if (sw &&
(i == RXPBS || i == TXPBS ||
(i >= EITR0 && i < EITR0 + IGB_INTR_NUM))) {
continue;
}
core->mac[i] = i < ARRAY_SIZE(igb_mac_reg_init) ?
igb_mac_reg_init[i] : 0;
}
if (qemu_get_queue(core->owner_nic)->link_down) {
igb_link_down(core);
}
e1000x_reset_mac_addr(core->owner_nic, core->mac, core->permanent_mac);
for (int vfn = 0; vfn < IGB_MAX_VF_FUNCTIONS; vfn++) {
/* Set RSTI, so VF can identify a PF reset is in progress */
core->mac[V2PMAILBOX0 + vfn] |= E1000_V2PMAILBOX_RSTI;
}
for (i = 0; i < ARRAY_SIZE(core->tx); i++) {
tx = &core->tx[i];
memset(tx->ctx, 0, sizeof(tx->ctx));
tx->first = true;
tx->skip_cp = false;
}
}
void
igb_core_reset(IGBCore *core)
{
igb_reset(core, false);
}
void igb_core_pre_save(IGBCore *core)
{
int i;
NetClientState *nc = qemu_get_queue(core->owner_nic);
/*
* If link is down and auto-negotiation is supported and ongoing,
* complete auto-negotiation immediately. This allows us to look
* at MII_BMSR_AN_COMP to infer link status on load.
*/
if (nc->link_down && igb_have_autoneg(core)) {
core->phy[MII_BMSR] |= MII_BMSR_AN_COMP;
igb_update_flowctl_status(core);
}
for (i = 0; i < ARRAY_SIZE(core->tx); i++) {
if (net_tx_pkt_has_fragments(core->tx[i].tx_pkt)) {
core->tx[i].skip_cp = true;
}
}
}
int
igb_core_post_load(IGBCore *core)
{
NetClientState *nc = qemu_get_queue(core->owner_nic);
/*
* nc.link_down can't be migrated, so infer link_down according
* to link status bit in core.mac[STATUS].
*/
nc->link_down = (core->mac[STATUS] & E1000_STATUS_LU) == 0;
/*
* we need to restart intrmgr timers, as an older version of
* QEMU can have stopped them before migration
*/
igb_intrmgr_resume(core);
igb_autoneg_resume(core);
return 0;
}