// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2023, Jaguar Micro. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #define DEBUG #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "crete_nic.h" #include "crete_nic_io.h" #include "crete_nic_ethtool.h" static int napi_weight = NAPI_POLL_WEIGHT; module_param(napi_weight, int, 0444); static bool csum = true, gso = true, napi_tx = true; module_param(csum, bool, 0444); module_param(gso, bool, 0444); module_param(napi_tx, bool, 0644); #define CRETE_NIC_DRV_DESCRIPTION "JaguarMicro/CRETE NIC Device Driver" #define CRETE_VNIC_DRV_NAME "crete_vnic" static const unsigned long crete_vnic_guest_offloads[] = { VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, VIRTIO_NET_F_GUEST_CSUM }; #define CRETE_VNIC_FEATURES \ VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \ VIRTIO_NET_F_MAC, \ VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_TSO6, \ VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \ VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \ VIRTIO_NET_F_CTRL_VLAN, \ VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \ VIRTIO_NET_F_CTRL_MAC_ADDR, \ VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_MRG_RXBUF static unsigned int crete_vnic_features[] = { CRETE_VNIC_FEATURES, }; #define CRETE_VIRTIO_DEVICE_ID 0x1C20 /* virtio net */ #define CRETE_VIRTIO_VENDOR_ID 0x1f53 /* virtio net */ struct crete_vnic_wq_ent { struct work_struct work; struct crete_vnic_priv *vnic_priv; }; static bool crete_vnic_get_link_state(struct crete_core_dev *core_dev, struct crete_vnic_priv *vnic_priv) { int ret; bool linkstat = false; ret = crete_get_dev_linkattr(core_dev, &linkstat); if (ret) return false; return linkstat; } static void update_carrier(struct work_struct *work) { struct crete_vnic_wq_ent *wqent; struct crete_vnic_priv *vnic_priv; struct crete_core_dev *core_dev; wqent = container_of(work, struct crete_vnic_wq_ent, work); vnic_priv = wqent->vnic_priv; core_dev = vnic_priv->coredev; if (crete_vnic_get_link_state(core_dev, vnic_priv)) netif_carrier_on(core_dev->netdev); else netif_carrier_off(core_dev->netdev); kfree(wqent); } static int queue_link_work(struct crete_vnic_priv *vnic_priv) { struct crete_vnic_wq_ent *wqent; wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); if (!wqent) return -ENOMEM; wqent->vnic_priv = vnic_priv; INIT_WORK(&wqent->work, update_carrier); queue_work(vnic_priv->wq, &wqent->work); return 0; } static int crete_vnic_notify_handler(struct notifier_block *nb, unsigned long ecode, void *data) { struct crete_nb *cnb = container_of(nb, struct crete_nb, nb); struct crete_vnic_priv *vnic_priv = container_of(cnb, struct crete_vnic_priv, cnb); struct crete_core_dev *core_dev = vnic_priv->coredev; struct crete_event_entry *cee = data; uint8_t event_sub_type = cee->event_sub_type; struct pci_dev *pdev = core_dev->pdev; unsigned int status; //dev_dbg(&pdev->dev, "%s: event code %lu\n sub_tupe %d\n", __func__, crete_info(&pdev->dev, "%s: event code %lu\n sub_tupe %d\n", __func__, ecode, event_sub_type); /* * about the event type is port link change */ if (ecode == CRETE_EVENT_PORT_LINK_CHANGE) { status = event_sub_type; /* * if status exceed RIGHT LINK status * or status not changed anymore. * just return now. */ if (status >= CRETE_EVENT_PORT_LINK_ERR || status == vnic_priv->status) return NOTIFY_DONE; vnic_priv->status = status; /* make sure other core will see it */ smp_mb(); if (queue_link_work(vnic_priv)) return NOTIFY_DONE; } return NOTIFY_OK; } static int crete_vnic_register_event_handler(struct net_device *netdev) { struct crete_vnic_priv *vnic_priv = netdev_priv(netdev); struct crete_core_dev *core_dev = vnic_priv->coredev; struct pci_dev *pdev = core_dev->pdev; struct crete_nb *cnb = &vnic_priv->cnb; struct notifier_block *nb = &cnb->nb; int err; if (!nb->notifier_call) { nb->notifier_call = crete_vnic_notify_handler; cnb->event_type = CRETE_EVENT_PORT_LINK_CHANGE; err = crete_event_notifier_register(core_dev, cnb); if (err) { nb->notifier_call = NULL; dev_err(&pdev->dev, "failed to register pds event handler: %ps\n", ERR_PTR(err)); return -EINVAL; } dev_dbg(&pdev->dev, "crete event handler registered\n"); } return 0; } static void crete_vnic_unregister_event_handler(struct net_device *netdev) { struct crete_vnic_priv *vnic_priv = netdev_priv(netdev); struct crete_core_dev *core_dev = vnic_priv->coredev; struct crete_nb *cnb = &vnic_priv->cnb; struct notifier_block *nb = &cnb->nb; if (nb->notifier_call) { crete_event_notifier_unregister(core_dev, cnb); nb->notifier_call = NULL; } } /* Converting between virtqueue no. and kernel tx/rx queue no. * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq */ static int txq2vq(int txq) { return txq * 2 + 1; } static int rxq2vq(int rxq) { return rxq * 2; } static inline void __vnic_set_bit(struct virtio_device *vdev, unsigned int fbit) { /* Did you forget to fix assumptions on max features? */ if (__builtin_constant_p(fbit)) BUILD_BUG_ON(fbit >= 64); else BUG_ON(fbit >= 64); vdev->features |= BIT_ULL(fbit); } /********************************virtio config ops*********************************************/ static inline struct crete_vnic_priv *to_vnic_net_device(struct virtio_device *vdev) { return container_of(vdev, struct crete_vnic_priv, vdev); } static inline struct crete_core_dev *to_vnic_core_device(struct virtio_device *dev) { struct crete_vnic_priv *cpriv; cpriv = to_vnic_net_device(dev); return cpriv->coredev; } static bool crete_vnic_kick_vq(struct virtqueue *vq) { iowrite16(vq->index, (void __iomem *)vq->priv); return true; } static irqreturn_t crete_vnic_intr_handler(int irq, void *arg) { struct crete_vnic_msix_info *info = arg; return vring_interrupt(irq, info->vq); } static void __iomem *crete_vnic_signal_queue_init_notify(struct crete_core_dev *core_dev, u16 index) { return core_dev->db_base; } static int crete_vnic_signal_queue_init_irq(struct crete_vnic_msix_info *info, struct crete_core_dev *core_dev, u16 index) { struct device *dev = core_dev->device; struct pci_dev *pdev = core_dev->pdev; int vectorno, ret = 0; struct crete_irq_info *irq_info; if (!info) { crete_err(dev, "info is null\n"); ret = -EINVAL; goto err; } vectorno = crete_req_msixirq(core_dev); if (vectorno < 0) { crete_err(dev, "request irq vector failed\n"); ret = -ENXIO; goto err; } irq_info = &core_dev->irq_info[vectorno]; info->msix_vector = vectorno; info->irq = irq_info->vector; snprintf(info->msix_name, 256, "crete-nic[%s]-%d", pci_name(pdev), index); ret = request_irq(irq_info->vector, crete_vnic_intr_handler, 0, info->msix_name, info); if (ret) { crete_free_msixirq(core_dev, vectorno); crete_err(dev, "enable irq failed\n"); goto err; } irq_info->handler = crete_vnic_intr_handler; irq_info->requested = 1; //info->ready = 1; err: return ret; } static void crete_vnic_signal_queue_freeirq(int msix_vector, struct crete_core_dev *core_dev, struct crete_vnic_msix_info *info) { struct crete_irq_info *irq_info; irq_info = &core_dev->irq_info[msix_vector]; if (irq_info->requested == 0) return; irq_info->requested = 0; free_irq(info->irq, info); crete_free_msixirq(core_dev, msix_vector); } static int crete_vnic_signal_queue_create(struct crete_core_dev *core_dev, struct virtqueue *vq, int qid, int msix_vector) { struct crete_queue_context qc; u64 desc_addr, driver_addr, device_addr; int ret = 0; int queue_size = core_dev->cap.qpcap.max_queue_size; struct pci_dev *pdev = core_dev->pdev; desc_addr = virtqueue_get_desc_addr(vq); driver_addr = virtqueue_get_avail_addr(vq); device_addr = virtqueue_get_used_addr(vq); dev_info(&pdev->dev, "qpid[%d] desc_addr[0x%llx] driver_addr[0x%llx] vector[%d]\n", qid, desc_addr, driver_addr, msix_vector); qc.qid = qid; qc.queue_size = queue_size; qc.cq_size = queue_size; qc.queue_vec = msix_vector; qc.queue_desc_base = desc_addr; qc.queue_used_base = desc_addr; //qc.queue_used_base = device_addr; ret = crete_cmd_create_signal_queue(core_dev, &qc); if (ret < 0) { crete_err(&pdev->dev, "crete create queue failed ret:%d\n", ret); ret = -EINVAL; goto err; } err: return ret; } static void crete_vnic_driver_resetqueue(struct crete_core_dev *core_dev, int qid) { int ret; ret = crete_reset_singlequeue(core_dev, qid); if (ret) { crete_err(core_dev->device, "crete reset queue id:%d failed\n", qid); ret = -EINVAL; } } static void crete_vnic_reset_io_queue(struct crete_core_dev *core_dev, int endqid) { int i; struct net_device *netdev = core_dev->netdev; struct crete_vnic_priv *vnic_info = netdev_priv(netdev); struct crete_vnic_msix_info *msix_info; for (i = endqid - 1; i >= 0; i--) { msix_info = &vnic_info->msix_info[i]; if (msix_info->irq) crete_vnic_signal_queue_freeirq(msix_info->msix_vector, core_dev, msix_info); if (msix_info->ready) crete_vnic_driver_resetqueue(core_dev, i); msix_info->ready = 0; msix_info->msix_vector = 0; msix_info->irq = 0; msix_info->vq = NULL; } } static void crete_vnic_del_vqs(struct virtio_device *vdev) { struct virtqueue *vq, *n; list_for_each_entry_safe(vq, n, &vdev->vqs, list) { vring_del_virtqueue(vq); } } static struct virtqueue *crete_vnic_setup_vq(struct virtio_device *vdev, unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, bool ctx) { struct crete_vnic_priv *vnic_info = to_vnic_net_device(vdev); struct crete_core_dev *core_dev = to_vnic_core_device(vdev); struct virtqueue *vq; struct pci_dev *pdev = core_dev->pdev; struct crete_vnic_msix_info *info; int err; int total_vqs = 2 * core_dev->cap.qpcap.max_qp_num; u32 max_queue_size = core_dev->cap.qpcap.max_queue_size; if (!name) return NULL; if (index >= total_vqs || index >= CRETE_VNIC_MQ_MAX) return ERR_PTR(-ENOENT); /* Queue shouldn't already be set up. */ vq = vring_create_virtqueue(index, max_queue_size, PAGE_SIZE, vdev, true, true, ctx, crete_vnic_kick_vq, callback, name); if (!vq) { err = -ENOMEM; goto error_new_virtqueue; } info = &vnic_info->msix_info[index]; info->qid = index; info->vq = vq; vq->priv = (void __force *)crete_vnic_signal_queue_init_notify(core_dev, index); if (!vq->priv) { err = -ENOMEM; goto error_new_virtqueue; } if (callback) { err = crete_vnic_signal_queue_init_irq(info, core_dev, index); if (err) { dev_err(&pdev->dev, "crete_vnic_signal_queue_init_irq failed\n"); err = -ENOMEM; goto error_new_virtqueue; } } else { info->irq = 0; info->msix_vector = VIRTIO_MSI_NO_VECTOR; info->ready = 0; } err = crete_vnic_signal_queue_create(core_dev, vq, index, info->msix_vector); if (err) { dev_err(&pdev->dev, "create queue is error qid[%d]\n", index); err = -ENOMEM; goto error_new_virtqueue; } info->ready = 1; err = crete_start_singlequeue(core_dev, index); if (err) { dev_err(&pdev->dev, "start queue [%d]is error\n", index); err = -ENOMEM; goto error_new_virtqueue; } return vq; error_new_virtqueue: return ERR_PTR(err); } static int crete_vnic_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char *const names[], const bool *ctx, struct irq_affinity *desc) { struct crete_core_dev *core_dev = to_vnic_core_device(vdev); int i, err, queue_idx = 0; for (i = 0; i < nvqs; ++i) { if (!names[i]) { vqs[i] = NULL; continue; } vqs[i] = crete_vnic_setup_vq(vdev, queue_idx++, callbacks[i], names[i], ctx ? ctx[i] : false); if (IS_ERR(vqs[i])) { err = PTR_ERR(vqs[i]); goto err_setup_vq; } } return 0; err_setup_vq: crete_vnic_reset_io_queue(core_dev, i); crete_vnic_del_vqs(vdev); return err; } static int crete_vnic_alloc_queues(struct crete_vnic_priv *vnic_info, struct crete_core_dev *core_dev) { struct net_device *netdev = core_dev->netdev; int i, max_queue_pairs; max_queue_pairs = core_dev->cap.qpcap.max_qp_num; vnic_info->sq = kcalloc(max_queue_pairs, sizeof(*vnic_info->sq), GFP_KERNEL); if (!vnic_info->sq) goto err; vnic_info->rq = kcalloc(max_queue_pairs, sizeof(*vnic_info->rq), GFP_KERNEL); if (!vnic_info->rq) goto err_rq; spin_lock_init(&vnic_info->refill_lock); INIT_DELAYED_WORK(&vnic_info->refill, crete_vnic_alloc_recv_buf); vnic_info->mergeable_rx_bufs = true; for (i = 0; i < max_queue_pairs; i++) { vnic_info->rq[i].pages = NULL; #ifdef NETIF_NAPI_ADD_NEWAPI netif_napi_add_weight(netdev, &vnic_info->rq[i].napi, crete_vnic_poll, napi_weight); netif_napi_add_tx_weight(netdev, &vnic_info->sq[i].napi, crete_vnic_poll_tx, napi_tx ? napi_weight : 0); #else netif_napi_add(netdev, &vnic_info->rq[i].napi, crete_vnic_poll, napi_weight); netif_tx_napi_add(netdev, &vnic_info->sq[i].napi, crete_vnic_poll_tx, napi_tx ? napi_weight : 0); #endif sg_init_table(vnic_info->rq[i].sg, ARRAY_SIZE(vnic_info->rq[i].sg)); ewma_pkt_len_init(&vnic_info->rq[i].mrg_avg_pkt_len); sg_init_table(vnic_info->sq[i].sg, ARRAY_SIZE(vnic_info->sq[i].sg)); u64_stats_init(&vnic_info->rq[i].stats.syncp); u64_stats_init(&vnic_info->sq[i].stats.syncp); } return 0; err_rq: kfree(vnic_info->sq); err: return -ENOMEM; } static void crete_vnic_free_queues(struct crete_vnic_priv *vnic_info, struct crete_core_dev *core_dev) { int i, max_queue_pairs; max_queue_pairs = core_dev->cap.qpcap.max_qp_num; for (i = 0; i < max_queue_pairs; i++) { __netif_napi_del(&vnic_info->rq[i].napi); __netif_napi_del(&vnic_info->sq[i].napi); } /* We called __netif_napi_del(), * we need to respect an RCU grace period before freeing vi->rq */ synchronize_net(); kfree(vnic_info->rq); kfree(vnic_info->sq); } static unsigned int crete_get_min_buf_len(struct virtqueue *vq) { const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); unsigned int rq_size = virtqueue_get_vring_size(vq); unsigned int packet_len = IP_MAX_MTU; unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len; unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size); unsigned int ret; ret = max(max(min_buf_len, hdr_len) - hdr_len, (unsigned int)GOOD_PACKET_LEN); return ret; } static int crete_vnic_setup_vqs(struct crete_vnic_priv *vnic_info) { vq_callback_t **callbacks; struct virtqueue **vqs; int ret = -ENOMEM; int i, total_vqs; const char **names; bool *ctx; struct crete_core_dev *core_dev = vnic_info->coredev; total_vqs = 2 * core_dev->cap.qpcap.max_qp_num; vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL); if (!vqs) goto err_vq; callbacks = kmalloc_array(total_vqs, sizeof(*callbacks), GFP_KERNEL); if (!callbacks) goto err_callback; names = kmalloc_array(total_vqs, sizeof(*names), GFP_KERNEL); if (!names) goto err_names; if (!vnic_info->big_packets || vnic_info->mergeable_rx_bufs) { ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL); if (!ctx) goto err_ctx; } else { ctx = NULL; } /* Parameters for control virtqueue, if any */ /* Allocate/initialize parameters for send/receive virtqueues */ for (i = 0; i < core_dev->cap.qpcap.max_qp_num; i++) { callbacks[rxq2vq(i)] = crete_vnic_recv_done; callbacks[txq2vq(i)] = crete_vnic_xmit_done; sprintf(vnic_info->rq[i].name, "vnic-input.%d", i); sprintf(vnic_info->sq[i].name, "vnic-output.%d", i); names[rxq2vq(i)] = vnic_info->rq[i].name; names[txq2vq(i)] = vnic_info->sq[i].name; if (ctx) ctx[rxq2vq(i)] = true; } ret = crete_vnic_find_vqs(&vnic_info->vdev, total_vqs, vqs, callbacks, names, ctx, NULL); if (ret) goto err_find; for (i = 0; i < core_dev->cap.qpcap.max_qp_num; i++) { vnic_info->rq[i].vq = vqs[rxq2vq(i)]; vnic_info->rq[i].min_buf_len = crete_get_min_buf_len(vnic_info->rq[i].vq); vnic_info->sq[i].vq = vqs[txq2vq(i)]; } err_find: kfree(ctx); err_ctx: kfree(names); err_names: kfree(callbacks); err_callback: kfree(vqs); err_vq: return ret; } static int crete_vnic_request_rings(struct crete_core_dev *core_dev) { struct net_device *netdev = core_dev->netdev; struct crete_vnic_priv *priv = netdev_priv(netdev); struct pci_dev *pdev = core_dev->pdev; int ret = 0; ret = crete_vnic_alloc_queues(priv, core_dev); if (ret) { dev_err(&pdev->dev, "crete_vnic_alloc_queues failed\n"); return -ENOMEM; } ret = crete_vnic_setup_vqs(priv); if (ret) { dev_err(&pdev->dev, "crete_vnic_setup_vqs failed\n"); goto alloc_queues; } return 0; alloc_queues: crete_vnic_free_queues(priv, core_dev); return ret; } static void crete_vnic_distroy_rings(struct crete_core_dev *core_dev) { struct net_device *netdev = core_dev->netdev; struct crete_vnic_priv *priv = netdev_priv(netdev); struct virtio_device *vdev = &priv->vdev; crete_vnic_del_vqs(vdev); crete_vnic_free_queues(priv, core_dev); } static void crete_nic_negotiate_driver_features(struct crete_core_dev *core_dev) { u64 device_features; u64 driver_features; int i, feature_table_size; device_features = core_dev->cap.hw_features; feature_table_size = ARRAY_SIZE(crete_vnic_features), /* Figure out what features the driver supports. */ driver_features = 0; for (i = 0; i < feature_table_size; i++) { unsigned int f = crete_vnic_features[i]; BUG_ON(f >= 64); driver_features |= (1ULL << f); } core_dev->cap.driver_features = driver_features & device_features; /* Transport features always preserved to pass to finalize_features. */ for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) if (device_features & (1ULL << i)) core_dev->cap.driver_features |= BIT_ULL(i); } static inline bool crete_vnic_has_feature(struct crete_core_dev *core_dev, unsigned int fbit) { bool ret = false; ret = !!(core_dev->cap.driver_features & BIT_ULL(fbit)); return ret; } static int crete_build_vnic_netdev_features(struct net_device *netdev) { struct crete_vnic_priv *priv = netdev_priv(netdev); struct crete_core_dev *core_dev = priv->coredev; struct pci_dev *pdev = core_dev->pdev; int i; netdev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE; netdev->features = NETIF_F_HIGHDMA; crete_build_ptys2ethtool_map(); crete_set_netdev_ops(netdev); crete_set_ethtool_ops(netdev); SET_NETDEV_DEV(netdev, &pdev->dev); /* Do we support "hardware" checksums? */ if (crete_vnic_has_feature(core_dev, VIRTIO_NET_F_CSUM)) { /* This opens up the world of extra features. */ netdev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG; if (csum) netdev->features |= NETIF_F_HW_CSUM | NETIF_F_SG; /* Individual feature bits: what can host handle? */ if (crete_vnic_has_feature(core_dev, VIRTIO_NET_F_HOST_TSO4)) netdev->hw_features |= NETIF_F_TSO; if (crete_vnic_has_feature(core_dev, VIRTIO_NET_F_HOST_TSO6)) netdev->hw_features |= NETIF_F_TSO6; if (crete_vnic_has_feature(core_dev, VIRTIO_NET_F_HOST_ECN)) netdev->hw_features |= NETIF_F_TSO_ECN; netdev->features |= NETIF_F_GSO_ROBUST; if (gso) netdev->features |= netdev->hw_features & NETIF_F_ALL_TSO; /* (!csum && gso) case will be fixed by register_netdev() */ } if (crete_vnic_has_feature(core_dev, VIRTIO_NET_F_GUEST_CSUM)) netdev->features |= NETIF_F_RXCSUM; if (crete_vnic_has_feature(core_dev, VIRTIO_NET_F_GUEST_TSO4) || crete_vnic_has_feature(core_dev, VIRTIO_NET_F_GUEST_TSO6)) netdev->features |= NETIF_F_GRO_HW; if (crete_vnic_has_feature(core_dev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) netdev->hw_features |= NETIF_F_GRO_HW; netdev->vlan_features = netdev->features; for (i = 0; i < ARRAY_SIZE(crete_vnic_guest_offloads); i++) if (crete_vnic_has_feature (core_dev, crete_vnic_guest_offloads[i])) set_bit(crete_vnic_guest_offloads[i], &priv->guest_offloads); priv->guest_offloads_capable = priv->guest_offloads; return 0; } static int crete_vnic_init_mac_addr(struct net_device *netdev) { struct crete_vnic_priv *priv = netdev_priv(netdev); struct crete_net_common_cfg *netcfg = &priv->net_cfg; struct crete_core_dev *core_dev = priv->coredev; if (crete_vnic_has_feature(core_dev, VIRTIO_NET_F_MAC)) memcpy(netdev->dev_addr, netcfg->mac, ETH_ALEN); else eth_hw_addr_random(netdev); return 0; } int crete_vnic_set_mac_address(struct net_device *dev, void *p); static int crete_vnic_init_mac_addr_new(struct net_device *netdev) { struct crete_vnic_priv *priv = netdev_priv(netdev); struct crete_core_dev *core_dev = priv->coredev; struct pci_dev *pdev = core_dev->pdev; struct sockaddr addr; int err = 0; memset(&addr, 0, sizeof(struct sockaddr)); /* init get the mac default address */ err = crete_get_dev_macattr(core_dev, &addr); if (err) { dev_err(&pdev->dev, "vnic get default mac address faile\n"); return -1; } /* config the default mac */ err = crete_vnic_set_mac_address(netdev, (void *)(&addr)); if (err) { dev_err(&pdev->dev, "vnic get default mac address faile\n"); return -1; } return 0; } static int crete_vnic_init_mtu(struct net_device *netdev) { struct crete_vnic_priv *priv = netdev_priv(netdev); struct crete_core_dev *core_dev = priv->coredev; struct pci_dev *pdev = core_dev->pdev; struct crete_net_common_cfg *netcfg = &priv->net_cfg; int vnic_mtu = 1500; int err = 0; /* MTU range: 68 - 65535 */ netdev->min_mtu = ETH_MIN_MTU; netdev->max_mtu = ETH_MAX_MTU; if (crete_vnic_has_feature(core_dev, VIRTIO_NET_F_MTU)) { vnic_mtu = netcfg->mtu; if (vnic_mtu < netdev->min_mtu) { /* Should never trigger: MTU was previously validated * in virtnet_validate. */ dev_err(&pdev->dev, "device MTU appears to have changed it is now %d < %d", vnic_mtu, netdev->min_mtu); err = -EINVAL; return err; } netdev->mtu = vnic_mtu; netdev->max_mtu = ETH_MAX_MTU; } else { netdev->mtu = 1500; netdev->max_mtu = ETH_MAX_MTU; } err = crete_set_port_mtu(core_dev, CRETE_SET_MTU_MACPORT, MAC_MTU(netdev->mtu)); if (err) { dev_err(&pdev->dev,"init port mtu (%u) error info\n", netdev->mtu); return err; } return 0; } static int crete_vnic_init_mru(struct net_device *netdev) { return 0; } #define NEW_MAC_ADDRESS "\x56\x48\x4d\x47\x00\x00" #define NEW_MAC_ADDRESS1 "\x56\x48\x4d\x47\x00\x01" int crete_vport_get_net_comm_cfg(struct crete_core_dev *cdev, struct crete_net_common_cfg *netcfg) { //int ret = 0; //todo :get form vdev static int mac_index = 0; netcfg->vportid = 0; netcfg->speed = 0x01; netcfg->duplex = 0x01; netcfg->mtu = 1500; netcfg->vlanid = 0xffff; memcpy(netcfg->mac, NEW_MAC_ADDRESS, ETH_ALEN); if( mac_index) memcpy(netcfg->mac, NEW_MAC_ADDRESS1, ETH_ALEN); mac_index ++ ; crete_info(cdev->device, "vportid[%d] speed[0x%x] duplex[0x%x] mtu[%d] vlanid[%d] mac[%pM]\n", netcfg->vportid, netcfg->speed, netcfg->duplex, netcfg->mtu, netcfg->vlanid, netcfg->mac); return 0; } static int crete_vnic_init_net_cfg(struct net_device *netdev) { struct crete_vnic_priv *priv = netdev_priv(netdev); struct crete_core_dev *core_dev = priv->coredev; struct pci_dev *pdev = core_dev->pdev; struct crete_net_common_cfg *netcfg = &priv->net_cfg; int ret = 0; ret = crete_vport_get_net_comm_cfg(core_dev, netcfg); if (ret) { dev_err(&pdev->dev, "get net cfg erro ret %d\n", ret); goto err; } ret = crete_vnic_init_mac_addr_new(netdev); if (ret) { dev_err(&pdev->dev, "Unable to initialize mac address. Use the ramdom mac\n"); crete_vnic_init_mac_addr(netdev); } ret = crete_vnic_init_mtu(netdev); if (ret) { dev_err(&pdev->dev, "Unable to initialize mtu.\n"); goto err; } ret = crete_vnic_init_mru(netdev); if (ret) { dev_err(&pdev->dev, "Unable to initialize mru.\n"); goto err; } err: return ret; } static void crete_virtio_release_dev(struct device *_d) { struct virtio_device *vdev = container_of(_d, struct virtio_device, dev); struct crete_vnic_priv *priv = container_of(vdev, struct crete_vnic_priv, vdev); struct crete_core_dev *core_dev = priv->coredev; dev_info(core_dev->device, "release virtio device\n"); } static int crete_vnic_set_priv(struct net_device *netdev) { struct crete_vnic_priv *priv = netdev_priv(netdev); struct crete_core_dev *core_dev = priv->coredev; struct pci_dev *pdev = core_dev->pdev; int maxqueue = 2 * core_dev->cap.qpcap.max_qp_num; struct virtio_device *vdev = &priv->vdev; int ret = 0; int i; vdev->priv = priv; vdev->dev.parent = &pdev->dev; vdev->dev.release = crete_virtio_release_dev; priv->curr_queue_pairs = core_dev->ring_size; priv->max_queue_pairs = core_dev->cap.qpcap.max_qp_num; //crete_vnic_priv->vdev.config = &crete_vnic_config_ops; //INIT_LIST_HEAD(&crete_vnic_priv->vvdev->featuresirtqueues); spin_lock_init(&priv->lock); vdev->id.vendor = CRETE_VIRTIO_VENDOR_ID; vdev->id.device = CRETE_VIRTIO_DEVICE_ID; vdev->features = core_dev->cap.driver_features; spin_lock_init(&vdev->config_lock); vdev->config_enabled = false; vdev->config_change_pending = false; INIT_LIST_HEAD(&vdev->vqs); spin_lock_init(&vdev->vqs_list_lock); for (i = 0; i < maxqueue; i++) { priv->msix_info[i].irq = 0; priv->msix_info[i].msix_vector = VIRTIO_MSI_NO_VECTOR; priv->msix_info[i].ready = 0; priv->msix_info[i].vq = NULL; } ret = crete_cmd_set_features(core_dev, CRETE_VIRTIO_NET_DEV_FEAT, vdev->features); if (ret) { dev_err(&pdev->dev, "crete set features 0x%llx failed\n", vdev->features); goto err; } ret = crete_set_status(core_dev, CRETE_NET_DEV_STATUS, CRETE_NET_DEV_FEATURE_OK); if (ret) { dev_err(&pdev->dev, "crete cmd set status %u failed\n", CRETE_NET_DEV_FEATURE_OK); goto err; } priv->big_packets = true; priv->mergeable_rx_bufs = true; priv->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); priv->any_header_sg = true; netif_set_real_num_tx_queues(netdev, priv->curr_queue_pairs); netif_set_real_num_rx_queues(netdev, priv->curr_queue_pairs); return ret; err: return ret; } int crete_vnic_trim_rings(struct crete_core_dev *cdev) { int cpunums = num_online_cpus(); int maxqpnums = cdev->cap.qpcap.max_qp_num; if (maxqpnums > CRETE_VNIC_MAX_QUEUES) { cdev->cap.qpcap.max_qp_num = CRETE_VNIC_MAX_QUEUES; crete_err(cdev->device, "max qp greater than %d\n", CRETE_VNIC_MAX_QUEUES); } cdev->ring_size = min(cpunums, maxqpnums); if (!cdev->ring_size) { crete_err(cdev->device, "ring size zero not right\n"); return -1; } return 0; } void crete_vnic_set_rx_mode_work(struct work_struct *work); static int crete_vnic_net_init(struct crete_core_dev *core_dev) { struct pci_dev *pdev = core_dev->pdev; struct device *dev = &pdev->dev; struct crete_vnic_priv *priv; struct net_device *netdev; int err; /* reserve the ring size */ err = crete_vnic_trim_rings(core_dev); if (err) return err; netdev = crete_vnic_create_netdev(core_dev); if (!netdev) { dev_err(dev, "crete_create_netdev failed\n"); return -ENOMEM; } priv = netdev_priv(netdev); crete_build_common_netdev(netdev); err = crete_build_vnic_netdev_features(netdev); if (err) { dev_err(dev, "build netdev features failed, %d\n", err); goto err_crete_create_netdev; } err = crete_get_func_caps(core_dev); if (err < 0) { dev_err(dev, "init func err\n"); goto err_crete_create_netdev; } err = crete_vnic_set_priv(netdev); if (err) { dev_err(dev, "Unable to initialize prive.\n"); goto err_crete_create_netdev; } err = crete_vnic_init_net_cfg(netdev); if (err) { dev_err(dev, "Unable to initialize mac address.\n"); goto err_init_set_priv; } err = crete_vnic_request_rings(core_dev); if (err) { dev_err(dev, "Reserve rings failed\n"); goto err_init_set_priv; } priv->wq = create_singlethread_workqueue("crete_nic_wq"); if (!priv->wq) { dev_err(dev, "Reserve workqueue faild\n"); err = -ENOMEM; goto err_init_set_priv; } /* init rx mode work */ INIT_WORK(&priv->set_rx_mode_work, crete_vnic_set_rx_mode_work); err = crete_vnic_register_event_handler(netdev); if (err) { dev_err(dev, "register evnet faild\n"); crete_vnic_unregister_event_handler(netdev); goto err_init_set_priv; } return 0; //err_request_rings: // crete_vnic_distroy_rings(core_dev); err_init_set_priv: err_crete_create_netdev: priv = netdev_priv(netdev); crete_vnic_priv_cleanup(priv); return err; } int crete_vnic_net_del(struct crete_core_dev *core_dev) { struct net_device *netdev = core_dev->netdev; struct crete_vnic_priv *priv; struct workqueue_struct *wq; priv = netdev_priv(netdev); wq = priv->wq; crete_vnic_unregister_event_handler(netdev); destroy_workqueue(wq); crete_vnic_distroy_rings(core_dev); crete_vnic_priv_cleanup(priv); priv->wq = NULL; return 0; } static int crete_vnic_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { struct crete_aux_dev *cadev = container_of(adev, struct crete_aux_dev, adev); struct crete_core_dev *core_dev = cadev->core_dev; struct pci_dev *pdev = core_dev->pdev; struct device *dev = &pdev->dev; enum crete_feature_opcode op = CRETE_VIRTIO_NET_DEV_FEAT; enum crete_dev_type crete_pci_type = CRETE_VNET_DEV; struct crete_vnic_priv *priv = NULL; int ret; ret = crete_nic_set_device_type(core_dev, crete_pci_type); if (ret) { dev_err(dev, "Failed to set device type pnic\n"); return -EINVAL; } ret = crete_set_status(core_dev, CRETE_NET_DEV_STATUS, CRETE_NET_DEV_STARTUP); if (ret) { dev_err(dev, "crete cmd set status %u failed\n", CRETE_NET_DEV_STARTUP); return -EINVAL; } /* get core dev cap init */ ret = crete_net_get_max_supported_vqs(core_dev); if (ret) { dev_err(dev, "crete core dev get qpcap failed\n"); return -EINVAL; } ret = crete_net_get_supported_features(core_dev, op); if (ret) { dev_err(dev, "crete core dev get features failed\n"); return -EINVAL; } crete_nic_negotiate_driver_features(core_dev); ret = crete_vnic_net_init(core_dev); if (ret) { dev_err(dev, "Failed to init net dev management\n"); goto err_return; } ret = register_netdev(core_dev->netdev); if (ret) { dev_err(dev, "register_netdev failed, %d\n", ret); goto err_net_init; } priv = netdev_priv(core_dev->netdev); /* config the netdev queue pairs */ ret = crete_vnic_set_queues(core_dev->netdev, priv->curr_queue_pairs); if (ret) { dev_err(dev, "crete vnic set queues %d failed\n", priv->curr_queue_pairs); goto err_net_registert; } netif_carrier_off(core_dev->netdev); ret = crete_set_status(core_dev, CRETE_NET_DEV_STATUS, CRETE_NET_DEV_DEV_OK); if (ret) { dev_err(dev, "Failed to set device CRETE_NET_DEV_DEV_OK\n"); goto err_net_registert; } #if 0 /* carrier off reporting is important to ethtool even BEFORE open */ if (crete_vnic_get_link_state(core_dev)) netif_carrier_on(core_dev->netdev); else netif_carrier_off(core_dev->netdev); #endif dev_set_drvdata(&adev->dev, core_dev); //crete_vnic_debugfs_add_dev(cvm); //crete_vnic_debugfs_add_hcap(cvm); return 0; err_net_registert: unregister_netdev(core_dev->netdev); err_net_init: crete_vnic_net_del(core_dev); err_return: return ret; } static void crete_vnic_reset_device(struct crete_core_dev *core_dev) { int ret; enum crete_net_status op = CRETE_NET_DEV_STARTUP; int max_queue = core_dev->cap.qpcap.max_qp_num * 2; crete_vnic_reset_io_queue(core_dev, max_queue); ret = crete_set_status(core_dev, CRETE_NET_DEV_STATUS, op); if (ret) { crete_err(core_dev->device, "crete cmd set status %u failed\n", op); WARN_ON(1); } } static void crete_remove_vq_common(struct crete_core_dev *core_dev) { struct net_device *netdev = core_dev->netdev; struct crete_vnic_priv *priv = netdev_priv(netdev); crete_vnic_reset_device(core_dev); /* Free unused buffers in both send and recv, if any. */ free_unused_bufs(priv); //todo free_receive_bufs(priv); //todo free_receive_page_frags(priv); //todo crete_vnic_net_del(core_dev); free_netdev(netdev); core_dev->netdev = NULL; } /* * about the anolis 5.10.134-14 kernel version * auxliary define with return int value */ #ifdef SNIC_ANOLIS_VERSION14 static int crete_vnic_remove(struct auxiliary_device *adev) { struct crete_aux_dev *cadev = container_of(adev, struct crete_aux_dev, adev); struct crete_core_dev *core_dev = cadev->core_dev; struct pci_dev *pdev = core_dev->pdev; struct device *dev = &pdev->dev; struct crete_vnic_priv *priv = netdev_priv(core_dev->netdev); cancel_work_sync(&priv->set_rx_mode_work); unregister_netdev(core_dev->netdev); crete_remove_vq_common(core_dev); dev_set_drvdata(&adev->dev, NULL); dev_info(dev, "crete vnic remove\n"); return 0; } #else static void crete_vnic_remove(struct auxiliary_device *adev) { struct crete_aux_dev *cadev = container_of(adev, struct crete_aux_dev, adev); struct crete_core_dev *core_dev = cadev->core_dev; struct pci_dev *pdev = core_dev->pdev; struct device *dev = &pdev->dev; struct crete_vnic_priv *priv = netdev_priv(core_dev->netdev); cancel_work_sync(&priv->set_rx_mode_work); unregister_netdev(core_dev->netdev); crete_remove_vq_common(core_dev); dev_set_drvdata(&adev->dev, NULL); dev_info(dev, "crete vnic remove\n"); } #endif static const struct auxiliary_device_id crete_vnic_id_table[] = { {.name = CRETE_VNIC_AUX_DEV_NAME,}, {}, }; static struct auxiliary_driver crete_vnic_driver = { .name = CRETE_VNIC_DRV_NAME, .probe = crete_vnic_probe, .remove = crete_vnic_remove, .id_table = crete_vnic_id_table, }; static void __exit crete_vnic_cleanup(void) { auxiliary_driver_unregister(&crete_vnic_driver); // crete_vnet_debugfs_destroy(); } module_exit(crete_vnic_cleanup); static int __init crete_vnic_init(void) { int err; // crete_nic_debugfs_create(); err = auxiliary_driver_register(&crete_vnic_driver); if (err) { pr_err("%s: aux driver register failed: %pe\n", CRETE_NIC_DRV_NAME, ERR_PTR(err)); //crete_nic_debugfs_destroy(); } return err; } module_init(crete_vnic_init); MODULE_DESCRIPTION(CRETE_NIC_DRV_DESCRIPTION); MODULE_AUTHOR("jaguarmicro.com"); MODULE_LICENSE("GPL");