xilinx xdma.rar

/* * This file is part of the Xilinx DMA IP Core driver for Linux * * Copyright (c) 2016-present, Xilinx, Inc. * All rights reserved. * * This source code is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * The full GNU General Public License is included in this distribution in * the file called "COPYING". */ #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ #include <linux/module.h> #include <linux/kernel.h> #include <linux/string.h> #include <linux/mm.h> #include <linux/errno.h> #include <linux/sched.h> #include <linux/vmalloc.h> #include "libxdma.h" #include "libxdma_api.h" #include "cdev_sgdma.h" #include "xdma_thread.h" /* Module Parameters */ static unsigned int poll_mode; module_param(poll_mode, uint, 0644); MODULE_PARM_DESC(poll_mode, "Set 1 for hw polling, default is 0 (interrupts)"); static unsigned int interrupt_mode; module_param(interrupt_mode, uint, 0644); MODULE_PARM_DESC(interrupt_mode, "0 - Auto , 1 - MSI, 2 - Legacy, 3 - MSI-x"); static unsigned int enable_st_c2h_credit = 0; module_param(enable_st_c2h_credit, uint, 0644); MODULE_PARM_DESC(enable_st_c2h_credit, "Set 1 to enable ST C2H engine credit feature, default is 0 ( credit control disabled)"); unsigned int desc_blen_max = XDMA_DESC_BLEN_MAX; module_param(desc_blen_max, uint, 0644); MODULE_PARM_DESC(desc_blen_max, "per descriptor max. buffer length, default is (1 << 28) - 1"); #define XDMA_PERF_NUM_DESC 128 /* Kernel version adaptative code */ #if HAS_SWAKE_UP_ONE /* since 4.18, using simple wait queues is not recommended * except for realtime constraint (see swait.h comments) * and will likely be removed in future kernel versions */ #define xlx_wake_up swake_up_one #define xlx_wait_event_interruptible_timeout \ swait_event_interruptible_timeout_exclusive #define xlx_wait_event_interruptible \ swait_event_interruptible_exclusive #elif HAS_SWAKE_UP #define xlx_wake_up swake_up #define xlx_wait_event_interruptible_timeout \ swait_event_interruptible_timeout #define xlx_wait_event_interruptible \ swait_event_interruptible #else #define xlx_wake_up wake_up_interruptible /* wait_event_interruptible_timeout() could return prematurely (-ERESTARTSYS) * if it is interrupted by a signal */ #define xlx_wait_event_interruptible_timeout(wq, condition, timeout) \ ({\ int __ret = 0; \ unsigned long expire = timeout + jiffies; \ do { \ __ret = wait_event_interruptible_timeout(wq, condition, \ timeout); \ } while ((__ret < 0) && (jiffies < expire)); \ __ret; \ }) #define xlx_wait_event_interruptible \ wait_event_interruptible #endif /* * xdma device management * maintains a list of the xdma devices */ static LIST_HEAD(xdev_list); static DEFINE_MUTEX(xdev_mutex); static LIST_HEAD(xdev_rcu_list); static DEFINE_SPINLOCK(xdev_rcu_lock); #ifndef list_last_entry #define list_last_entry(ptr, type, member) list_entry((ptr)->prev, type, member) #endif static inline int xdev_list_add(struct xdma_dev *xdev) { mutex_lock(&xdev_mutex); if (list_empty(&xdev_list)) { xdev->idx = 0; if (poll_mode) { int rv = xdma_threads_create(xdev->h2c_channel_max + xdev->c2h_channel_max); if (rv < 0) { mutex_unlock(&xdev_mutex); return rv; } } } else { struct xdma_dev *last; last = list_last_entry(&xdev_list, struct xdma_dev, list_head); xdev->idx = last->idx + 1; } list_add_tail(&xdev->list_head, &xdev_list); mutex_unlock(&xdev_mutex); dbg_init("dev %s, xdev 0x%p, xdma idx %d.\n", dev_name(&xdev->pdev->dev), xdev, xdev->idx); spin_lock(&xdev_rcu_lock); list_add_tail_rcu(&xdev->rcu_node, &xdev_rcu_list); spin_unlock(&xdev_rcu_lock); return 0; } #undef list_last_entry static inline void xdev_list_remove(struct xdma_dev *xdev) { mutex_lock(&xdev_mutex); list_del(&xdev->list_head); if (poll_mode && list_empty(&xdev_list)) xdma_threads_destroy(); mutex_unlock(&xdev_mutex); spin_lock(&xdev_rcu_lock); list_del_rcu(&xdev->rcu_node); spin_unlock(&xdev_rcu_lock); synchronize_rcu(); } struct xdma_dev *xdev_find_by_pdev(struct pci_dev *pdev) { struct xdma_dev *xdev, *tmp; mutex_lock(&xdev_mutex); list_for_each_entry_safe(xdev, tmp, &xdev_list, list_head) { if (xdev->pdev == pdev) { mutex_unlock(&xdev_mutex); return xdev; } } mutex_unlock(&xdev_mutex); return NULL; } static inline int debug_check_dev_hndl(const char *fname, struct pci_dev *pdev, void *hndl) { struct xdma_dev *xdev; if (!pdev) return -EINVAL; xdev = xdev_find_by_pdev(pdev); if (!xdev) { pr_info("%s pdev 0x%p, hndl 0x%p, NO match found!\n", fname, pdev, hndl); return -EINVAL; } if (xdev != hndl) { pr_err("%s pdev 0x%p, hndl 0x%p != 0x%p!\n", fname, pdev, hndl, xdev); return -EINVAL; } return 0; } #ifdef __LIBXDMA_DEBUG__ /* SECTION: Function definitions */ inline void __write_register(const char *fn, u32 value, void *iomem, unsigned long off) { pr_err("%s: w reg 0x%lx(0x%p), 0x%x.\n", fn, off, iomem, value); iowrite32(value, iomem); } #define write_register(v, mem, off) __write_register(__func__, v, mem, off) #else #define write_register(v, mem, off) iowrite32(v, mem) #endif inline u32 read_register(void *iomem) { return ioread32(iomem); } static inline u32 build_u32(u32 hi, u32 lo) { return ((hi & 0xFFFFUL) << 16) | (lo & 0xFFFFUL); } static inline u64 build_u64(u64 hi, u64 lo) { return ((hi & 0xFFFFFFFULL) << 32) | (lo & 0xFFFFFFFFULL); } static void check_nonzero_interrupt_status(struct xdma_dev *xdev) { struct interrupt_regs *reg = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); u32 w; w = read_register(&reg->user_int_enable); if (w) pr_info("%s xdma%d user_int_enable = 0x%08x\n", dev_name(&xdev->pdev->dev), xdev->idx, w); w = read_register(&reg->channel_int_enable); if (w) pr_info("%s xdma%d channel_int_enable = 0x%08x\n", dev_name(&xdev->pdev->dev), xdev->idx, w); w = read_register(&reg->user_int_request); if (w) pr_info("%s xdma%d user_int_request = 0x%08x\n", dev_name(&xdev->pdev->dev), xdev->idx, w); w = read_register(&reg->channel_int_request); if (w) pr_info("%s xdma%d channel_int_request = 0x%08x\n", dev_name(&xdev->pdev->dev), xdev->idx, w); w = read_register(&reg->user_int_pending); if (w) pr_info("%s xdma%d user_int_pending = 0x%08x\n", dev_name(&xdev->pdev->dev), xdev->idx, w); w = read_register(&reg->channel_int_pending); if (w) pr_info("%s xdma%d channel_int_pending = 0x%08x\n", dev_name(&xdev->pdev->dev), xdev->idx, w); } /* channel_interrupts_enable -- Enable interrupts we are interested in */ static void channel_interrupts_enable(struct xdma_dev *xdev, u32 mask) { struct interrupt_regs *reg = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); write_register(mask, &reg->channel_int_enable_w1s, XDMA_OFS_INT_CTRL); } /* channel_interrupts_disable -- Disable interrupts we not interested in */ static void channel_interrupts_disable(struct xdma_dev *xdev, u32 mask) { struct interrupt_regs *reg = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); write_register(mask, &reg->channel_int_enable_w1c, XDMA_OFS_INT_CTRL); } /* user_interrupts_enable -- Enable interrupts we are interested in */ static void user_interrupts_enable(struct xdma_dev *xdev, u32 mask) { struct interrupt_regs *reg = (struct interrupt_regs *)(xdev->bar[xdev->config_bar_idx] + XDMA_OFS_INT_CTRL); write_register(mask, &reg->user_int_enable_w1s, XDMA_OFS_INT_CTRL); } /* user_interrupts_disable -- Disable i