drivers/misc/mic/host/mic_intr.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Intel MIC Platform Software Stack (MPSS)
  *
  * Copyright(c) 2013 Intel Corporation.
  *
  * Intel MIC Host driver.
  */
 #include <linux/pci.h>
 #include <linux/interrupt.h>

 #include "../common/mic_dev.h"
 #include "mic_device.h"

 static irqreturn_t mic_thread_fn(int irq, void *dev)
 {
 	struct mic_device *mdev = dev;
 	struct mic_intr_info *intr_info = mdev->intr_info;
 	struct mic_irq_info *irq_info = &mdev->irq_info;
 	struct mic_intr_cb *intr_cb;
 	struct pci_dev *pdev = mdev->pdev;
 	int i;

 	spin_lock(&irq_info->mic_thread_lock);
 	for (i = intr_info->intr_start_idx[MIC_INTR_DB];
 			i < intr_info->intr_len[MIC_INTR_DB]; i++)
 		if (test_and_clear_bit(i, &irq_info->mask)) {
 			list_for_each_entry(intr_cb, &irq_info->cb_list[i],
 					    list)
 				if (intr_cb->thread_fn)
 					intr_cb->thread_fn(pdev->irq,
 							 intr_cb->data);
 		}
 	spin_unlock(&irq_info->mic_thread_lock);
 	return IRQ_HANDLED;
 }
 /**
  * mic_interrupt - Generic interrupt handler for
  * MSI and INTx based interrupts.
  */
 static irqreturn_t mic_interrupt(int irq, void *dev)
 {
 	struct mic_device *mdev = dev;
 	struct mic_intr_info *intr_info = mdev->intr_info;
 	struct mic_irq_info *irq_info = &mdev->irq_info;
 	struct mic_intr_cb *intr_cb;
 	struct pci_dev *pdev = mdev->pdev;
 	u32 mask;
 	int i;

 	mask = mdev->ops->ack_interrupt(mdev);
 	if (!mask)
 		return IRQ_NONE;

 	spin_lock(&irq_info->mic_intr_lock);
 	for (i = intr_info->intr_start_idx[MIC_INTR_DB];
 			i < intr_info->intr_len[MIC_INTR_DB]; i++)
 		if (mask & BIT(i)) {
 			list_for_each_entry(intr_cb, &irq_info->cb_list[i],
 					    list)
 				if (intr_cb->handler)
 					intr_cb->handler(pdev->irq,
 							 intr_cb->data);
 			set_bit(i, &irq_info->mask);
 		}
 	spin_unlock(&irq_info->mic_intr_lock);
 	return IRQ_WAKE_THREAD;
 }

 /* Return the interrupt offset from the index. Index is 0 based. */
 static u16 mic_map_src_to_offset(struct mic_device *mdev,
 				 int intr_src, enum mic_intr_type type)
 {
 	if (type >= MIC_NUM_INTR_TYPES)
 		return MIC_NUM_OFFSETS;
 	if (intr_src >= mdev->intr_info->intr_len[type])
 		return MIC_NUM_OFFSETS;

 	return mdev->intr_info->intr_start_idx[type] + intr_src;
 }

 /* Return next available msix_entry. */
 static struct msix_entry *mic_get_available_vector(struct mic_device *mdev)
 {
 	int i;
 	struct mic_irq_info *info = &mdev->irq_info;

 	for (i = 0; i < info->num_vectors; i++)
 		if (!info->mic_msi_map[i])
 			return &info->msix_entries[i];
 	return NULL;
 }

 /**
  * mic_register_intr_callback - Register a callback handler for the
  * given source id.
  *
  * @mdev: pointer to the mic_device instance
  * @idx: The source id to be registered.
  * @handler: The function to be called when the source id receives
  * the interrupt.
  * @thread_fn: thread fn. corresponding to the handler
  * @data: Private data of the requester.
  * Return the callback structure that was registered or an
  * appropriate error on failure.
  */
 static struct mic_intr_cb *mic_register_intr_callback(struct mic_device *mdev,
 			u8 idx, irq_handler_t handler, irq_handler_t thread_fn,
 			void *data)
 {
 	struct mic_intr_cb *intr_cb;
 	unsigned long flags;
 	int rc;
 	intr_cb = kmalloc(sizeof(*intr_cb), GFP_KERNEL);

 	if (!intr_cb)
 		return ERR_PTR(-ENOMEM);

 	intr_cb->handler = handler;
 	intr_cb->thread_fn = thread_fn;
 	intr_cb->data = data;
 	intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida,
 		0, 0, GFP_KERNEL);
 	if (intr_cb->cb_id < 0) {
 		rc = intr_cb->cb_id;
 		goto ida_fail;
 	}

 	spin_lock(&mdev->irq_info.mic_thread_lock);
 	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
 	list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]);
 	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
 	spin_unlock(&mdev->irq_info.mic_thread_lock);

 	return intr_cb;
 ida_fail:
 	kfree(intr_cb);
 	return ERR_PTR(rc);
 }

 /**
  * mic_unregister_intr_callback - Unregister the callback handler
  * identified by its callback id.
  *
  * @mdev: pointer to the mic_device instance
  * @idx: The callback structure id to be unregistered.
  * Return the source id that was unregistered or MIC_NUM_OFFSETS if no
  * such callback handler was found.
  */
 static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx)
 {
 	struct list_head *pos, *tmp;
 	struct mic_intr_cb *intr_cb;
 	unsigned long flags;
 	int i;

 	spin_lock(&mdev->irq_info.mic_thread_lock);
 	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
 	for (i = 0;  i < MIC_NUM_OFFSETS; i++) {
 		list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
 			intr_cb = list_entry(pos, struct mic_intr_cb, list);
 			if (intr_cb->cb_id == idx) {
 				list_del(pos);
 				ida_simple_remove(&mdev->irq_info.cb_ida,
 						  intr_cb->cb_id);
 				kfree(intr_cb);
 				spin_unlock_irqrestore(
 					&mdev->irq_info.mic_intr_lock, flags);
 				spin_unlock(&mdev->irq_info.mic_thread_lock);
 				return i;
 			}
 		}
 	}
 	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
 	spin_unlock(&mdev->irq_info.mic_thread_lock);
 	return MIC_NUM_OFFSETS;
 }

 /**
  * mic_setup_msix - Initializes MSIx interrupts.
  *
  * @mdev: pointer to mic_device instance
  *
  *
  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  */
 static int mic_setup_msix(struct mic_device *mdev, struct pci_dev *pdev)
 {
 	int rc, i;
 	int entry_size = sizeof(*mdev->irq_info.msix_entries);

 	mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX,
 						    entry_size, GFP_KERNEL);
 	if (!mdev->irq_info.msix_entries) {
 		rc = -ENOMEM;
 		goto err_nomem1;
 	}

 	for (i = 0; i < MIC_MIN_MSIX; i++)
 		mdev->irq_info.msix_entries[i].entry = i;

 	rc = pci_enable_msix_exact(pdev, mdev->irq_info.msix_entries,
 				   MIC_MIN_MSIX);
 	if (rc) {
 		dev_dbg(&pdev->dev, "Error enabling MSIx. rc = %d\n", rc);
 		goto err_enable_msix;
 	}

 	mdev->irq_info.num_vectors = MIC_MIN_MSIX;
 	mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
 		mdev->irq_info.num_vectors), GFP_KERNEL);

 	if (!mdev->irq_info.mic_msi_map) {
 		rc = -ENOMEM;
 		goto err_nomem2;
 	}

 	dev_dbg(&mdev->pdev->dev,
 		"%d MSIx irqs setup\n", mdev->irq_info.num_vectors);
 	return 0;
 err_nomem2:
 	pci_disable_msix(pdev);
 err_enable_msix:
 	kfree(mdev->irq_info.msix_entries);
 err_nomem1:
 	mdev->irq_info.num_vectors = 0;
 	return rc;
 }

 /**
  * mic_setup_callbacks - Initialize data structures needed
  * to handle callbacks.
  *
  * @mdev: pointer to mic_device instance
  */
 static int mic_setup_callbacks(struct mic_device *mdev)
 {
 	int i;

 	mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS,
 					       sizeof(*mdev->irq_info.cb_list),
 					       GFP_KERNEL);
 	if (!mdev->irq_info.cb_list)
 		return -ENOMEM;

 	for (i = 0; i < MIC_NUM_OFFSETS; i++)
 		INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]);
 	ida_init(&mdev->irq_info.cb_ida);
 	spin_lock_init(&mdev->irq_info.mic_intr_lock);
 	spin_lock_init(&mdev->irq_info.mic_thread_lock);
 	return 0;
 }

 /**
  * mic_release_callbacks - Uninitialize data structures needed
  * to handle callbacks.
  *
  * @mdev: pointer to mic_device instance
  */
 static void mic_release_callbacks(struct mic_device *mdev)
 {
 	unsigned long flags;
 	struct list_head *pos, *tmp;
 	struct mic_intr_cb *intr_cb;
 	int i;

 	spin_lock(&mdev->irq_info.mic_thread_lock);
 	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
 	for (i = 0; i < MIC_NUM_OFFSETS; i++) {
 		if (list_empty(&mdev->irq_info.cb_list[i]))
 			break;

 		list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
 			intr_cb = list_entry(pos, struct mic_intr_cb, list);
 			list_del(pos);
 			ida_simple_remove(&mdev->irq_info.cb_ida,
 					  intr_cb->cb_id);
 			kfree(intr_cb);
 		}
 	}
 	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
 	spin_unlock(&mdev->irq_info.mic_thread_lock);
 	ida_destroy(&mdev->irq_info.cb_ida);
 	kfree(mdev->irq_info.cb_list);
 }

 /**
  * mic_setup_msi - Initializes MSI interrupts.
  *
  * @mdev: pointer to mic_device instance
  * @pdev: PCI device structure
  *
  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  */
 static int mic_setup_msi(struct mic_device *mdev, struct pci_dev *pdev)
 {
 	int rc;

 	rc = pci_enable_msi(pdev);
 	if (rc) {
 		dev_dbg(&pdev->dev, "Error enabling MSI. rc = %d\n", rc);
 		return rc;
 	}

 	mdev->irq_info.num_vectors = 1;
 	mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
 		mdev->irq_info.num_vectors), GFP_KERNEL);

 	if (!mdev->irq_info.mic_msi_map) {
 		rc = -ENOMEM;
 		goto err_nomem1;
 	}

 	rc = mic_setup_callbacks(mdev);
 	if (rc) {
 		dev_err(&pdev->dev, "Error setting up callbacks\n");
 		goto err_nomem2;
 	}

 	rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn,
 				  0, "mic-msi", mdev);
 	if (rc) {
 		dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
 		goto err_irq_req_fail;
 	}

 	dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors);
 	return 0;
 err_irq_req_fail:
 	mic_release_callbacks(mdev);
 err_nomem2:
 	kfree(mdev->irq_info.mic_msi_map);
 err_nomem1:
 	pci_disable_msi(pdev);
 	mdev->irq_info.num_vectors = 0;
 	return rc;
 }

 /**
  * mic_setup_intx - Initializes legacy interrupts.
  *
  * @mdev: pointer to mic_device instance
  * @pdev: PCI device structure
  *
  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  */
 static int mic_setup_intx(struct mic_device *mdev, struct pci_dev *pdev)
 {
 	int rc;

 	/* Enable intx */
 	pci_intx(pdev, 1);
 	rc = mic_setup_callbacks(mdev);
 	if (rc) {
 		dev_err(&pdev->dev, "Error setting up callbacks\n");
 		goto err_nomem;
 	}

 	rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn,
 				  IRQF_SHARED, "mic-intx", mdev);
 	if (rc)
 		goto err;

 	dev_dbg(&pdev->dev, "intx irq setup\n");
 	return 0;
 err:
 	mic_release_callbacks(mdev);
 err_nomem:
 	return rc;
 }

 /**
  * mic_next_db - Retrieve the next doorbell interrupt source id.
  * The id is picked sequentially from the available pool of
  * doorlbell ids.
  *
  * @mdev: pointer to the mic_device instance.
  *
  * Returns the next doorbell interrupt source.
  */
 int mic_next_db(struct mic_device *mdev)
 {
 	int next_db;

 	next_db = mdev->irq_info.next_avail_src %
 		mdev->intr_info->intr_len[MIC_INTR_DB];
 	mdev->irq_info.next_avail_src++;
 	return next_db;
 }

 #define COOKIE_ID_SHIFT 16
 #define GET_ENTRY(cookie) ((cookie) & 0xFFFF)
 #define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT)
 #define MK_COOKIE(x, y) ((x) | (y) << COOKIE_ID_SHIFT)

 /**
  * mic_request_threaded_irq - request an irq. mic_mutex needs
  * to be held before calling this function.
  *
  * @mdev: pointer to mic_device instance
  * @handler: The callback function that handles the interrupt.
  * The function needs to call ack_interrupts
  * (mdev->ops->ack_interrupt(mdev)) when handling the interrupts.
  * @thread_fn: thread fn required by request_threaded_irq.
  * @name: The ASCII name of the callee requesting the irq.
  * @data: private data that is returned back when calling the
  * function handler.
  * @intr_src: The source id of the requester. Its the doorbell id
  * for Doorbell interrupts and DMA channel id for DMA interrupts.
  * @type: The type of interrupt. Values defined in mic_intr_type
  *
  * returns: The cookie that is transparent to the caller. Passed
  * back when calling mic_free_irq. An appropriate error code
  * is returned on failure. Caller needs to use IS_ERR(return_val)
  * to check for failure and PTR_ERR(return_val) to obtained the
  * error code.
  *
  */
 struct mic_irq *
 mic_request_threaded_irq(struct mic_device *mdev,
 			 irq_handler_t handler, irq_handler_t thread_fn,
 			 const char *name, void *data, int intr_src,
 			 enum mic_intr_type type)
 {
 	u16 offset;
 	int rc = 0;
 	struct msix_entry *msix = NULL;
 	unsigned long cookie = 0;
 	u16 entry;
 	struct mic_intr_cb *intr_cb;
 	struct pci_dev *pdev = mdev->pdev;

 	offset = mic_map_src_to_offset(mdev, intr_src, type);
 	if (offset >= MIC_NUM_OFFSETS) {
 		dev_err(&mdev->pdev->dev,
 			"Error mapping index %d to a valid source id.\n",
 			intr_src);
 		rc = -EINVAL;
 		goto err;
 	}

 	if (mdev->irq_info.num_vectors > 1) {
 		msix = mic_get_available_vector(mdev);
 		if (!msix) {
 			dev_err(&mdev->pdev->dev,
 				"No MSIx vectors available for use.\n");
 			rc = -ENOSPC;
 			goto err;
 		}

 		rc = request_threaded_irq(msix->vector, handler, thread_fn,
 					  0, name, data);
 		if (rc) {
 			dev_dbg(&mdev->pdev->dev,
 				"request irq failed rc = %d\n", rc);
 			goto err;
 		}
 		entry = msix->entry;
 		mdev->irq_info.mic_msi_map[entry] |= BIT(offset);
 		mdev->intr_ops->program_msi_to_src_map(mdev,
 				entry, offset, true);
 		cookie = MK_COOKIE(entry, offset);
 		dev_dbg(&mdev->pdev->dev, "irq: %d assigned for src: %d\n",
 			msix->vector, intr_src);
 	} else {
 		intr_cb = mic_register_intr_callback(mdev, offset, handler,
 						     thread_fn, data);
 		if (IS_ERR(intr_cb)) {
 			dev_err(&mdev->pdev->dev,
 				"No available callback entries for use\n");
 			rc = PTR_ERR(intr_cb);
 			goto err;
 		}

 		entry = 0;
 		if (pci_dev_msi_enabled(pdev)) {
 			mdev->irq_info.mic_msi_map[entry] |= (1 << offset);
 			mdev->intr_ops->program_msi_to_src_map(mdev,
 				entry, offset, true);
 		}
 		cookie = MK_COOKIE(entry, intr_cb->cb_id);
 		dev_dbg(&mdev->pdev->dev, "callback %d registered for src: %d\n",
 			intr_cb->cb_id, intr_src);
 	}
 	return (struct mic_irq *)cookie;
 err:
 	return ERR_PTR(rc);
 }

 /**
  * mic_free_irq - free irq. mic_mutex
  *  needs to be held before calling this function.
  *
  * @mdev: pointer to mic_device instance
  * @cookie: cookie obtained during a successful call to mic_request_threaded_irq
  * @data: private data specified by the calling function during the
  * mic_request_threaded_irq
  *
  * returns: none.
  */
 void mic_free_irq(struct mic_device *mdev,
 		  struct mic_irq *cookie, void *data)
 {
 	u32 offset;
 	u32 entry;
 	u8 src_id;
 	unsigned int irq;
 	struct pci_dev *pdev = mdev->pdev;

 	entry = GET_ENTRY((unsigned long)cookie);
 	offset = GET_OFFSET((unsigned long)cookie);
 	if (mdev->irq_info.num_vectors > 1) {
 		if (entry >= mdev->irq_info.num_vectors) {
 			dev_warn(&mdev->pdev->dev,
 				 "entry %d should be < num_irq %d\n",
 				entry, mdev->irq_info.num_vectors);
 			return;
 		}
 		irq = mdev->irq_info.msix_entries[entry].vector;
 		free_irq(irq, data);
 		mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset));
 		mdev->intr_ops->program_msi_to_src_map(mdev,
 			entry, offset, false);

 		dev_dbg(&mdev->pdev->dev, "irq: %d freed\n", irq);
 	} else {
 		irq = pdev->irq;
 		src_id = mic_unregister_intr_callback(mdev, offset);
 		if (src_id >= MIC_NUM_OFFSETS) {
 			dev_warn(&mdev->pdev->dev, "Error unregistering callback\n");
 			return;
 		}
 		if (pci_dev_msi_enabled(pdev)) {
 			mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id));
 			mdev->intr_ops->program_msi_to_src_map(mdev,
 				entry, src_id, false);
 		}
 		dev_dbg(&mdev->pdev->dev, "callback %d unregistered for src: %d\n",
 			offset, src_id);
 	}
 }

 /**
  * mic_setup_interrupts - Initializes interrupts.
  *
  * @mdev: pointer to mic_device instance
  * @pdev: PCI device structure
  *
  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
  */
 int mic_setup_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
 {
 	int rc;

 	rc = mic_setup_msix(mdev, pdev);
 	if (!rc)
 		goto done;

 	rc = mic_setup_msi(mdev, pdev);
 	if (!rc)
 		goto done;

 	rc = mic_setup_intx(mdev, pdev);
 	if (rc) {
 		dev_err(&mdev->pdev->dev, "no usable interrupts\n");
 		return rc;
 	}
 done:
 	mdev->intr_ops->enable_interrupts(mdev);
 	return 0;
 }

 /**
  * mic_free_interrupts - Frees interrupts setup by mic_setup_interrupts
  *
  * @mdev: pointer to mic_device instance
  * @pdev: PCI device structure
  *
  * returns none.
  */
 void mic_free_interrupts(struct mic_device *mdev, struct pci_dev *pdev)
 {
 	int i;

 	mdev->intr_ops->disable_interrupts(mdev);
 	if (mdev->irq_info.num_vectors > 1) {
 		for (i = 0; i < mdev->irq_info.num_vectors; i++) {
 			if (mdev->irq_info.mic_msi_map[i])
 				dev_warn(&pdev->dev, "irq %d may still be in use.\n",
 					 mdev->irq_info.msix_entries[i].vector);
 		}
 		kfree(mdev->irq_info.mic_msi_map);
 		kfree(mdev->irq_info.msix_entries);
 		pci_disable_msix(pdev);
 	} else {
 		if (pci_dev_msi_enabled(pdev)) {
 			free_irq(pdev->irq, mdev);
 			kfree(mdev->irq_info.mic_msi_map);
 			pci_disable_msi(pdev);
 		} else {
 			free_irq(pdev->irq, mdev);
 		}
 		mic_release_callbacks(mdev);
 	}
 }

 /**
  * mic_intr_restore - Restore MIC interrupt registers.
  *
  * @mdev: pointer to mic_device instance.
  *
  * Restore the interrupt registers to values previously
  * stored in the SW data structures. mic_mutex needs to
  * be held before calling this function.
  *
  * returns None.
  */
 void mic_intr_restore(struct mic_device *mdev)
 {
 	int entry, offset;
 	struct pci_dev *pdev = mdev->pdev;

 	if (!pci_dev_msi_enabled(pdev))
 		return;

 	for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) {
 		for (offset = 0; offset < MIC_NUM_OFFSETS; offset++) {
 			if (mdev->irq_info.mic_msi_map[entry] & BIT(offset))
 				mdev->intr_ops->program_msi_to_src_map(mdev,
 					entry, offset, true);
 		}
 	}
 }
	// SPDX-License-Identifier: GPL-2.0-only
	/*
	* Intel MIC Platform Software Stack (MPSS)
	*
	* Copyright(c) 2013 Intel Corporation.
	*
	* Intel MIC Host driver.
	*/
	#include <linux/pci.h>
	#include <linux/interrupt.h>

	#include "../common/mic_dev.h"
	#include "mic_device.h"

	static irqreturn_t mic_thread_fn(int irq, void *dev)
	{
	struct mic_device *mdev = dev;
	struct mic_intr_info *intr_info = mdev->intr_info;
	struct mic_irq_info *irq_info = &mdev->irq_info;
	struct mic_intr_cb *intr_cb;
	struct pci_dev *pdev = mdev->pdev;
	int i;

	spin_lock(&irq_info->mic_thread_lock);
	for (i = intr_info->intr_start_idx[MIC_INTR_DB];
	i < intr_info->intr_len[MIC_INTR_DB]; i++)
	if (test_and_clear_bit(i, &irq_info->mask)) {
	list_for_each_entry(intr_cb, &irq_info->cb_list[i],
	list)
	if (intr_cb->thread_fn)
	intr_cb->thread_fn(pdev->irq,
	intr_cb->data);
	}
	spin_unlock(&irq_info->mic_thread_lock);
	return IRQ_HANDLED;
	}
	/**
	* mic_interrupt - Generic interrupt handler for
	* MSI and INTx based interrupts.
	*/
	static irqreturn_t mic_interrupt(int irq, void *dev)
	{
	struct mic_device *mdev = dev;
	struct mic_intr_info *intr_info = mdev->intr_info;
	struct mic_irq_info *irq_info = &mdev->irq_info;
	struct mic_intr_cb *intr_cb;
	struct pci_dev *pdev = mdev->pdev;
	u32 mask;
	int i;

	mask = mdev->ops->ack_interrupt(mdev);
	if (!mask)
	return IRQ_NONE;

	spin_lock(&irq_info->mic_intr_lock);
	for (i = intr_info->intr_start_idx[MIC_INTR_DB];
	i < intr_info->intr_len[MIC_INTR_DB]; i++)
	if (mask & BIT(i)) {
	list_for_each_entry(intr_cb, &irq_info->cb_list[i],
	list)
	if (intr_cb->handler)
	intr_cb->handler(pdev->irq,
	intr_cb->data);
	set_bit(i, &irq_info->mask);
	}
	spin_unlock(&irq_info->mic_intr_lock);
	return IRQ_WAKE_THREAD;
	}

	/* Return the interrupt offset from the index. Index is 0 based. */
	static u16 mic_map_src_to_offset(struct mic_device *mdev,
	int intr_src, enum mic_intr_type type)
	{
	if (type >= MIC_NUM_INTR_TYPES)
	return MIC_NUM_OFFSETS;
	if (intr_src >= mdev->intr_info->intr_len[type])
	return MIC_NUM_OFFSETS;

	return mdev->intr_info->intr_start_idx[type] + intr_src;
	}

	/* Return next available msix_entry. */
	static struct msix_entry mic_get_available_vector(struct mic_device mdev)
	{
	int i;
	struct mic_irq_info *info = &mdev->irq_info;

	for (i = 0; i < info->num_vectors; i++)
	if (!info->mic_msi_map[i])
	return &info->msix_entries[i];
	return NULL;
	}

	/**
	* mic_register_intr_callback - Register a callback handler for the
	* given source id.
	*
	* @mdev: pointer to the mic_device instance
	* @idx: The source id to be registered.
	* @handler: The function to be called when the source id receives
	* the interrupt.
	* @thread_fn: thread fn. corresponding to the handler
	* @data: Private data of the requester.
	* Return the callback structure that was registered or an
	* appropriate error on failure.
	*/
	static struct mic_intr_cb mic_register_intr_callback(struct mic_device mdev,
	u8 idx, irq_handler_t handler, irq_handler_t thread_fn,
	void *data)
	{
	struct mic_intr_cb *intr_cb;
	unsigned long flags;
	int rc;
	intr_cb = kmalloc(sizeof(*intr_cb), GFP_KERNEL);

	if (!intr_cb)
	return ERR_PTR(-ENOMEM);

	intr_cb->handler = handler;
	intr_cb->thread_fn = thread_fn;
	intr_cb->data = data;
	intr_cb->cb_id = ida_simple_get(&mdev->irq_info.cb_ida,
	0, 0, GFP_KERNEL);
	if (intr_cb->cb_id < 0) {
	rc = intr_cb->cb_id;
	goto ida_fail;
	}

	spin_lock(&mdev->irq_info.mic_thread_lock);
	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
	list_add_tail(&intr_cb->list, &mdev->irq_info.cb_list[idx]);
	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
	spin_unlock(&mdev->irq_info.mic_thread_lock);

	return intr_cb;
	ida_fail:
	kfree(intr_cb);
	return ERR_PTR(rc);
	}

	/**
	* mic_unregister_intr_callback - Unregister the callback handler
	* identified by its callback id.
	*
	* @mdev: pointer to the mic_device instance
	* @idx: The callback structure id to be unregistered.
	* Return the source id that was unregistered or MIC_NUM_OFFSETS if no
	* such callback handler was found.
	*/
	static u8 mic_unregister_intr_callback(struct mic_device *mdev, u32 idx)
	{
	struct list_head pos, tmp;
	struct mic_intr_cb *intr_cb;
	unsigned long flags;
	int i;

	spin_lock(&mdev->irq_info.mic_thread_lock);
	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
	for (i = 0; i < MIC_NUM_OFFSETS; i++) {
	list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
	intr_cb = list_entry(pos, struct mic_intr_cb, list);
	if (intr_cb->cb_id == idx) {
	list_del(pos);
	ida_simple_remove(&mdev->irq_info.cb_ida,
	intr_cb->cb_id);
	kfree(intr_cb);
	spin_unlock_irqrestore(
	&mdev->irq_info.mic_intr_lock, flags);
	spin_unlock(&mdev->irq_info.mic_thread_lock);
	return i;
	}
	}
	}
	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
	spin_unlock(&mdev->irq_info.mic_thread_lock);
	return MIC_NUM_OFFSETS;
	}

	/**
	* mic_setup_msix - Initializes MSIx interrupts.
	*
	* @mdev: pointer to mic_device instance
	*
	*
	* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
	*/
	static int mic_setup_msix(struct mic_device mdev, struct pci_dev pdev)
	{
	int rc, i;
	int entry_size = sizeof(*mdev->irq_info.msix_entries);

	mdev->irq_info.msix_entries = kmalloc_array(MIC_MIN_MSIX,
	entry_size, GFP_KERNEL);
	if (!mdev->irq_info.msix_entries) {
	rc = -ENOMEM;
	goto err_nomem1;
	}

	for (i = 0; i < MIC_MIN_MSIX; i++)
	mdev->irq_info.msix_entries[i].entry = i;

	rc = pci_enable_msix_exact(pdev, mdev->irq_info.msix_entries,
	MIC_MIN_MSIX);
	if (rc) {
	dev_dbg(&pdev->dev, "Error enabling MSIx. rc = %d\n", rc);
	goto err_enable_msix;
	}

	mdev->irq_info.num_vectors = MIC_MIN_MSIX;
	mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
	mdev->irq_info.num_vectors), GFP_KERNEL);

	if (!mdev->irq_info.mic_msi_map) {
	rc = -ENOMEM;
	goto err_nomem2;
	}

	dev_dbg(&mdev->pdev->dev,
	"%d MSIx irqs setup\n", mdev->irq_info.num_vectors);
	return 0;
	err_nomem2:
	pci_disable_msix(pdev);
	err_enable_msix:
	kfree(mdev->irq_info.msix_entries);
	err_nomem1:
	mdev->irq_info.num_vectors = 0;
	return rc;
	}

	/**
	* mic_setup_callbacks - Initialize data structures needed
	* to handle callbacks.
	*
	* @mdev: pointer to mic_device instance
	*/
	static int mic_setup_callbacks(struct mic_device *mdev)
	{
	int i;

	mdev->irq_info.cb_list = kmalloc_array(MIC_NUM_OFFSETS,
	sizeof(*mdev->irq_info.cb_list),
	GFP_KERNEL);
	if (!mdev->irq_info.cb_list)
	return -ENOMEM;

	for (i = 0; i < MIC_NUM_OFFSETS; i++)
	INIT_LIST_HEAD(&mdev->irq_info.cb_list[i]);
	ida_init(&mdev->irq_info.cb_ida);
	spin_lock_init(&mdev->irq_info.mic_intr_lock);
	spin_lock_init(&mdev->irq_info.mic_thread_lock);
	return 0;
	}

	/**
	* mic_release_callbacks - Uninitialize data structures needed
	* to handle callbacks.
	*
	* @mdev: pointer to mic_device instance
	*/
	static void mic_release_callbacks(struct mic_device *mdev)
	{
	unsigned long flags;
	struct list_head pos, tmp;
	struct mic_intr_cb *intr_cb;
	int i;

	spin_lock(&mdev->irq_info.mic_thread_lock);
	spin_lock_irqsave(&mdev->irq_info.mic_intr_lock, flags);
	for (i = 0; i < MIC_NUM_OFFSETS; i++) {
	if (list_empty(&mdev->irq_info.cb_list[i]))
	break;

	list_for_each_safe(pos, tmp, &mdev->irq_info.cb_list[i]) {
	intr_cb = list_entry(pos, struct mic_intr_cb, list);
	list_del(pos);
	ida_simple_remove(&mdev->irq_info.cb_ida,
	intr_cb->cb_id);
	kfree(intr_cb);
	}
	}
	spin_unlock_irqrestore(&mdev->irq_info.mic_intr_lock, flags);
	spin_unlock(&mdev->irq_info.mic_thread_lock);
	ida_destroy(&mdev->irq_info.cb_ida);
	kfree(mdev->irq_info.cb_list);
	}

	/**
	* mic_setup_msi - Initializes MSI interrupts.
	*
	* @mdev: pointer to mic_device instance
	* @pdev: PCI device structure
	*
	* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
	*/
	static int mic_setup_msi(struct mic_device mdev, struct pci_dev pdev)
	{
	int rc;

	rc = pci_enable_msi(pdev);
	if (rc) {
	dev_dbg(&pdev->dev, "Error enabling MSI. rc = %d\n", rc);
	return rc;
	}

	mdev->irq_info.num_vectors = 1;
	mdev->irq_info.mic_msi_map = kzalloc((sizeof(u32) *
	mdev->irq_info.num_vectors), GFP_KERNEL);

	if (!mdev->irq_info.mic_msi_map) {
	rc = -ENOMEM;
	goto err_nomem1;
	}

	rc = mic_setup_callbacks(mdev);
	if (rc) {
	dev_err(&pdev->dev, "Error setting up callbacks\n");
	goto err_nomem2;
	}

	rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn,
	0, "mic-msi", mdev);
	if (rc) {
	dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
	goto err_irq_req_fail;
	}

	dev_dbg(&pdev->dev, "%d MSI irqs setup\n", mdev->irq_info.num_vectors);
	return 0;
	err_irq_req_fail:
	mic_release_callbacks(mdev);
	err_nomem2:
	kfree(mdev->irq_info.mic_msi_map);
	err_nomem1:
	pci_disable_msi(pdev);
	mdev->irq_info.num_vectors = 0;
	return rc;
	}

	/**
	* mic_setup_intx - Initializes legacy interrupts.
	*
	* @mdev: pointer to mic_device instance
	* @pdev: PCI device structure
	*
	* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
	*/
	static int mic_setup_intx(struct mic_device mdev, struct pci_dev pdev)
	{
	int rc;

	/* Enable intx */
	pci_intx(pdev, 1);
	rc = mic_setup_callbacks(mdev);
	if (rc) {
	dev_err(&pdev->dev, "Error setting up callbacks\n");
	goto err_nomem;
	}

	rc = request_threaded_irq(pdev->irq, mic_interrupt, mic_thread_fn,
	IRQF_SHARED, "mic-intx", mdev);
	if (rc)
	goto err;

	dev_dbg(&pdev->dev, "intx irq setup\n");
	return 0;
	err:
	mic_release_callbacks(mdev);
	err_nomem:
	return rc;
	}

	/**
	* mic_next_db - Retrieve the next doorbell interrupt source id.
	* The id is picked sequentially from the available pool of
	* doorlbell ids.
	*
	* @mdev: pointer to the mic_device instance.
	*
	* Returns the next doorbell interrupt source.
	*/
	int mic_next_db(struct mic_device *mdev)
	{
	int next_db;

	next_db = mdev->irq_info.next_avail_src %
	mdev->intr_info->intr_len[MIC_INTR_DB];
	mdev->irq_info.next_avail_src++;
	return next_db;
	}

	#define COOKIE_ID_SHIFT 16
	#define GET_ENTRY(cookie) ((cookie) & 0xFFFF)
	#define GET_OFFSET(cookie) ((cookie) >> COOKIE_ID_SHIFT)
	#define MK_COOKIE(x, y) ((x) \| (y) << COOKIE_ID_SHIFT)

	/**
	* mic_request_threaded_irq - request an irq. mic_mutex needs
	* to be held before calling this function.
	*
	* @mdev: pointer to mic_device instance
	* @handler: The callback function that handles the interrupt.
	* The function needs to call ack_interrupts
	* (mdev->ops->ack_interrupt(mdev)) when handling the interrupts.
	* @thread_fn: thread fn required by request_threaded_irq.
	* @name: The ASCII name of the callee requesting the irq.
	* @data: private data that is returned back when calling the
	* function handler.
	* @intr_src: The source id of the requester. Its the doorbell id
	* for Doorbell interrupts and DMA channel id for DMA interrupts.
	* @type: The type of interrupt. Values defined in mic_intr_type
	*
	* returns: The cookie that is transparent to the caller. Passed
	* back when calling mic_free_irq. An appropriate error code
	* is returned on failure. Caller needs to use IS_ERR(return_val)
	* to check for failure and PTR_ERR(return_val) to obtained the
	* error code.
	*
	*/
	struct mic_irq *
	mic_request_threaded_irq(struct mic_device *mdev,
	irq_handler_t handler, irq_handler_t thread_fn,
	const char name, void data, int intr_src,
	enum mic_intr_type type)
	{
	u16 offset;
	int rc = 0;
	struct msix_entry *msix = NULL;
	unsigned long cookie = 0;
	u16 entry;
	struct mic_intr_cb *intr_cb;
	struct pci_dev *pdev = mdev->pdev;

	offset = mic_map_src_to_offset(mdev, intr_src, type);
	if (offset >= MIC_NUM_OFFSETS) {
	dev_err(&mdev->pdev->dev,
	"Error mapping index %d to a valid source id.\n",
	intr_src);
	rc = -EINVAL;
	goto err;
	}

	if (mdev->irq_info.num_vectors > 1) {
	msix = mic_get_available_vector(mdev);
	if (!msix) {
	dev_err(&mdev->pdev->dev,
	"No MSIx vectors available for use.\n");
	rc = -ENOSPC;
	goto err;
	}

	rc = request_threaded_irq(msix->vector, handler, thread_fn,
	0, name, data);
	if (rc) {
	dev_dbg(&mdev->pdev->dev,
	"request irq failed rc = %d\n", rc);
	goto err;
	}
	entry = msix->entry;
	mdev->irq_info.mic_msi_map[entry] \|= BIT(offset);
	mdev->intr_ops->program_msi_to_src_map(mdev,
	entry, offset, true);
	cookie = MK_COOKIE(entry, offset);
	dev_dbg(&mdev->pdev->dev, "irq: %d assigned for src: %d\n",
	msix->vector, intr_src);
	} else {
	intr_cb = mic_register_intr_callback(mdev, offset, handler,
	thread_fn, data);
	if (IS_ERR(intr_cb)) {
	dev_err(&mdev->pdev->dev,
	"No available callback entries for use\n");
	rc = PTR_ERR(intr_cb);
	goto err;
	}

	entry = 0;
	if (pci_dev_msi_enabled(pdev)) {
	mdev->irq_info.mic_msi_map[entry] \|= (1 << offset);
	mdev->intr_ops->program_msi_to_src_map(mdev,
	entry, offset, true);
	}
	cookie = MK_COOKIE(entry, intr_cb->cb_id);
	dev_dbg(&mdev->pdev->dev, "callback %d registered for src: %d\n",
	intr_cb->cb_id, intr_src);
	}
	return (struct mic_irq *)cookie;
	err:
	return ERR_PTR(rc);
	}

	/**
	* mic_free_irq - free irq. mic_mutex
	* needs to be held before calling this function.
	*
	* @mdev: pointer to mic_device instance
	* @cookie: cookie obtained during a successful call to mic_request_threaded_irq
	* @data: private data specified by the calling function during the
	* mic_request_threaded_irq
	*
	* returns: none.
	*/
	void mic_free_irq(struct mic_device *mdev,
	struct mic_irq cookie, void data)
	{
	u32 offset;
	u32 entry;
	u8 src_id;
	unsigned int irq;
	struct pci_dev *pdev = mdev->pdev;

	entry = GET_ENTRY((unsigned long)cookie);
	offset = GET_OFFSET((unsigned long)cookie);
	if (mdev->irq_info.num_vectors > 1) {
	if (entry >= mdev->irq_info.num_vectors) {
	dev_warn(&mdev->pdev->dev,
	"entry %d should be < num_irq %d\n",
	entry, mdev->irq_info.num_vectors);
	return;
	}
	irq = mdev->irq_info.msix_entries[entry].vector;
	free_irq(irq, data);
	mdev->irq_info.mic_msi_map[entry] &= ~(BIT(offset));
	mdev->intr_ops->program_msi_to_src_map(mdev,
	entry, offset, false);

	dev_dbg(&mdev->pdev->dev, "irq: %d freed\n", irq);
	} else {
	irq = pdev->irq;
	src_id = mic_unregister_intr_callback(mdev, offset);
	if (src_id >= MIC_NUM_OFFSETS) {
	dev_warn(&mdev->pdev->dev, "Error unregistering callback\n");
	return;
	}
	if (pci_dev_msi_enabled(pdev)) {
	mdev->irq_info.mic_msi_map[entry] &= ~(BIT(src_id));
	mdev->intr_ops->program_msi_to_src_map(mdev,
	entry, src_id, false);
	}
	dev_dbg(&mdev->pdev->dev, "callback %d unregistered for src: %d\n",
	offset, src_id);
	}
	}

	/**
	* mic_setup_interrupts - Initializes interrupts.
	*
	* @mdev: pointer to mic_device instance
	* @pdev: PCI device structure
	*
	* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
	*/
	int mic_setup_interrupts(struct mic_device mdev, struct pci_dev pdev)
	{
	int rc;

	rc = mic_setup_msix(mdev, pdev);
	if (!rc)
	goto done;

	rc = mic_setup_msi(mdev, pdev);
	if (!rc)
	goto done;

	rc = mic_setup_intx(mdev, pdev);
	if (rc) {
	dev_err(&mdev->pdev->dev, "no usable interrupts\n");
	return rc;
	}
	done:
	mdev->intr_ops->enable_interrupts(mdev);
	return 0;
	}

	/**
	* mic_free_interrupts - Frees interrupts setup by mic_setup_interrupts
	*
	* @mdev: pointer to mic_device instance
	* @pdev: PCI device structure
	*
	* returns none.
	*/
	void mic_free_interrupts(struct mic_device mdev, struct pci_dev pdev)
	{
	int i;

	mdev->intr_ops->disable_interrupts(mdev);
	if (mdev->irq_info.num_vectors > 1) {
	for (i = 0; i < mdev->irq_info.num_vectors; i++) {
	if (mdev->irq_info.mic_msi_map[i])
	dev_warn(&pdev->dev, "irq %d may still be in use.\n",
	mdev->irq_info.msix_entries[i].vector);
	}
	kfree(mdev->irq_info.mic_msi_map);
	kfree(mdev->irq_info.msix_entries);
	pci_disable_msix(pdev);
	} else {
	if (pci_dev_msi_enabled(pdev)) {
	free_irq(pdev->irq, mdev);
	kfree(mdev->irq_info.mic_msi_map);
	pci_disable_msi(pdev);
	} else {
	free_irq(pdev->irq, mdev);
	}
	mic_release_callbacks(mdev);
	}
	}

	/**
	* mic_intr_restore - Restore MIC interrupt registers.
	*
	* @mdev: pointer to mic_device instance.
	*
	* Restore the interrupt registers to values previously
	* stored in the SW data structures. mic_mutex needs to
	* be held before calling this function.
	*
	* returns None.
	*/
	void mic_intr_restore(struct mic_device *mdev)
	{
	int entry, offset;
	struct pci_dev *pdev = mdev->pdev;

	if (!pci_dev_msi_enabled(pdev))
	return;

	for (entry = 0; entry < mdev->irq_info.num_vectors; entry++) {
	for (offset = 0; offset < MIC_NUM_OFFSETS; offset++) {
	if (mdev->irq_info.mic_msi_map[entry] & BIT(offset))
	mdev->intr_ops->program_msi_to_src_map(mdev,
	entry, offset, true);
	}
	}
	}