drivers/net/ethernet/altera/altera_sgdma.c - kernel/common.git - Git at Google

 /* Altera TSE SGDMA and MSGDMA Linux driver
  * Copyright (C) 2014 Altera Corporation. All rights reserved
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
  * version 2, as published by the Free Software Foundation.
  *
  * This program is distributed in the hope it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  * more details.
  *
  * You should have received a copy of the GNU General Public License along with
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */

 #include <linux/list.h>
 #include "altera_utils.h"
 #include "altera_tse.h"
 #include "altera_sgdmahw.h"
 #include "altera_sgdma.h"

 static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc,
 				struct sgdma_descrip __iomem *ndesc,
 				dma_addr_t ndesc_phys,
 				dma_addr_t raddr,
 				dma_addr_t waddr,
 				u16 length,
 				int generate_eop,
 				int rfixed,
 				int wfixed);

 static int sgdma_async_write(struct altera_tse_private *priv,
 			      struct sgdma_descrip __iomem *desc);

 static int sgdma_async_read(struct altera_tse_private *priv);

 static dma_addr_t
 sgdma_txphysaddr(struct altera_tse_private *priv,
 		 struct sgdma_descrip __iomem *desc);

 static dma_addr_t
 sgdma_rxphysaddr(struct altera_tse_private *priv,
 		 struct sgdma_descrip __iomem *desc);

 static int sgdma_txbusy(struct altera_tse_private *priv);

 static int sgdma_rxbusy(struct altera_tse_private *priv);

 static void
 queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer);

 static void
 queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer);

 static struct tse_buffer *
 dequeue_tx(struct altera_tse_private *priv);

 static struct tse_buffer *
 dequeue_rx(struct altera_tse_private *priv);

 static struct tse_buffer *
 queue_rx_peekhead(struct altera_tse_private *priv);

 int sgdma_initialize(struct altera_tse_private *priv)
 {
 	priv->txctrlreg = SGDMA_CTRLREG_ILASTD |
 		      SGDMA_CTRLREG_INTEN;

 	priv->rxctrlreg = SGDMA_CTRLREG_IDESCRIP |
 		      SGDMA_CTRLREG_INTEN |
 		      SGDMA_CTRLREG_ILASTD;

 	INIT_LIST_HEAD(&priv->txlisthd);
 	INIT_LIST_HEAD(&priv->rxlisthd);

 	priv->rxdescphys = (dma_addr_t) 0;
 	priv->txdescphys = (dma_addr_t) 0;

 	priv->rxdescphys = dma_map_single(priv->device,
 					  (void __force *)priv->rx_dma_desc,
 					  priv->rxdescmem, DMA_BIDIRECTIONAL);

 	if (dma_mapping_error(priv->device, priv->rxdescphys)) {
 		sgdma_uninitialize(priv);
 		netdev_err(priv->dev, "error mapping rx descriptor memory\n");
 		return -EINVAL;
 	}

 	priv->txdescphys = dma_map_single(priv->device,
 					  (void __force *)priv->tx_dma_desc,
 					  priv->txdescmem, DMA_TO_DEVICE);

 	if (dma_mapping_error(priv->device, priv->txdescphys)) {
 		sgdma_uninitialize(priv);
 		netdev_err(priv->dev, "error mapping tx descriptor memory\n");
 		return -EINVAL;
 	}

 	/* Initialize descriptor memory to all 0's, sync memory to cache */
 	memset_io(priv->tx_dma_desc, 0, priv->txdescmem);
 	memset_io(priv->rx_dma_desc, 0, priv->rxdescmem);

 	dma_sync_single_for_device(priv->device, priv->txdescphys,
 				   priv->txdescmem, DMA_TO_DEVICE);

 	dma_sync_single_for_device(priv->device, priv->rxdescphys,
 				   priv->rxdescmem, DMA_TO_DEVICE);

 	return 0;
 }

 void sgdma_uninitialize(struct altera_tse_private *priv)
 {
 	if (priv->rxdescphys)
 		dma_unmap_single(priv->device, priv->rxdescphys,
 				 priv->rxdescmem, DMA_BIDIRECTIONAL);

 	if (priv->txdescphys)
 		dma_unmap_single(priv->device, priv->txdescphys,
 				 priv->txdescmem, DMA_TO_DEVICE);
 }

 /* This function resets the SGDMA controller and clears the
  * descriptor memory used for transmits and receives.
  */
 void sgdma_reset(struct altera_tse_private *priv)
 {
 	/* Initialize descriptor memory to 0 */
 	memset_io(priv->tx_dma_desc, 0, priv->txdescmem);
 	memset_io(priv->rx_dma_desc, 0, priv->rxdescmem);

 	csrwr32(SGDMA_CTRLREG_RESET, priv->tx_dma_csr, sgdma_csroffs(control));
 	csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control));

 	csrwr32(SGDMA_CTRLREG_RESET, priv->rx_dma_csr, sgdma_csroffs(control));
 	csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control));
 }

 /* For SGDMA, interrupts remain enabled after initially enabling,
  * so no need to provide implementations for abstract enable
  * and disable
  */

 void sgdma_enable_rxirq(struct altera_tse_private *priv)
 {
 }

 void sgdma_enable_txirq(struct altera_tse_private *priv)
 {
 }

 void sgdma_disable_rxirq(struct altera_tse_private *priv)
 {
 }

 void sgdma_disable_txirq(struct altera_tse_private *priv)
 {
 }

 void sgdma_clear_rxirq(struct altera_tse_private *priv)
 {
 	tse_set_bit(priv->rx_dma_csr, sgdma_csroffs(control),
 		    SGDMA_CTRLREG_CLRINT);
 }

 void sgdma_clear_txirq(struct altera_tse_private *priv)
 {
 	tse_set_bit(priv->tx_dma_csr, sgdma_csroffs(control),
 		    SGDMA_CTRLREG_CLRINT);
 }

 /* transmits buffer through SGDMA. Returns number of buffers
  * transmitted, 0 if not possible.
  *
  * tx_lock is held by the caller
  */
 int sgdma_tx_buffer(struct altera_tse_private *priv, struct tse_buffer *buffer)
 {
 	struct sgdma_descrip __iomem *descbase =
 		(struct sgdma_descrip __iomem *)priv->tx_dma_desc;

 	struct sgdma_descrip __iomem *cdesc = &descbase[0];
 	struct sgdma_descrip __iomem *ndesc = &descbase[1];

 	/* wait 'til the tx sgdma is ready for the next transmit request */
 	if (sgdma_txbusy(priv))
 		return 0;

 	sgdma_setup_descrip(cdesc,			/* current descriptor */
 			    ndesc,			/* next descriptor */
 			    sgdma_txphysaddr(priv, ndesc),
 			    buffer->dma_addr,		/* address of packet to xmit */
 			    0,				/* write addr 0 for tx dma */
 			    buffer->len,		/* length of packet */
 			    SGDMA_CONTROL_EOP,		/* Generate EOP */
 			    0,				/* read fixed */
 			    SGDMA_CONTROL_WR_FIXED);	/* Generate SOP */

 	sgdma_async_write(priv, cdesc);

 	/* enqueue the request to the pending transmit queue */
 	queue_tx(priv, buffer);

 	return 1;
 }


 /* tx_lock held to protect access to queued tx list
  */
 u32 sgdma_tx_completions(struct altera_tse_private *priv)
 {
 	u32 ready = 0;

 	if (!sgdma_txbusy(priv) &&
 	    ((csrrd8(priv->tx_dma_desc, sgdma_descroffs(control))
 	     & SGDMA_CONTROL_HW_OWNED) == 0) &&
 	    (dequeue_tx(priv))) {
 		ready = 1;
 	}

 	return ready;
 }

 void sgdma_start_rxdma(struct altera_tse_private *priv)
 {
 	sgdma_async_read(priv);
 }

 void sgdma_add_rx_desc(struct altera_tse_private *priv,
 		       struct tse_buffer *rxbuffer)
 {
 	queue_rx(priv, rxbuffer);
 }

 /* status is returned on upper 16 bits,
  * length is returned in lower 16 bits
  */
 u32 sgdma_rx_status(struct altera_tse_private *priv)
 {
 	struct sgdma_descrip __iomem *base =
 		(struct sgdma_descrip __iomem *)priv->rx_dma_desc;
 	struct sgdma_descrip __iomem *desc = NULL;
 	struct tse_buffer *rxbuffer = NULL;
 	unsigned int rxstatus = 0;

 	u32 sts = csrrd32(priv->rx_dma_csr, sgdma_csroffs(status));

 	desc = &base[0];
 	if (sts & SGDMA_STSREG_EOP) {
 		unsigned int pktlength = 0;
 		unsigned int pktstatus = 0;
 		dma_sync_single_for_cpu(priv->device,
 					priv->rxdescphys,
 					SGDMA_DESC_LEN,
 					DMA_FROM_DEVICE);

 		pktlength = csrrd16(desc, sgdma_descroffs(bytes_xferred));
 		pktstatus = csrrd8(desc, sgdma_descroffs(status));
 		rxstatus = pktstatus & ~SGDMA_STATUS_EOP;
 		rxstatus = rxstatus << 16;
 		rxstatus |= (pktlength & 0xffff);

 		if (rxstatus) {
 			csrwr8(0, desc, sgdma_descroffs(status));

 			rxbuffer = dequeue_rx(priv);
 			if (rxbuffer == NULL)
 				netdev_info(priv->dev,
 					    "sgdma rx and rx queue empty!\n");

 			/* Clear control */
 			csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control));
 			/* clear status */
 			csrwr32(0xf, priv->rx_dma_csr, sgdma_csroffs(status));

 			/* kick the rx sgdma after reaping this descriptor */
 			sgdma_async_read(priv);

 		} else {
 			/* If the SGDMA indicated an end of packet on recv,
 			 * then it's expected that the rxstatus from the
 			 * descriptor is non-zero - meaning a valid packet
 			 * with a nonzero length, or an error has been
 			 * indicated. if not, then all we can do is signal
 			 * an error and return no packet received. Most likely
 			 * there is a system design error, or an error in the
 			 * underlying kernel (cache or cache management problem)
 			 */
 			netdev_err(priv->dev,
 				   "SGDMA RX Error Info: %x, %x, %x\n",
 				   sts, csrrd8(desc, sgdma_descroffs(status)),
 				   rxstatus);
 		}
 	} else if (sts == 0) {
 		sgdma_async_read(priv);
 	}

 	return rxstatus;
 }


 /* Private functions */
 static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc,
 				struct sgdma_descrip __iomem *ndesc,
 				dma_addr_t ndesc_phys,
 				dma_addr_t raddr,
 				dma_addr_t waddr,
 				u16 length,
 				int generate_eop,
 				int rfixed,
 				int wfixed)
 {
 	/* Clear the next descriptor as not owned by hardware */

 	u32 ctrl = csrrd8(ndesc, sgdma_descroffs(control));
 	ctrl &= ~SGDMA_CONTROL_HW_OWNED;
 	csrwr8(ctrl, ndesc, sgdma_descroffs(control));

 	ctrl = SGDMA_CONTROL_HW_OWNED;
 	ctrl |= generate_eop;
 	ctrl |= rfixed;
 	ctrl |= wfixed;

 	/* Channel is implicitly zero, initialized to 0 by default */
 	csrwr32(lower_32_bits(raddr), desc, sgdma_descroffs(raddr));
 	csrwr32(lower_32_bits(waddr), desc, sgdma_descroffs(waddr));

 	csrwr32(0, desc, sgdma_descroffs(pad1));
 	csrwr32(0, desc, sgdma_descroffs(pad2));
 	csrwr32(lower_32_bits(ndesc_phys), desc, sgdma_descroffs(next));

 	csrwr8(ctrl, desc, sgdma_descroffs(control));
 	csrwr8(0, desc, sgdma_descroffs(status));
 	csrwr8(0, desc, sgdma_descroffs(wburst));
 	csrwr8(0, desc, sgdma_descroffs(rburst));
 	csrwr16(length, desc, sgdma_descroffs(bytes));
 	csrwr16(0, desc, sgdma_descroffs(bytes_xferred));
 }

 /* If hardware is busy, don't restart async read.
  * if status register is 0 - meaning initial state, restart async read,
  * probably for the first time when populating a receive buffer.
  * If read status indicate not busy and a status, restart the async
  * DMA read.
  */
 static int sgdma_async_read(struct altera_tse_private *priv)
 {
 	struct sgdma_descrip __iomem *descbase =
 		(struct sgdma_descrip __iomem *)priv->rx_dma_desc;

 	struct sgdma_descrip __iomem *cdesc = &descbase[0];
 	struct sgdma_descrip __iomem *ndesc = &descbase[1];
 	struct tse_buffer *rxbuffer = NULL;

 	if (!sgdma_rxbusy(priv)) {
 		rxbuffer = queue_rx_peekhead(priv);
 		if (rxbuffer == NULL) {
 			netdev_err(priv->dev, "no rx buffers available\n");
 			return 0;
 		}

 		sgdma_setup_descrip(cdesc,		/* current descriptor */
 				    ndesc,		/* next descriptor */
 				    sgdma_rxphysaddr(priv, ndesc),
 				    0,			/* read addr 0 for rx dma */
 				    rxbuffer->dma_addr, /* write addr for rx dma */
 				    0,			/* read 'til EOP */
 				    0,			/* EOP: NA for rx dma */
 				    0,			/* read fixed: NA for rx dma */
 				    0);			/* SOP: NA for rx DMA */

 		dma_sync_single_for_device(priv->device,
 					   priv->rxdescphys,
 					   SGDMA_DESC_LEN,
 					   DMA_TO_DEVICE);

 		csrwr32(lower_32_bits(sgdma_rxphysaddr(priv, cdesc)),
 			priv->rx_dma_csr,
 			sgdma_csroffs(next_descrip));

 		csrwr32((priv->rxctrlreg | SGDMA_CTRLREG_START),
 			priv->rx_dma_csr,
 			sgdma_csroffs(control));

 		return 1;
 	}

 	return 0;
 }

 static int sgdma_async_write(struct altera_tse_private *priv,
 			     struct sgdma_descrip __iomem *desc)
 {
 	if (sgdma_txbusy(priv))
 		return 0;

 	/* clear control and status */
 	csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control));
 	csrwr32(0x1f, priv->tx_dma_csr, sgdma_csroffs(status));

 	dma_sync_single_for_device(priv->device, priv->txdescphys,
 				   SGDMA_DESC_LEN, DMA_TO_DEVICE);

 	csrwr32(lower_32_bits(sgdma_txphysaddr(priv, desc)),
 		priv->tx_dma_csr,
 		sgdma_csroffs(next_descrip));

 	csrwr32((priv->txctrlreg | SGDMA_CTRLREG_START),
 		priv->tx_dma_csr,
 		sgdma_csroffs(control));

 	return 1;
 }

 static dma_addr_t
 sgdma_txphysaddr(struct altera_tse_private *priv,
 		 struct sgdma_descrip __iomem *desc)
 {
 	dma_addr_t paddr = priv->txdescmem_busaddr;
 	uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->tx_dma_desc;
 	return (dma_addr_t)((uintptr_t)paddr + offs);
 }

 static dma_addr_t
 sgdma_rxphysaddr(struct altera_tse_private *priv,
 		 struct sgdma_descrip __iomem *desc)
 {
 	dma_addr_t paddr = priv->rxdescmem_busaddr;
 	uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->rx_dma_desc;
 	return (dma_addr_t)((uintptr_t)paddr + offs);
 }

 #define list_remove_head(list, entry, type, member)			\
 	do {								\
 		entry = NULL;						\
 		if (!list_empty(list)) {				\
 			entry = list_entry((list)->next, type, member);	\
 			list_del_init(&entry->member);			\
 		}							\
 	} while (0)

 #define list_peek_head(list, entry, type, member)			\
 	do {								\
 		entry = NULL;						\
 		if (!list_empty(list)) {				\
 			entry = list_entry((list)->next, type, member);	\
 		}							\
 	} while (0)

 /* adds a tse_buffer to the tail of a tx buffer list.
  * assumes the caller is managing and holding a mutual exclusion
  * primitive to avoid simultaneous pushes/pops to the list.
  */
 static void
 queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer)
 {
 	list_add_tail(&buffer->lh, &priv->txlisthd);
 }


 /* adds a tse_buffer to the tail of a rx buffer list
  * assumes the caller is managing and holding a mutual exclusion
  * primitive to avoid simultaneous pushes/pops to the list.
  */
 static void
 queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer)
 {
 	list_add_tail(&buffer->lh, &priv->rxlisthd);
 }

 /* dequeues a tse_buffer from the transmit buffer list, otherwise
  * returns NULL if empty.
  * assumes the caller is managing and holding a mutual exclusion
  * primitive to avoid simultaneous pushes/pops to the list.
  */
 static struct tse_buffer *
 dequeue_tx(struct altera_tse_private *priv)
 {
 	struct tse_buffer *buffer = NULL;
 	list_remove_head(&priv->txlisthd, buffer, struct tse_buffer, lh);
 	return buffer;
 }

 /* dequeues a tse_buffer from the receive buffer list, otherwise
  * returns NULL if empty
  * assumes the caller is managing and holding a mutual exclusion
  * primitive to avoid simultaneous pushes/pops to the list.
  */
 static struct tse_buffer *
 dequeue_rx(struct altera_tse_private *priv)
 {
 	struct tse_buffer *buffer = NULL;
 	list_remove_head(&priv->rxlisthd, buffer, struct tse_buffer, lh);
 	return buffer;
 }

 /* dequeues a tse_buffer from the receive buffer list, otherwise
  * returns NULL if empty
  * assumes the caller is managing and holding a mutual exclusion
  * primitive to avoid simultaneous pushes/pops to the list while the
  * head is being examined.
  */
 static struct tse_buffer *
 queue_rx_peekhead(struct altera_tse_private *priv)
 {
 	struct tse_buffer *buffer = NULL;
 	list_peek_head(&priv->rxlisthd, buffer, struct tse_buffer, lh);
 	return buffer;
 }

 /* check and return rx sgdma status without polling
  */
 static int sgdma_rxbusy(struct altera_tse_private *priv)
 {
 	return csrrd32(priv->rx_dma_csr, sgdma_csroffs(status))
 		       & SGDMA_STSREG_BUSY;
 }

 /* waits for the tx sgdma to finish it's current operation, returns 0
  * when it transitions to nonbusy, returns 1 if the operation times out
  */
 static int sgdma_txbusy(struct altera_tse_private *priv)
 {
 	int delay = 0;

 	/* if DMA is busy, wait for current transactino to finish */
 	while ((csrrd32(priv->tx_dma_csr, sgdma_csroffs(status))
 		& SGDMA_STSREG_BUSY) && (delay++ < 100))
 		udelay(1);

 	if (csrrd32(priv->tx_dma_csr, sgdma_csroffs(status))
 	    & SGDMA_STSREG_BUSY) {
 		netdev_err(priv->dev, "timeout waiting for tx dma\n");
 		return 1;
 	}
 	return 0;
 }
	/* Altera TSE SGDMA and MSGDMA Linux driver
	* Copyright (C) 2014 Altera Corporation. All rights reserved
	*
	* This program is free software; you can redistribute it and/or modify it
	* under the terms and conditions of the GNU General Public License,
	* version 2, as published by the Free Software Foundation.
	*
	* This program is distributed in the hope it will be useful, but WITHOUT
	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	* more details.
	*
	* You should have received a copy of the GNU General Public License along with
	* this program. If not, see <http://www.gnu.org/licenses/>.
	*/

	#include <linux/list.h>
	#include "altera_utils.h"
	#include "altera_tse.h"
	#include "altera_sgdmahw.h"
	#include "altera_sgdma.h"

	static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc,
	struct sgdma_descrip __iomem *ndesc,
	dma_addr_t ndesc_phys,
	dma_addr_t raddr,
	dma_addr_t waddr,
	u16 length,
	int generate_eop,
	int rfixed,
	int wfixed);

	static int sgdma_async_write(struct altera_tse_private *priv,
	struct sgdma_descrip __iomem *desc);

	static int sgdma_async_read(struct altera_tse_private *priv);

	static dma_addr_t
	sgdma_txphysaddr(struct altera_tse_private *priv,
	struct sgdma_descrip __iomem *desc);

	static dma_addr_t
	sgdma_rxphysaddr(struct altera_tse_private *priv,
	struct sgdma_descrip __iomem *desc);

	static int sgdma_txbusy(struct altera_tse_private *priv);

	static int sgdma_rxbusy(struct altera_tse_private *priv);

	static void
	queue_tx(struct altera_tse_private priv, struct tse_buffer buffer);

	static void
	queue_rx(struct altera_tse_private priv, struct tse_buffer buffer);

	static struct tse_buffer *
	dequeue_tx(struct altera_tse_private *priv);

	static struct tse_buffer *
	dequeue_rx(struct altera_tse_private *priv);

	static struct tse_buffer *
	queue_rx_peekhead(struct altera_tse_private *priv);

	int sgdma_initialize(struct altera_tse_private *priv)
	{
	priv->txctrlreg = SGDMA_CTRLREG_ILASTD \|
	SGDMA_CTRLREG_INTEN;

	priv->rxctrlreg = SGDMA_CTRLREG_IDESCRIP \|
	SGDMA_CTRLREG_INTEN \|
	SGDMA_CTRLREG_ILASTD;

	INIT_LIST_HEAD(&priv->txlisthd);
	INIT_LIST_HEAD(&priv->rxlisthd);

	priv->rxdescphys = (dma_addr_t) 0;
	priv->txdescphys = (dma_addr_t) 0;

	priv->rxdescphys = dma_map_single(priv->device,
	(void __force *)priv->rx_dma_desc,
	priv->rxdescmem, DMA_BIDIRECTIONAL);

	if (dma_mapping_error(priv->device, priv->rxdescphys)) {
	sgdma_uninitialize(priv);
	netdev_err(priv->dev, "error mapping rx descriptor memory\n");
	return -EINVAL;
	}

	priv->txdescphys = dma_map_single(priv->device,
	(void __force *)priv->tx_dma_desc,
	priv->txdescmem, DMA_TO_DEVICE);

	if (dma_mapping_error(priv->device, priv->txdescphys)) {
	sgdma_uninitialize(priv);
	netdev_err(priv->dev, "error mapping tx descriptor memory\n");
	return -EINVAL;
	}

	/* Initialize descriptor memory to all 0's, sync memory to cache */
	memset_io(priv->tx_dma_desc, 0, priv->txdescmem);
	memset_io(priv->rx_dma_desc, 0, priv->rxdescmem);

	dma_sync_single_for_device(priv->device, priv->txdescphys,
	priv->txdescmem, DMA_TO_DEVICE);

	dma_sync_single_for_device(priv->device, priv->rxdescphys,
	priv->rxdescmem, DMA_TO_DEVICE);

	return 0;
	}

	void sgdma_uninitialize(struct altera_tse_private *priv)
	{
	if (priv->rxdescphys)
	dma_unmap_single(priv->device, priv->rxdescphys,
	priv->rxdescmem, DMA_BIDIRECTIONAL);

	if (priv->txdescphys)
	dma_unmap_single(priv->device, priv->txdescphys,
	priv->txdescmem, DMA_TO_DEVICE);
	}

	/* This function resets the SGDMA controller and clears the
	* descriptor memory used for transmits and receives.
	*/
	void sgdma_reset(struct altera_tse_private *priv)
	{
	/* Initialize descriptor memory to 0 */
	memset_io(priv->tx_dma_desc, 0, priv->txdescmem);
	memset_io(priv->rx_dma_desc, 0, priv->rxdescmem);

	csrwr32(SGDMA_CTRLREG_RESET, priv->tx_dma_csr, sgdma_csroffs(control));
	csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control));

	csrwr32(SGDMA_CTRLREG_RESET, priv->rx_dma_csr, sgdma_csroffs(control));
	csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control));
	}

	/* For SGDMA, interrupts remain enabled after initially enabling,
	* so no need to provide implementations for abstract enable
	* and disable
	*/

	void sgdma_enable_rxirq(struct altera_tse_private *priv)
	{
	}

	void sgdma_enable_txirq(struct altera_tse_private *priv)
	{
	}

	void sgdma_disable_rxirq(struct altera_tse_private *priv)
	{
	}

	void sgdma_disable_txirq(struct altera_tse_private *priv)
	{
	}

	void sgdma_clear_rxirq(struct altera_tse_private *priv)
	{
	tse_set_bit(priv->rx_dma_csr, sgdma_csroffs(control),
	SGDMA_CTRLREG_CLRINT);
	}

	void sgdma_clear_txirq(struct altera_tse_private *priv)
	{
	tse_set_bit(priv->tx_dma_csr, sgdma_csroffs(control),
	SGDMA_CTRLREG_CLRINT);
	}

	/* transmits buffer through SGDMA. Returns number of buffers
	* transmitted, 0 if not possible.
	*
	* tx_lock is held by the caller
	*/
	int sgdma_tx_buffer(struct altera_tse_private priv, struct tse_buffer buffer)
	{
	struct sgdma_descrip __iomem *descbase =
	(struct sgdma_descrip __iomem *)priv->tx_dma_desc;

	struct sgdma_descrip __iomem *cdesc = &descbase[0];
	struct sgdma_descrip __iomem *ndesc = &descbase[1];

	/* wait 'til the tx sgdma is ready for the next transmit request */
	if (sgdma_txbusy(priv))
	return 0;

	sgdma_setup_descrip(cdesc, /* current descriptor */
	ndesc, /* next descriptor */
	sgdma_txphysaddr(priv, ndesc),
	buffer->dma_addr, /* address of packet to xmit */
	0, /* write addr 0 for tx dma */
	buffer->len, /* length of packet */
	SGDMA_CONTROL_EOP, /* Generate EOP */
	0, /* read fixed */
	SGDMA_CONTROL_WR_FIXED); /* Generate SOP */

	sgdma_async_write(priv, cdesc);

	/* enqueue the request to the pending transmit queue */
	queue_tx(priv, buffer);

	return 1;
	}


	/* tx_lock held to protect access to queued tx list
	*/
	u32 sgdma_tx_completions(struct altera_tse_private *priv)
	{
	u32 ready = 0;

	if (!sgdma_txbusy(priv) &&
	((csrrd8(priv->tx_dma_desc, sgdma_descroffs(control))
	& SGDMA_CONTROL_HW_OWNED) == 0) &&
	(dequeue_tx(priv))) {
	ready = 1;
	}

	return ready;
	}

	void sgdma_start_rxdma(struct altera_tse_private *priv)
	{
	sgdma_async_read(priv);
	}

	void sgdma_add_rx_desc(struct altera_tse_private *priv,
	struct tse_buffer *rxbuffer)
	{
	queue_rx(priv, rxbuffer);
	}

	/* status is returned on upper 16 bits,
	* length is returned in lower 16 bits
	*/
	u32 sgdma_rx_status(struct altera_tse_private *priv)
	{
	struct sgdma_descrip __iomem *base =
	(struct sgdma_descrip __iomem *)priv->rx_dma_desc;
	struct sgdma_descrip __iomem *desc = NULL;
	struct tse_buffer *rxbuffer = NULL;
	unsigned int rxstatus = 0;

	u32 sts = csrrd32(priv->rx_dma_csr, sgdma_csroffs(status));

	desc = &base[0];
	if (sts & SGDMA_STSREG_EOP) {
	unsigned int pktlength = 0;
	unsigned int pktstatus = 0;
	dma_sync_single_for_cpu(priv->device,
	priv->rxdescphys,
	SGDMA_DESC_LEN,
	DMA_FROM_DEVICE);

	pktlength = csrrd16(desc, sgdma_descroffs(bytes_xferred));
	pktstatus = csrrd8(desc, sgdma_descroffs(status));
	rxstatus = pktstatus & ~SGDMA_STATUS_EOP;
	rxstatus = rxstatus << 16;
	rxstatus \|= (pktlength & 0xffff);

	if (rxstatus) {
	csrwr8(0, desc, sgdma_descroffs(status));

	rxbuffer = dequeue_rx(priv);
	if (rxbuffer == NULL)
	netdev_info(priv->dev,
	"sgdma rx and rx queue empty!\n");

	/* Clear control */
	csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control));
	/* clear status */
	csrwr32(0xf, priv->rx_dma_csr, sgdma_csroffs(status));

	/* kick the rx sgdma after reaping this descriptor */
	sgdma_async_read(priv);

	} else {
	/* If the SGDMA indicated an end of packet on recv,
	* then it's expected that the rxstatus from the
	* descriptor is non-zero - meaning a valid packet
	* with a nonzero length, or an error has been
	* indicated. if not, then all we can do is signal
	* an error and return no packet received. Most likely
	* there is a system design error, or an error in the
	* underlying kernel (cache or cache management problem)
	*/
	netdev_err(priv->dev,
	"SGDMA RX Error Info: %x, %x, %x\n",
	sts, csrrd8(desc, sgdma_descroffs(status)),
	rxstatus);
	}
	} else if (sts == 0) {
	sgdma_async_read(priv);
	}

	return rxstatus;
	}


	/* Private functions */
	static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc,
	struct sgdma_descrip __iomem *ndesc,
	dma_addr_t ndesc_phys,
	dma_addr_t raddr,
	dma_addr_t waddr,
	u16 length,
	int generate_eop,
	int rfixed,
	int wfixed)
	{
	/* Clear the next descriptor as not owned by hardware */

	u32 ctrl = csrrd8(ndesc, sgdma_descroffs(control));
	ctrl &= ~SGDMA_CONTROL_HW_OWNED;
	csrwr8(ctrl, ndesc, sgdma_descroffs(control));

	ctrl = SGDMA_CONTROL_HW_OWNED;
	ctrl \|= generate_eop;
	ctrl \|= rfixed;
	ctrl \|= wfixed;

	/* Channel is implicitly zero, initialized to 0 by default */
	csrwr32(lower_32_bits(raddr), desc, sgdma_descroffs(raddr));
	csrwr32(lower_32_bits(waddr), desc, sgdma_descroffs(waddr));

	csrwr32(0, desc, sgdma_descroffs(pad1));
	csrwr32(0, desc, sgdma_descroffs(pad2));
	csrwr32(lower_32_bits(ndesc_phys), desc, sgdma_descroffs(next));

	csrwr8(ctrl, desc, sgdma_descroffs(control));
	csrwr8(0, desc, sgdma_descroffs(status));
	csrwr8(0, desc, sgdma_descroffs(wburst));
	csrwr8(0, desc, sgdma_descroffs(rburst));
	csrwr16(length, desc, sgdma_descroffs(bytes));
	csrwr16(0, desc, sgdma_descroffs(bytes_xferred));
	}

	/* If hardware is busy, don't restart async read.
	* if status register is 0 - meaning initial state, restart async read,
	* probably for the first time when populating a receive buffer.
	* If read status indicate not busy and a status, restart the async
	* DMA read.
	*/
	static int sgdma_async_read(struct altera_tse_private *priv)
	{
	struct sgdma_descrip __iomem *descbase =
	(struct sgdma_descrip __iomem *)priv->rx_dma_desc;

	struct sgdma_descrip __iomem *cdesc = &descbase[0];
	struct sgdma_descrip __iomem *ndesc = &descbase[1];
	struct tse_buffer *rxbuffer = NULL;

	if (!sgdma_rxbusy(priv)) {
	rxbuffer = queue_rx_peekhead(priv);
	if (rxbuffer == NULL) {
	netdev_err(priv->dev, "no rx buffers available\n");
	return 0;
	}

	sgdma_setup_descrip(cdesc, /* current descriptor */
	ndesc, /* next descriptor */
	sgdma_rxphysaddr(priv, ndesc),
	0, /* read addr 0 for rx dma */
	rxbuffer->dma_addr, /* write addr for rx dma */
	0, /* read 'til EOP */
	0, /* EOP: NA for rx dma */
	0, /* read fixed: NA for rx dma */
	0); /* SOP: NA for rx DMA */

	dma_sync_single_for_device(priv->device,
	priv->rxdescphys,
	SGDMA_DESC_LEN,
	DMA_TO_DEVICE);

	csrwr32(lower_32_bits(sgdma_rxphysaddr(priv, cdesc)),
	priv->rx_dma_csr,
	sgdma_csroffs(next_descrip));

	csrwr32((priv->rxctrlreg \| SGDMA_CTRLREG_START),
	priv->rx_dma_csr,
	sgdma_csroffs(control));

	return 1;
	}

	return 0;
	}

	static int sgdma_async_write(struct altera_tse_private *priv,
	struct sgdma_descrip __iomem *desc)
	{
	if (sgdma_txbusy(priv))
	return 0;

	/* clear control and status */
	csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control));
	csrwr32(0x1f, priv->tx_dma_csr, sgdma_csroffs(status));

	dma_sync_single_for_device(priv->device, priv->txdescphys,
	SGDMA_DESC_LEN, DMA_TO_DEVICE);

	csrwr32(lower_32_bits(sgdma_txphysaddr(priv, desc)),
	priv->tx_dma_csr,
	sgdma_csroffs(next_descrip));

	csrwr32((priv->txctrlreg \| SGDMA_CTRLREG_START),
	priv->tx_dma_csr,
	sgdma_csroffs(control));

	return 1;
	}

	static dma_addr_t
	sgdma_txphysaddr(struct altera_tse_private *priv,
	struct sgdma_descrip __iomem *desc)
	{
	dma_addr_t paddr = priv->txdescmem_busaddr;
	uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->tx_dma_desc;
	return (dma_addr_t)((uintptr_t)paddr + offs);
	}

	static dma_addr_t
	sgdma_rxphysaddr(struct altera_tse_private *priv,
	struct sgdma_descrip __iomem *desc)
	{
	dma_addr_t paddr = priv->rxdescmem_busaddr;
	uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->rx_dma_desc;
	return (dma_addr_t)((uintptr_t)paddr + offs);
	}

	#define list_remove_head(list, entry, type, member) \
	do { \
	entry = NULL; \
	if (!list_empty(list)) { \
	entry = list_entry((list)->next, type, member); \
	list_del_init(&entry->member); \
	} \
	} while (0)

	#define list_peek_head(list, entry, type, member) \
	do { \
	entry = NULL; \
	if (!list_empty(list)) { \
	entry = list_entry((list)->next, type, member); \
	} \
	} while (0)

	/* adds a tse_buffer to the tail of a tx buffer list.
	* assumes the caller is managing and holding a mutual exclusion
	* primitive to avoid simultaneous pushes/pops to the list.
	*/
	static void
	queue_tx(struct altera_tse_private priv, struct tse_buffer buffer)
	{
	list_add_tail(&buffer->lh, &priv->txlisthd);
	}


	/* adds a tse_buffer to the tail of a rx buffer list
	* assumes the caller is managing and holding a mutual exclusion
	* primitive to avoid simultaneous pushes/pops to the list.
	*/
	static void
	queue_rx(struct altera_tse_private priv, struct tse_buffer buffer)
	{
	list_add_tail(&buffer->lh, &priv->rxlisthd);
	}

	/* dequeues a tse_buffer from the transmit buffer list, otherwise
	* returns NULL if empty.
	* assumes the caller is managing and holding a mutual exclusion
	* primitive to avoid simultaneous pushes/pops to the list.
	*/
	static struct tse_buffer *
	dequeue_tx(struct altera_tse_private *priv)
	{
	struct tse_buffer *buffer = NULL;
	list_remove_head(&priv->txlisthd, buffer, struct tse_buffer, lh);
	return buffer;
	}

	/* dequeues a tse_buffer from the receive buffer list, otherwise
	* returns NULL if empty
	* assumes the caller is managing and holding a mutual exclusion
	* primitive to avoid simultaneous pushes/pops to the list.
	*/
	static struct tse_buffer *
	dequeue_rx(struct altera_tse_private *priv)
	{
	struct tse_buffer *buffer = NULL;
	list_remove_head(&priv->rxlisthd, buffer, struct tse_buffer, lh);
	return buffer;
	}

	/* dequeues a tse_buffer from the receive buffer list, otherwise
	* returns NULL if empty
	* assumes the caller is managing and holding a mutual exclusion
	* primitive to avoid simultaneous pushes/pops to the list while the
	* head is being examined.
	*/
	static struct tse_buffer *
	queue_rx_peekhead(struct altera_tse_private *priv)
	{
	struct tse_buffer *buffer = NULL;
	list_peek_head(&priv->rxlisthd, buffer, struct tse_buffer, lh);
	return buffer;
	}

	/* check and return rx sgdma status without polling
	*/
	static int sgdma_rxbusy(struct altera_tse_private *priv)
	{
	return csrrd32(priv->rx_dma_csr, sgdma_csroffs(status))
	& SGDMA_STSREG_BUSY;
	}

	/* waits for the tx sgdma to finish it's current operation, returns 0
	* when it transitions to nonbusy, returns 1 if the operation times out
	*/
	static int sgdma_txbusy(struct altera_tse_private *priv)
	{
	int delay = 0;

	/* if DMA is busy, wait for current transactino to finish */
	while ((csrrd32(priv->tx_dma_csr, sgdma_csroffs(status))
	& SGDMA_STSREG_BUSY) && (delay++ < 100))
	udelay(1);

	if (csrrd32(priv->tx_dma_csr, sgdma_csroffs(status))
	& SGDMA_STSREG_BUSY) {
	netdev_err(priv->dev, "timeout waiting for tx dma\n");
	return 1;
	}
	return 0;
	}