[PATCH] xtensa: Architecture support for Tensilica Xtensa Part 4

#

# Makefile for Xtensa-specific library files.

#

lib-y	+= memcopy.o memset.o checksum.o strcasecmp.o \

	   usercopy.o strncpy_user.o strnlen_user.o

lib-$(CONFIG_PCI) += pci-auto.o

/*

 * INET		An implementation of the TCP/IP protocol suite for the LINUX

 *		operating system.  INET is implemented using the  BSD Socket

 *		interface as the means of communication with the user level.

 *

 *		IP/TCP/UDP checksumming routines

 *

 * Xtensa version:  Copyright (C) 2001 Tensilica, Inc. by Kevin Chea

 *                  Optimized by Joe Taylor

 *

 *		This program is free software; you can redistribute it and/or

 *		modify it under the terms of the GNU General Public License

 *		as published by the Free Software Foundation; either version

 *		2 of the License, or (at your option) any later version.

 */

#include <asm/errno.h>

#include <linux/linkage.h>

#define _ASMLANGUAGE

#include <xtensa/config/core.h>

/*

 * computes a partial checksum, e.g. for TCP/UDP fragments

 */

/*

 * unsigned int csum_partial(const unsigned char *buf, int len,

 *                           unsigned int sum);

 *    a2 = buf

 *    a3 = len

 *    a4 = sum

 *

 * This function assumes 2- or 4-byte alignment.  Other alignments will fail!

 */

/* ONES_ADD converts twos-complement math to ones-complement. */

#define ONES_ADD(sum, val)	  \

	add	sum, sum, val	; \

	bgeu	sum, val, 99f	; \

	addi	sum, sum, 1	; \

99:				;

.text

ENTRY(csum_partial)

	  /*

	   * Experiments with Ethernet and SLIP connections show that buf

	   * is aligned on either a 2-byte or 4-byte boundary.

	   */

	entry	sp, 32

	extui	a5, a2, 0, 2

	bnez	a5, 8f		/* branch if 2-byte aligned */

	/* Fall-through on common case, 4-byte alignment */

1:

	srli	a5, a3, 5	/* 32-byte chunks */

#if XCHAL_HAVE_LOOPS

	loopgtz	a5, 2f

#else

	beqz	a5, 2f

	slli	a5, a5, 5

	add	a5, a5, a2	/* a5 = end of last 32-byte chunk */

.Loop1:

#endif

	l32i	a6, a2, 0

	l32i	a7, a2, 4

	ONES_ADD(a4, a6)

	ONES_ADD(a4, a7)

	l32i	a6, a2, 8

	l32i	a7, a2, 12

	ONES_ADD(a4, a6)

	ONES_ADD(a4, a7)

	l32i	a6, a2, 16

	l32i	a7, a2, 20

	ONES_ADD(a4, a6)

	ONES_ADD(a4, a7)

	l32i	a6, a2, 24

	l32i	a7, a2, 28

	ONES_ADD(a4, a6)

	ONES_ADD(a4, a7)

	addi	a2, a2, 4*8

#if !XCHAL_HAVE_LOOPS

	blt	a2, a5, .Loop1

#endif

2:

	extui	a5, a3, 2, 3	/* remaining 4-byte chunks */

#if XCHAL_HAVE_LOOPS

	loopgtz	a5, 3f

#else

	beqz	a5, 3f

	slli	a5, a5, 2

	add	a5, a5, a2	/* a5 = end of last 4-byte chunk */

.Loop2:

#endif

	l32i	a6, a2, 0

	ONES_ADD(a4, a6)

	addi	a2, a2, 4

#if !XCHAL_HAVE_LOOPS

	blt	a2, a5, .Loop2

#endif

3:

	_bbci.l	a3, 1, 5f	/* remaining 2-byte chunk */

	l16ui	a6, a2, 0

	ONES_ADD(a4, a6)

	addi	a2, a2, 2

5:

	_bbci.l	a3, 0, 7f	/* remaining 1-byte chunk */

6:	l8ui	a6, a2, 0

#ifdef __XTENSA_EB__

	slli	a6, a6, 8	/* load byte into bits 8..15 */

#endif

	ONES_ADD(a4, a6)

7:

	mov	a2, a4

	retw

	/* uncommon case, buf is 2-byte aligned */

8:

	beqz	a3, 7b		/* branch if len == 0 */

	beqi	a3, 1, 6b	/* branch if len == 1 */

	extui	a5, a2, 0, 1

	bnez	a5, 8f		/* branch if 1-byte aligned */

	l16ui	a6, a2, 0	/* common case, len >= 2 */

	ONES_ADD(a4, a6)

	addi	a2, a2, 2	/* adjust buf */

	addi	a3, a3, -2	/* adjust len */

	j	1b		/* now buf is 4-byte aligned */

	/* case: odd-byte aligned, len > 1

	 * This case is dog slow, so don't give us an odd address.

	 * (I don't think this ever happens, but just in case.)

	 */

8:

	srli	a5, a3, 2	/* 4-byte chunks */

#if XCHAL_HAVE_LOOPS

	loopgtz	a5, 2f

#else

	beqz	a5, 2f

	slli	a5, a5, 2

	add	a5, a5, a2	/* a5 = end of last 4-byte chunk */

.Loop3:

#endif

	l8ui	a6, a2, 0	/* bits 24..31 */

	l16ui	a7, a2, 1	/* bits  8..23 */

	l8ui	a8, a2, 3	/* bits  0.. 8 */

#ifdef	__XTENSA_EB__

	slli	a6, a6, 24

#else

	slli	a8, a8, 24

#endif

	slli	a7, a7, 8

	or	a7, a7, a6

	or	a7, a7, a8

	ONES_ADD(a4, a7)

	addi	a2, a2, 4

#if !XCHAL_HAVE_LOOPS

	blt	a2, a5, .Loop3

#endif

2:

	_bbci.l	a3, 1, 3f	/* remaining 2-byte chunk, still odd addr */

	l8ui	a6, a2, 0

	l8ui	a7, a2, 1

#ifdef	__XTENSA_EB__

	slli	a6, a6, 8

#else

	slli	a7, a7, 8

#endif

	or	a7, a7, a6

	ONES_ADD(a4, a7)

	addi	a2, a2, 2

3:

	j	5b		/* branch to handle the remaining byte */

/*

 * Copy from ds while checksumming, otherwise like csum_partial

 *

 * The macros SRC and DST specify the type of access for the instruction.

 * thus we can call a custom exception handler for each access type.

 */

#define SRC(y...)			\

	9999: y;			\

	.section __ex_table, "a";	\

	.long 9999b, 6001f	;	\

	.previous

#define DST(y...)			\

	9999: y;			\

	.section __ex_table, "a";	\

	.long 9999b, 6002f	;	\

	.previous

/*

unsigned int csum_partial_copy_generic (const char *src, char *dst, int len,

					int sum, int *src_err_ptr, int *dst_err_ptr)

	a2  = src

	a3  = dst

	a4  = len

	a5  = sum

	a6  = src_err_ptr

	a7  = dst_err_ptr

	a8  = temp

	a9  = temp

	a10 = temp

	a11 = original len for exception handling

	a12 = original dst for exception handling

    This function is optimized for 4-byte aligned addresses.  Other

    alignments work, but not nearly as efficiently.

 */

ENTRY(csum_partial_copy_generic)

	entry	sp, 32

	mov	a12, a3

	mov	a11, a4

	or	a10, a2, a3

	/* We optimize the following alignment tests for the 4-byte

	aligned case.  Two bbsi.l instructions might seem more optimal

	(commented out below).  However, both labels 5: and 3: are out

	of the imm8 range, so the assembler relaxes them into

	equivalent bbci.l, j combinations, which is actually

	slower. */

	extui	a9, a10, 0, 2

	beqz	a9, 1f		/* branch if both are 4-byte aligned */

	bbsi.l	a10, 0, 5f	/* branch if one address is odd */

	j	3f		/* one address is 2-byte aligned */

/*	_bbsi.l	a10, 0, 5f */	/* branch if odd address */

/*	_bbsi.l	a10, 1, 3f */	/* branch if 2-byte-aligned address */

1:

	/* src and dst are both 4-byte aligned */

	srli	a10, a4, 5	/* 32-byte chunks */

#if XCHAL_HAVE_LOOPS

	loopgtz	a10, 2f

#else

	beqz	a10, 2f

	slli	a10, a10, 5

	add	a10, a10, a2	/* a10 = end of last 32-byte src chunk */

.Loop5:

#endif

SRC(	l32i	a9, a2, 0	)

SRC(	l32i	a8, a2, 4	)

DST(	s32i	a9, a3, 0	)

DST(	s32i	a8, a3, 4	)

	ONES_ADD(a5, a9)

	ONES_ADD(a5, a8)

SRC(	l32i	a9, a2, 8	)

SRC(	l32i	a8, a2, 12	)

DST(	s32i	a9, a3, 8	)

DST(	s32i	a8, a3, 12	)

	ONES_ADD(a5, a9)

	ONES_ADD(a5, a8)

SRC(	l32i	a9, a2, 16	)

SRC(	l32i	a8, a2, 20	)

DST(	s32i	a9, a3, 16	)

DST(	s32i	a8, a3, 20	)

	ONES_ADD(a5, a9)

	ONES_ADD(a5, a8)

SRC(	l32i	a9, a2, 24	)

SRC(	l32i	a8, a2, 28	)

DST(	s32i	a9, a3, 24	)

DST(	s32i	a8, a3, 28	)

	ONES_ADD(a5, a9)

	ONES_ADD(a5, a8)

	addi	a2, a2, 32

	addi	a3, a3, 32

#if !XCHAL_HAVE_LOOPS

	blt	a2, a10, .Loop5

#endif

2:

	extui	a10, a4, 2, 3	/* remaining 4-byte chunks */

	extui	a4, a4, 0, 2	/* reset len for general-case, 2-byte chunks */

#if XCHAL_HAVE_LOOPS

	loopgtz	a10, 3f

#else

	beqz	a10, 3f

	slli	a10, a10, 2

	add	a10, a10, a2	/* a10 = end of last 4-byte src chunk */

.Loop6:

#endif

SRC(	l32i	a9, a2, 0	)

DST(	s32i	a9, a3, 0	)

	ONES_ADD(a5, a9)

	addi	a2, a2, 4

	addi	a3, a3, 4

#if !XCHAL_HAVE_LOOPS

	blt	a2, a10, .Loop6

#endif

3:

	/*

	Control comes to here in two cases: (1) It may fall through

	to here from the 4-byte alignment case to process, at most,

	one 2-byte chunk.  (2) It branches to here from above if

	either src or dst is 2-byte aligned, and we process all bytes

	here, except for perhaps a trailing odd byte.  It's

	inefficient, so align your addresses to 4-byte boundaries.

	a2 = src

	a3 = dst

	a4 = len

	a5 = sum

	*/

	srli	a10, a4, 1	/* 2-byte chunks */

#if XCHAL_HAVE_LOOPS

	loopgtz	a10, 4f

#else

	beqz	a10, 4f

	slli	a10, a10, 1

	add	a10, a10, a2	/* a10 = end of last 2-byte src chunk */

.Loop7:

#endif

SRC(	l16ui	a9, a2, 0	)

DST(	s16i	a9, a3, 0	)

	ONES_ADD(a5, a9)

	addi	a2, a2, 2

	addi	a3, a3, 2

#if !XCHAL_HAVE_LOOPS

	blt	a2, a10, .Loop7

#endif

4:

	/* This section processes a possible trailing odd byte. */

	_bbci.l	a4, 0, 8f	/* 1-byte chunk */

SRC(	l8ui	a9, a2, 0	)

DST(	s8i	a9, a3, 0	)

#ifdef __XTENSA_EB__

	slli	a9, a9, 8	/* shift byte to bits 8..15 */

#endif

	ONES_ADD(a5, a9)

8:

	mov	a2, a5

	retw

5:

	/* Control branch to here when either src or dst is odd.  We

	process all bytes using 8-bit accesses.  Grossly inefficient,

	so don't feed us an odd address. */

	srli	a10, a4, 1	/* handle in pairs for 16-bit csum */

#if XCHAL_HAVE_LOOPS

	loopgtz	a10, 6f

#else

	beqz	a10, 6f

	slli	a10, a10, 1

	add	a10, a10, a2	/* a10 = end of last odd-aligned, 2-byte src chunk */

.Loop8:

#endif

SRC(	l8ui	a9, a2, 0	)

SRC(	l8ui	a8, a2, 1	)

DST(	s8i	a9, a3, 0	)

DST(	s8i	a8, a3, 1	)

#ifdef __XTENSA_EB__

	slli	a9, a9, 8	/* combine into a single 16-bit value */

#else				/* for checksum computation */

	slli	a8, a8, 8

#endif

	or	a9, a9, a8

	ONES_ADD(a5, a9)

	addi	a2, a2, 2

	addi	a3, a3, 2

#if !XCHAL_HAVE_LOOPS

	blt	a2, a10, .Loop8

#endif

6:

	j	4b		/* process the possible trailing odd byte */

# Exception handler:

.section .fixup, "ax"

/*

	a6  = src_err_ptr

	a7  = dst_err_ptr

	a11 = original len for exception handling

	a12 = original dst for exception handling

*/

6001:

	_movi	a2, -EFAULT

	s32i	a2, a6, 0	/* src_err_ptr */

	# clear the complete destination - computing the rest

	# is too much work

	movi	a2, 0

#if XCHAL_HAVE_LOOPS

	loopgtz	a11, 2f

#else

	beqz	a11, 2f

	add	a11, a11, a12	/* a11 = ending address */

.Leloop:

#endif

	s8i	a2, a12, 0

	addi	a12, a12, 1

#if !XCHAL_HAVE_LOOPS

	blt	a12, a11, .Leloop

#endif

2:

	retw

6002:

	movi	a2, -EFAULT

	s32i	a2, a7, 0	/* dst_err_ptr */

	movi	a2, 0

	retw

.previous

/*

 * arch/xtensa/lib/hal/memcopy.S -- Core HAL library functions

 * xthal_memcpy and xthal_bcopy

 *

 * This file is subject to the terms and conditions of the GNU General Public

 * License.  See the file "COPYING" in the main directory of this archive

 * for more details.

 *

 * Copyright (C) 2002 - 2005 Tensilica Inc.

 */

#include <xtensa/coreasm.h>

	.macro	src_b	r, w0, w1

#ifdef __XTENSA_EB__

	src	\r, \w0, \w1

#else

	src	\r, \w1, \w0

#endif

	.endm

	.macro	ssa8	r

#ifdef __XTENSA_EB__

	ssa8b	\r

#else

	ssa8l	\r

#endif

	.endm

/*

 * void *memcpy(void *dst, const void *src, size_t len);

 * void *memmove(void *dst, const void *src, size_t len);

 * void *bcopy(const void *src, void *dst, size_t len);

 *

 * This function is intended to do the same thing as the standard

 * library function memcpy() (or bcopy()) for most cases.

 * However, where the source and/or destination references

 * an instruction RAM or ROM or a data RAM or ROM, that

 * source and/or destination will always be accessed with

 * 32-bit load and store instructions (as required for these

 * types of devices).

 *

 * !!!!!!!  XTFIXME:

 * !!!!!!!  Handling of IRAM/IROM has not yet

 * !!!!!!!  been implemented.

 *

 * The bcopy version is provided here to avoid the overhead

 * of an extra call, for callers that require this convention.

 *

 * The (general case) algorithm is as follows:

 *   If destination is unaligned, align it by conditionally

 *     copying 1 and 2 bytes.

 *   If source is aligned,

 *     do 16 bytes with a loop, and then finish up with

 *     8, 4, 2, and 1 byte copies conditional on the length;

 *   else (if source is unaligned),

 *     do the same, but use SRC to align the source data.

 *   This code tries to use fall-through branches for the common

 *     case of aligned source and destination and multiple

 *     of 4 (or 8) length.

 *

 * Register use:

 *	a0/ return address

 *	a1/ stack pointer

 *	a2/ return value

 *	a3/ src

 *	a4/ length

 *	a5/ dst

 *	a6/ tmp

 *	a7/ tmp

 *	a8/ tmp

 *	a9/ tmp

 *	a10/ tmp

 *	a11/ tmp

 */

	.text

	.align	4

	.global	bcopy

	.type   bcopy,@function

bcopy:

	entry	sp, 16		# minimal stack frame

	# a2=src, a3=dst, a4=len

	mov	a5, a3		# copy dst so that a2 is return value

	mov	a3, a2

	mov	a2, a5

	j	.Lcommon	# go to common code for memcpy+bcopy

/*

 * Byte by byte copy

 */

	.align	4

	.byte	0		# 1 mod 4 alignment for LOOPNEZ

				# (0 mod 4 alignment for LBEG)

.Lbytecopy:

#if XCHAL_HAVE_LOOPS

	loopnez	a4, .Lbytecopydone

#else /* !XCHAL_HAVE_LOOPS */

	beqz	a4, .Lbytecopydone

	add	a7, a3, a4	# a7 = end address for source

#endif /* !XCHAL_HAVE_LOOPS */

.Lnextbyte:

	l8ui	a6, a3, 0

	addi	a3, a3, 1

	s8i	a6, a5, 0

	addi	a5, a5, 1

#if !XCHAL_HAVE_LOOPS

	blt	a3, a7, .Lnextbyte

#endif /* !XCHAL_HAVE_LOOPS */

.Lbytecopydone:

	retw

/*

 * Destination is unaligned

 */

	.align	4

.Ldst1mod2:	# dst is only byte aligned

	_bltui	a4, 7, .Lbytecopy	# do short copies byte by byte

	# copy 1 byte

	l8ui	a6, a3,  0

	addi	a3, a3,  1

	addi	a4, a4, -1

	s8i	a6, a5,  0

	addi	a5, a5,  1

	_bbci.l	a5, 1, .Ldstaligned	# if dst is now aligned, then

					# return to main algorithm

.Ldst2mod4:	# dst 16-bit aligned

	# copy 2 bytes

	_bltui	a4, 6, .Lbytecopy	# do short copies byte by byte

	l8ui	a6, a3,  0

	l8ui	a7, a3,  1

	addi	a3, a3,  2

	addi	a4, a4, -2

	s8i	a6, a5,  0

	s8i	a7, a5,  1

	addi	a5, a5,  2

	j	.Ldstaligned	# dst is now aligned, return to main algorithm

	.align	4

	.global	memcpy

	.type   memcpy,@function

memcpy:

	.global	memmove

	.type   memmove,@function

memmove:

	entry	sp, 16		# minimal stack frame

	# a2/ dst, a3/ src, a4/ len

	mov	a5, a2		# copy dst so that a2 is return value

.Lcommon:

	_bbsi.l	a2, 0, .Ldst1mod2	# if dst is 1 mod 2

	_bbsi.l	a2, 1, .Ldst2mod4	# if dst is 2 mod 4

.Ldstaligned:	# return here from .Ldst?mod? once dst is aligned

	srli	a7, a4, 4	# number of loop iterations with 16B

				# per iteration

	movi	a8, 3		# if source is not aligned,

	_bany	a3, a8, .Lsrcunaligned	# then use shifting copy

	/*

	 * Destination and source are word-aligned, use word copy.

	 */

	# copy 16 bytes per iteration for word-aligned dst and word-aligned src

#if XCHAL_HAVE_LOOPS

	loopnez	a7, .Loop1done

#else /* !XCHAL_HAVE_LOOPS */

	beqz	a7, .Loop1done

	slli	a8, a7, 4

	add	a8, a8, a3	# a8 = end of last 16B source chunk

#endif /* !XCHAL_HAVE_LOOPS */

.Loop1:

	l32i	a6, a3,  0

	l32i	a7, a3,  4

	s32i	a6, a5,  0

	l32i	a6, a3,  8

	s32i	a7, a5,  4

	l32i	a7, a3, 12

	s32i	a6, a5,  8

	addi	a3, a3, 16

	s32i	a7, a5, 12

	addi	a5, a5, 16

#if !XCHAL_HAVE_LOOPS

	blt	a3, a8, .Loop1

#endif /* !XCHAL_HAVE_LOOPS */

.Loop1done:

	bbci.l	a4, 3, .L2

	# copy 8 bytes

	l32i	a6, a3,  0

	l32i	a7, a3,  4

	addi	a3, a3,  8

	s32i	a6, a5,  0