Commit 0928368f authored by Kristof Beyls's avatar Kristof Beyls Committed by Siva Chandra Reddy
Browse files

[libc] Provide Arm Optimized Routines for the LLVM libc project. 

This adds the Arm Optimized Routines (see
https://github.com/ARM-software/optimized-routines) source code under the
the LLVM license. The version of the code provided in this patch is v20.02
of the Arm Optimized Routines project.

This entire contribution is being committed as is even though it does
not currently fit the LLVM libc model and does not follow the LLVM
coding style. In the near future, implementations from this patch will be
moved over to their right place in the LLVM-libc tree. This will be done
over many small patches, all of which will go through the normal LLVM code
review process. See this libc-dev post for the plan:
http://lists.llvm.org/pipermail/libc-dev/2020-March/000044.html

Differential revision of the original upload: https://reviews.llvm.org/D75355
parent 9b151386

libc/AOR_v20.02/Makefile

0 → 100644
+90 −0
Original line number Diff line number Diff line 
# Makefile - requires GNU make

#

# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

# See https://llvm.org/LICENSE.txt for license information.

# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception



srcdir = .

prefix = /usr

bindir = $(prefix)/bin

libdir = $(prefix)/lib

includedir = $(prefix)/include



# Configure these in config.mk, do not make changes in this file.

SUBS = math string networking

HOST_CC = cc

HOST_CFLAGS = -std=c99 -O2

HOST_LDFLAGS =

HOST_LDLIBS =

EMULATOR =

CPPFLAGS =

CFLAGS = -std=c99 -O2

CFLAGS_SHARED = -fPIC

CFLAGS_ALL = -Ibuild/include $(CPPFLAGS) $(CFLAGS)

LDFLAGS =

LDLIBS =

AR = $(CROSS_COMPILE)ar

RANLIB = $(CROSS_COMPILE)ranlib

INSTALL = install



all:



-include config.mk



$(foreach sub,$(SUBS),$(eval include $(srcdir)/$(sub)/Dir.mk))



# Required targets of subproject foo:

#   all-foo

#   check-foo

#   clean-foo

#   install-foo

# Required make variables of subproject foo:

#   foo-files: Built files (all in build/).

# Make variables used by subproject foo:

#   foo-...: Variables defined in foo/Dir.mk or by config.mk.



all: $(SUBS:%=all-%)



ALL_FILES = $(foreach sub,$(SUBS),$($(sub)-files))

DIRS = $(sort $(patsubst %/,%,$(dir $(ALL_FILES))))

$(ALL_FILES): | $(DIRS)

$(DIRS):

	mkdir -p $@



$(filter %.os,$(ALL_FILES)): CFLAGS_ALL += $(CFLAGS_SHARED)



build/%.o: $(srcdir)/%.S

	$(CC) $(CFLAGS_ALL) -c -o $@ $<



build/%.o: $(srcdir)/%.c

	$(CC) $(CFLAGS_ALL) -c -o $@ $<



build/%.os: $(srcdir)/%.S

	$(CC) $(CFLAGS_ALL) -c -o $@ $<



build/%.os: $(srcdir)/%.c

	$(CC) $(CFLAGS_ALL) -c -o $@ $<



clean: $(SUBS:%=clean-%)

	rm -rf build



distclean: clean

	rm -f config.mk



$(DESTDIR)$(bindir)/%: build/bin/%

	$(INSTALL) -D $< $@



$(DESTDIR)$(libdir)/%.so: build/lib/%.so

	$(INSTALL) -D $< $@



$(DESTDIR)$(libdir)/%: build/lib/%

	$(INSTALL) -m 644 -D $< $@



$(DESTDIR)$(includedir)/%: build/include/%

	$(INSTALL) -m 644 -D $< $@



install: $(SUBS:%=install-%)



check: $(SUBS:%=check-%)



.PHONY: all clean distclean install check

libc/AOR_v20.02/config.mk.dist

0 → 100644
+68 −0
Original line number Diff line number Diff line 
# Example config.mk

#

# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

# See https://llvm.org/LICENSE.txt for license information.

# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception



# Subprojects to build

SUBS = math string networking



# Target architecture: aarch64, arm or x86_64

ARCH = aarch64



# Compiler for the target

CC = $(CROSS_COMPILE)gcc

CFLAGS = -std=c99 -pipe -O3

CFLAGS += -Wall -Wno-missing-braces

CFLAGS += -Werror=implicit-function-declaration



# Used for test case generator that is executed on the host

HOST_CC = gcc

HOST_CFLAGS = -std=c99 -O2

HOST_CFLAGS += -Wall -Wno-unused-function



# Enable debug info.

HOST_CFLAGS += -g

CFLAGS += -g



# Optimize the shared libraries on aarch64 assuming they fit in 1M.

#CFLAGS_SHARED = -fPIC -mcmodel=tiny



# Use for cross compilation with gcc.

#CROSS_COMPILE = aarch64-none-linux-gnu-



# Use with cross testing.

#EMULATOR = qemu-aarch64-static

#EMULATOR = sh -c 'scp $$1 user@host:/dir && ssh user@host /dir/"$$@"' --



# Additional flags for subprojects.

math-cflags =

math-ldlibs =

math-ulpflags =

math-testflags =

string-cflags =

networking-cflags =



# Use if mpfr is available on the target for ulp error checking.

#math-ldlibs += -lmpfr -lgmp

#math-cflags += -DUSE_MPFR



# Use with gcc.

math-cflags += -frounding-math -fexcess-precision=standard -fno-stack-protector

math-cflags += -ffp-contract=fast -fno-math-errno



# Use with clang.

#math-cflags += -ffp-contract=fast



# Disable vector math code

#math-cflags += -DWANT_VMATH=0



# Disable fenv checks

#math-ulpflags = -q -f

#math-testflags = -nostatus



# Enable assertion checks.

#networking-cflags += -DWANT_ASSERT



# Avoid auto-vectorization of scalar code and unroll loops

networking-cflags += -O2 -fno-tree-vectorize -funroll-loops

libc/AOR_v20.02/math/Dir.mk

0 → 100644
+111 −0
Original line number Diff line number Diff line 
# Makefile fragment - requires GNU make

#

# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

# See https://llvm.org/LICENSE.txt for license information.

# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception



S := $(srcdir)/math

B := build/math



math-lib-srcs := $(wildcard $(S)/*.[cS])

math-test-srcs := \

	$(S)/test/mathtest.c \

	$(S)/test/mathbench.c \

	$(S)/test/ulp.c \



math-test-host-srcs := $(wildcard $(S)/test/rtest/*.[cS])



math-includes := $(patsubst $(S)/%,build/%,$(wildcard $(S)/include/*.h))



math-libs := \

	build/lib/libmathlib.so \

	build/lib/libmathlib.a \



math-tools := \

	build/bin/mathtest \

	build/bin/mathbench \

	build/bin/mathbench_libc \

	build/bin/runulp.sh \

	build/bin/ulp \



math-host-tools := \

	build/bin/rtest \



math-lib-objs := $(patsubst $(S)/%,$(B)/%.o,$(basename $(math-lib-srcs)))

math-test-objs := $(patsubst $(S)/%,$(B)/%.o,$(basename $(math-test-srcs)))

math-host-objs := $(patsubst $(S)/%,$(B)/%.o,$(basename $(math-test-host-srcs)))

math-target-objs := $(math-lib-objs) $(math-test-objs)

math-objs := $(math-target-objs) $(math-target-objs:%.o=%.os) $(math-host-objs)



math-files := \

	$(math-objs) \

	$(math-libs) \

	$(math-tools) \

	$(math-host-tools) \

	$(math-includes) \



all-math: $(math-libs) $(math-tools) $(math-includes)



$(math-objs): $(math-includes)

$(math-objs): CFLAGS_ALL += $(math-cflags)

$(B)/test/mathtest.o: CFLAGS_ALL += -fmath-errno

$(math-host-objs): CC = $(HOST_CC)

$(math-host-objs): CFLAGS_ALL = $(HOST_CFLAGS)



$(B)/test/ulp.o: $(S)/test/ulp.h



build/lib/libmathlib.so: $(math-lib-objs:%.o=%.os)

	$(CC) $(CFLAGS_ALL) $(LDFLAGS) -shared -o $@ $^



build/lib/libmathlib.a: $(math-lib-objs)

	rm -f $@

	$(AR) rc $@ $^

	$(RANLIB) $@



$(math-host-tools): HOST_LDLIBS += -lm -lmpfr -lmpc

$(math-tools): LDLIBS += $(math-ldlibs) -lm



build/bin/rtest: $(math-host-objs)

	$(HOST_CC) $(HOST_CFLAGS) $(HOST_LDFLAGS) -o $@ $^ $(HOST_LDLIBS)



build/bin/mathtest: $(B)/test/mathtest.o build/lib/libmathlib.a

	$(CC) $(CFLAGS_ALL) $(LDFLAGS) -static -o $@ $^ $(LDLIBS)



build/bin/mathbench: $(B)/test/mathbench.o build/lib/libmathlib.a

	$(CC) $(CFLAGS_ALL) $(LDFLAGS) -static -o $@ $^ $(LDLIBS)



# This is not ideal, but allows custom symbols in mathbench to get resolved.

build/bin/mathbench_libc: $(B)/test/mathbench.o build/lib/libmathlib.a

	$(CC) $(CFLAGS_ALL) $(LDFLAGS) -static -o $@ $< $(LDLIBS) -lc build/lib/libmathlib.a -lm



build/bin/ulp: $(B)/test/ulp.o build/lib/libmathlib.a

	$(CC) $(CFLAGS_ALL) $(LDFLAGS) -static -o $@ $^ $(LDLIBS)



build/include/%.h: $(S)/include/%.h

	cp $< $@



build/bin/%.sh: $(S)/test/%.sh

	cp $< $@



math-tests := $(wildcard $(S)/test/testcases/directed/*.tst)

math-rtests := $(wildcard $(S)/test/testcases/random/*.tst)



check-math-test: $(math-tools)

	cat $(math-tests) | $(EMULATOR) build/bin/mathtest $(math-testflags)



check-math-rtest: $(math-host-tools) $(math-tools)

	cat $(math-rtests) | build/bin/rtest | $(EMULATOR) build/bin/mathtest $(math-testflags)



check-math-ulp: $(math-tools)

	ULPFLAGS="$(math-ulpflags)" build/bin/runulp.sh $(EMULATOR)



check-math: check-math-test check-math-rtest check-math-ulp



install-math: \

 $(math-libs:build/lib/%=$(DESTDIR)$(libdir)/%) \

 $(math-includes:build/include/%=$(DESTDIR)$(includedir)/%)



clean-math:

	rm -f $(math-files)



.PHONY: all-math check-math-test check-math-rtest check-math-ulp check-math install-math clean-math

libc/AOR_v20.02/math/cosf.c

0 → 100644
+64 −0
Original line number Diff line number Diff line 
/*

 * Single-precision cos function.

 *

 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

 * See https://llvm.org/LICENSE.txt for license information.

 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

 */



#include <stdint.h>

#include <math.h>

#include "math_config.h"

#include "sincosf.h"



/* Fast cosf implementation.  Worst-case ULP is 0.5607, maximum relative

   error is 0.5303 * 2^-23.  A single-step range reduction is used for

   small values.  Large inputs have their range reduced using fast integer

   arithmetic.  */

float

cosf (float y)

{

  double x = y;

  double s;

  int n;

  const sincos_t *p = &__sincosf_table[0];



  if (abstop12 (y) < abstop12 (pio4))

    {

      double x2 = x * x;



      if (unlikely (abstop12 (y) < abstop12 (0x1p-12f)))

	return 1.0f;



      return sinf_poly (x, x2, p, 1);

    }

  else if (likely (abstop12 (y) < abstop12 (120.0f)))

    {

      x = reduce_fast (x, p, &n);



      /* Setup the signs for sin and cos.  */

      s = p->sign[n & 3];



      if (n & 2)

	p = &__sincosf_table[1];



      return sinf_poly (x * s, x * x, p, n ^ 1);

    }

  else if (abstop12 (y) < abstop12 (INFINITY))

    {

      uint32_t xi = asuint (y);

      int sign = xi >> 31;



      x = reduce_large (xi, &n);



      /* Setup signs for sin and cos - include original sign.  */

      s = p->sign[(n + sign) & 3];



      if ((n + sign) & 2)

	p = &__sincosf_table[1];



      return sinf_poly (x * s, x * x, p, n ^ 1);

    }

  else

    return __math_invalidf (y);

}

libc/AOR_v20.02/math/exp.c

0 → 100644
+177 −0
Original line number Diff line number Diff line 
/*

 * Double-precision e^x function.

 *

 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.

 * See https://llvm.org/LICENSE.txt for license information.

 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

 */



#include <float.h>

#include <math.h>

#include <stdint.h>

#include "math_config.h"



#define N (1 << EXP_TABLE_BITS)

#define InvLn2N __exp_data.invln2N

#define NegLn2hiN __exp_data.negln2hiN

#define NegLn2loN __exp_data.negln2loN

#define Shift __exp_data.shift

#define T __exp_data.tab

#define C2 __exp_data.poly[5 - EXP_POLY_ORDER]

#define C3 __exp_data.poly[6 - EXP_POLY_ORDER]

#define C4 __exp_data.poly[7 - EXP_POLY_ORDER]

#define C5 __exp_data.poly[8 - EXP_POLY_ORDER]

#define C6 __exp_data.poly[9 - EXP_POLY_ORDER]



/* Handle cases that may overflow or underflow when computing the result that

   is scale*(1+TMP) without intermediate rounding.  The bit representation of

   scale is in SBITS, however it has a computed exponent that may have

   overflown into the sign bit so that needs to be adjusted before using it as

   a double.  (int32_t)KI is the k used in the argument reduction and exponent

   adjustment of scale, positive k here means the result may overflow and

   negative k means the result may underflow.  */

static inline double

specialcase (double_t tmp, uint64_t sbits, uint64_t ki)

{

  double_t scale, y;



  if ((ki & 0x80000000) == 0)

    {

      /* k > 0, the exponent of scale might have overflowed by <= 460.  */

      sbits -= 1009ull << 52;

      scale = asdouble (sbits);

      y = 0x1p1009 * (scale + scale * tmp);

      return check_oflow (eval_as_double (y));

    }

  /* k < 0, need special care in the subnormal range.  */

  sbits += 1022ull << 52;

  scale = asdouble (sbits);

  y = scale + scale * tmp;

  if (y < 1.0)

    {

      /* Round y to the right precision before scaling it into the subnormal

	 range to avoid double rounding that can cause 0.5+E/2 ulp error where

	 E is the worst-case ulp error outside the subnormal range.  So this

	 is only useful if the goal is better than 1 ulp worst-case error.  */

      double_t hi, lo;

      lo = scale - y + scale * tmp;

      hi = 1.0 + y;

      lo = 1.0 - hi + y + lo;

      y = eval_as_double (hi + lo) - 1.0;

      /* Avoid -0.0 with downward rounding.  */

      if (WANT_ROUNDING && y == 0.0)

	y = 0.0;

      /* The underflow exception needs to be signaled explicitly.  */

      force_eval_double (opt_barrier_double (0x1p-1022) * 0x1p-1022);

    }

  y = 0x1p-1022 * y;

  return check_uflow (eval_as_double (y));

}



/* Top 12 bits of a double (sign and exponent bits).  */

static inline uint32_t

top12 (double x)

{

  return asuint64 (x) >> 52;

}



/* Computes exp(x+xtail) where |xtail| < 2^-8/N and |xtail| <= |x|.

   If hastail is 0 then xtail is assumed to be 0 too.  */

static inline double

exp_inline (double x, double xtail, int hastail)

{

  uint32_t abstop;

  uint64_t ki, idx, top, sbits;

  /* double_t for better performance on targets with FLT_EVAL_METHOD==2.  */

  double_t kd, z, r, r2, scale, tail, tmp;



  abstop = top12 (x) & 0x7ff;

  if (unlikely (abstop - top12 (0x1p-54) >= top12 (512.0) - top12 (0x1p-54)))

    {

      if (abstop - top12 (0x1p-54) >= 0x80000000)

	/* Avoid spurious underflow for tiny x.  */

	/* Note: 0 is common input.  */

	return WANT_ROUNDING ? 1.0 + x : 1.0;

      if (abstop >= top12 (1024.0))

	{

	  if (asuint64 (x) == asuint64 (-INFINITY))

	    return 0.0;

	  if (abstop >= top12 (INFINITY))

	    return 1.0 + x;

	  if (asuint64 (x) >> 63)

	    return __math_uflow (0);

	  else

	    return __math_oflow (0);

	}

      /* Large x is special cased below.  */

      abstop = 0;

    }



  /* exp(x) = 2^(k/N) * exp(r), with exp(r) in [2^(-1/2N),2^(1/2N)].  */

  /* x = ln2/N*k + r, with int k and r in [-ln2/2N, ln2/2N].  */

  z = InvLn2N * x;

#if TOINT_INTRINSICS

  kd = roundtoint (z);

  ki = converttoint (z);

#elif EXP_USE_TOINT_NARROW

  /* z - kd is in [-0.5-2^-16, 0.5] in all rounding modes.  */

  kd = eval_as_double (z + Shift);

  ki = asuint64 (kd) >> 16;

  kd = (double_t) (int32_t) ki;

#else

  /* z - kd is in [-1, 1] in non-nearest rounding modes.  */

  kd = eval_as_double (z + Shift);

  ki = asuint64 (kd);

  kd -= Shift;

#endif

  r = x + kd * NegLn2hiN + kd * NegLn2loN;

  /* The code assumes 2^-200 < |xtail| < 2^-8/N.  */

  if (hastail)

    r += xtail;

  /* 2^(k/N) ~= scale * (1 + tail).  */

  idx = 2 * (ki % N);

  top = ki << (52 - EXP_TABLE_BITS);

  tail = asdouble (T[idx]);

  /* This is only a valid scale when -1023*N < k < 1024*N.  */

  sbits = T[idx + 1] + top;

  /* exp(x) = 2^(k/N) * exp(r) ~= scale + scale * (tail + exp(r) - 1).  */

  /* Evaluation is optimized assuming superscalar pipelined execution.  */

  r2 = r * r;

  /* Without fma the worst case error is 0.25/N ulp larger.  */

  /* Worst case error is less than 0.5+1.11/N+(abs poly error * 2^53) ulp.  */

#if EXP_POLY_ORDER == 4

  tmp = tail + r + r2 * C2 + r * r2 * (C3 + r * C4);

#elif EXP_POLY_ORDER == 5

  tmp = tail + r + r2 * (C2 + r * C3) + r2 * r2 * (C4 + r * C5);

#elif EXP_POLY_ORDER == 6

  tmp = tail + r + r2 * (0.5 + r * C3) + r2 * r2 * (C4 + r * C5 + r2 * C6);

#endif

  if (unlikely (abstop == 0))

    return specialcase (tmp, sbits, ki);

  scale = asdouble (sbits);

  /* Note: tmp == 0 or |tmp| > 2^-200 and scale > 2^-739, so there

     is no spurious underflow here even without fma.  */

  return eval_as_double (scale + scale * tmp);

}



double

exp (double x)

{

  return exp_inline (x, 0, 0);

}



/* May be useful for implementing pow where more than double

   precision input is needed.  */

double

__exp_dd (double x, double xtail)

{

  return exp_inline (x, xtail, 1);

}

#if USE_GLIBC_ABI

strong_alias (exp, __exp_finite)

hidden_alias (exp, __ieee754_exp)

hidden_alias (__exp_dd, __exp1)

# if LDBL_MANT_DIG == 53

long double expl (long double x) { return exp (x); }

# endif

#endif