ossl: port to arm64

Enable in-kernel acceleration of SHA1 and SHA2 operations on arm64 by adding
support for the ossl(4) crypto driver. This uses OpenSSL's assembly routines
under the hood, which will detect and use SHA intrinsics if they are
supported by the CPU.

Reviewed by:	jhb
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D27390

share/man/man4/ossl.4

@@ -26,12 +26,12 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 19, 2020
+.Dd December 4, 2020
 .Dt OSSL 4
 .Os
 .Sh NAME
 .Nm ossl
-.Nd "driver using OpenSSL assembly routines on x86 CPUs"
+.Nd "driver using OpenSSL assembly routines"
 .Sh SYNOPSIS
 To compile this driver into the kernel,
 place the following lines in your
@@ -61,6 +61,8 @@ architectures:
 .Pp
 .Bl -bullet -compact
 .It
+arm64
+.It
 amd64
 .It
 i386

sys/conf/files.arm64

@@ -246,6 +246,13 @@ armv8_crypto_wrap.o		optional	armv8crypto		\
 	no-implicit-rule						\
 	clean		"armv8_crypto_wrap.o"
 crypto/des/des_enc.c		optional	netsmb
+crypto/openssl/ossl_aarch64.c	optional	ossl
+crypto/openssl/aarch64/sha1-armv8.S	optional ossl		\
+	compile-with	"${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}"
+crypto/openssl/aarch64/sha256-armv8.S	optional ossl		\
+	compile-with	"${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}"
+crypto/openssl/aarch64/sha512-armv8.S	optional ossl		\
+	compile-with	"${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}"
 dev/acpica/acpi_bus_if.m	optional	acpi
 dev/acpica/acpi_if.m		optional	acpi
 dev/acpica/acpi_pci_link.c	optional	acpi pci

sys/crypto/openssl/aarch64/arm_arch.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2011-2018 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#ifndef OSSL_CRYPTO_ARM_ARCH_H
+# define OSSL_CRYPTO_ARM_ARCH_H
+
+# if !defined(__ARM_ARCH__)
+#  if defined(__CC_ARM)
+#   define __ARM_ARCH__ __TARGET_ARCH_ARM
+#   if defined(__BIG_ENDIAN)
+#    define __ARMEB__
+#   else
+#    define __ARMEL__
+#   endif
+#  elif defined(__GNUC__)
+#   if   defined(__aarch64__)
+#    define __ARM_ARCH__ 8
+#    if __BYTE_ORDER__==__ORDER_BIG_ENDIAN__
+#     define __ARMEB__
+#    else
+#     define __ARMEL__
+#    endif
+  /*
+   * Why doesn't gcc define __ARM_ARCH__? Instead it defines
+   * bunch of below macros. See all_architectures[] table in
+   * gcc/config/arm/arm.c. On a side note it defines
+   * __ARMEL__/__ARMEB__ for little-/big-endian.
+   */
+#   elif defined(__ARM_ARCH)
+#    define __ARM_ARCH__ __ARM_ARCH
+#   elif defined(__ARM_ARCH_8A__)
+#    define __ARM_ARCH__ 8
+#   elif defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)     || \
+        defined(__ARM_ARCH_7R__)|| defined(__ARM_ARCH_7M__)     || \
+        defined(__ARM_ARCH_7EM__)
+#    define __ARM_ARCH__ 7
+#   elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__)     || \
+        defined(__ARM_ARCH_6K__)|| defined(__ARM_ARCH_6M__)     || \
+        defined(__ARM_ARCH_6Z__)|| defined(__ARM_ARCH_6ZK__)    || \
+        defined(__ARM_ARCH_6T2__)
+#    define __ARM_ARCH__ 6
+#   elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__)     || \
+        defined(__ARM_ARCH_5E__)|| defined(__ARM_ARCH_5TE__)    || \
+        defined(__ARM_ARCH_5TEJ__)
+#    define __ARM_ARCH__ 5
+#   elif defined(__ARM_ARCH_4__) || defined(__ARM_ARCH_4T__)
+#    define __ARM_ARCH__ 4
+#   else
+#    error "unsupported ARM architecture"
+#   endif
+#  endif
+# endif
+
+# if !defined(__ARM_MAX_ARCH__)
+#  define __ARM_MAX_ARCH__ __ARM_ARCH__
+# endif
+
+# if __ARM_MAX_ARCH__<__ARM_ARCH__
+#  error "__ARM_MAX_ARCH__ can't be less than __ARM_ARCH__"
+# elif __ARM_MAX_ARCH__!=__ARM_ARCH__
+#  if __ARM_ARCH__<7 && __ARM_MAX_ARCH__>=7 && defined(__ARMEB__)
+#   error "can't build universal big-endian binary"
+#  endif
+# endif
+
+# ifndef __ASSEMBLER__
+extern unsigned int OPENSSL_armcap_P;
+# endif
+
+# define ARMV7_NEON      (1<<0)
+# define ARMV7_TICK      (1<<1)
+# define ARMV8_AES       (1<<2)
+# define ARMV8_SHA1      (1<<3)
+# define ARMV8_SHA256    (1<<4)
+# define ARMV8_PMULL     (1<<5)
+# define ARMV8_SHA512    (1<<6)
+
+#endif

sys/crypto/openssl/ossl_aarch64.c

@@ -0,0 +1,62 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2020 The FreeBSD Foundation
+ *
+ * This software was developed by Mitchell Horne <mhorne@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/types.h>
+
+#include <machine/elf.h>
+#include <machine/md_var.h>
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/aarch64/arm_arch.h>
+
+/*
+ * Feature bits defined in arm_arch.h
+ */
+unsigned int OPENSSL_armcap_P;
+
+void
+ossl_cpuid(void)
+{
+	/* SHA features */
+	if ((elf_hwcap & HWCAP_SHA1) != 0)
+		OPENSSL_armcap_P |= ARMV8_SHA1;
+	if ((elf_hwcap & HWCAP_SHA2) != 0)
+		OPENSSL_armcap_P |= ARMV8_SHA256;
+	if ((elf_hwcap & HWCAP_SHA512) != 0)
+		OPENSSL_armcap_P |= ARMV8_SHA512;
+
+	/* AES features */
+	if ((elf_hwcap & HWCAP_AES) != 0)
+		OPENSSL_armcap_P |= ARMV8_AES;
+	if ((elf_hwcap & HWCAP_PMULL) != 0)
+		OPENSSL_armcap_P |= ARMV8_PMULL;
+}

sys/modules/Makefile

@@ -518,6 +518,7 @@ _mthca=		mthca
 _mlx4ib=	mlx4ib
 _mlx5ib=	mlx5ib
 .endif
+_ossl=		ossl
 _vmware=	vmware
 .endif
 
@@ -634,7 +635,6 @@ _nctgpio=	nctgpio
 _ndis=		ndis
 _ntb=		ntb
 _ocs_fc=	ocs_fc
-_ossl=		ossl
 _pccard=	pccard
 _qat=		qat
 _qatfw=		qatfw

sys/modules/ossl/Makefile

@@ -13,6 +13,12 @@ SRCS=	bus_if.h \
 	ossl_sha512.c \
 	${SRCS.${MACHINE_CPUARCH}}
 
+SRCS.aarch64= \
+	sha1-armv8.S \
+	sha256-armv8.S \
+	sha512-armv8.S \
+	ossl_aarch64.c
+
 SRCS.amd64= \
 	sha1-x86_64.S \
 	sha256-x86_64.S \
@@ -25,4 +31,10 @@ SRCS.i386= \
 	sha512-586.S \
 	ossl_x86.c
 
+# For arm64, we are forced to rewrite the compiler invocation for the assembly
+# files, to remove -mgeneral-regs-only.
+${SRCS.aarch64:M*.S:S/S/o/}: ${.TARGET:R}.S
+	${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC}
+	${CTFCONVERT_CMD}
+
 .include <bsd.kmod.mk>

tests/sys/opencrypto/runtests.sh

@@ -65,7 +65,7 @@ cpu_module=
 
 case ${cpu_type} in
 aarch64)
-	cpu_module=nexus/armv8crypto
+	cpu_module="nexus/armv8crypto nexus/ossl"
 	;;
 amd64|i386)
 	cpu_module="nexus/aesni nexus/ossl"