mirror/freebsd
1
0
Fork 0
mirror of https://git.FreeBSD.org/src.git synced 2024-10-19 02:29:40 +00:00
Code Issues Releases Activity

Merge ^/head r308227 through r308490.

Browse Source
This commit is contained in:
Dimitry Andric 2016-11-10 22:12:19 +00:00
commit 2828dafcf3
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/projects/clang390-import/; revision=308491
392 changed files with 27673 additions and 11368 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Makefile.inc1
View File

@ -1953,6 +1953,7 @@ native-xtools: .PHONY
usr.bin/gzip \
usr.bin/id \
usr.bin/lex \
usr.bin/limits \
usr.bin/lorder \
usr.bin/mktemp \
usr.bin/mt \
@ -2000,7 +2001,7 @@ libraries: .MAKE .PHONY
#
# static libgcc.a prerequisite for shared libc
#
_prereq_libs= gnu/lib/libgcc lib/libcompiler_rt
_prereq_libs= lib/libcompiler_rt
.if ${MK_SSP} != "no"
_prereq_libs+= gnu/lib/libssp/libssp_nonshared
.endif
@ -2012,7 +2013,6 @@ _prereq_libs+= gnu/lib/libssp/libssp_nonshared
#
_startup_libs= gnu/lib/csu
_startup_libs+= lib/csu
_startup_libs+= gnu/lib/libgcc
_startup_libs+= lib/libcompiler_rt
_startup_libs+= lib/libc
_startup_libs+= lib/libc_nonshared
@ -2020,11 +2020,27 @@ _startup_libs+= lib/libc_nonshared
_startup_libs+= lib/libcxxrt
.endif
.if ${MK_LLVM_LIBUNWIND} != "no"
_prereq_libs+= lib/libgcc_eh lib/libgcc_s
_startup_libs+= lib/libgcc_eh lib/libgcc_s
lib/libgcc_s__L: lib/libc__L
lib/libgcc_s__L: lib/libc_nonshared__L
.if ${MK_LIBCPLUSPLUS} != "no"
lib/libcxxrt__L: lib/libgcc_s__L
.endif
.else # MK_LLVM_LIBUNWIND == no
_prereq_libs+= gnu/lib/libgcc
_startup_libs+= gnu/lib/libgcc
gnu/lib/libgcc__L: lib/libc__L
gnu/lib/libgcc__L: lib/libc_nonshared__L
.if ${MK_LIBCPLUSPLUS} != "no"
lib/libcxxrt__L: gnu/lib/libgcc__L
.endif
.endif
_prebuild_libs= ${_kerberos5_lib_libasn1} \
${_kerberos5_lib_libhdb} \

5
bin/echo/echo.c
View File

@ -45,6 +45,8 @@ __FBSDID("$FreeBSD$");
#include <sys/uio.h>
#include <assert.h>
#include <capsicum_helpers.h>
#include <err.h>
#include <errno.h>
#include <limits.h>
#include <stdlib.h>
@ -78,6 +80,9 @@ main(int argc, char *argv[])
char newline[] = "\n";
char *progname = argv[0];
if (caph_limit_stdio() < 0 || (cap_enter() < 0 && errno != ENOSYS))
err(1, "capsicum");
/* This utility may NOT do getopt(3) option parsing. */
if (*++argv && !strcmp(*argv, "-n")) {
++argv;

4
bin/freebsd-version/freebsd-version.sh.in
View File

@ -36,7 +36,7 @@ USERLAND_VERSION="@@REVISION@@-@@BRANCH@@"
: ${LOADER_CONF_FILES:=$LOADER_DIR/defaults/loader.conf $LOADER_DIR/loader.conf $LOADER_DIR/loader.conf.local}
LOADER_RE1='^\([A-Z_a-z][0-9A-Z_a-z]*=[-./0-9A-Z_a-z]\{1,\}\).*$'
LOADER_RE2='^\([A-Z_a-z][0-9A-Z_a-z]*="[-./0-9A-Z_a-z]\{1,\}"\).*$'
KERNEL_RE='^@(#)@@TYPE@@ \([-.0-9A-Za-z]\{1,\}\) .*$'
KERNEL_RE='^@@TYPE@@ \([-.0-9A-Za-z]\{1,\}\) .*$'
progname=$(basename $0)
@ -67,7 +67,7 @@ kernel_version() {
if [ ! -f "$kernfile" -o ! -r "$kernfile" ] ; then
error "unable to locate kernel"
fi
strings "$kernfile" | sed -n "s/$KERNEL_RE/\\1/p"
what -qs "$kernfile" | sed -n "s/$KERNEL_RE/\\1/p"
}
#

7
bin/hostname/hostname.1
View File

@ -29,7 +29,7 @@
.\" @(#)hostname.1 8.2 (Berkeley) 4/28/95
.\" $FreeBSD$
.\"
.Dd December 7, 2006
.Dd November 10, 2016
.Dt HOSTNAME 1
.Os
.Sh NAME
@ -37,7 +37,8 @@
.Nd set or print name of current host system
.Sh SYNOPSIS
.Nm
.Op Fl fs
.Op Fl f
.Op Fl s | d
.Op Ar name-of-host
.Sh DESCRIPTION
The
@ -62,6 +63,8 @@ This is the default behavior.
.It Fl s
Trim off any domain information from the printed
name.
.It Fl d
Only print domain information.
.El
.Sh SEE ALSO
.Xr gethostname 3 ,

16
bin/hostname/hostname.c
View File

@ -54,11 +54,12 @@ static void usage(void) __dead2;
int
main(int argc, char *argv[])
{
int ch, sflag;
int ch, sflag, dflag;
char *p, hostname[MAXHOSTNAMELEN];
sflag = 0;
while ((ch = getopt(argc, argv, "fs")) != -1)
dflag = 0;
while ((ch = getopt(argc, argv, "fsd")) != -1)
switch (ch) {
case 'f':
/*
@ -70,6 +71,9 @@ main(int argc, char *argv[])
case 's':
sflag = 1;
break;
case 'd':
dflag = 1;
break;
case '?':
default:
usage();
@ -77,7 +81,7 @@ main(int argc, char *argv[])
argc -= optind;
argv += optind;
if (argc > 1)
if (argc > 1 || (sflag && dflag))
usage();
if (*argv) {
@ -90,6 +94,10 @@ main(int argc, char *argv[])
p = strchr(hostname, '.');
if (p != NULL)
*p = '\0';
} else if (dflag) {
p = strchr(hostname, '.');
if (p != NULL)
strcpy(hostname, ++p);
}
(void)printf("%s\n", hostname);
}
@ -100,6 +108,6 @@ static void
usage(void)
{
(void)fprintf(stderr, "usage: hostname [-fs] [name-of-host]\n");
(void)fprintf(stderr, "usage: hostname [-f] [-s | -d] [name-of-host]\n");
exit(1);
}

1
bin/sh/tests/execution/Makefile
View File

@ -37,6 +37,7 @@ ${PACKAGE}FILES+= redir4.0
${PACKAGE}FILES+= redir5.0
${PACKAGE}FILES+= redir6.0
${PACKAGE}FILES+= redir7.0
${PACKAGE}FILES+= set-C1.0
${PACKAGE}FILES+= set-n1.0
${PACKAGE}FILES+= set-n2.0
${PACKAGE}FILES+= set-n3.0

12
bin/sh/tests/execution/set-C1.0 Normal file
View File

@ -0,0 +1,12 @@
# $FreeBSD$
T=$(mktemp -d "${TMPDIR:-/tmp}/sh-test.XXXXXXXX") || exit
trap 'rm -rf "$T"' 0
set -C
echo . >"$T/a" &&
[ -s "$T/a" ] &&
{ ! true >"$T/a"; } 2>/dev/null &&
[ -s "$T/a" ] &&
ln -s /dev/null "$T/b" &&
true >"$T/b"

4
bin/sleep/sleep.c
View File

@ -41,6 +41,7 @@ static char sccsid[] = "@(#)sleep.c 8.3 (Berkeley) 4/2/94";
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <capsicum_helpers.h>
#include <ctype.h>
#include <err.h>
#include <errno.h>
@ -69,6 +70,9 @@ main(int argc, char *argv[])
time_t original;
char buf[2];
if (caph_limit_stdio() < 0 || (cap_enter() < 0 && errno != ENOSYS))
err(1, "capsicum");
if (argc != 2)
usage();

42
cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.c
View File

@ -596,8 +596,9 @@ recv_read(int fd, void *buf, int ilen)
}
static int
lzc_receive_impl(const char *snapname, nvlist_t *props, const char *origin,
boolean_t force, boolean_t resumable, int fd)
recv_impl(const char *snapname, nvlist_t *props, const char *origin,
boolean_t force, boolean_t resumable, int fd,
const dmu_replay_record_t *begin_record)
{
/*
* The receive ioctl is still legacy, so we need to construct our own
@ -642,9 +643,14 @@ lzc_receive_impl(const char *snapname, nvlist_t *props, const char *origin,
(void) strlcpy(zc.zc_string, origin, sizeof (zc.zc_string));
/* zc_begin_record is non-byteswapped BEGIN record */
error = recv_read(fd, &zc.zc_begin_record, sizeof (zc.zc_begin_record));
if (error != 0)
goto out;
if (begin_record == NULL) {
error = recv_read(fd, &zc.zc_begin_record,
sizeof (zc.zc_begin_record));
if (error != 0)
goto out;
} else {
zc.zc_begin_record = *begin_record;
}
/* zc_cookie is fd to read from */
zc.zc_cookie = fd;
@ -685,7 +691,7 @@ int
lzc_receive(const char *snapname, nvlist_t *props, const char *origin,
boolean_t force, int fd)
{
return (lzc_receive_impl(snapname, props, origin, force, B_FALSE, fd));
return (recv_impl(snapname, props, origin, force, B_FALSE, fd, NULL));
}
/*
@ -698,7 +704,29 @@ int
lzc_receive_resumable(const char *snapname, nvlist_t *props, const char *origin,
boolean_t force, int fd)
{
return (lzc_receive_impl(snapname, props, origin, force, B_TRUE, fd));
return (recv_impl(snapname, props, origin, force, B_TRUE, fd, NULL));
}
/*
* Like lzc_receive, but allows the caller to read the begin record and then to
* pass it in. That could be useful if the caller wants to derive, for example,
* the snapname or the origin parameters based on the information contained in
* the begin record.
* The begin record must be in its original form as read from the stream,
* in other words, it should not be byteswapped.
*
* The 'resumable' parameter allows to obtain the same behavior as with
* lzc_receive_resumable.
*/
int
lzc_receive_with_header(const char *snapname, nvlist_t *props,
const char *origin, boolean_t force, boolean_t resumable, int fd,
const dmu_replay_record_t *begin_record)
{
if (begin_record == NULL)
return (EINVAL);
return (recv_impl(snapname, props, origin, force, resumable, fd,
begin_record));
}
/*

7
cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.h
View File

@ -68,10 +68,15 @@ enum lzc_send_flags {
int lzc_send(const char *, const char *, int, enum lzc_send_flags);
int lzc_send_resume(const char *, const char *, int,
enum lzc_send_flags, uint64_t, uint64_t);
int lzc_send_space(const char *, const char *, uint64_t *);
struct dmu_replay_record;
int lzc_receive(const char *, nvlist_t *, const char *, boolean_t, int);
int lzc_receive_resumable(const char *, nvlist_t *, const char *,
boolean_t, int);
int lzc_send_space(const char *, const char *, uint64_t *);
int lzc_receive_with_header(const char *, nvlist_t *, const char *, boolean_t,
boolean_t, int, const struct dmu_replay_record *);
boolean_t lzc_exists(const char *);

5
contrib/bsnmp/lib/snmpagent.c
View File

@ -171,7 +171,10 @@ snmp_pdu_create_response(const struct snmp_pdu *pdu, struct snmp_pdu *resp)
memset(resp, 0, sizeof(*resp));
strcpy(resp->community, pdu->community);
resp->version = pdu->version;
resp->type = SNMP_PDU_RESPONSE;
if (pdu->flags & SNMP_MSG_AUTODISCOVER)
resp->type = SNMP_PDU_REPORT; /* RFC 3414.4 */
else
resp->type = SNMP_PDU_RESPONSE;
resp->request_id = pdu->request_id;
resp->version = pdu->version;

2
contrib/elftoolchain/cxxfilt/cxxfilt.c
View File

@ -188,6 +188,8 @@ main(int argc, char **argv)
if (c == EOF)
break;
putchar(c);
if (c == '\n')
fflush(stdout);
} else {
if ((size_t) p >= sizeof(buf) - 1)
warnx("buffer overflowed");

2
contrib/elftoolchain/libelftc/elftc_bfd_find_target.3
View File

@ -82,6 +82,7 @@ Known descriptor names and their properties include:
.It Li elf32-shbig-linux Ta ELF Ta MSB Ta 32
.It Li elf32-shl-linux Ta ELF Ta LSB Ta 32
.It Li elf32-sparc Ta ELF Ta MSB Ta 32
.It Li elf32-tradbigmips Ta ELF Ta MSB Ta 32
.It Li elf64-alpha Ta ELF Ta LSB Ta 64
.It Li elf64-alpha-freebsd Ta ELF Ta LSB Ta 64
.It Li elf64-big Ta ELF Ta MSB Ta 64
@ -101,6 +102,7 @@ Known descriptor names and their properties include:
.It Li elf64-sh64-linux Ta ELF Ta LSB Ta 64
.It Li elf64-sparc Ta ELF Ta MSB Ta 64
.It Li elf64-sparc-freebsd Ta ELF Ta MSB Ta 64
.It Li elf64-tradbigmips Ta ELF Ta MSB Ta 64
.It Li elf64-x86-64 Ta ELF Ta LSB Ta 64
.It Li elf64-x86-64-freebsd Ta ELF Ta LSB Ta 64
.It Li ihex Ta IHEX Ta - Ta -

16
contrib/elftoolchain/libelftc/libelftc_bfdtarget.c
View File

@ -194,6 +194,14 @@ struct _Elftc_Bfd_Target _libelftc_targets[] = {
.bt_machine = EM_SPARC,
},
{
.bt_name = "elf32-tradbigmips",
.bt_type = ETF_ELF,
.bt_byteorder = ELFDATA2MSB,
.bt_elfclass = ELFCLASS32,
.bt_machine = EM_MIPS,
},
{
.bt_name = "elf64-alpha",
.bt_type = ETF_ELF,
@ -350,6 +358,14 @@ struct _Elftc_Bfd_Target _libelftc_targets[] = {
.bt_osabi = ELFOSABI_FREEBSD
},
{
.bt_name = "elf64-tradbigmips",
.bt_type = ETF_ELF,
.bt_byteorder = ELFDATA2MSB,
.bt_elfclass = ELFCLASS64,
.bt_machine = EM_MIPS,
},
{
.bt_name = "elf64-x86-64",
.bt_type = ETF_ELF,

29
contrib/file/ChangeLog
View File

@ -1,3 +1,32 @@
2016-10-25 10:40 Christos Zoulas <christos@zoulas.com>
* release 5.28
2016-10-24 11:20 Christos Zoulas <christos@zoulas.com>
* der getlength overflow (Jonas Wagner)
* multiple magic file load failure (Christoph Biedl)
2016-10-17 11:26 Christos Zoulas <christos@zoulas.com>
* CDF parsing improvements (Guy Helmer)
2016-07-20 7:26 Christos Zoulas <christos@zoulas.com>
* Add support for signed indirect offsets
2016-07-18 7:41 Christos Zoulas <christos@zoulas.com>
* cat /dev/null | file - should print empty (Christoph Biedl)
2016-07-05 15:20 Christos Zoulas <christos@zoulas.com>
* Bump string size from 64 to 96.
2016-06-13 20:20 Christos Zoulas <christos@zoulas.com>
* PR/556: Fix separators on annotations.
2016-06-13 19:40 Christos Zoulas <christos@zoulas.com>
* release 5.28

20
contrib/file/TODO
View File

@ -6,7 +6,6 @@ file, not here. More speculative material can live here.
listed in the BUGS section of the man page had been fixed!)
---
It would be nice to simplify file considerably. For example,
reimplement the apprentice and non-pattern magic methods in Python,
and compile the magic patterns to a giant regex (or something similar;
@ -15,8 +14,23 @@ small amount of C is needed (because fast execution is typically only
required for soft magic, not the more detailed information given by
hard-wired routines). In this regard, note that hplip, which is
BSD-licensed, has a magic reimplementation in Python.
---
Read the kerberos magic entry for more ideas.
---
Write a string merger to make magic entry sizes dynamic.
Strings will be converted to offsets from the string table.
---
Programming language support, we can introduce the concept of a group
of rules where n rules need to match before the rule is positive. This
could require structural changes to the matching code :-(
0 group 2 # require 2 matches
# rule 1
>0 ....
...
# rule 2
>0 ....
...
christos

376
contrib/file/config.guess vendored
View File

@ -1,14 +1,12 @@
#! /bin/sh
# Attempt to guess a canonical system name.
# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
# 2011, 2012 Free Software Foundation, Inc.
# Copyright 1992-2015 Free Software Foundation, Inc.
timestamp='2012-02-10'
timestamp='2015-03-04'
# This file is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
@ -22,19 +20,17 @@ timestamp='2012-02-10'
# As a special exception to the GNU General Public License, if you
# distribute this file as part of a program that contains a
# configuration script generated by Autoconf, you may include it under
# the same distribution terms that you use for the rest of that program.
# Originally written by Per Bothner. Please send patches (context
# diff format) to <config-patches@gnu.org> and include a ChangeLog
# entry.
# the same distribution terms that you use for the rest of that
# program. This Exception is an additional permission under section 7
# of the GNU General Public License, version 3 ("GPLv3").
#
# This script attempts to guess a canonical system name similar to
# config.sub. If it succeeds, it prints the system name on stdout, and
# exits with 0. Otherwise, it exits with 1.
# Originally written by Per Bothner; maintained since 2000 by Ben Elliston.
#
# You can get the latest version of this script from:
# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
#
# Please send patches to <config-patches@gnu.org>.
me=`echo "$0" | sed -e 's,.*/,,'`
@ -54,9 +50,7 @@ version="\
GNU config.guess ($timestamp)
Originally written by Per Bothner.
Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
Free Software Foundation, Inc.
Copyright 1992-2015 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@ -138,6 +132,27 @@ UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown
UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown
UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown
case "${UNAME_SYSTEM}" in
Linux|GNU|GNU/*)
# If the system lacks a compiler, then just pick glibc.
# We could probably try harder.
LIBC=gnu
eval $set_cc_for_build
cat <<-EOF > $dummy.c
#include <features.h>
#if defined(__UCLIBC__)
LIBC=uclibc
#elif defined(__dietlibc__)
LIBC=dietlibc
#else
LIBC=gnu
#endif
EOF
eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`
;;
esac
# Note: order is significant - the case branches are not exclusive.
case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
@ -153,20 +168,27 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
# Note: NetBSD doesn't particularly care about the vendor
# portion of the name. We always set it to "unknown".
sysctl="sysctl -n hw.machine_arch"
UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \
/usr/sbin/$sysctl 2>/dev/null || echo unknown)`
UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \
/sbin/$sysctl 2>/dev/null || \
/usr/sbin/$sysctl 2>/dev/null || \
echo unknown)`
case "${UNAME_MACHINE_ARCH}" in
armeb) machine=armeb-unknown ;;
arm*) machine=arm-unknown ;;
sh3el) machine=shl-unknown ;;
sh3eb) machine=sh-unknown ;;
sh5el) machine=sh5le-unknown ;;
earmv*)
arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'`
endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'`
machine=${arch}${endian}-unknown
;;
*) machine=${UNAME_MACHINE_ARCH}-unknown ;;
esac
# The Operating System including object format, if it has switched
# to ELF recently, or will in the future.
case "${UNAME_MACHINE_ARCH}" in
arm*|i386|m68k|ns32k|sh3*|sparc|vax)
arm*|earm*|i386|m68k|ns32k|sh3*|sparc|vax)
eval $set_cc_for_build
if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \
| grep -q __ELF__
@ -182,6 +204,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
os=netbsd
;;
esac
# Determine ABI tags.
case "${UNAME_MACHINE_ARCH}" in
earm*)
expr='s/^earmv[0-9]/-eabi/;s/eb$//'
abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"`
;;
esac
# The OS release
# Debian GNU/NetBSD machines have a different userland, and
# thus, need a distinct triplet. However, they do not need
@ -198,7 +227,11 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
# Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM:
# contains redundant information, the shorter form:
# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used.
echo "${machine}-${os}${release}"
echo "${machine}-${os}${release}${abi}"
exit ;;
*:Bitrig:*:*)
UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'`
echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE}
exit ;;
*:OpenBSD:*:*)
UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'`
@ -302,7 +335,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*)
echo arm-acorn-riscix${UNAME_RELEASE}
exit ;;
arm:riscos:*:*|arm:RISCOS:*:*)
arm*:riscos:*:*|arm*:RISCOS:*:*)
echo arm-unknown-riscos
exit ;;
SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*)
@ -560,8 +593,9 @@ EOF
else
IBM_ARCH=powerpc
fi
if [ -x /usr/bin/oslevel ] ; then
IBM_REV=`/usr/bin/oslevel`
if [ -x /usr/bin/lslpp ] ; then
IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc |
awk -F: '{ print $3 }' | sed s/[0-9]*$/0/`
else
IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
fi
@ -801,10 +835,13 @@ EOF
i*:CYGWIN*:*)
echo ${UNAME_MACHINE}-pc-cygwin
exit ;;
*:MINGW64*:*)
echo ${UNAME_MACHINE}-pc-mingw64
exit ;;
*:MINGW*:*)
echo ${UNAME_MACHINE}-pc-mingw32
exit ;;
i*:MSYS*:*)
*:MSYS*:*)
echo ${UNAME_MACHINE}-pc-msys
exit ;;
i*:windows32*:*)
@ -852,21 +889,21 @@ EOF
exit ;;
*:GNU:*:*)
# the GNU system
echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-${LIBC}`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'`
exit ;;
*:GNU/*:*:*)
# other systems with GNU libc and userland
echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu
echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC}
exit ;;
i*86:Minix:*:*)
echo ${UNAME_MACHINE}-pc-minix
exit ;;
aarch64:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
aarch64_be:Linux:*:*)
UNAME_MACHINE=aarch64_be
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
alpha:Linux:*:*)
case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
@ -879,59 +916,57 @@ EOF
EV68*) UNAME_MACHINE=alphaev68 ;;
esac
objdump --private-headers /bin/sh | grep -q ld.so.1
if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi
echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC}
if test "$?" = 0 ; then LIBC="gnulibc1" ; fi
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
arc:Linux:*:* | arceb:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
arm*:Linux:*:*)
eval $set_cc_for_build
if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \
| grep -q __ARM_EABI__
then
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
else
if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \
| grep -q __ARM_PCS_VFP
then
echo ${UNAME_MACHINE}-unknown-linux-gnueabi
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi
else
echo ${UNAME_MACHINE}-unknown-linux-gnueabihf
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabihf
fi
fi
exit ;;
avr32*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
cris:Linux:*:*)
echo ${UNAME_MACHINE}-axis-linux-gnu
echo ${UNAME_MACHINE}-axis-linux-${LIBC}
exit ;;
crisv32:Linux:*:*)
echo ${UNAME_MACHINE}-axis-linux-gnu
echo ${UNAME_MACHINE}-axis-linux-${LIBC}
exit ;;
e2k:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
frv:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
hexagon:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
i*86:Linux:*:*)
LIBC=gnu
eval $set_cc_for_build
sed 's/^ //' << EOF >$dummy.c
#ifdef __dietlibc__
LIBC=dietlibc
#endif
EOF
eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'`
echo "${UNAME_MACHINE}-pc-linux-${LIBC}"
echo ${UNAME_MACHINE}-pc-linux-${LIBC}
exit ;;
ia64:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
m32r*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
m68*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
mips:Linux:*:* | mips64:Linux:*:*)
eval $set_cc_for_build
@ -950,54 +985,63 @@ EOF
#endif
EOF
eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'`
test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; }
test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; }
;;
or32:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
openrisc*:Linux:*:*)
echo or1k-unknown-linux-${LIBC}
exit ;;
or32:Linux:*:* | or1k*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
padre:Linux:*:*)
echo sparc-unknown-linux-gnu
echo sparc-unknown-linux-${LIBC}
exit ;;
parisc64:Linux:*:* | hppa64:Linux:*:*)
echo hppa64-unknown-linux-gnu
echo hppa64-unknown-linux-${LIBC}
exit ;;
parisc:Linux:*:* | hppa:Linux:*:*)
# Look for CPU level
case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in
PA7*) echo hppa1.1-unknown-linux-gnu ;;
PA8*) echo hppa2.0-unknown-linux-gnu ;;
*) echo hppa-unknown-linux-gnu ;;
PA7*) echo hppa1.1-unknown-linux-${LIBC} ;;
PA8*) echo hppa2.0-unknown-linux-${LIBC} ;;
*) echo hppa-unknown-linux-${LIBC} ;;
esac
exit ;;
ppc64:Linux:*:*)
echo powerpc64-unknown-linux-gnu
echo powerpc64-unknown-linux-${LIBC}
exit ;;
ppc:Linux:*:*)
echo powerpc-unknown-linux-gnu
echo powerpc-unknown-linux-${LIBC}
exit ;;
ppc64le:Linux:*:*)
echo powerpc64le-unknown-linux-${LIBC}
exit ;;
ppcle:Linux:*:*)
echo powerpcle-unknown-linux-${LIBC}
exit ;;
s390:Linux:*:* | s390x:Linux:*:*)
echo ${UNAME_MACHINE}-ibm-linux
echo ${UNAME_MACHINE}-ibm-linux-${LIBC}
exit ;;
sh64*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
sh*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
sparc:Linux:*:* | sparc64:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
tile*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
vax:Linux:*:*)
echo ${UNAME_MACHINE}-dec-linux-gnu
echo ${UNAME_MACHINE}-dec-linux-${LIBC}
exit ;;
x86_64:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
xtensa*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-gnu
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
i*86:DYNIX/ptx:4*:*)
# ptx 4.0 does uname -s correctly, with DYNIX/ptx in there.
@ -1201,6 +1245,9 @@ EOF
BePC:Haiku:*:*) # Haiku running on Intel PC compatible.
echo i586-pc-haiku
exit ;;
x86_64:Haiku:*:*)
echo x86_64-unknown-haiku
exit ;;
SX-4:SUPER-UX:*:*)
echo sx4-nec-superux${UNAME_RELEASE}
exit ;;
@ -1227,19 +1274,31 @@ EOF
exit ;;
*:Darwin:*:*)
UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown
case $UNAME_PROCESSOR in
i386)
eval $set_cc_for_build
if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
grep IS_64BIT_ARCH >/dev/null
then
UNAME_PROCESSOR="x86_64"
fi
fi ;;
unknown) UNAME_PROCESSOR=powerpc ;;
esac
eval $set_cc_for_build
if test "$UNAME_PROCESSOR" = unknown ; then
UNAME_PROCESSOR=powerpc
fi
if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then
if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then
if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \
(CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \
grep IS_64BIT_ARCH >/dev/null
then
case $UNAME_PROCESSOR in
i386) UNAME_PROCESSOR=x86_64 ;;
powerpc) UNAME_PROCESSOR=powerpc64 ;;
esac
fi
fi
elif test "$UNAME_PROCESSOR" = i386 ; then
# Avoid executing cc on OS X 10.9, as it ships with a stub
# that puts up a graphical alert prompting to install
# developer tools. Any system running Mac OS X 10.7 or
# later (Darwin 11 and later) is required to have a 64-bit
# processor. This is not true of the ARM version of Darwin
# that Apple uses in portable devices.
UNAME_PROCESSOR=x86_64
fi
echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE}
exit ;;
*:procnto*:*:* | *:QNX:[0123456789]*:*)
@ -1256,7 +1315,7 @@ EOF
NEO-?:NONSTOP_KERNEL:*:*)
echo neo-tandem-nsk${UNAME_RELEASE}
exit ;;
NSE-?:NONSTOP_KERNEL:*:*)
NSE-*:NONSTOP_KERNEL:*:*)
echo nse-tandem-nsk${UNAME_RELEASE}
exit ;;
NSR-?:NONSTOP_KERNEL:*:*)
@ -1330,157 +1389,6 @@ EOF
exit ;;
esac
#echo '(No uname command or uname output not recognized.)' 1>&2
#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2
eval $set_cc_for_build
cat >$dummy.c <<EOF
#ifdef _SEQUENT_
# include <sys/types.h>
# include <sys/utsname.h>
#endif
main ()
{
#if defined (sony)
#if defined (MIPSEB)
/* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed,
I don't know.... */
printf ("mips-sony-bsd\n"); exit (0);
#else
#include <sys/param.h>
printf ("m68k-sony-newsos%s\n",
#ifdef NEWSOS4
"4"
#else
""
#endif
); exit (0);
#endif
#endif
#if defined (__arm) && defined (__acorn) && defined (__unix)
printf ("arm-acorn-riscix\n"); exit (0);
#endif
#if defined (hp300) && !defined (hpux)
printf ("m68k-hp-bsd\n"); exit (0);
#endif
#if defined (NeXT)
#if !defined (__ARCHITECTURE__)
#define __ARCHITECTURE__ "m68k"
#endif
int version;
version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`;
if (version < 4)
printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version);
else
printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version);
exit (0);
#endif
#if defined (MULTIMAX) || defined (n16)
#if defined (UMAXV)
printf ("ns32k-encore-sysv\n"); exit (0);
#else
#if defined (CMU)
printf ("ns32k-encore-mach\n"); exit (0);
#else
printf ("ns32k-encore-bsd\n"); exit (0);
#endif
#endif
#endif
#if defined (__386BSD__)
printf ("i386-pc-bsd\n"); exit (0);
#endif
#if defined (sequent)
#if defined (i386)
printf ("i386-sequent-dynix\n"); exit (0);
#endif
#if defined (ns32000)
printf ("ns32k-sequent-dynix\n"); exit (0);
#endif
#endif
#if defined (_SEQUENT_)
struct utsname un;
uname(&un);
if (strncmp(un.version, "V2", 2) == 0) {
printf ("i386-sequent-ptx2\n"); exit (0);
}
if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */
printf ("i386-sequent-ptx1\n"); exit (0);
}
printf ("i386-sequent-ptx\n"); exit (0);
#endif
#if defined (vax)
# if !defined (ultrix)
# include <sys/param.h>
# if defined (BSD)
# if BSD == 43
printf ("vax-dec-bsd4.3\n"); exit (0);
# else
# if BSD == 199006
printf ("vax-dec-bsd4.3reno\n"); exit (0);
# else
printf ("vax-dec-bsd\n"); exit (0);
# endif
# endif
# else
printf ("vax-dec-bsd\n"); exit (0);
# endif
# else
printf ("vax-dec-ultrix\n"); exit (0);
# endif
#endif
#if defined (alliant) && defined (i860)
printf ("i860-alliant-bsd\n"); exit (0);
#endif
exit (1);
}
EOF
$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` &&
{ echo "$SYSTEM_NAME"; exit; }
# Apollos put the system type in the environment.
test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; }
# Convex versions that predate uname can use getsysinfo(1)
if [ -x /usr/convex/getsysinfo ]
then
case `getsysinfo -f cpu_type` in
c1*)
echo c1-convex-bsd
exit ;;
c2*)
if getsysinfo -f scalar_acc
then echo c32-convex-bsd
else echo c2-convex-bsd
fi
exit ;;
c34*)
echo c34-convex-bsd
exit ;;
c38*)
echo c38-convex-bsd
exit ;;
c4*)
echo c4-convex-bsd
exit ;;
esac
fi
cat >&2 <<EOF
$0: unable to guess system type

137
contrib/file/config.sub vendored
View File

@ -1,24 +1,18 @@
#! /bin/sh
# Configuration validation subroutine script.
# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
# 2011, 2012 Free Software Foundation, Inc.
# Copyright 1992-2015 Free Software Foundation, Inc.
timestamp='2012-04-18'
timestamp='2015-03-08'
# This file is (in principle) common to ALL GNU software.
# The presence of a machine in this file suggests that SOME GNU software
# can handle that machine. It does not imply ALL GNU software can.
#
# This file is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# This file is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.
@ -26,11 +20,12 @@ timestamp='2012-04-18'
# As a special exception to the GNU General Public License, if you
# distribute this file as part of a program that contains a
# configuration script generated by Autoconf, you may include it under
# the same distribution terms that you use for the rest of that program.
# the same distribution terms that you use for the rest of that
# program. This Exception is an additional permission under section 7
# of the GNU General Public License, version 3 ("GPLv3").
# Please send patches to <config-patches@gnu.org>. Submit a context
# diff and a properly formatted GNU ChangeLog entry.
# Please send patches to <config-patches@gnu.org>.
#
# Configuration subroutine to validate and canonicalize a configuration type.
# Supply the specified configuration type as an argument.
@ -73,9 +68,7 @@ Report bugs and patches to <config-patches@gnu.org>."
version="\
GNU config.sub ($timestamp)
Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
Free Software Foundation, Inc.
Copyright 1992-2015 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@ -123,8 +116,8 @@ esac
maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
case $maybe_os in
nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \
linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
knetbsd*-gnu* | netbsd*-gnu* | \
linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \
kopensolaris*-gnu* | \
storm-chaos* | os2-emx* | rtmk-nova*)
os=-$maybe_os
@ -156,7 +149,7 @@ case $os in
-convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\
-c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \
-harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \
-apple | -axis | -knuth | -cray | -microblaze)
-apple | -axis | -knuth | -cray | -microblaze*)
os=
basic_machine=$1
;;
@ -259,21 +252,24 @@ case $basic_machine in
| alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \
| alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \
| am33_2.0 \
| arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr | avr32 \
| be32 | be64 \
| arc | arceb \
| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \
| avr | avr32 \
| be32 | be64 \
| bfin \
| c4x | clipper \
| c4x | c8051 | clipper \
| d10v | d30v | dlx | dsp16xx \
| epiphany \
| fido | fr30 | frv \
| e2k | epiphany \
| fido | fr30 | frv | ft32 \
| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
| hexagon \
| i370 | i860 | i960 | ia64 \
| ip2k | iq2000 \
| k1om \
| le32 | le64 \
| lm32 \
| m32c | m32r | m32rle | m68000 | m68k | m88k \
| maxq | mb | microblaze | mcore | mep | metag \
| maxq | mb | microblaze | microblazeel | mcore | mep | metag \
| mips | mipsbe | mipseb | mipsel | mipsle \
| mips16 \
| mips64 | mips64el \
@ -287,23 +283,26 @@ case $basic_machine in
| mips64vr5900 | mips64vr5900el \
| mipsisa32 | mipsisa32el \
| mipsisa32r2 | mipsisa32r2el \
| mipsisa32r6 | mipsisa32r6el \
| mipsisa64 | mipsisa64el \
| mipsisa64r2 | mipsisa64r2el \
| mipsisa64r6 | mipsisa64r6el \
| mipsisa64sb1 | mipsisa64sb1el \
| mipsisa64sr71k | mipsisa64sr71kel \
| mipsr5900 | mipsr5900el \
| mipstx39 | mipstx39el \
| mn10200 | mn10300 \
| moxie \
| mt \
| msp430 \
| nds32 | nds32le | nds32be \
| nios | nios2 \
| nios | nios2 | nios2eb | nios2el \
| ns16k | ns32k \
| open8 \
| or32 \
| open8 | or1k | or1knd | or32 \
| pdp10 | pdp11 | pj | pjl \
| powerpc | powerpc64 | powerpc64le | powerpcle \
| pyramid \
| riscv32 | riscv64 \
| rl78 | rx \
| score \
| sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \
@ -314,6 +313,7 @@ case $basic_machine in
| tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
| ubicom32 \
| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
| visium \
| we32k \
| x86 | xc16x | xstormy16 | xtensa \
| z8k | z80)
@ -328,7 +328,10 @@ case $basic_machine in
c6x)
basic_machine=tic6x-unknown
;;
m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | picochip)
leon|leon[3-9])
basic_machine=sparc-$basic_machine
;;
m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip)
basic_machine=$basic_machine-unknown
os=-none
;;
@ -370,26 +373,28 @@ case $basic_machine in
| aarch64-* | aarch64_be-* \
| alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \
| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
| alphapca5[67]-* | alpha64pca5[67]-* | arc-* \
| alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \
| arm-* | armbe-* | armle-* | armeb-* | armv*-* \
| avr-* | avr32-* \
| be32-* | be64-* \
| bfin-* | bs2000-* \
| c[123]* | c30-* | [cjt]90-* | c4x-* \
| clipper-* | craynv-* | cydra-* \
| c8051-* | clipper-* | craynv-* | cydra-* \
| d10v-* | d30v-* | dlx-* \
| elxsi-* \
| e2k-* | elxsi-* \
| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
| h8300-* | h8500-* \
| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
| hexagon-* \
| i*86-* | i860-* | i960-* | ia64-* \
| ip2k-* | iq2000-* \
| k1om-* \
| le32-* | le64-* \
| lm32-* \
| m32c-* | m32r-* | m32rle-* \
| m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \
| m88110-* | m88k-* | maxq-* | mcore-* | metag-* | microblaze-* \
| m88110-* | m88k-* | maxq-* | mcore-* | metag-* \
| microblaze-* | microblazeel-* \
| mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \
| mips16-* \
| mips64-* | mips64el-* \
@ -403,18 +408,22 @@ case $basic_machine in
| mips64vr5900-* | mips64vr5900el-* \
| mipsisa32-* | mipsisa32el-* \
| mipsisa32r2-* | mipsisa32r2el-* \
| mipsisa32r6-* | mipsisa32r6el-* \
| mipsisa64-* | mipsisa64el-* \
| mipsisa64r2-* | mipsisa64r2el-* \
| mipsisa64r6-* | mipsisa64r6el-* \
| mipsisa64sb1-* | mipsisa64sb1el-* \
| mipsisa64sr71k-* | mipsisa64sr71kel-* \
| mipsr5900-* | mipsr5900el-* \
| mipstx39-* | mipstx39el-* \
| mmix-* \
| mt-* \
| msp430-* \
| nds32-* | nds32le-* | nds32be-* \
| nios-* | nios2-* \
| nios-* | nios2-* | nios2eb-* | nios2el-* \
| none-* | np1-* | ns16k-* | ns32k-* \
| open8-* \
| or1k*-* \
| orion-* \
| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \
@ -432,6 +441,7 @@ case $basic_machine in
| ubicom32-* \
| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
| vax-* \
| visium-* \
| we32k-* \
| x86-* | x86_64-* | xc16x-* | xps100-* \
| xstormy16-* | xtensa*-* \
@ -508,6 +518,9 @@ case $basic_machine in
basic_machine=i386-pc
os=-aros
;;
asmjs)
basic_machine=asmjs-unknown
;;
aux)
basic_machine=m68k-apple
os=-aux
@ -769,6 +782,9 @@ case $basic_machine in
basic_machine=m68k-isi
os=-sysv
;;
leon-*|leon[3-9]-*)
basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'`
;;
m68knommu)
basic_machine=m68k-unknown
os=-linux
@ -788,11 +804,15 @@ case $basic_machine in
basic_machine=ns32k-utek
os=-sysv
;;
microblaze)
microblaze*)
basic_machine=microblaze-xilinx
;;
mingw64)
basic_machine=x86_64-pc
os=-mingw64
;;
mingw32)
basic_machine=i386-pc
basic_machine=i686-pc
os=-mingw32
;;
mingw32ce)
@ -820,6 +840,10 @@ case $basic_machine in
basic_machine=powerpc-unknown
os=-morphos
;;
moxiebox)
basic_machine=moxie-unknown
os=-moxiebox
;;
msdos)
basic_machine=i386-pc
os=-msdos
@ -828,7 +852,7 @@ case $basic_machine in
basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'`
;;
msys)
basic_machine=i386-pc
basic_machine=i686-pc
os=-msys
;;
mvs)
@ -1019,7 +1043,11 @@ case $basic_machine in
basic_machine=i586-unknown
os=-pw32
;;
rdos)
rdos | rdos64)
basic_machine=x86_64-pc
os=-rdos
;;
rdos32)
basic_machine=i386-pc
os=-rdos
;;
@ -1346,29 +1374,29 @@ case $os in
-gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \
| -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\
| -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \
| -sym* | -kopensolaris* \
| -sym* | -kopensolaris* | -plan9* \
| -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
| -aos* | -aros* \
| -aos* | -aros* | -cloudabi* \
| -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
| -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
| -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
| -openbsd* | -solidbsd* \
| -bitrig* | -openbsd* | -solidbsd* \
| -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \
| -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \
| -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \
| -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \
| -chorusos* | -chorusrdb* | -cegcc* \
| -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \
| -mingw32* | -linux-gnu* | -linux-android* \
| -linux-newlib* | -linux-uclibc* \
| -uxpv* | -beos* | -mpeix* | -udk* \
| -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \
| -linux-newlib* | -linux-musl* | -linux-uclibc* \
| -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \
| -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \
| -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \
| -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \
| -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \
| -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \
| -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \
| -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*)
| -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* | -tirtos*)
# Remember, each alternative MUST END IN *, to match a version number.
;;
-qnx*)
@ -1492,9 +1520,6 @@ case $os in
-aros*)
os=-aros
;;
-kaos*)
os=-kaos
;;
-zvmoe)
os=-zvmoe
;;
@ -1543,6 +1568,12 @@ case $basic_machine in
c4x-* | tic4x-*)
os=-coff
;;
c8051-*)
os=-elf
;;
hexagon-*)
os=-elf
;;
tic54x-*)
os=-coff
;;

20
contrib/file/configure vendored
View File

@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for file 5.28.
# Generated by GNU Autoconf 2.69 for file 5.29.
#
# Report bugs to <christos@astron.com>.
#
@ -590,8 +590,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='file'
PACKAGE_TARNAME='file'
PACKAGE_VERSION='5.28'
PACKAGE_STRING='file 5.28'
PACKAGE_VERSION='5.29'
PACKAGE_STRING='file 5.29'
PACKAGE_BUGREPORT='christos@astron.com'
PACKAGE_URL=''
@ -1328,7 +1328,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures file 5.28 to adapt to many kinds of systems.
\`configure' configures file 5.29 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1398,7 +1398,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of file 5.28:";;
short | recursive ) echo "Configuration of file 5.29:";;
esac
cat <<\_ACEOF
@ -1509,7 +1509,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
file configure 5.28
file configure 5.29
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@ -2165,7 +2165,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by file $as_me 5.28, which was
It was created by file $as_me 5.29, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@ -3031,7 +3031,7 @@ fi
# Define the identity of the package.
PACKAGE='file'
VERSION='5.28'
VERSION='5.29'
cat >>confdefs.h <<_ACEOF
@ -15075,7 +15075,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by file $as_me 5.28, which was
This file was extended by file $as_me 5.29, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -15141,7 +15141,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
file config.status 5.28
file config.status 5.29
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"

2
contrib/file/configure.ac
View File

@ -1,5 +1,5 @@
dnl Process this file with autoconf to produce a configure script.
AC_INIT([file],[5.28],[christos@astron.com])
AC_INIT([file],[5.29],[christos@astron.com])
AM_INIT_AUTOMAKE([subdir-objects foreign])
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])

13
contrib/file/doc/file.man
View File

@ -1,5 +1,5 @@
.\" $File: file.man,v 1.121 2016/06/07 22:09:20 rrt Exp $
.Dd March 13, 2016
.\" $File: file.man,v 1.124 2016/10/19 20:52:45 christos Exp $
.Dd October 19, 2016
.Dt FILE __CSECTION__
.Os
.Sh NAME
@ -8,7 +8,7 @@
.Sh SYNOPSIS
.Nm
.Bk -words
.Op Fl bcEhiklLNnprsvzZ0
.Op Fl bcdEhiklLNnprsvzZ0
.Op Fl Fl apple
.Op Fl Fl extension
.Op Fl Fl mime-encoding
@ -181,6 +181,8 @@ Cause a checking printout of the parsed form of the magic file.
This is usually used in conjunction with the
.Fl m
flag to debug a new magic file before installing it.
.It Fl d
Prints internal debugging information to stderr.
.It Fl E
On filesystem errors (file not found etc), instead of handling the error
as regular output as POSIX mandates and keep going, issue an error message
@ -208,7 +210,8 @@ Prints details of Compound Document Files.
.It compress
Checks for, and looks inside, compressed files.
.It elf
Prints ELF file details.
Prints ELF file details, provided soft magic tests are enabled and the
elf magic is found.
.It soft
Consults magic files.
.It tar
@ -312,7 +315,7 @@ attempt to preserve the access time of files analyzed, to pretend that
never read them.
.It Fl P , Fl Fl parameter Ar name=value
Set various parameter limits.
.Bl -column "elf_phnum" "Default" "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" -offset indent
.Bl -column "elf_phnum" "Default" "XXXXXXXXXXXXXXXXXXXXXXXXXXX" -offset indent
.It Sy "Name" Ta Sy "Default" Ta Sy "Explanation"
.It Li indir Ta 15 Ta recursion limit for indirect magic
.It Li name Ta 30 Ta use count limit for name/use magic

17
contrib/file/doc/magic.man
View File

@ -1,5 +1,5 @@
.\" $File: magic.man,v 1.86 2015/09/08 13:48:44 christos Exp $
.Dd January 1, 2015
.\" $File: magic.man,v 1.88 2016/07/27 09:42:49 rrt Exp $
.Dd July 20, 2016
.Dt MAGIC __FSECTION__
.Os
.\" install as magic.4 on USG, magic.5 on V7, Berkeley and Linux systems.
@ -535,13 +535,18 @@ the file.
The value at that offset is read, and is used again as an offset
in the file.
Indirect offsets are of the form:
.Em (( x [.[bislBISL]][+\-][ y ]) .
.Em (( x [[.,][bislBISL]][+\-][ y ]) .
The value of
.Em x
is used as an offset in the file.
A byte, id3 length, short or long is read at that offset depending on the
.Em [bislBISLm]
type specifier.
The value is treated as signed if
.Dq ,
is specified or unsigned if
.Dq .
is specified.
The capitalized types interpret the number as a big endian
value, whereas the small letter versions interpret the number as a little
endian value;
@ -564,9 +569,9 @@ That way variable length structures can be examined:
\*[Gt]\*[Gt](0x3c.l) string LX\e0\e0 LX executable (OS/2)
.Ed
.Pp
This strategy of examining has a drawback: You must make sure that
you eventually print something, or users may get empty output (like, when
there is neither PE\e0\e0 nor LE\e0\e0 in the above example)
This strategy of examining has a drawback: you must make sure that you
eventually print something, or users may get empty output (such as when
there is neither PE\e0\e0 nor LE\e0\e0 in the above example).
.Pp
If this indirect offset cannot be used directly, simple calculations are
possible: appending

3
contrib/file/ltmain.sh
View File

@ -9432,7 +9432,8 @@ dlpreopen='$dlprefiles'
# Directory that this library needs to be installed in:
libdir='$install_libdir'"
if test "$installed" = no && test "$need_relink" = yes; then
if test "$installed" = no && test "$need_relink" = yes && \
test -n "$relink_command"; then
$ECHO >> $output "\
relink_command=\"$relink_command\""
fi

19
contrib/file/magic/Magdir/algol68 Normal file
View File

@ -0,0 +1,19 @@
#------------------------------------------------------------------------------
# $File: algol68,v 1.2 2016/10/17 14:17:48 christos Exp $
# algol68: file(1) magic for Algol 68 source
#
0 search/8192 (input, Algol 68 source text
!:mime text/x-Algol68
0 regex \^PROC Algol 68 source text
!:mime text/x-Algol68
0 regex MODE[\t\ ] Algol 68 source text
!:mime text/x-Algol68
0 regex REF[\t\ ] Algol 68 source text
!:mime text/x-Algol68
0 regex FLEX[\t\ ]\*\\[ Algol 68 source text
!:mime text/x-Algol68
#0 regex [\t\ ]OD Algol 68 source text
#!:mime text/x-Algol68
#0 regex [\t\ ]FI Algol 68 source text
#!:mime text/x-Algol68

10
contrib/file/magic/Magdir/animation
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: animation,v 1.57 2015/11/29 22:11:07 christos Exp $
# $File: animation,v 1.58 2016/07/03 14:13:11 christos Exp $
# animation: file(1) magic for animation/movie formats
#
# animation formats
@ -1008,3 +1008,11 @@
>4 lelong x %d x
>8 lelong x %d,
>12 lelong x %d frames
# Material Exchange Format
# More information:
# https://en.wikipedia.org/wiki/Material_Exchange_Format
# http://www.freemxf.org/
0 string \x06\x0e\x2b\x34\x02\x05\x01\x01\x0d\x01\x02\x01\x01\x02 Material exchange container format
!:ext mxf
!:mime application/mxf

172
contrib/file/magic/Magdir/apple
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: apple,v 1.32 2015/12/04 20:40:10 christos Exp $
# $File: apple,v 1.35 2016/08/17 09:45:13 christos Exp $
# apple: file(1) magic for Apple file formats
#
0 search/1/t FiLeStArTfIlEsTaRt binscii (apple ][) text
@ -285,49 +285,118 @@
# .vdi
4 string innotek\ VirtualBox\ Disk\ Image %s
# Apple disk partition stuff, strengthen the magic using byte 4
# Apple disk partition stuff
# URL: https://en.wikipedia.org/wiki/Apple_Partition_Map
# Reference: https://ftp.netbsd.org/pub/NetBSD/NetBSD-current/src/sys/sys/bootblock.h
# Update: Joerg Jenderek
# "ER" is APPLE_DRVR_MAP_MAGIC signature
0 beshort 0x4552
>4 byte 0 Apple Driver Map
# display Apple Driver Map (strength=50) after Syslinux bootloader (71)
#!:strength +0
# strengthen the magic by looking for used blocksizes 512 2048
>2 ubeshort&0xf1FF 0 Apple Driver Map
# last 6 bytes for padding found are 0 or end with 55AAh marker for MBR hybrid
#>>504 ubequad&0x0000FFffFFff0000 0
!:mime application/x-apple-diskimage
!:apple ????devr
# https://en.wikipedia.org/wiki/Apple_Disk_Image
!:ext dmg/iso
# sbBlkSize for driver descriptor map 512 2048
>>2 beshort x \b, blocksize %d
>>4 belong x \b, blockcount %d
>>10 beshort x \b, devtype %d
>>12 beshort x \b, devid %d
>>20 beshort x \b, descriptors %d
# Assume 8 partitions each at a multiple of the sector size.
# We could glean this from the partition descriptors, but they are empty!?!?
>>(2.S*1) indirect x \b, contains[@0x%x]:
>>(2.S*2) indirect x \b, contains[@0x%x]:
>>(2.S*3) indirect x \b, contains[@0x%x]:
>>(2.S*4) indirect x \b, contains[@0x%x]:
>>(2.S*5) indirect x \b, contains[@0x%x]:
>>(2.S*6) indirect x \b, contains[@0x%x]:
>>(2.S*7) indirect x \b, contains[@0x%x]:
>>(2.S*8) indirect x \b, contains[@0x%x]:
# sbBlkCount sometimes garbish like
# 0xb0200000 for unzlibed install_flash_player_19.0.0.245_osx.dmg
# 0xf2720100 for bunziped Firefox 48.0-2.dmg
# 0xeb02ffff for super_grub2_disk_hybrid_2.02s3.iso
# 0x00009090 by syslinux-6.03/utils/isohybrid.c
>>4 ubelong x \b, blockcount %u
# following device/driver information not very useful
# device type 0 1 (37008 garbage for super_grub2_disk_hybrid_2.02s3.iso)
>>8 ubeshort x \b, devtype %u
# device id 0 1 (37008 garbage for super_grub2_disk_hybrid_2.02s3.iso)
>>10 ubeshort x \b, devid %u
# driver data 0 (2425393296 garbage for super_grub2_disk_hybrid_2.02s3.iso)
>>12 ubelong >0
>>>12 ubelong x \b, driver data %u
# number of driver descriptors sbDrvrCount <= 61
# (37008 garbage for super_grub2_disk_hybrid_2.02s3.iso)
>>16 ubeshort x \b, driver count %u
# 61 * apple_drvr_descriptor[8]. information not very useful or same as in partition map
# >>18 use apple-driver-map
# >>26 use apple-driver-map
# # ...
# >>500 use apple-driver-map
# number of partitions is always same in every partition (map block count)
#>>0x0204 ubelong x \b, %u partitions
>>0x0204 ubelong >0 \b, contains[@0x200]:
>>>0x0200 use apple-apm
>>0x0204 ubelong >1 \b, contains[@0x400]:
>>>0x0400 use apple-apm
>>0x0204 ubelong >2 \b, contains[@0x600]:
>>>0x0600 use apple-apm
>>0x0204 ubelong >3 \b, contains[@0x800]:
>>>0x0800 use apple-apm
>>0x0204 ubelong >4 \b, contains[@0xA00]:
>>>0x0A00 use apple-apm
>>0x0204 ubelong >5 \b, contains[@0xC00]:
>>>0x0C00 use apple-apm
>>0x0204 ubelong >6 \b, contains[@0xE00]:
>>>0x0E00 use apple-apm
>>0x0204 ubelong >7 \b, contains[@0x1000]:
>>>0x1000 use apple-apm
# display apple driver descriptor map (start-block, # blocks in sbBlkSize sizes, type)
0 name apple-driver-map
>0 ubequad !0
# descBlock first block of driver
>>0 ubelong x \b, driver start block %u
# descSize driver size in blocks
>>4 ubeshort x \b, size %u
# descType driver system type 1 701h F8FFh FFFFh
>>6 ubeshort x \b, type 0x%x
# Yes, the 3rd and 4th bytes are reserved, but we use them to make the
# URL: https://en.wikipedia.org/wiki/Apple_Partition_Map
# Reference: http://opensource.apple.com/source/IOStorageFamily/IOStorageFamily-116/IOApplePartitionScheme.h
# Update: Joerg Jenderek
# Yes, the 3rd and 4th bytes pmSigPad are reserved, but we use them to make the
# magic stronger.
0 belong 0x504d0000 Apple Partition Map
>4 belong x \b, map block count %d
>8 belong x \b, start block %d
>12 belong x \b, block count %d
>16 string >0 \b, name %s
>48 string >0 \b, type %s
>124 string >0 \b, processor %s
>140 string >0 \b, boot arguments %s
>92 belong & 1 \b, valid
>92 belong & 2 \b, allocated
>92 belong & 4 \b, in use
>92 belong & 8 \b, has boot info
>92 belong & 16 \b, readable
>92 belong & 32 \b, writable
>92 belong & 64 \b, pic boot code
>92 belong & 128 \b, chain compatible driver
>92 belong & 256 \b, real driver
>92 belong & 512 \b, chain driver
>92 belong & 1024 \b, mount at startup
>92 belong & 2048 \b, is the startup partition
# for apple partition map stored as a single file
0 belong 0x504d0000
# to display Apple Partition Map (strength=70) after Syslinux bootloader (71)
#!:strength +0
>0 use apple-apm
# magic/Magdir/apple14.test, 365: Warning: Current entry does not yet have a description for adding a EXTENSION type
# file: could not find any valid magic files!
#!:ext bin
# display apple partition map. Normally called after Apple driver map
0 name apple-apm
>0 belong 0x504d0000 Apple Partition Map
# number of partitions
>>4 ubelong x \b, map block count %u
# logical block (512 bytes) start of partition
>>8 ubelong x \b, start block %u
>>12 ubelong x \b, block count %u
>>16 string >0 \b, name %s
>>48 string >0 \b, type %s
# processor type dpme_process_id[16] e.g. "68000" "68020"
>>120 string >0 \b, processor %s
# A/UX boot arguments BootArgs[128]
>>136 string >0 \b, boot arguments %s
# status of partition dpme_flags
>>88 belong & 1 \b, valid
>>88 belong & 2 \b, allocated
>>88 belong & 4 \b, in use
>>88 belong & 8 \b, has boot info
>>88 belong & 16 \b, readable
>>88 belong & 32 \b, writable
>>88 belong & 64 \b, pic boot code
>>88 belong & 128 \b, chain compatible driver
>>88 belong & 256 \b, real driver
>>88 belong & 512 \b, chain driver
# mount automatically at startup APPLE_PS_AUTO_MOUNT
>>88 ubelong &0x40000000 \b, mount at startup
# is the startup partition APPLE_PS_STARTUP
>>88 ubelong &0x80000000 \b, is the startup partition
#http://wiki.mozilla.org/DS_Store_File_Format`
#http://wiki.mozilla.org/DS_Store_File_Format
#http://en.wikipedia.org/wiki/.DS_Store
0 string \0\0\0\1Bud1\0 Apple Desktop Services Store
@ -337,5 +406,28 @@
# extensions rsr/rsrc
0 string \000\000\001\000
>4 leshort 0
>>16 lelong 0 Apple HFS/HFS+ resource fork
>>16 lelong 0 Apple HFS/HFS+ resource fork
#https://en.wikipedia.org/wiki/AppleScript
0 string FasdUAS AppleScript compiled
# AppleWorks/ClarisWorks
# https://github.com/joshenders/appleworks_format
# http://fileformats.archiveteam.org/wiki/AppleWorks
0 name appleworks
>0 belong&0x00ffffff 0x07e100 AppleWorks CWK Document
>0 belong&0x00ffffff 0x008803 ClarisWorks CWK Document
>0 default x
>>0 belong x AppleWorks/ClarisWorks CWK Document
>0 byte x \b, version %d
>30 beshort x \b, %d
>32 beshort x \bx%d
!:ext cwk
4 string BOBO
>0 byte >4
>>12 belong 0
>>>26 belong 0
>>>>0 use appleworks
>0 belong 0x0481ad00
>>0 use appleworks

7
contrib/file/magic/Magdir/application Normal file
View File

@ -0,0 +1,7 @@
#------------------------------------------------------------------------------
# $File: application,v 1.1 2016/10/17 12:13:01 christos Exp $
# application: file(1) magic for applications on small devices
#
# Pebble Application
0 string PBLAPP\000\000 Pebble application

52
contrib/file/magic/Magdir/apt Normal file
View File

@ -0,0 +1,52 @@
#------------------------------------------------------------------------------
# $File: apt,v 1.1 2016/10/17 19:51:57 christos Exp $
# apt: file(1) magic for APT Cache files
# <http://www.fifi.org/doc/libapt-pkg-doc/cache.html/ch2.html>
# <https://anonscm.debian.org/cgit/apt/apt.git/tree/apt-pkg/pkgcache.h#n292>
# before version 10 ("old format"), data was in arch-specific long/short
# old format 64 bit
0 name apt-cache-64bit-be
>12 beshort 1 \b, dirty
>40 bequad x \b, %llu packages
>48 bequad x \b, %llu versions
# old format 32 bit
0 name apt-cache-32bit-be
>8 beshort 1 \b, dirty
>40 belong x \b, %u packages
>44 belong x \b, %u versions
# new format
0 name apt-cache-be
>6 byte 1 \b, dirty
>24 belong x \b, %u packages
>28 belong x \b, %u versions
0 bequad 0x98FE76DC
>8 ubeshort <10 APT cache data, version %u
>>10 beshort x \b.%u, 64 bit big-endian
>>0 use apt-cache-64bit-be
0 lequad 0x98FE76DC
>8 uleshort <10 APT cache data, version %u
>>10 leshort x \b.%u, 64 bit little-endian
>>0 use \^apt-cache-64bit-be
0 belong 0x98FE76DC
>4 ubeshort <10 APT cache data, version %u
>>6 ubeshort x \b.%u, 32 bit big-endian
>>0 use apt-cache-32bit-be
>4 ubyte >9 APT cache data, version %u
>>5 ubyte x \b.%u, big-endian
>>0 use apt-cache-be
0 lelong 0x98FE76DC
>4 uleshort <10 APT cache data, version %u
>>6 uleshort x \b.%u, 32 bit little-endian
>>0 use \^apt-cache-32bit-be
>4 ubyte >9 APT cache data, version %u
>>5 ubyte x \b.%u, little-endian
>>0 use \^apt-cache-be

60
contrib/file/magic/Magdir/bioinformatics
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: bioinformatics,v 1.2 2016/02/14 15:53:53 christos Exp $
# $File: bioinformatics,v 1.4 2016/06/20 16:13:46 christos Exp $
# bioinfomatics: file(1) magic for Bioinfomatics file formats
###############################################################################
@ -16,32 +16,32 @@
###############################################################################
# Tabix index file
# Tabix index file
# used by SAMtools bgzip/tabix (http://samtools.sourceforge.net/tabix.shtml)
###############################################################################
0 string TBI\1 SAMtools TBI (Tabix index format)
>0x04 lelong =1 \b, with %d reference sequence
>0x04 lelong >1 \b, with %d reference sequences
>0x08 lelong &0x10000 \b, using half-closed-half-open coordinates (BED style)
>0x08 lelong ^0x10000
>0x08 lelong ^0x10000
>>0x08 lelong =0 \b, using closed and one based coordinates (GFF style)
>>0x08 lelong =1 \b, using SAM format
>>0x08 lelong =2 \b, using VCF format
>0x0c lelong x \b, sequence name column: %d
>0x10 lelong x \b, region start column: %d
>0x08 lelong =0
>0x08 lelong =0
>>0x14 lelong x \b, region end column: %d
>0x18 byte x \b, comment character: %c
>0x1c lelong x \b, skip line count: %d
###############################################################################
# BAM (Binary Sequence Alignment/Map format)
# used by SAMtools (http://samtools.sourceforge.net/SAM1.pdf)
# BAM (Binary Sequence Alignment/Map format)
# used by SAMtools (http://samtools.sourceforge.net/SAM1.pdf)
# data is normally present only within compressed BGZF blocks (CDATA), so use file -z to examine it
###############################################################################
0 string BAM\1 SAMtools BAM (Binary Sequence Alignment/Map)
>0x04 lelong >0
>0x04 lelong >0
>>&0x00 regex =^[@]HD\t.*VN: \b, with SAM header
>>>&0 regex =[0-9.]+ \b version %s
>>&(0x04) lelong >0 \b, with %d reference sequences
@ -49,14 +49,14 @@
###############################################################################
# BAI (BAM indexing format)
# used by SAMtools (http://samtools.sourceforge.net/SAM1.pdf)
# used by SAMtools (http://samtools.sourceforge.net/SAM1.pdf)
###############################################################################
0 string BAI\1 SAMtools BAI (BAM indexing format)
>0x04 lelong >0 \b, with %d reference sequences
###############################################################################
# CRAM (Binary Sequence Alignment/Map format)
# CRAM (Binary Sequence Alignment/Map format)
###############################################################################
0 string CRAM CRAM
>0x04 byte >-1 version %d.
@ -69,13 +69,13 @@
# used by SAMtools & VCFtools (http://vcftools.sourceforge.net/bcf.pdf)
# data is normally present only within compressed BGZF blocks (CDATA), so use file -z to examine it
###############################################################################
0 string BCF\4
0 string BCF\4
# length of seqnm data in bytes is positive
>&0x00 lelong >0
>&0x00 lelong >0
# length of smpl data in bytes is positive
>>&(&-0x04) lelong >0 SAMtools BCF (Binary Call Format)
# length of meta in bytes
>>>&(&-0x04) lelong >0
>>>&(&-0x04) lelong >0
# have meta text string
>>>>&0x00 search ##samtoolsVersion=
>>>>>&0x00 string x \b, generated by SAMtools version %s
@ -88,7 +88,7 @@
###############################################################################
0 string BCF\2\1 Binary Call Format (BCF) version 2.1
# length of header text
>&0x00 lelong >0
>&0x00 lelong >0
# have header string
>>&0x00 search ##samtoolsVersion=
>>>&0x00 string x \b, generated by SAMtools version %s
@ -101,7 +101,7 @@
###############################################################################
0 string BCF\2\2 Binary Call Format (BCF) version 2.2
# length of header text
>&0x00 lelong >0
>&0x00 lelong >0
# have header string
>>&0x00 search ##samtoolsVersion=
>>>&0x00 string x \b, generated by SAMtools version %s
@ -119,11 +119,11 @@
###############################################################################
# XXX Broken?
# @<seqname>
#0 regex =^@[A-Za-z0-9_.:-]+\?\n
#0 regex =^@[A-Za-z0-9_.:-]+\?\n
# <seq>
#>&1 regex =^[A-Za-z\n.~]++
# +[<seqname>]
#>>&1 regex =^[A-Za-z0-9_.:-]*\?\n
#>>&1 regex =^[A-Za-z0-9_.:-]*\?\n
# <qual>
#>>>&1 regex =^[!-~\n]+\n FASTQ
@ -132,7 +132,7 @@
# used by FASTA (http://fasta.bioch.virginia.edu/fasta_www2/fasta_guide.pdf)
###############################################################################
#0 byte 0x3e
# q>0 regex =^[>][!-~\t\ ]+$
# q>0 regex =^[>][!-~\t\ ]+$
# Amino Acid codes: [A-IK-Z*-]+
#>>1 regex !=[!-'Jj;:=?@^`|~\\] FASTA
# IUPAC codes/gaps: [ACGTURYKMSWBDHVNX-]+
@ -141,37 +141,37 @@
#>>>1 regex =^[EFIJLOPQZefijlopqz]+$ \b, with Amino Acid codes
###############################################################################
# SAM (Sequence Alignment/Map format)
# used by SAMtools (http://samtools.sourceforge.net/SAM1.pdf)
# SAM (Sequence Alignment/Map format)
# used by SAMtools (http://samtools.sourceforge.net/SAM1.pdf)
###############################################################################
# Short-cut version to recognise SAM files with (optional) header at beginning
###############################################################################
0 string @HD\t
0 string @HD\t
>4 search VN: Sequence Alignment/Map (SAM), with header
>>&0 regex [0-9.]+ \b version %s
###############################################################################
# Longer version to recognise SAM alignment lines using (many) regexes
###############################################################################
# SAM Alignment QNAME
0 regex =^[!-?A-~]{1,255}(\t[^\t]+){11}
0 regex =^[!-?A-~]{1,255}(\t[^\t]+){11}
# SAM Alignment FLAG
>0 regex =^([^\t]+\t){1}[0-9]{1,5}\t
>0 regex =^([^\t]+\t){1}[0-9]{1,5}\t
# SAM Alignment RNAME
>>0 regex =^([^\t]+\t){2}\\*|[^*=]*\t
>>0 regex =^([^\t]+\t){2}\\*|[^*=]*\t
# SAM Alignment POS
>>>0 regex =^([^\t]+\t){3}[0-9]{1,9}\t
>>>0 regex =^([^\t]+\t){3}[0-9]{1,9}\t
# SAM Alignment MAPQ
>>>>0 regex =^([^\t]+\t){4}[0-9]{1,3}\t
>>>>0 regex =^([^\t]+\t){4}[0-9]{1,3}\t
# SAM Alignment CIGAR
>>>>>0 regex =\t\\*|([0-9]+[MIDNSHPX=])+)\t
>>>>>0 regex =\t(\\*|([0-9]+[MIDNSHPX=])+)\t
# SAM Alignment RNEXT
>>>>>>0 regex =\t(\\*|=|[!-()+->?-~][!-~]*)\t
>>>>>>0 regex =\t(\\*|=|[!-()+->?-~][!-~]*)\t
# SAM Alignment PNEXT
>>>>>>>0 regex =^([^\t]+\t){7}[0-9]{1,9}\t
>>>>>>>0 regex =^([^\t]+\t){7}[0-9]{1,9}\t
# SAM Alignment TLEN
>>>>>>>>0 regex =\t[+-]{0,1}[0-9]{1,9}\t.*\t
>>>>>>>>0 regex =\t[+-]{0,1}[0-9]{1,9}\t.*\t
# SAM Alignment SEQ
>>>>>>>>>0 regex =^([^\t]+\t){9}(\\*|[A-Za-z=.]+)\t
>>>>>>>>>0 regex =^([^\t]+\t){9}(\\*|[A-Za-z=.]+)\t
# SAM Alignment QUAL
>>>>>>>>>>0 regex =^([^\t]+\t){10}[!-~]+ Sequence Alignment/Map (SAM)
>>>>>>>>>>>0 regex =^[@]HD\t.*VN: \b, with header

7
contrib/file/magic/Magdir/c-lang
View File

@ -1,5 +1,5 @@
#------------------------------------------------------------------------------
# $File: c-lang,v 1.23 2016/05/21 14:28:27 christos Exp $
# $File: c-lang,v 1.24 2016/07/01 23:31:13 christos Exp $
# c-lang: file(1) magic for C and related languages programs
#
# The strength is to beat standard HTML
@ -48,6 +48,11 @@
!:strength + 30
!:mime text/x-c++
# Objective-C
0 regex \^#import Objective-C source text
!:strength +25
!:mime text/x-objective-c
# From: Mikhail Teterin <mi@aldan.algebra.com>
0 string cscope cscope reference data
>7 string x version %.2s

5
contrib/file/magic/Magdir/claris
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: claris,v 1.7 2014/06/03 19:17:27 christos Exp $
# $File: claris,v 1.8 2016/07/18 19:23:38 christos Exp $
# claris: file(1) magic for claris
# "H. Nanosecond" <aldomel@ix.netcom.com>
# Claris Works a word processor, etc.
@ -18,7 +18,8 @@
# Claris works files
# .cwk
0 string \002\000\210\003\102\117\102\117\000\001\206 Claris works document
# Moved to Apple AppleWorks document
#0 string \002\000\210\003\102\117\102\117\000\001\206 Claris works document
# .plt
0 string \020\341\000\000\010\010 Claris Works palette files .plt

6
contrib/file/magic/Magdir/commands
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: commands,v 1.54 2016/04/19 13:40:02 christos Exp $
# $File: commands,v 1.56 2016/07/14 19:01:12 christos Exp $
# commands: file(1) magic for various shells and interpreters
#
#0 string/w : shell archive or script for antique kernel text
@ -112,3 +112,7 @@
# URL: http://packages.debian.org/pdmenu
# From: Edward Betts <edward@debian.org>
0 string #!/usr/bin/pdmenu Pdmenu configuration file text
# From Danny Weldon
0 string \x0b\x13\x08\x00
>0x04 uleshort <4 ksh byte-code version %d

52
contrib/file/magic/Magdir/compress
View File

@ -1,5 +1,5 @@
#------------------------------------------------------------------------------
# $File: compress,v 1.65 2015/12/04 20:48:03 christos Exp $
# $File: compress,v 1.66 2016/09/16 12:12:05 christos Exp $
# compress: file(1) magic for pure-compression formats (no archives)
#
# compress, gzip, pack, compact, huf, squeeze, crunch, freeze, yabba, etc.
@ -220,6 +220,56 @@
0 lelong 0x184c2102 LZ4 compressed data (v0.1-v0.9)
!:mime application/x-lz4
# Zstandard/LZ4 skippable frames
# https://github.com/facebook/zstd/blob/dev/zstd_compression_format.md
0 lelong&0xFFFFFFF0 0x184D2A50
>(4.l+8) indirect
# Zstandard Dictionary ID subroutine
0 name zstd-dictionary-id
# Single Segment = True
>0 byte &0x20 \b, Dictionary ID:
>>0 byte&0x03 0 None
>>0 byte&0x03 1
>>>1 byte x %u
>>0 byte&0x03 2
>>>1 leshort x %u
>>0 byte&0x03 3
>>>1 lelong x %u
# Single Segment = False
>0 byte ^0x20 \b, Dictionary ID:
>>0 byte&0x03 0 None
>>0 byte&0x03 1
>>>2 byte x %u
>>0 byte&0x03 2
>>>2 leshort x %u
>>0 byte&0x03 3
>>>2 lelong x %u
# Zstandard compressed data
# https://github.com/facebook/zstd/blob/dev/zstd_compression_format.md
0 lelong 0xFD2FB522 Zstandard compressed data (v0.2)
!:mime application/x-zstd
0 lelong 0xFD2FB523 Zstandard compressed data (v0.3)
!:mime application/x-zstd
0 lelong 0xFD2FB524 Zstandard compressed data (v0.4)
!:mime application/x-zstd
0 lelong 0xFD2FB525 Zstandard compressed data (v0.5)
!:mime application/x-zstd
0 lelong 0xFD2FB526 Zstandard compressed data (v0.6)
!:mime application/x-zstd
0 lelong 0xFD2FB527 Zstandard compressed data (v0.7)
!:mime application/x-zstd
>4 use zstd-dictionary-id
0 lelong 0xFD2FB528 Zstandard compressed data (v0.8+)
!:mime application/x-zstd
>4 use zstd-dictionary-id
# https://github.com/facebook/zstd/blob/dev/zstd_compression_format.md
0 lelong 0xEC30A437 Zstandard dictionary
!:mime application/x-zstd-dictionary
>4 lelong x (ID %u)
# AFX compressed files (Wolfram Kleff)
2 string -afx- AFX compressed file data

4
contrib/file/magic/Magdir/cups
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: cups,v 1.3 2014/05/28 19:50:41 christos Exp $
# $File: cups,v 1.4 2016/10/17 18:51:02 christos Exp $
# Cups: file(1) magic for the cups raster file format
# From: Laurent Martelli <martellilaurent@gmail.com>
# http://www.cups.org/documentation.php/spec-raster.html
@ -44,7 +44,7 @@
>3 string 2 Cups Raster version 2, Big Endian
>3 string 3 Cups Raster version 3, Big Endian
!:mime application/vnd.cups-raster
>0 use ^cups-le
>0 use \^cups-le
# Cups Raster image format, Little Endian

27
contrib/file/magic/Magdir/editors
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: editors,v 1.8 2009/09/19 16:28:09 christos Exp $
# $File: editors,v 1.10 2016/07/18 17:44:49 christos Exp $
# T602 editor documents
# by David Necas <yeti@physics.muni.cz>
0 string @CT\ T602 document data,
@ -12,7 +12,28 @@
# Vi IMproved Encrypted file
# by David Necas <yeti@physics.muni.cz>
0 string VimCrypt~ Vim encrypted file data
0 name vimnanoswap
>67 byte 0
>>107 byte 0
#>>>2 string x %s swap file
>>>24 ulelong x \b, pid %d
>>>28 string >\0 \b, user %s
>>>68 string >\0 \b, host %s
>>>108 string >\0 \b, file %s
>>>1007 byte 0x55 \b, modified
# Vi IMproved Swap file
# by Sven Wegener <swegener@gentoo.org>
0 string b0VIM\ Vim swap file
>&0 string >\0 \b, version %s
0 string b0VIM\ Vim swap file
>&0 string >\0 \b, version %s
>0 use vimnanoswap
# Lock/swap file for several editors, at least
# Vi IMproved and nano
0 string b0nano Nano swap file
>0 use vimnanoswap
# kate (K Advanced Text Editor)
0 string \x00\x00\x00\x12Kate\ Swap\ File\ 2.0\x00 Kate swap file

5
contrib/file/magic/Magdir/filesystems
View File

@ -1,5 +1,5 @@
#------------------------------------------------------------------------------
# $File: filesystems,v 1.113 2016/02/14 14:38:24 christos Exp $
# $File: filesystems,v 1.114 2016/09/05 08:34:25 christos Exp $
# filesystems: file(1) magic for different filesystems
#
0 name partid
@ -539,8 +539,6 @@
>>>514 string !HdrS
# not BeOS
>>>>422 string !Be\ Boot\ Loader
>>>>>32769 string CD001
>>>>>>0 use cdrom
# jump over BPB instruction implies DOS bootsector or AdvanceMAME mbr
>>>>>0 ubelong&0xFD000000 =0xE9000000
# AdvanceMAME mbr
@ -1929,6 +1927,7 @@
>>38917 string 3 (version 2.0)
>>38917 byte >0x33 (unknown version, ID 0x%X)
>>38917 byte <0x31 (unknown version, ID 0x%X)
# The next line is not necessary because the MBR staff is done looking for boot signature
>0x1FE leshort 0xAA55 (DOS/MBR boot sector)
# "application id" which appears to be used as a volume label
>32808 string/T >\0 '%s'

59
contrib/file/magic/Magdir/fonts
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: fonts,v 1.30 2016/03/22 22:27:47 christos Exp $
# $File: fonts,v 1.33 2016/09/14 01:26:26 christos Exp $
# fonts: file(1) magic for font data
#
0 search/1 FONT ASCII vfont text
@ -15,6 +15,56 @@
6 string %!FontType1 PostScript Type 1 font program data
0 string %!PS-Adobe-3.0\ Resource-Font PostScript Type 1 font text
# Summary: PostScript Type 1 Printer Font Metrics
# URL: https://en.wikipedia.org/wiki/PostScript_fonts
# Reference: http://partners.adobe.com/public/developer/en/font/5178.PFM.pdf
# Modified by: Joerg Jenderek
# Note: moved from ./msdos magic
# dfVersion 256=0100h
0 uleshort 0x0100
# GRR: line above is too general as it catches also TrueType font,
# raw G3 data FAX, WhatsApp encrypted and Panorama database
# dfType 129=0081h
>66 uleshort 0x0081
# dfVertRes 300=012Ch not needed as additional test
#>>70 uleshort 0x012c
# dfHorizRes 300=012Ch
#>>>72 uleshort 0x012c
# dfDriverInfo points to postscript information section
>>(101.l) string/c Postscript Printer Font Metrics
# above labeled "PFM data" by ./msdos (version 5.28) or "Adobe Printer Font Metrics" by TrID
!:mime application/x-font-pfm
# AppleShare Print Server
#!:apple ASPS????
!:ext pfm
# dfCopyright 60 byte null padded Copyright string. uncomment it to get old looking
#>>>6 string >\060 - %-.60s
# dfDriverInfo
>>>139 ulelong >0
# often abbreviated and same as filename
>>>>(139.l) string x %s
# dfSize
>>>2 ulelong x \b, %d bytes
# dfFace 210=D2h 9Eh
>>>105 ulelong >0
# Windows font name
>>>>(105.l) string x \b, %s
# dfItalic
>>>80 ubyte 1 italic
# dfUnderline
>>>81 ubyte 1 underline
# dfStrikeOut
>>>82 ubyte 1 strikeout
# dfWeight 400=0x0190 300=0x012c 500=0x01f4 600=0x0258 700=0x02bc
>>>83 uleshort >699 bold
# dfPitchAndFamily 16 17 48 49 64 65
>>>90 ubyte 16 serif
>>>90 ubyte 17 serif proportional
#>>>90 ubyte 48 other
>>>90 ubyte 49 proportional
>>>90 ubyte 64 script
>>>90 ubyte 65 script proportional
# X11 font files in SNF (Server Natural Format) format
# updated by Joerg Jenderek at Feb 2013
# http://computer-programming-forum.com/51-perl/8f22fb96d2e34bab.htm
@ -110,7 +160,12 @@
>14 string x version %s
# EOT
34 string LP Embedded OpenType (EOT)
0x40 string \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0
>0x22 string LP Embedded OpenType (EOT)
# workaround until there's lepstring16
# >>0x52 lepstring16/h >\0 \b, %s family
>>0x52 short !0
>>>0x54 lestring16 x \b, %s family
!:mime application/vnd.ms-fontobject
# Web Open Font Format (.woff)

6
contrib/file/magic/Magdir/gcc
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: gcc,v 1.4 2009/09/19 16:28:09 christos Exp $
# $File: gcc,v 1.5 2016/07/01 23:31:13 christos Exp $
# gcc: file(1) magic for GCC special files
#
0 string gpch GCC precompiled header
@ -12,6 +12,6 @@
# 67 = 'C', 111 = 'o', 43 = '+', 79 = 'O'
>4 byte 67 for C
>4 byte 111 for Objective C
>4 byte 111 for Objective-C
>4 byte 43 for C++
>4 byte 79 for Objective C++
>4 byte 79 for Objective-C++

9
contrib/file/magic/Magdir/gnu
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: gnu,v 1.16 2015/04/19 22:59:25 christos Exp $
# $File: gnu,v 1.17 2016/07/16 22:17:04 christos Exp $
# gnu: file(1) magic for various GNU tools
#
# GNU nlsutils message catalog file format
@ -34,8 +34,11 @@
# This magic is not particularly good, as the keyrings don't have true
# magic. Nevertheless, it covers many keyrings.
0 beshort 0x9901 GPG key public ring
!:mime application/x-gnupg-keyring
0 ubeshort-0x9901 <2
>3 byte 4
>>4 bedate x GPG key public ring, created %s
!:mime application/x-gnupg-keyring
# Symmetric encryption
0 leshort 0x0d8c

32
contrib/file/magic/Magdir/images
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: images,v 1.116 2016/03/23 15:29:20 christos Exp $
# $File: images,v 1.117 2016/07/05 19:12:21 christos Exp $
# images: file(1) magic for image formats (see also "iff", and "c-lang" for
# XPM bitmaps)
#
@ -171,38 +171,44 @@
>>>&0 regex =[0-9]{1,50} \b %s
0 search/1 P1
>0 use netpbm
>>0 string x \b, bitmap
>0 regex/4 P1\\s
>>0 use netpbm
>>>0 string x \b, bitmap
!:strength + 45
!:mime image/x-portable-bitmap
0 search/1 P2
>0 use netpbm
>>0 string x \b, greymap
>0 regex/4 P2\\s
>>0 use netpbm
>>>0 string x \b, greymap
!:strength + 45
!:mime image/x-portable-greymap
0 search/1 P3
>0 use netpbm
>>0 string x \b, pixmap
>0 regex/4 P3\\s
>>0 use netpbm
>>>0 string x \b, pixmap
!:strength + 45
!:mime image/x-portable-pixmap
0 string P4
>0 use netpbm
>>0 string x \b, rawbits, bitmap
>0 regex/4 P4\\s
>>0 use netpbm
>>>0 string x \b, rawbits, bitmap
!:strength + 45
!:mime image/x-portable-bitmap
0 string P5
>0 use netpbm
>>0 string x \b, rawbits, greymap
>0 regex/4 P5\\s
>>0 use netpbm
>>>0 string x \b, rawbits, greymap
!:strength + 45
!:mime image/x-portable-greymap
0 string P6
>0 use netpbm
>>0 string x \b, rawbits, pixmap
>0 regex/4 P6\\s
>>0 use netpbm
>>>0 string x \b, rawbits, pixmap
!:strength + 45
!:mime image/x-portable-pixmap

9
contrib/file/magic/Magdir/jpeg
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: jpeg,v 1.29 2015/04/10 15:36:02 christos Exp $
# $File: jpeg,v 1.30 2016/07/04 15:18:23 christos Exp $
# JPEG images
# SunOS 5.5.1 had
#
@ -117,3 +117,10 @@
# From: Mathieu Malaterre <mathieu.malaterre@gmail.com>
0 belong 0xff4fff51 JPEG 2000 codestream
45 beshort 0xff52
# JPEG extended range
0 string \x49\x49\xbc
>3 byte 1
>>4 lelong%2 0 JPEG-XR
!:mime image/jxr
!:ext jxr

230
contrib/file/magic/Magdir/msdos
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: msdos,v 1.106 2016/06/11 00:52:14 christos Exp $
# $File: msdos,v 1.111 2016/09/14 01:26:26 christos Exp $
# msdos: file(1) magic for MS-DOS files
#
@ -328,15 +328,6 @@
0 string \xffKEYB\ \ \ \0\0\0\0
>12 string \0\0\0\0`\004\360 MS-DOS KEYBoard Layout file
# .COM formats (Daniel Quinlan, quinlan@yggdrasil.com)
# Uncommenting only the first two lines will cover about 2/3 of COM files,
# but it isn't feasible to match all COM files since there must be at least
# two dozen different one-byte "magics".
# test too generic ?
0 byte 0xe9 DOS executable (COM)
>0x1FE leshort 0xAA55 \b, boot code
>6 string SFX\ of\ LHarc (%s)
# DOS device driver updated by Joerg Jenderek at May 2011
# http://maben.homeip.net/static/S100/IBM/software/DOS/DOS%20techref/CHAPTER.009
0 ulequad&0x07a0ffffffff 0xffffffff DOS executable (
@ -439,12 +430,37 @@
# byte 0xeb conflicts with "sequent" magic leshort 0xn2eb
0 ubeshort&0xeb8d >0xeb00
# DR-DOS STACKER.COM SCREATE.SYS missed
>0 byte 0xeb
>>0x1FE leshort 0xAA55 DOS executable (COM), boot code
>>85 string UPX DOS executable (COM), UPX compressed
>>4 string \ $ARX DOS executable (COM), ARX self-extracting archive
>>4 string \ $LHarc DOS executable (COM), LHarc self-extracting archive
>>0x20e string SFX\ by\ LARC DOS executable (COM), LARC self-extracting archive
0 name msdos-com
>0 byte x DOS executable (COM)
>6 string SFX\ of\ LHarc \b, %s
>0x1FE leshort 0xAA55 \b, boot code
>85 string UPX \b, UPX compressed
>4 string \ $ARX \b, ARX self-extracting archive
>4 string \ $LHarc \b, LHarc self-extracting archive
>0x20e string SFX\ by\ LARC \b, LARC self-extracting archive
# JMP 8bit
0 byte 0xeb
# allow forward jumps only
>1 byte >-1
# that offset must be accessible
>>(1.b+2) byte x
>>>0 use msdos-com
# JMP 16bit
0 byte 0xe9
# forward jumps
>1 short >-1
# that offset must be accessible
>>(1.s+3) byte x
>>>0 use msdos-com
# negative offset, must not lead into PSP
>1 short <-259
# that offset must be accessible
>>(1,s+65539) byte x
>>>0 use msdos-com
# updated by Joerg Jenderek at Oct 2008,2015
# following line is too general
0 ubyte 0xb8
@ -578,17 +594,158 @@
0 string/b \x09\x04\x06\x00\x00\x00\x10\x00 Microsoft Excel Worksheet
!:mime application/vnd.ms-excel
#
0 belong 0x00001a00 Lotus 1-2-3
!:mime application/x-123
>4 belong 0x00100400 wk3 document data
>4 belong 0x02100400 wk4 document data
>4 belong 0x07800100 fm3 or fmb document data
>4 belong 0x07800000 fm3 or fmb document data
# Update: Joerg Jenderek
# URL: https://en.wikipedia.org/wiki/Lotus_1-2-3
# Reference: http://www.aboutvb.de/bas/formate/pdf/wk3.pdf
# Note: newer Lotus versions >2 use longer BOF record
# record type (BeginningOfFile=0000h) + length (001Ah)
0 belong 0x00001a00
# reserved should be 0h but 8c0dh for TUTMAC.WK3, 5h for SAMPADNS.WK3, 1h for a_readme.wk3, 1eh for K&G86.WK3
#>18 uleshort&0x73E0 0
# Lotus Multi Byte Character Set (LMBCS=1-31)
>20 ubyte >0
>>20 ubyte <32 Lotus 1-2-3
#!:mime application/x-123
!:mime application/vnd.lotus-1-2-3
!:apple ????L123
# (version 5.26) labeled the entry as "Lotus 1-2-3 wk3 document data"
>>>4 uleshort 0x1000 WorKsheet, version 3
!:ext wk3
# (version 5.26) labeled the entry as "Lotus 1-2-3 wk4 document data"
>>>4 uleshort 0x1002 WorKsheet, version 4
# also worksheet template 4 (.wt4)
!:ext wk4/wt4
# no example or documentation for wk5
#>>4 uleshort 0x???? WorKsheet, version 4
#!:ext wk5
# only MacrotoScript.123 example
>>>4 uleshort 0x1003 WorKsheet, version 97
# also worksheet template Smartmaster (.12M)?
!:ext 123
# only Set_Y2K.123 example
>>>4 uleshort 0x1005 WorKsheet, version 9.8 Millennium
!:ext 123
# no example for this version
>>>4 uleshort 0x8001 FoRMatting data
!:ext frm
# (version 5.26) labeled the entry as "Lotus 1-2-3 fm3 or fmb document data"
# TrID labeles the entry as "Formatting Data for Lotus 1-2-3 worksheet"
>>>4 uleshort 0x8007 ForMatting data, version 3
!:ext fm3
>>>4 default x unknown
# file revision sub code 0004h for worksheets
>>>>6 uleshort =0x0004 worksheet
!:ext wXX
>>>>6 uleshort !0x0004 formatting data
!:ext fXX
# main revision number
>>>>4 uleshort x \b, revision 0x%x
>>>6 uleshort =0x0004 \b, cell range
# active cellcoord range (start row, page,column ; end row, page, column)
# start values normally 0~1st sheet A1
>>>>8 ulelong !0
>>>>>10 ubyte >0 \b%d*
>>>>>8 uleshort x \b%d,
>>>>>11 ubyte x \b%d-
# end page mostly 0
>>>>14 ubyte >0 \b%d*
# end raw, column normally not 0
>>>>12 uleshort x \b%d,
>>>>15 ubyte x \b%d
# Lotus Multi Byte Character Set (1~cp850,2~cp851,...,16~japan,...,31~??)
>>>>20 ubyte >1 \b, character set 0x%x
# flags
>>>>21 ubyte x \b, flags 0x%x
>>>6 uleshort !0x0004
# record type (FONTNAME=00AEh)
>>>>30 search/29 \0\xAE
# variable length m (2) + entries (1) + ?? (1) + LCMBS string (n)
>>>>>&4 string >\0 \b, 1st font "%s"
#
0 belong 0x00000200 Lotus 1-2-3
!:mime application/x-123
>4 belong 0x06040600 wk1 document data
>4 belong 0x06800200 fmt document data
# Update: Joerg Jenderek
# URL: http://fileformats.archiveteam.org/wiki/Lotus_1-2-3
# Reference: http://www.schnarff.com/file-formats/lotus-1-2-3/WSFF2.TXT
# Note: Used by both old Lotus 1-2-3 and Lotus Symphony (DOS) til version 2.x
# record type (BeginningOfFile=0000h) + length (0002h)
0 belong 0x00000200
# GRR: line above is too general as it catches also MS Windows CURsor
# to display MS Windows cursor (strength=70) before Lotus 1-2-3 (strength=70-1)
!:strength -1
# skip Windows cursors with image height <256 and keep Lotus with low opcode 0001-0083h
>7 ubyte 0
# skip Windows cursors with image width 256 and keep Lotus with positiv opcode
>>6 ubyte >0 Lotus
# !:mime application/x-123
!:mime application/vnd.lotus-1-2-3
!:apple ????L123
# revision number (0404h = 123 1A, 0405h = Lotus Symphony , 0406h = 123 2.x wk1 , 8006h = fmt , ...)
# undocumented; (version 5.26) labeled the configurations as "Lotus 1-2-3"
>>>4 uleshort 0x0007 1-2-3 CoNFiguration, version 2.x (PGRAPH.CNF)
!:ext cnf
>>>4 uleshort 0x0C05 1-2-3 CoNFiguration, version 2.4J
!:ext cnf
>>>4 uleshort 0x0801 1-2-3 CoNFiguration, version 1-2.1
!:ext cnf
>>>4 uleshort 0x0802 Symphony CoNFiguration
!:ext cnf
>>>4 uleshort 0x0804 1-2-3 CoNFiguration, version 2.2
!:ext cnf
>>>4 uleshort 0x080A 1-2-3 CoNFiguration, version 2.3-2.4
!:ext cnf
>>>4 uleshort 0x1402 1-2-3 CoNFiguration, version 3.x
!:ext cnf
>>>4 uleshort 0x1450 1-2-3 CoNFiguration, version 4.x
!:ext cnf
# (version 5.26) labeled the entry as "Lotus 123"
# TrID labeles the entry as "Lotus 123 Worksheet (generic)"
>>>4 uleshort 0x0404 1-2-3 WorKSheet, version 1
# extension "wks" also for Microsoft Works document
!:ext wks
# (version 5.26) labeled the entry as "Lotus 123"
# TrID labeles the entry as "Lotus 123 Worksheet (generic)"
>>>4 uleshort 0x0405 Symphony WoRksheet, version 1.0
!:ext wrk/wr1
# (version 5.26) labeled the entry as "Lotus 1-2-3 wk1 document data"
# TrID labeles the entry as "Lotus 123 Worksheet (V2)"
>>>4 uleshort 0x0406 1-2-3/Symphony worksheet, version 2
# Symphony (.wr1)
!:ext wk1/wr1
# no example for this japan version
>>>4 uleshort 0x0600 1-2-3 WorKsheet, version 1.xJ
!:ext wj1
# no example or documentation for wk2
#>>>4 uleshort 0x???? 1-2-3 WorKsheet, version 2
#!:ext wk2
# undocumented japan version
>>>4 uleshort 0x0602 1-2-3 worksheet, version 2.4J
!:ext wj3
# (version 5.26) labeled the entry as "Lotus 1-2-3 fmt document data"
>>>4 uleshort 0x8006 1-2-3 ForMaTting data, version 2.x
# japan version 2.4J (fj3)
!:ext fmt/fj3
# no example for this version
>>>4 uleshort 0x8007 1-2-3 FoRMatting data, version 2.0
!:ext frm
# (version 5.26) labeled the entry as "Lotus 1-2-3"
>>>4 default x unknown worksheet or configuration
!:ext cnf
>>>>4 uleshort x \b, revision 0x%x
# 2nd record for most worksheets describes cells range
>>>6 use lotus-cells
# 3nd record for most japan worksheets describes cells range
>>>(8.s+10) use lotus-cells
# check and then display Lotus worksheet cells range
0 name lotus-cells
# look for type (RANGE=0006h) + length (0008h) at record begin
>0 ubelong 0x06000800 \b, cell range
# cell range (start column, row, end column, row) start values normally 0,0~A1 cell
>>4 ulong !0
>>>4 uleshort x \b%d,
>>>6 uleshort x \b%d-
# end of cell range
>>8 uleshort x \b%d,
>>10 uleshort x \b%d
# EndOfLotus123
0 string/b WordPro\0 Lotus WordPro
!:mime application/vnd.lotus-wordpro
0 string/b WordPro\r\373 Lotus WordPro
@ -738,24 +895,6 @@
0 lelong 0x00000005
>12 lelong 0x00000320 Windows Recycle Bin INFO2 file (Win2k - WinXP)
##### put in Either Magic/font or Magic/news
# Acroread or something files wrongly identified as G3 .pfm
# these have the form \000 \001 any? \002 \000 \000
# or \000 \001 any? \022 \000 \000
0 belong&0xffff00ff 0x00010012 PFM data
>4 string \000\000
>6 string >\060 - %s
0 belong&0xffff00ff 0x00010002 PFM data
>4 string \000\000
>6 string >\060 - %s
#0 string \000\001 pfm?
#>3 string \022\000\000Copyright\ yes
#>3 string \002\000\000Copyright\ yes
#>3 string >\0 oops, not a font file. Cancel that.
#it clashes with ttf files so put it lower down.
# From Doug Lee via a FreeBSD pr
9 string GERBILDOC First Choice document
9 string GERBILDB First Choice database
@ -940,7 +1079,8 @@
# Type: Microsoft Document Imaging Format (.mdi)
# URL: http://en.wikipedia.org/wiki/Microsoft_Document_Imaging_Format
# From: Daniele Sempione <scrows@oziosi.org>
0 short 0x5045 Microsoft Document Imaging Format
# Too weak (EP)
#0 short 0x5045 Microsoft Document Imaging Format
# MS eBook format (.lit)
0 string/b ITOLITLS Microsoft Reader eBook Data

59
contrib/file/magic/Magdir/pgp
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: pgp,v 1.11 2014/11/11 21:32:38 christos Exp $
# $File: pgp,v 1.12 2016/10/07 20:22:12 christos Exp $
# pgp: file(1) magic for Pretty Good Privacy
# see http://lists.gnupg.org/pipermail/gnupg-devel/1999-September/016052.html
#
@ -194,6 +194,23 @@
>0 byte 0x0a SHA512
>0 byte 0x0b SHA224
# display public key algorithms as human readable text
0 name key_algo
>0 byte 0x01 RSA (Encrypt or Sign)
# keep old look of version 5.28 without parentheses
>0 byte 0x02 RSA Encrypt-Only
>0 byte 0x03 RSA (Sign-Only)
>0 byte 16 ElGamal (Encrypt-Only)
>0 byte 17 DSA
>0 byte 18 Elliptic Curve
>0 byte 19 ECDSA
>0 byte 20 ElGamal (Encrypt or Sign)
>0 byte 21 Diffie-Hellman
>0 default x
>>0 ubyte <22 unknown (pub %d)
# this should never happen
>>0 ubyte >21 invalid (%d)
# pgp symmetric encrypted data
0 byte 0x8c PGP symmetric key encrypted data -
@ -465,5 +482,41 @@
>1 use pgpkey
0 byte 0x97 PGP Secret Sub-key -
>1 use pgpkey
0 byte 0x9d PGP Secret Sub-key -
>1 use pgpkey
0 byte 0x9d
# Update: Joerg Jenderek
# secret subkey packet (tag 7) with same structure as secret key packet (tag 5)
# skip Fetus.Sys16 CALIBUS.MAIN OrbFix.Sys16.Ex by looking for positive len
>1 ubeshort >0
#>1 ubeshort x \b, body length 0x%x
# next packet type often 88h,89h~(tag 2)~Signature Packet
#>>(1.S+3) ubyte x \b, next packet type 0x%x
# skip Dragon.SHR DEMO.INIT by looking for positive version
>>3 ubyte >0
# skip BUISSON.13 GUITAR1 by looking for low version number
>>>3 ubyte <5 PGP Secret Sub-key
# sub-key are normally part of secret key. So it does not occur as standalone file
#!:ext bin
# version 2,3~old 4~new . Comment following line for version 5.28 look
>>>>3 ubyte x (v%d)
>>>>3 ubyte x -
# old versions 2 or 3 but no real example found
>>>>3 ubyte <4
# 2 byte for key bits in version 5.28 look
>>>>>11 ubeshort x %db
>>>>>4 beldate x created on %s -
# old versions use 2 additional bytes after time stamp
#>>>>>8 ubeshort x 0x%x
# display key algorithm 1~RSA Encrypt|Sign - 21~Diffie-Hellman
>>>>>10 use key_algo
>>>>>(11.S/8) ubequad x
# look after first key
>>>>>>&5 use keyend
# new version
>>>>3 ubyte >3
>>>>>9 ubeshort x %db
>>>>>4 beldate x created on %s -
# display key algorithm
>>>>>8 use key_algo
>>>>>(9.S/8) ubequad x
# look after first key for something like s2k
>>>>>>&3 use keyend

14
contrib/file/magic/Magdir/psl Normal file
View File

@ -0,0 +1,14 @@
#------------------------------------------------------------------------------
# $File: psl,v 1.2 2016/07/14 17:34:27 christos Exp $
# psl: file(1) magic for Public Suffix List representations
# From: Daniel Kahn Gillmor <dkg@fifthhorseman.net>
# URL: https://publicsuffix.org
# see also: http://thread.gmane.org/gmane.network.dns.libpsl.bugs/162/focus=166
0 search/512 \n\n//\ ===BEGIN\ ICANN\ DOMAINS===\n\n Public Suffix List data
0 string .DAFSA@PSL_
>15 string \n Public Suffix List data (optimized)
>>11 byte >0x2f
>>>11 byte <0x3a (Version %c)

5
contrib/file/magic/Magdir/python
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: python,v 1.28 2015/09/16 22:19:54 christos Exp $
# $File: python,v 1.29 2016/07/27 09:42:16 rrt Exp $
# python: file(1) magic for python
#
# Outlook puts """ too for urgent messages
@ -9,7 +9,7 @@
0 string/t """ Python script text executable
# MAGIC as specified in Python/import.c (1.5 to 2.7a0 and 3.1a0, assuming
# that Py_UnicodeFlag is off for Python 2)
# 20121 ( YEAR - 1995 ) + MONTH + DAY (little endian followed by "\r\n"
# two bytes of magic followed by "\r\n" in little endian order
0 belong 0x994e0d0a python 1.5/1.6 byte-compiled
0 belong 0x87c60d0a python 2.0 byte-compiled
0 belong 0x2aeb0d0a python 2.1 byte-compiled
@ -24,6 +24,7 @@
0 belong 0x6c0c0d0a python 3.2 byte-compiled
0 belong 0x9e0c0d0a python 3.3 byte-compiled
0 belong 0xee0c0d0a python 3.4 byte-compiled
0 belong 0x160d0d0a python 3.5 byte-compiled
0 search/1/w #!\ /usr/bin/python Python script text executable
!:strength + 15

6
contrib/file/magic/Magdir/ruby
View File

@ -1,18 +1,22 @@
#------------------------------------------------------------------------------
# $File: ruby,v 1.5 2010/07/21 16:47:17 christos Exp $
# $File: ruby,v 1.6 2016/07/27 09:46:29 rrt Exp $
# ruby: file(1) magic for Ruby scripting language
# URL: http://www.ruby-lang.org/
# From: Reuben Thomas <rrt@sc3d.org>
# Ruby scripts
0 search/1/w #!\ /usr/bin/ruby Ruby script text executable
!:strength + 15
!:mime text/x-ruby
0 search/1/w #!\ /usr/local/bin/ruby Ruby script text executable
!:strength + 15
!:mime text/x-ruby
0 search/1 #!/usr/bin/env\ ruby Ruby script text executable
!:strength + 15
!:mime text/x-ruby
0 search/1 #!\ /usr/bin/env\ ruby Ruby script text executable
!:strength + 15
!:mime text/x-ruby
# What looks like ruby, but does not have a shebang

4
contrib/file/magic/Magdir/sgml
View File

@ -1,4 +1,4 @@
#------------------------------------------------------------------------------ # $File: sgml,v 1.33 2015/11/29 22:14:49 christos Exp $
#------------------------------------------------------------------------------ # $File: sgml,v 1.34 2016/09/11 13:56:42 christos Exp $
# Type: SVG Vectorial Graphics
# From: Noel Torres <tecnico@ejerciciosresueltos.com>
0 string \<?xml\ version="
@ -7,6 +7,8 @@
!:mime image/svg+xml
>>19 search/4096 \<gnc-v2 GnuCash file
!:mime application/x-gnucash
0 string \<svg SVG Scalable Vector Graphics image
!:mime image/svg
# Sitemap file
0 string/t \<?xml\ version="

71
contrib/file/magic/Magdir/sql
View File

@ -1,6 +1,6 @@
#------------------------------------------------------------------------------
# $File: sql,v 1.18 2015/12/04 20:38:43 christos Exp $
# $File: sql,v 1.20 2016/07/05 19:49:59 christos Exp $
# sql: file(1) magic for SQL files
#
# From: "Marty Leisner" <mleisner@eng.mc.xerox.com>
@ -10,25 +10,67 @@
#
0 beshort 0xfe01 MySQL table definition file
>2 byte x Version %d
0 belong&0xffffff00 0xfefe0700 MySQL MyISAM index file
>3 byte x Version %d
0 belong&0xffffff00 0xfefe0800 MySQL MyISAM compressed data file
>3 byte x Version %d
0 belong&0xffffff00 0xfefe0900 MySQL Maria index file
>3 byte x Version %d
0 belong&0xffffff00 0xfefe0A00 MySQL Maria compressed data file
>3 byte x Version %d
>3 byte 0 \b, type UNKNOWN
>3 byte 1 \b, type DIAM_ISAM
>3 byte 2 \b, type HASH
>3 byte 3 \b, type MISAM
>3 byte 4 \b, type PISAM
>3 byte 5 \b, type RMS_ISAM
>3 byte 6 \b, type HEAP
>3 byte 7 \b, type ISAM
>3 byte 8 \b, type MRG_ISAM
>3 byte 9 \b, type MYISAM
>3 byte 10 \b, type MRG_MYISAM
>3 byte 11 \b, type BERKELEY_DB
>3 byte 12 \b, type INNODB
>3 byte 13 \b, type GEMINI
>3 byte 14 \b, type NDBCLUSTER
>3 byte 15 \b, type EXAMPLE_DB
>3 byte 16 \b, type CSV_DB
>3 byte 17 \b, type FEDERATED_DB
>3 byte 18 \b, type BLACKHOLE_DB
>3 byte 19 \b, type PARTITION_DB
>3 byte 20 \b, type BINLOG
>3 byte 21 \b, type SOLID
>3 byte 22 \b, type PBXT
>3 byte 23 \b, type TABLE_FUNCTION
>3 byte 24 \b, type MEMCACHE
>3 byte 25 \b, type FALCON
>3 byte 26 \b, type MARIA
>3 byte 27 \b, type PERFORMANCE_SCHEMA
>3 byte 127 \b, type DEFAULT
>0x0033 ulong x \b, MySQL version %d
0 belong&0xffffff00 0xfefe0500 MySQL ISAM index file
>3 byte x Version %d
0 belong&0xffffff00 0xfefe0600 MySQL ISAM compressed data file
>3 byte x Version %d
0 string \376bin MySQL replication log
0 belong&0xffffff00 0xfefe0b00
>4 string MARIALOG MySQL Maria transaction log file
>>3 byte x Version %d
0 belong&0xffffff00 0xfefe0700 MySQL MyISAM index file
>3 byte x Version %d
>14 beshort x \b, %d key parts
>16 beshort x \b, %d unique key parts
>18 byte x \b, %d keys
>28 bequad x \b, %lld records
>36 bequad x \b, %lld deleted records
0 belong&0xffffff00 0xfefe0800 MySQL MyISAM compressed data file
>3 byte x Version %d
0 belong&0xffffff00 0xfefe0900 MySQL Maria index file
>3 byte x Version %d
0 belong&0xffffff00 0xfefe0a00 MySQL Maria compressed data file
>3 byte x Version %d
0 belong&0xffffff00 0xfefe0c00
>4 string MACF MySQL Maria control file
>>3 byte x Version %d
0 string \376bin MySQL replication log,
>9 long x server id %d
>8 byte 1
>>13 long 69 \b, MySQL V3.2.3
>>>19 string x \b, server version %s
>>13 long 75 \b, MySQL V4.0.2-V4.1
>>>25 string x \b, server version %s
>8 byte 15 MySQL V5+,
>>25 string x server version %s
>4 string MARIALOG MySQL Maria transaction log file
>>3 byte x Version %d
#------------------------------------------------------------------------------
# iRiver H Series database file
@ -94,3 +136,6 @@
>126 string SQLite\ format\ 3
#!:mime application/x-panasonic-sqlite3
>>&-15 indirect x \b; contains
# H2 Database from http://www.h2database.com/
0 string --\ H2\ 0.5/B\ --\ \n H2 Database file

6
contrib/file/magic/Makefile.am
View File

@ -1,5 +1,5 @@
#
# $File: Makefile.am,v 1.116 2016/06/13 19:09:31 christos Exp $
# $File: Makefile.am,v 1.120 2016/10/17 19:52:29 christos Exp $
#
MAGIC_FRAGMENT_BASE = Magdir
MAGIC_DIR = $(top_srcdir)/magic
@ -13,6 +13,7 @@ $(MAGIC_DIR)/Localstuff \
$(MAGIC_FRAGMENT_DIR)/acorn \
$(MAGIC_FRAGMENT_DIR)/adi \
$(MAGIC_FRAGMENT_DIR)/adventure \
$(MAGIC_FRAGMENT_DIR)/algol68 \
$(MAGIC_FRAGMENT_DIR)/allegro \
$(MAGIC_FRAGMENT_DIR)/alliant \
$(MAGIC_FRAGMENT_DIR)/amanda \
@ -22,7 +23,9 @@ $(MAGIC_FRAGMENT_DIR)/animation \
$(MAGIC_FRAGMENT_DIR)/aout \
$(MAGIC_FRAGMENT_DIR)/apl \
$(MAGIC_FRAGMENT_DIR)/apple \
$(MAGIC_FRAGMENT_DIR)/application \
$(MAGIC_FRAGMENT_DIR)/applix \
$(MAGIC_FRAGMENT_DIR)/apt \
$(MAGIC_FRAGMENT_DIR)/archive \
$(MAGIC_FRAGMENT_DIR)/assembler \
$(MAGIC_FRAGMENT_DIR)/asterix \
@ -207,6 +210,7 @@ $(MAGIC_FRAGMENT_DIR)/polyml \
$(MAGIC_FRAGMENT_DIR)/printer \
$(MAGIC_FRAGMENT_DIR)/project \
$(MAGIC_FRAGMENT_DIR)/psdbms \
$(MAGIC_FRAGMENT_DIR)/psl \
$(MAGIC_FRAGMENT_DIR)/pulsar \
$(MAGIC_FRAGMENT_DIR)/pwsafe \
$(MAGIC_FRAGMENT_DIR)/pyramid \

6
contrib/file/magic/Makefile.in
View File

@ -273,7 +273,7 @@ top_builddir = @top_builddir@
top_srcdir = @top_srcdir@
#
# $File: Makefile.am,v 1.116 2016/06/13 19:09:31 christos Exp $
# $File: Makefile.am,v 1.120 2016/10/17 19:52:29 christos Exp $
#
MAGIC_FRAGMENT_BASE = Magdir
MAGIC_DIR = $(top_srcdir)/magic
@ -285,6 +285,7 @@ $(MAGIC_DIR)/Localstuff \
$(MAGIC_FRAGMENT_DIR)/acorn \
$(MAGIC_FRAGMENT_DIR)/adi \
$(MAGIC_FRAGMENT_DIR)/adventure \
$(MAGIC_FRAGMENT_DIR)/algol68 \
$(MAGIC_FRAGMENT_DIR)/allegro \
$(MAGIC_FRAGMENT_DIR)/alliant \
$(MAGIC_FRAGMENT_DIR)/amanda \
@ -294,7 +295,9 @@ $(MAGIC_FRAGMENT_DIR)/animation \
$(MAGIC_FRAGMENT_DIR)/aout \
$(MAGIC_FRAGMENT_DIR)/apl \
$(MAGIC_FRAGMENT_DIR)/apple \
$(MAGIC_FRAGMENT_DIR)/application \
$(MAGIC_FRAGMENT_DIR)/applix \
$(MAGIC_FRAGMENT_DIR)/apt \
$(MAGIC_FRAGMENT_DIR)/archive \
$(MAGIC_FRAGMENT_DIR)/assembler \
$(MAGIC_FRAGMENT_DIR)/asterix \
@ -479,6 +482,7 @@ $(MAGIC_FRAGMENT_DIR)/polyml \
$(MAGIC_FRAGMENT_DIR)/printer \
$(MAGIC_FRAGMENT_DIR)/project \
$(MAGIC_FRAGMENT_DIR)/psdbms \
$(MAGIC_FRAGMENT_DIR)/psl \
$(MAGIC_FRAGMENT_DIR)/pulsar \
$(MAGIC_FRAGMENT_DIR)/pwsafe \
$(MAGIC_FRAGMENT_DIR)/pyramid \

6
contrib/file/python/magic.py
View File

@ -134,7 +134,7 @@ def file(self, filename):
if isinstance(r, str):
return r
else:
return str(r).encode('utf-8')
return str(r, 'utf-8')
def descriptor(self, fd):
"""
@ -152,7 +152,7 @@ def buffer(self, buf):
if isinstance(r, str):
return r
else:
return str(r).encode('utf-8')
return str(r, 'utf-8')
def error(self):
"""
@ -163,7 +163,7 @@ def error(self):
if isinstance(e, str):
return e
else:
return str(e).encode('utf-8')
return str(e, 'utf-8')
def setflags(self, flags):
"""

37
contrib/file/src/apprentice.c
View File

@ -32,7 +32,7 @@
#include "file.h"
#ifndef lint
FILE_RCSID("@(#)$File: apprentice.c,v 1.249 2016/05/17 21:43:07 christos Exp $")
FILE_RCSID("@(#)$File: apprentice.c,v 1.255 2016/10/24 18:02:17 christos Exp $")
#endif /* lint */
#include "magic.h"
@ -408,11 +408,11 @@ add_mlist(struct mlist *mlp, struct magic_map *map, size_t idx)
{
struct mlist *ml;
mlp->map = idx == 0 ? map : NULL;
mlp->map = NULL;
if ((ml = CAST(struct mlist *, malloc(sizeof(*ml)))) == NULL)
return -1;
ml->map = NULL;
ml->map = idx == 0 ? map : NULL;
ml->magic = map->magic[idx];
ml->nmagic = map->nmagic[idx];
@ -451,6 +451,8 @@ apprentice_1(struct magic_set *ms, const char *fn, int action)
#ifndef COMPILE_ONLY
map = apprentice_map(ms, fn);
if (map == (struct magic_map *)-1)
return -1;
if (map == NULL) {
if (ms->flags & MAGIC_CHECK)
file_magwarn(ms, "using regular magic file `%s'", fn);
@ -462,7 +464,7 @@ apprentice_1(struct magic_set *ms, const char *fn, int action)
for (i = 0; i < MAGIC_SETS; i++) {
if (add_mlist(ms->mlist[i], map, i) == -1) {
file_oomem(ms, sizeof(*ml));
goto fail;
return -1;
}
}
@ -476,12 +478,6 @@ apprentice_1(struct magic_set *ms, const char *fn, int action)
}
}
return 0;
fail:
for (i = 0; i < MAGIC_SETS; i++) {
mlist_free(ms->mlist[i]);
ms->mlist[i] = NULL;
}
return -1;
#else
return 0;
#endif /* COMPILE_ONLY */
@ -554,7 +550,7 @@ apprentice_unmap(struct magic_map *map)
case MAP_TYPE_MALLOC:
for (i = 0; i < MAGIC_SETS; i++) {
if ((char *)map->magic[i] >= (char *)map->p &&
(char *)map->magic[i] < (char *)map->p + map->len)
(char *)map->magic[i] <= (char *)map->p + map->len)
continue;
free(map->magic[i]);
}
@ -594,7 +590,7 @@ mlist_free(struct mlist *mlist)
ml = mlist->next;
for (ml = mlist->next; (next = ml->next) != NULL; ml = next) {
if (ml->map)
apprentice_unmap(ml->map);
apprentice_unmap(CAST(struct magic_map *, ml->map));
free(ml);
if (ml == mlist)
break;
@ -1879,10 +1875,13 @@ parse(struct magic_set *ms, struct magic_entry *me, const char *line,
if (m->flag & INDIR) {
m->in_type = FILE_LONG;
m->in_offset = 0;
m->in_op = 0;
/*
* read [.lbs][+-]nnnnn)
* read [.,lbs][+-]nnnnn)
*/
if (*l == '.') {
if (*l == '.' || *l == ',') {
if (*l == ',')
m->in_op |= FILE_OPSIGNED;
l++;
switch (*l) {
case 'l':
@ -1934,7 +1933,6 @@ parse(struct magic_set *ms, struct magic_entry *me, const char *line,
l++;
}
m->in_op = 0;
if (*l == '~') {
m->in_op |= FILE_OPINVERSE;
l++;
@ -2930,6 +2928,7 @@ apprentice_map(struct magic_set *ms, const char *fn)
struct stat st;
char *dbname = NULL;
struct magic_map *map;
struct magic_map *rv = NULL;
fd = -1;
if ((map = CAST(struct magic_map *, calloc(1, sizeof(*map)))) == NULL) {
@ -2978,8 +2977,10 @@ apprentice_map(struct magic_set *ms, const char *fn)
(void)close(fd);
fd = -1;
if (check_buffer(ms, map, dbname) != 0)
if (check_buffer(ms, map, dbname) != 0) {
rv = (struct magic_map *)-1;
goto error;
}
#ifdef QUICK
if (mprotect(map->p, (size_t)st.st_size, PROT_READ) == -1) {
file_error(ms, errno, "cannot mprotect `%s'", dbname);
@ -2995,7 +2996,7 @@ apprentice_map(struct magic_set *ms, const char *fn)
(void)close(fd);
apprentice_unmap(map);
free(dbname);
return NULL;
return rv;
}
private int
@ -3151,7 +3152,7 @@ mkdbname(struct magic_set *ms, const char *fn, int strip)
return NULL;
/* Compatibility with old code that looked in .mime */
if (strstr(p, ".mime") != NULL)
if (strstr(fn, ".mime") != NULL)
ms->flags &= MAGIC_MIME_TYPE;
return buf;
}

8
contrib/file/src/ascmagic.c
View File

@ -35,7 +35,7 @@
#include "file.h"
#ifndef lint
FILE_RCSID("@(#)$File: ascmagic.c,v 1.95 2016/05/03 16:10:37 christos Exp $")
FILE_RCSID("@(#)$File: ascmagic.c,v 1.97 2016/06/27 20:56:25 christos Exp $")
#endif /* lint */
#include "magic.h"
@ -144,8 +144,10 @@ file_ascmagic_with_encoding(struct magic_set *ms, const unsigned char *buf,
(size_t)(utf8_end - utf8_buf), NULL, NULL,
TEXTTEST, text)) == 0)
rv = -1;
if ((ms->flags & (MAGIC_APPLE|MAGIC_EXTENSION)))
return rv == -1 ? 0 : 1;
if ((ms->flags & (MAGIC_APPLE|MAGIC_EXTENSION))) {
rv = rv == -1 ? 0 : 1;
goto done;
}
}
if ((ms->flags & (MAGIC_APPLE|MAGIC_EXTENSION)))
return 0;

32
contrib/file/src/cdf.c
View File

@ -35,7 +35,7 @@
#include "file.h"
#ifndef lint
FILE_RCSID("@(#)$File: cdf.c,v 1.82 2016/06/01 22:25:25 christos Exp $")
FILE_RCSID("@(#)$File: cdf.c,v 1.85 2016/10/24 18:02:17 christos Exp $")
#endif
#include <assert.h>
@ -266,7 +266,7 @@ cdf_unpack_dir(cdf_directory_t *d, char *buf)
CDF_UNPACK(d->d_unused0);
}
static int
int
cdf_zero_stream(cdf_stream_t *scn)
{
scn->sst_len = 0;
@ -731,22 +731,25 @@ cdf_read_short_stream(const cdf_info_t *info, const cdf_header_t *h,
break;
/* If the it is not there, just fake it; some docs don't have it */
if (i == dir->dir_len)
if (i == dir->dir_len) {
DPRINTF(("Cannot find root storage dir\n"));
goto out;
}
d = &dir->dir_tab[i];
*root = d;
/* If the it is not there, just fake it; some docs don't have it */
if (d->d_stream_first_sector < 0)
if (d->d_stream_first_sector < 0) {
DPRINTF(("No first secror in dir\n"));
goto out;
}
return cdf_read_long_sector_chain(info, h, sat,
d->d_stream_first_sector, d->d_size, scn);
out:
scn->sst_tab = NULL;
(void)cdf_zero_stream(scn);
errno = EFTYPE;
return -1;
return 0;
}
static int
@ -758,6 +761,15 @@ cdf_namecmp(const char *d, const uint16_t *s, size_t l)
return 0;
}
int
cdf_read_doc_summary_info(const cdf_info_t *info, const cdf_header_t *h,
const cdf_sat_t *sat, const cdf_sat_t *ssat, const cdf_stream_t *sst,
const cdf_dir_t *dir, cdf_stream_t *scn)
{
return cdf_read_user_stream(info, h, sat, ssat, sst, dir,
"\05DocumentSummaryInformation", scn);
}
int
cdf_read_summary_info(const cdf_info_t *info, const cdf_header_t *h,
const cdf_sat_t *sat, const cdf_sat_t *ssat, const cdf_stream_t *sst,
@ -1098,7 +1110,7 @@ cdf_unpack_catalog(const cdf_header_t *h, const cdf_stream_t *sst,
cep->ce_namlen = rlen;
np = CAST(const uint16_t *, CAST(const void *, (b + 16)));
if (CAST(const char *, np + cep->ce_namlen) > eb) {
if (RCAST(const char *, np + cep->ce_namlen) > eb) {
cep->ce_namlen = 0;
break;
}
@ -1275,7 +1287,7 @@ cdf_dump(const void *v, size_t len)
}
void
cdf_dump_stream(const cdf_header_t *h, const cdf_stream_t *sst)
cdf_dump_stream(const cdf_stream_t *sst)
{
size_t ss = sst->sst_ss;
cdf_dump(sst->sst_tab, ss * sst->sst_len);
@ -1331,7 +1343,7 @@ cdf_dump_dir(const cdf_info_t *info, const cdf_header_t *h,
name, d->d_stream_first_sector, d->d_size);
break;
}
cdf_dump_stream(h, &scn);
cdf_dump_stream(&scn);
free(scn.sst_tab);
break;
default:
@ -1507,7 +1519,7 @@ main(int argc, char *argv[])
== -1)
err(1, "Cannot read short stream");
#ifdef CDF_DEBUG
cdf_dump_stream(&h, &sst);
cdf_dump_stream(&sst);
#endif
#ifdef CDF_DEBUG

6
contrib/file/src/cdf.h
View File

@ -316,6 +316,10 @@ int cdf_read_user_stream(const cdf_info_t *, const cdf_header_t *,
const cdf_sat_t *, const cdf_sat_t *, const cdf_stream_t *,
const cdf_dir_t *, const char *, cdf_stream_t *);
int cdf_find_stream(const cdf_dir_t *, const char *, int);
int cdf_zero_stream(cdf_stream_t *);
int cdf_read_doc_summary_info(const cdf_info_t *, const cdf_header_t *,
const cdf_sat_t *, const cdf_sat_t *, const cdf_stream_t *,
const cdf_dir_t *, cdf_stream_t *);
int cdf_read_summary_info(const cdf_info_t *, const cdf_header_t *,
const cdf_sat_t *, const cdf_sat_t *, const cdf_stream_t *,
const cdf_dir_t *, cdf_stream_t *);
@ -336,7 +340,7 @@ char *cdf_u16tos8(char *, size_t, const uint16_t *);
void cdf_dump_header(const cdf_header_t *);
void cdf_dump_sat(const char *, const cdf_sat_t *, size_t);
void cdf_dump(const void *, size_t);
void cdf_dump_stream(const cdf_header_t *, const cdf_stream_t *);
void cdf_dump_stream(const cdf_stream_t *);
void cdf_dump_dir(const cdf_info_t *, const cdf_header_t *, const cdf_sat_t *,
const cdf_sat_t *, const cdf_stream_t *, const cdf_dir_t *);
void cdf_dump_property_info(const cdf_property_info_t *, size_t);

14
contrib/file/src/compress.c
View File

@ -35,7 +35,7 @@
#include "file.h"
#ifndef lint
FILE_RCSID("@(#)$File: compress.c,v 1.97 2016/05/13 23:02:28 christos Exp $")
FILE_RCSID("@(#)$File: compress.c,v 1.100 2016/10/24 18:02:17 christos Exp $")
#endif
#include "magic.h"
@ -132,6 +132,9 @@ static const char *lrzip_args[] = {
static const char *lz4_args[] = {
"lz4", "-cd", NULL
};
static const char *zstd_args[] = {
"zstd", "-cd", NULL
};
private const struct {
const void *magic;
@ -154,8 +157,9 @@ private const struct {
{ "\3757zXZ\0", 6, xz_args }, /* XZ Utils */
{ "LRZI", 4, lrzip_args }, /* LRZIP */
{ "\004\"M\030",4, lz4_args }, /* LZ4 */
{ "\x28\xB5\x2F\xFD", 4, zstd_args }, /* zstd */
#ifdef ZLIBSUPPORT
{ zlibcmp, 0, zlib_args }, /* zlib */
{ RCAST(const void *, zlibcmp), 0, zlib_args }, /* zlib */
#endif
};
@ -204,7 +208,7 @@ file_zmagic(struct magic_set *ms, int fd, const char *name,
continue;
#ifdef ZLIBSUPPORT
if (compr[i].maglen == 0)
zm = (CAST(int (*)(const unsigned char *),
zm = (RCAST(int (*)(const unsigned char *),
CCAST(void *, compr[i].magic)))(buf);
else
#endif
@ -362,7 +366,7 @@ sread(int fd, void *buf, size_t n, int canbepipe __attribute__((__unused__)))
return rn - n;
default:
n -= rv;
buf = ((char *)buf) + rv;
buf = CAST(char *, CCAST(void *, buf)) + rv;
break;
}
while (n > 0);
@ -517,7 +521,7 @@ uncompresszlib(const unsigned char *old, unsigned char **newch,
return OKDATA;
err:
strlcpy((char *)*newch, z.msg, bytes_max);
strlcpy((char *)*newch, z.msg ? z.msg : zError(rc), bytes_max);
*n = strlen((char *)*newch);
return ERRDATA;
}

14
contrib/file/src/der.c
View File

@ -35,13 +35,11 @@
#include "file.h"
#ifndef lint
FILE_RCSID("@(#)$File: der.c,v 1.7 2016/06/01 22:01:15 christos Exp $")
FILE_RCSID("@(#)$File: der.c,v 1.10 2016/10/24 18:02:17 christos Exp $")
#endif
#endif
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include <stdio.h>
#include <fcntl.h>
@ -53,6 +51,8 @@ FILE_RCSID("@(#)$File: der.c,v 1.7 2016/06/01 22:01:15 christos Exp $")
#include "magic.h"
#include "der.h"
#else
#include <sys/mman.h>
#include <sys/stat.h>
#include <err.h>
#endif
@ -181,6 +181,8 @@ getlength(const uint8_t *c, size_t *p, size_t l)
for (i = 0; i < digits; i++)
len = (len << 8) | c[(*p)++];
if (*p + len >= l)
return DER_BAD;
return len;
}
@ -198,7 +200,7 @@ der_tag(char *buf, size_t len, uint32_t tag)
static int
der_data(char *buf, size_t blen, uint32_t tag, const void *q, uint32_t len)
{
const uint8_t *d = q;
const uint8_t *d = CAST(const uint8_t *, q);
switch (tag) {
case DER_TAG_PRINTABLE_STRING:
case DER_TAG_UTF8_STRING:
@ -220,7 +222,7 @@ der_data(char *buf, size_t blen, uint32_t tag, const void *q, uint32_t len)
int32_t
der_offs(struct magic_set *ms, struct magic *m, size_t nbytes)
{
const uint8_t *b = CAST(const void *, ms->search.s);
const uint8_t *b = RCAST(const uint8_t *, ms->search.s);
size_t offs = 0, len = ms->search.s_len ? ms->search.s_len : nbytes;
if (gettag(b, &offs, len) == DER_BAD)
@ -251,7 +253,7 @@ der_offs(struct magic_set *ms, struct magic *m, size_t nbytes)
int
der_cmp(struct magic_set *ms, struct magic *m)
{
const uint8_t *b = CAST(const void *, ms->search.s);
const uint8_t *b = RCAST(const uint8_t *, ms->search.s);
const char *s = m->value.s;
size_t offs = 0, len = ms->search.s_len;
uint32_t tag, tlen;

6
contrib/file/src/file.c
View File

@ -32,7 +32,7 @@
#include "file.h"
#ifndef lint
FILE_RCSID("@(#)$File: file.c,v 1.171 2016/05/17 15:52:45 christos Exp $")
FILE_RCSID("@(#)$File: file.c,v 1.172 2016/10/24 15:21:07 christos Exp $")
#endif /* lint */
#include "magic.h"
@ -431,6 +431,8 @@ private struct magic_set *
load(const char *magicfile, int flags)
{
struct magic_set *magic = magic_open(flags);
const char *e;
if (magic == NULL) {
(void)fprintf(stderr, "%s: %s\n", progname, strerror(errno));
return NULL;
@ -441,6 +443,8 @@ load(const char *magicfile, int flags)
magic_close(magic);
return NULL;
}
if ((e = magic_error(magic)) != NULL)
(void)fprintf(stderr, "%s: Warning: %s\n", progname, e);
return magic;
}

10
contrib/file/src/file.h
View File

@ -27,7 +27,7 @@
*/
/*
* file.h - definitions for file(1) program
* @(#)$File: file.h,v 1.178 2016/03/31 17:51:12 christos Exp $
* @(#)$File: file.h,v 1.180 2016/07/20 11:27:08 christos Exp $
*/
#ifndef __file_h__
@ -134,11 +134,11 @@
or directory */
#define MAXDESC 64 /* max len of text description/MIME type */
#define MAXMIME 80 /* max len of text MIME type */
#define MAXstring 64 /* max len of "string" types */
#define MAXstring 96 /* max len of "string" types */
#define MAGICNO 0xF11E041C
#define VERSIONNO 13
#define FILE_MAGICSIZE 312
#define VERSIONNO 14
#define FILE_MAGICSIZE 344
#define FILE_LOAD 0
#define FILE_CHECK 1
@ -275,7 +275,7 @@ struct magic {
#define FILE_OPS_MASK 0x07 /* mask for above ops */
#define FILE_UNUSED_1 0x08
#define FILE_UNUSED_2 0x10
#define FILE_UNUSED_3 0x20
#define FILE_OPSIGNED 0x20
#define FILE_OPINVERSE 0x40
#define FILE_OPINDIRECT 0x80

5
contrib/file/src/funcs.c
View File

@ -27,7 +27,7 @@
#include "file.h"
#ifndef lint
FILE_RCSID("@(#)$File: funcs.c,v 1.89 2016/03/21 15:56:53 christos Exp $")
FILE_RCSID("@(#)$File: funcs.c,v 1.90 2016/10/19 20:51:17 christos Exp $")
#endif /* lint */
#include "magic.h"
@ -250,7 +250,7 @@ file_buffer(struct magic_set *ms, int fd, const char *inname __attribute__ ((__u
}
/* try soft magic tests */
if ((ms->flags & MAGIC_NO_CHECK_SOFT) == 0)
if ((ms->flags & MAGIC_NO_CHECK_SOFT) == 0) {
m = file_softmagic(ms, ubuf, nb, NULL, NULL, BINTEST,
looks_text);
if ((ms->flags & MAGIC_DEBUG) != 0)
@ -277,6 +277,7 @@ file_buffer(struct magic_set *ms, int fd, const char *inname __attribute__ ((__u
if (checkdone(ms, &rv))
goto done;
}
}
/* try text properties */
if ((ms->flags & MAGIC_NO_CHECK_TEXT) == 0) {

4
contrib/file/src/magic.c
View File

@ -33,7 +33,7 @@
#include "file.h"
#ifndef lint
FILE_RCSID("@(#)$File: magic.c,v 1.99 2016/05/03 16:09:38 christos Exp $")
FILE_RCSID("@(#)$File: magic.c,v 1.100 2016/07/18 11:43:05 christos Exp $")
#endif /* lint */
#include "magic.h"
@ -492,7 +492,7 @@ file_or_fd(struct magic_set *ms, const char *inname, int fd)
if (r < PIPE_BUF) break;
}
if (nbytes == 0) {
if (nbytes == 0 && inname) {
/* We can not read it, but we were able to stat it. */
if (unreadable_info(ms, sb.st_mode, inname) == -1)
goto done;

185
contrib/file/src/readcdf.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2008 Christos Zoulas
* Copyright (c) 2008, 2016 Christos Zoulas
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -26,7 +26,7 @@
#include "file.h"
#ifndef lint
FILE_RCSID("@(#)$File: readcdf.c,v 1.57 2016/05/03 16:08:49 christos Exp $")
FILE_RCSID("@(#)$File: readcdf.c,v 1.63 2016/10/18 22:25:42 christos Exp $")
#endif
#include <assert.h>
@ -373,13 +373,61 @@ cdf_file_catalog_info(struct magic_set *ms, const cdf_info_t *info,
dir, "Catalog", scn)) == -1)
return i;
#ifdef CDF_DEBUG
cdf_dump_catalog(&h, scn);
cdf_dump_catalog(h, scn);
#endif
if ((i = cdf_file_catalog(ms, h, scn)) == -1)
return -1;
return i;
}
private int
cdf_check_summary_info(struct magic_set *ms, const cdf_info_t *info,
const cdf_header_t *h, const cdf_sat_t *sat, const cdf_sat_t *ssat,
const cdf_stream_t *sst, const cdf_dir_t *dir, cdf_stream_t *scn,
const cdf_directory_t *root_storage, const char **expn)
{
int i;
const char *str = NULL;
cdf_directory_t *d;
char name[__arraycount(d->d_name)];
size_t j, k;
#ifdef CDF_DEBUG
cdf_dump_summary_info(h, scn);
#endif
if ((i = cdf_file_summary_info(ms, h, scn, root_storage)) < 0) {
*expn = "Can't expand summary_info";
return i;
}
if (i == 1)
return i;
for (j = 0; str == NULL && j < dir->dir_len; j++) {
d = &dir->dir_tab[j];
for (k = 0; k < sizeof(name); k++)
name[k] = (char)cdf_tole2(d->d_name[k]);
str = cdf_app_to_mime(name,
NOTMIME(ms) ? name2desc : name2mime);
}
if (NOTMIME(ms)) {
if (str != NULL) {
if (file_printf(ms, "%s", str) == -1)
return -1;
i = 1;
}
} else {
if (str == NULL)
str = "vnd.ms-office";
if (file_printf(ms, "application/%s", str) == -1)
return -1;
i = 1;
}
if (i <= 0) {
i = cdf_file_catalog_info(ms, info, h, sat, ssat, sst,
dir, scn);
}
return i;
}
private struct sinfo {
const char *name;
const char *mime;
@ -388,10 +436,13 @@ private struct sinfo {
} sectioninfo[] = {
{ "Encrypted", "encrypted",
{
"EncryptedPackage", NULL, NULL, NULL, NULL,
"EncryptedPackage", "EncryptedSummary",
NULL, NULL, NULL,
},
{
CDF_DIR_TYPE_USER_STREAM, 0, 0, 0, 0,
CDF_DIR_TYPE_USER_STREAM,
CDF_DIR_TYPE_USER_STREAM,
0, 0, 0,
},
},
@ -412,6 +463,46 @@ private struct sinfo {
0, 0, 0, 0
},
},
{ "Microsoft Excel", "vnd.ms-excel",
{
"Book", "Workbook", NULL, NULL, NULL,
},
{
CDF_DIR_TYPE_USER_STREAM,
CDF_DIR_TYPE_USER_STREAM,
0, 0, 0,
},
},
{ "Microsoft Word", "msword",
{
"WordDocument", NULL, NULL, NULL, NULL,
},
{
CDF_DIR_TYPE_USER_STREAM,
0, 0, 0, 0,
},
},
{ "Microsoft PowerPoint", "vnd.ms-powerpoint",
{
"PowerPoint", NULL, NULL, NULL, NULL,
},
{
CDF_DIR_TYPE_USER_STREAM,
0, 0, 0, 0,
},
},
{ "Microsoft Outlook Message", "vnd.ms-outlook",
{
"__properties_version1.0",
"__recip_version1.0_#00000000",
NULL, NULL, NULL,
},
{
CDF_DIR_TYPE_USER_STREAM,
CDF_DIR_TYPE_USER_STORAGE,
0, 0, 0,
},
},
};
private int
@ -423,22 +514,19 @@ cdf_file_dir_info(struct magic_set *ms, const cdf_dir_t *dir)
const struct sinfo *si = &sectioninfo[sd];
for (j = 0; si->sections[j]; j++) {
if (cdf_find_stream(dir, si->sections[j], si->types[j])
<= 0) {
#ifdef CDF_DEBUG
fprintf(stderr, "Can't read %s\n",
si->sections[j]);
#endif
> 0)
break;
}
#ifdef CDF_DEBUG
fprintf(stderr, "Can't read %s\n", si->sections[j]);
#endif
}
if (si->sections[j] != NULL)
if (si->sections[j] == NULL)
continue;
if (NOTMIME(ms)) {
if (file_printf(ms, "CDFV2 %s", si->name) == -1)
return -1;
} else {
if (file_printf(ms, "application/CDFV2-%s",
si->mime) == -1)
if (file_printf(ms, "application/%s", si->mime) == -1)
return -1;
}
return 1;
@ -459,6 +547,7 @@ file_trycdf(struct magic_set *ms, int fd, const unsigned char *buf,
const char *expn = "";
const cdf_directory_t *root_storage;
scn.sst_tab = NULL;
info.i_fd = fd;
info.i_buf = buf;
info.i_len = nbytes;
@ -528,10 +617,7 @@ file_trycdf(struct magic_set *ms, int fd, const unsigned char *buf,
i = 1;
goto out5;
} else {
free(scn.sst_tab);
scn.sst_tab = NULL;
scn.sst_len = 0;
scn.sst_dirlen = 0;
cdf_zero_stream(&scn);
}
}
@ -539,56 +625,31 @@ file_trycdf(struct magic_set *ms, int fd, const unsigned char *buf,
&scn)) == -1) {
if (errno != ESRCH) {
expn = "Cannot read summary info";
goto out4;
}
i = cdf_file_catalog_info(ms, &info, &h, &sat, &ssat, &sst,
&dir, &scn);
if (i > 0)
goto out4;
i = cdf_file_dir_info(ms, &dir);
if (i < 0)
expn = "Cannot read section info";
goto out4;
} else {
i = cdf_check_summary_info(ms, &info, &h,
&sat, &ssat, &sst, &dir, &scn, root_storage, &expn);
cdf_zero_stream(&scn);
}
#ifdef CDF_DEBUG
cdf_dump_summary_info(&h, &scn);
#endif
if ((i = cdf_file_summary_info(ms, &h, &scn, root_storage)) < 0)
expn = "Can't expand summary_info";
if (i == 0) {
const char *str = NULL;
cdf_directory_t *d;
char name[__arraycount(d->d_name)];
size_t j, k;
for (j = 0; str == NULL && j < dir.dir_len; j++) {
d = &dir.dir_tab[j];
for (k = 0; k < sizeof(name); k++)
name[k] = (char)cdf_tole2(d->d_name[k]);
str = cdf_app_to_mime(name,
NOTMIME(ms) ? name2desc : name2mime);
}
if (NOTMIME(ms)) {
if (str != NULL) {
if (file_printf(ms, "%s", str) == -1)
return -1;
i = 1;
if (i <= 0) {
if ((i = cdf_read_doc_summary_info(&info, &h, &sat, &ssat,
&sst, &dir, &scn)) == -1) {
if (errno != ESRCH) {
expn = "Cannot read summary info";
}
} else {
if (str == NULL)
str = "vnd.ms-office";
if (file_printf(ms, "application/%s", str) == -1)
return -1;
i = 1;
i = cdf_check_summary_info(ms, &info, &h, &sat, &ssat,
&sst, &dir, &scn, root_storage, &expn);
}
}
if (i <= 0) {
i = cdf_file_dir_info(ms, &dir);
if (i < 0)
expn = "Cannot read section info";
}
out5:
free(scn.sst_tab);
out4:
free(sst.sst_tab);
cdf_zero_stream(&scn);
cdf_zero_stream(&sst);
out3:
free(dir.dir_tab);
out2:
@ -605,7 +666,7 @@ file_trycdf(struct magic_set *ms, int fd, const unsigned char *buf,
if (file_printf(ms, ", %s", expn) == -1)
return -1;
} else {
if (file_printf(ms, "application/CDFV2-unknown") == -1)
if (file_printf(ms, "application/CDFV2") == -1)
return -1;
}
i = 1;

22
contrib/file/src/readelf.c
View File

@ -27,7 +27,7 @@
#include "file.h"
#ifndef lint
FILE_RCSID("@(#)$File: readelf.c,v 1.127 2015/11/18 12:29:29 christos Exp $")
FILE_RCSID("@(#)$File: readelf.c,v 1.128 2016/10/04 21:43:10 christos Exp $")
#endif
#ifdef BUILTIN_ELF
@ -509,12 +509,26 @@ do_bid_note(struct magic_set *ms, unsigned char *nbuf, uint32_t type,
size_t noff, size_t doff, int *flags)
{
if (namesz == 4 && strcmp((char *)&nbuf[noff], "GNU") == 0 &&
type == NT_GNU_BUILD_ID && (descsz == 16 || descsz == 20)) {
type == NT_GNU_BUILD_ID && (descsz >= 4 || descsz <= 20)) {
uint8_t desc[20];
const char *btype;
uint32_t i;
*flags |= FLAGS_DID_BUILD_ID;
if (file_printf(ms, ", BuildID[%s]=", descsz == 16 ? "md5/uuid" :
"sha1") == -1)
switch (descsz) {
case 8:
btype = "xxHash";
break;
case 16:
btype = "md5/uuid";
break;
case 20:
btype = "sha1";
break;
default:
btype = "unknown";
break;
}
if (file_printf(ms, ", BuildID[%s]=", btype) == -1)
return 1;
(void)memcpy(desc, &nbuf[doff], descsz);
for (i = 0; i < descsz; i++)

353
contrib/file/src/softmagic.c
View File

@ -32,7 +32,7 @@
#include "file.h"
#ifndef lint
FILE_RCSID("@(#)$File: softmagic.c,v 1.234 2016/06/13 12:02:06 christos Exp $")
FILE_RCSID("@(#)$File: softmagic.c,v 1.238 2016/10/24 18:02:17 christos Exp $")
#endif /* lint */
#include "magic.h"
@ -57,13 +57,13 @@ private int mcopy(struct magic_set *, union VALUETYPE *, int, int,
const unsigned char *, uint32_t, size_t, struct magic *);
private int mconvert(struct magic_set *, struct magic *, int);
private int print_sep(struct magic_set *, int);
private int handle_annotation(struct magic_set *, struct magic *);
private int handle_annotation(struct magic_set *, struct magic *, int);
private int cvt_8(union VALUETYPE *, const struct magic *);
private int cvt_16(union VALUETYPE *, const struct magic *);
private int cvt_32(union VALUETYPE *, const struct magic *);
private int cvt_64(union VALUETYPE *, const struct magic *);
#define OFFSET_OOB(n, o, i) ((n) < (o) || (i) > ((n) - (o)))
#define OFFSET_OOB(n, o, i) ((n) < (uint32_t)(o) || (i) > ((n) - (o)))
#define BE64(p) (((uint64_t)(p)->hq[0]<<56)|((uint64_t)(p)->hq[1]<<48)| \
((uint64_t)(p)->hq[2]<<40)|((uint64_t)(p)->hq[3]<<32)| \
((uint64_t)(p)->hq[4]<<24)|((uint64_t)(p)->hq[5]<<16)| \
@ -80,6 +80,7 @@ private int cvt_64(union VALUETYPE *, const struct magic *);
((uint32_t)(p)->hl[3]<<8)|((uint32_t)(p)->hl[2]))
#define BE16(p) (((uint16_t)(p)->hs[0]<<8)|((uint16_t)(p)->hs[1]))
#define LE16(p) (((uint16_t)(p)->hs[1]<<8)|((uint16_t)(p)->hs[0]))
#define SEXT(s,v,p) ((s)?(intmax_t)(int##v##_t)(p):(intmax_t)(uint##v##_t)(p))
/*
* softmagic - lookup one file in parsed, in-memory copy of database
@ -230,7 +231,7 @@ match(struct magic_set *ms, struct magic *magic, uint32_t nmagic,
goto flush;
}
if ((e = handle_annotation(ms, m)) != 0) {
if ((e = handle_annotation(ms, m, firstline)) != 0) {
*need_separator = 1;
*printed_something = 1;
*returnval = 1;
@ -328,7 +329,7 @@ match(struct magic_set *ms, struct magic *magic, uint32_t nmagic,
} else
ms->c.li[cont_level].got_match = 1;
if ((e = handle_annotation(ms, m)) != 0) {
if ((e = handle_annotation(ms, m, firstline)) != 0) {
*need_separator = 1;
*printed_something = 1;
*returnval = 1;
@ -1226,7 +1227,7 @@ mcopy(struct magic_set *ms, union VALUETYPE *p, int type, int indir,
memchr(c, '\r', CAST(size_t, (end - c))))));
lines--, b++) {
last = b;
if (b[0] == '\r' && b[1] == '\n')
if (b < end - 1 && b[0] == '\r' && b[1] == '\n')
b++;
}
if (lines)
@ -1294,6 +1295,45 @@ mcopy(struct magic_set *ms, union VALUETYPE *p, int type, int indir,
return 0;
}
private uint32_t
do_ops(struct magic *m, intmax_t lhs, intmax_t off)
{
intmax_t offset;
if (off) {
switch (m->in_op & FILE_OPS_MASK) {
case FILE_OPAND:
offset = lhs & off;
break;
case FILE_OPOR:
offset = lhs | off;
break;
case FILE_OPXOR:
offset = lhs ^ off;
break;
case FILE_OPADD:
offset = lhs + off;
break;
case FILE_OPMINUS:
offset = lhs - off;
break;
case FILE_OPMULTIPLY:
offset = lhs * off;
break;
case FILE_OPDIVIDE:
offset = lhs / off;
break;
case FILE_OPMODULO:
offset = lhs % off;
break;
}
} else
offset = lhs;
if (m->in_op & FILE_OPINVERSE)
offset = ~offset;
return (uint32_t)offset;
}
private int
mget(struct magic_set *ms, const unsigned char *s, struct magic *m,
size_t nbytes, size_t o, unsigned int cont_level, int mode, int text,
@ -1301,7 +1341,7 @@ mget(struct magic_set *ms, const unsigned char *s, struct magic *m,
int *printed_something, int *need_separator, int *returnval)
{
uint32_t offset = ms->offset;
uint32_t lhs;
intmax_t lhs;
file_pushbuf_t *pb;
int rv, oneed_separator, in_type;
char *rbuf;
@ -1337,7 +1377,8 @@ mget(struct magic_set *ms, const unsigned char *s, struct magic *m,
}
if (m->flag & INDIR) {
int off = m->in_offset;
intmax_t off = m->in_offset;
const int sgn = m->in_op & FILE_OPSIGNED;
if (m->in_op & FILE_OPINDIRECT) {
const union VALUETYPE *q = CAST(const union VALUETYPE *,
((const void *)(s + offset + off)));
@ -1345,178 +1386,55 @@ mget(struct magic_set *ms, const unsigned char *s, struct magic *m,
return 0;
switch (cvt_flip(m->in_type, flip)) {
case FILE_BYTE:
off = q->b;
off = SEXT(sgn,8,q->b);
break;
case FILE_SHORT:
off = q->h;
off = SEXT(sgn,16,q->h);
break;
case FILE_BESHORT:
off = (short)BE16(q);
off = SEXT(sgn,16,BE16(q));
break;
case FILE_LESHORT:
off = (short)LE16(q);
off = SEXT(sgn,16,LE16(q));
break;
case FILE_LONG:
off = q->l;
off = SEXT(sgn,32,q->l);
break;
case FILE_BELONG:
case FILE_BEID3:
off = (int32_t)BE32(q);
off = SEXT(sgn,32,BE32(q));
break;
case FILE_LEID3:
case FILE_LELONG:
off = (int32_t)LE32(q);
off = SEXT(sgn,32,LE32(q));
break;
case FILE_MELONG:
off = (int32_t)ME32(q);
off = SEXT(sgn,32,ME32(q));
break;
}
if ((ms->flags & MAGIC_DEBUG) != 0)
fprintf(stderr, "indirect offs=%u\n", off);
fprintf(stderr, "indirect offs=%jd\n", off);
}
switch (in_type = cvt_flip(m->in_type, flip)) {
case FILE_BYTE:
if (OFFSET_OOB(nbytes, offset, 1))
return 0;
if (off) {
switch (m->in_op & FILE_OPS_MASK) {
case FILE_OPAND:
offset = p->b & off;
break;
case FILE_OPOR:
offset = p->b | off;
break;
case FILE_OPXOR:
offset = p->b ^ off;
break;
case FILE_OPADD:
offset = p->b + off;
break;
case FILE_OPMINUS:
offset = p->b - off;
break;
case FILE_OPMULTIPLY:
offset = p->b * off;
break;
case FILE_OPDIVIDE:
offset = p->b / off;
break;
case FILE_OPMODULO:
offset = p->b % off;
break;
}
} else
offset = p->b;
if (m->in_op & FILE_OPINVERSE)
offset = ~offset;
offset = do_ops(m, SEXT(sgn,8,p->b), off);
break;
case FILE_BESHORT:
if (OFFSET_OOB(nbytes, offset, 2))
return 0;
lhs = (p->hs[0] << 8) | p->hs[1];
if (off) {
switch (m->in_op & FILE_OPS_MASK) {
case FILE_OPAND:
offset = lhs & off;
break;
case FILE_OPOR:
offset = lhs | off;
break;
case FILE_OPXOR:
offset = lhs ^ off;
break;
case FILE_OPADD:
offset = lhs + off;
break;
case FILE_OPMINUS:
offset = lhs - off;
break;
case FILE_OPMULTIPLY:
offset = lhs * off;
break;
case FILE_OPDIVIDE:
offset = lhs / off;
break;
case FILE_OPMODULO:
offset = lhs % off;
break;
}
} else
offset = lhs;
if (m->in_op & FILE_OPINVERSE)
offset = ~offset;
offset = do_ops(m, SEXT(sgn,16,BE16(p)), off);
break;
case FILE_LESHORT:
if (OFFSET_OOB(nbytes, offset, 2))
return 0;
lhs = (p->hs[1] << 8) | p->hs[0];
if (off) {
switch (m->in_op & FILE_OPS_MASK) {
case FILE_OPAND:
offset = lhs & off;
break;
case FILE_OPOR:
offset = lhs | off;
break;
case FILE_OPXOR:
offset = lhs ^ off;
break;
case FILE_OPADD:
offset = lhs + off;
break;
case FILE_OPMINUS:
offset = lhs - off;
break;
case FILE_OPMULTIPLY:
offset = lhs * off;
break;
case FILE_OPDIVIDE:
offset = lhs / off;
break;
case FILE_OPMODULO:
offset = lhs % off;
break;
}
} else
offset = lhs;
if (m->in_op & FILE_OPINVERSE)
offset = ~offset;
offset = do_ops(m, SEXT(sgn,16,LE16(p)), off);
break;
case FILE_SHORT:
if (OFFSET_OOB(nbytes, offset, 2))
return 0;
if (off) {
switch (m->in_op & FILE_OPS_MASK) {
case FILE_OPAND:
offset = p->h & off;
break;
case FILE_OPOR:
offset = p->h | off;
break;
case FILE_OPXOR:
offset = p->h ^ off;
break;
case FILE_OPADD:
offset = p->h + off;
break;
case FILE_OPMINUS:
offset = p->h - off;
break;
case FILE_OPMULTIPLY:
offset = p->h * off;
break;
case FILE_OPDIVIDE:
offset = p->h / off;
break;
case FILE_OPMODULO:
offset = p->h % off;
break;
}
}
else
offset = p->h;
if (m->in_op & FILE_OPINVERSE)
offset = ~offset;
offset = do_ops(m, SEXT(sgn,16,p->h), off);
break;
case FILE_BELONG:
case FILE_BEID3:
@ -1524,38 +1442,8 @@ mget(struct magic_set *ms, const unsigned char *s, struct magic *m,
return 0;
lhs = BE32(p);
if (in_type == FILE_BEID3)
lhs = cvt_id3(ms, lhs);
if (off) {
switch (m->in_op & FILE_OPS_MASK) {
case FILE_OPAND:
offset = lhs & off;
break;
case FILE_OPOR:
offset = lhs | off;
break;
case FILE_OPXOR:
offset = lhs ^ off;
break;
case FILE_OPADD:
offset = lhs + off;
break;
case FILE_OPMINUS:
offset = lhs - off;
break;
case FILE_OPMULTIPLY:
offset = lhs * off;
break;
case FILE_OPDIVIDE:
offset = lhs / off;
break;
case FILE_OPMODULO:
offset = lhs % off;
break;
}
} else
offset = lhs;
if (m->in_op & FILE_OPINVERSE)
offset = ~offset;
lhs = cvt_id3(ms, (uint32_t)lhs);
offset = do_ops(m, SEXT(sgn,32,lhs), off);
break;
case FILE_LELONG:
case FILE_LEID3:
@ -1563,109 +1451,18 @@ mget(struct magic_set *ms, const unsigned char *s, struct magic *m,
return 0;
lhs = LE32(p);
if (in_type == FILE_LEID3)
lhs = cvt_id3(ms, lhs);
if (off) {
switch (m->in_op & FILE_OPS_MASK) {
case FILE_OPAND:
offset = lhs & off;
break;
case FILE_OPOR:
offset = lhs | off;
break;
case FILE_OPXOR:
offset = lhs ^ off;
break;
case FILE_OPADD:
offset = lhs + off;
break;
case FILE_OPMINUS:
offset = lhs - off;
break;
case FILE_OPMULTIPLY:
offset = lhs * off;
break;
case FILE_OPDIVIDE:
offset = lhs / off;
break;
case FILE_OPMODULO:
offset = lhs % off;
break;
}
} else
offset = lhs;
if (m->in_op & FILE_OPINVERSE)
offset = ~offset;
lhs = cvt_id3(ms, (uint32_t)lhs);
offset = do_ops(m, SEXT(sgn,32,lhs), off);
break;
case FILE_MELONG:
if (OFFSET_OOB(nbytes, offset, 4))
return 0;
lhs = ME32(p);
if (off) {
switch (m->in_op & FILE_OPS_MASK) {
case FILE_OPAND:
offset = lhs & off;
break;
case FILE_OPOR:
offset = lhs | off;
break;
case FILE_OPXOR:
offset = lhs ^ off;
break;
case FILE_OPADD:
offset = lhs + off;
break;
case FILE_OPMINUS:
offset = lhs - off;
break;
case FILE_OPMULTIPLY:
offset = lhs * off;
break;
case FILE_OPDIVIDE:
offset = lhs / off;
break;
case FILE_OPMODULO:
offset = lhs % off;
break;
}
} else
offset = lhs;
if (m->in_op & FILE_OPINVERSE)
offset = ~offset;
offset = do_ops(m, SEXT(sgn,32,ME32(p)), off);
break;
case FILE_LONG:
if (OFFSET_OOB(nbytes, offset, 4))
return 0;
if (off) {
switch (m->in_op & FILE_OPS_MASK) {
case FILE_OPAND:
offset = p->l & off;
break;
case FILE_OPOR:
offset = p->l | off;
break;
case FILE_OPXOR:
offset = p->l ^ off;
break;
case FILE_OPADD:
offset = p->l + off;
break;
case FILE_OPMINUS:
offset = p->l - off;
break;
case FILE_OPMULTIPLY:
offset = p->l * off;
break;
case FILE_OPDIVIDE:
offset = p->l / off;
break;
case FILE_OPMODULO:
offset = p->l % off;
break;
}
} else
offset = p->l;
if (m->in_op & FILE_OPINVERSE)
offset = ~offset;
offset = do_ops(m, SEXT(sgn,32,p->l), off);
break;
default:
break;
@ -2078,7 +1875,7 @@ magiccheck(struct magic_set *ms, struct magic *m)
size_t slen = ms->search.s_len;
char *copy;
if (slen != 0) {
copy = malloc(slen);
copy = CAST(char *, malloc(slen));
if (copy == NULL) {
file_regfree(&rx);
file_error(ms, errno,
@ -2231,19 +2028,25 @@ magiccheck(struct magic_set *ms, struct magic *m)
}
private int
handle_annotation(struct magic_set *ms, struct magic *m)
handle_annotation(struct magic_set *ms, struct magic *m, int firstline)
{
if ((ms->flags & MAGIC_APPLE) && m->apple[0]) {
if (!firstline && file_printf(ms, "\n- ") == -1)
return -1;
if (file_printf(ms, "%.8s", m->apple) == -1)
return -1;
return 1;
}
if ((ms->flags & MAGIC_EXTENSION) && m->ext[0]) {
if (!firstline && file_printf(ms, "\n- ") == -1)
return -1;
if (file_printf(ms, "%s", m->ext) == -1)
return -1;
return 1;
}
if ((ms->flags & MAGIC_MIME_TYPE) && m->mimetype[0]) {
if (!firstline && file_printf(ms, "\n- ") == -1)
return -1;
if (file_printf(ms, "%s", m->mimetype) == -1)
return -1;
return 1;
@ -2254,8 +2057,8 @@ handle_annotation(struct magic_set *ms, struct magic *m)
private int
print_sep(struct magic_set *ms, int firstline)
{
if (ms->flags & MAGIC_NODESC)
return 0;
// if (ms->flags & MAGIC_NODESC)
// return 0;
if (firstline)
return 0;
/*

53
contrib/jemalloc/ChangeLog
View File

@ -4,6 +4,59 @@ brevity. Much more detail can be found in the git revision history:
https://github.com/jemalloc/jemalloc
* 4.3.1 (November 7, 2016)
Bug fixes:
- Fix a severe virtual memory leak. This regression was first released in
4.3.0. (@interwq, @jasone)
- Refactor atomic and prng APIs to restore support for 32-bit platforms that
use pre-C11 toolchains, e.g. FreeBSD's mips. (@jasone)
* 4.3.0 (November 4, 2016)
This is the first release that passes the test suite for multiple Windows
configurations, thanks in large part to @glandium setting up continuous
integration via AppVeyor (and Travis CI for Linux and OS X).
New features:
- Add "J" (JSON) support to malloc_stats_print(). (@jasone)
- Add Cray compiler support. (@ronawho)
Optimizations:
- Add/use adaptive spinning for bootstrapping and radix tree node
initialization. (@jasone)
Bug fixes:
- Fix large allocation to search starting in the optimal size class heap,
which can substantially reduce virtual memory churn and fragmentation. This
regression was first released in 4.0.0. (@mjp41, @jasone)
- Fix stats.arenas.<i>.nthreads accounting. (@interwq)
- Fix and simplify decay-based purging. (@jasone)
- Make DSS (sbrk(2)-related) operations lockless, which resolves potential
deadlocks during thread exit. (@jasone)
- Fix over-sized allocation of radix tree leaf nodes. (@mjp41, @ogaun,
@jasone)
- Fix over-sized allocation of arena_t (plus associated stats) data
structures. (@jasone, @interwq)
- Fix EXTRA_CFLAGS to not affect configuration. (@jasone)
- Fix a Valgrind integration bug. (@ronawho)
- Disallow 0x5a junk filling when running in Valgrind. (@jasone)
- Fix a file descriptor leak on Linux. This regression was first released in
4.2.0. (@vsarunas, @jasone)
- Fix static linking of jemalloc with glibc. (@djwatson)
- Use syscall(2) rather than {open,read,close}(2) during boot on Linux. This
works around other libraries' system call wrappers performing reentrant
allocation. (@kspinka, @Whissi, @jasone)
- Fix OS X default zone replacement to work with OS X 10.12. (@glandium,
@jasone)
- Fix cached memory management to avoid needless commit/decommit operations
during purging, which resolves permanent virtual memory map fragmentation
issues on Windows. (@mjp41, @jasone)
- Fix TSD fetches to avoid (recursive) allocation. This is relevant to
non-TLS and Windows configurations. (@jasone)
- Fix malloc_conf overriding to work on Windows. (@jasone)
- Forcibly disable lazy-lock on Windows (was forcibly *enabled*). (@jasone)
* 4.2.1 (June 8, 2016)
Bug fixes:

2
contrib/jemalloc/FREEBSD-Xlist
View File

@ -1,6 +1,8 @@
$FreeBSD$
.appveyor.yml
.autom4te.cfg
.git*
.travis.yml
FREEBSD-*
INSTALL
Makefile*

66
contrib/jemalloc/FREEBSD-diffs
View File

@ -1,11 +1,11 @@
diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
index c4a44e3..4626e9b 100644
index 3d2e721..b361db2 100644
--- a/doc/jemalloc.xml.in
+++ b/doc/jemalloc.xml.in
@@ -53,11 +53,23 @@
<para>This manual describes jemalloc @jemalloc_version@. More information
can be found at the <ulink
url="http://www.canonware.com/jemalloc/">jemalloc website</ulink>.</para>
url="http://jemalloc.net/">jemalloc website</ulink>.</para>
+
+ <para>The following configuration options are enabled in libc's built-in
+ jemalloc: <option>--enable-fill</option>,
@ -27,30 +27,30 @@ index c4a44e3..4626e9b 100644
<refsect2>
<title>Standard API</title>
<funcprototype>
@@ -2961,4 +2973,18 @@ malloc_conf = "lg_chunk:24";]]></programlisting></para>
<para>The <function>posix_memalign<parameter/></function> function conforms
to IEEE Std 1003.1-2001 (&ldquo;POSIX.1&rdquo;).</para>
@@ -2963,4 +2975,18 @@ malloc_conf = "lg_chunk:24";]]></programlisting></para>
<para>The <function>posix_memalign()</function> function conforms
to IEEE Std 1003.1-2001 (<quote>POSIX.1</quote>).</para>
</refsect1>
+ <refsect1 id="history">
+ <title>HISTORY</title>
+ <para>The <function>malloc_usable_size<parameter/></function> and
+ <function>posix_memalign<parameter/></function> functions first appeared in
+ FreeBSD 7.0.</para>
+ <para>The <function>malloc_usable_size()</function> and
+ <function>posix_memalign()</function> functions first appeared in FreeBSD
+ 7.0.</para>
+
+ <para>The <function>aligned_alloc<parameter/></function>,
+ <function>malloc_stats_print<parameter/></function>, and
+ <function>mallctl*<parameter/></function> functions first appeared in
+ FreeBSD 10.0.</para>
+ <para>The <function>aligned_alloc()</function>,
+ <function>malloc_stats_print()</function>, and
+ <function>mallctl*()</function> functions first appeared in FreeBSD
+ 10.0.</para>
+
+ <para>The <function>*allocx<parameter/></function> functions first appeared
+ in FreeBSD 11.0.</para>
+ <para>The <function>*allocx()</function> functions first appeared in FreeBSD
+ 11.0.</para>
+ </refsect1>
</refentry>
diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h
index b1de2b6..da6b6d2 100644
index f39ce54..a3ba55d 100644
--- a/include/jemalloc/internal/arena.h
+++ b/include/jemalloc/internal/arena.h
@@ -718,8 +718,13 @@ arena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind)
@@ -719,8 +719,13 @@ arena_miscelm_get_mutable(arena_chunk_t *chunk, size_t pageind)
JEMALLOC_ALWAYS_INLINE const arena_chunk_map_misc_t *
arena_miscelm_get_const(const arena_chunk_t *chunk, size_t pageind)
{
@ -64,7 +64,7 @@ index b1de2b6..da6b6d2 100644
}
JEMALLOC_ALWAYS_INLINE size_t
@@ -778,8 +783,13 @@ arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind)
@@ -779,8 +784,13 @@ arena_mapbitsp_get_mutable(arena_chunk_t *chunk, size_t pageind)
JEMALLOC_ALWAYS_INLINE const size_t *
arena_mapbitsp_get_const(const arena_chunk_t *chunk, size_t pageind)
{
@ -79,7 +79,7 @@ index b1de2b6..da6b6d2 100644
JEMALLOC_ALWAYS_INLINE size_t
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index 8f82edd..78e2df2 100644
index fdc8fef..56a35a4 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -8,6 +8,9 @@
@ -108,7 +108,7 @@ index 8f82edd..78e2df2 100644
static const bool config_prof =
#ifdef JEMALLOC_PROF
diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h
index 2b8ca5d..42d97f2 100644
index c907d91..4626632 100644
--- a/include/jemalloc/internal/jemalloc_internal_decls.h
+++ b/include/jemalloc/internal/jemalloc_internal_decls.h
@@ -1,6 +1,9 @@
@ -122,10 +122,10 @@ index 2b8ca5d..42d97f2 100644
#ifdef _WIN32
# include <windows.h>
diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h
index 5221799..60ab041 100644
index b442d2d..76518db 100644
--- a/include/jemalloc/internal/mutex.h
+++ b/include/jemalloc/internal/mutex.h
@@ -52,9 +52,6 @@ struct malloc_mutex_s {
@@ -57,9 +57,6 @@ struct malloc_mutex_s {
#ifdef JEMALLOC_LAZY_LOCK
extern bool isthreaded;
@ -135,7 +135,7 @@ index 5221799..60ab041 100644
#endif
bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
@@ -62,6 +59,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
@@ -67,6 +64,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
@ -144,10 +144,10 @@ index 5221799..60ab041 100644
#endif /* JEMALLOC_H_EXTERNS */
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index f2b6a55..69369c9 100644
index 87c8c9b..df576f6 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -311,7 +311,6 @@ iralloct_realign
@@ -307,7 +307,6 @@ iralloct_realign
isalloc
isdalloct
isqalloc
@ -335,7 +335,7 @@ index f943891..47d032c 100755
+#include "jemalloc_FreeBSD.h"
EOF
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 5d1f493..46dd1d1 100644
index 38650ff..f659b55 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -4,6 +4,10 @@
@ -347,9 +347,9 @@ index 5d1f493..46dd1d1 100644
+__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
+
/* Runtime configuration options. */
const char *je_malloc_conf JEMALLOC_ATTR(weak);
bool opt_abort =
@@ -2673,6 +2677,107 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
const char *je_malloc_conf
#ifndef _WIN32
@@ -2756,6 +2760,107 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
*/
/******************************************************************************/
/*
@ -457,7 +457,7 @@ index 5d1f493..46dd1d1 100644
* The following functions are used by threading libraries for protection of
* malloc during fork().
*/
@@ -2814,4 +2919,11 @@ jemalloc_postfork_child(void)
@@ -2894,4 +2999,11 @@ jemalloc_postfork_child(void)
ctl_postfork_child(tsd_tsdn(tsd));
}
@ -470,7 +470,7 @@ index 5d1f493..46dd1d1 100644
+
/******************************************************************************/
diff --git a/src/mutex.c b/src/mutex.c
index a1fac34..a24e420 100644
index 6333e73..13f8d79 100644
--- a/src/mutex.c
+++ b/src/mutex.c
@@ -66,6 +66,17 @@ pthread_create(pthread_t *__restrict thread,
@ -491,7 +491,7 @@ index a1fac34..a24e420 100644
#endif
bool
@@ -140,7 +151,7 @@ malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex)
@@ -142,7 +153,7 @@ malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex)
}
bool
@ -500,7 +500,7 @@ index a1fac34..a24e420 100644
{
#ifdef JEMALLOC_MUTEX_INIT_CB
@@ -154,3 +165,14 @@ malloc_mutex_boot(void)
@@ -156,3 +167,14 @@ malloc_mutex_boot(void)
#endif
return (false);
}
@ -516,7 +516,7 @@ index a1fac34..a24e420 100644
+#endif
+}
diff --git a/src/util.c b/src/util.c
index a1c4a2a..04f9153 100644
index 7905267..bee1c77 100644
--- a/src/util.c
+++ b/src/util.c
@@ -67,6 +67,22 @@ wrtmessage(void *cbopaque, const char *s)

2
contrib/jemalloc/VERSION
View File

@ -1 +1 @@
4.2.1-0-g3de035335255d553bdb344c32ffdb603816195d8
4.3.1-0-g0110fa8451af905affd77c3bea0d545fee2251b2

724
contrib/jemalloc/doc/jemalloc.3
View File

File diff suppressed because it is too large Load Diff

105
contrib/jemalloc/include/jemalloc/internal/arena.h
View File

@ -42,6 +42,7 @@ typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t;
typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t;
typedef struct arena_chunk_s arena_chunk_t;
typedef struct arena_bin_info_s arena_bin_info_t;
typedef struct arena_decay_s arena_decay_t;
typedef struct arena_bin_s arena_bin_t;
typedef struct arena_s arena_t;
typedef struct arena_tdata_s arena_tdata_t;
@ -257,6 +258,49 @@ struct arena_bin_info_s {
uint32_t reg0_offset;
};
struct arena_decay_s {
/*
* Approximate time in seconds from the creation of a set of unused
* dirty pages until an equivalent set of unused dirty pages is purged
* and/or reused.
*/
ssize_t time;
/* time / SMOOTHSTEP_NSTEPS. */
nstime_t interval;
/*
* Time at which the current decay interval logically started. We do
* not actually advance to a new epoch until sometime after it starts
* because of scheduling and computation delays, and it is even possible
* to completely skip epochs. In all cases, during epoch advancement we
* merge all relevant activity into the most recently recorded epoch.
*/
nstime_t epoch;
/* Deadline randomness generator. */
uint64_t jitter_state;
/*
* Deadline for current epoch. This is the sum of interval and per
* epoch jitter which is a uniform random variable in [0..interval).
* Epochs always advance by precise multiples of interval, but we
* randomize the deadline to reduce the likelihood of arenas purging in
* lockstep.
*/
nstime_t deadline;
/*
* Number of dirty pages at beginning of current epoch. During epoch
* advancement we use the delta between arena->decay.ndirty and
* arena->ndirty to determine how many dirty pages, if any, were
* generated.
*/
size_t ndirty;
/*
* Trailing log of how many unused dirty pages were generated during
* each of the past SMOOTHSTEP_NSTEPS decay epochs, where the last
* element is the most recent epoch. Corresponding epoch times are
* relative to epoch.
*/
size_t backlog[SMOOTHSTEP_NSTEPS];
};
struct arena_bin_s {
/*
* All operations on runcur, runs, and stats require that lock be
@ -326,7 +370,7 @@ struct arena_s {
* PRNG state for cache index randomization of large allocation base
* pointers.
*/
uint64_t offset_state;
size_t offset_state;
dss_prec_t dss_prec;
@ -394,52 +438,8 @@ struct arena_s {
arena_runs_dirty_link_t runs_dirty;
extent_node_t chunks_cache;
/*
* Approximate time in seconds from the creation of a set of unused
* dirty pages until an equivalent set of unused dirty pages is purged
* and/or reused.
*/
ssize_t decay_time;
/* decay_time / SMOOTHSTEP_NSTEPS. */
nstime_t decay_interval;
/*
* Time at which the current decay interval logically started. We do
* not actually advance to a new epoch until sometime after it starts
* because of scheduling and computation delays, and it is even possible
* to completely skip epochs. In all cases, during epoch advancement we
* merge all relevant activity into the most recently recorded epoch.
*/
nstime_t decay_epoch;
/* decay_deadline randomness generator. */
uint64_t decay_jitter_state;
/*
* Deadline for current epoch. This is the sum of decay_interval and
* per epoch jitter which is a uniform random variable in
* [0..decay_interval). Epochs always advance by precise multiples of
* decay_interval, but we randomize the deadline to reduce the
* likelihood of arenas purging in lockstep.
*/
nstime_t decay_deadline;
/*
* Number of dirty pages at beginning of current epoch. During epoch
* advancement we use the delta between decay_ndirty and ndirty to
* determine how many dirty pages, if any, were generated, and record
* the result in decay_backlog.
*/
size_t decay_ndirty;
/*
* Memoized result of arena_decay_backlog_npages_limit() corresponding
* to the current contents of decay_backlog, i.e. the limit on how many
* pages are allowed to exist for the decay epochs.
*/
size_t decay_backlog_npages_limit;
/*
* Trailing log of how many unused dirty pages were generated during
* each of the past SMOOTHSTEP_NSTEPS decay epochs, where the last
* element is the most recent epoch. Corresponding epoch times are
* relative to decay_epoch.
*/
size_t decay_backlog[SMOOTHSTEP_NSTEPS];
/* Decay-based purging state. */
arena_decay_t decay;
/* Extant huge allocations. */
ql_head(extent_node_t) huge;
@ -470,10 +470,12 @@ struct arena_s {
arena_bin_t bins[NBINS];
/*
* Quantized address-ordered heaps of this arena's available runs. The
* heaps are used for first-best-fit run allocation.
* Size-segregated address-ordered heaps of this arena's available runs,
* used for first-best-fit run allocation. Runs are quantized, i.e.
* they reside in the last heap which corresponds to a size class less
* than or equal to the run size.
*/
arena_run_heap_t runs_avail[1]; /* Dynamically sized. */
arena_run_heap_t runs_avail[NPSIZES];
};
/* Used in conjunction with tsd for fast arena-related context lookup. */
@ -505,7 +507,6 @@ extern size_t map_bias; /* Number of arena chunk header pages. */
extern size_t map_misc_offset;
extern size_t arena_maxrun; /* Max run size for arenas. */
extern size_t large_maxclass; /* Max large size class. */
extern size_t run_quantize_max; /* Max run_quantize_*() input. */
extern unsigned nlclasses; /* Number of large size classes. */
extern unsigned nhclasses; /* Number of huge size classes. */
@ -601,7 +602,7 @@ unsigned arena_nthreads_get(arena_t *arena, bool internal);
void arena_nthreads_inc(arena_t *arena, bool internal);
void arena_nthreads_dec(arena_t *arena, bool internal);
arena_t *arena_new(tsdn_t *tsdn, unsigned ind);
bool arena_boot(void);
void arena_boot(void);
void arena_prefork0(tsdn_t *tsdn, arena_t *arena);
void arena_prefork1(tsdn_t *tsdn, arena_t *arena);
void arena_prefork2(tsdn_t *tsdn, arena_t *arena);

5
contrib/jemalloc/include/jemalloc/internal/chunk.h
View File

@ -58,7 +58,7 @@ void chunk_deregister(const void *chunk, const extent_node_t *node);
void *chunk_alloc_base(size_t size);
void *chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero, bool dalloc_node);
bool *zero, bool *commit, bool dalloc_node);
void *chunk_alloc_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *new_addr, size_t size, size_t alignment,
bool *zero, bool *commit);
@ -71,9 +71,6 @@ bool chunk_purge_wrapper(tsdn_t *tsdn, arena_t *arena,
chunk_hooks_t *chunk_hooks, void *chunk, size_t size, size_t offset,
size_t length);
bool chunk_boot(void);
void chunk_prefork(tsdn_t *tsdn);
void chunk_postfork_parent(tsdn_t *tsdn);
void chunk_postfork_child(tsdn_t *tsdn);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/

12
contrib/jemalloc/include/jemalloc/internal/chunk_dss.h
View File

@ -21,15 +21,13 @@ extern const char *dss_prec_names[];
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
dss_prec_t chunk_dss_prec_get(tsdn_t *tsdn);
bool chunk_dss_prec_set(tsdn_t *tsdn, dss_prec_t dss_prec);
dss_prec_t chunk_dss_prec_get(void);
bool chunk_dss_prec_set(dss_prec_t dss_prec);
void *chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr,
size_t size, size_t alignment, bool *zero, bool *commit);
bool chunk_in_dss(tsdn_t *tsdn, void *chunk);
bool chunk_dss_boot(void);
void chunk_dss_prefork(tsdn_t *tsdn);
void chunk_dss_postfork_parent(tsdn_t *tsdn);
void chunk_dss_postfork_child(tsdn_t *tsdn);
bool chunk_in_dss(void *chunk);
bool chunk_dss_mergeable(void *chunk_a, void *chunk_b);
void chunk_dss_boot(void);
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/

8
contrib/jemalloc/include/jemalloc/internal/ckh.h
View File

@ -64,13 +64,13 @@ struct ckh_s {
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
bool ckh_new(tsdn_t *tsdn, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
bool ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
ckh_keycomp_t *keycomp);
void ckh_delete(tsdn_t *tsdn, ckh_t *ckh);
void ckh_delete(tsd_t *tsd, ckh_t *ckh);
size_t ckh_count(ckh_t *ckh);
bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data);
bool ckh_insert(tsdn_t *tsdn, ckh_t *ckh, const void *key, const void *data);
bool ckh_remove(tsdn_t *tsdn, ckh_t *ckh, const void *searchkey, void **key,
bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data);
bool ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
void **data);
bool ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data);
void ckh_string_hash(const void *key, size_t r_hash[2]);

2
contrib/jemalloc/include/jemalloc/internal/huge.h
View File

@ -17,7 +17,7 @@ bool huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize,
void *huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize,
size_t usize, size_t alignment, bool zero, tcache_t *tcache);
#ifdef JEMALLOC_JET
typedef void (huge_dalloc_junk_t)(tsdn_t *, void *, size_t);
typedef void (huge_dalloc_junk_t)(void *, size_t);
extern huge_dalloc_junk_t *huge_dalloc_junk;
#endif
void huge_dalloc(tsdn_t *tsdn, void *ptr);

124
contrib/jemalloc/include/jemalloc/internal/jemalloc_internal.h
View File

@ -159,7 +159,9 @@ static const bool config_cache_oblivious =
#endif
#include "jemalloc/internal/ph.h"
#ifndef __PGI
#define RB_COMPACT
#endif
#include "jemalloc/internal/rb.h"
#include "jemalloc/internal/qr.h"
#include "jemalloc/internal/ql.h"
@ -182,6 +184,9 @@ static const bool config_cache_oblivious =
#include "jemalloc/internal/jemalloc_internal_macros.h"
/* Page size index type. */
typedef unsigned pszind_t;
/* Size class index type. */
typedef unsigned szind_t;
@ -231,7 +236,7 @@ typedef unsigned szind_t;
# ifdef __alpha__
# define LG_QUANTUM 4
# endif
# if (defined(__sparc64__) || defined(__sparcv9))
# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__))
# define LG_QUANTUM 4
# endif
# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64))
@ -361,6 +366,7 @@ typedef unsigned szind_t;
#include "jemalloc/internal/valgrind.h"
#include "jemalloc/internal/util.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/spin.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ticker.h"
#include "jemalloc/internal/ckh.h"
@ -393,6 +399,7 @@ typedef unsigned szind_t;
#include "jemalloc/internal/valgrind.h"
#include "jemalloc/internal/util.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/spin.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ticker.h"
#include "jemalloc/internal/ckh.h"
@ -452,11 +459,16 @@ extern unsigned narenas_auto;
*/
extern arena_t **arenas;
/*
* pind2sz_tab encodes the same information as could be computed by
* pind2sz_compute().
*/
extern size_t const pind2sz_tab[NPSIZES];
/*
* index2size_tab encodes the same information as could be computed (at
* unacceptable cost in some code paths) by index2size_compute().
*/
extern size_t const index2size_tab[NSIZES+1];
extern size_t const index2size_tab[NSIZES];
/*
* size2index_tab is a compact lookup table that rounds request sizes up to
* size classes. In order to reduce cache footprint, the table is compressed,
@ -464,6 +476,7 @@ extern size_t const index2size_tab[NSIZES+1];
*/
extern uint8_t const size2index_tab[];
arena_t *a0get(void);
void *a0malloc(size_t size);
void a0dalloc(void *ptr);
void *bootstrap_malloc(size_t size);
@ -489,6 +502,7 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/valgrind.h"
#include "jemalloc/internal/util.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/spin.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ticker.h"
#include "jemalloc/internal/ckh.h"
@ -521,6 +535,7 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/valgrind.h"
#include "jemalloc/internal/util.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/spin.h"
#include "jemalloc/internal/prng.h"
#include "jemalloc/internal/ticker.h"
#include "jemalloc/internal/ckh.h"
@ -540,6 +555,11 @@ void jemalloc_postfork_child(void);
#include "jemalloc/internal/huge.h"
#ifndef JEMALLOC_ENABLE_INLINE
pszind_t psz2ind(size_t psz);
size_t pind2sz_compute(pszind_t pind);
size_t pind2sz_lookup(pszind_t pind);
size_t pind2sz(pszind_t pind);
size_t psz2u(size_t psz);
szind_t size2index_compute(size_t size);
szind_t size2index_lookup(size_t size);
szind_t size2index(size_t size);
@ -552,7 +572,7 @@ size_t s2u(size_t size);
size_t sa2u(size_t size, size_t alignment);
arena_t *arena_choose_impl(tsd_t *tsd, arena_t *arena, bool internal);
arena_t *arena_choose(tsd_t *tsd, arena_t *arena);
arena_t *arena_ichoose(tsdn_t *tsdn, arena_t *arena);
arena_t *arena_ichoose(tsd_t *tsd, arena_t *arena);
arena_tdata_t *arena_tdata_get(tsd_t *tsd, unsigned ind,
bool refresh_if_missing);
arena_t *arena_get(tsdn_t *tsdn, unsigned ind, bool init_if_missing);
@ -560,10 +580,90 @@ ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
JEMALLOC_INLINE pszind_t
psz2ind(size_t psz)
{
if (unlikely(psz > HUGE_MAXCLASS))
return (NPSIZES);
{
pszind_t x = lg_floor((psz<<1)-1);
pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x -
(LG_SIZE_CLASS_GROUP + LG_PAGE);
pszind_t grp = shift << LG_SIZE_CLASS_GROUP;
pszind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ?
LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1;
size_t delta_inverse_mask = ZI(-1) << lg_delta;
pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) &
((ZU(1) << LG_SIZE_CLASS_GROUP) - 1);
pszind_t ind = grp + mod;
return (ind);
}
}
JEMALLOC_INLINE size_t
pind2sz_compute(pszind_t pind)
{
{
size_t grp = pind >> LG_SIZE_CLASS_GROUP;
size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1);
size_t grp_size_mask = ~((!!grp)-1);
size_t grp_size = ((ZU(1) << (LG_PAGE +
(LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask;
size_t shift = (grp == 0) ? 1 : grp;
size_t lg_delta = shift + (LG_PAGE-1);
size_t mod_size = (mod+1) << lg_delta;
size_t sz = grp_size + mod_size;
return (sz);
}
}
JEMALLOC_INLINE size_t
pind2sz_lookup(pszind_t pind)
{
size_t ret = (size_t)pind2sz_tab[pind];
assert(ret == pind2sz_compute(pind));
return (ret);
}
JEMALLOC_INLINE size_t
pind2sz(pszind_t pind)
{
assert(pind < NPSIZES);
return (pind2sz_lookup(pind));
}
JEMALLOC_INLINE size_t
psz2u(size_t psz)
{
if (unlikely(psz > HUGE_MAXCLASS))
return (0);
{
size_t x = lg_floor((psz<<1)-1);
size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ?
LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1;
size_t delta = ZU(1) << lg_delta;
size_t delta_mask = delta - 1;
size_t usize = (psz + delta_mask) & ~delta_mask;
return (usize);
}
}
JEMALLOC_INLINE szind_t
size2index_compute(size_t size)
{
if (unlikely(size > HUGE_MAXCLASS))
return (NSIZES);
#if (NTBINS != 0)
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
szind_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
@ -572,9 +672,7 @@ size2index_compute(size_t size)
}
#endif
{
szind_t x = unlikely(ZI(size) < 0) ? ((size<<1) ?
(ZU(1)<<(LG_SIZEOF_PTR+3)) : ((ZU(1)<<(LG_SIZEOF_PTR+3))-1))
: lg_floor((size<<1)-1);
szind_t x = lg_floor((size<<1)-1);
szind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 :
x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM);
szind_t grp = shift << LG_SIZE_CLASS_GROUP;
@ -660,6 +758,8 @@ JEMALLOC_ALWAYS_INLINE size_t
s2u_compute(size_t size)
{
if (unlikely(size > HUGE_MAXCLASS))
return (0);
#if (NTBINS > 0)
if (size <= (ZU(1) << LG_TINY_MAXCLASS)) {
size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1;
@ -669,9 +769,7 @@ s2u_compute(size_t size)
}
#endif
{
size_t x = unlikely(ZI(size) < 0) ? ((size<<1) ?
(ZU(1)<<(LG_SIZEOF_PTR+3)) : ((ZU(1)<<(LG_SIZEOF_PTR+3))-1))
: lg_floor((size<<1)-1);
size_t x = lg_floor((size<<1)-1);
size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1)
? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1;
size_t delta = ZU(1) << lg_delta;
@ -812,14 +910,10 @@ arena_choose(tsd_t *tsd, arena_t *arena)
}
JEMALLOC_INLINE arena_t *
arena_ichoose(tsdn_t *tsdn, arena_t *arena)
arena_ichoose(tsd_t *tsd, arena_t *arena)
{
assert(!tsdn_null(tsdn) || arena != NULL);
if (!tsdn_null(tsdn))
return (arena_choose_impl(tsdn_tsd(tsdn), NULL, true));
return (arena);
return (arena_choose_impl(tsd, arena, true));
}
JEMALLOC_INLINE arena_tdata_t *

10
contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h
View File

@ -20,8 +20,18 @@
# include <sys/uio.h>
# endif
# include <pthread.h>
# ifdef JEMALLOC_OS_UNFAIR_LOCK
# include <os/lock.h>
# endif
# ifdef JEMALLOC_GLIBC_MALLOC_HOOK
# include <sched.h>
# endif
# include <errno.h>
# include <sys/time.h>
# include <time.h>
# ifdef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME
# include <mach/mach_time.h>
# endif
#endif
#include <sys/types.h>

29
contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h
View File

@ -61,12 +61,20 @@
*/
#define JEMALLOC_HAVE_MADVISE
/*
* Defined if os_unfair_lock_*() functions are available, as provided by Darwin.
*/
/* #undef JEMALLOC_OS_UNFAIR_LOCK */
/*
* Defined if OSSpin*() functions are available, as provided by Darwin, and
* documented in the spinlock(3) manual page.
*/
/* #undef JEMALLOC_OSSPIN */
/* Defined if syscall(2) is available. */
#define JEMALLOC_HAVE_SYSCALL
/*
* Defined if secure_getenv(3) is available.
*/
@ -77,6 +85,21 @@
*/
#define JEMALLOC_HAVE_ISSETUGID
/*
* Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available.
*/
/* #undef JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE */
/*
* Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available.
*/
#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1
/*
* Defined if mach_absolute_time() is available.
*/
/* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */
/*
* Defined if _malloc_thread_cleanup() exists. At least in the case of
* FreeBSD, pthread_key_create() allocates, which if used during malloc
@ -189,6 +212,12 @@
/* TLS is used to map arenas and magazine caches to threads. */
#define JEMALLOC_TLS
/*
* Used to mark unreachable code to quiet "end of non-void" compiler warnings.
* Don't use this directly; instead use unreachable() from util.h
*/
#define JEMALLOC_INTERNAL_UNREACHABLE abort
/*
* ffs*() functions to use for bitmapping. Don't use these directly; instead,
* use ffs_*() from util.h.

4
contrib/jemalloc/include/jemalloc/internal/mb.h
View File

@ -105,8 +105,8 @@ mb_write(void)
malloc_mutex_t mtx;
malloc_mutex_init(&mtx, "mb", WITNESS_RANK_OMIT);
malloc_mutex_lock(NULL, &mtx);
malloc_mutex_unlock(NULL, &mtx);
malloc_mutex_lock(TSDN_NULL, &mtx);
malloc_mutex_unlock(TSDN_NULL, &mtx);
}
#endif
#endif

9
contrib/jemalloc/include/jemalloc/internal/mutex.h
View File

@ -5,6 +5,9 @@ typedef struct malloc_mutex_s malloc_mutex_t;
#ifdef _WIN32
# define MALLOC_MUTEX_INITIALIZER
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
# define MALLOC_MUTEX_INITIALIZER \
{OS_UNFAIR_LOCK_INIT, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
#elif (defined(JEMALLOC_OSSPIN))
# define MALLOC_MUTEX_INITIALIZER {0, WITNESS_INITIALIZER(WITNESS_RANK_OMIT)}
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
@ -35,6 +38,8 @@ struct malloc_mutex_s {
# else
CRITICAL_SECTION lock;
# endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
os_unfair_lock lock;
#elif (defined(JEMALLOC_OSSPIN))
OSSpinLock lock;
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
@ -86,6 +91,8 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex)
# else
EnterCriticalSection(&mutex->lock);
# endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
os_unfair_lock_lock(&mutex->lock);
#elif (defined(JEMALLOC_OSSPIN))
OSSpinLockLock(&mutex->lock);
#else
@ -107,6 +114,8 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex)
# else
LeaveCriticalSection(&mutex->lock);
# endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
os_unfair_lock_unlock(&mutex->lock);
#elif (defined(JEMALLOC_OSSPIN))
OSSpinLockUnlock(&mutex->lock);
#else

9
contrib/jemalloc/include/jemalloc/internal/nstime.h
View File

@ -1,12 +1,6 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
#if defined(_POSIX_MONOTONIC_CLOCK) && _POSIX_MONOTONIC_CLOCK >= 0
#define JEMALLOC_CLOCK_GETTIME 1
#else
#define JEMALLOC_CLOCK_GETTIME 0
#endif
typedef struct nstime_s nstime_t;
/* Maximum supported number of seconds (~584 years). */
@ -37,9 +31,12 @@ void nstime_imultiply(nstime_t *time, uint64_t multiplier);
void nstime_idivide(nstime_t *time, uint64_t divisor);
uint64_t nstime_divide(const nstime_t *time, const nstime_t *divisor);
#ifdef JEMALLOC_JET
typedef bool (nstime_monotonic_t)(void);
extern nstime_monotonic_t *nstime_monotonic;
typedef bool (nstime_update_t)(nstime_t *);
extern nstime_update_t *nstime_update;
#else
bool nstime_monotonic(void);
bool nstime_update(nstime_t *time);
#endif

34
contrib/jemalloc/include/jemalloc/internal/private_namespace.h
View File

@ -1,4 +1,5 @@
#define a0dalloc JEMALLOC_N(a0dalloc)
#define a0get JEMALLOC_N(a0get)
#define a0malloc JEMALLOC_N(a0malloc)
#define arena_aalloc JEMALLOC_N(arena_aalloc)
#define arena_alloc_junk_small JEMALLOC_N(arena_alloc_junk_small)
@ -167,20 +168,15 @@
#define chunk_dalloc_wrapper JEMALLOC_N(chunk_dalloc_wrapper)
#define chunk_deregister JEMALLOC_N(chunk_deregister)
#define chunk_dss_boot JEMALLOC_N(chunk_dss_boot)
#define chunk_dss_postfork_child JEMALLOC_N(chunk_dss_postfork_child)
#define chunk_dss_postfork_parent JEMALLOC_N(chunk_dss_postfork_parent)
#define chunk_dss_mergeable JEMALLOC_N(chunk_dss_mergeable)
#define chunk_dss_prec_get JEMALLOC_N(chunk_dss_prec_get)
#define chunk_dss_prec_set JEMALLOC_N(chunk_dss_prec_set)
#define chunk_dss_prefork JEMALLOC_N(chunk_dss_prefork)
#define chunk_hooks_default JEMALLOC_N(chunk_hooks_default)
#define chunk_hooks_get JEMALLOC_N(chunk_hooks_get)
#define chunk_hooks_set JEMALLOC_N(chunk_hooks_set)
#define chunk_in_dss JEMALLOC_N(chunk_in_dss)
#define chunk_lookup JEMALLOC_N(chunk_lookup)
#define chunk_npages JEMALLOC_N(chunk_npages)
#define chunk_postfork_child JEMALLOC_N(chunk_postfork_child)
#define chunk_postfork_parent JEMALLOC_N(chunk_postfork_parent)
#define chunk_prefork JEMALLOC_N(chunk_prefork)
#define chunk_purge_wrapper JEMALLOC_N(chunk_purge_wrapper)
#define chunk_register JEMALLOC_N(chunk_register)
#define chunks_rtree JEMALLOC_N(chunks_rtree)
@ -359,6 +355,7 @@
#define nstime_imultiply JEMALLOC_N(nstime_imultiply)
#define nstime_init JEMALLOC_N(nstime_init)
#define nstime_init2 JEMALLOC_N(nstime_init2)
#define nstime_monotonic JEMALLOC_N(nstime_monotonic)
#define nstime_ns JEMALLOC_N(nstime_ns)
#define nstime_nsec JEMALLOC_N(nstime_nsec)
#define nstime_sec JEMALLOC_N(nstime_sec)
@ -400,11 +397,22 @@
#define pages_purge JEMALLOC_N(pages_purge)
#define pages_trim JEMALLOC_N(pages_trim)
#define pages_unmap JEMALLOC_N(pages_unmap)
#define pind2sz JEMALLOC_N(pind2sz)
#define pind2sz_compute JEMALLOC_N(pind2sz_compute)
#define pind2sz_lookup JEMALLOC_N(pind2sz_lookup)
#define pind2sz_tab JEMALLOC_N(pind2sz_tab)
#define pow2_ceil_u32 JEMALLOC_N(pow2_ceil_u32)
#define pow2_ceil_u64 JEMALLOC_N(pow2_ceil_u64)
#define pow2_ceil_zu JEMALLOC_N(pow2_ceil_zu)
#define prng_lg_range JEMALLOC_N(prng_lg_range)
#define prng_range JEMALLOC_N(prng_range)
#define prng_lg_range_u32 JEMALLOC_N(prng_lg_range_u32)
#define prng_lg_range_u64 JEMALLOC_N(prng_lg_range_u64)
#define prng_lg_range_zu JEMALLOC_N(prng_lg_range_zu)
#define prng_range_u32 JEMALLOC_N(prng_range_u32)
#define prng_range_u64 JEMALLOC_N(prng_range_u64)
#define prng_range_zu JEMALLOC_N(prng_range_zu)
#define prng_state_next_u32 JEMALLOC_N(prng_state_next_u32)
#define prng_state_next_u64 JEMALLOC_N(prng_state_next_u64)
#define prng_state_next_zu JEMALLOC_N(prng_state_next_zu)
#define prof_active JEMALLOC_N(prof_active)
#define prof_active_get JEMALLOC_N(prof_active_get)
#define prof_active_get_unlocked JEMALLOC_N(prof_active_get_unlocked)
@ -453,12 +461,13 @@
#define prof_thread_active_set JEMALLOC_N(prof_thread_active_set)
#define prof_thread_name_get JEMALLOC_N(prof_thread_name_get)
#define prof_thread_name_set JEMALLOC_N(prof_thread_name_set)
#define psz2ind JEMALLOC_N(psz2ind)
#define psz2u JEMALLOC_N(psz2u)
#define purge_mode_names JEMALLOC_N(purge_mode_names)
#define quarantine JEMALLOC_N(quarantine)
#define quarantine_alloc_hook JEMALLOC_N(quarantine_alloc_hook)
#define quarantine_alloc_hook_work JEMALLOC_N(quarantine_alloc_hook_work)
#define quarantine_cleanup JEMALLOC_N(quarantine_cleanup)
#define register_zone JEMALLOC_N(register_zone)
#define rtree_child_read JEMALLOC_N(rtree_child_read)
#define rtree_child_read_hard JEMALLOC_N(rtree_child_read_hard)
#define rtree_child_tryread JEMALLOC_N(rtree_child_tryread)
@ -476,7 +485,6 @@
#define rtree_val_write JEMALLOC_N(rtree_val_write)
#define run_quantize_ceil JEMALLOC_N(run_quantize_ceil)
#define run_quantize_floor JEMALLOC_N(run_quantize_floor)
#define run_quantize_max JEMALLOC_N(run_quantize_max)
#define s2u JEMALLOC_N(s2u)
#define s2u_compute JEMALLOC_N(s2u_compute)
#define s2u_lookup JEMALLOC_N(s2u_lookup)
@ -486,6 +494,8 @@
#define size2index_compute JEMALLOC_N(size2index_compute)
#define size2index_lookup JEMALLOC_N(size2index_lookup)
#define size2index_tab JEMALLOC_N(size2index_tab)
#define spin_adaptive JEMALLOC_N(spin_adaptive)
#define spin_init JEMALLOC_N(spin_init)
#define stats_cactive JEMALLOC_N(stats_cactive)
#define stats_cactive_add JEMALLOC_N(stats_cactive_add)
#define stats_cactive_get JEMALLOC_N(stats_cactive_get)
@ -544,7 +554,9 @@
#define tsd_cleanup JEMALLOC_N(tsd_cleanup)
#define tsd_cleanup_wrapper JEMALLOC_N(tsd_cleanup_wrapper)
#define tsd_fetch JEMALLOC_N(tsd_fetch)
#define tsd_fetch_impl JEMALLOC_N(tsd_fetch_impl)
#define tsd_get JEMALLOC_N(tsd_get)
#define tsd_get_allocates JEMALLOC_N(tsd_get_allocates)
#define tsd_iarena_get JEMALLOC_N(tsd_iarena_get)
#define tsd_iarena_set JEMALLOC_N(tsd_iarena_set)
#define tsd_iarenap_get JEMALLOC_N(tsd_iarenap_get)
@ -603,9 +615,11 @@
#define witness_lock_error JEMALLOC_N(witness_lock_error)
#define witness_lockless_error JEMALLOC_N(witness_lockless_error)
#define witness_not_owner_error JEMALLOC_N(witness_not_owner_error)
#define witness_owner JEMALLOC_N(witness_owner)
#define witness_owner_error JEMALLOC_N(witness_owner_error)
#define witness_postfork_child JEMALLOC_N(witness_postfork_child)
#define witness_postfork_parent JEMALLOC_N(witness_postfork_parent)
#define witness_prefork JEMALLOC_N(witness_prefork)
#define witness_unlock JEMALLOC_N(witness_unlock)
#define witnesses_cleanup JEMALLOC_N(witnesses_cleanup)
#define zone_register JEMALLOC_N(zone_register)

152
contrib/jemalloc/include/jemalloc/internal/prng.h
View File

@ -19,8 +19,12 @@
* the next has a cycle of 4, etc. For this reason, we prefer to use the upper
* bits.
*/
#define PRNG_A UINT64_C(6364136223846793005)
#define PRNG_C UINT64_C(1442695040888963407)
#define PRNG_A_32 UINT32_C(1103515241)
#define PRNG_C_32 UINT32_C(12347)
#define PRNG_A_64 UINT64_C(6364136223846793005)
#define PRNG_C_64 UINT64_C(1442695040888963407)
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
@ -35,28 +39,133 @@
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
uint64_t prng_lg_range(uint64_t *state, unsigned lg_range);
uint64_t prng_range(uint64_t *state, uint64_t range);
uint32_t prng_state_next_u32(uint32_t state);
uint64_t prng_state_next_u64(uint64_t state);
size_t prng_state_next_zu(size_t state);
uint32_t prng_lg_range_u32(uint32_t *state, unsigned lg_range,
bool atomic);
uint64_t prng_lg_range_u64(uint64_t *state, unsigned lg_range);
size_t prng_lg_range_zu(size_t *state, unsigned lg_range, bool atomic);
uint32_t prng_range_u32(uint32_t *state, uint32_t range, bool atomic);
uint64_t prng_range_u64(uint64_t *state, uint64_t range);
size_t prng_range_zu(size_t *state, size_t range, bool atomic);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PRNG_C_))
JEMALLOC_ALWAYS_INLINE uint64_t
prng_lg_range(uint64_t *state, unsigned lg_range)
JEMALLOC_ALWAYS_INLINE uint32_t
prng_state_next_u32(uint32_t state)
{
uint64_t ret;
return ((state * PRNG_A_32) + PRNG_C_32);
}
JEMALLOC_ALWAYS_INLINE uint64_t
prng_state_next_u64(uint64_t state)
{
return ((state * PRNG_A_64) + PRNG_C_64);
}
JEMALLOC_ALWAYS_INLINE size_t
prng_state_next_zu(size_t state)
{
#if LG_SIZEOF_PTR == 2
return ((state * PRNG_A_32) + PRNG_C_32);
#elif LG_SIZEOF_PTR == 3
return ((state * PRNG_A_64) + PRNG_C_64);
#else
#error Unsupported pointer size
#endif
}
JEMALLOC_ALWAYS_INLINE uint32_t
prng_lg_range_u32(uint32_t *state, unsigned lg_range, bool atomic)
{
uint32_t ret, state1;
assert(lg_range > 0);
assert(lg_range <= 32);
if (atomic) {
uint32_t state0;
do {
state0 = atomic_read_uint32(state);
state1 = prng_state_next_u32(state0);
} while (atomic_cas_uint32(state, state0, state1));
} else {
state1 = prng_state_next_u32(*state);
*state = state1;
}
ret = state1 >> (32 - lg_range);
return (ret);
}
/* 64-bit atomic operations cannot be supported on all relevant platforms. */
JEMALLOC_ALWAYS_INLINE uint64_t
prng_lg_range_u64(uint64_t *state, unsigned lg_range)
{
uint64_t ret, state1;
assert(lg_range > 0);
assert(lg_range <= 64);
ret = (*state * PRNG_A) + PRNG_C;
*state = ret;
ret >>= (64 - lg_range);
state1 = prng_state_next_u64(*state);
*state = state1;
ret = state1 >> (64 - lg_range);
return (ret);
}
JEMALLOC_ALWAYS_INLINE size_t
prng_lg_range_zu(size_t *state, unsigned lg_range, bool atomic)
{
size_t ret, state1;
assert(lg_range > 0);
assert(lg_range <= ZU(1) << (3 + LG_SIZEOF_PTR));
if (atomic) {
size_t state0;
do {
state0 = atomic_read_z(state);
state1 = prng_state_next_zu(state0);
} while (atomic_cas_z(state, state0, state1));
} else {
state1 = prng_state_next_zu(*state);
*state = state1;
}
ret = state1 >> ((ZU(1) << (3 + LG_SIZEOF_PTR)) - lg_range);
return (ret);
}
JEMALLOC_ALWAYS_INLINE uint32_t
prng_range_u32(uint32_t *state, uint32_t range, bool atomic)
{
uint32_t ret;
unsigned lg_range;
assert(range > 1);
/* Compute the ceiling of lg(range). */
lg_range = ffs_u32(pow2_ceil_u32(range)) - 1;
/* Generate a result in [0..range) via repeated trial. */
do {
ret = prng_lg_range_u32(state, lg_range, atomic);
} while (ret >= range);
return (ret);
}
JEMALLOC_ALWAYS_INLINE uint64_t
prng_range(uint64_t *state, uint64_t range)
prng_range_u64(uint64_t *state, uint64_t range)
{
uint64_t ret;
unsigned lg_range;
@ -68,7 +177,26 @@ prng_range(uint64_t *state, uint64_t range)
/* Generate a result in [0..range) via repeated trial. */
do {
ret = prng_lg_range(state, lg_range);
ret = prng_lg_range_u64(state, lg_range);
} while (ret >= range);
return (ret);
}
JEMALLOC_ALWAYS_INLINE size_t
prng_range_zu(size_t *state, size_t range, bool atomic)
{
size_t ret;
unsigned lg_range;
assert(range > 1);
/* Compute the ceiling of lg(range). */
lg_range = ffs_u64(pow2_ceil_u64(range)) - 1;
/* Generate a result in [0..range) via repeated trial. */
do {
ret = prng_lg_range_zu(state, lg_range, atomic);
} while (ret >= range);
return (ret);

8
contrib/jemalloc/include/jemalloc/internal/prof.h
View File

@ -299,9 +299,9 @@ extern prof_dump_header_t *prof_dump_header;
void prof_idump(tsdn_t *tsdn);
bool prof_mdump(tsd_t *tsd, const char *filename);
void prof_gdump(tsdn_t *tsdn);
prof_tdata_t *prof_tdata_init(tsdn_t *tsdn);
prof_tdata_t *prof_tdata_init(tsd_t *tsd);
prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
void prof_reset(tsdn_t *tsdn, size_t lg_sample);
void prof_reset(tsd_t *tsd, size_t lg_sample);
void prof_tdata_cleanup(tsd_t *tsd);
bool prof_active_get(tsdn_t *tsdn);
bool prof_active_set(tsdn_t *tsdn, bool active);
@ -315,7 +315,7 @@ bool prof_gdump_get(tsdn_t *tsdn);
bool prof_gdump_set(tsdn_t *tsdn, bool active);
void prof_boot0(void);
void prof_boot1(void);
bool prof_boot2(tsdn_t *tsdn);
bool prof_boot2(tsd_t *tsd);
void prof_prefork0(tsdn_t *tsdn);
void prof_prefork1(tsdn_t *tsdn);
void prof_postfork_parent(tsdn_t *tsdn);
@ -384,7 +384,7 @@ prof_tdata_get(tsd_t *tsd, bool create)
if (create) {
if (unlikely(tdata == NULL)) {
if (tsd_nominal(tsd)) {
tdata = prof_tdata_init(tsd_tsdn(tsd));
tdata = prof_tdata_init(tsd);
tsd_prof_tdata_set(tsd, tdata);
}
} else if (unlikely(tdata->expired)) {

10175
contrib/jemalloc/include/jemalloc/internal/size_classes.h
View File

File diff suppressed because it is too large Load Diff

51
contrib/jemalloc/include/jemalloc/internal/spin.h Normal file
View File

@ -0,0 +1,51 @@
/******************************************************************************/
#ifdef JEMALLOC_H_TYPES
typedef struct spin_s spin_t;
#endif /* JEMALLOC_H_TYPES */
/******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS
struct spin_s {
unsigned iteration;
};
#endif /* JEMALLOC_H_STRUCTS */
/******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS
#endif /* JEMALLOC_H_EXTERNS */
/******************************************************************************/
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
void spin_init(spin_t *spin);
void spin_adaptive(spin_t *spin);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_SPIN_C_))
JEMALLOC_INLINE void
spin_init(spin_t *spin)
{
spin->iteration = 0;
}
JEMALLOC_INLINE void
spin_adaptive(spin_t *spin)
{
volatile uint64_t i;
for (i = 0; i < (KQU(1) << spin->iteration); i++)
CPU_SPINWAIT;
if (spin->iteration < 63)
spin->iteration++;
}
#endif
#endif /* JEMALLOC_H_INLINES */
/******************************************************************************/

2
contrib/jemalloc/include/jemalloc/internal/tcache.h
View File

@ -145,7 +145,7 @@ tcache_t *tcache_create(tsdn_t *tsdn, arena_t *arena);
void tcache_cleanup(tsd_t *tsd);
void tcache_enabled_cleanup(tsd_t *tsd);
void tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena);
bool tcaches_create(tsdn_t *tsdn, unsigned *r_ind);
bool tcaches_create(tsd_t *tsd, unsigned *r_ind);
void tcaches_flush(tsd_t *tsd, unsigned ind);
void tcaches_destroy(tsd_t *tsd, unsigned ind);
bool tcache_boot(tsdn_t *tsdn);

74
contrib/jemalloc/include/jemalloc/internal/tsd.h
View File

@ -48,7 +48,7 @@ typedef enum {
*
* bool example_tsd_boot(void) {...}
* bool example_tsd_booted_get(void) {...}
* example_t *example_tsd_get() {...}
* example_t *example_tsd_get(bool init) {...}
* void example_tsd_set(example_t *val) {...}
*
* Note that all of the functions deal in terms of (a_type *) rather than
@ -105,7 +105,7 @@ a_name##tsd_boot(void); \
a_attr bool \
a_name##tsd_booted_get(void); \
a_attr a_type * \
a_name##tsd_get(void); \
a_name##tsd_get(bool init); \
a_attr void \
a_name##tsd_set(a_type *val);
@ -213,9 +213,15 @@ a_name##tsd_booted_get(void) \
\
return (a_name##tsd_booted); \
} \
a_attr bool \
a_name##tsd_get_allocates(void) \
{ \
\
return (false); \
} \
/* Get/set. */ \
a_attr a_type * \
a_name##tsd_get(void) \
a_name##tsd_get(bool init) \
{ \
\
assert(a_name##tsd_booted); \
@ -264,9 +270,15 @@ a_name##tsd_booted_get(void) \
\
return (a_name##tsd_booted); \
} \
a_attr bool \
a_name##tsd_get_allocates(void) \
{ \
\
return (false); \
} \
/* Get/set. */ \
a_attr a_type * \
a_name##tsd_get(void) \
a_name##tsd_get(bool init) \
{ \
\
assert(a_name##tsd_booted); \
@ -325,14 +337,14 @@ a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
} \
} \
a_attr a_name##tsd_wrapper_t * \
a_name##tsd_wrapper_get(void) \
a_name##tsd_wrapper_get(bool init) \
{ \
DWORD error = GetLastError(); \
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
TlsGetValue(a_name##tsd_tsd); \
SetLastError(error); \
\
if (unlikely(wrapper == NULL)) { \
if (init && unlikely(wrapper == NULL)) { \
wrapper = (a_name##tsd_wrapper_t *) \
malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \
if (wrapper == NULL) { \
@ -392,14 +404,22 @@ a_name##tsd_booted_get(void) \
\
return (a_name##tsd_booted); \
} \
a_attr bool \
a_name##tsd_get_allocates(void) \
{ \
\
return (true); \
} \
/* Get/set. */ \
a_attr a_type * \
a_name##tsd_get(void) \
a_name##tsd_get(bool init) \
{ \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_wrapper_get(); \
wrapper = a_name##tsd_wrapper_get(init); \
if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \
return (NULL); \
return (&wrapper->val); \
} \
a_attr void \
@ -408,7 +428,7 @@ a_name##tsd_set(a_type *val) \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_wrapper_get(); \
wrapper = a_name##tsd_wrapper_get(true); \
wrapper->val = *(val); \
if (a_cleanup != malloc_tsd_no_cleanup) \
wrapper->initialized = true; \
@ -452,12 +472,12 @@ a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \
} \
} \
a_attr a_name##tsd_wrapper_t * \
a_name##tsd_wrapper_get(void) \
a_name##tsd_wrapper_get(bool init) \
{ \
a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \
pthread_getspecific(a_name##tsd_tsd); \
\
if (unlikely(wrapper == NULL)) { \
if (init && unlikely(wrapper == NULL)) { \
tsd_init_block_t block; \
wrapper = tsd_init_check_recursion( \
&a_name##tsd_init_head, &block); \
@ -520,14 +540,22 @@ a_name##tsd_booted_get(void) \
\
return (a_name##tsd_booted); \
} \
a_attr bool \
a_name##tsd_get_allocates(void) \
{ \
\
return (true); \
} \
/* Get/set. */ \
a_attr a_type * \
a_name##tsd_get(void) \
a_name##tsd_get(bool init) \
{ \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_wrapper_get(); \
wrapper = a_name##tsd_wrapper_get(init); \
if (a_name##tsd_get_allocates() && !init && wrapper == NULL) \
return (NULL); \
return (&wrapper->val); \
} \
a_attr void \
@ -536,7 +564,7 @@ a_name##tsd_set(a_type *val) \
a_name##tsd_wrapper_t *wrapper; \
\
assert(a_name##tsd_booted); \
wrapper = a_name##tsd_wrapper_get(); \
wrapper = a_name##tsd_wrapper_get(true); \
wrapper->val = *(val); \
if (a_cleanup != malloc_tsd_no_cleanup) \
wrapper->initialized = true; \
@ -639,6 +667,7 @@ void tsd_cleanup(void *arg);
#ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t)
tsd_t *tsd_fetch_impl(bool init);
tsd_t *tsd_fetch(void);
tsdn_t *tsd_tsdn(tsd_t *tsd);
bool tsd_nominal(tsd_t *tsd);
@ -658,9 +687,13 @@ malloc_tsd_externs(, tsd_t)
malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup)
JEMALLOC_ALWAYS_INLINE tsd_t *
tsd_fetch(void)
tsd_fetch_impl(bool init)
{
tsd_t *tsd = tsd_get();
tsd_t *tsd = tsd_get(init);
if (!init && tsd_get_allocates() && tsd == NULL)
return (NULL);
assert(tsd != NULL);
if (unlikely(tsd->state != tsd_state_nominal)) {
if (tsd->state == tsd_state_uninitialized) {
@ -677,6 +710,13 @@ tsd_fetch(void)
return (tsd);
}
JEMALLOC_ALWAYS_INLINE tsd_t *
tsd_fetch(void)
{
return (tsd_fetch_impl(true));
}
JEMALLOC_ALWAYS_INLINE tsdn_t *
tsd_tsdn(tsd_t *tsd)
{
@ -723,7 +763,7 @@ tsdn_fetch(void)
if (!tsd_booted_get())
return (NULL);
return (tsd_tsdn(tsd_fetch()));
return (tsd_tsdn(tsd_fetch_impl(false)));
}
JEMALLOC_ALWAYS_INLINE bool

22
contrib/jemalloc/include/jemalloc/internal/util.h
View File

@ -61,30 +61,20 @@
# define JEMALLOC_CC_SILENCE_INIT(v)
#endif
#define JEMALLOC_GNUC_PREREQ(major, minor) \
(!defined(__clang__) && \
(__GNUC__ > (major) || (__GNUC__ == (major) && __GNUC_MINOR__ >= (minor))))
#ifndef __has_builtin
# define __has_builtin(builtin) (0)
#endif
#define JEMALLOC_CLANG_HAS_BUILTIN(builtin) \
(defined(__clang__) && __has_builtin(builtin))
#ifdef __GNUC__
# define likely(x) __builtin_expect(!!(x), 1)
# define unlikely(x) __builtin_expect(!!(x), 0)
# if JEMALLOC_GNUC_PREREQ(4, 6) || \
JEMALLOC_CLANG_HAS_BUILTIN(__builtin_unreachable)
# define unreachable() __builtin_unreachable()
# else
# define unreachable() abort()
# endif
#else
# define likely(x) !!(x)
# define unlikely(x) !!(x)
# define unreachable() abort()
#endif
#if !defined(JEMALLOC_INTERNAL_UNREACHABLE)
# error JEMALLOC_INTERNAL_UNREACHABLE should have been defined by configure
#endif
#define unreachable() JEMALLOC_INTERNAL_UNREACHABLE()
#include "jemalloc/internal/assert.h"
/* Use to assert a particular configuration, e.g., cassert(config_debug). */

39
contrib/jemalloc/include/jemalloc/internal/witness.h
View File

@ -108,6 +108,7 @@ void witness_postfork_child(tsd_t *tsd);
#ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE
bool witness_owner(tsd_t *tsd, const witness_t *witness);
void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
void witness_assert_lockless(tsdn_t *tsdn);
@ -116,12 +117,25 @@ void witness_unlock(tsdn_t *tsdn, witness_t *witness);
#endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
JEMALLOC_INLINE bool
witness_owner(tsd_t *tsd, const witness_t *witness)
{
witness_list_t *witnesses;
witness_t *w;
witnesses = tsd_witnessesp_get(tsd);
ql_foreach(w, witnesses, link) {
if (w == witness)
return (true);
}
return (false);
}
JEMALLOC_INLINE void
witness_assert_owner(tsdn_t *tsdn, const witness_t *witness)
{
tsd_t *tsd;
witness_list_t *witnesses;
witness_t *w;
if (!config_debug)
return;
@ -132,11 +146,8 @@ witness_assert_owner(tsdn_t *tsdn, const witness_t *witness)
if (witness->rank == WITNESS_RANK_OMIT)
return;
witnesses = tsd_witnessesp_get(tsd);
ql_foreach(w, witnesses, link) {
if (w == witness)
return;
}
if (witness_owner(tsd, witness))
return;
witness_owner_error(witness);
}
@ -238,10 +249,16 @@ witness_unlock(tsdn_t *tsdn, witness_t *witness)
if (witness->rank == WITNESS_RANK_OMIT)
return;
witness_assert_owner(tsdn, witness);
witnesses = tsd_witnessesp_get(tsd);
ql_remove(witnesses, witness, link);
/*
* Check whether owner before removal, rather than relying on
* witness_assert_owner() to abort, so that unit tests can test this
* function's failure mode without causing undefined behavior.
*/
if (witness_owner(tsd, witness)) {
witnesses = tsd_witnessesp_get(tsd);
ql_remove(witnesses, witness, link);
} else
witness_assert_owner(tsdn, witness);
}
#endif

6
contrib/jemalloc/include/jemalloc/jemalloc.h
View File

@ -87,12 +87,12 @@ extern "C" {
#include <limits.h>
#include <strings.h>
#define JEMALLOC_VERSION "4.2.1-0-g3de035335255d553bdb344c32ffdb603816195d8"
#define JEMALLOC_VERSION "4.3.1-0-g0110fa8451af905affd77c3bea0d545fee2251b2"
#define JEMALLOC_VERSION_MAJOR 4
#define JEMALLOC_VERSION_MINOR 2
#define JEMALLOC_VERSION_MINOR 3
#define JEMALLOC_VERSION_BUGFIX 1
#define JEMALLOC_VERSION_NREV 0
#define JEMALLOC_VERSION_GID "3de035335255d553bdb344c32ffdb603816195d8"
#define JEMALLOC_VERSION_GID "0110fa8451af905affd77c3bea0d545fee2251b2"
# define MALLOCX_LG_ALIGN(la) ((int)(la))
# if LG_SIZEOF_PTR == 2

434
contrib/jemalloc/src/arena.c
View File

@ -21,15 +21,8 @@ size_t map_bias;
size_t map_misc_offset;
size_t arena_maxrun; /* Max run size for arenas. */
size_t large_maxclass; /* Max large size class. */
size_t run_quantize_max; /* Max run_quantize_*() input. */
static size_t small_maxrun; /* Max run size for small size classes. */
static bool *small_run_tab; /* Valid small run page multiples. */
static size_t *run_quantize_floor_tab; /* run_quantize_floor() memoization. */
static size_t *run_quantize_ceil_tab; /* run_quantize_ceil() memoization. */
unsigned nlclasses; /* Number of large size classes. */
unsigned nhclasses; /* Number of huge size classes. */
static szind_t runs_avail_bias; /* Size index for first runs_avail tree. */
static szind_t runs_avail_nclasses; /* Number of runs_avail trees. */
/******************************************************************************/
/*
@ -37,6 +30,8 @@ static szind_t runs_avail_nclasses; /* Number of runs_avail trees. */
* definition.
*/
static void arena_chunk_dalloc(tsdn_t *tsdn, arena_t *arena,
arena_chunk_t *chunk);
static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena,
size_t ndirty_limit);
static void arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run,
@ -77,83 +72,6 @@ arena_run_addr_comp(const arena_chunk_map_misc_t *a,
ph_gen(static UNUSED, arena_run_heap_, arena_run_heap_t, arena_chunk_map_misc_t,
ph_link, arena_run_addr_comp)
static size_t
run_quantize_floor_compute(size_t size)
{
size_t qsize;
assert(size != 0);
assert(size == PAGE_CEILING(size));
/* Don't change sizes that are valid small run sizes. */
if (size <= small_maxrun && small_run_tab[size >> LG_PAGE])
return (size);
/*
* Round down to the nearest run size that can actually be requested
* during normal large allocation. Add large_pad so that cache index
* randomization can offset the allocation from the page boundary.
*/
qsize = index2size(size2index(size - large_pad + 1) - 1) + large_pad;
if (qsize <= SMALL_MAXCLASS + large_pad)
return (run_quantize_floor_compute(size - large_pad));
assert(qsize <= size);
return (qsize);
}
static size_t
run_quantize_ceil_compute_hard(size_t size)
{
size_t large_run_size_next;
assert(size != 0);
assert(size == PAGE_CEILING(size));
/*
* Return the next quantized size greater than the input size.
* Quantized sizes comprise the union of run sizes that back small
* region runs, and run sizes that back large regions with no explicit
* alignment constraints.
*/
if (size > SMALL_MAXCLASS) {
large_run_size_next = PAGE_CEILING(index2size(size2index(size -
large_pad) + 1) + large_pad);
} else
large_run_size_next = SIZE_T_MAX;
if (size >= small_maxrun)
return (large_run_size_next);
while (true) {
size += PAGE;
assert(size <= small_maxrun);
if (small_run_tab[size >> LG_PAGE]) {
if (large_run_size_next < size)
return (large_run_size_next);
return (size);
}
}
}
static size_t
run_quantize_ceil_compute(size_t size)
{
size_t qsize = run_quantize_floor_compute(size);
if (qsize < size) {
/*
* Skip a quantization that may have an adequately large run,
* because under-sized runs may be mixed in. This only happens
* when an unusual size is requested, i.e. for aligned
* allocation, and is just one of several places where linear
* search would potentially find sufficiently aligned available
* memory somewhere lower.
*/
qsize = run_quantize_ceil_compute_hard(qsize);
}
return (qsize);
}
#ifdef JEMALLOC_JET
#undef run_quantize_floor
#define run_quantize_floor JEMALLOC_N(n_run_quantize_floor)
@ -162,13 +80,27 @@ static size_t
run_quantize_floor(size_t size)
{
size_t ret;
pszind_t pind;
assert(size > 0);
assert(size <= run_quantize_max);
assert(size <= HUGE_MAXCLASS);
assert((size & PAGE_MASK) == 0);
ret = run_quantize_floor_tab[(size >> LG_PAGE) - 1];
assert(ret == run_quantize_floor_compute(size));
assert(size != 0);
assert(size == PAGE_CEILING(size));
pind = psz2ind(size - large_pad + 1);
if (pind == 0) {
/*
* Avoid underflow. This short-circuit would also do the right
* thing for all sizes in the range for which there are
* PAGE-spaced size classes, but it's simplest to just handle
* the one case that would cause erroneous results.
*/
return (size);
}
ret = pind2sz(pind - 1) + large_pad;
assert(ret <= size);
return (ret);
}
#ifdef JEMALLOC_JET
@ -187,11 +119,21 @@ run_quantize_ceil(size_t size)
size_t ret;
assert(size > 0);
assert(size <= run_quantize_max);
assert(size <= HUGE_MAXCLASS);
assert((size & PAGE_MASK) == 0);
ret = run_quantize_ceil_tab[(size >> LG_PAGE) - 1];
assert(ret == run_quantize_ceil_compute(size));
ret = run_quantize_floor(size);
if (ret < size) {
/*
* Skip a quantization that may have an adequately large run,
* because under-sized runs may be mixed in. This only happens
* when an unusual size is requested, i.e. for aligned
* allocation, and is just one of several places where linear
* search would potentially find sufficiently aligned available
* memory somewhere lower.
*/
ret = pind2sz(psz2ind(ret - large_pad + 1)) + large_pad;
}
return (ret);
}
#ifdef JEMALLOC_JET
@ -200,25 +142,17 @@ run_quantize_ceil(size_t size)
run_quantize_t *run_quantize_ceil = JEMALLOC_N(n_run_quantize_ceil);
#endif
static arena_run_heap_t *
arena_runs_avail_get(arena_t *arena, szind_t ind)
{
assert(ind >= runs_avail_bias);
assert(ind - runs_avail_bias < runs_avail_nclasses);
return (&arena->runs_avail[ind - runs_avail_bias]);
}
static void
arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
size_t npages)
{
szind_t ind = size2index(run_quantize_floor(arena_miscelm_size_get(
pszind_t pind = psz2ind(run_quantize_floor(arena_miscelm_size_get(
arena_miscelm_get_const(chunk, pageind))));
assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
LG_PAGE));
arena_run_heap_insert(arena_runs_avail_get(arena, ind),
assert((npages << LG_PAGE) < chunksize);
assert(pind2sz(pind) <= chunksize);
arena_run_heap_insert(&arena->runs_avail[pind],
arena_miscelm_get_mutable(chunk, pageind));
}
@ -226,11 +160,13 @@ static void
arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind,
size_t npages)
{
szind_t ind = size2index(run_quantize_floor(arena_miscelm_size_get(
pszind_t pind = psz2ind(run_quantize_floor(arena_miscelm_size_get(
arena_miscelm_get_const(chunk, pageind))));
assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >>
LG_PAGE));
arena_run_heap_remove(arena_runs_avail_get(arena, ind),
assert((npages << LG_PAGE) < chunksize);
assert(pind2sz(pind) <= chunksize);
arena_run_heap_remove(&arena->runs_avail[pind],
arena_miscelm_get_mutable(chunk, pageind));
}
@ -649,14 +585,13 @@ arena_chunk_alloc_internal(tsdn_t *tsdn, arena_t *arena, bool *zero,
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
chunk = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, chunksize,
chunksize, zero, true);
chunksize, zero, commit, true);
if (chunk != NULL) {
if (arena_chunk_register(tsdn, arena, chunk, *zero)) {
chunk_dalloc_cache(tsdn, arena, &chunk_hooks, chunk,
chunksize, true);
return (NULL);
}
*commit = true;
}
if (chunk == NULL) {
chunk = arena_chunk_alloc_internal_hard(tsdn, arena,
@ -953,6 +888,7 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
void *ret;
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
size_t csize = CHUNK_CEILING(usize);
bool commit = true;
malloc_mutex_lock(tsdn, &arena->lock);
@ -964,7 +900,7 @@ arena_chunk_alloc_huge(tsdn_t *tsdn, arena_t *arena, size_t usize,
arena_nactive_add(arena, usize >> LG_PAGE);
ret = chunk_alloc_cache(tsdn, arena, &chunk_hooks, NULL, csize,
alignment, zero, true);
alignment, zero, &commit, true);
malloc_mutex_unlock(tsdn, &arena->lock);
if (ret == NULL) {
ret = arena_chunk_alloc_huge_hard(tsdn, arena, &chunk_hooks,
@ -1074,6 +1010,7 @@ arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk,
void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize));
size_t udiff = usize - oldsize;
size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
bool commit = true;
malloc_mutex_lock(tsdn, &arena->lock);
@ -1085,7 +1022,7 @@ arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk,
arena_nactive_add(arena, udiff >> LG_PAGE);
err = (chunk_alloc_cache(tsdn, arena, &chunk_hooks, nchunk, cdiff,
chunksize, zero, true) == NULL);
chunksize, zero, &commit, true) == NULL);
malloc_mutex_unlock(tsdn, &arena->lock);
if (err) {
err = arena_chunk_ralloc_huge_expand_hard(tsdn, arena,
@ -1109,12 +1046,13 @@ arena_chunk_ralloc_huge_expand(tsdn_t *tsdn, arena_t *arena, void *chunk,
static arena_run_t *
arena_run_first_best_fit(arena_t *arena, size_t size)
{
szind_t ind, i;
pszind_t pind, i;
ind = size2index(run_quantize_ceil(size));
for (i = ind; i < runs_avail_nclasses + runs_avail_bias; i++) {
pind = psz2ind(run_quantize_ceil(size));
for (i = pind; pind2sz(i) <= chunksize; i++) {
arena_chunk_map_misc_t *miscelm = arena_run_heap_first(
arena_runs_avail_get(arena, i));
&arena->runs_avail[i]);
if (miscelm != NULL)
return (&miscelm->run);
}
@ -1125,7 +1063,7 @@ arena_run_first_best_fit(arena_t *arena, size_t size)
static arena_run_t *
arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero)
{
arena_run_t *run = arena_run_first_best_fit(arena, s2u(size));
arena_run_t *run = arena_run_first_best_fit(arena, size);
if (run != NULL) {
if (arena_run_split_large(arena, run, size, zero))
run = NULL;
@ -1256,14 +1194,14 @@ arena_decay_deadline_init(arena_t *arena)
* Generate a new deadline that is uniformly random within the next
* epoch after the current one.
*/
nstime_copy(&arena->decay_deadline, &arena->decay_epoch);
nstime_add(&arena->decay_deadline, &arena->decay_interval);
if (arena->decay_time > 0) {
nstime_copy(&arena->decay.deadline, &arena->decay.epoch);
nstime_add(&arena->decay.deadline, &arena->decay.interval);
if (arena->decay.time > 0) {
nstime_t jitter;
nstime_init(&jitter, prng_range(&arena->decay_jitter_state,
nstime_ns(&arena->decay_interval)));
nstime_add(&arena->decay_deadline, &jitter);
nstime_init(&jitter, prng_range_u64(&arena->decay.jitter_state,
nstime_ns(&arena->decay.interval)));
nstime_add(&arena->decay.deadline, &jitter);
}
}
@ -1273,7 +1211,7 @@ arena_decay_deadline_reached(const arena_t *arena, const nstime_t *time)
assert(opt_purge == purge_mode_decay);
return (nstime_compare(&arena->decay_deadline, time) <= 0);
return (nstime_compare(&arena->decay.deadline, time) <= 0);
}
static size_t
@ -1298,92 +1236,103 @@ arena_decay_backlog_npages_limit(const arena_t *arena)
*/
sum = 0;
for (i = 0; i < SMOOTHSTEP_NSTEPS; i++)
sum += arena->decay_backlog[i] * h_steps[i];
sum += arena->decay.backlog[i] * h_steps[i];
npages_limit_backlog = (size_t)(sum >> SMOOTHSTEP_BFP);
return (npages_limit_backlog);
}
static void
arena_decay_epoch_advance(arena_t *arena, const nstime_t *time)
arena_decay_backlog_update_last(arena_t *arena)
{
uint64_t nadvance_u64;
nstime_t delta;
size_t ndirty_delta;
size_t ndirty_delta = (arena->ndirty > arena->decay.ndirty) ?
arena->ndirty - arena->decay.ndirty : 0;
arena->decay.backlog[SMOOTHSTEP_NSTEPS-1] = ndirty_delta;
}
assert(opt_purge == purge_mode_decay);
assert(arena_decay_deadline_reached(arena, time));
static void
arena_decay_backlog_update(arena_t *arena, uint64_t nadvance_u64)
{
nstime_copy(&delta, time);
nstime_subtract(&delta, &arena->decay_epoch);
nadvance_u64 = nstime_divide(&delta, &arena->decay_interval);
assert(nadvance_u64 > 0);
/* Add nadvance_u64 decay intervals to epoch. */
nstime_copy(&delta, &arena->decay_interval);
nstime_imultiply(&delta, nadvance_u64);
nstime_add(&arena->decay_epoch, &delta);
/* Set a new deadline. */
arena_decay_deadline_init(arena);
/* Update the backlog. */
if (nadvance_u64 >= SMOOTHSTEP_NSTEPS) {
memset(arena->decay_backlog, 0, (SMOOTHSTEP_NSTEPS-1) *
memset(arena->decay.backlog, 0, (SMOOTHSTEP_NSTEPS-1) *
sizeof(size_t));
} else {
size_t nadvance_z = (size_t)nadvance_u64;
assert((uint64_t)nadvance_z == nadvance_u64);
memmove(arena->decay_backlog, &arena->decay_backlog[nadvance_z],
memmove(arena->decay.backlog, &arena->decay.backlog[nadvance_z],
(SMOOTHSTEP_NSTEPS - nadvance_z) * sizeof(size_t));
if (nadvance_z > 1) {
memset(&arena->decay_backlog[SMOOTHSTEP_NSTEPS -
memset(&arena->decay.backlog[SMOOTHSTEP_NSTEPS -
nadvance_z], 0, (nadvance_z-1) * sizeof(size_t));
}
}
ndirty_delta = (arena->ndirty > arena->decay_ndirty) ? arena->ndirty -
arena->decay_ndirty : 0;
arena->decay_ndirty = arena->ndirty;
arena->decay_backlog[SMOOTHSTEP_NSTEPS-1] = ndirty_delta;
arena->decay_backlog_npages_limit =
arena_decay_backlog_npages_limit(arena);
arena_decay_backlog_update_last(arena);
}
static size_t
arena_decay_npages_limit(arena_t *arena)
static void
arena_decay_epoch_advance_helper(arena_t *arena, const nstime_t *time)
{
size_t npages_limit;
uint64_t nadvance_u64;
nstime_t delta;
assert(opt_purge == purge_mode_decay);
assert(arena_decay_deadline_reached(arena, time));
npages_limit = arena->decay_backlog_npages_limit;
nstime_copy(&delta, time);
nstime_subtract(&delta, &arena->decay.epoch);
nadvance_u64 = nstime_divide(&delta, &arena->decay.interval);
assert(nadvance_u64 > 0);
/* Add in any dirty pages created during the current epoch. */
if (arena->ndirty > arena->decay_ndirty)
npages_limit += arena->ndirty - arena->decay_ndirty;
/* Add nadvance_u64 decay intervals to epoch. */
nstime_copy(&delta, &arena->decay.interval);
nstime_imultiply(&delta, nadvance_u64);
nstime_add(&arena->decay.epoch, &delta);
return (npages_limit);
/* Set a new deadline. */
arena_decay_deadline_init(arena);
/* Update the backlog. */
arena_decay_backlog_update(arena, nadvance_u64);
}
static void
arena_decay_epoch_advance_purge(tsdn_t *tsdn, arena_t *arena)
{
size_t ndirty_limit = arena_decay_backlog_npages_limit(arena);
if (arena->ndirty > ndirty_limit)
arena_purge_to_limit(tsdn, arena, ndirty_limit);
arena->decay.ndirty = arena->ndirty;
}
static void
arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, const nstime_t *time)
{
arena_decay_epoch_advance_helper(arena, time);
arena_decay_epoch_advance_purge(tsdn, arena);
}
static void
arena_decay_init(arena_t *arena, ssize_t decay_time)
{
arena->decay_time = decay_time;
arena->decay.time = decay_time;
if (decay_time > 0) {
nstime_init2(&arena->decay_interval, decay_time, 0);
nstime_idivide(&arena->decay_interval, SMOOTHSTEP_NSTEPS);
nstime_init2(&arena->decay.interval, decay_time, 0);
nstime_idivide(&arena->decay.interval, SMOOTHSTEP_NSTEPS);
}
nstime_init(&arena->decay_epoch, 0);
nstime_update(&arena->decay_epoch);
arena->decay_jitter_state = (uint64_t)(uintptr_t)arena;
nstime_init(&arena->decay.epoch, 0);
nstime_update(&arena->decay.epoch);
arena->decay.jitter_state = (uint64_t)(uintptr_t)arena;
arena_decay_deadline_init(arena);
arena->decay_ndirty = arena->ndirty;
arena->decay_backlog_npages_limit = 0;
memset(arena->decay_backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t));
arena->decay.ndirty = arena->ndirty;
memset(arena->decay.backlog, 0, SMOOTHSTEP_NSTEPS * sizeof(size_t));
}
static bool
@ -1403,7 +1352,7 @@ arena_decay_time_get(tsdn_t *tsdn, arena_t *arena)
ssize_t decay_time;
malloc_mutex_lock(tsdn, &arena->lock);
decay_time = arena->decay_time;
decay_time = arena->decay.time;
malloc_mutex_unlock(tsdn, &arena->lock);
return (decay_time);
@ -1464,35 +1413,44 @@ static void
arena_maybe_purge_decay(tsdn_t *tsdn, arena_t *arena)
{
nstime_t time;
size_t ndirty_limit;
assert(opt_purge == purge_mode_decay);
/* Purge all or nothing if the option is disabled. */
if (arena->decay_time <= 0) {
if (arena->decay_time == 0)
if (arena->decay.time <= 0) {
if (arena->decay.time == 0)
arena_purge_to_limit(tsdn, arena, 0);
return;
}
nstime_copy(&time, &arena->decay_epoch);
if (unlikely(nstime_update(&time))) {
/* Time went backwards. Force an epoch advance. */
nstime_copy(&time, &arena->decay_deadline);
nstime_init(&time, 0);
nstime_update(&time);
if (unlikely(!nstime_monotonic() && nstime_compare(&arena->decay.epoch,
&time) > 0)) {
/*
* Time went backwards. Move the epoch back in time and
* generate a new deadline, with the expectation that time
* typically flows forward for long enough periods of time that
* epochs complete. Unfortunately, this strategy is susceptible
* to clock jitter triggering premature epoch advances, but
* clock jitter estimation and compensation isn't feasible here
* because calls into this code are event-driven.
*/
nstime_copy(&arena->decay.epoch, &time);
arena_decay_deadline_init(arena);
} else {
/* Verify that time does not go backwards. */
assert(nstime_compare(&arena->decay.epoch, &time) <= 0);
}
if (arena_decay_deadline_reached(arena, &time))
arena_decay_epoch_advance(arena, &time);
ndirty_limit = arena_decay_npages_limit(arena);
/*
* Don't try to purge unless the number of purgeable pages exceeds the
* current limit.
* If the deadline has been reached, advance to the current epoch and
* purge to the new limit if necessary. Note that dirty pages created
* during the current epoch are not subject to purge until a future
* epoch, so as a result purging only happens during epoch advances.
*/
if (arena->ndirty <= ndirty_limit)
return;
arena_purge_to_limit(tsdn, arena, ndirty_limit);
if (arena_decay_deadline_reached(arena, &time))
arena_decay_epoch_advance(tsdn, arena, &time);
}
void
@ -1561,7 +1519,7 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
if (rdelm == &chunkselm->rd) {
extent_node_t *chunkselm_next;
bool zero;
bool zero, commit;
UNUSED void *chunk;
npages = extent_node_size_get(chunkselm) >> LG_PAGE;
@ -1575,10 +1533,11 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
* dalloc_node=false argument to chunk_alloc_cache().
*/
zero = false;
commit = false;
chunk = chunk_alloc_cache(tsdn, arena, chunk_hooks,
extent_node_addr_get(chunkselm),
extent_node_size_get(chunkselm), chunksize, &zero,
false);
&commit, false);
assert(chunk == extent_node_addr_get(chunkselm));
assert(zero == extent_node_zeroed_get(chunkselm));
extent_node_dirty_insert(chunkselm, purge_runs_sentinel,
@ -1967,7 +1926,7 @@ arena_reset(tsd_t *tsd, arena_t *arena)
assert(!arena->purging);
arena->nactive = 0;
for(i = 0; i < runs_avail_nclasses; i++)
for (i = 0; i < NPSIZES; i++)
arena_run_heap_new(&arena->runs_avail[i]);
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->lock);
@ -2606,7 +2565,8 @@ arena_malloc_large(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero)
* that is a multiple of the cacheline size, e.g. [0 .. 63) * 64
* for 4 KiB pages and 64-byte cachelines.
*/
r = prng_lg_range(&arena->offset_state, LG_PAGE - LG_CACHELINE);
r = prng_lg_range_zu(&arena->offset_state, LG_PAGE -
LG_CACHELINE, false);
random_offset = ((uintptr_t)r) << LG_CACHELINE;
} else
random_offset = 0;
@ -3391,7 +3351,7 @@ arena_basic_stats_merge_locked(arena_t *arena, unsigned *nthreads,
*nthreads += arena_nthreads_get(arena, false);
*dss = dss_prec_names[arena->dss_prec];
*lg_dirty_mult = arena->lg_dirty_mult;
*decay_time = arena->decay_time;
*decay_time = arena->decay.time;
*nactive += arena->nactive;
*ndirty += arena->ndirty;
}
@ -3496,23 +3456,19 @@ arena_t *
arena_new(tsdn_t *tsdn, unsigned ind)
{
arena_t *arena;
size_t arena_size;
unsigned i;
/* Compute arena size to incorporate sufficient runs_avail elements. */
arena_size = offsetof(arena_t, runs_avail) + (sizeof(arena_run_heap_t) *
runs_avail_nclasses);
/*
* Allocate arena, arena->lstats, and arena->hstats contiguously, mainly
* because there is no way to clean up if base_alloc() OOMs.
*/
if (config_stats) {
arena = (arena_t *)base_alloc(tsdn,
CACHELINE_CEILING(arena_size) + QUANTUM_CEILING(nlclasses *
sizeof(malloc_large_stats_t) + nhclasses) *
sizeof(malloc_huge_stats_t));
CACHELINE_CEILING(sizeof(arena_t)) +
QUANTUM_CEILING((nlclasses * sizeof(malloc_large_stats_t)))
+ (nhclasses * sizeof(malloc_huge_stats_t)));
} else
arena = (arena_t *)base_alloc(tsdn, arena_size);
arena = (arena_t *)base_alloc(tsdn, sizeof(arena_t));
if (arena == NULL)
return (NULL);
@ -3524,11 +3480,11 @@ arena_new(tsdn_t *tsdn, unsigned ind)
if (config_stats) {
memset(&arena->stats, 0, sizeof(arena_stats_t));
arena->stats.lstats = (malloc_large_stats_t *)((uintptr_t)arena
+ CACHELINE_CEILING(arena_size));
+ CACHELINE_CEILING(sizeof(arena_t)));
memset(arena->stats.lstats, 0, nlclasses *
sizeof(malloc_large_stats_t));
arena->stats.hstats = (malloc_huge_stats_t *)((uintptr_t)arena
+ CACHELINE_CEILING(arena_size) +
+ CACHELINE_CEILING(sizeof(arena_t)) +
QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t)));
memset(arena->stats.hstats, 0, nhclasses *
sizeof(malloc_huge_stats_t));
@ -3548,10 +3504,10 @@ arena_new(tsdn_t *tsdn, unsigned ind)
* deterministic seed.
*/
arena->offset_state = config_debug ? ind :
(uint64_t)(uintptr_t)arena;
(size_t)(uintptr_t)arena;
}
arena->dss_prec = chunk_dss_prec_get(tsdn);
arena->dss_prec = chunk_dss_prec_get();
ql_new(&arena->achunks);
@ -3562,8 +3518,9 @@ arena_new(tsdn_t *tsdn, unsigned ind)
arena->nactive = 0;
arena->ndirty = 0;
for(i = 0; i < runs_avail_nclasses; i++)
for (i = 0; i < NPSIZES; i++)
arena_run_heap_new(&arena->runs_avail[i]);
qr_new(&arena->runs_dirty, rd_link);
qr_new(&arena->chunks_cache, cc_link);
@ -3693,9 +3650,6 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info)
bin_info->reg0_offset = (uint32_t)(actual_run_size - (actual_nregs *
bin_info->reg_interval) - pad_size + bin_info->redzone_size);
if (actual_run_size > small_maxrun)
small_maxrun = actual_run_size;
assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs
* bin_info->reg_interval) + pad_size == bin_info->run_size);
}
@ -3711,7 +3665,7 @@ bin_info_init(void)
bin_info_run_size_calc(bin_info); \
bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs);
#define BIN_INFO_INIT_bin_no(index, size)
#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \
BIN_INFO_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))
SIZE_CLASSES
#undef BIN_INFO_INIT_bin_yes
@ -3719,62 +3673,7 @@ bin_info_init(void)
#undef SC
}
static bool
small_run_size_init(void)
{
assert(small_maxrun != 0);
small_run_tab = (bool *)base_alloc(NULL, sizeof(bool) * (small_maxrun >>
LG_PAGE));
if (small_run_tab == NULL)
return (true);
#define TAB_INIT_bin_yes(index, size) { \
arena_bin_info_t *bin_info = &arena_bin_info[index]; \
small_run_tab[bin_info->run_size >> LG_PAGE] = true; \
}
#define TAB_INIT_bin_no(index, size)
#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
TAB_INIT_bin_##bin(index, (ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))
SIZE_CLASSES
#undef TAB_INIT_bin_yes
#undef TAB_INIT_bin_no
#undef SC
return (false);
}
static bool
run_quantize_init(void)
{
unsigned i;
run_quantize_max = chunksize + large_pad;
run_quantize_floor_tab = (size_t *)base_alloc(NULL, sizeof(size_t) *
(run_quantize_max >> LG_PAGE));
if (run_quantize_floor_tab == NULL)
return (true);
run_quantize_ceil_tab = (size_t *)base_alloc(NULL, sizeof(size_t) *
(run_quantize_max >> LG_PAGE));
if (run_quantize_ceil_tab == NULL)
return (true);
for (i = 1; i <= run_quantize_max >> LG_PAGE; i++) {
size_t run_size = i << LG_PAGE;
run_quantize_floor_tab[i-1] =
run_quantize_floor_compute(run_size);
run_quantize_ceil_tab[i-1] =
run_quantize_ceil_compute(run_size);
}
return (false);
}
bool
void
arena_boot(void)
{
unsigned i;
@ -3822,15 +3721,6 @@ arena_boot(void)
nhclasses = NSIZES - nlclasses - NBINS;
bin_info_init();
if (small_run_size_init())
return (true);
if (run_quantize_init())
return (true);
runs_avail_bias = size2index(PAGE);
runs_avail_nclasses = size2index(run_quantize_max)+1 - runs_avail_bias;
return (false);
}
void

57
contrib/jemalloc/src/chunk.c
View File

@ -316,10 +316,11 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
size_t i;
size_t *p = (size_t *)(uintptr_t)ret;
JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, size);
for (i = 0; i < size / sizeof(size_t); i++)
assert(p[i] == 0);
}
if (config_valgrind)
JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, size);
}
return (ret);
}
@ -384,23 +385,21 @@ chunk_alloc_base(size_t size)
void *
chunk_alloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
void *new_addr, size_t size, size_t alignment, bool *zero, bool dalloc_node)
void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit,
bool dalloc_node)
{
void *ret;
bool commit;
assert(size != 0);
assert((size & chunksize_mask) == 0);
assert(alignment != 0);
assert((alignment & chunksize_mask) == 0);
commit = true;
ret = chunk_recycle(tsdn, arena, chunk_hooks,
&arena->chunks_szad_cached, &arena->chunks_ad_cached, true,
new_addr, size, alignment, zero, &commit, dalloc_node);
new_addr, size, alignment, zero, commit, dalloc_node);
if (ret == NULL)
return (NULL);
assert(commit);
if (config_valgrind)
JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size);
return (ret);
@ -610,10 +609,10 @@ chunk_dalloc_cache(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
}
static bool
chunk_dalloc_default_impl(tsdn_t *tsdn, void *chunk, size_t size)
chunk_dalloc_default_impl(void *chunk, size_t size)
{
if (!have_dss || !chunk_in_dss(tsdn, chunk))
if (!have_dss || !chunk_in_dss(chunk))
return (chunk_dalloc_mmap(chunk, size));
return (true);
}
@ -622,11 +621,8 @@ static bool
chunk_dalloc_default(void *chunk, size_t size, bool committed,
unsigned arena_ind)
{
tsdn_t *tsdn;
tsdn = tsdn_fetch();
return (chunk_dalloc_default_impl(tsdn, chunk, size));
return (chunk_dalloc_default_impl(chunk, size));
}
void
@ -644,7 +640,7 @@ chunk_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
/* Try to deallocate. */
if (chunk_hooks->dalloc == chunk_dalloc_default) {
/* Call directly to propagate tsdn. */
err = chunk_dalloc_default_impl(tsdn, chunk, size);
err = chunk_dalloc_default_impl(chunk, size);
} else
err = chunk_hooks->dalloc(chunk, size, committed, arena->ind);
@ -717,13 +713,12 @@ chunk_split_default(void *chunk, size_t size, size_t size_a, size_t size_b,
}
static bool
chunk_merge_default_impl(tsdn_t *tsdn, void *chunk_a, void *chunk_b)
chunk_merge_default_impl(void *chunk_a, void *chunk_b)
{
if (!maps_coalesce)
return (true);
if (have_dss && chunk_in_dss(tsdn, chunk_a) != chunk_in_dss(tsdn,
chunk_b))
if (have_dss && !chunk_dss_mergeable(chunk_a, chunk_b))
return (true);
return (false);
@ -733,11 +728,8 @@ static bool
chunk_merge_default(void *chunk_a, size_t size_a, void *chunk_b, size_t size_b,
bool committed, unsigned arena_ind)
{
tsdn_t *tsdn;
tsdn = tsdn_fetch();
return (chunk_merge_default_impl(tsdn, chunk_a, chunk_b));
return (chunk_merge_default_impl(chunk_a, chunk_b));
}
static rtree_node_elm_t *
@ -781,32 +773,11 @@ chunk_boot(void)
chunksize_mask = chunksize - 1;
chunk_npages = (chunksize >> LG_PAGE);
if (have_dss && chunk_dss_boot())
return (true);
if (have_dss)
chunk_dss_boot();
if (rtree_new(&chunks_rtree, (unsigned)((ZU(1) << (LG_SIZEOF_PTR+3)) -
opt_lg_chunk), chunks_rtree_node_alloc, NULL))
return (true);
return (false);
}
void
chunk_prefork(tsdn_t *tsdn)
{
chunk_dss_prefork(tsdn);
}
void
chunk_postfork_parent(tsdn_t *tsdn)
{
chunk_dss_postfork_parent(tsdn);
}
void
chunk_postfork_child(tsdn_t *tsdn)
{
chunk_dss_postfork_child(tsdn);
}

193
contrib/jemalloc/src/chunk_dss.c
View File

@ -10,20 +10,19 @@ const char *dss_prec_names[] = {
"N/A"
};
/* Current dss precedence default, used when creating new arenas. */
static dss_prec_t dss_prec_default = DSS_PREC_DEFAULT;
/*
* Protects sbrk() calls. This avoids malloc races among threads, though it
* does not protect against races with threads that call sbrk() directly.
* Current dss precedence default, used when creating new arenas. NB: This is
* stored as unsigned rather than dss_prec_t because in principle there's no
* guarantee that sizeof(dss_prec_t) is the same as sizeof(unsigned), and we use
* atomic operations to synchronize the setting.
*/
static malloc_mutex_t dss_mtx;
static unsigned dss_prec_default = (unsigned)DSS_PREC_DEFAULT;
/* Base address of the DSS. */
static void *dss_base;
/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */
static void *dss_prev;
/* Current upper limit on DSS addresses. */
/* Atomic boolean indicating whether the DSS is exhausted. */
static unsigned dss_exhausted;
/* Atomic current upper limit on DSS addresses. */
static void *dss_max;
/******************************************************************************/
@ -41,30 +40,59 @@ chunk_dss_sbrk(intptr_t increment)
}
dss_prec_t
chunk_dss_prec_get(tsdn_t *tsdn)
chunk_dss_prec_get(void)
{
dss_prec_t ret;
if (!have_dss)
return (dss_prec_disabled);
malloc_mutex_lock(tsdn, &dss_mtx);
ret = dss_prec_default;
malloc_mutex_unlock(tsdn, &dss_mtx);
ret = (dss_prec_t)atomic_read_u(&dss_prec_default);
return (ret);
}
bool
chunk_dss_prec_set(tsdn_t *tsdn, dss_prec_t dss_prec)
chunk_dss_prec_set(dss_prec_t dss_prec)
{
if (!have_dss)
return (dss_prec != dss_prec_disabled);
malloc_mutex_lock(tsdn, &dss_mtx);
dss_prec_default = dss_prec;
malloc_mutex_unlock(tsdn, &dss_mtx);
atomic_write_u(&dss_prec_default, (unsigned)dss_prec);
return (false);
}
static void *
chunk_dss_max_update(void *new_addr)
{
void *max_cur;
spin_t spinner;
/*
* Get the current end of the DSS as max_cur and assure that dss_max is
* up to date.
*/
spin_init(&spinner);
while (true) {
void *max_prev = atomic_read_p(&dss_max);
max_cur = chunk_dss_sbrk(0);
if ((uintptr_t)max_prev > (uintptr_t)max_cur) {
/*
* Another thread optimistically updated dss_max. Wait
* for it to finish.
*/
spin_adaptive(&spinner);
continue;
}
if (!atomic_cas_p(&dss_max, max_prev, max_cur))
break;
}
/* Fixed new_addr can only be supported if it is at the edge of DSS. */
if (new_addr != NULL && max_cur != new_addr)
return (NULL);
return (max_cur);
}
void *
chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit)
@ -80,28 +108,20 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
if ((intptr_t)size < 0)
return (NULL);
malloc_mutex_lock(tsdn, &dss_mtx);
if (dss_prev != (void *)-1) {
if (!atomic_read_u(&dss_exhausted)) {
/*
* The loop is necessary to recover from races with other
* threads that are using the DSS for something other than
* malloc.
*/
do {
void *ret, *cpad, *dss_next;
while (true) {
void *ret, *cpad, *max_cur, *dss_next, *dss_prev;
size_t gap_size, cpad_size;
intptr_t incr;
/* Avoid an unnecessary system call. */
if (new_addr != NULL && dss_max != new_addr)
break;
/* Get the current end of the DSS. */
dss_max = chunk_dss_sbrk(0);
/* Make sure the earlier condition still holds. */
if (new_addr != NULL && dss_max != new_addr)
break;
max_cur = chunk_dss_max_update(new_addr);
if (max_cur == NULL)
goto label_oom;
/*
* Calculate how much padding is necessary to
@ -120,17 +140,23 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
cpad_size = (uintptr_t)ret - (uintptr_t)cpad;
dss_next = (void *)((uintptr_t)ret + size);
if ((uintptr_t)ret < (uintptr_t)dss_max ||
(uintptr_t)dss_next < (uintptr_t)dss_max) {
/* Wrap-around. */
malloc_mutex_unlock(tsdn, &dss_mtx);
return (NULL);
}
(uintptr_t)dss_next < (uintptr_t)dss_max)
goto label_oom; /* Wrap-around. */
incr = gap_size + cpad_size + size;
/*
* Optimistically update dss_max, and roll back below if
* sbrk() fails. No other thread will try to extend the
* DSS while dss_max is greater than the current DSS
* max reported by sbrk(0).
*/
if (atomic_cas_p(&dss_max, max_cur, dss_next))
continue;
/* Try to allocate. */
dss_prev = chunk_dss_sbrk(incr);
if (dss_prev == dss_max) {
if (dss_prev == max_cur) {
/* Success. */
dss_max = dss_next;
malloc_mutex_unlock(tsdn, &dss_mtx);
if (cpad_size != 0) {
chunk_hooks_t chunk_hooks =
CHUNK_HOOKS_INITIALIZER;
@ -147,68 +173,65 @@ chunk_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size,
*commit = pages_decommit(ret, size);
return (ret);
}
} while (dss_prev != (void *)-1);
}
malloc_mutex_unlock(tsdn, &dss_mtx);
/*
* Failure, whether due to OOM or a race with a raw
* sbrk() call from outside the allocator. Try to roll
* back optimistic dss_max update; if rollback fails,
* it's due to another caller of this function having
* succeeded since this invocation started, in which
* case rollback is not necessary.
*/
atomic_cas_p(&dss_max, dss_next, max_cur);
if (dss_prev == (void *)-1) {
/* OOM. */
atomic_write_u(&dss_exhausted, (unsigned)true);
goto label_oom;
}
}
}
label_oom:
return (NULL);
}
bool
chunk_in_dss(tsdn_t *tsdn, void *chunk)
static bool
chunk_in_dss_helper(void *chunk, void *max)
{
bool ret;
cassert(have_dss);
malloc_mutex_lock(tsdn, &dss_mtx);
if ((uintptr_t)chunk >= (uintptr_t)dss_base
&& (uintptr_t)chunk < (uintptr_t)dss_max)
ret = true;
else
ret = false;
malloc_mutex_unlock(tsdn, &dss_mtx);
return (ret);
return ((uintptr_t)chunk >= (uintptr_t)dss_base && (uintptr_t)chunk <
(uintptr_t)max);
}
bool
chunk_in_dss(void *chunk)
{
cassert(have_dss);
return (chunk_in_dss_helper(chunk, atomic_read_p(&dss_max)));
}
bool
chunk_dss_mergeable(void *chunk_a, void *chunk_b)
{
void *max;
cassert(have_dss);
max = atomic_read_p(&dss_max);
return (chunk_in_dss_helper(chunk_a, max) ==
chunk_in_dss_helper(chunk_b, max));
}
void
chunk_dss_boot(void)
{
cassert(have_dss);
if (malloc_mutex_init(&dss_mtx, "dss", WITNESS_RANK_DSS))
return (true);
dss_base = chunk_dss_sbrk(0);
dss_prev = dss_base;
dss_exhausted = (unsigned)(dss_base == (void *)-1);
dss_max = dss_base;
return (false);
}
void
chunk_dss_prefork(tsdn_t *tsdn)
{
if (have_dss)
malloc_mutex_prefork(tsdn, &dss_mtx);
}
void
chunk_dss_postfork_parent(tsdn_t *tsdn)
{
if (have_dss)
malloc_mutex_postfork_parent(tsdn, &dss_mtx);
}
void
chunk_dss_postfork_child(tsdn_t *tsdn)
{
if (have_dss)
malloc_mutex_postfork_child(tsdn, &dss_mtx);
}
/******************************************************************************/

47
contrib/jemalloc/src/ckh.c
View File

@ -40,8 +40,8 @@
/******************************************************************************/
/* Function prototypes for non-inline static functions. */
static bool ckh_grow(tsdn_t *tsdn, ckh_t *ckh);
static void ckh_shrink(tsdn_t *tsdn, ckh_t *ckh);
static bool ckh_grow(tsd_t *tsd, ckh_t *ckh);
static void ckh_shrink(tsd_t *tsd, ckh_t *ckh);
/******************************************************************************/
@ -99,7 +99,8 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key,
* Cycle through the cells in the bucket, starting at a random position.
* The randomness avoids worst-case search overhead as buckets fill up.
*/
offset = (unsigned)prng_lg_range(&ckh->prng_state, LG_CKH_BUCKET_CELLS);
offset = (unsigned)prng_lg_range_u64(&ckh->prng_state,
LG_CKH_BUCKET_CELLS);
for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) {
cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) +
((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))];
@ -141,7 +142,7 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey,
* were an item for which both hashes indicated the same
* bucket.
*/
i = (unsigned)prng_lg_range(&ckh->prng_state,
i = (unsigned)prng_lg_range_u64(&ckh->prng_state,
LG_CKH_BUCKET_CELLS);
cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i];
assert(cell->key != NULL);
@ -244,7 +245,7 @@ ckh_rebuild(ckh_t *ckh, ckhc_t *aTab)
}
static bool
ckh_grow(tsdn_t *tsdn, ckh_t *ckh)
ckh_grow(tsd_t *tsd, ckh_t *ckh)
{
bool ret;
ckhc_t *tab, *ttab;
@ -270,8 +271,8 @@ ckh_grow(tsdn_t *tsdn, ckh_t *ckh)
ret = true;
goto label_return;
}
tab = (ckhc_t *)ipallocztm(tsdn, usize, CACHELINE, true, NULL,
true, arena_ichoose(tsdn, NULL));
tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE,
true, NULL, true, arena_ichoose(tsd, NULL));
if (tab == NULL) {
ret = true;
goto label_return;
@ -283,12 +284,12 @@ ckh_grow(tsdn_t *tsdn, ckh_t *ckh)
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
if (!ckh_rebuild(ckh, tab)) {
idalloctm(tsdn, tab, NULL, true, true);
idalloctm(tsd_tsdn(tsd), tab, NULL, true, true);
break;
}
/* Rebuilding failed, so back out partially rebuilt table. */
idalloctm(tsdn, ckh->tab, NULL, true, true);
idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, true, true);
ckh->tab = tab;
ckh->lg_curbuckets = lg_prevbuckets;
}
@ -299,7 +300,7 @@ ckh_grow(tsdn_t *tsdn, ckh_t *ckh)
}
static void
ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
ckh_shrink(tsd_t *tsd, ckh_t *ckh)
{
ckhc_t *tab, *ttab;
size_t usize;
@ -314,8 +315,8 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE);
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
return;
tab = (ckhc_t *)ipallocztm(tsdn, usize, CACHELINE, true, NULL, true,
arena_ichoose(tsdn, NULL));
tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, NULL,
true, arena_ichoose(tsd, NULL));
if (tab == NULL) {
/*
* An OOM error isn't worth propagating, since it doesn't
@ -330,7 +331,7 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
if (!ckh_rebuild(ckh, tab)) {
idalloctm(tsdn, tab, NULL, true, true);
idalloctm(tsd_tsdn(tsd), tab, NULL, true, true);
#ifdef CKH_COUNT
ckh->nshrinks++;
#endif
@ -338,7 +339,7 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
}
/* Rebuilding failed, so back out partially rebuilt table. */
idalloctm(tsdn, ckh->tab, NULL, true, true);
idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, true, true);
ckh->tab = tab;
ckh->lg_curbuckets = lg_prevbuckets;
#ifdef CKH_COUNT
@ -347,7 +348,7 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
}
bool
ckh_new(tsdn_t *tsdn, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
ckh_keycomp_t *keycomp)
{
bool ret;
@ -391,8 +392,8 @@ ckh_new(tsdn_t *tsdn, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
ret = true;
goto label_return;
}
ckh->tab = (ckhc_t *)ipallocztm(tsdn, usize, CACHELINE, true, NULL,
true, arena_ichoose(tsdn, NULL));
ckh->tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true,
NULL, true, arena_ichoose(tsd, NULL));
if (ckh->tab == NULL) {
ret = true;
goto label_return;
@ -404,7 +405,7 @@ ckh_new(tsdn_t *tsdn, ckh_t *ckh, size_t minitems, ckh_hash_t *hash,
}
void
ckh_delete(tsdn_t *tsdn, ckh_t *ckh)
ckh_delete(tsd_t *tsd, ckh_t *ckh)
{
assert(ckh != NULL);
@ -421,7 +422,7 @@ ckh_delete(tsdn_t *tsdn, ckh_t *ckh)
(unsigned long long)ckh->nrelocs);
#endif
idalloctm(tsdn, ckh->tab, NULL, true, true);
idalloctm(tsd_tsdn(tsd), ckh->tab, NULL, true, true);
if (config_debug)
memset(ckh, JEMALLOC_FREE_JUNK, sizeof(ckh_t));
}
@ -456,7 +457,7 @@ ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data)
}
bool
ckh_insert(tsdn_t *tsdn, ckh_t *ckh, const void *key, const void *data)
ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data)
{
bool ret;
@ -468,7 +469,7 @@ ckh_insert(tsdn_t *tsdn, ckh_t *ckh, const void *key, const void *data)
#endif
while (ckh_try_insert(ckh, &key, &data)) {
if (ckh_grow(tsdn, ckh)) {
if (ckh_grow(tsd, ckh)) {
ret = true;
goto label_return;
}
@ -480,7 +481,7 @@ ckh_insert(tsdn_t *tsdn, ckh_t *ckh, const void *key, const void *data)
}
bool
ckh_remove(tsdn_t *tsdn, ckh_t *ckh, const void *searchkey, void **key,
ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key,
void **data)
{
size_t cell;
@ -502,7 +503,7 @@ ckh_remove(tsdn_t *tsdn, ckh_t *ckh, const void *searchkey, void **key,
+ LG_CKH_BUCKET_CELLS - 2)) && ckh->lg_curbuckets
> ckh->lg_minbuckets) {
/* Ignore error due to OOM. */
ckh_shrink(tsdn, ckh);
ckh_shrink(tsd, ckh);
}
return (false);

8
contrib/jemalloc/src/ctl.c
View File

@ -1478,7 +1478,7 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
READONLY();
if (tcaches_create(tsd_tsdn(tsd), &tcache_ind)) {
if (tcaches_create(tsd, &tcache_ind)) {
ret = EFAULT;
goto label_return;
}
@ -1685,11 +1685,11 @@ arena_i_dss_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
dss_prec_old = arena_dss_prec_get(tsd_tsdn(tsd), arena);
} else {
if (dss_prec != dss_prec_limit &&
chunk_dss_prec_set(tsd_tsdn(tsd), dss_prec)) {
chunk_dss_prec_set(dss_prec)) {
ret = EFAULT;
goto label_return;
}
dss_prec_old = chunk_dss_prec_get(tsd_tsdn(tsd));
dss_prec_old = chunk_dss_prec_get();
}
dss = dss_prec_names[dss_prec_old];
@ -2100,7 +2100,7 @@ prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
if (lg_sample >= (sizeof(uint64_t) << 3))
lg_sample = (sizeof(uint64_t) << 3) - 1;
prof_reset(tsd_tsdn(tsd), lg_sample);
prof_reset(tsd, lg_sample);
ret = 0;
label_return:

12
contrib/jemalloc/src/huge.c
View File

@ -54,6 +54,7 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
{
void *ret;
size_t ausize;
arena_t *iarena;
extent_node_t *node;
bool is_zeroed;
@ -67,8 +68,9 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
assert(ausize >= chunksize);
/* Allocate an extent node with which to track the chunk. */
iarena = (!tsdn_null(tsdn)) ? arena_ichoose(tsdn_tsd(tsdn), NULL) : a0get();
node = ipallocztm(tsdn, CACHELINE_CEILING(sizeof(extent_node_t)),
CACHELINE, false, NULL, true, arena_ichoose(tsdn, arena));
CACHELINE, false, NULL, true, iarena);
if (node == NULL)
return (NULL);
@ -114,7 +116,7 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl)
#endif
static void
huge_dalloc_junk(tsdn_t *tsdn, void *ptr, size_t usize)
huge_dalloc_junk(void *ptr, size_t usize)
{
if (config_fill && have_dss && unlikely(opt_junk_free)) {
@ -122,7 +124,7 @@ huge_dalloc_junk(tsdn_t *tsdn, void *ptr, size_t usize)
* Only bother junk filling if the chunk isn't about to be
* unmapped.
*/
if (!config_munmap || (have_dss && chunk_in_dss(tsdn, ptr)))
if (!config_munmap || (have_dss && chunk_in_dss(ptr)))
memset(ptr, JEMALLOC_FREE_JUNK, usize);
}
}
@ -221,7 +223,7 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
if (oldsize > usize) {
size_t sdiff = oldsize - usize;
if (config_fill && unlikely(opt_junk_free)) {
huge_dalloc_junk(tsdn, (void *)((uintptr_t)ptr + usize),
huge_dalloc_junk((void *)((uintptr_t)ptr + usize),
sdiff);
post_zeroed = false;
} else {
@ -402,7 +404,7 @@ huge_dalloc(tsdn_t *tsdn, void *ptr)
ql_remove(&arena->huge, node, ql_link);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
huge_dalloc_junk(tsdn, extent_node_addr_get(node),
huge_dalloc_junk(extent_node_addr_get(node),
extent_node_size_get(node));
arena_chunk_dalloc_huge(tsdn, extent_node_arena_get(node),
extent_node_addr_get(node), extent_node_size_get(node));

138
contrib/jemalloc/src/jemalloc.c
View File

@ -9,7 +9,11 @@ const char *__malloc_options_1_0 = NULL;
__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
/* Runtime configuration options. */
const char *je_malloc_conf JEMALLOC_ATTR(weak);
const char *je_malloc_conf
#ifndef _WIN32
JEMALLOC_ATTR(weak)
#endif
;
bool opt_abort =
#ifdef JEMALLOC_DEBUG
true
@ -89,14 +93,25 @@ enum {
};
static uint8_t malloc_slow_flags;
/* Last entry for overflow detection only. */
JEMALLOC_ALIGNED(CACHELINE)
const size_t index2size_tab[NSIZES+1] = {
#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
const size_t pind2sz_tab[NPSIZES] = {
#define PSZ_yes(lg_grp, ndelta, lg_delta) \
(((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta))),
#define PSZ_no(lg_grp, ndelta, lg_delta)
#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \
PSZ_##psz(lg_grp, ndelta, lg_delta)
SIZE_CLASSES
#undef PSZ_yes
#undef PSZ_no
#undef SC
};
JEMALLOC_ALIGNED(CACHELINE)
const size_t index2size_tab[NSIZES] = {
#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \
((ZU(1)<<lg_grp) + (ZU(ndelta)<<lg_delta)),
SIZE_CLASSES
#undef SC
ZU(0)
};
JEMALLOC_ALIGNED(CACHELINE)
@ -165,7 +180,7 @@ const uint8_t size2index_tab[] = {
#define S2B_11(i) S2B_10(i) S2B_10(i)
#endif
#define S2B_no(i)
#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \
#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, lg_delta_lookup) \
S2B_##lg_delta_lookup(index)
SIZE_CLASSES
#undef S2B_3
@ -333,6 +348,13 @@ a0idalloc(void *ptr, bool is_metadata)
idalloctm(TSDN_NULL, ptr, false, is_metadata, true);
}
arena_t *
a0get(void)
{
return (a0);
}
void *
a0malloc(size_t size)
{
@ -459,15 +481,16 @@ arena_bind(tsd_t *tsd, unsigned ind, bool internal)
{
arena_t *arena;
if (!tsd_nominal(tsd))
return;
arena = arena_get(tsd_tsdn(tsd), ind, false);
arena_nthreads_inc(arena, internal);
if (tsd_nominal(tsd)) {
if (internal)
tsd_iarena_set(tsd, arena);
else
tsd_arena_set(tsd, arena);
}
if (internal)
tsd_iarena_set(tsd, arena);
else
tsd_arena_set(tsd, arena);
}
void
@ -793,6 +816,20 @@ malloc_ncpus(void)
SYSTEM_INFO si;
GetSystemInfo(&si);
result = si.dwNumberOfProcessors;
#elif defined(JEMALLOC_GLIBC_MALLOC_HOOK) && defined(CPU_COUNT)
/*
* glibc >= 2.6 has the CPU_COUNT macro.
*
* glibc's sysconf() uses isspace(). glibc allocates for the first time
* *before* setting up the isspace tables. Therefore we need a
* different method to get the number of CPUs.
*/
{
cpu_set_t set;
pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
result = CPU_COUNT(&set);
}
#else
result = sysconf(_SC_NPROCESSORS_ONLN);
#endif
@ -1107,8 +1144,7 @@ malloc_conf_init(void)
for (i = 0; i < dss_prec_limit; i++) {
if (strncmp(dss_prec_names[i], v, vlen)
== 0) {
if (chunk_dss_prec_set(NULL,
i)) {
if (chunk_dss_prec_set(i)) {
malloc_conf_error(
"Error setting dss",
k, klen, v, vlen);
@ -1153,9 +1189,20 @@ malloc_conf_init(void)
if (config_fill) {
if (CONF_MATCH("junk")) {
if (CONF_MATCH_VALUE("true")) {
opt_junk = "true";
opt_junk_alloc = opt_junk_free =
true;
if (config_valgrind &&
unlikely(in_valgrind)) {
malloc_conf_error(
"Deallocation-time "
"junk filling cannot "
"be enabled while "
"running inside "
"Valgrind", k, klen, v,
vlen);
} else {
opt_junk = "true";
opt_junk_alloc = true;
opt_junk_free = true;
}
} else if (CONF_MATCH_VALUE("false")) {
opt_junk = "false";
opt_junk_alloc = opt_junk_free =
@ -1165,9 +1212,20 @@ malloc_conf_init(void)
opt_junk_alloc = true;
opt_junk_free = false;
} else if (CONF_MATCH_VALUE("free")) {
opt_junk = "free";
opt_junk_alloc = false;
opt_junk_free = true;
if (config_valgrind &&
unlikely(in_valgrind)) {
malloc_conf_error(
"Deallocation-time "
"junk filling cannot "
"be enabled while "
"running inside "
"Valgrind", k, klen, v,
vlen);
} else {
opt_junk = "free";
opt_junk_alloc = false;
opt_junk_free = true;
}
} else {
malloc_conf_error(
"Invalid conf value", k,
@ -1253,11 +1311,14 @@ malloc_init_hard_needed(void)
}
#ifdef JEMALLOC_THREADED_INIT
if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) {
spin_t spinner;
/* Busy-wait until the initializing thread completes. */
spin_init(&spinner);
do {
malloc_mutex_unlock(NULL, &init_lock);
CPU_SPINWAIT;
malloc_mutex_lock(NULL, &init_lock);
malloc_mutex_unlock(TSDN_NULL, &init_lock);
spin_adaptive(&spinner);
malloc_mutex_lock(TSDN_NULL, &init_lock);
} while (!malloc_initialized());
return (false);
}
@ -1291,8 +1352,7 @@ malloc_init_hard_a0_locked()
return (true);
if (config_prof)
prof_boot1();
if (arena_boot())
return (true);
arena_boot();
if (config_tcache && tcache_boot(TSDN_NULL))
return (true);
if (malloc_mutex_init(&arenas_lock, "arenas", WITNESS_RANK_ARENAS))
@ -1423,7 +1483,7 @@ malloc_init_hard(void)
return (true);
malloc_mutex_lock(tsd_tsdn(tsd), &init_lock);
if (config_prof && prof_boot2(tsd_tsdn(tsd))) {
if (config_prof && prof_boot2(tsd)) {
malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock);
return (true);
}
@ -1998,6 +2058,29 @@ JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc;
JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) =
je_memalign;
# endif
#ifdef CPU_COUNT
/*
* To enable static linking with glibc, the libc specific malloc interface must
* be implemented also, so none of glibc's malloc.o functions are added to the
* link.
*/
#define ALIAS(je_fn) __attribute__((alias (#je_fn), used))
/* To force macro expansion of je_ prefix before stringification. */
#define PREALIAS(je_fn) ALIAS(je_fn)
void *__libc_malloc(size_t size) PREALIAS(je_malloc);
void __libc_free(void* ptr) PREALIAS(je_free);
void *__libc_realloc(void* ptr, size_t size) PREALIAS(je_realloc);
void *__libc_calloc(size_t n, size_t size) PREALIAS(je_calloc);
void *__libc_memalign(size_t align, size_t s) PREALIAS(je_memalign);
void *__libc_valloc(size_t size) PREALIAS(je_valloc);
int __posix_memalign(void** r, size_t a, size_t s)
PREALIAS(je_posix_memalign);
#undef PREALIAS
#undef ALIAS
#endif
#endif
/*
@ -2852,7 +2935,6 @@ _malloc_prefork(void)
}
}
base_prefork(tsd_tsdn(tsd));
chunk_prefork(tsd_tsdn(tsd));
for (i = 0; i < narenas; i++) {
if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL)
arena_prefork3(tsd_tsdn(tsd), arena);
@ -2881,7 +2963,6 @@ _malloc_postfork(void)
witness_postfork_parent(tsd);
/* Release all mutexes, now that fork() has completed. */
chunk_postfork_parent(tsd_tsdn(tsd));
base_postfork_parent(tsd_tsdn(tsd));
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena;
@ -2906,7 +2987,6 @@ jemalloc_postfork_child(void)
witness_postfork_child(tsd);
/* Release all mutexes, now that fork() has completed. */
chunk_postfork_child(tsd_tsdn(tsd));
base_postfork_child(tsd_tsdn(tsd));
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena;

2
contrib/jemalloc/src/mutex.c
View File

@ -91,6 +91,8 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank)
_CRT_SPINCOUNT))
return (true);
# endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
mutex->lock = OS_UNFAIR_LOCK_INIT;
#elif (defined(JEMALLOC_OSSPIN))
mutex->lock = 0;
#elif (defined(JEMALLOC_MUTEX_INIT_CB))

98
contrib/jemalloc/src/nstime.c
View File

@ -97,6 +97,76 @@ nstime_divide(const nstime_t *time, const nstime_t *divisor)
return (time->ns / divisor->ns);
}
#ifdef _WIN32
# define NSTIME_MONOTONIC true
static void
nstime_get(nstime_t *time)
{
FILETIME ft;
uint64_t ticks_100ns;
GetSystemTimeAsFileTime(&ft);
ticks_100ns = (((uint64_t)ft.dwHighDateTime) << 32) | ft.dwLowDateTime;
nstime_init(time, ticks_100ns * 100);
}
#elif JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE
# define NSTIME_MONOTONIC true
static void
nstime_get(nstime_t *time)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC_COARSE, &ts);
nstime_init2(time, ts.tv_sec, ts.tv_nsec);
}
#elif JEMALLOC_HAVE_CLOCK_MONOTONIC
# define NSTIME_MONOTONIC true
static void
nstime_get(nstime_t *time)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
nstime_init2(time, ts.tv_sec, ts.tv_nsec);
}
#elif JEMALLOC_HAVE_MACH_ABSOLUTE_TIME
# define NSTIME_MONOTONIC true
static void
nstime_get(nstime_t *time)
{
nstime_init(time, mach_absolute_time());
}
#else
# define NSTIME_MONOTONIC false
static void
nstime_get(nstime_t *time)
{
struct timeval tv;
gettimeofday(&tv, NULL);
nstime_init2(time, tv.tv_sec, tv.tv_usec * 1000);
}
#endif
#ifdef JEMALLOC_JET
#undef nstime_monotonic
#define nstime_monotonic JEMALLOC_N(n_nstime_monotonic)
#endif
bool
nstime_monotonic(void)
{
return (NSTIME_MONOTONIC);
#undef NSTIME_MONOTONIC
}
#ifdef JEMALLOC_JET
#undef nstime_monotonic
#define nstime_monotonic JEMALLOC_N(nstime_monotonic)
nstime_monotonic_t *nstime_monotonic = JEMALLOC_N(n_nstime_monotonic);
#endif
#ifdef JEMALLOC_JET
#undef nstime_update
#define nstime_update JEMALLOC_N(n_nstime_update)
@ -107,33 +177,7 @@ nstime_update(nstime_t *time)
nstime_t old_time;
nstime_copy(&old_time, time);
#ifdef _WIN32
{
FILETIME ft;
uint64_t ticks;
GetSystemTimeAsFileTime(&ft);
ticks = (((uint64_t)ft.dwHighDateTime) << 32) |
ft.dwLowDateTime;
time->ns = ticks * 100;
}
#elif JEMALLOC_CLOCK_GETTIME
{
struct timespec ts;
if (sysconf(_SC_MONOTONIC_CLOCK) > 0)
clock_gettime(CLOCK_MONOTONIC, &ts);
else
clock_gettime(CLOCK_REALTIME, &ts);
time->ns = ts.tv_sec * BILLION + ts.tv_nsec;
}
#else
{
struct timeval tv;
gettimeofday(&tv, NULL);
time->ns = tv.tv_sec * BILLION + tv.tv_usec * 1000;
}
#endif
nstime_get(time);
/* Handle non-monotonic clocks. */
if (unlikely(nstime_compare(&old_time, time) > 0)) {

20
contrib/jemalloc/src/pages.c
View File

@ -207,6 +207,11 @@ os_overcommits_sysctl(void)
#endif
#ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
/*
* Use syscall(2) rather than {open,read,close}(2) when possible to avoid
* reentry during bootstrapping if another library has interposed system call
* wrappers.
*/
static bool
os_overcommits_proc(void)
{
@ -214,11 +219,26 @@ os_overcommits_proc(void)
char buf[1];
ssize_t nread;
#if defined(JEMALLOC_HAVE_SYSCALL) && defined(SYS_open)
fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY);
#else
fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
#endif
if (fd == -1)
return (false); /* Error. */
#if defined(JEMALLOC_HAVE_SYSCALL) && defined(SYS_read)
nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf));
#else
nread = read(fd, &buf, sizeof(buf));
#endif
#if defined(JEMALLOC_HAVE_SYSCALL) && defined(SYS_close)
syscall(SYS_close, fd);
#else
close(fd);
#endif
if (nread < 1)
return (false); /* Error. */
/*

Some files were not shown because too many files have changed in this diff Show More