Fully support constructors for the purpose of code coverage analysis.

This involves:
1.  Have the loader pass the start and size of the .ctors section to the
    kernel in 2 new metadata elements.
2.  Have the linker backends look for and record the start and size of
    the .ctors section in dynamically loaded modules.
3.  Have the linker backends call the constructors as part of the final
    work of initializing preloaded or dynamically loaded modules.

Note that LLVM appends the priority of the constructors to the name of
the .ctors section. Not so when compiling with GCC. The code currently
works for GCC and not for LLVM.

Submitted by:	Dmitry Mikulin <dmitrym@juniper.net>
Obtained from:	Juniper Networks, Inc.

sys/boot/common/load_elf.c

@@ -240,6 +240,7 @@ __elfN(loadimage)(struct preloaded_file *fp, elf_file_t ef, u_int64_t off)
     Elf_Ehdr	*ehdr;
     Elf_Phdr	*phdr, *php;
     Elf_Shdr	*shdr;
+    char	*shstr;
     int		ret;
     vm_offset_t firstaddr;
     vm_offset_t lastaddr;
@@ -248,6 +249,7 @@ __elfN(loadimage)(struct preloaded_file *fp, elf_file_t ef, u_int64_t off)
     Elf_Addr	ssym, esym;
     Elf_Dyn	*dp;
     Elf_Addr	adp;
+    Elf_Addr	ctors;
     int		ndp;
     int		symstrindex;
     int		symtabindex;
@@ -383,10 +385,11 @@ __elfN(loadimage)(struct preloaded_file *fp, elf_file_t ef, u_int64_t off)
     lastaddr = roundup(lastaddr, sizeof(long));
 
     /*
-     * Now grab the symbol tables.  This isn't easy if we're reading a
-     * .gz file.  I think the rule is going to have to be that you must
-     * strip a file to remove symbols before gzipping it so that we do not
-     * try to lseek() on it.
+     * Get the section headers.  We need this for finding the .ctors
+     * section as well as for loading any symbols.  Both may be hard
+     * to do if reading from a .gz file as it involves seeking.  I
+     * think the rule is going to have to be that you must strip a
+     * file to remove symbols before gzipping it.
      */
     chunk = ehdr->e_shnum * ehdr->e_shentsize;
     if (chunk == 0 || ehdr->e_shoff == 0)
@@ -399,6 +402,33 @@ __elfN(loadimage)(struct preloaded_file *fp, elf_file_t ef, u_int64_t off)
     }
     file_addmetadata(fp, MODINFOMD_SHDR, chunk, shdr);
 
+    /*
+     * Read the section string table and look for the .ctors section.
+     * We need to tell the kernel where it is so that it can call the
+     * ctors.
+     */
+    chunk = shdr[ehdr->e_shstrndx].sh_size;
+    if (chunk) {
+	shstr = alloc_pread(ef->fd, shdr[ehdr->e_shstrndx].sh_offset, chunk);
+	if (shstr) {
+	    for (i = 0; i < ehdr->e_shnum; i++) {
+		if (strcmp(shstr + shdr[i].sh_name, ".ctors") != 0)
+		    continue;
+		ctors = shdr[i].sh_addr;
+		file_addmetadata(fp, MODINFOMD_CTORS_ADDR, sizeof(ctors),
+		    &ctors);
+		size = shdr[i].sh_size;
+		file_addmetadata(fp, MODINFOMD_CTORS_SIZE, sizeof(size),
+		    &size);
+		break;
+	    }
+	    free(shstr);
+	}
+    }
+
+    /*
+     * Now load any symbols.
+     */
     symtabindex = -1;
     symstrindex = -1;
     for (i = 0; i < ehdr->e_shnum; i++) {

sys/kern/kern_linker.c

@@ -573,6 +573,8 @@ linker_make_file(const char *pathname, linker_class_t lc)
 	lf = (linker_file_t)kobj_create((kobj_class_t)lc, M_LINKER, M_WAITOK);
 	if (lf == NULL)
 		return (NULL);
+	lf->ctors_addr = 0;
+	lf->ctors_size = 0;
 	lf->refs = 1;
 	lf->userrefs = 0;
 	lf->flags = 0;

sys/kern/link_elf.c

@@ -331,6 +331,22 @@ link_elf_error(const char *filename, const char *s)
 		printf("kldload: %s: %s\n", filename, s);
 }
 
+static void
+link_elf_invoke_ctors(caddr_t addr, size_t size)
+{
+	void (**ctor)(void);
+	size_t i, cnt;
+
+	if (addr == NULL || size == 0)
+		return;
+	cnt = size / sizeof(*ctor);
+	ctor = (void *)addr;
+	for (i = 0; i < cnt; i++) {
+		if (ctor[i] != NULL)
+			(*ctor[i])();
+	}
+}
+
 /*
  * Actions performed after linking/loading both the preloaded kernel and any
  * modules; whether preloaded or dynamicly loaded.
@@ -360,6 +376,8 @@ link_elf_link_common_finish(linker_file_t lf)
 	GDB_STATE(RT_CONSISTENT);
 #endif
 
+	/* Invoke .ctors */
+	link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size);
 	return (0);
 }
 
@@ -367,6 +385,8 @@ static void
 link_elf_init(void* arg)
 {
 	Elf_Dyn *dp;
+	Elf_Addr *ctors_addrp;
+	Elf_Size *ctors_sizep;
 	caddr_t modptr, baseptr, sizeptr;
 	elf_file_t ef;
 	char *modname;
@@ -408,6 +428,15 @@ link_elf_init(void* arg)
 		sizeptr = preload_search_info(modptr, MODINFO_SIZE);
 		if (sizeptr != NULL)
 			linker_kernel_file->size = *(size_t *)sizeptr;
+		ctors_addrp = (Elf_Addr *)preload_search_info(modptr,
+			MODINFO_METADATA | MODINFOMD_CTORS_ADDR);
+		ctors_sizep = (Elf_Size *)preload_search_info(modptr,
+			MODINFO_METADATA | MODINFOMD_CTORS_SIZE);
+		if (ctors_addrp != NULL && ctors_sizep != NULL) {
+			linker_kernel_file->ctors_addr = ef->address +
+			    *ctors_addrp;
+			linker_kernel_file->ctors_size = *ctors_sizep;
+		}
 	}
 	(void)link_elf_preload_parse_symbols(ef);
 
@@ -635,6 +664,8 @@ static int
 link_elf_link_preload(linker_class_t cls,
     const char* filename, linker_file_t *result)
 {
+	Elf_Addr *ctors_addrp;
+	Elf_Size *ctors_sizep;
 	caddr_t modptr, baseptr, sizeptr, dynptr;
 	char *type;
 	elf_file_t ef;
@@ -675,6 +706,15 @@ link_elf_link_preload(linker_class_t cls,
 	lf->address = ef->address;
 	lf->size = *(size_t *)sizeptr;
 
+	ctors_addrp = (Elf_Addr *)preload_search_info(modptr,
+	    MODINFO_METADATA | MODINFOMD_CTORS_ADDR);
+	ctors_sizep = (Elf_Size *)preload_search_info(modptr,
+	    MODINFO_METADATA | MODINFOMD_CTORS_SIZE);
+	if (ctors_addrp != NULL && ctors_sizep != NULL) {
+		lf->ctors_addr = ef->address + *ctors_addrp;
+		lf->ctors_size = *ctors_sizep;
+	}
+
 	error = parse_dynamic(ef);
 	if (error == 0)
 		error = parse_dpcpu(ef);
@@ -734,11 +774,14 @@ link_elf_load_file(linker_class_t cls, const char* filename,
 	Elf_Shdr *shdr;
 	int symtabindex;
 	int symstrindex;
+	int shstrindex;
 	int symcnt;
 	int strcnt;
+	char *shstrs;
 
 	shdr = NULL;
 	lf = NULL;
+	shstrs = NULL;
 
 	NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, filename, td);
 	flags = FREAD;
@@ -977,12 +1020,31 @@ link_elf_load_file(linker_class_t cls, const char* filename,
 	    &resid, td);
 	if (error != 0)
 		goto out;
+
+	/* Read section string table */
+	shstrindex = hdr->e_shstrndx;
+	if (shstrindex != 0 && shdr[shstrindex].sh_type == SHT_STRTAB &&
+	    shdr[shstrindex].sh_size != 0) {
+		nbytes = shdr[shstrindex].sh_size;
+		shstrs = malloc(nbytes, M_LINKER, M_WAITOK | M_ZERO);
+		error = vn_rdwr(UIO_READ, nd.ni_vp, (caddr_t)shstrs, nbytes,
+		    shdr[shstrindex].sh_offset, UIO_SYSSPACE, IO_NODELOCKED,
+		    td->td_ucred, NOCRED, &resid, td);
+		if (error)
+			goto out;
+	}
+
 	symtabindex = -1;
 	symstrindex = -1;
 	for (i = 0; i < hdr->e_shnum; i++) {
 		if (shdr[i].sh_type == SHT_SYMTAB) {
 			symtabindex = i;
 			symstrindex = shdr[i].sh_link;
+		} else if (shstrs != NULL && shdr[i].sh_name != 0 &&
+		    strcmp(shstrs + shdr[i].sh_name, ".ctors") == 0) {
+			/* Record relocated address and size of .ctors. */
+			lf->ctors_addr = mapbase + shdr[i].sh_addr - base_vaddr;
+			lf->ctors_size = shdr[i].sh_size;
 		}
 	}
 	if (symtabindex < 0 || symstrindex < 0)
@@ -1027,6 +1089,8 @@ link_elf_load_file(linker_class_t cls, const char* filename,
 		free(shdr, M_LINKER);
 	if (firstpage != NULL)
 		free(firstpage, M_LINKER);
+	if (shstrs != NULL)
+		free(shstrs, M_LINKER);
 
 	return (error);
 }

sys/kern/link_elf_obj.c

@@ -363,6 +363,10 @@ link_elf_link_preload(linker_class_t cls, const char *filename,
 				vnet_data_copy(vnet_data, shdr[i].sh_size);
 				ef->progtab[pb].addr = vnet_data;
 #endif
+			} else if (ef->progtab[pb].name != NULL &&
+			    !strcmp(ef->progtab[pb].name, ".ctors")) {
+				lf->ctors_addr = ef->progtab[pb].addr;
+				lf->ctors_size = shdr[i].sh_size;
 			}
 
 			/* Update all symbol values with the offset. */
@@ -408,6 +412,22 @@ link_elf_link_preload(linker_class_t cls, const char *filename,
 	return (error);
 }
 
+static void
+link_elf_invoke_ctors(caddr_t addr, size_t size)
+{
+	void (**ctor)(void);
+	size_t i, cnt;
+
+	if (addr == NULL || size == 0)
+		return;
+	cnt = size / sizeof(*ctor);
+	ctor = (void *)addr;
+	for (i = 0; i < cnt; i++) {
+		if (ctor[i] != NULL)
+			(*ctor[i])();
+	}
+}
+
 static int
 link_elf_link_preload_finish(linker_file_t lf)
 {
@@ -424,6 +444,8 @@ link_elf_link_preload_finish(linker_file_t lf)
 	if (error)
 		return (error);
 
+	/* Invoke .ctors */
+	link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size);
 	return (0);
 }
 
@@ -727,10 +749,14 @@ link_elf_load_file(linker_class_t cls, const char *filename,
 			alignmask = shdr[i].sh_addralign - 1;
 			mapbase += alignmask;
 			mapbase &= ~alignmask;
-			if (ef->shstrtab && shdr[i].sh_name != 0)
+			if (ef->shstrtab != NULL && shdr[i].sh_name != 0) {
 				ef->progtab[pb].name =
 				    ef->shstrtab + shdr[i].sh_name;
-			else if (shdr[i].sh_type == SHT_PROGBITS)
+				if (!strcmp(ef->progtab[pb].name, ".ctors")) {
+					lf->ctors_addr = (caddr_t)mapbase;
+					lf->ctors_size = shdr[i].sh_size;
+				}
+			} else if (shdr[i].sh_type == SHT_PROGBITS)
 				ef->progtab[pb].name = "<<PROGBITS>>";
 			else
 				ef->progtab[pb].name = "<<NOBITS>>";
@@ -860,6 +886,9 @@ link_elf_load_file(linker_class_t cls, const char *filename,
 	if (error)
 		goto out;
 
+	/* Invoke .ctors */
+	link_elf_invoke_ctors(lf->ctors_addr, lf->ctors_size);
+
 	*result = lf;
 
 out:

sys/kern/subr_prof.c

@@ -540,50 +540,3 @@ addupc_task(struct thread *td, uintfptr_t pc, u_int ticks)
 		stopprofclock(p);
 	PROC_UNLOCK(p);
 }
-
-#if (defined(__amd64__) || defined(__i386__)) && \
-	defined(__GNUCLIKE_CTOR_SECTION_HANDLING)
-/*
- * Support for "--test-coverage --profile-arcs" in GCC.
- *
- * We need to call all the functions in the .ctor section, in order
- * to get all the counter-arrays strung into a list.
- *
- * XXX: the .ctors call __bb_init_func which is located in over in 
- * XXX: i386/i386/support.s for historical reasons.  There is probably
- * XXX: no reason for that to be assembler anymore, but doing it right
- * XXX: in MI C code requires one to reverse-engineer the type-selection
- * XXX: inside GCC.  Have fun.
- *
- * XXX: Worrisome perspective: Calling the .ctors may make C++ in the
- * XXX: kernel feasible.  Don't.
- */
-typedef void (*ctor_t)(void);
-extern ctor_t _start_ctors, _stop_ctors;
-
-static void
-tcov_init(void *foo __unused)
-{
-	ctor_t *p, q;
-
-	for (p = &_start_ctors; p < &_stop_ctors; p++) {
-		q = *p;
-		q();
-	}
-}
-
-SYSINIT(tcov_init, SI_SUB_KPROF, SI_ORDER_SECOND, tcov_init, NULL);
-
-/*
- * GCC contains magic to recognize calls to for instance execve() and
- * puts in calls to this function to preserve the profile counters.
- * XXX: Put zinging punchline here.
- */
-void __bb_fork_func(void);
-void
-__bb_fork_func(void)
-{
-}
-
-#endif
-

sys/sys/linker.h

@@ -79,6 +79,8 @@ struct linker_file {
     int			id;		/* unique id */
     caddr_t		address;	/* load address */
     size_t		size;		/* size of file */
+    caddr_t		ctors_addr;	/* address of .ctors */
+    size_t		ctors_size;	/* size of .ctors */
     int			ndeps;		/* number of dependencies */
     linker_file_t*	deps;		/* list of dependencies */
     STAILQ_HEAD(, common_symbol) common; /* list of common symbols */
@@ -211,6 +213,8 @@ void *linker_hwpmc_list_objects(void);
 #define MODINFOMD_KERNEND	0x0008		/* kernend */
 #endif
 #define MODINFOMD_SHDR		0x0009		/* section header table */
+#define MODINFOMD_CTORS_ADDR	0x000a		/* address of .ctors */
+#define MODINFOMD_CTORS_SIZE	0x000b		/* size of .ctors */
 #define MODINFOMD_NOCOPY	0x8000		/* don't copy this metadata to the kernel */
 
 #define MODINFOMD_DEPLIST	(0x4001 | MODINFOMD_NOCOPY)	/* depends on */