Add MAP_NOCORE to mmap(2), and MADV_NOCORE and MADV_CORE to madvise(2).

This
This feature allows you to specify if mmap'd data is included in
an application's corefile.

Change the type of eflags in struct vm_map_entry from u_char to
vm_eflags_t (an unsigned int).

Reviewed by:	dillon,jdp,alfred
Approved by:	jkh

lib/libc/sys/madvise.2

@@ -60,6 +60,8 @@ The known behaviors are given in
 #define	MADV_FREE	5 /* data is now unimportant */
 #define MADV_NOSYNC	6 /* no explicit commit to physical backing store */
 #define MADV_AUTOSYNC	7 /* default commit method to physical backing store */
+#define MADV_NOCORE	8 /* do not include these pages in a core file */
+#define MADV_CORE	9 /* revert to including pages in a core file */
 .Ed
 .Pp
 .Bl -tag -width MADV_SEQUENTIAL 
@@ -123,6 +125,10 @@ may or may not be reverted.  You can guarentee reversion by using the
 or
 .Xr fsync 2
 system calls.
+.It Dv MADV_NOCORE
+Region is not included in a core file.
+.It Dv MADV_CORE
+Include region in a core file.
 .El
 .Sh RETURN VALUES
 Upon successful completion,

lib/libc/sys/mmap.2

@@ -179,6 +179,8 @@ system call is obsolete since
 implements a coherent filesystem buffer cache.  However, it may be 
 used to associate dirty VM pages with filesystem buffers and thus cause
 them to be flushed to physical media sooner rather then later.
+.It Dv MAP_NOCORE
+Region is not included in a core file.
 .El
 .Pp
 The 

sys/kern/imgact_elf.c

@@ -816,6 +816,13 @@ each_writable_segment(p, func, closure)
 		    (VM_PROT_READ|VM_PROT_WRITE))
 			continue;
 
+		/*
+		** Dont include mmapped data in the coredump if MAP_NOCORE 
+		** is set in mmap(2).
+		*/
+		if (entry->eflags & MAP_ENTRY_NOCOREDUMP)
+			continue;
+
 		if ((obj = entry->object.vm_object) == NULL)
 			continue;
 

sys/sys/mman.h

@@ -65,7 +65,12 @@
 #define	MAP_NOEXTEND	 0x0100	/* for MAP_FILE, don't change file size */
 #define	MAP_HASSEMAPHORE 0x0200	/* region may contain semaphores */
 #define	MAP_STACK	 0x0400	/* region grows down, like a stack */
-#define MAP_NOSYNC	 0x0800 /* page to but do not sync underlying file */
+#define	MAP_NOSYNC	 0x0800 /* page to but do not sync underlying file */
+
+/*
+ * Extended flags
+ */
+#define	MAP_NOCORE	 0x00020000 /* dont include these pages in a coredump */
 
 #ifdef _P1003_1B_VISIBLE
 /*
@@ -106,6 +111,8 @@
 #define	MADV_FREE	5	/* dont need these pages, and junk contents */
 #define	MADV_NOSYNC	6	/* try to avoid flushes to physical media */
 #define	MADV_AUTOSYNC	7	/* revert to default flushing strategy */
+#define	MADV_NOCORE	8	/* do not include these pages in a core file */
+#define	MADV_CORE	9	/* revert to including pages in a core file */
 
 /*
  * Return bits from mincore

sys/vm/vm_map.c

@@ -427,7 +427,7 @@ vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
 	vm_map_entry_t new_entry;
 	vm_map_entry_t prev_entry;
 	vm_map_entry_t temp_entry;
-	u_char protoeflags;
+	vm_eflags_t protoeflags;
 
 	/*
 	 * Check that the start and end points are not bogus.
@@ -468,6 +468,8 @@ vm_map_insert(vm_map_t map, vm_object_t object, vm_ooffset_t offset,
 	}
 	if (cow & MAP_DISABLE_SYNCER)
 		protoeflags |= MAP_ENTRY_NOSYNC;
+	if (cow & MAP_DISABLE_COREDUMP)
+		protoeflags |= MAP_ENTRY_NOCOREDUMP;
 
 	if (object) {
 		/*
@@ -1039,6 +1041,8 @@ vm_map_madvise(map, start, end, behav)
 	case MADV_RANDOM:
 	case MADV_NOSYNC:
 	case MADV_AUTOSYNC:
+	case MADV_NOCORE:
+	case MADV_CORE:
 		modify_map = 1;
 		vm_map_lock(map);
 		break;
@@ -1096,6 +1100,12 @@ vm_map_madvise(map, start, end, behav)
 			case MADV_AUTOSYNC:
 				current->eflags &= ~MAP_ENTRY_NOSYNC;
 				break;
+			case MADV_NOCORE:
+				current->eflags |= MAP_ENTRY_NOCOREDUMP;
+				break;
+			case MADV_CORE:
+				current->eflags &= ~MAP_ENTRY_NOCOREDUMP;
+				break;
 			default:
 				break;
 			}

sys/vm/vm_map.h

@@ -78,6 +78,8 @@
  *	vm_map_entry_t		an entry in an address map.
  */
 
+typedef u_int vm_eflags_t;
+
 /*
  *	Objects which live in maps may be either VM objects, or
  *	another map (called a "sharing map") which denotes read-write
@@ -103,7 +105,7 @@ struct vm_map_entry {
 	vm_offset_t avail_ssize;	/* amt can grow if this is a stack */
 	union vm_map_object object;	/* object I point to */
 	vm_ooffset_t offset;		/* offset into object */
-	u_char eflags;			/* map entry flags */
+	vm_eflags_t eflags;		/* map entry flags */
 	/* Only in task maps: */
 	vm_prot_t protection;		/* protection code */
 	vm_prot_t max_protection;	/* maximum protection */
@@ -112,19 +114,21 @@ struct vm_map_entry {
 	vm_pindex_t lastr;		/* last read */
 };
 
-#define MAP_ENTRY_NOSYNC		0x1
-#define MAP_ENTRY_IS_SUB_MAP		0x2
-#define MAP_ENTRY_COW			0x4
-#define MAP_ENTRY_NEEDS_COPY		0x8
-#define MAP_ENTRY_NOFAULT		0x10
-#define MAP_ENTRY_USER_WIRED		0x20
+#define MAP_ENTRY_NOSYNC		0x0001
+#define MAP_ENTRY_IS_SUB_MAP		0x0002
+#define MAP_ENTRY_COW			0x0004
+#define MAP_ENTRY_NEEDS_COPY		0x0008
+#define MAP_ENTRY_NOFAULT		0x0010
+#define MAP_ENTRY_USER_WIRED		0x0020
 
-#define MAP_ENTRY_BEHAV_NORMAL		0x00	/* default behavior */
-#define MAP_ENTRY_BEHAV_SEQUENTIAL	0x40	/* expect sequential access */
-#define MAP_ENTRY_BEHAV_RANDOM		0x80	/* expect random access */
-#define MAP_ENTRY_BEHAV_RESERVED	0xC0	/* future use */
+#define MAP_ENTRY_BEHAV_NORMAL		0x0000	/* default behavior */
+#define MAP_ENTRY_BEHAV_SEQUENTIAL	0x0040	/* expect sequential access */
+#define MAP_ENTRY_BEHAV_RANDOM		0x0080	/* expect random access */
+#define MAP_ENTRY_BEHAV_RESERVED	0x00C0	/* future use */
 
-#define MAP_ENTRY_BEHAV_MASK		0xC0
+#define MAP_ENTRY_BEHAV_MASK		0x00C0
+
+#define MAP_ENTRY_NOCOREDUMP		0x0400	/* don't include in a core */
 
 static __inline u_char   
 vm_map_entry_behavior(struct vm_map_entry *entry)
@@ -324,12 +328,13 @@ vmspace_resident_count(struct vmspace *vmspace)
 /*
  * Copy-on-write flags for vm_map operations
  */
-#define MAP_UNUSED_01		0x1
-#define MAP_COPY_ON_WRITE	0x2
-#define MAP_NOFAULT		0x4
-#define MAP_PREFAULT		0x8
-#define MAP_PREFAULT_PARTIAL	0x10
-#define MAP_DISABLE_SYNCER	0x20
+#define MAP_UNUSED_01		0x0001
+#define MAP_COPY_ON_WRITE	0x0002
+#define MAP_NOFAULT		0x0004
+#define MAP_PREFAULT		0x0008
+#define MAP_PREFAULT_PARTIAL	0x0010
+#define MAP_DISABLE_SYNCER	0x0020
+#define MAP_DISABLE_COREDUMP	0x0100
 
 /*
  * vm_fault option flags

sys/vm/vm_mmap.c

@@ -664,7 +664,7 @@ madvise(p, uap)
 	/*
 	 * Check for illegal behavior
 	 */
-	if (uap->behav < 0 || uap->behav > MADV_AUTOSYNC)
+	if (uap->behav < 0 || uap->behav > MADV_CORE)
 		return (EINVAL);
 	/*
 	 * Check for illegal addresses.  Watch out for address wrap... Note
@@ -1095,6 +1095,8 @@ vm_mmap(vm_map_t map, vm_offset_t *addr, vm_size_t size, vm_prot_t prot,
 		docow |= MAP_COPY_ON_WRITE;
 	if (flags & MAP_NOSYNC)
 		docow |= MAP_DISABLE_SYNCER;
+	if (flags & MAP_NOCORE)
+		docow |= MAP_DISABLE_COREDUMP;
 
 #if defined(VM_PROT_READ_IS_EXEC)
 	if (prot & VM_PROT_READ)