From: Brian Maly <bmaly@redhat.com>
Subject: Re: [RHEL5 patch] x86_64: fix execshield randomization for heap
Date: Wed, 08 Nov 2006 13:34:11 -0500
Bugzilla: 214548
Message-Id: <45522323.7020909@redhat.com>
Changelog: x86_64: fix execshield randomization for heap

Currently execshield randomization for heap (i.e. /proc/self/maps) works
only on i386, but randomization is broken on x86_64. This patch adds
this heap randomization functionality for x86_64 and resolves the issue.

This item was a beta blocker (see BZ 213492). Randomization is currenly
broken on x86_64 for heap, vdso and mmap. Since each is a different
issue, I am breaking this BZ down into seperate BZ's. This issue was the
only one straigtforward enough to resolve by codefreeze.

I tested this with randtest.c (as well as observing randomization in
/proc/self/maps)

Brian

--- linux-2.6.18.noarch/arch/x86_64/kernel/process.c.orig
+++ linux-2.6.18.noarch/arch/x86_64/kernel/process.c
@@ -826,3 +826,45 @@ unsigned long arch_align_stack(unsigned 
 		sp -= get_random_int() % 8192;
 	return sp & ~0xf;
 }
+
+/*
+ * Generate random brk address between 128MB and 196MB. (if the layout
+ * allows it.)
+ */
+void randomize_brk(unsigned long old_brk)
+{
+        unsigned long new_brk, range_start, range_end;
+
+#ifdef CONFIG_IA32_EMULATION
+	/* randomize_brk is called after SET_PERSONALITY, but before
+	   flush_thread.  So, the test if the process will be 32-bit
+	   or 64-bit is uglier.  */
+	if (test_thread_flag(TIF_ABI_PENDING) ^ test_thread_flag(TIF_IA32))
+	{
+		/* i?86 ELF32 binaries start at 0x8048000 or sometimes
+		   a little bit lower when prelinked.  */
+		range_start = 0x08000000;
+		if (current->mm->brk >= range_start)
+			range_start = current->mm->brk;
+		range_end = range_start + 0x02000000;
+		if (range_start >= 0xfe000000UL)
+			return;
+	} else
+#endif
+	{
+		/* x86-64 ELF64 binaries start at 4MB */
+		range_start = 0x00400000;
+		if (current->mm->brk >= range_start)
+			range_start = current->mm->brk;
+		/* For most binaries, try to keep heap start within low .5GB
+		   (so that usually whole heap fits within first 1GB (first PUD
+		   table)).  */
+		range_end = 0x20000000
+			    + ((range_start > 0x10000000) ? range_start : 0);
+	}
+
+	new_brk = randomize_range(range_start, range_end, 0);
+	if (new_brk)
+		current->mm->brk = new_brk;
+
+}
--- linux-2.6.18.noarch/arch/x86_64/kernel/process-xen.c.orig
+++ linux-2.6.18.noarch/arch/x86_64/kernel/process-xen.c
@@ -780,3 +780,45 @@ void _restore_vcpu(void)
 {
 }
 #endif
+
+/*
+ * Generate random brk address between 128MB and 196MB. (if the layout
+ * allows it.)
+ */
+void randomize_brk(unsigned long old_brk)
+{
+	unsigned long new_brk, range_start, range_end;
+
+#ifdef CONFIG_IA32_EMULATION
+	/* randomize_brk is called after SET_PERSONALITY, but before
+	   flush_thread.  So, the test if the process will be 32-bit
+	   or 64-bit is uglier.  */
+	if (test_thread_flag(TIF_ABI_PENDING) ^ test_thread_flag(TIF_IA32))
+	{
+		/* i?86 ELF32 binaries start at 0x8048000 or sometimes
+		   a little bit lower when prelinked.  */
+		range_start = 0x08000000;
+		if (current->mm->brk >= range_start)
+			range_start = current->mm->brk;
+		range_end = range_start + 0x02000000;
+		if (range_start >= 0xfe000000UL)
+			return;
+	} else
+#endif
+	{
+		/* x86-64 ELF64 binaries start at 4MB */
+		range_start = 0x00400000;
+		if (current->mm->brk >= range_start)
+			range_start = current->mm->brk;
+		/* For most binaries, try to keep heap start within low .5GB
+		   (so that usually whole heap fits within first 1GB (first PUD
+		   table)).  */
+		range_end = 0x20000000
+			    + ((range_start > 0x10000000) ? range_start : 0);
+	}
+
+	new_brk = randomize_range(range_start, range_end, 0);
+	if (new_brk)
+		current->mm->brk = new_brk;
+
+}
--- linux-2.6.18.noarch/include/asm-x86_64/elf.h.orig
+++ linux-2.6.18.noarch/include/asm-x86_64/elf.h
@@ -165,4 +165,7 @@ extern int dump_task_fpu (struct task_st
 
 #endif
 
+#define __HAVE_ARCH_RANDOMIZE_BRK
+extern void randomize_brk(unsigned long old_brk);
+
 #endif