[PATCH][2.5] VMWare doesn't like sysenter

[Zwane Mwaikambo]

I get a monitor error in VMWare4 with a sysenter syscall enabled kernel, 
this patch simply disables sysenter based syscalls but doesn't clear the 
SEP bit in the capabilities.

Index: linux-2.5.69-mm5/arch/i386/kernel/sysenter.c
===================================================================
RCS file: /build/cvsroot/linux-2.5.69/arch/i386/kernel/sysenter.c,v
retrieving revision 1.1.1.1
diff -u -p -B -r1.1.1.1 sysenter.c
--- linux-2.5.69-mm5/arch/i386/kernel/sysenter.c	6 May 2003 12:20:51 -0000	1.1.1.1
+++ linux-2.5.69-mm5/arch/i386/kernel/sysenter.c	15 May 2003 07:46:05 -0000
@@ -20,6 +20,7 @@
 #include <asm/unistd.h>
 
 extern asmlinkage void sysenter_entry(void);
+static int nosysenter __initdata;
 
 /*
  * Create a per-cpu fake "SEP thread" stack, so that we can
@@ -51,6 +52,13 @@ void enable_sep_cpu(void *info)
 	put_cpu();	
 }
 
+static int __init do_nosysenter(char *s)
+{
+	nosysenter = 1;
+	return 1;
+}
+__setup("nosysenter", do_nosysenter);
+
 /*
  * These symbols are defined by vsyscall.o to mark the bounds
  * of the ELF DSO images included therein.
@@ -64,7 +72,7 @@ static int __init sysenter_setup(void)
 
 	__set_fixmap(FIX_VSYSCALL, __pa(page), PAGE_READONLY);
 
-	if (!boot_cpu_has(X86_FEATURE_SEP)) {
+	if (nosysenter || !boot_cpu_has(X86_FEATURE_SEP)) {
 		memcpy((void *) page,
 		       &vsyscall_int80_start,
 		       &vsyscall_int80_end - &vsyscall_int80_start);

-- 
function.linuxpower.ca