[PATCH 0/15] KVM userspace interface updates

[PATCH 0/15] KVM userspace interface updates

[Avi Kivity]

This patchset updates the kvm userspace interface to what I hope will
be the long-term stable interface.  Provisions are included for extending
the interface later.  The patches address performance and cleanliness
concerns.

One patch is missing -- I'd like the string pio transfers not to include
guest virtual addresses.  To date all my attempts to write the patch ended
with me losing consiousness.  Hopefully I'll manage it soon.

I'd like to submit the patchset post 2.6.21.  Comments are welcome.

[PATCH 01/15] KVM: Use a shared page for kernel/user communication when runing a vcpu

[Avi Kivity]

Instead of passing a 'struct kvm_run' back and forth between the kernel and
userspace, allocate a page and allow the user to mmap() it.  This reduces
needless copying and makes the interface expandable by providing lots of
free space.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/kvm.h      |    1 +
 drivers/kvm/kvm_main.c |   54 +++++++++++++++++++++++++++++++++++------------
 include/linux/kvm.h    |    6 ++--
 3 files changed, 44 insertions(+), 17 deletions(-)
 mode change 100755 => 100644 drivers/kvm/kvm_main.c

diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h
index 0d122bf..901b8d9 100644
--- a/drivers/kvm/kvm.h
+++ b/drivers/kvm/kvm.h
@@ -228,6 +228,7 @@ struct kvm_vcpu {
 	struct mutex mutex;
 	int   cpu;
 	int   launched;
+	struct kvm_run *run;
 	int interrupt_window_open;
 	unsigned long irq_summary; /* bit vector: 1 per word in irq_pending */
 #define NR_IRQ_WORDS KVM_IRQ_BITMAP_SIZE(unsigned long)
diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
old mode 100755
new mode 100644
index 946ed86..42be8a8
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -355,6 +355,8 @@ static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
 	kvm_mmu_destroy(vcpu);
 	vcpu_put(vcpu);
 	kvm_arch_ops->vcpu_free(vcpu);
+	free_page((unsigned long)vcpu->run);
+	vcpu->run = NULL;
 }
 
 static void kvm_free_vcpus(struct kvm *kvm)
@@ -1887,6 +1889,33 @@ static int kvm_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu,
 	return r;
 }
 
+static struct page *kvm_vcpu_nopage(struct vm_area_struct *vma,
+				    unsigned long address,
+				    int *type)
+{
+	struct kvm_vcpu *vcpu = vma->vm_file->private_data;
+	unsigned long pgoff;
+	struct page *page;
+
+	*type = VM_FAULT_MINOR;
+	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
+	if (pgoff != 0)
+		return NOPAGE_SIGBUS;
+	page = virt_to_page(vcpu->run);
+	get_page(page);
+	return page;
+}
+
+static struct vm_operations_struct kvm_vcpu_vm_ops = {
+	.nopage = kvm_vcpu_nopage,
+};
+
+static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	vma->vm_ops = &kvm_vcpu_vm_ops;
+	return 0;
+}
+
 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
@@ -1899,6 +1928,7 @@ static struct file_operations kvm_vcpu_fops = {
 	.release        = kvm_vcpu_release,
 	.unlocked_ioctl = kvm_vcpu_ioctl,
 	.compat_ioctl   = kvm_vcpu_ioctl,
+	.mmap           = kvm_vcpu_mmap,
 };
 
 /*
@@ -1947,6 +1977,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
 {
 	int r;
 	struct kvm_vcpu *vcpu;
+	struct page *page;
 
 	r = -EINVAL;
 	if (!valid_vcpu(n))
@@ -1961,6 +1992,12 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
 		return -EEXIST;
 	}
 
+	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	r = -ENOMEM;
+	if (!page)
+		goto out_unlock;
+	vcpu->run = page_address(page);
+
 	vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
 					   FX_IMAGE_ALIGN);
 	vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
@@ -1990,6 +2027,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
 
 out_free_vcpus:
 	kvm_free_vcpu(vcpu);
+out_unlock:
 	mutex_unlock(&vcpu->mutex);
 out:
 	return r;
@@ -2003,21 +2041,9 @@ static long kvm_vcpu_ioctl(struct file *filp,
 	int r = -EINVAL;
 
 	switch (ioctl) {
-	case KVM_RUN: {
-		struct kvm_run kvm_run;
-
-		r = -EFAULT;
-		if (copy_from_user(&kvm_run, argp, sizeof kvm_run))
-			goto out;
-		r = kvm_vcpu_ioctl_run(vcpu, &kvm_run);
-		if (r < 0 &&  r != -EINTR)
-			goto out;
-		if (copy_to_user(argp, &kvm_run, sizeof kvm_run)) {
-			r = -EFAULT;
-			goto out;
-		}
+	case KVM_RUN:
+		r = kvm_vcpu_ioctl_run(vcpu, vcpu->run);
 		break;
-	}
 	case KVM_GET_REGS: {
 		struct kvm_regs kvm_regs;
 
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index 275354f..d88e750 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -11,7 +11,7 @@
 #include <asm/types.h>
 #include <linux/ioctl.h>
 
-#define KVM_API_VERSION 4
+#define KVM_API_VERSION 5
 
 /*
  * Architectural interrupt line count, and the size of the bitmap needed
@@ -49,7 +49,7 @@ enum kvm_exit_reason {
 	KVM_EXIT_SHUTDOWN         = 8,
 };
 
-/* for KVM_RUN */
+/* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */
 struct kvm_run {
 	/* in */
 	__u32 emulated;  /* skip current instruction */
@@ -233,7 +233,7 @@ struct kvm_dirty_log {
 /*
  * ioctls for vcpu fds
  */
-#define KVM_RUN                   _IOWR(KVMIO, 2, struct kvm_run)
+#define KVM_RUN                   _IO(KVMIO, 16)
 #define KVM_GET_REGS              _IOR(KVMIO, 3, struct kvm_regs)
 #define KVM_SET_REGS              _IOW(KVMIO, 4, struct kvm_regs)
 #define KVM_GET_SREGS             _IOR(KVMIO, 5, struct kvm_sregs)
-- 
1.5.0.2

[PATCH 02/15] KVM: Do not communicate to userspace through cpu registers during PIO

[Avi Kivity]

Currently when passing the a PIO emulation request to userspace, we
rely on userspace updating %rax (on 'in' instructions) and %rsi/%rdi/%rcx
(on string instructions).  This (a) requires two extra ioctls for getting
and setting the registers and (b) is unfriendly to non-x86 archs, when
they get kvm ports.

So fix by doing the register fixups in the kernel and passing to userspace
only an abstract description of the PIO to be done.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/kvm.h      |    1 +
 drivers/kvm/kvm_main.c |   48 +++++++++++++++++++++++++++++++++++++++++++++---
 drivers/kvm/svm.c      |    1 +
 drivers/kvm/vmx.c      |    1 +
 include/linux/kvm.h    |    6 +++---
 5 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h
index 901b8d9..59cbc5b 100644
--- a/drivers/kvm/kvm.h
+++ b/drivers/kvm/kvm.h
@@ -274,6 +274,7 @@ struct kvm_vcpu {
 	int mmio_size;
 	unsigned char mmio_data[8];
 	gpa_t mmio_phys_addr;
+	int pio_pending;
 
 	struct {
 		int active;
diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index 42be8a8..8a4984d 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -1504,6 +1504,44 @@ void save_msrs(struct vmx_msr_entry *e, int n)
 }
 EXPORT_SYMBOL_GPL(save_msrs);
 
+static void complete_pio(struct kvm_vcpu *vcpu)
+{
+	struct kvm_io *io = &vcpu->run->io;
+	long delta;
+
+	kvm_arch_ops->cache_regs(vcpu);
+
+	if (!io->string) {
+		if (io->direction == KVM_EXIT_IO_IN)
+			memcpy(&vcpu->regs[VCPU_REGS_RAX], &io->value,
+			       io->size);
+	} else {
+		delta = 1;
+		if (io->rep) {
+			delta *= io->count;
+			/*
+			 * The size of the register should really depend on
+			 * current address size.
+			 */
+			vcpu->regs[VCPU_REGS_RCX] -= delta;
+		}
+		if (io->string_down)
+			delta = -delta;
+		delta *= io->size;
+		if (io->direction == KVM_EXIT_IO_IN)
+			vcpu->regs[VCPU_REGS_RDI] += delta;
+		else
+			vcpu->regs[VCPU_REGS_RSI] += delta;
+	}
+	
+	vcpu->pio_pending = 0;
+	vcpu->run->io_completed = 0;
+
+	kvm_arch_ops->decache_regs(vcpu);
+
+	kvm_arch_ops->skip_emulated_instruction(vcpu);
+}
+
 static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	int r;
@@ -1518,9 +1556,13 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 		kvm_run->emulated = 0;
 	}
 
-	if (kvm_run->mmio_completed) {
-		memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
-		vcpu->mmio_read_completed = 1;
+	if (kvm_run->io_completed) {
+		if (vcpu->pio_pending)
+			complete_pio(vcpu);
+		else {
+			memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
+			vcpu->mmio_read_completed = 1;
+		}
 	}
 
 	vcpu->mmio_needed = 0;
diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c
index 6787f11..b176f5a 100644
--- a/drivers/kvm/svm.c
+++ b/drivers/kvm/svm.c
@@ -1056,6 +1056,7 @@ static int io_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 		}
 	} else
 		kvm_run->io.value = vcpu->svm->vmcb->save.rax;
+	vcpu->pio_pending = 1;
 	return 0;
 }
 
diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c
index 910535d..7fd572a 100644
--- a/drivers/kvm/vmx.c
+++ b/drivers/kvm/vmx.c
@@ -1465,6 +1465,7 @@ static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 		kvm_run->io.address = vmcs_readl(GUEST_LINEAR_ADDRESS);
 	} else
 		kvm_run->io.value = vcpu->regs[VCPU_REGS_RAX]; /* rax */
+	vcpu->pio_pending = 1;
 	return 0;
 }
 
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index d88e750..19aeb33 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -11,7 +11,7 @@
 #include <asm/types.h>
 #include <linux/ioctl.h>
 
-#define KVM_API_VERSION 5
+#define KVM_API_VERSION 6
 
 /*
  * Architectural interrupt line count, and the size of the bitmap needed
@@ -53,7 +53,7 @@ enum kvm_exit_reason {
 struct kvm_run {
 	/* in */
 	__u32 emulated;  /* skip current instruction */
-	__u32 mmio_completed; /* mmio request completed */
+	__u32 io_completed; /* mmio/pio request completed */
 	__u8 request_interrupt_window;
 	__u8 padding1[7];
 
@@ -80,7 +80,7 @@ struct kvm_run {
 			__u32 error_code;
 		} ex;
 		/* KVM_EXIT_IO */
-		struct {
+		struct kvm_io {
 #define KVM_EXIT_IO_IN  0
 #define KVM_EXIT_IO_OUT 1
 			__u8 direction;
-- 
1.5.0.2

[PATCH 03/15] KVM: Initialize PIO I/O count

[Avi Kivity]

This allows userspace to ignore the io.rep field.  No a big deal, but
friendly.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/svm.c |    1 +
 drivers/kvm/vmx.c |    1 +
 2 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c
index b176f5a..c35b8c8 100644
--- a/drivers/kvm/svm.c
+++ b/drivers/kvm/svm.c
@@ -1037,6 +1037,7 @@ static int io_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 	kvm_run->io.size = ((io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT);
 	kvm_run->io.string = (io_info & SVM_IOIO_STR_MASK) != 0;
 	kvm_run->io.rep = (io_info & SVM_IOIO_REP_MASK) != 0;
+	kvm_run->io.count = 1;
 
 	if (kvm_run->io.string) {
 		unsigned addr_mask;
diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c
index 7fd572a..d4c9f33 100644
--- a/drivers/kvm/vmx.c
+++ b/drivers/kvm/vmx.c
@@ -1459,6 +1459,7 @@ static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 		= (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_DF) != 0;
 	kvm_run->io.rep = (exit_qualification & 32) != 0;
 	kvm_run->io.port = exit_qualification >> 16;
+	kvm_run->io.count = 1;
 	if (kvm_run->io.string) {
 		if (!get_io_count(vcpu, &kvm_run->io.count))
 			return 1;
-- 
1.5.0.2

[PATCH 04/15] KVM: Handle cpuid in the kernel instead of punting to userspace

[Avi Kivity]

KVM used to handle cpuid by letting userspace decide what values to
return to the guest.  We now handle cpuid completely in the kernel.  We
still let userspace decide which values the guest will see by having
userspace set up the value table beforehand (this is necessary to allow
management software to set the cpu features to the least common denominator,
so that live migration can work).

The motivation for the change is that kvm kernel code can be impacted by
cpuid features, for example the x86 emulator.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/kvm.h      |    5 +++
 drivers/kvm/kvm_main.c |   69 ++++++++++++++++++++++++++++++++++++++++++++++++
 drivers/kvm/svm.c      |    4 +-
 drivers/kvm/vmx.c      |    4 +-
 include/linux/kvm.h    |   18 ++++++++++++-
 5 files changed, 95 insertions(+), 5 deletions(-)

diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h
index 59cbc5b..be3a0e7 100644
--- a/drivers/kvm/kvm.h
+++ b/drivers/kvm/kvm.h
@@ -55,6 +55,7 @@
 #define KVM_NUM_MMU_PAGES 256
 #define KVM_MIN_FREE_MMU_PAGES 5
 #define KVM_REFILL_PAGES 25
+#define KVM_MAX_CPUID_ENTRIES 40
 
 #define FX_IMAGE_SIZE 512
 #define FX_IMAGE_ALIGN 16
@@ -286,6 +287,9 @@ struct kvm_vcpu {
 			u32 ar;
 		} tr, es, ds, fs, gs;
 	} rmode;
+
+	int cpuid_nent;
+	struct kvm_cpuid_entry cpuid_entries[KVM_MAX_CPUID_ENTRIES];
 };
 
 struct kvm_memory_slot {
@@ -446,6 +450,7 @@ void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long value,
 
 struct x86_emulate_ctxt;
 
+void kvm_emulate_cpuid(struct kvm_vcpu *vcpu);
 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address);
 int emulate_clts(struct kvm_vcpu *vcpu);
 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr,
diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index 8a4984d..347467e 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -1504,6 +1504,43 @@ void save_msrs(struct vmx_msr_entry *e, int n)
 }
 EXPORT_SYMBOL_GPL(save_msrs);
 
+void kvm_emulate_cpuid(struct kvm_vcpu *vcpu)
+{
+	int i;
+	u32 function;
+	struct kvm_cpuid_entry *e, *best;
+
+	kvm_arch_ops->cache_regs(vcpu);
+	function = vcpu->regs[VCPU_REGS_RAX];
+	vcpu->regs[VCPU_REGS_RAX] = 0;
+	vcpu->regs[VCPU_REGS_RBX] = 0;
+	vcpu->regs[VCPU_REGS_RCX] = 0;
+	vcpu->regs[VCPU_REGS_RDX] = 0;
+	best = NULL;
+	for (i = 0; i < vcpu->cpuid_nent; ++i) {
+		e = &vcpu->cpuid_entries[i];
+		if (e->function == function) {
+			best = e;
+			break;
+		}
+		/*
+		 * Both basic or both extended?
+		 */
+		if (((e->function ^ function) & 0x80000000) == 0)
+			if (!best || e->function > best->function)
+				best = e;
+	}
+	if (best) {
+		vcpu->regs[VCPU_REGS_RAX] = best->eax;
+		vcpu->regs[VCPU_REGS_RBX] = best->ebx;
+		vcpu->regs[VCPU_REGS_RCX] = best->ecx;
+		vcpu->regs[VCPU_REGS_RDX] = best->edx;
+	}
+	kvm_arch_ops->decache_regs(vcpu);
+	kvm_arch_ops->skip_emulated_instruction(vcpu);
+}
+EXPORT_SYMBOL_GPL(kvm_emulate_cpuid);
+
 static void complete_pio(struct kvm_vcpu *vcpu)
 {
 	struct kvm_io *io = &vcpu->run->io;
@@ -2075,6 +2112,26 @@ out:
 	return r;
 }
 
+static int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu,
+				    struct kvm_cpuid *cpuid,
+				    struct kvm_cpuid_entry __user *entries)
+{
+	int r;
+
+	r = -E2BIG;
+	if (cpuid->nent > KVM_MAX_CPUID_ENTRIES)
+		goto out;
+	r = -EFAULT;
+	if (copy_from_user(&vcpu->cpuid_entries, entries,
+			   cpuid->nent * sizeof(struct kvm_cpuid_entry)))
+		goto out;
+	vcpu->cpuid_nent = cpuid->nent;
+	return 0;
+
+out:
+	return r;
+}
+
 static long kvm_vcpu_ioctl(struct file *filp,
 			   unsigned int ioctl, unsigned long arg)
 {
@@ -2181,6 +2238,18 @@ static long kvm_vcpu_ioctl(struct file *filp,
 	case KVM_SET_MSRS:
 		r = msr_io(vcpu, argp, do_set_msr, 0);
 		break;
+	case KVM_SET_CPUID: {
+		struct kvm_cpuid __user *cpuid_arg = argp;
+		struct kvm_cpuid cpuid;
+
+		r = -EFAULT;
+		if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid))
+			goto out;
+		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
+		if (r)
+			goto out;
+		break;
+	}
 	default:
 		;
 	}
diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c
index c35b8c8..d4b2936 100644
--- a/drivers/kvm/svm.c
+++ b/drivers/kvm/svm.c
@@ -1101,8 +1101,8 @@ static int task_switch_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_r
 static int cpuid_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	vcpu->svm->next_rip = vcpu->svm->vmcb->save.rip + 2;
-	kvm_run->exit_reason = KVM_EXIT_CPUID;
-	return 0;
+	kvm_emulate_cpuid(vcpu);
+	return 1;
 }
 
 static int emulate_on_interception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c
index d4c9f33..e093892 100644
--- a/drivers/kvm/vmx.c
+++ b/drivers/kvm/vmx.c
@@ -1585,8 +1585,8 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
 static int handle_cpuid(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
-	kvm_run->exit_reason = KVM_EXIT_CPUID;
-	return 0;
+	kvm_emulate_cpuid(vcpu);
+	return 1;
 }
 
 static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index 19aeb33..15e23bc 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -41,7 +41,6 @@ enum kvm_exit_reason {
 	KVM_EXIT_UNKNOWN          = 0,
 	KVM_EXIT_EXCEPTION        = 1,
 	KVM_EXIT_IO               = 2,
-	KVM_EXIT_CPUID            = 3,
 	KVM_EXIT_DEBUG            = 4,
 	KVM_EXIT_HLT              = 5,
 	KVM_EXIT_MMIO             = 6,
@@ -210,6 +209,22 @@ struct kvm_dirty_log {
 	};
 };
 
+struct kvm_cpuid_entry {
+	__u32 function;
+	__u32 eax;
+	__u32 ebx;
+	__u32 ecx;
+	__u32 edx;
+	__u32 padding;
+};
+
+/* for KVM_SET_CPUID */
+struct kvm_cpuid {
+	__u32 nent;
+	__u32 padding;
+	struct kvm_cpuid_entry entries[0];
+};
+
 #define KVMIO 0xAE
 
 /*
@@ -243,5 +258,6 @@ struct kvm_dirty_log {
 #define KVM_DEBUG_GUEST           _IOW(KVMIO, 9, struct kvm_debug_guest)
 #define KVM_GET_MSRS              _IOWR(KVMIO, 13, struct kvm_msrs)
 #define KVM_SET_MSRS              _IOW(KVMIO, 14, struct kvm_msrs)
+#define KVM_SET_CPUID             _IOW(KVMIO, 17, struct kvm_cpuid)
 
 #endif
-- 
1.5.0.2

[PATCH 05/15] KVM: Remove the 'emulated' field from the userspace interface

[Avi Kivity]

We no longer emulate single instructions in userspace.  Instead, we service
mmio or pio requests.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/kvm_main.c |    5 -----
 include/linux/kvm.h    |    3 +--
 2 files changed, 1 insertions(+), 7 deletions(-)

diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index 347467e..747966e 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -1588,11 +1588,6 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 	/* re-sync apic's tpr */
 	vcpu->cr8 = kvm_run->cr8;
 
-	if (kvm_run->emulated) {
-		kvm_arch_ops->skip_emulated_instruction(vcpu);
-		kvm_run->emulated = 0;
-	}
-
 	if (kvm_run->io_completed) {
 		if (vcpu->pio_pending)
 			complete_pio(vcpu);
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index 15e23bc..c6dd4a7 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -51,10 +51,9 @@ enum kvm_exit_reason {
 /* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */
 struct kvm_run {
 	/* in */
-	__u32 emulated;  /* skip current instruction */
 	__u32 io_completed; /* mmio/pio request completed */
 	__u8 request_interrupt_window;
-	__u8 padding1[7];
+	__u8 padding1[3];
 
 	/* out */
 	__u32 exit_type;
-- 
1.5.0.2

[PATCH 06/15] KVM: Remove minor wart from KVM_CREATE_VCPU ioctl

[Avi Kivity]

That ioctl does not transfer any data, so it should be an _IO rather than an
_IOW.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 include/linux/kvm.h |    2 +-
 1 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index c6dd4a7..d89189a 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -241,7 +241,7 @@ struct kvm_cpuid {
  * KVM_CREATE_VCPU receives as a parameter the vcpu slot, and returns
  * a vcpu fd.
  */
-#define KVM_CREATE_VCPU           _IOW(KVMIO, 11, int)
+#define KVM_CREATE_VCPU           _IO(KVMIO, 11)
 #define KVM_GET_DIRTY_LOG         _IOW(KVMIO, 12, struct kvm_dirty_log)
 
 /*
-- 
1.5.0.2

[PATCH 07/15] KVM: Renumber ioctls

[Avi Kivity]

The recent changes have left the ioctl numbers in complete disarray.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 include/linux/kvm.h |   34 +++++++++++++++++-----------------
 1 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index d89189a..93472da 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -229,34 +229,34 @@ struct kvm_cpuid {
 /*
  * ioctls for /dev/kvm fds:
  */
-#define KVM_GET_API_VERSION       _IO(KVMIO, 1)
-#define KVM_CREATE_VM             _IO(KVMIO, 2) /* returns a VM fd */
-#define KVM_GET_MSR_INDEX_LIST    _IOWR(KVMIO, 15, struct kvm_msr_list)
+#define KVM_GET_API_VERSION       _IO(KVMIO,   0x00)
+#define KVM_CREATE_VM             _IO(KVMIO,   0x01) /* returns a VM fd */
+#define KVM_GET_MSR_INDEX_LIST    _IOWR(KVMIO, 0x02, struct kvm_msr_list)
 
 /*
  * ioctls for VM fds
  */
-#define KVM_SET_MEMORY_REGION     _IOW(KVMIO, 10, struct kvm_memory_region)
+#define KVM_SET_MEMORY_REGION     _IOW(KVMIO, 0x40, struct kvm_memory_region)
 /*
  * KVM_CREATE_VCPU receives as a parameter the vcpu slot, and returns
  * a vcpu fd.
  */
-#define KVM_CREATE_VCPU           _IO(KVMIO, 11)
-#define KVM_GET_DIRTY_LOG         _IOW(KVMIO, 12, struct kvm_dirty_log)
+#define KVM_CREATE_VCPU           _IO(KVMIO,  0x41)
+#define KVM_GET_DIRTY_LOG         _IOW(KVMIO, 0x42, struct kvm_dirty_log)
 
 /*
  * ioctls for vcpu fds
  */
-#define KVM_RUN                   _IO(KVMIO, 16)
-#define KVM_GET_REGS              _IOR(KVMIO, 3, struct kvm_regs)
-#define KVM_SET_REGS              _IOW(KVMIO, 4, struct kvm_regs)
-#define KVM_GET_SREGS             _IOR(KVMIO, 5, struct kvm_sregs)
-#define KVM_SET_SREGS             _IOW(KVMIO, 6, struct kvm_sregs)
-#define KVM_TRANSLATE             _IOWR(KVMIO, 7, struct kvm_translation)
-#define KVM_INTERRUPT             _IOW(KVMIO, 8, struct kvm_interrupt)
-#define KVM_DEBUG_GUEST           _IOW(KVMIO, 9, struct kvm_debug_guest)
-#define KVM_GET_MSRS              _IOWR(KVMIO, 13, struct kvm_msrs)
-#define KVM_SET_MSRS              _IOW(KVMIO, 14, struct kvm_msrs)
-#define KVM_SET_CPUID             _IOW(KVMIO, 17, struct kvm_cpuid)
+#define KVM_RUN                   _IO(KVMIO,   0x80)
+#define KVM_GET_REGS              _IOR(KVMIO,  0x81, struct kvm_regs)
+#define KVM_SET_REGS              _IOW(KVMIO,  0x82, struct kvm_regs)
+#define KVM_GET_SREGS             _IOR(KVMIO,  0x83, struct kvm_sregs)
+#define KVM_SET_SREGS             _IOW(KVMIO,  0x84, struct kvm_sregs)
+#define KVM_TRANSLATE             _IOWR(KVMIO, 0x85, struct kvm_translation)
+#define KVM_INTERRUPT             _IOW(KVMIO,  0x86, struct kvm_interrupt)
+#define KVM_DEBUG_GUEST           _IOW(KVMIO,  0x87, struct kvm_debug_guest)
+#define KVM_GET_MSRS              _IOWR(KVMIO, 0x88, struct kvm_msrs)
+#define KVM_SET_MSRS              _IOW(KVMIO,  0x89, struct kvm_msrs)
+#define KVM_SET_CPUID             _IOW(KVMIO,  0x8a, struct kvm_cpuid)
 
 #endif
-- 
1.5.0.2

[PATCH 08/15] KVM: Add method to check for backwards-compatible API extensions

[Avi Kivity]

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/kvm_main.c |    6 ++++++
 include/linux/kvm.h    |    5 +++++
 2 files changed, 11 insertions(+), 0 deletions(-)

diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index 747966e..376538c 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -2416,6 +2416,12 @@ static long kvm_dev_ioctl(struct file *filp,
 		r = 0;
 		break;
 	}
+	case KVM_CHECK_EXTENSION:
+		/*
+		 * No extensions defined at present.
+		 */
+		r = 0;
+		break;
 	default:
 		;
 	}
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index 93472da..c93cf53 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -232,6 +232,11 @@ struct kvm_cpuid {
 #define KVM_GET_API_VERSION       _IO(KVMIO,   0x00)
 #define KVM_CREATE_VM             _IO(KVMIO,   0x01) /* returns a VM fd */
 #define KVM_GET_MSR_INDEX_LIST    _IOWR(KVMIO, 0x02, struct kvm_msr_list)
+/*
+ * Check if a kvm extension is available.  Argument is extension number,
+ * return is 1 (yes) or 0 (no, sorry).
+ */
+#define KVM_CHECK_EXTENSION       _IO(KVMIO,   0x03)
 
 /*
  * ioctls for VM fds
-- 
1.5.0.2

[PATCH 09/15] KVM: Allow userspace to process hypercalls which have no kernel handler

[Avi Kivity]

This is useful for paravirtualized graphics devices, for example.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/kvm_main.c |   18 +++++++++++++++++-
 include/linux/kvm.h    |   10 +++++++++-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index 376538c..2220e49 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -1203,7 +1203,16 @@ int kvm_hypercall(struct kvm_vcpu *vcpu, struct kvm_run *run)
 	}
 	switch (nr) {
 	default:
-		;
+		run->hypercall.args[0] = a0;
+		run->hypercall.args[1] = a1;
+		run->hypercall.args[2] = a2;
+		run->hypercall.args[3] = a3;
+		run->hypercall.args[4] = a4;
+		run->hypercall.args[5] = a5;
+		run->hypercall.ret = ret;
+		run->hypercall.longmode = is_long_mode(vcpu);
+		kvm_arch_ops->decache_regs(vcpu);
+		return 0;
 	}
 	vcpu->regs[VCPU_REGS_RAX] = ret;
 	kvm_arch_ops->decache_regs(vcpu);
@@ -1599,6 +1608,13 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
 	vcpu->mmio_needed = 0;
 
+	if (kvm_run->exit_type == KVM_EXIT_TYPE_VM_EXIT
+	    && kvm_run->exit_type == KVM_EXIT_HYPERCALL) {
+		kvm_arch_ops->cache_regs(vcpu);
+		vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
+		kvm_arch_ops->decache_regs(vcpu);
+	}
+
 	r = kvm_arch_ops->run(vcpu, kvm_run);
 
 	vcpu_put(vcpu);
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index c93cf53..9151ebf 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -11,7 +11,7 @@
 #include <asm/types.h>
 #include <linux/ioctl.h>
 
-#define KVM_API_VERSION 6
+#define KVM_API_VERSION 7
 
 /*
  * Architectural interrupt line count, and the size of the bitmap needed
@@ -41,6 +41,7 @@ enum kvm_exit_reason {
 	KVM_EXIT_UNKNOWN          = 0,
 	KVM_EXIT_EXCEPTION        = 1,
 	KVM_EXIT_IO               = 2,
+	KVM_EXIT_HYPERCALL        = 3,
 	KVM_EXIT_DEBUG            = 4,
 	KVM_EXIT_HLT              = 5,
 	KVM_EXIT_MMIO             = 6,
@@ -103,6 +104,13 @@ struct kvm_run {
 			__u32 len;
 			__u8  is_write;
 		} mmio;
+		/* KVM_EXIT_HYPERCALL */
+		struct {
+			__u64 args[6];
+			__u64 ret;
+			__u32 longmode;
+			__u32 pad;
+		} hypercall;
 	};
 };
 
-- 
1.5.0.2

[PATCH 10/15] KVM: Fold kvm_run::exit_type into kvm_run::exit_reason

[Avi Kivity]

Currently, userspace is told about the nature of the last exit from the
guest using two fields, exit_type and exit_reason, where exit_type has
just two enumerations (and no need for more).  So fold exit_type into
exit_reason, reducing the complexity of determining what really happened.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/kvm_main.c |    3 +--
 drivers/kvm/svm.c      |    7 +++----
 drivers/kvm/vmx.c      |    7 +++----
 include/linux/kvm.h    |   15 ++++++++-------
 4 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index 2220e49..0e28f58 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -1608,8 +1608,7 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
 	vcpu->mmio_needed = 0;
 
-	if (kvm_run->exit_type == KVM_EXIT_TYPE_VM_EXIT
-	    && kvm_run->exit_type == KVM_EXIT_HYPERCALL) {
+	if (kvm_run->exit_reason == KVM_EXIT_HYPERCALL) {
 		kvm_arch_ops->cache_regs(vcpu);
 		vcpu->regs[VCPU_REGS_RAX] = kvm_run->hypercall.ret;
 		kvm_arch_ops->decache_regs(vcpu);
diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c
index d4b2936..b09928f 100644
--- a/drivers/kvm/svm.c
+++ b/drivers/kvm/svm.c
@@ -1298,8 +1298,6 @@ static int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	u32 exit_code = vcpu->svm->vmcb->control.exit_code;
 
-	kvm_run->exit_type = KVM_EXIT_TYPE_VM_EXIT;
-
 	if (is_external_interrupt(vcpu->svm->vmcb->control.exit_int_info) &&
 	    exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR)
 		printk(KERN_ERR "%s: unexpected exit_ini_info 0x%x "
@@ -1609,8 +1607,9 @@ again:
 	vcpu->svm->next_rip = 0;
 
 	if (vcpu->svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
-		kvm_run->exit_type = KVM_EXIT_TYPE_FAIL_ENTRY;
-		kvm_run->exit_reason = vcpu->svm->vmcb->control.exit_code;
+		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
+		kvm_run->fail_entry.hardware_entry_failure_reason
+			= vcpu->svm->vmcb->control.exit_code;
 		post_kvm_run_save(vcpu, kvm_run);
 		return 0;
 	}
diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c
index e093892..ba7a98b 100644
--- a/drivers/kvm/vmx.c
+++ b/drivers/kvm/vmx.c
@@ -1901,10 +1901,10 @@ again:
 
 	asm ("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
 
-	kvm_run->exit_type = 0;
 	if (fail) {
-		kvm_run->exit_type = KVM_EXIT_TYPE_FAIL_ENTRY;
-		kvm_run->exit_reason = vmcs_read32(VM_INSTRUCTION_ERROR);
+		kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
+		kvm_run->fail_entry.hardware_entry_failure_reason
+			= vmcs_read32(VM_INSTRUCTION_ERROR);
 		r = 0;
 	} else {
 		if (fs_gs_ldt_reload_needed) {
@@ -1930,7 +1930,6 @@ again:
 			profile_hit(KVM_PROFILING, (void *)vmcs_readl(GUEST_RIP));
 
 		vcpu->launched = 1;
-		kvm_run->exit_type = KVM_EXIT_TYPE_VM_EXIT;
 		r = kvm_handle_exit(kvm_run, vcpu);
 		if (r > 0) {
 			/* Give scheduler a change to reschedule. */
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index 9151ebf..57f47ef 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -11,7 +11,7 @@
 #include <asm/types.h>
 #include <linux/ioctl.h>
 
-#define KVM_API_VERSION 7
+#define KVM_API_VERSION 8
 
 /*
  * Architectural interrupt line count, and the size of the bitmap needed
@@ -34,9 +34,6 @@ struct kvm_memory_region {
 #define KVM_MEM_LOG_DIRTY_PAGES  1UL
 
 
-#define KVM_EXIT_TYPE_FAIL_ENTRY 1
-#define KVM_EXIT_TYPE_VM_EXIT    2
-
 enum kvm_exit_reason {
 	KVM_EXIT_UNKNOWN          = 0,
 	KVM_EXIT_EXCEPTION        = 1,
@@ -47,6 +44,7 @@ enum kvm_exit_reason {
 	KVM_EXIT_MMIO             = 6,
 	KVM_EXIT_IRQ_WINDOW_OPEN  = 7,
 	KVM_EXIT_SHUTDOWN         = 8,
+	KVM_EXIT_FAIL_ENTRY       = 9,
 };
 
 /* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */
@@ -57,12 +55,11 @@ struct kvm_run {
 	__u8 padding1[3];
 
 	/* out */
-	__u32 exit_type;
 	__u32 exit_reason;
 	__u32 instruction_length;
 	__u8 ready_for_interrupt_injection;
 	__u8 if_flag;
-	__u16 padding2;
+	__u8 padding2[6];
 
 	/* in (pre_kvm_run), out (post_kvm_run) */
 	__u64 cr8;
@@ -71,8 +68,12 @@ struct kvm_run {
 	union {
 		/* KVM_EXIT_UNKNOWN */
 		struct {
-			__u32 hardware_exit_reason;
+			__u64 hardware_exit_reason;
 		} hw;
+		/* KVM_EXIT_FAIL_ENTRY */
+		struct {
+			__u64 hardware_entry_failure_reason;
+		} fail_entry;
 		/* KVM_EXIT_EXCEPTION */
 		struct {
 			__u32 exception;
-- 
1.5.0.2

[PATCH 11/15] KVM: Add a special exit reason when exiting due to an interrupt

[Avi Kivity]

This is redundant, as we also return -EINTR from the ioctl, but it
allows us to examine the exit_reason field on resume without seeing
old data.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/svm.c   |    2 ++
 drivers/kvm/vmx.c   |    2 ++
 include/linux/kvm.h |    3 ++-
 3 files changed, 6 insertions(+), 1 deletions(-)

diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c
index b09928f..0311665 100644
--- a/drivers/kvm/svm.c
+++ b/drivers/kvm/svm.c
@@ -1619,12 +1619,14 @@ again:
 		if (signal_pending(current)) {
 			++kvm_stat.signal_exits;
 			post_kvm_run_save(vcpu, kvm_run);
+			kvm_run->exit_reason = KVM_EXIT_INTR;
 			return -EINTR;
 		}
 
 		if (dm_request_for_irq_injection(vcpu, kvm_run)) {
 			++kvm_stat.request_irq_exits;
 			post_kvm_run_save(vcpu, kvm_run);
+			kvm_run->exit_reason = KVM_EXIT_INTR;
 			return -EINTR;
 		}
 		kvm_resched(vcpu);
diff --git a/drivers/kvm/vmx.c b/drivers/kvm/vmx.c
index ba7a98b..0d1c8cf 100644
--- a/drivers/kvm/vmx.c
+++ b/drivers/kvm/vmx.c
@@ -1936,12 +1936,14 @@ again:
 			if (signal_pending(current)) {
 				++kvm_stat.signal_exits;
 				post_kvm_run_save(vcpu, kvm_run);
+				kvm_run->exit_reason = KVM_EXIT_INTR;
 				return -EINTR;
 			}
 
 			if (dm_request_for_irq_injection(vcpu, kvm_run)) {
 				++kvm_stat.request_irq_exits;
 				post_kvm_run_save(vcpu, kvm_run);
+				kvm_run->exit_reason = KVM_EXIT_INTR;
 				return -EINTR;
 			}
 
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index 57f47ef..b3af92e 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -11,7 +11,7 @@
 #include <asm/types.h>
 #include <linux/ioctl.h>
 
-#define KVM_API_VERSION 8
+#define KVM_API_VERSION 9
 
 /*
  * Architectural interrupt line count, and the size of the bitmap needed
@@ -45,6 +45,7 @@ enum kvm_exit_reason {
 	KVM_EXIT_IRQ_WINDOW_OPEN  = 7,
 	KVM_EXIT_SHUTDOWN         = 8,
 	KVM_EXIT_FAIL_ENTRY       = 9,
+	KVM_EXIT_INTR             = 10,
 };
 
 /* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */
-- 
1.5.0.2

[PATCH 12/15] KVM: Initialize the apic_base msr on svm too

[Avi Kivity]

Older userspace didn't care, but newer userspace (with the cpuid changes)
does.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/svm.c |    3 +++
 1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/drivers/kvm/svm.c b/drivers/kvm/svm.c
index 0311665..2396ada 100644
--- a/drivers/kvm/svm.c
+++ b/drivers/kvm/svm.c
@@ -582,6 +582,9 @@ static int svm_create_vcpu(struct kvm_vcpu *vcpu)
 	init_vmcb(vcpu->svm->vmcb);
 
 	fx_init(vcpu);
+	vcpu->apic_base = 0xfee00000 |
+			/*for vcpu 0*/ MSR_IA32_APICBASE_BSP |
+			MSR_IA32_APICBASE_ENABLE;
 
 	return 0;
 
-- 
1.5.0.2

[PATCH 13/15] KVM: Add guest mode signal mask

[Avi Kivity]

Allow a special signal mask to be used while executing in guest mode.  This
allows signals to be used to interrupt a vcpu without requiring signal
delivery to a userspace handler, which is quite expensive.  Userspace still
receives -EINTR and can get the signal via sigwait().

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/kvm.h      |    3 +++
 drivers/kvm/kvm_main.c |   41 +++++++++++++++++++++++++++++++++++++++++
 include/linux/kvm.h    |    7 +++++++
 3 files changed, 51 insertions(+), 0 deletions(-)

diff --git a/drivers/kvm/kvm.h b/drivers/kvm/kvm.h
index be3a0e7..1c4a581 100644
--- a/drivers/kvm/kvm.h
+++ b/drivers/kvm/kvm.h
@@ -277,6 +277,9 @@ struct kvm_vcpu {
 	gpa_t mmio_phys_addr;
 	int pio_pending;
 
+	int sigset_active;
+	sigset_t sigset;
+
 	struct {
 		int active;
 		u8 save_iopl;
diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index 0e28f58..ed95c9b 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -1591,9 +1591,13 @@ static void complete_pio(struct kvm_vcpu *vcpu)
 static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 {
 	int r;
+	sigset_t sigsaved;
 
 	vcpu_load(vcpu);
 
+	if (vcpu->sigset_active)
+		sigprocmask(SIG_SETMASK, &vcpu->sigset, &sigsaved);
+
 	/* re-sync apic's tpr */
 	vcpu->cr8 = kvm_run->cr8;
 
@@ -1616,6 +1620,9 @@ static int kvm_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
 
 	r = kvm_arch_ops->run(vcpu, kvm_run);
 
+	if (vcpu->sigset_active)
+		sigprocmask(SIG_SETMASK, &sigsaved, NULL);
+
 	vcpu_put(vcpu);
 	return r;
 }
@@ -2142,6 +2149,17 @@ out:
 	return r;
 }
 
+static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
+{
+	if (sigset) {
+		sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
+		vcpu->sigset_active = 1;
+		vcpu->sigset = *sigset;
+	} else
+		vcpu->sigset_active = 0;
+	return 0;
+}
+
 static long kvm_vcpu_ioctl(struct file *filp,
 			   unsigned int ioctl, unsigned long arg)
 {
@@ -2260,6 +2278,29 @@ static long kvm_vcpu_ioctl(struct file *filp,
 			goto out;
 		break;
 	}
+	case KVM_SET_SIGNAL_MASK: {
+		struct kvm_signal_mask __user *sigmask_arg = argp;
+		struct kvm_signal_mask kvm_sigmask;
+		sigset_t sigset, *p;
+
+		p = NULL;
+		if (argp) {
+			r = -EFAULT;
+			if (copy_from_user(&kvm_sigmask, argp,
+					   sizeof kvm_sigmask))
+				goto out;
+			r = -EINVAL;
+			if (kvm_sigmask.len != sizeof sigset)
+				goto out;
+			r = -EFAULT;
+			if (copy_from_user(&sigset, sigmask_arg->sigset,
+					   sizeof sigset))
+				goto out;
+			p = &sigset;
+		}
+		r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
+		break;
+	}
 	default:
 		;
 	}
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index b3af92e..c0d10cd 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -234,6 +234,12 @@ struct kvm_cpuid {
 	struct kvm_cpuid_entry entries[0];
 };
 
+/* for KVM_SET_SIGNAL_MASK */
+struct kvm_signal_mask {
+	__u32 len;
+	__u8  sigset[0];
+};
+
 #define KVMIO 0xAE
 
 /*
@@ -273,5 +279,6 @@ struct kvm_cpuid {
 #define KVM_GET_MSRS              _IOWR(KVMIO, 0x88, struct kvm_msrs)
 #define KVM_SET_MSRS              _IOW(KVMIO,  0x89, struct kvm_msrs)
 #define KVM_SET_CPUID             _IOW(KVMIO,  0x8a, struct kvm_cpuid)
+#define KVM_SET_SIGNAL_MASK       _IOW(KVMIO,  0x8b, struct kvm_signal_mask)
 
 #endif
-- 
1.5.0.2

[PATCH 14/15] KVM: Allow kernel to select size of mmap() buffer

[Avi Kivity]

This allows us to store offsets in the kernel/user kvm_run area, and be
sure that userspace has them mapped.  As offsets can be outside the
kvm_run struct, userspace has no way of knowing how much to mmap.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/kvm_main.c |    8 +++++++-
 include/linux/kvm.h    |    4 ++++
 2 files changed, 11 insertions(+), 1 deletions(-)

diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index ed95c9b..b81f007 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -2436,7 +2436,7 @@ static long kvm_dev_ioctl(struct file *filp,
 			  unsigned int ioctl, unsigned long arg)
 {
 	void __user *argp = (void __user *)arg;
-	int r = -EINVAL;
+	long r = -EINVAL;
 
 	switch (ioctl) {
 	case KVM_GET_API_VERSION:
@@ -2478,6 +2478,12 @@ static long kvm_dev_ioctl(struct file *filp,
 		 */
 		r = 0;
 		break;
+	case KVM_GET_VCPU_MMAP_SIZE:
+		r = -EINVAL;
+		if (arg)
+			goto out;
+		r = PAGE_SIZE;
+		break;
 	default:
 		;
 	}
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index c0d10cd..dad9081 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -253,6 +253,10 @@ struct kvm_signal_mask {
  * return is 1 (yes) or 0 (no, sorry).
  */
 #define KVM_CHECK_EXTENSION       _IO(KVMIO,   0x03)
+/*
+ * Get size for mmap(vcpu_fd)
+ */
+#define KVM_GET_VCPU_MMAP_SIZE    _IO(KVMIO,   0x04) /* in bytes */
 
 /*
  * ioctls for VM fds
-- 
1.5.0.2

[PATCH 15/15] KVM: Future-proof argument-less ioctls

[Avi Kivity]

Some ioctls ignore their arguments.  By requiring them to be zero now,
we allow a nonzero value to have some special meaning in the future.

Signed-off-by: Avi Kivity <avi@qumranet.com>
---
 drivers/kvm/kvm_main.c |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
index b81f007..bf8403e 100644
--- a/drivers/kvm/kvm_main.c
+++ b/drivers/kvm/kvm_main.c
@@ -2169,6 +2169,9 @@ static long kvm_vcpu_ioctl(struct file *filp,
 
 	switch (ioctl) {
 	case KVM_RUN:
+		r = -EINVAL;
+		if (arg)
+			goto out;
 		r = kvm_vcpu_ioctl_run(vcpu, vcpu->run);
 		break;
 	case KVM_GET_REGS: {
@@ -2440,9 +2443,15 @@ static long kvm_dev_ioctl(struct file *filp,
 
 	switch (ioctl) {
 	case KVM_GET_API_VERSION:
+		r = -EINVAL;
+		if (arg)
+			goto out;
 		r = KVM_API_VERSION;
 		break;
 	case KVM_CREATE_VM:
+		r = -EINVAL;
+		if (arg)
+			goto out;
 		r = kvm_dev_ioctl_create_vm();
 		break;
 	case KVM_GET_MSR_INDEX_LIST: {
-- 
1.5.0.2

Re: [kvm-devel] [PATCH 01/15] KVM: Use a shared page for kernel/user communication when runing a vcpu

[Hollis Blanchard]

On Sun, 2007-03-11 at 15:53 +0200, Avi Kivity wrote:
> Instead of passing a 'struct kvm_run' back and forth between the
> kernel and userspace, allocate a page and allow the user to mmap() it.
> This reduces needless copying and makes the interface expandable by
> providing lots of free space.

Do you provide for another means of accessing guest memory from host
userspace? For example, how do you attach a host debugger to the guest?

Xen uses an ioctl followed by mmap for this purpose, which is why I
wonder about using mmap(/dev/kvm) for another purpose.

-Hollis

Re: [kvm-devel] [PATCH 01/15] KVM: Use a shared page for kernel/user communication when runing a vcpu

[Hollis Blanchard]

On Wed, 2007-03-14 at 21:38 -0500, Hollis Blanchard wrote:
> On Sun, 2007-03-11 at 15:53 +0200, Avi Kivity wrote:
> > Instead of passing a 'struct kvm_run' back and forth between the
> > kernel and userspace, allocate a page and allow the user to mmap() it.
> > This reduces needless copying and makes the interface expandable by
> > providing lots of free space.
> 
> Do you provide for another means of accessing guest memory from host
> userspace? For example, how do you attach a host debugger to the guest?
> 
> Xen uses an ioctl followed by mmap for this purpose, which is why I
> wonder about using mmap(/dev/kvm) for another purpose.

Never mind: I see now that you have separate device nodes for the VM vs
each vcpu.

-Hollis

Re: [kvm-devel] [PATCH 0/15] KVM userspace interface updates

[Heiko Carstens]

On Sun, Mar 11, 2007 at 03:53:12PM +0200, Avi Kivity wrote:
> This patchset updates the kvm userspace interface to what I hope will
> be the long-term stable interface.  Provisions are included for extending
> the interface later.  The patches address performance and cleanliness
> concerns.

Searching the mailing list I figured that as soons as the interface seems
to be stable, kvm should/would switch to a system call based interface.
I assume the userspace interface might still change a lot, especially if
kvm is ported to new architectures.
But the general question is: do you still plan to switch to a syscall
interface?

Re: [kvm-devel] [PATCH 0/15] KVM userspace interface updates

[Anthony Liguori]

Heiko Carstens wrote:
> On Sun, Mar 11, 2007 at 03:53:12PM +0200, Avi Kivity wrote:
>   
>> This patchset updates the kvm userspace interface to what I hope will
>> be the long-term stable interface.  Provisions are included for extending
>> the interface later.  The patches address performance and cleanliness
>> concerns.
>>     
>
> Searching the mailing list I figured that as soons as the interface seems
> to be stable, kvm should/would switch to a system call based interface.
> I assume the userspace interface might still change a lot, especially if
> kvm is ported to new architectures.
> But the general question is: do you still plan to switch to a syscall
> interface?
>   

What benefit would a syscall interface have?

Regards,

Anthony Liguori

> -------------------------------------------------------------------------
> Take Surveys. Earn Cash. Influence the Future of IT
> Join SourceForge.net's Techsay panel and you'll get the chance to share your
> opinions on IT & business topics through brief surveys-and earn cash
> http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
> _______________________________________________
> kvm-devel mailing list
> kvm-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/kvm-devel
>
>   

Re: [kvm-devel] [PATCH 0/15] KVM userspace interface updates

[Heiko Carstens]

On Fri, Mar 16, 2007 at 09:03:08AM -0500, Anthony Liguori wrote:
> Heiko Carstens wrote:
> >On Sun, Mar 11, 2007 at 03:53:12PM +0200, Avi Kivity wrote:
> >  
> >>This patchset updates the kvm userspace interface to what I hope will
> >>be the long-term stable interface.  Provisions are included for extending
> >>the interface later.  The patches address performance and cleanliness
> >>concerns.
> >>    
> >
> >Searching the mailing list I figured that as soons as the interface seems
> >to be stable, kvm should/would switch to a system call based interface.
> >I assume the userspace interface might still change a lot, especially if
> >kvm is ported to new architectures.
> >But the general question is: do you still plan to switch to a syscall
> >interface?
> >  
> 
> What benefit would a syscall interface have?

First of all: it's faster and doesn't burn a bunch of additional cpu
cycles like sys_ioctl and the large switch statements do.

Another thing is that this patch set already introduces a way to pass a
sigset. Passing a sigset to a device node is sort of strange.

In addition, if we would port kvm to s390, then we would need to
make sure that each virtual cpu only gets executed from the thread
that created it. That is simply because the upper half of our page
tables contain information about the guest page states. This is yet
another thing that would be strange to do via an ioctl based interface.

Of course everthing can be done via an iotcl interface too, but IMHO
that's just the wrong interface.

Re: [kvm-devel] [PATCH 08/15] KVM: Add method to check for backwards-compatible API extensions

[Heiko Carstens]

> diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
> index 747966e..376538c 100644
> --- a/drivers/kvm/kvm_main.c
> +++ b/drivers/kvm/kvm_main.c
> @@ -2416,6 +2416,12 @@ static long kvm_dev_ioctl(struct file *filp,
>  		r = 0;
>  		break;
>  	}
> +	case KVM_CHECK_EXTENSION:
> +		/*
> +		 * No extensions defined at present.
> +		 */
> +		r = 0;
> +		break;
>  	default:

What exactly is this good for?

Re: [kvm-devel] [PATCH 0/15] KVM userspace interface updates

[Avi Kivity]

Heiko Carstens wrote:
> On Sun, Mar 11, 2007 at 03:53:12PM +0200, Avi Kivity wrote:
>   
>> This patchset updates the kvm userspace interface to what I hope will
>> be the long-term stable interface.  Provisions are included for extending
>> the interface later.  The patches address performance and cleanliness
>> concerns.
>>     
>
> Searching the mailing list I figured that as soons as the interface seems
> to be stable, kvm should/would switch to a system call based interface.
> I assume the userspace interface might still change a lot, especially if
> kvm is ported to new architectures.
> But the general question is: do you still plan to switch to a syscall
> interface?
>   

I don't have any present plans for that.  Maybe when the interface 
starts to evolve at a slower pace, or if it is shown to be significantly 
faster.

Not that interface stabilization here doesn't mean a freeze; it means 
that backwards compatibility starts when this gets merged.

-- 
Do not meddle in the internals of kernels, for they are subtle and quick to panic.

Re: [kvm-devel] [PATCH 08/15] KVM: Add method to check for backwards-compatible API extensions

[Avi Kivity]

Heiko Carstens wrote:
>> diff --git a/drivers/kvm/kvm_main.c b/drivers/kvm/kvm_main.c
>> index 747966e..376538c 100644
>> --- a/drivers/kvm/kvm_main.c
>> +++ b/drivers/kvm/kvm_main.c
>> @@ -2416,6 +2416,12 @@ static long kvm_dev_ioctl(struct file *filp,
>>  		r = 0;
>>  		break;
>>  	}
>> +	case KVM_CHECK_EXTENSION:
>> +		/*
>> +		 * No extensions defined at present.
>> +		 */
>> +		r = 0;
>> +		break;
>>  	default:
>>     
>
> What exactly is this good for?
>   

Extending the interface in a backwards compatible manner.  When new 
interfaces are added, it will return 1 (depending on arg) for 
implemented extensions.

-- 
error compiling committee.c: too many arguments to function

Re: [kvm-devel] [PATCH 0/15] KVM userspace interface updates

[Heiko Carstens]

On Sun, Mar 18, 2007 at 07:20:57AM +0200, Avi Kivity wrote:
> Heiko Carstens wrote:
> >On Sun, Mar 11, 2007 at 03:53:12PM +0200, Avi Kivity wrote:
> >  
> >>This patchset updates the kvm userspace interface to what I hope will
> >>be the long-term stable interface.  Provisions are included for extending
> >>the interface later.  The patches address performance and cleanliness
> >>concerns.
> > [...]
> >But the general question is: do you still plan to switch to a syscall
> >interface?
> 
> I don't have any present plans for that.  Maybe when the interface starts
> to evolve at a slower pace, or if it is shown to be significantly faster.
> 
> Not that interface stabilization here doesn't mean a freeze; it means that
> backwards compatibility starts when this gets merged.

If the interface is considered to be stable you can get rid of the
KVM_GET_API_VERSION ioctl, since the version can't change anymore, right?

Re: [kvm-devel] [PATCH 0/15] KVM userspace interface updates

[Avi Kivity]

Heiko Carstens wrote:
> On Sun, Mar 18, 2007 at 07:20:57AM +0200, Avi Kivity wrote:
>   
>> Heiko Carstens wrote:
>>     
>>> On Sun, Mar 11, 2007 at 03:53:12PM +0200, Avi Kivity wrote:
>>>  
>>>       
>>>> This patchset updates the kvm userspace interface to what I hope will
>>>> be the long-term stable interface.  Provisions are included for extending
>>>> the interface later.  The patches address performance and cleanliness
>>>> concerns.
>>>>         
>>> [...]
>>> But the general question is: do you still plan to switch to a syscall
>>> interface?
>>>       
>> I don't have any present plans for that.  Maybe when the interface starts
>> to evolve at a slower pace, or if it is shown to be significantly faster.
>>
>> Not that interface stabilization here doesn't mean a freeze; it means that
>> backwards compatibility starts when this gets merged.
>>     
>
> If the interface is considered to be stable you can get rid of the
> KVM_GET_API_VERSION ioctl, since the version can't change anymore, right?
>   

It's needed in case pre-stabilization userspace tries to use the 
stabilized interface.  It's true the version won't change.

But maybe we can get rid of it, and the old userspace will just fail on 
the ioctl (we need to keep it reserved for that).

-- 
error compiling committee.c: too many arguments to function

Re: [kvm-devel] [PATCH 0/15] KVM userspace interface updates

[Avi Kivity]

Heiko Carstens wrote:
>>>  
>>>       
>> What benefit would a syscall interface have?
>>     
>
> Another thing is that this patch set already introduces a way to pass a
> sigset. Passing a sigset to a device node is sort of strange.
>   

The sigset is passed to the device node just for safekeeping, as it 
doesn't normally change.  It's only used when switching to guest mode.

> In addition, if we would port kvm to s390, then we would need to
> make sure that each virtual cpu only gets executed from the thread
> that created it. That is simply because the upper half of our page
> tables contain information about the guest page states. This is yet
> another thing that would be strange to do via an ioctl based interface.
>   

Right.  I agree it's more natural to associate a vcpu with a task 
instead of a vcpu being an independent entry.  We'd still need a handle 
for it, and in Linux that's an fd (pid doesn't cut it as it's racy, and 
probably slower too as it has to go through a global structure).

> Of course everthing can be done via an iotcl interface too, but IMHO
> that's just the wrong interface.
>   

I guess once we have smp, and preferably an additional arch port, we can 
do another round of API consolidation around a syscall based API.  We'll 
need to support the ioctl based API in parallel until the distros flush 
out older userspace.

-- 
error compiling committee.c: too many arguments to function

Re: [kvm-devel] [PATCH 0/15] KVM userspace interface updates

[Heiko Carstens]

On Sun, Mar 18, 2007 at 12:42:00PM +0200, Avi Kivity wrote:
> Heiko Carstens wrote:
> >In addition, if we would port kvm to s390, then we would need to
> >make sure that each virtual cpu only gets executed from the thread
> >that created it. That is simply because the upper half of our page
> >tables contain information about the guest page states. This is yet
> >another thing that would be strange to do via an ioctl based interface.
> 
> Right.  I agree it's more natural to associate a vcpu with a task
> instead of a vcpu being an independent entry.  We'd still need a
> handle for it, and in Linux that's an fd (pid doesn't cut it as it's
> racy, and probably slower too as it has to go through a global structure).

If you go for: only one VM per thread group and only one vcpu per thread
you don't need any identifier.
All relevant information or a pointer to it would be saved in the
thread_info structure.
That would give you two system calls to add/remove cpus which implicitely
create a VM if none is present. This add_vcpu syscall would also map
a memory range to user space which would be used to communicate between
user/kernel space to avoid frequent copy_to/from_user just like your
latest patches for KVM_RUN do.

We implemented a prototype on s390 based on a system call interface
and which does have full smp support.

This is a simplified version of how a add_cpu system call would look like.
Please note that I left out all error checkings etc. E.g. checking if the
vcpu already exists in the VM.

asmlinkage long sys_kvm_add_cpu(int vcpu, unsigned long addr)
{
	struct kvm *kvm;

	if (current_thread_info()->vcpu != -1)
		return -EINVAL;

	mutex_lock(&kvm_mutex);

	write_lock_bh(&tasklist_lock);
	/*
	 * Check all thread_infos in thread group if a VM context
	 * was already created.
	 */
	kvm = search_threads_for_kvm();
	write_unlock_bh(&tasklist_lock);

	if (!kvm) {
		kvm = create_kvm_context();
		current_thread_info()->kvm = kvm;
	}

	arch_add_cpu(vcpu);
	current_thread_info()->vcpu = vcpu;

	/*
	 * Map vcpu data to userspace at addr.
	 */
	arch_create_kvm_area(addr);

	mutex_unlock(&kvm_mutex);

	return 0;
}

asmlinkage long sys_kvm_remove_cpu(void)
{
	int vcpu;

	vcpu = current_thread_info()->vcpu;
	if (cpu == -1)
		return -EINVAL;

	mutex_lock(&kvm_mutex);
	arch_remove_cpu(vcpu);
	current_thread_info()->vcpu = -1;
	mutex_unlock(&kvm_mutex);
	return 0;
}

The interesting part with this is that you don't need any locking
for a kvm_run system call, simply because only the thread itself can
create/remove the vcpu:

asmlinkage long sys_kvm_run(void)
{
	int vcpu;

	vcpu = current_thread_info()->vcpu;
	if (vcpu == -1)
		return -EINVAL;
	return arch_kvm_run();
}

Of course all this is rather simplified, but should give a good idea
why I think that a syscall based interface should be the way to go.

Re: [kvm-devel] [PATCH 0/15] KVM userspace interface updates

[Avi Kivity]

Heiko Carstens wrote:
>> Right.  I agree it's more natural to associate a vcpu with a task
>> instead of a vcpu being an independent entry.  We'd still need a
>> handle for it, and in Linux that's an fd (pid doesn't cut it as it's
>> racy, and probably slower too as it has to go through a global structure).
>>     
>
> If you go for: only one VM per thread group and only one vcpu per thread
> you don't need any identifier.
>   

That's the idea, but if I want to send an inter-processor-interrupt to 
another cpu, I need to be able to identify it.  The pid [tid] is 
natural, but racy if the thread can die.

> All relevant information or a pointer to it would be saved in the
> thread_info structure.
> That would give you two system calls to add/remove cpus which implicitely
> create a VM if none is present. This add_vcpu syscall would also map
> a memory range to user space which would be used to communicate between
> user/kernel space to avoid frequent copy_to/from_user just like your
> latest patches for KVM_RUN do.
>
> We implemented a prototype on s390 based on a system call interface
> and which does have full smp support.

[...]

> The interesting part with this is that you don't need any locking
> for a kvm_run system call, simply because only the thread itself can
> create/remove the vcpu:
>   

Yes!  The vcpu is an implied parameter (current->vcpu).

> Of course all this is rather simplified, but should give a good idea
> why I think that a syscall based interface should be the way to go.
>   

I agree with all of the above, and in addition, integration to the 
scheduler will allow us to reduce vcpu migration rate, and maybe do 
things like gang scheduling.

But that doesn't mean it can be done now: we really need to see how it 
works out with smp and with an additional arch, and then we can 
stabilize it.  Meanwhile I'd like a stable ABI so distros can start 
shipping kvm without worrying about upgrade headaches.

So the plan is:
- get the /dev/kvm ABI into 2.6.22
- implement smp
- add another arch
- move to a syscall based interface in parallel; userspace should work 
with both
- deprecate the old ABI and external modules.

-- 
error compiling committee.c: too many arguments to function

Re: [kvm-devel] [PATCH 0/15] KVM userspace interface updates

[Heiko Carstens]

On Mon, Mar 19, 2007 at 06:02:57PM +0200, Avi Kivity wrote:
> Heiko Carstens wrote:
> I agree with all of the above, and in addition, integration to the
> scheduler will allow us to reduce vcpu migration rate, and maybe do
> things like gang scheduling.
> 
> But that doesn't mean it can be done now: we really need to see how it
> works out with smp and with an additional arch, and then we can stabilize
> it.  Meanwhile I'd like a stable ABI so distros can start shipping
> kvm without worrying about upgrade headaches.
> 
> So the plan is:
> - get the /dev/kvm ABI into 2.6.22
> - implement smp
> - add another arch
> - move to a syscall based interface in parallel; userspace should work with both
> - deprecate the old ABI and external modules.

I wasn't asking to change the ABI right now, just wanted to make sure that
it is not the 'final' interface. So I agree with your plan.

Re: [kvm-devel] [PATCH 0/15] KVM userspace interface updates

[Avi Kivity]

Avi Kivity wrote:
>
> So the plan is:
> - get the /dev/kvm ABI into 2.6.22
> - implement smp
> - add another arch
> - move to a syscall based interface in parallel; userspace should work 
> with both
> - deprecate the old ABI and external modules.

I would also like to add using arbitrary vmas as guest memory (and 
paging guest memory) to that list.

-- 
error compiling committee.c: too many arguments to function