Re: [PATCH] KVM swapping with mmu notifiers #v7

[izik eidus <izike@qumranet.com>]

ציטוט Andrea Arcangeli:
> Same as before but one one hand ported to #v7 API and on the other
> hand ported to latest kvm.git.
>
> Signed-off-by: Andrea Arcangeli <andrea@qumranet.com>
>
> diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
> index 41962e7..e1287ab 100644
> --- a/arch/x86/kvm/Kconfig
> +++ b/arch/x86/kvm/Kconfig
> @@ -21,6 +21,7 @@ config KVM
>  	tristate "Kernel-based Virtual Machine (KVM) support"
>  	depends on HAVE_KVM && EXPERIMENTAL
>  	select PREEMPT_NOTIFIERS
> +	select MMU_NOTIFIER
>  	select ANON_INODES
>  	---help---
>  	  Support hosting fully virtualized guest machines using hardware
> diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
> index 4583329..4067b0f 100644
> --- a/arch/x86/kvm/mmu.c
> +++ b/arch/x86/kvm/mmu.c
> @@ -642,6 +642,110 @@ static void rmap_write_protect(struct kvm *kvm, u64 gfn)
>  	account_shadowed(kvm, gfn);
>  }
>  
> +static void kvm_unmap_spte(struct kvm *kvm, u64 *spte)
> +{
> +	struct page *page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
> +	get_page(page);
> +	rmap_remove(kvm, spte);
> +	set_shadow_pte(spte, shadow_trap_nonpresent_pte);
> +	kvm_flush_remote_tlbs(kvm);
> +	__free_page(page);
>   

with large page support i think we need here put_page...

> +}
> +
> +static void kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp)
> +{
> +	u64 *spte, *curr_spte;
> +
> +	spte = rmap_next(kvm, rmapp, NULL);
> +	while (spte) {
> +		BUG_ON(!(*spte & PT_PRESENT_MASK));
> +		rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
> +		curr_spte = spte;
> +		spte = rmap_next(kvm, rmapp, spte);
> +		kvm_unmap_spte(kvm, curr_spte);
> +	}
> +}
> +
> +void kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
> +{
> +	int i;
> +
> +	/*
> +	 * If mmap_sem isn't taken, we can look the memslots with only
> +	 * the mmu_lock by skipping over the slots with userspace_addr == 0.
> +	 */
> +	spin_lock(&kvm->mmu_lock);
> +	for (i = 0; i < kvm->nmemslots; i++) {
> +		struct kvm_memory_slot *memslot = &kvm->memslots[i];
> +		unsigned long start = memslot->userspace_addr;
> +		unsigned long end;
> +
> +		/* mmu_lock protects userspace_addr */
> +		if (!start)
> +			continue;
> +
> +		end = start + (memslot->npages << PAGE_SHIFT);
> +		if (hva >= start && hva < end) {
> +			gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
> +			kvm_unmap_rmapp(kvm, &memslot->rmap[gfn_offset]);
> +		}
> +	}
> +	spin_unlock(&kvm->mmu_lock);
> +}
> +
> +static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp)
> +{
> +	u64 *spte;
> +	int young = 0;
> +
> +	spte = rmap_next(kvm, rmapp, NULL);
> +	while (spte) {
> +		int _young;
> +		u64 _spte = *spte;
> +		BUG_ON(!(_spte & PT_PRESENT_MASK));
> +		_young = _spte & PT_ACCESSED_MASK;
> +		if (_young) {
> +			young = !!_young;
> +			set_shadow_pte(spte, _spte & ~PT_ACCESSED_MASK);
> +		}
> +		spte = rmap_next(kvm, rmapp, spte);
> +	}
> +	return young;
> +}
> +
> +int kvm_age_hva(struct kvm *kvm, unsigned long hva)
> +{
> +	int i;
> +	int young = 0;
> +
> +	/*
> +	 * If mmap_sem isn't taken, we can look the memslots with only
> +	 * the mmu_lock by skipping over the slots with userspace_addr == 0.
> +	 */
> +	spin_lock(&kvm->mmu_lock);
> +	for (i = 0; i < kvm->nmemslots; i++) {
> +		struct kvm_memory_slot *memslot = &kvm->memslots[i];
> +		unsigned long start = memslot->userspace_addr;
> +		unsigned long end;
> +
> +		/* mmu_lock protects userspace_addr */
> +		if (!start)
> +			continue;
> +
> +		end = start + (memslot->npages << PAGE_SHIFT);
> +		if (hva >= start && hva < end) {
> +			gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
> +			young |= kvm_age_rmapp(kvm, &memslot->rmap[gfn_offset]);
> +		}
> +	}
> +	spin_unlock(&kvm->mmu_lock);
> +
> +	if (young)
> +		kvm_flush_remote_tlbs(kvm);
> +
> +	return young;
> +}
> +
>  #ifdef MMU_DEBUG
>  static int is_empty_shadow_page(u64 *spt)
>  {
> diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
> index 17f9d16..b014b19 100644
> --- a/arch/x86/kvm/paging_tmpl.h
> +++ b/arch/x86/kvm/paging_tmpl.h
> @@ -380,6 +380,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
>  	int r;
>  	struct page *page;
>  	int largepage = 0;
> +	unsigned mmu_seq;
>  
>  	pgprintk("%s: addr %lx err %x\n", __FUNCTION__, addr, error_code);
>  	kvm_mmu_audit(vcpu, "pre page fault");
> @@ -415,6 +416,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
>  			largepage = 1;
>  		}
>  	}
> +	mmu_seq = read_seqbegin(&vcpu->kvm->arch.mmu_notifier_invalidate_lock);
>  	page = gfn_to_page(vcpu->kvm, walker.gfn);
>  	up_read(&current->mm->mmap_sem);
>  
> @@ -440,6 +442,15 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gva_t addr,
>  	++vcpu->stat.pf_fixed;
>  	kvm_mmu_audit(vcpu, "post page fault (fixed)");
>  	spin_unlock(&vcpu->kvm->mmu_lock);
> +
> +	if (read_seqretry(&vcpu->kvm->arch.mmu_notifier_invalidate_lock, mmu_seq)) {
> +		down_read(&current->mm->mmap_sem);
> +		if (page != gfn_to_page(vcpu->kvm, walker.gfn))
> +			BUG();
> +		up_read(&current->mm->mmap_sem);
> +		kvm_release_page_clean(page);
> +	}
> +
>  	up_read(&vcpu->kvm->slots_lock);
>  
>  	return write_pt;
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 6f09840..6eafb74 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -3319,6 +3319,47 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
>  	free_page((unsigned long)vcpu->arch.pio_data);
>  }
>  
> +static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
> +{
> +	struct kvm_arch *kvm_arch;
> +	kvm_arch = container_of(mn, struct kvm_arch, mmu_notifier);
> +	return container_of(kvm_arch, struct kvm, arch);
> +}
> +
> +void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
> +				      struct mm_struct *mm,
> +				      unsigned long address)
> +{
> +	struct kvm *kvm = mmu_notifier_to_kvm(mn);
> +	BUG_ON(mm != kvm->mm);
> +	write_seqlock(&kvm->arch.mmu_notifier_invalidate_lock);
> +	kvm_unmap_hva(kvm, address);
> +	write_sequnlock(&kvm->arch.mmu_notifier_invalidate_lock);
> +}
> +
> +void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
> +					   struct mm_struct *mm,
> +					   unsigned long start, unsigned long end)
> +{
> +	for (; start < end; start += PAGE_SIZE)
> +		kvm_mmu_notifier_invalidate_page(mn, mm, start);
> +}
> +
> +int kvm_mmu_notifier_age_page(struct mmu_notifier *mn,
> +			      struct mm_struct *mm,
> +			      unsigned long address)
> +{
> +	struct kvm *kvm = mmu_notifier_to_kvm(mn);
> +	BUG_ON(mm != kvm->mm);
> +	return kvm_age_hva(kvm, address);
> +}
> +
> +static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
> +	.invalidate_page	= kvm_mmu_notifier_invalidate_page,
> +	.invalidate_range_end	= kvm_mmu_notifier_invalidate_range_end,
> +	.age_page		= kvm_mmu_notifier_age_page,
> +};
> +
>  struct  kvm *kvm_arch_create_vm(void)
>  {
>  	struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
> @@ -3328,6 +3369,10 @@ struct  kvm *kvm_arch_create_vm(void)
>  
>  	INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
>  
> +	kvm->arch.mmu_notifier.ops = &kvm_mmu_notifier_ops;
> +	mmu_notifier_register(&kvm->arch.mmu_notifier, current->mm);
> +	seqlock_init(&kvm->arch.mmu_notifier_invalidate_lock);
> +
>  	return kvm;
>  }
>  
> diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h
> index 024b57c..305b7c3 100644
> --- a/include/asm-x86/kvm_host.h
> +++ b/include/asm-x86/kvm_host.h
> @@ -13,6 +13,7 @@
>  
>  #include <linux/types.h>
>  #include <linux/mm.h>
> +#include <linux/mmu_notifier.h>
>  
>  #include <linux/kvm.h>
>  #include <linux/kvm_para.h>
> @@ -303,6 +304,9 @@ struct kvm_arch{
>  	struct page *apic_access_page;
>  
>  	gpa_t wall_clock;
> +
> +	struct mmu_notifier mmu_notifier;
> +	seqlock_t mmu_notifier_invalidate_lock;
>  };
>  
>  struct kvm_vm_stat {
> @@ -422,6 +426,8 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu);
>  int kvm_mmu_setup(struct kvm_vcpu *vcpu);
>  void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte);
>  
> +void kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
> +int kvm_age_hva(struct kvm *kvm, unsigned long hva);
>  int kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
>  void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot);
>  void kvm_mmu_zap_all(struct kvm *kvm);
>
>
> As usual (for completeness) I append the change to the memslot
> readonly locking through kvm->mmu_lock:
>
> Signed-off-by: Andrea Arcangeli <andrea@qumranet.com>
>
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 6f09840..a519fd8 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -3379,16 +3379,23 @@ int kvm_arch_set_memory_region(struct kvm *kvm,
>  	 */
>  	if (!user_alloc) {
>  		if (npages && !old.rmap) {
> +			unsigned long userspace_addr;
> +
>  			down_write(&current->mm->mmap_sem);
> -			memslot->userspace_addr = do_mmap(NULL, 0,
> -						     npages * PAGE_SIZE,
> -						     PROT_READ | PROT_WRITE,
> -						     MAP_SHARED | MAP_ANONYMOUS,
> -						     0);
> +			userspace_addr = do_mmap(NULL, 0,
> +						 npages * PAGE_SIZE,
> +						 PROT_READ | PROT_WRITE,
> +						 MAP_SHARED | MAP_ANONYMOUS,
> +						 0);
>  			up_write(&current->mm->mmap_sem);
>  
> -			if (IS_ERR((void *)memslot->userspace_addr))
> -				return PTR_ERR((void *)memslot->userspace_addr);
> +			if (IS_ERR((void *)userspace_addr))
> +				return PTR_ERR((void *)userspace_addr);
> +
> +			/* set userspace_addr atomically for kvm_hva_to_rmapp */
> +			spin_lock(&kvm->mmu_lock);
> +			memslot->userspace_addr = userspace_addr;
> +			spin_unlock(&kvm->mmu_lock);
>  		} else {
>  			if (!old.user_alloc && old.rmap) {
>  				int ret;
> diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
> index 30bf832..8f3b6d6 100644
> --- a/virt/kvm/kvm_main.c
> +++ b/virt/kvm/kvm_main.c
> @@ -326,7 +326,15 @@ int __kvm_set_memory_region(struct kvm *kvm,
>  		memset(new.rmap, 0, npages * sizeof(*new.rmap));
>  
>  		new.user_alloc = user_alloc;
> -		new.userspace_addr = mem->userspace_addr;
> +		/*
> +		 * hva_to_rmmap() serialzies with the mmu_lock and to be
> +		 * safe it has to ignore memslots with !user_alloc &&
> +		 * !userspace_addr.
> +		 */
> +		if (user_alloc)
> +			new.userspace_addr = mem->userspace_addr;
> +		else
> +			new.userspace_addr = 0;
>  	}
>  	if (npages && !new.lpage_info) {
>  		int largepages = npages / KVM_PAGES_PER_HPAGE;
> @@ -355,14 +363,18 @@ int __kvm_set_memory_region(struct kvm *kvm,
>  		memset(new.dirty_bitmap, 0, dirty_bytes);
>  	}
>  
> +	spin_lock(&kvm->mmu_lock);
>  	if (mem->slot >= kvm->nmemslots)
>  		kvm->nmemslots = mem->slot + 1;
>  
>  	*memslot = new;
> +	spin_unlock(&kvm->mmu_lock);
>  
>  	r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc);
>  	if (r) {
> +		spin_lock(&kvm->mmu_lock);
>  		*memslot = old;
> +		spin_unlock(&kvm->mmu_lock);
>  		goto out_free;
>  	}
>  
>