AcoSail/kernel_optimize_test
8
0
Fork 0
forked from luck/tmp_suning_uos_patched
Code Pull Requests Activity

mm/gup: remove task_struct pointer for all gup code

Browse Source 
After the cleanup of page fault accounting, gup does not need to pass
task_struct around any more.  Remove that parameter in the whole gup
stack.

Signed-off-by: Peter Xu <peterx@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Reviewed-by: John Hubbard <jhubbard@nvidia.com>
Link: http://lkml.kernel.org/r/20200707225021.200906-26-peterx@redhat.com
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Peter Xu 2020-08-11 18:39:01 -07:00 committed by Linus Torvalds
parent a2beb5f1ef
commit 64019a2e46
18 changed files with 69 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/arc/kernel/process.c
View File

@ -91,7 +91,7 @@ SYSCALL_DEFINE3(arc_usr_cmpxchg, int *, uaddr, int, expected, int, new)
goto fail; goto fail;
mmap_read_lock(current->mm); mmap_read_lock(current->mm);
ret = fixup_user_fault(current, current->mm, (unsigned long) uaddr, ret = fixup_user_fault(current->mm, (unsigned long) uaddr,
FAULT_FLAG_WRITE, NULL); FAULT_FLAG_WRITE, NULL);
mmap_read_unlock(current->mm); mmap_read_unlock(current->mm);

2
arch/s390/kvm/interrupt.c
View File

@ -2768,7 +2768,7 @@ static struct page *get_map_page(struct kvm *kvm, u64 uaddr)
struct page *page = NULL; struct page *page = NULL;
mmap_read_lock(kvm->mm); mmap_read_lock(kvm->mm);
get_user_pages_remote(NULL, kvm->mm, uaddr, 1, FOLL_WRITE, get_user_pages_remote(kvm->mm, uaddr, 1, FOLL_WRITE,
&page, NULL, NULL); &page, NULL, NULL);
mmap_read_unlock(kvm->mm); mmap_read_unlock(kvm->mm);
return page; return page;

2
arch/s390/kvm/kvm-s390.c
View File

@ -1892,7 +1892,7 @@ static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
r = set_guest_storage_key(current->mm, hva, keys[i], 0); r = set_guest_storage_key(current->mm, hva, keys[i], 0);
if (r) { if (r) {
r = fixup_user_fault(current, current->mm, hva, r = fixup_user_fault(current->mm, hva,
FAULT_FLAG_WRITE, &unlocked); FAULT_FLAG_WRITE, &unlocked);
if (r) if (r)
break; break;

8
arch/s390/kvm/priv.c
View File

@ -273,7 +273,7 @@ static int handle_iske(struct kvm_vcpu *vcpu)
rc = get_guest_storage_key(current->mm, vmaddr, &key); rc = get_guest_storage_key(current->mm, vmaddr, &key);
if (rc) { if (rc) {
rc = fixup_user_fault(current, current->mm, vmaddr, rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked); FAULT_FLAG_WRITE, &unlocked);
if (!rc) { if (!rc) {
mmap_read_unlock(current->mm); mmap_read_unlock(current->mm);
@ -319,7 +319,7 @@ static int handle_rrbe(struct kvm_vcpu *vcpu)
mmap_read_lock(current->mm); mmap_read_lock(current->mm);
rc = reset_guest_reference_bit(current->mm, vmaddr); rc = reset_guest_reference_bit(current->mm, vmaddr);
if (rc < 0) { if (rc < 0) {
rc = fixup_user_fault(current, current->mm, vmaddr, rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked); FAULT_FLAG_WRITE, &unlocked);
if (!rc) { if (!rc) {
mmap_read_unlock(current->mm); mmap_read_unlock(current->mm);
@ -390,7 +390,7 @@ static int handle_sske(struct kvm_vcpu *vcpu)
m3 & SSKE_MC); m3 & SSKE_MC);
if (rc < 0) { if (rc < 0) {
rc = fixup_user_fault(current, current->mm, vmaddr, rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked); FAULT_FLAG_WRITE, &unlocked);
rc = !rc ? -EAGAIN : rc; rc = !rc ? -EAGAIN : rc;
} }
@ -1094,7 +1094,7 @@ static int handle_pfmf(struct kvm_vcpu *vcpu)
rc = cond_set_guest_storage_key(current->mm, vmaddr, rc = cond_set_guest_storage_key(current->mm, vmaddr,
key, NULL, nq, mr, mc); key, NULL, nq, mr, mc);
if (rc < 0) { if (rc < 0) {
rc = fixup_user_fault(current, current->mm, vmaddr, rc = fixup_user_fault(current->mm, vmaddr,
FAULT_FLAG_WRITE, &unlocked); FAULT_FLAG_WRITE, &unlocked);
rc = !rc ? -EAGAIN : rc; rc = !rc ? -EAGAIN : rc;
} }

4
arch/s390/mm/gmap.c
View File

@ -649,7 +649,7 @@ int gmap_fault(struct gmap *gmap, unsigned long gaddr,
rc = vmaddr; rc = vmaddr;
goto out_up; goto out_up;
} }
if (fixup_user_fault(current, gmap->mm, vmaddr, fault_flags, if (fixup_user_fault(gmap->mm, vmaddr, fault_flags,
&unlocked)) { &unlocked)) {
rc = -EFAULT; rc = -EFAULT;
goto out_up; goto out_up;
@ -879,7 +879,7 @@ static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr,
BUG_ON(gmap_is_shadow(gmap)); BUG_ON(gmap_is_shadow(gmap));
fault_flags = (prot == PROT_WRITE) ? FAULT_FLAG_WRITE : 0; fault_flags = (prot == PROT_WRITE) ? FAULT_FLAG_WRITE : 0;
if (fixup_user_fault(current, mm, vmaddr, fault_flags, &unlocked)) if (fixup_user_fault(mm, vmaddr, fault_flags, &unlocked))
return -EFAULT; return -EFAULT;
if (unlocked) if (unlocked)
/* lost mmap_lock, caller has to retry __gmap_translate */ /* lost mmap_lock, caller has to retry __gmap_translate */

2
drivers/gpu/drm/i915/gem/i915_gem_userptr.c
View File

@ -469,7 +469,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
locked = 1; locked = 1;
} }
ret = pin_user_pages_remote ret = pin_user_pages_remote
(work->task, mm, (mm,
obj->userptr.ptr + pinned * PAGE_SIZE, obj->userptr.ptr + pinned * PAGE_SIZE,
npages - pinned, npages - pinned,
flags, flags,

2
drivers/infiniband/core/umem_odp.c
View File

@ -439,7 +439,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
* complex (and doesn't gain us much performance in most use * complex (and doesn't gain us much performance in most use
* cases). * cases).
*/ */
npages = get_user_pages_remote(owning_process, owning_mm, npages = get_user_pages_remote(owning_mm,
user_virt, gup_num_pages, user_virt, gup_num_pages,
flags, local_page_list, NULL, NULL); flags, local_page_list, NULL, NULL);
mmap_read_unlock(owning_mm); mmap_read_unlock(owning_mm);

4
drivers/vfio/vfio_iommu_type1.c
View File

@ -425,7 +425,7 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm,
if (ret) { if (ret) {
bool unlocked = false; bool unlocked = false;
ret = fixup_user_fault(NULL, mm, vaddr, ret = fixup_user_fault(mm, vaddr,
FAULT_FLAG_REMOTE | FAULT_FLAG_REMOTE |
(write_fault ? FAULT_FLAG_WRITE : 0), (write_fault ? FAULT_FLAG_WRITE : 0),
&unlocked); &unlocked);
@ -453,7 +453,7 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
flags |= FOLL_WRITE; flags |= FOLL_WRITE;
mmap_read_lock(mm); mmap_read_lock(mm);
ret = pin_user_pages_remote(NULL, mm, vaddr, 1, flags | FOLL_LONGTERM, ret = pin_user_pages_remote(mm, vaddr, 1, flags | FOLL_LONGTERM,
page, NULL, NULL); page, NULL, NULL);
if (ret == 1) { if (ret == 1) {
*pfn = page_to_pfn(page[0]); *pfn = page_to_pfn(page[0]);

2
fs/exec.c
View File

@ -217,7 +217,7 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos,
* We are doing an exec(). 'current' is the process * We are doing an exec(). 'current' is the process
* doing the exec and bprm->mm is the new process's mm. * doing the exec and bprm->mm is the new process's mm.
*/ */
ret = get_user_pages_remote(current, bprm->mm, pos, 1, gup_flags, ret = get_user_pages_remote(bprm->mm, pos, 1, gup_flags,
&page, NULL, NULL); &page, NULL, NULL);
if (ret <= 0) if (ret <= 0)
return NULL; return NULL;

9
include/linux/mm.h
View File

@ -1661,7 +1661,7 @@ int invalidate_inode_page(struct page *page);
extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma, extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
unsigned long address, unsigned int flags, unsigned long address, unsigned int flags,
struct pt_regs *regs); struct pt_regs *regs);
extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, extern int fixup_user_fault(struct mm_struct *mm,
unsigned long address, unsigned int fault_flags, unsigned long address, unsigned int fault_flags,
bool *unlocked); bool *unlocked);
void unmap_mapping_pages(struct address_space *mapping, void unmap_mapping_pages(struct address_space *mapping,
@ -1677,8 +1677,7 @@ static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
BUG(); BUG();
return VM_FAULT_SIGBUS; return VM_FAULT_SIGBUS;
} }
static inline int fixup_user_fault(struct task_struct *tsk, static inline int fixup_user_fault(struct mm_struct *mm, unsigned long address,
struct mm_struct *mm, unsigned long address,
unsigned int fault_flags, bool *unlocked) unsigned int fault_flags, bool *unlocked)
{ {
/* should never happen if there's no MMU */ /* should never happen if there's no MMU */
@ -1704,11 +1703,11 @@ extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
extern int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, extern int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
unsigned long addr, void *buf, int len, unsigned int gup_flags); unsigned long addr, void *buf, int len, unsigned int gup_flags);
long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, long get_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked); struct vm_area_struct **vmas, int *locked);
long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, long pin_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked); struct vm_area_struct **vmas, int *locked);

6
kernel/events/uprobes.c
View File

@ -376,7 +376,7 @@ __update_ref_ctr(struct mm_struct *mm, unsigned long vaddr, short d)
if (!vaddr || !d) if (!vaddr || !d)
return -EINVAL; return -EINVAL;
ret = get_user_pages_remote(NULL, mm, vaddr, 1, ret = get_user_pages_remote(mm, vaddr, 1,
FOLL_WRITE, &page, &vma, NULL); FOLL_WRITE, &page, &vma, NULL);
if (unlikely(ret <= 0)) { if (unlikely(ret <= 0)) {
/* /*
@ -477,7 +477,7 @@ int uprobe_write_opcode(struct arch_uprobe *auprobe, struct mm_struct *mm,
if (is_register) if (is_register)
gup_flags |= FOLL_SPLIT_PMD; gup_flags |= FOLL_SPLIT_PMD;
/* Read the page with vaddr into memory */ /* Read the page with vaddr into memory */
ret = get_user_pages_remote(NULL, mm, vaddr, 1, gup_flags, ret = get_user_pages_remote(mm, vaddr, 1, gup_flags,
&old_page, &vma, NULL); &old_page, &vma, NULL);
if (ret <= 0) if (ret <= 0)
return ret; return ret;
@ -2029,7 +2029,7 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
* but we treat this as a 'remote' access since it is * but we treat this as a 'remote' access since it is
* essentially a kernel access to the memory. * essentially a kernel access to the memory.
*/ */
result = get_user_pages_remote(NULL, mm, vaddr, 1, FOLL_FORCE, &page, result = get_user_pages_remote(mm, vaddr, 1, FOLL_FORCE, &page,
NULL, NULL); NULL, NULL);
if (result < 0) if (result < 0)
return result; return result;

2
kernel/futex.c
View File

@ -678,7 +678,7 @@ static int fault_in_user_writeable(u32 __user *uaddr)
int ret; int ret;
mmap_read_lock(mm); mmap_read_lock(mm);
ret = fixup_user_fault(current, mm, (unsigned long)uaddr, ret = fixup_user_fault(mm, (unsigned long)uaddr,
FAULT_FLAG_WRITE, NULL); FAULT_FLAG_WRITE, NULL);
mmap_read_unlock(mm); mmap_read_unlock(mm);

101
mm/gup.c
View File

@ -859,7 +859,7 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address,
* does not include FOLL_NOWAIT, the mmap_lock may be released. If it * does not include FOLL_NOWAIT, the mmap_lock may be released. If it
* is, *@locked will be set to 0 and -EBUSY returned. * is, *@locked will be set to 0 and -EBUSY returned.
*/ */
static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, static int faultin_page(struct vm_area_struct *vma,
unsigned long address, unsigned int *flags, int *locked) unsigned long address, unsigned int *flags, int *locked)
{ {
unsigned int fault_flags = 0; unsigned int fault_flags = 0;
@ -962,7 +962,6 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
/** /**
* __get_user_pages() - pin user pages in memory * __get_user_pages() - pin user pages in memory
* @tsk: task_struct of target task
* @mm: mm_struct of target mm * @mm: mm_struct of target mm
* @start: starting user address * @start: starting user address
* @nr_pages: number of pages from start to pin * @nr_pages: number of pages from start to pin
@ -1021,7 +1020,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
* instead of __get_user_pages. __get_user_pages should be used only if * instead of __get_user_pages. __get_user_pages should be used only if
* you need some special @gup_flags. * you need some special @gup_flags.
*/ */
static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, static long __get_user_pages(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
@ -1103,8 +1102,7 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
page = follow_page_mask(vma, start, foll_flags, &ctx); page = follow_page_mask(vma, start, foll_flags, &ctx);
if (!page) { if (!page) {
ret = faultin_page(tsk, vma, start, &foll_flags, ret = faultin_page(vma, start, &foll_flags, locked);
locked);
switch (ret) { switch (ret) {
case 0: case 0:
goto retry; goto retry;
@ -1178,8 +1176,6 @@ static bool vma_permits_fault(struct vm_area_struct *vma,
/** /**
* fixup_user_fault() - manually resolve a user page fault * fixup_user_fault() - manually resolve a user page fault
* @tsk: the task_struct to use for page fault accounting, or
* NULL if faults are not to be recorded.
* @mm: mm_struct of target mm * @mm: mm_struct of target mm
* @address: user address * @address: user address
* @fault_flags:flags to pass down to handle_mm_fault() * @fault_flags:flags to pass down to handle_mm_fault()
@ -1207,7 +1203,7 @@ static bool vma_permits_fault(struct vm_area_struct *vma,
* This function will not return with an unlocked mmap_lock. So it has not the * This function will not return with an unlocked mmap_lock. So it has not the
* same semantics wrt the @mm->mmap_lock as does filemap_fault(). * same semantics wrt the @mm->mmap_lock as does filemap_fault().
*/ */
int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, int fixup_user_fault(struct mm_struct *mm,
unsigned long address, unsigned int fault_flags, unsigned long address, unsigned int fault_flags,
bool *unlocked) bool *unlocked)
{ {
@ -1256,8 +1252,7 @@ EXPORT_SYMBOL_GPL(fixup_user_fault);
* Please note that this function, unlike __get_user_pages will not * Please note that this function, unlike __get_user_pages will not
* return 0 for nr_pages > 0 without FOLL_NOWAIT * return 0 for nr_pages > 0 without FOLL_NOWAIT
*/ */
static __always_inline long __get_user_pages_locked(struct task_struct *tsk, static __always_inline long __get_user_pages_locked(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long start,
unsigned long nr_pages, unsigned long nr_pages,
struct page **pages, struct page **pages,
@ -1290,7 +1285,7 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
pages_done = 0; pages_done = 0;
lock_dropped = false; lock_dropped = false;
for (;;) { for (;;) {
ret = __get_user_pages(tsk, mm, start, nr_pages, flags, pages, ret = __get_user_pages(mm, start, nr_pages, flags, pages,
vmas, locked); vmas, locked);
if (!locked) if (!locked)
/* VM_FAULT_RETRY couldn't trigger, bypass */ /* VM_FAULT_RETRY couldn't trigger, bypass */
@ -1350,7 +1345,7 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
} }
*locked = 1; *locked = 1;
ret = __get_user_pages(tsk, mm, start, 1, flags | FOLL_TRIED, ret = __get_user_pages(mm, start, 1, flags | FOLL_TRIED,
pages, NULL, locked); pages, NULL, locked);
if (!*locked) { if (!*locked) {
/* Continue to retry until we succeeded */ /* Continue to retry until we succeeded */
@ -1437,7 +1432,7 @@ long populate_vma_page_range(struct vm_area_struct *vma,
* We made sure addr is within a VMA, so the following will * We made sure addr is within a VMA, so the following will
* not result in a stack expansion that recurses back here. * not result in a stack expansion that recurses back here.
*/ */
return __get_user_pages(current, mm, start, nr_pages, gup_flags, return __get_user_pages(mm, start, nr_pages, gup_flags,
NULL, NULL, locked); NULL, NULL, locked);
} }
@ -1521,7 +1516,7 @@ struct page *get_dump_page(unsigned long addr)
struct vm_area_struct *vma; struct vm_area_struct *vma;
struct page *page; struct page *page;
if (__get_user_pages(current, current->mm, addr, 1, if (__get_user_pages(current->mm, addr, 1,
FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma, FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma,
NULL) < 1) NULL) < 1)
return NULL; return NULL;
@ -1530,8 +1525,7 @@ struct page *get_dump_page(unsigned long addr)
} }
#endif /* CONFIG_ELF_CORE */ #endif /* CONFIG_ELF_CORE */
#else /* CONFIG_MMU */ #else /* CONFIG_MMU */
static long __get_user_pages_locked(struct task_struct *tsk, static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
struct mm_struct *mm, unsigned long start,
unsigned long nr_pages, struct page **pages, unsigned long nr_pages, struct page **pages,
struct vm_area_struct **vmas, int *locked, struct vm_area_struct **vmas, int *locked,
unsigned int foll_flags) unsigned int foll_flags)
@ -1596,8 +1590,7 @@ static bool check_dax_vmas(struct vm_area_struct **vmas, long nr_pages)
} }
#ifdef CONFIG_CMA #ifdef CONFIG_CMA
static long check_and_migrate_cma_pages(struct task_struct *tsk, static long check_and_migrate_cma_pages(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long start,
unsigned long nr_pages, unsigned long nr_pages,
struct page **pages, struct page **pages,
@ -1675,7 +1668,7 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk,
* again migrating any new CMA pages which we failed to isolate * again migrating any new CMA pages which we failed to isolate
* earlier. * earlier.
*/ */
ret = __get_user_pages_locked(tsk, mm, start, nr_pages, ret = __get_user_pages_locked(mm, start, nr_pages,
pages, vmas, NULL, pages, vmas, NULL,
gup_flags); gup_flags);
@ -1689,8 +1682,7 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk,
return ret; return ret;
} }
#else #else
static long check_and_migrate_cma_pages(struct task_struct *tsk, static long check_and_migrate_cma_pages(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long start,
unsigned long nr_pages, unsigned long nr_pages,
struct page **pages, struct page **pages,
@ -1705,8 +1697,7 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk,
* __gup_longterm_locked() is a wrapper for __get_user_pages_locked which * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which
* allows us to process the FOLL_LONGTERM flag. * allows us to process the FOLL_LONGTERM flag.
*/ */
static long __gup_longterm_locked(struct task_struct *tsk, static long __gup_longterm_locked(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long start,
unsigned long nr_pages, unsigned long nr_pages,
struct page **pages, struct page **pages,
@ -1731,7 +1722,7 @@ static long __gup_longterm_locked(struct task_struct *tsk,
flags = memalloc_nocma_save(); flags = memalloc_nocma_save();
} }
rc = __get_user_pages_locked(tsk, mm, start, nr_pages, pages, rc = __get_user_pages_locked(mm, start, nr_pages, pages,
vmas_tmp, NULL, gup_flags); vmas_tmp, NULL, gup_flags);
if (gup_flags & FOLL_LONGTERM) { if (gup_flags & FOLL_LONGTERM) {
@ -1745,7 +1736,7 @@ static long __gup_longterm_locked(struct task_struct *tsk,
goto out; goto out;
} }
rc = check_and_migrate_cma_pages(tsk, mm, start, rc, pages, rc = check_and_migrate_cma_pages(mm, start, rc, pages,
vmas_tmp, gup_flags); vmas_tmp, gup_flags);
out: out:
memalloc_nocma_restore(flags); memalloc_nocma_restore(flags);
@ -1756,22 +1747,20 @@ static long __gup_longterm_locked(struct task_struct *tsk,
return rc; return rc;
} }
#else /* !CONFIG_FS_DAX && !CONFIG_CMA */ #else /* !CONFIG_FS_DAX && !CONFIG_CMA */
static __always_inline long __gup_longterm_locked(struct task_struct *tsk, static __always_inline long __gup_longterm_locked(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long start,
unsigned long nr_pages, unsigned long nr_pages,
struct page **pages, struct page **pages,
struct vm_area_struct **vmas, struct vm_area_struct **vmas,
unsigned int flags) unsigned int flags)
{ {
return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas, return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
NULL, flags); NULL, flags);
} }
#endif /* CONFIG_FS_DAX || CONFIG_CMA */ #endif /* CONFIG_FS_DAX || CONFIG_CMA */
#ifdef CONFIG_MMU #ifdef CONFIG_MMU
static long __get_user_pages_remote(struct task_struct *tsk, static long __get_user_pages_remote(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
@ -1790,20 +1779,18 @@ static long __get_user_pages_remote(struct task_struct *tsk,
* This will check the vmas (even if our vmas arg is NULL) * This will check the vmas (even if our vmas arg is NULL)
* and return -ENOTSUPP if DAX isn't allowed in this case: * and return -ENOTSUPP if DAX isn't allowed in this case:
*/ */
return __gup_longterm_locked(tsk, mm, start, nr_pages, pages, return __gup_longterm_locked(mm, start, nr_pages, pages,
vmas, gup_flags | FOLL_TOUCH | vmas, gup_flags | FOLL_TOUCH |
FOLL_REMOTE); FOLL_REMOTE);
} }
return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas, return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
locked, locked,
gup_flags | FOLL_TOUCH | FOLL_REMOTE); gup_flags | FOLL_TOUCH | FOLL_REMOTE);
} }
/** /**
* get_user_pages_remote() - pin user pages in memory * get_user_pages_remote() - pin user pages in memory
* @tsk: the task_struct to use for page fault accounting, or
* NULL if faults are not to be recorded.
* @mm: mm_struct of target mm * @mm: mm_struct of target mm
* @start: starting user address * @start: starting user address
* @nr_pages: number of pages from start to pin * @nr_pages: number of pages from start to pin
@ -1862,7 +1849,7 @@ static long __get_user_pages_remote(struct task_struct *tsk,
* should use get_user_pages_remote because it cannot pass * should use get_user_pages_remote because it cannot pass
* FAULT_FLAG_ALLOW_RETRY to handle_mm_fault. * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault.
*/ */
long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, long get_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
@ -1874,13 +1861,13 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
return -EINVAL; return -EINVAL;
return __get_user_pages_remote(tsk, mm, start, nr_pages, gup_flags, return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
pages, vmas, locked); pages, vmas, locked);
} }
EXPORT_SYMBOL(get_user_pages_remote); EXPORT_SYMBOL(get_user_pages_remote);
#else /* CONFIG_MMU */ #else /* CONFIG_MMU */
long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, long get_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
@ -1888,8 +1875,7 @@ long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
return 0; return 0;
} }
static long __get_user_pages_remote(struct task_struct *tsk, static long __get_user_pages_remote(struct mm_struct *mm,
struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
@ -1909,11 +1895,10 @@ static long __get_user_pages_remote(struct task_struct *tsk,
* @vmas: array of pointers to vmas corresponding to each page. * @vmas: array of pointers to vmas corresponding to each page.
* Or NULL if the caller does not require them. * Or NULL if the caller does not require them.
* *
* This is the same as get_user_pages_remote(), just with a * This is the same as get_user_pages_remote(), just with a less-flexible
* less-flexible calling convention where we assume that the task * calling convention where we assume that the mm being operated on belongs to
* and mm being operated on are the current task's and don't allow * the current task, and doesn't allow passing of a locked parameter. We also
* passing of a locked parameter. We also obviously don't pass * obviously don't pass FOLL_REMOTE in here.
* FOLL_REMOTE in here.
*/ */
long get_user_pages(unsigned long start, unsigned long nr_pages, long get_user_pages(unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
@ -1926,7 +1911,7 @@ long get_user_pages(unsigned long start, unsigned long nr_pages,
if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
return -EINVAL; return -EINVAL;
return __gup_longterm_locked(current, current->mm, start, nr_pages, return __gup_longterm_locked(current->mm, start, nr_pages,
pages, vmas, gup_flags | FOLL_TOUCH); pages, vmas, gup_flags | FOLL_TOUCH);
} }
EXPORT_SYMBOL(get_user_pages); EXPORT_SYMBOL(get_user_pages);
@ -1936,7 +1921,7 @@ EXPORT_SYMBOL(get_user_pages);
* *
* mmap_read_lock(mm); * mmap_read_lock(mm);
* do_something() * do_something()
* get_user_pages(tsk, mm, ..., pages, NULL); * get_user_pages(mm, ..., pages, NULL);
* mmap_read_unlock(mm); * mmap_read_unlock(mm);
* *
* to: * to:
@ -1944,7 +1929,7 @@ EXPORT_SYMBOL(get_user_pages);
* int locked = 1; * int locked = 1;
* mmap_read_lock(mm); * mmap_read_lock(mm);
* do_something() * do_something()
* get_user_pages_locked(tsk, mm, ..., pages, &locked); * get_user_pages_locked(mm, ..., pages, &locked);
* if (locked) * if (locked)
* mmap_read_unlock(mm); * mmap_read_unlock(mm);
* *
@ -1982,7 +1967,7 @@ long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
return -EINVAL; return -EINVAL;
return __get_user_pages_locked(current, current->mm, start, nr_pages, return __get_user_pages_locked(current->mm, start, nr_pages,
pages, NULL, locked, pages, NULL, locked,
gup_flags | FOLL_TOUCH); gup_flags | FOLL_TOUCH);
} }
@ -1992,12 +1977,12 @@ EXPORT_SYMBOL(get_user_pages_locked);
* get_user_pages_unlocked() is suitable to replace the form: * get_user_pages_unlocked() is suitable to replace the form:
* *
* mmap_read_lock(mm); * mmap_read_lock(mm);
* get_user_pages(tsk, mm, ..., pages, NULL); * get_user_pages(mm, ..., pages, NULL);
* mmap_read_unlock(mm); * mmap_read_unlock(mm);
* *
* with: * with:
* *
* get_user_pages_unlocked(tsk, mm, ..., pages); * get_user_pages_unlocked(mm, ..., pages);
* *
* It is functionally equivalent to get_user_pages_fast so * It is functionally equivalent to get_user_pages_fast so
* get_user_pages_fast should be used instead if specific gup_flags * get_user_pages_fast should be used instead if specific gup_flags
@ -2020,7 +2005,7 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
return -EINVAL; return -EINVAL;
mmap_read_lock(mm); mmap_read_lock(mm);
ret = __get_user_pages_locked(current, mm, start, nr_pages, pages, NULL, ret = __get_user_pages_locked(mm, start, nr_pages, pages, NULL,
&locked, gup_flags | FOLL_TOUCH); &locked, gup_flags | FOLL_TOUCH);
if (locked) if (locked)
mmap_read_unlock(mm); mmap_read_unlock(mm);
@ -2665,7 +2650,7 @@ static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
*/ */
if (gup_flags & FOLL_LONGTERM) { if (gup_flags & FOLL_LONGTERM) {
mmap_read_lock(current->mm); mmap_read_lock(current->mm);
ret = __gup_longterm_locked(current, current->mm, ret = __gup_longterm_locked(current->mm,
start, nr_pages, start, nr_pages,
pages, NULL, gup_flags); pages, NULL, gup_flags);
mmap_read_unlock(current->mm); mmap_read_unlock(current->mm);
@ -2908,10 +2893,8 @@ int pin_user_pages_fast_only(unsigned long start, int nr_pages,
EXPORT_SYMBOL_GPL(pin_user_pages_fast_only); EXPORT_SYMBOL_GPL(pin_user_pages_fast_only);
/** /**
* pin_user_pages_remote() - pin pages of a remote process (task != current) * pin_user_pages_remote() - pin pages of a remote process
* *
* @tsk: the task_struct to use for page fault accounting, or
* NULL if faults are not to be recorded.
* @mm: mm_struct of target mm * @mm: mm_struct of target mm
* @start: starting user address * @start: starting user address
* @nr_pages: number of pages from start to pin * @nr_pages: number of pages from start to pin
@ -2932,7 +2915,7 @@ EXPORT_SYMBOL_GPL(pin_user_pages_fast_only);
* FOLL_PIN means that the pages must be released via unpin_user_page(). Please * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
* see Documentation/core-api/pin_user_pages.rst for details. * see Documentation/core-api/pin_user_pages.rst for details.
*/ */
long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm, long pin_user_pages_remote(struct mm_struct *mm,
unsigned long start, unsigned long nr_pages, unsigned long start, unsigned long nr_pages,
unsigned int gup_flags, struct page **pages, unsigned int gup_flags, struct page **pages,
struct vm_area_struct **vmas, int *locked) struct vm_area_struct **vmas, int *locked)
@ -2942,7 +2925,7 @@ long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
return -EINVAL; return -EINVAL;
gup_flags |= FOLL_PIN; gup_flags |= FOLL_PIN;
return __get_user_pages_remote(tsk, mm, start, nr_pages, gup_flags, return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
pages, vmas, locked); pages, vmas, locked);
} }
EXPORT_SYMBOL(pin_user_pages_remote); EXPORT_SYMBOL(pin_user_pages_remote);
@ -2974,7 +2957,7 @@ long pin_user_pages(unsigned long start, unsigned long nr_pages,
return -EINVAL; return -EINVAL;
gup_flags |= FOLL_PIN; gup_flags |= FOLL_PIN;
return __gup_longterm_locked(current, current->mm, start, nr_pages, return __gup_longterm_locked(current->mm, start, nr_pages,
pages, vmas, gup_flags); pages, vmas, gup_flags);
} }
EXPORT_SYMBOL(pin_user_pages); EXPORT_SYMBOL(pin_user_pages);
@ -3019,7 +3002,7 @@ long pin_user_pages_locked(unsigned long start, unsigned long nr_pages,
return -EINVAL; return -EINVAL;
gup_flags |= FOLL_PIN; gup_flags |= FOLL_PIN;
return __get_user_pages_locked(current, current->mm, start, nr_pages, return __get_user_pages_locked(current->mm, start, nr_pages,
pages, NULL, locked, pages, NULL, locked,
gup_flags | FOLL_TOUCH); gup_flags | FOLL_TOUCH);
} }

2
mm/memory.c
View File

@ -4742,7 +4742,7 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
void *maddr; void *maddr;
struct page *page = NULL; struct page *page = NULL;
ret = get_user_pages_remote(tsk, mm, addr, 1, ret = get_user_pages_remote(mm, addr, 1,
gup_flags, &page, &vma, NULL); gup_flags, &page, &vma, NULL);
if (ret <= 0) { if (ret <= 0) {
#ifndef CONFIG_HAVE_IOREMAP_PROT #ifndef CONFIG_HAVE_IOREMAP_PROT

2
mm/process_vm_access.c
View File

@ -105,7 +105,7 @@ static int process_vm_rw_single_vec(unsigned long addr,
* current/current->mm * current/current->mm
*/ */
mmap_read_lock(mm); mmap_read_lock(mm);
pinned_pages = pin_user_pages_remote(task, mm, pa, pinned_pages, pinned_pages = pin_user_pages_remote(mm, pa, pinned_pages,
flags, process_pages, flags, process_pages,
NULL, &locked); NULL, &locked);
if (locked) if (locked)

2
security/tomoyo/domain.c
View File

@ -914,7 +914,7 @@ bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos,
* (represented by bprm). 'current' is the process doing * (represented by bprm). 'current' is the process doing
* the execve(). * the execve().
*/ */
if (get_user_pages_remote(current, bprm->mm, pos, 1, if (get_user_pages_remote(bprm->mm, pos, 1,
FOLL_FORCE, &page, NULL, NULL) <= 0) FOLL_FORCE, &page, NULL, NULL) <= 0)
return false; return false;
#else #else

2
virt/kvm/async_pf.c
View File

@ -61,7 +61,7 @@ static void async_pf_execute(struct work_struct *work)
* access remotely. * access remotely.
*/ */
mmap_read_lock(mm); mmap_read_lock(mm);
get_user_pages_remote(NULL, mm, addr, 1, FOLL_WRITE, NULL, NULL, get_user_pages_remote(mm, addr, 1, FOLL_WRITE, NULL, NULL,
&locked); &locked);
if (locked) if (locked)
mmap_read_unlock(mm); mmap_read_unlock(mm);

2
virt/kvm/kvm_main.c
View File

@ -1893,7 +1893,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma,
* not call the fault handler, so do it here. * not call the fault handler, so do it here.
*/ */
bool unlocked = false; bool unlocked = false;
r = fixup_user_fault(current, current->mm, addr, r = fixup_user_fault(current->mm, addr,
(write_fault ? FAULT_FLAG_WRITE : 0), (write_fault ? FAULT_FLAG_WRITE : 0),
&unlocked); &unlocked);
if (unlocked) if (unlocked)