hardenedbsd/HardenedBSD
1
0
Fork 0
mirror of https://git.hardenedbsd.org/hardenedbsd/HardenedBSD.git synced 2024-11-23 17:41:04 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity
5fa4151e92
HardenedBSD/sys/vm/vm_domainset.c
Jason A. Harmening 2619c5ccfe Avoid waiting on physical allocations that can't possibly be satisfied 
- Change vm_page_reclaim_contig[_domain] to return an errno instead
  of a boolean.  0 indicates a successful reclaim, ENOMEM indicates
  lack of available memory to reclaim, with any other error (currently
  only ERANGE) indicating that reclamation is impossible for the
  specified address range.  Change all callers to only follow
  up with vm_page_wait* in the ENOMEM case.

- Introduce vm_domainset_iter_ignore(), which marks the specified
  domain as unavailable for further use by the iterator.  Use this
  function to ignore domains that can't possibly satisfy a physical
  allocation request.  Since WAITOK allocations run the iterators
  repeatedly, this avoids the possibility of infinitely spinning
  in domain iteration if no available domain can satisfy the
  allocation request.

PR:		274252
Reported by:	kevans
Tested by:	kevans
Reviewed by:	markj
Differential Revision: https://reviews.freebsd.org/D42706
2023-12-23 23:01:40 -06:00

383 lines
9.9 KiB
C
Raw Blame History

/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2017, Jeffrey Roberson <jeff@freebsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice unmodified, this list of conditions, and the following
* disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include <sys/cdefs.h>
#include "opt_vm.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bitset.h>
#include <sys/domainset.h>
#include <sys/proc.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/malloc.h>
#include <sys/rwlock.h>
#include <sys/vmmeter.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_domainset.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_phys.h>
#ifdef NUMA
/*
* Iterators are written such that the first nowait pass has as short a
* codepath as possible to eliminate bloat from the allocator. It is
* assumed that most allocations are successful.
*/
static int vm_domainset_default_stride = 64;
/*
* Determine which policy is to be used for this allocation.
*/
static void
vm_domainset_iter_init(struct vm_domainset_iter *di, struct domainset *ds,
int *iter, struct vm_object *obj, vm_pindex_t pindex)
{
di->di_domain = ds;
di->di_iter = iter;
di->di_policy = ds->ds_policy;
DOMAINSET_COPY(&ds->ds_mask, &di->di_valid_mask);
if (di->di_policy == DOMAINSET_POLICY_INTERLEAVE) {
#if VM_NRESERVLEVEL > 0
if (vm_object_reserv(obj)) {
/*
* Color the pindex so we end up on the correct
* reservation boundary.
*/
pindex += obj->pg_color;
pindex >>= VM_LEVEL_0_ORDER;
} else
#endif
pindex /= vm_domainset_default_stride;
/*
* Offset pindex so the first page of each object does
* not end up in domain 0.
*/
if (obj != NULL)
pindex += (((uintptr_t)obj) / sizeof(*obj));
di->di_offset = pindex;
}
/* Skip domains below min on the first pass. */
di->di_minskip = true;
}
static void
vm_domainset_iter_rr(struct vm_domainset_iter *di, int *domain)
{
*domain = di->di_domain->ds_order[
++(*di->di_iter) % di->di_domain->ds_cnt];
}
static void
vm_domainset_iter_prefer(struct vm_domainset_iter *di, int *domain)
{
int d;
do {
d = di->di_domain->ds_order[
++(*di->di_iter) % di->di_domain->ds_cnt];
} while (d == di->di_domain->ds_prefer);
*domain = d;
}
static void
vm_domainset_iter_interleave(struct vm_domainset_iter *di, int *domain)
{
int d;
d = di->di_offset % di->di_domain->ds_cnt;
*di->di_iter = d;
*domain = di->di_domain->ds_order[d];
}
static void
vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain)
{
KASSERT(di->di_n > 0,
("vm_domainset_iter_first: Invalid n %d", di->di_n));
switch (di->di_policy) {
case DOMAINSET_POLICY_FIRSTTOUCH:
/*
* To prevent impossible allocations we convert an invalid
* first-touch to round-robin.
*/
/* FALLTHROUGH */
case DOMAINSET_POLICY_INTERLEAVE:
/* FALLTHROUGH */
case DOMAINSET_POLICY_ROUNDROBIN:
vm_domainset_iter_rr(di, domain);
break;
case DOMAINSET_POLICY_PREFER:
vm_domainset_iter_prefer(di, domain);
break;
default:
panic("vm_domainset_iter_first: Unknown policy %d",
di->di_policy);
}
KASSERT(*domain < vm_ndomains,
("vm_domainset_iter_next: Invalid domain %d", *domain));
}
static void
vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain)
{
switch (di->di_policy) {
case DOMAINSET_POLICY_FIRSTTOUCH:
*domain = PCPU_GET(domain);
if (DOMAINSET_ISSET(*domain, &di->di_valid_mask)) {
/*
* Add an extra iteration because we will visit the
* current domain a second time in the rr iterator.
*/
di->di_n = di->di_domain->ds_cnt + 1;
break;
}
/*
* To prevent impossible allocations we convert an invalid
* first-touch to round-robin.
*/
/* FALLTHROUGH */
case DOMAINSET_POLICY_ROUNDROBIN:
di->di_n = di->di_domain->ds_cnt;
vm_domainset_iter_rr(di, domain);
break;
case DOMAINSET_POLICY_PREFER:
*domain = di->di_domain->ds_prefer;
di->di_n = di->di_domain->ds_cnt;
break;
case DOMAINSET_POLICY_INTERLEAVE:
vm_domainset_iter_interleave(di, domain);
di->di_n = di->di_domain->ds_cnt;
break;
default:
panic("vm_domainset_iter_first: Unknown policy %d",
di->di_policy);
}
KASSERT(di->di_n > 0,
("vm_domainset_iter_first: Invalid n %d", di->di_n));
KASSERT(*domain < vm_ndomains,
("vm_domainset_iter_first: Invalid domain %d", *domain));
}
void
vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
vm_pindex_t pindex, int *domain, int *req)
{
struct domainset_ref *dr;
/*
* Object policy takes precedence over thread policy. The policies
* are immutable and unsynchronized. Updates can race but pointer
* loads are assumed to be atomic.
*/
if (obj != NULL && obj->domain.dr_policy != NULL)
dr = &obj->domain;
else
dr = &curthread->td_domain;
vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, obj, pindex);
di->di_flags = *req;
*req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) |
VM_ALLOC_NOWAIT;
vm_domainset_iter_first(di, domain);
if (vm_page_count_min_domain(*domain))
vm_domainset_iter_page(di, obj, domain);
}
int
vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
int *domain)
{
if (__predict_false(DOMAINSET_EMPTY(&di->di_valid_mask)))
return (ENOMEM);
/* If there are more domains to visit we run the iterator. */
while (--di->di_n != 0) {
vm_domainset_iter_next(di, domain);
if (DOMAINSET_ISSET(*domain, &di->di_valid_mask) &&
(!di->di_minskip || !vm_page_count_min_domain(*domain)))
return (0);
}
/* If we skipped domains below min restart the search. */
if (di->di_minskip) {
di->di_minskip = false;
vm_domainset_iter_first(di, domain);
return (0);
}
/* If we visited all domains and this was a NOWAIT we return error. */
if ((di->di_flags & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) == 0)
return (ENOMEM);
/* Wait for one of the domains to accumulate some free pages. */
if (obj != NULL)
VM_OBJECT_WUNLOCK(obj);
vm_wait_doms(&di->di_valid_mask, 0);
if (obj != NULL)
VM_OBJECT_WLOCK(obj);
if ((di->di_flags & VM_ALLOC_WAITFAIL) != 0)
return (ENOMEM);
/* Restart the search. */
vm_domainset_iter_first(di, domain);
return (0);
}
static void
_vm_domainset_iter_policy_init(struct vm_domainset_iter *di, int *domain,
int *flags)
{
di->di_flags = *flags;
*flags = (di->di_flags & ~M_WAITOK) | M_NOWAIT;
vm_domainset_iter_first(di, domain);
if (vm_page_count_min_domain(*domain))
vm_domainset_iter_policy(di, domain);
}
void
vm_domainset_iter_policy_init(struct vm_domainset_iter *di,
struct domainset *ds, int *domain, int *flags)
{
vm_domainset_iter_init(di, ds, &curthread->td_domain.dr_iter, NULL, 0);
_vm_domainset_iter_policy_init(di, domain, flags);
}
void
vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,
struct domainset_ref *dr, int *domain, int *flags)
{
vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, NULL, 0);
_vm_domainset_iter_policy_init(di, domain, flags);
}
int
vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
{
if (DOMAINSET_EMPTY(&di->di_valid_mask))
return (ENOMEM);
/* If there are more domains to visit we run the iterator. */
while (--di->di_n != 0) {
vm_domainset_iter_next(di, domain);
if (DOMAINSET_ISSET(*domain, &di->di_valid_mask) &&
(!di->di_minskip || !vm_page_count_min_domain(*domain)))
return (0);
}
/* If we skipped domains below min restart the search. */
if (di->di_minskip) {
di->di_minskip = false;
vm_domainset_iter_first(di, domain);
return (0);
}
/* If we visited all domains and this was a NOWAIT we return error. */
if ((di->di_flags & M_WAITOK) == 0)
return (ENOMEM);
/* Wait for one of the domains to accumulate some free pages. */
vm_wait_doms(&di->di_valid_mask, 0);
/* Restart the search. */
vm_domainset_iter_first(di, domain);
return (0);
}
void
vm_domainset_iter_ignore(struct vm_domainset_iter *di, int domain)
{
KASSERT(DOMAINSET_ISSET(domain, &di->di_valid_mask),
("%s: domain %d not present in di_valid_mask for di %p",
__func__, domain, di));
DOMAINSET_CLR(domain, &di->di_valid_mask);
}
#else /* !NUMA */
int
vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,
int *domain)
{
return (EJUSTRETURN);
}
void
vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,
vm_pindex_t pindex, int *domain, int *flags)
{
*domain = 0;
}
int
vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)
{
return (EJUSTRETURN);
}
void
vm_domainset_iter_policy_init(struct vm_domainset_iter *di,
struct domainset *ds, int *domain, int *flags)
{
*domain = 0;
}
void
vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,
struct domainset_ref *dr, int *domain, int *flags)
{
*domain = 0;
}
void
vm_domainset_iter_ignore(struct vm_domainset_iter *di __unused,
int domain __unused)
{
}
#endif /* NUMA */
Reference in New Issue View Git Blame Copy Permalink