dsa.h File Reference

#include "port/atomics.h"
#include "storage/dsm.h"

Include dependency graph for dsa.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros
#define	SIZEOF_DSA_POINTER   8
#define	dsa_pointer_atomic_init   pg_atomic_init_u64
#define	dsa_pointer_atomic_read   pg_atomic_read_u64
#define	dsa_pointer_atomic_write   pg_atomic_write_u64
#define	dsa_pointer_atomic_fetch_add   pg_atomic_fetch_add_u64
#define	dsa_pointer_atomic_compare_exchange   pg_atomic_compare_exchange_u64
#define	DSA_POINTER_FORMAT   "%016" PRIx64
#define	DSA_ALLOC_HUGE   0x01 /* allow huge allocation (> 1 GB) */
#define	DSA_ALLOC_NO_OOM   0x02 /* no failure if out-of-memory */
#define	DSA_ALLOC_ZERO   0x04 /* zero allocated memory */
#define	InvalidDsaPointer   ((dsa_pointer) 0)
#define	DSA_OFFSET_WIDTH   40 /* 1024 segments of size up to 1TB */
#define	DSA_DEFAULT_INIT_SEGMENT_SIZE   ((size_t) (1 * 1024 * 1024))
#define	DSA_MIN_SEGMENT_SIZE   ((size_t) (256 * 1024))
#define	DSA_MAX_SEGMENT_SIZE   ((size_t) 1 << DSA_OFFSET_WIDTH)
#define	DsaPointerIsValid(x)   ((x) != InvalidDsaPointer)
#define	dsa_allocate(area, size)    dsa_allocate_extended(area, size, 0)
#define	dsa_allocate0(area, size)    dsa_allocate_extended(area, size, DSA_ALLOC_ZERO)
#define	dsa_create(tranche_id)
#define	dsa_create_in_place(place, size, tranche_id, segment)
#define	DSA_HANDLE_INVALID   ((dsa_handle) DSM_HANDLE_INVALID)

Typedefs
typedef struct dsa_area	dsa_area
typedef uint64	dsa_pointer
typedef pg_atomic_uint64	dsa_pointer_atomic
typedef dsm_handle	dsa_handle

Functions
dsa_area *	dsa_create_ext (int tranche_id, size_t init_segment_size, size_t max_segment_size)
dsa_area *	dsa_create_in_place_ext (void *place, size_t size, int tranche_id, dsm_segment *segment, size_t init_segment_size, size_t max_segment_size)
dsa_area *	dsa_attach (dsa_handle handle)
bool	dsa_is_attached (dsa_handle handle)
dsa_area *	dsa_attach_in_place (void *place, dsm_segment *segment)
void	dsa_release_in_place (void *place)
void	dsa_on_dsm_detach_release_in_place (dsm_segment *, Datum)
void	dsa_on_shmem_exit_release_in_place (int, Datum)
void	dsa_pin_mapping (dsa_area *area)
void	dsa_detach (dsa_area *area)
void	dsa_pin (dsa_area *area)
void	dsa_unpin (dsa_area *area)
void	dsa_set_size_limit (dsa_area *area, size_t limit)
size_t	dsa_minimum_size (void)
dsa_handle	dsa_get_handle (dsa_area *area)
dsa_pointer	dsa_allocate_extended (dsa_area *area, size_t size, int flags)
void	dsa_free (dsa_area *area, dsa_pointer dp)
void *	dsa_get_address (dsa_area *area, dsa_pointer dp)
size_t	dsa_get_total_size (dsa_area *area)
void	dsa_trim (dsa_area *area)
void	dsa_dump (dsa_area *area)

Macro Definition Documentation

DSA_ALLOC_HUGE

#define DSA_ALLOC_HUGE   0x01 /* allow huge allocation (> 1 GB) */

Definition at line 73 of file dsa.h.

DSA_ALLOC_NO_OOM

#define DSA_ALLOC_NO_OOM   0x02 /* no failure if out-of-memory */

Definition at line 74 of file dsa.h.

DSA_ALLOC_ZERO

#define DSA_ALLOC_ZERO   0x04 /* zero allocated memory */

Definition at line 75 of file dsa.h.

dsa_allocate

#define dsa_allocate	(		area,
			size
	)		dsa_allocate_extended(area, size, 0)

Definition at line 109 of file dsa.h.

dsa_allocate0

#define dsa_allocate0	(		area,
			size
	)		dsa_allocate_extended(area, size, DSA_ALLOC_ZERO)

Definition at line 113 of file dsa.h.

dsa_create

#define dsa_create

(

tranche_id

)

Value:

    dsa_create_ext(tranche_id, DSA_DEFAULT_INIT_SEGMENT_SIZE, \
                   DSA_MAX_SEGMENT_SIZE)

Definition at line 117 of file dsa.h.

dsa_create_in_place

#define dsa_create_in_place	(		place,
			size,
			tranche_id,
			segment
	)

Value:

    dsa_create_in_place_ext(place, size, tranche_id, segment, \
                            DSA_DEFAULT_INIT_SEGMENT_SIZE, \
                            DSA_MAX_SEGMENT_SIZE)

Definition at line 122 of file dsa.h.

DSA_DEFAULT_INIT_SEGMENT_SIZE

#define DSA_DEFAULT_INIT_SEGMENT_SIZE   ((size_t) (1 * 1024 * 1024))

Definition at line 97 of file dsa.h.

DSA_HANDLE_INVALID

#define DSA_HANDLE_INVALID   ((dsa_handle) DSM_HANDLE_INVALID)

Definition at line 139 of file dsa.h.

DSA_MAX_SEGMENT_SIZE

#define DSA_MAX_SEGMENT_SIZE   ((size_t) 1 << DSA_OFFSET_WIDTH)

Definition at line 103 of file dsa.h.

DSA_MIN_SEGMENT_SIZE

#define DSA_MIN_SEGMENT_SIZE   ((size_t) (256 * 1024))

Definition at line 100 of file dsa.h.

DSA_OFFSET_WIDTH

#define DSA_OFFSET_WIDTH   40 /* 1024 segments of size up to 1TB */

Definition at line 88 of file dsa.h.

dsa_pointer_atomic_compare_exchange

#define dsa_pointer_atomic_compare_exchange   pg_atomic_compare_exchange_u64

Definition at line 68 of file dsa.h.

dsa_pointer_atomic_fetch_add

#define dsa_pointer_atomic_fetch_add   pg_atomic_fetch_add_u64

Definition at line 67 of file dsa.h.

dsa_pointer_atomic_init

#define dsa_pointer_atomic_init   pg_atomic_init_u64

Definition at line 64 of file dsa.h.

dsa_pointer_atomic_read

#define dsa_pointer_atomic_read   pg_atomic_read_u64

Definition at line 65 of file dsa.h.

dsa_pointer_atomic_write

#define dsa_pointer_atomic_write   pg_atomic_write_u64

Definition at line 66 of file dsa.h.

DSA_POINTER_FORMAT

#define DSA_POINTER_FORMAT   "%016" PRIx64

Definition at line 69 of file dsa.h.

DsaPointerIsValid

#define DsaPointerIsValid

(

x

)

((x) != InvalidDsaPointer)

Definition at line 106 of file dsa.h.

InvalidDsaPointer

#define InvalidDsaPointer   ((dsa_pointer) 0)

Definition at line 78 of file dsa.h.

SIZEOF_DSA_POINTER

#define SIZEOF_DSA_POINTER   8

Definition at line 42 of file dsa.h.

Typedef Documentation

dsa_area

typedef struct dsa_area dsa_area

Definition at line 22 of file dsa.h.

dsa_handle

typedef dsm_handle dsa_handle

Definition at line 136 of file dsa.h.

dsa_pointer

typedef uint64 dsa_pointer

Definition at line 62 of file dsa.h.

dsa_pointer_atomic

typedef pg_atomic_uint64 dsa_pointer_atomic

Definition at line 63 of file dsa.h.

Function Documentation

dsa_allocate_extended()

dsa_pointer dsa_allocate_extended	(	dsa_area *	area,
		size_t	size,
		int	flags
	)

Definition at line 686 of file dsa.c.

{
    uint16      size_class;
    dsa_pointer start_pointer;
    dsa_segment_map *segment_map;
    dsa_pointer result;
 
    Assert(size > 0);
 
    /* Sanity check on huge individual allocation size. */
    if (((flags & DSA_ALLOC_HUGE) != 0 && !AllocHugeSizeIsValid(size)) ||
        ((flags & DSA_ALLOC_HUGE) == 0 && !AllocSizeIsValid(size)))
        elog(ERROR, "invalid DSA memory alloc request size %zu", size);
 
    /*
     * If bigger than the largest size class, just grab a run of pages from
     * the free page manager, instead of allocating an object from a pool.
     * There will still be a span, but it's a special class of span that
     * manages this whole allocation and simply gives all pages back to the
     * free page manager when dsa_free is called.
     */
    if (size > dsa_size_classes[lengthof(dsa_size_classes) - 1])
    {
        size_t      npages = fpm_size_to_pages(size);
        size_t      first_page;
        dsa_pointer span_pointer;
        dsa_area_pool *pool = &area->control->pools[DSA_SCLASS_SPAN_LARGE];
 
        /* Obtain a span object. */
        span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
        if (!DsaPointerIsValid(span_pointer))
        {
            /* Raise error unless asked not to. */
            if ((flags & DSA_ALLOC_NO_OOM) == 0)
                ereport(ERROR,
                        (errcode(ERRCODE_OUT_OF_MEMORY),
                         errmsg("out of memory"),
                         errdetail("Failed on DSA request of size %zu.",
                                   size)));
            return InvalidDsaPointer;
        }
 
        LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
 
        /* Find a segment from which to allocate. */
        segment_map = get_best_segment(area, npages);
        if (segment_map == NULL)
            segment_map = make_new_segment(area, npages);
        if (segment_map == NULL)
        {
            /* Can't make any more segments: game over. */
            LWLockRelease(DSA_AREA_LOCK(area));
            dsa_free(area, span_pointer);
 
            /* Raise error unless asked not to. */
            if ((flags & DSA_ALLOC_NO_OOM) == 0)
                ereport(ERROR,
                        (errcode(ERRCODE_OUT_OF_MEMORY),
                         errmsg("out of memory"),
                         errdetail("Failed on DSA request of size %zu.",
                                   size)));
            return InvalidDsaPointer;
        }
 
        /*
         * Ask the free page manager for a run of pages.  This should always
         * succeed, since both get_best_segment and make_new_segment should
         * only return a non-NULL pointer if it actually contains enough
         * contiguous freespace.  If it does fail, something in our backend
         * private state is out of whack, so use FATAL to kill the process.
         */
        if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
            elog(FATAL,
                 "dsa_allocate could not find %zu free pages", npages);
        LWLockRelease(DSA_AREA_LOCK(area));
 
        start_pointer = DSA_MAKE_POINTER(get_segment_index(area, segment_map),
                                         first_page * FPM_PAGE_SIZE);
 
        /* Initialize span and pagemap. */
        LWLockAcquire(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE),
                      LW_EXCLUSIVE);
        init_span(area, span_pointer, pool, start_pointer, npages,
                  DSA_SCLASS_SPAN_LARGE);
        segment_map->pagemap[first_page] = span_pointer;
        LWLockRelease(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE));
 
        /* Zero-initialize the memory if requested. */
        if ((flags & DSA_ALLOC_ZERO) != 0)
            memset(dsa_get_address(area, start_pointer), 0, size);
 
        return start_pointer;
    }
 
    /* Map allocation to a size class. */
    if (size < lengthof(dsa_size_class_map) * DSA_SIZE_CLASS_MAP_QUANTUM)
    {
        int         mapidx;
 
        /* For smaller sizes we have a lookup table... */
        mapidx = ((size + DSA_SIZE_CLASS_MAP_QUANTUM - 1) /
                  DSA_SIZE_CLASS_MAP_QUANTUM) - 1;
        size_class = dsa_size_class_map[mapidx];
    }
    else
    {
        uint16      min;
        uint16      max;
 
        /* ... and for the rest we search by binary chop. */
        min = dsa_size_class_map[lengthof(dsa_size_class_map) - 1];
        max = lengthof(dsa_size_classes) - 1;
 
        while (min < max)
        {
            uint16      mid = (min + max) / 2;
            uint16      class_size = dsa_size_classes[mid];
 
            if (class_size < size)
                min = mid + 1;
            else
                max = mid;
        }
 
        size_class = min;
    }
    Assert(size <= dsa_size_classes[size_class]);
    Assert(size_class == 0 || size > dsa_size_classes[size_class - 1]);
 
    /* Attempt to allocate an object from the appropriate pool. */
    result = alloc_object(area, size_class);
 
    /* Check for failure to allocate. */
    if (!DsaPointerIsValid(result))
    {
        /* Raise error unless asked not to. */
        if ((flags & DSA_ALLOC_NO_OOM) == 0)
            ereport(ERROR,
                    (errcode(ERRCODE_OUT_OF_MEMORY),
                     errmsg("out of memory"),
                     errdetail("Failed on DSA request of size %zu.", size)));
        return InvalidDsaPointer;
    }
 
    /* Zero-initialize the memory if requested. */
    if ((flags & DSA_ALLOC_ZERO) != 0)
        memset(dsa_get_address(area, result), 0, size);
 
    return result;
}

References alloc_object(), AllocHugeSizeIsValid, AllocSizeIsValid, Assert(), dsa_area::control, DSA_ALLOC_HUGE, DSA_ALLOC_NO_OOM, DSA_ALLOC_ZERO, DSA_AREA_LOCK, dsa_free(), dsa_get_address(), DSA_MAKE_POINTER, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_class_map, DSA_SIZE_CLASS_MAP_QUANTUM, dsa_size_classes, DsaPointerIsValid, elog, ereport, errcode(), errdetail(), errmsg(), ERROR, FATAL, dsa_segment_map::fpm, FPM_PAGE_SIZE, fpm_size_to_pages, FreePageManagerGet(), get_best_segment(), get_segment_index, init_span(), InvalidDsaPointer, lengthof, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), make_new_segment(), dsa_segment_map::pagemap, and dsa_area_control::pools.

Referenced by dshash_create(), pagetable_allocate(), pgstat_init_entry(), and resize().

dsa_attach()

dsa_area * dsa_attach

(

dsa_handle

handle

)

Definition at line 510 of file dsa.c.

{
    dsm_segment *segment;
    dsa_area   *area;
 
    /*
     * An area handle is really a DSM segment handle for the first segment, so
     * we go ahead and attach to that.
     */
    segment = dsm_attach(handle);
    if (segment == NULL)
        ereport(ERROR,
                (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                 errmsg("could not attach to dynamic shared area")));
 
    area = attach_internal(dsm_segment_address(segment), segment, handle);
 
    /* Clean up when the control segment detaches. */
    on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                  PointerGetDatum(dsm_segment_address(segment)));
 
    return area;
}

References attach_internal(), dsa_on_dsm_detach_release_in_place(), dsm_attach(), dsm_segment_address(), ereport, errcode(), errmsg(), ERROR, on_dsm_detach(), and PointerGetDatum().

Referenced by GetNamedDSA(), GetNamedDSHash(), init_dsm_registry(), logicalrep_launcher_attach_dshmem(), and TidStoreAttach().

dsa_attach_in_place()

dsa_area * dsa_attach_in_place	(	void *	place,
		dsm_segment *	segment
	)

Definition at line 560 of file dsa.c.

{
    dsa_area   *area;
 
    area = attach_internal(place, NULL, DSA_HANDLE_INVALID);
 
    /*
     * Clean up when the control segment detaches, if a containing DSM segment
     * was provided.
     */
    if (segment != NULL)
        on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                      PointerGetDatum(place));
 
    return area;
}

References attach_internal(), DSA_HANDLE_INVALID, dsa_on_dsm_detach_release_in_place(), on_dsm_detach(), and PointerGetDatum().

Referenced by AttachSession(), ParallelQueryMain(), and pgstat_attach_shmem().

dsa_create_ext()

dsa_area * dsa_create_ext	(	int	tranche_id,
		size_t	init_segment_size,
		size_t	max_segment_size
	)

Definition at line 421 of file dsa.c.

{
    dsm_segment *segment;
    dsa_area   *area;
 
    /*
     * Create the DSM segment that will hold the shared control object and the
     * first segment of usable space.
     */
    segment = dsm_create(init_segment_size, 0);
 
    /*
     * All segments backing this area are pinned, so that DSA can explicitly
     * control their lifetime (otherwise a newly created segment belonging to
     * this area might be freed when the only backend that happens to have it
     * mapped in ends, corrupting the area).
     */
    dsm_pin_segment(segment);
 
    /* Create a new DSA area with the control object in this segment. */
    area = create_internal(dsm_segment_address(segment),
                           init_segment_size,
                           tranche_id,
                           dsm_segment_handle(segment), segment,
                           init_segment_size, max_segment_size);
 
    /* Clean up when the control segment detaches. */
    on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                  PointerGetDatum(dsm_segment_address(segment)));
 
    return area;
}

References create_internal(), dsa_on_dsm_detach_release_in_place(), dsm_create(), dsm_pin_segment(), dsm_segment_address(), dsm_segment_handle(), on_dsm_detach(), and PointerGetDatum().

Referenced by TidStoreCreateShared().

dsa_create_in_place_ext()

dsa_area * dsa_create_in_place_ext	(	void *	place,
		size_t	size,
		int	tranche_id,
		dsm_segment *	segment,
		size_t	init_segment_size,
		size_t	max_segment_size
	)

Definition at line 471 of file dsa.c.

{
    dsa_area   *area;
 
    area = create_internal(place, size, tranche_id,
                           DSM_HANDLE_INVALID, NULL,
                           init_segment_size, max_segment_size);
 
    /*
     * Clean up when the control segment detaches, if a containing DSM segment
     * was provided.
     */
    if (segment != NULL)
        on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
                      PointerGetDatum(place));
 
    return area;
}

References create_internal(), dsa_on_dsm_detach_release_in_place(), DSM_HANDLE_INVALID, on_dsm_detach(), and PointerGetDatum().

dsa_detach()

void dsa_detach

(

dsa_area *

area

)

Definition at line 1967 of file dsa.c.

{
    int         i;
 
    /* Detach from all segments. */
    for (i = 0; i <= area->high_segment_index; ++i)
        if (area->segment_maps[i].segment != NULL)
            dsm_detach(area->segment_maps[i].segment);
 
    /*
     * Note that 'detaching' (= detaching from DSM segments) doesn't include
     * 'releasing' (= adjusting the reference count).  It would be nice to
     * combine these operations, but client code might never get around to
     * calling dsa_detach because of an error path, and a detach hook on any
     * particular segment is too late to detach other segments in the area
     * without risking a 'leak' warning in the non-error path.
     */
 
    /* Free the backend-local area object. */
    pfree(area);
}

References dsm_detach(), dsa_area::high_segment_index, i, pfree(), dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by DetachSession(), ExecParallelCleanup(), ParallelQueryMain(), pgstat_detach_shmem(), StatsShmemInit(), test_basic(), test_dsa_basic(), test_dsa_resowners(), test_empty(), test_random(), TidStoreDestroy(), and TidStoreDetach().

dsa_dump()

void dsa_dump

(

dsa_area *

area

)

Definition at line 1103 of file dsa.c.

{
    size_t      i,
                j;
 
    /*
     * Note: This gives an inconsistent snapshot as it acquires and releases
     * individual locks as it goes...
     */
 
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    check_for_freed_segments_locked(area);
    fprintf(stderr, "dsa_area handle %x:\n", area->control->handle);
    fprintf(stderr, "  max_total_segment_size: %zu\n",
            area->control->max_total_segment_size);
    fprintf(stderr, "  total_segment_size: %zu\n",
            area->control->total_segment_size);
    fprintf(stderr, "  refcnt: %d\n", area->control->refcnt);
    fprintf(stderr, "  pinned: %c\n", area->control->pinned ? 't' : 'f');
    fprintf(stderr, "  segment bins:\n");
    for (i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
    {
        if (area->control->segment_bins[i] != DSA_SEGMENT_INDEX_NONE)
        {
            dsa_segment_index segment_index;
 
            if (i == 0)
                fprintf(stderr,
                        "    segment bin %zu (no contiguous free pages):\n", i);
            else
                fprintf(stderr,
                        "    segment bin %zu (at least %d contiguous pages free):\n",
                        i, 1 << (i - 1));
            segment_index = area->control->segment_bins[i];
            while (segment_index != DSA_SEGMENT_INDEX_NONE)
            {
                dsa_segment_map *segment_map;
 
                segment_map =
                    get_segment_by_index(area, segment_index);
 
                fprintf(stderr,
                        "      segment index %zu, usable_pages = %zu, "
                        "contiguous_pages = %zu, mapped at %p\n",
                        segment_index,
                        segment_map->header->usable_pages,
                        fpm_largest(segment_map->fpm),
                        segment_map->mapped_address);
                segment_index = segment_map->header->next;
            }
        }
    }
    LWLockRelease(DSA_AREA_LOCK(area));
 
    fprintf(stderr, "  pools:\n");
    for (i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
    {
        bool        found = false;
 
        LWLockAcquire(DSA_SCLASS_LOCK(area, i), LW_EXCLUSIVE);
        for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
            if (DsaPointerIsValid(area->control->pools[i].spans[j]))
                found = true;
        if (found)
        {
            if (i == DSA_SCLASS_BLOCK_OF_SPANS)
                fprintf(stderr, "    pool for blocks of span objects:\n");
            else if (i == DSA_SCLASS_SPAN_LARGE)
                fprintf(stderr, "    pool for large object spans:\n");
            else
                fprintf(stderr,
                        "    pool for size class %zu (object size %hu bytes):\n",
                        i, dsa_size_classes[i]);
            for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
            {
                if (!DsaPointerIsValid(area->control->pools[i].spans[j]))
                    fprintf(stderr, "      fullness class %zu is empty\n", j);
                else
                {
                    dsa_pointer span_pointer = area->control->pools[i].spans[j];
 
                    fprintf(stderr, "      fullness class %zu:\n", j);
                    while (DsaPointerIsValid(span_pointer))
                    {
                        dsa_area_span *span;
 
                        span = dsa_get_address(area, span_pointer);
                        fprintf(stderr,
                                "        span descriptor at "
                                DSA_POINTER_FORMAT ", superblock at "
                                DSA_POINTER_FORMAT
                                ", pages = %zu, objects free = %hu/%hu\n",
                                span_pointer, span->start, span->npages,
                                span->nallocatable, span->nmax);
                        span_pointer = span->nextspan;
                    }
                }
            }
        }
        LWLockRelease(DSA_SCLASS_LOCK(area, i));
    }
}

References check_for_freed_segments_locked(), dsa_area::control, DSA_AREA_LOCK, DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_NUM_SEGMENT_BINS, DSA_NUM_SIZE_CLASSES, DSA_POINTER_FORMAT, DSA_SCLASS_BLOCK_OF_SPANS, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DSA_SEGMENT_INDEX_NONE, dsa_size_classes, DsaPointerIsValid, dsa_segment_map::fpm, fpm_largest, fprintf, get_segment_by_index(), dsa_area_control::handle, dsa_segment_map::header, i, j, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_map::mapped_address, dsa_area_control::max_total_segment_size, dsa_area_span::nallocatable, dsa_segment_header::next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_span::npages, dsa_area_control::pinned, dsa_area_control::pools, dsa_area_control::refcnt, dsa_area_control::segment_bins, dsa_area_pool::spans, dsa_area_span::start, dsa_area_control::total_segment_size, and dsa_segment_header::usable_pages.

dsa_free()

void dsa_free	(	dsa_area *	area,
		dsa_pointer	dp
	)

Definition at line 841 of file dsa.c.

{
    dsa_segment_map *segment_map;
    int         pageno;
    dsa_pointer span_pointer;
    dsa_area_span *span;
    char       *superblock;
    char       *object;
    size_t      size;
    int         size_class;
 
    /* Make sure we don't have a stale segment in the slot 'dp' refers to. */
    check_for_freed_segments(area);
 
    /* Locate the object, span and pool. */
    segment_map = get_segment_by_index(area, DSA_EXTRACT_SEGMENT_NUMBER(dp));
    pageno = DSA_EXTRACT_OFFSET(dp) / FPM_PAGE_SIZE;
    span_pointer = segment_map->pagemap[pageno];
    span = dsa_get_address(area, span_pointer);
    superblock = dsa_get_address(area, span->start);
    object = dsa_get_address(area, dp);
    size_class = span->size_class;
    size = dsa_size_classes[size_class];
 
    /*
     * Special case for large objects that live in a special span: we return
     * those pages directly to the free page manager and free the span.
     */
    if (span->size_class == DSA_SCLASS_SPAN_LARGE)
    {
 
#ifdef CLOBBER_FREED_MEMORY
        memset(object, 0x7f, span->npages * FPM_PAGE_SIZE);
#endif
 
        /* Give pages back to free page manager. */
        LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
        FreePageManagerPut(segment_map->fpm,
                           DSA_EXTRACT_OFFSET(span->start) / FPM_PAGE_SIZE,
                           span->npages);
 
        /* Move segment to appropriate bin if necessary. */
        rebin_segment(area, segment_map);
        LWLockRelease(DSA_AREA_LOCK(area));
 
        /* Unlink span. */
        LWLockAcquire(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE),
                      LW_EXCLUSIVE);
        unlink_span(area, span);
        LWLockRelease(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE));
        /* Free the span object so it can be reused. */
        dsa_free(area, span_pointer);
        return;
    }
 
#ifdef CLOBBER_FREED_MEMORY
    memset(object, 0x7f, size);
#endif
 
    LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
 
    /* Put the object on the span's freelist. */
    Assert(object >= superblock);
    Assert(object < superblock + DSA_SUPERBLOCK_SIZE);
    Assert((object - superblock) % size == 0);
    NextFreeObjectIndex(object) = span->firstfree;
    span->firstfree = (object - superblock) / size;
    ++span->nallocatable;
 
    /*
     * See if the span needs to moved to a different fullness class, or be
     * freed so its pages can be given back to the segment.
     */
    if (span->nallocatable == 1 && span->fclass == DSA_FULLNESS_CLASSES - 1)
    {
        /*
         * The block was completely full and is located in the
         * highest-numbered fullness class, which is never scanned for free
         * chunks.  We must move it to the next-lower fullness class.
         */
        unlink_span(area, span);
        add_span_to_fullness_class(area, span, span_pointer,
                                   DSA_FULLNESS_CLASSES - 2);
 
        /*
         * If this is the only span, and there is no active span, then we
         * should probably move this span to fullness class 1.  (Otherwise if
         * you allocate exactly all the objects in the only span, it moves to
         * class 3, then you free them all, it moves to 2, and then is given
         * back, leaving no active span).
         */
    }
    else if (span->nallocatable == span->nmax &&
             (span->fclass != 1 || span->prevspan != InvalidDsaPointer))
    {
        /*
         * This entire block is free, and it's not the active block for this
         * size class.  Return the memory to the free page manager. We don't
         * do this for the active block to prevent hysteresis: if we
         * repeatedly allocate and free the only chunk in the active block, it
         * will be very inefficient if we deallocate and reallocate the block
         * every time.
         */
        destroy_superblock(area, span_pointer);
    }
 
    LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
}

References add_span_to_fullness_class(), Assert(), check_for_freed_segments(), destroy_superblock(), DSA_AREA_LOCK, DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, dsa_free(), DSA_FULLNESS_CLASSES, dsa_get_address(), DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, dsa_size_classes, DSA_SUPERBLOCK_SIZE, dsa_area_span::fclass, dsa_area_span::firstfree, dsa_segment_map::fpm, FPM_PAGE_SIZE, FreePageManagerPut(), get_segment_by_index(), InvalidDsaPointer, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, NextFreeObjectIndex, dsa_area_span::nmax, dsa_area_span::npages, dsa_segment_map::pagemap, dsa_area_span::prevspan, rebin_segment(), dsa_area_span::size_class, dsa_area_span::start, and unlink_span().

Referenced by delete_item_from_bucket(), delete_key_from_bucket(), destroy_superblock(), dsa_allocate_extended(), dsa_free(), dshash_create(), dshash_destroy(), ExecHashTableDetach(), ExecHashTableDetachBatch(), ExecParallelCleanup(), ExecParallelHashIncreaseNumBatches(), ExecParallelHashIncreaseNumBuckets(), ExecParallelHashRepartitionFirst(), ExecParallelReinitialize(), find_or_make_matching_shared_tupledesc(), pagetable_free(), pgstat_free_entry(), resize(), RT_FREE(), RT_FREE_LEAF(), RT_FREE_NODE(), set_val_in_hash(), tbm_free_shared_area(), test_dsa_basic(), and test_dsa_resowners().

dsa_get_address()

void * dsa_get_address	(	dsa_area *	area,
		dsa_pointer	dp
	)

Definition at line 957 of file dsa.c.

{
    dsa_segment_index index;
    size_t      offset;
 
    /* Convert InvalidDsaPointer to NULL. */
    if (!DsaPointerIsValid(dp))
        return NULL;
 
    /* Process any requests to detach from freed segments. */
    check_for_freed_segments(area);
 
    /* Break the dsa_pointer into its components. */
    index = DSA_EXTRACT_SEGMENT_NUMBER(dp);
    offset = DSA_EXTRACT_OFFSET(dp);
    Assert(index < DSA_MAX_SEGMENTS);
 
    /* Check if we need to cause this segment to be mapped in. */
    if (unlikely(area->segment_maps[index].mapped_address == NULL))
    {
        /* Call for effect (we don't need the result). */
        get_segment_by_index(area, index);
    }
 
    return area->segment_maps[index].mapped_address + offset;
}

References Assert(), check_for_freed_segments(), DSA_EXTRACT_OFFSET, DSA_EXTRACT_SEGMENT_NUMBER, DSA_MAX_SEGMENTS, DsaPointerIsValid, get_segment_by_index(), dsa_segment_map::mapped_address, dsa_area::segment_maps, and unlikely.

Referenced by add_span_to_fullness_class(), alloc_object(), delete_item_from_bucket(), delete_key_from_bucket(), destroy_superblock(), dsa_allocate_extended(), dsa_dump(), dsa_free(), dsa_trim(), dshash_attach(), dshash_create(), dshash_destroy(), dshash_dump(), dshash_seq_next(), ensure_active_superblock(), ensure_valid_bucket_pointers(), ExecHashTableDetachBatch(), ExecParallelHashEnsureBatchAccessors(), ExecParallelHashFirstTuple(), ExecParallelHashIncreaseNumBatches(), ExecParallelHashIncreaseNumBuckets(), ExecParallelHashJoinSetUpBatches(), ExecParallelHashNextTuple(), ExecParallelHashPopChunkQueue(), ExecParallelHashRepartitionRest(), ExecParallelHashTableAlloc(), ExecParallelHashTableSetCurrentBatch(), ExecParallelHashTupleAlloc(), find_in_bucket(), find_or_make_matching_shared_tupledesc(), get_val_in_hash(), init_span(), insert_into_bucket(), insert_item_into_bucket(), lookup_rowtype_tupdesc_internal(), pagetable_allocate(), ParallelQueryMain(), pgstat_build_snapshot(), pgstat_get_entry_ref(), pgstat_init_entry(), pgstat_reinit_entry(), pgstat_reset_matching_entries(), pgstat_write_statsfile(), resize(), RT_PTR_SET_LOCAL(), SerializeParamExecParams(), set_val_in_hash(), share_tupledesc(), shared_record_table_compare(), shared_record_table_hash(), tbm_attach_shared_iterate(), tbm_free_shared_area(), tbm_prepare_shared_iterate(), test_dsa_basic(), test_dsa_resowners(), transfer_first_span(), and unlink_span().

dsa_get_handle()

dsa_handle dsa_get_handle

(

dsa_area *

area

)

Definition at line 498 of file dsa.c.

{
    Assert(area->control->handle != DSA_HANDLE_INVALID);
    return area->control->handle;
}

References Assert(), dsa_area::control, DSA_HANDLE_INVALID, and dsa_area_control::handle.

Referenced by GetNamedDSA(), GetNamedDSHash(), init_dsm_registry(), logicalrep_launcher_attach_dshmem(), parallel_vacuum_init(), and parallel_vacuum_reset_dead_items().

dsa_get_total_size()

size_t dsa_get_total_size

(

dsa_area *

area

)

Definition at line 1042 of file dsa.c.

{
    size_t      size;
 
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    size = area->control->total_segment_size;
    LWLockRelease(DSA_AREA_LOCK(area));
 
    return size;
}

References dsa_area::control, DSA_AREA_LOCK, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), and dsa_area_control::total_segment_size.

Referenced by RT_MEMORY_USAGE().

dsa_is_attached()

bool dsa_is_attached

(

dsa_handle

handle

)

Definition at line 540 of file dsa.c.

{
    /*
     * An area handle is really a DSM segment handle for the first segment, so
     * we can just search for that.
     */
    return dsm_find_mapping(handle) != NULL;
}

References dsm_find_mapping().

Referenced by GetNamedDSA(), and GetNamedDSHash().

dsa_minimum_size()

size_t dsa_minimum_size

(

void

)

Definition at line 1211 of file dsa.c.

{
    size_t      size;
    int         pages = 0;
 
    size = MAXALIGN(sizeof(dsa_area_control)) +
        MAXALIGN(sizeof(FreePageManager));
 
    /* Figure out how many pages we need, including the page map... */
    while (((size + FPM_PAGE_SIZE - 1) / FPM_PAGE_SIZE) > pages)
    {
        ++pages;
        size += sizeof(dsa_pointer);
    }
 
    return pages * FPM_PAGE_SIZE;
}

References FPM_PAGE_SIZE, and MAXALIGN.

Referenced by create_internal(), ExecInitParallelPlan(), and pgstat_dsa_init_size().

dsa_on_dsm_detach_release_in_place()

void dsa_on_dsm_detach_release_in_place	(	dsm_segment *	segment,
		Datum	place
	)

Definition at line 591 of file dsa.c.

{
    dsa_release_in_place(DatumGetPointer(place));
}

References DatumGetPointer(), and dsa_release_in_place().

Referenced by dsa_attach(), dsa_attach_in_place(), dsa_create_ext(), and dsa_create_in_place_ext().

dsa_on_shmem_exit_release_in_place()

void dsa_on_shmem_exit_release_in_place	(	int	code,
		Datum	place
	)

Definition at line 605 of file dsa.c.

{
    dsa_release_in_place(DatumGetPointer(place));
}

References DatumGetPointer(), and dsa_release_in_place().

dsa_pin()

void dsa_pin

(

dsa_area *

area

)

Definition at line 990 of file dsa.c.

{
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    if (area->control->pinned)
    {
        LWLockRelease(DSA_AREA_LOCK(area));
        elog(ERROR, "dsa_area already pinned");
    }
    area->control->pinned = true;
    ++area->control->refcnt;
    LWLockRelease(DSA_AREA_LOCK(area));
}

References dsa_area::control, DSA_AREA_LOCK, elog, ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.

Referenced by GetNamedDSA(), GetNamedDSHash(), init_dsm_registry(), logicalrep_launcher_attach_dshmem(), and StatsShmemInit().

dsa_pin_mapping()

void dsa_pin_mapping

(

dsa_area *

area

)

Definition at line 650 of file dsa.c.

{
    int         i;
 
    if (area->resowner != NULL)
    {
        area->resowner = NULL;
 
        for (i = 0; i <= area->high_segment_index; ++i)
            if (area->segment_maps[i].segment != NULL)
                dsm_pin_mapping(area->segment_maps[i].segment);
    }
}

References dsm_pin_mapping(), dsa_area::high_segment_index, i, dsa_area::resowner, dsa_segment_map::segment, and dsa_area::segment_maps.

Referenced by AttachSession(), GetNamedDSA(), GetNamedDSHash(), GetSessionDsmHandle(), init_dsm_registry(), logicalrep_launcher_attach_dshmem(), pgstat_attach_shmem(), and test_create().

dsa_release_in_place()

void dsa_release_in_place

(

void *

place

)

Definition at line 620 of file dsa.c.

{
    dsa_area_control *control = (dsa_area_control *) place;
    int         i;
 
    LWLockAcquire(&control->lock, LW_EXCLUSIVE);
    Assert(control->segment_header.magic ==
           (DSA_SEGMENT_HEADER_MAGIC ^ control->handle ^ 0));
    Assert(control->refcnt > 0);
    if (--control->refcnt == 0)
    {
        for (i = 0; i <= control->high_segment_index; ++i)
        {
            dsm_handle  handle;
 
            handle = control->segment_handles[i];
            if (handle != DSM_HANDLE_INVALID)
                dsm_unpin_segment(handle);
        }
    }
    LWLockRelease(&control->lock);
}

References Assert(), DSA_SEGMENT_HEADER_MAGIC, DSM_HANDLE_INVALID, dsm_unpin_segment(), dsa_area_control::handle, dsa_area_control::high_segment_index, i, dsa_area_control::lock, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_segment_header::magic, dsa_area_control::refcnt, dsa_area_control::segment_handles, and dsa_area_control::segment_header.

Referenced by dsa_on_dsm_detach_release_in_place(), dsa_on_shmem_exit_release_in_place(), and pgstat_detach_shmem().

dsa_set_size_limit()

void dsa_set_size_limit	(	dsa_area *	area,
		size_t	limit
	)

Definition at line 1033 of file dsa.c.

{
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    area->control->max_total_segment_size = limit;
    LWLockRelease(DSA_AREA_LOCK(area));
}

References dsa_area::control, DSA_AREA_LOCK, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), and dsa_area_control::max_total_segment_size.

Referenced by StatsShmemInit().

dsa_trim()

void dsa_trim

(

dsa_area *

area

)

Definition at line 1058 of file dsa.c.

{
    int         size_class;
 
    /*
     * Trim in reverse pool order so we get to the spans-of-spans last, just
     * in case any become entirely free while processing all the other pools.
     */
    for (size_class = DSA_NUM_SIZE_CLASSES - 1; size_class >= 0; --size_class)
    {
        dsa_area_pool *pool = &area->control->pools[size_class];
        dsa_pointer span_pointer;
 
        if (size_class == DSA_SCLASS_SPAN_LARGE)
        {
            /* Large object frees give back segments aggressively already. */
            continue;
        }
 
        /*
         * Search fullness class 1 only.  That is where we expect to find an
         * entirely empty superblock (entirely empty superblocks in other
         * fullness classes are returned to the free page map by dsa_free).
         */
        LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
        span_pointer = pool->spans[1];
        while (DsaPointerIsValid(span_pointer))
        {
            dsa_area_span *span = dsa_get_address(area, span_pointer);
            dsa_pointer next = span->nextspan;
 
            if (span->nallocatable == span->nmax)
                destroy_superblock(area, span_pointer);
 
            span_pointer = next;
        }
        LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
    }
}

References dsa_area::control, destroy_superblock(), dsa_get_address(), DSA_NUM_SIZE_CLASSES, DSA_SCLASS_LOCK, DSA_SCLASS_SPAN_LARGE, DsaPointerIsValid, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_span::nallocatable, next, dsa_area_span::nextspan, dsa_area_span::nmax, dsa_area_control::pools, and dsa_area_pool::spans.

dsa_unpin()

void dsa_unpin

(

dsa_area *

area

)

Definition at line 1009 of file dsa.c.

{
    LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    Assert(area->control->refcnt > 1);
    if (!area->control->pinned)
    {
        LWLockRelease(DSA_AREA_LOCK(area));
        elog(ERROR, "dsa_area not pinned");
    }
    area->control->pinned = false;
    --area->control->refcnt;
    LWLockRelease(DSA_AREA_LOCK(area));
}

References Assert(), dsa_area::control, DSA_AREA_LOCK, elog, ERROR, LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), dsa_area_control::pinned, and dsa_area_control::refcnt.