diff --git a/app/boards/intel_adsp_ace15_mtpm.conf b/app/boards/intel_adsp_ace15_mtpm.conf index e95b18d97169..4f124eee5285 100644 --- a/app/boards/intel_adsp_ace15_mtpm.conf +++ b/app/boards/intel_adsp_ace15_mtpm.conf @@ -71,7 +71,7 @@ CONFIG_DMA_DW_LLI_POOL_SIZE=50 CONFIG_DMA_INTEL_ADSP_GPDMA=y CONFIG_MEMORY_WIN_2_SIZE=12288 CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=38400000 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 diff --git a/app/boards/intel_adsp_ace20_lnl.conf b/app/boards/intel_adsp_ace20_lnl.conf index beb441d6e6c6..2aef09fb9a4a 100644 --- a/app/boards/intel_adsp_ace20_lnl.conf +++ b/app/boards/intel_adsp_ace20_lnl.conf @@ -49,7 +49,7 @@ CONFIG_DAI_INIT_PRIORITY=70 CONFIG_DMA_INTEL_ADSP_GPDMA=n CONFIG_MEMORY_WIN_2_SIZE=12288 CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=38400000 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index 12aa78162c06..a79ca6e67764 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -53,7 +53,7 @@ CONFIG_DMA_INTEL_ADSP_GPDMA=n CONFIG_DMA_DW_LLI_POOL_SIZE=50 CONFIG_MEMORY_WIN_2_SIZE=12288 CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_XTENSA_MMU_NUM_L2_TABLES=128 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 diff --git a/app/boards/intel_adsp_ace30_wcl.conf b/app/boards/intel_adsp_ace30_wcl.conf index 621eb719de9f..999a3c309d41 100644 --- a/app/boards/intel_adsp_ace30_wcl.conf +++ b/app/boards/intel_adsp_ace30_wcl.conf @@ -50,7 +50,7 @@ CONFIG_DAI_DMIC_HAS_MULTIPLE_LINE_SYNC=y CONFIG_DMA_INTEL_ADSP_GPDMA=n CONFIG_MEMORY_WIN_2_SIZE=12288 CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 # Zephyr / power settings diff --git a/app/boards/intel_adsp_ace40_nvl.conf b/app/boards/intel_adsp_ace40_nvl.conf index 0f60dc5569b1..becef1a65f10 100644 --- a/app/boards/intel_adsp_ace40_nvl.conf +++ b/app/boards/intel_adsp_ace40_nvl.conf @@ -42,7 +42,7 @@ CONFIG_DAI_DMIC_HAS_OWNERSHIP=n CONFIG_DAI_DMIC_HAS_MULTIPLE_LINE_SYNC=y CONFIG_DMA_INTEL_ADSP_GPDMA=n CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 # Zephyr / power settings diff --git a/app/boards/intel_adsp_ace40_nvls.conf b/app/boards/intel_adsp_ace40_nvls.conf index 0f60dc5569b1..becef1a65f10 100644 --- a/app/boards/intel_adsp_ace40_nvls.conf +++ b/app/boards/intel_adsp_ace40_nvls.conf @@ -42,7 +42,7 @@ CONFIG_DAI_DMIC_HAS_OWNERSHIP=n CONFIG_DAI_DMIC_HAS_MULTIPLE_LINE_SYNC=y CONFIG_DMA_INTEL_ADSP_GPDMA=n CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 # Zephyr / power settings diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 233c93f3e070..f3e8ef740116 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -289,8 +289,12 @@ if (CONFIG_SOC_SERIES_INTEL_ADSP_ACE) ${SOF_PLATFORM_PATH}/novalake/lib/clk.c ) - # Sources for virtual heap management - zephyr_library_sources( + zephyr_library_sources_ifdef(CONFIG_SOF_VREGIONS + lib/vpages.c + lib/vregion.c + ) + + zephyr_library_sources_ifdef(CONFIG_VIRTUAL_HEAP lib/regions_mm.c ) diff --git a/zephyr/Kconfig b/zephyr/Kconfig index c744b965a07f..e25789748dfe 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -93,10 +93,18 @@ config SOF_ZEPHYR_VIRTUAL_HEAP_SIZE config SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE hex "Size in bytes of virtual memory region for virtual heap shared for all cores" depends on MM_DRV_INTEL_ADSP_MTL_TLB - default 0x100000 + default 0xc0000 help This config defines size of virtual heap region shared between all cores +config SOF_ZEPHYR_VIRTUAL_PAGE_REGION_SIZE + hex "Size in bytes of virtual memory regions" + depends on MM_DRV_INTEL_ADSP_MTL_TLB + default 0x40000 + help + This config defines size of the memory region, available to the + vregion / vpage API + config SOF_USERSPACE_USE_DRIVER_HEAP bool "Use driver heap for SOF userspace modules" depends on USERSPACE @@ -145,6 +153,19 @@ config SOF_USERSPACE_APPLICATION Not manually settable. This is effectively a shortcut to replace numerous checks for (CONFIG_USERSPACE && !CONFIG_SOF_USERSPACE_PROXY) +config SOF_VPAGE_MAX_ALLOCS + int "Number of virtual memory page allocations" + default 128 + help + This setting defines the maximum number of virtual memory page + allocations that can be tracked. Each allocation represents a + contiguous block of virtual memory allocated from the virtual memory + region. This API isn't intended for end-user allocations, instead it + should be used by the framework to allocate memory, which is then + used, e.g. to create one or multiple heap allocators in it. Increasing + this number allows for more simultaneous allocations, but also + increases the memory overhead for tracking these allocations. + config ZEPHYR_NATIVE_DRIVERS bool "Use Zephyr native drivers" help @@ -237,8 +258,19 @@ config SOF_ZEPHYR_NO_SOF_CLOCK Do not use SOF clk.h interface to set the DSP clock frequency. Requires implementation of platform/lib/clk.h. +config SOF_VREGIONS + bool "Enable virtual memory regions" + default y if ACE + depends on ACE + help + Enable the virtual regions memory allocator for pipeline resource management. + This provides a way to manage memory resources for audio pipelines, + including + 1) multiple pipeline static lifetime allocations. + 2) runtime pipeline allocations. + config VIRTUAL_HEAP - bool "Use virtual memory heap to allocate a buffers" + bool "Use virtual memory heap to allocate buffers" default y if ACE depends on ACE help diff --git a/zephyr/include/sof/lib/regions_mm.h b/zephyr/include/sof/lib/regions_mm.h index 9e960e304b65..abaaf158d3d4 100644 --- a/zephyr/include/sof/lib/regions_mm.h +++ b/zephyr/include/sof/lib/regions_mm.h @@ -18,8 +18,9 @@ #include /* Attributes for memory regions */ -#define VIRTUAL_REGION_SHARED_HEAP_ATTR 1U /*< region dedicated for shared virtual heap */ -#define VIRTUAL_REGION_LLEXT_LIBRARIES_ATTR 2U /*< region dedicated for LLEXT libraries */ +#define VIRTUAL_REGION_SHARED_HEAP_ATTR 1U /*< region for shared virtual heap */ +#define VIRTUAL_REGION_LLEXT_LIBRARIES_ATTR 2U /*< region for LLEXT libraries */ +#define VIRTUAL_REGION_VPAGES_ATTR 3U /*< region for virtual page allocator */ /* Dependency on ipc/topology.h created due to memory capability definitions * that are defined there diff --git a/zephyr/include/sof/lib/vpages.h b/zephyr/include/sof/lib/vpages.h new file mode 100644 index 000000000000..09225db0e5f2 --- /dev/null +++ b/zephyr/include/sof/lib/vpages.h @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright(c) 2025 Intel Corporation. + +/* Virtual Page Allocator API */ +#ifndef __SOF_LIB_VPAGE_H__ +#define __SOF_LIB_VPAGE_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Allocate virtual pages + * Allocates a specified number of contiguous virtual memory pages by mapping + * physical pages. + * + * @param[in] pages Number of 4kB pages to allocate. + * + * @return Pointer to the allocated virtual memory region, or NULL on failure. + */ +void *vpage_alloc(unsigned int pages); + +/** + * @brief Free virtual pages + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param[in] ptr Pointer to the memory pages to free. + */ +void vpage_free(void *ptr); + +/** + * @brief Initialize virtual page allocator + * + * Initializes a virtual page allocator that manages a virtual memory region + * using a page table and block structures. + * + * @retval 0 if successful. + * @retval -ENOMEM on creation failure. + */ +int vpage_init(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __SOF_LIB_VPAGE_H__ */ diff --git a/zephyr/include/sof/lib/vregion.h b/zephyr/include/sof/lib/vregion.h new file mode 100644 index 000000000000..3e36f4e870ff --- /dev/null +++ b/zephyr/include/sof/lib/vregion.h @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright(c) 2025 Intel Corporation. + +/* Pre Allocated Contiguous Virtual Region */ +#ifndef __SOF_LIB_VREGION_H__ +#define __SOF_LIB_VREGION_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct vregion; + +/** + * @brief Create a new virtual region instance. + * + * Create a new virtual region instance with specified static and dynamic partitions. + * Total size is the sum of static and dynamic sizes. + * + * @param[in] lifetime_size Size of the virtual region lifetime partition. + * @param[in] interim_size Size of the virtual region interim partition. + * @return struct vregion* Pointer to the new virtual region instance, or NULL on failure. + */ +struct vregion *vregion_create(size_t lifetime_size, size_t interim_size); + +/** + * @brief Destroy a virtual region instance. + * + * Free all associated resources and deallocate the virtual region instance. + * + * @param[in] vr Pointer to the virtual region instance to destroy. + */ +void vregion_destroy(struct vregion *vr); + +/** + * @brief Memory types for virtual region allocations. + * Used to specify the type of memory allocation within a virtual region. + * + * @note + * - interim: allocation that can be freed i.e. get/set large config, kcontrols. + * - lifetime: allocation that cannot be freed i.e. init data, pipeline data. + */ +enum vregion_mem_type { + VREGION_MEM_TYPE_INTERIM, /* interim allocation that can be freed */ + VREGION_MEM_TYPE_LIFETIME, /* lifetime allocation */ +}; + +/** + * @brief Allocate memory from the specified virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Type of memory to allocate (static, dynamic, or shared static). + * @param[in] size Size of memory to allocate in bytes. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size); + +/** + * @brief Allocate aligned memory from the specified virtual region. + * + * Allocate aligned memory from the specified virtual region based on the memory type. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Type of memory to allocate (static, dynamic, or shared static). + * @param[in] size Size of memory to allocate in bytes. + * @param[in] alignment Alignment of memory to allocate in bytes. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment); + +/** + * @brief Free memory allocated from the specified virtual region. + * + * Free memory previously allocated from the specified virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] ptr Pointer to the memory to free. + */ +void vregion_free(struct vregion *vr, void *ptr); + +/** + * @brief Log virtual region memory usage. + * + * @param[in] vr Pointer to the virtual region instance. + */ +void vregion_info(struct vregion *vr); + +#ifdef __cplusplus +} +#endif + +#endif /* __SOF_LIB_VREGION_H__ */ diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index f95f66fbdb7f..80f1a50ed717 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -338,15 +338,14 @@ static void virtual_heap_free(void *ptr) static const struct vmh_heap_config static_hp_buffers = { { - { 128, 32}, - { 512, 8}, - { 1024, 44}, - { 2048, 8}, - { 4096, 11}, - { 8192, 10}, - { 65536, 3}, - { 131072, 1}, - { 524288, 1} /* buffer for kpb */ + {128, 32}, /* 4k */ + {512, 8}, /* 4k -> 8k */ + {1024, 28}, /* 28k -> 36k */ + {2048, 4}, /* 8k -> 44k */ + {4096, 7}, /* 28k -> 72k */ + {8192, 7}, /* 56k -> 128k */ + {65536, 2}, /* 128k -> 256k */ + {524288, 1} /* 512k -> 768k */ /* buffer for kpb */ }, }; diff --git a/zephyr/lib/vpages.c b/zephyr/lib/vpages.c new file mode 100644 index 000000000000..66bc7bee8403 --- /dev/null +++ b/zephyr/lib/vpages.c @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. + * + * Author: Liam Girdwood + */ + +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(vpage, CONFIG_SOF_LOG_LEVEL); + +/* Simple Page Allocator. + * + * This allocator manages the allocation and deallocation of virtual memory pages from + * a predefined virtual memory region which is larger than the physical memory region. + * + * Both memory regions are divided into 4kB pages that are represented as blocks in a + * bitmap using the zephyr sys_mem_blocks API. The virtual block map tracks the allocation + * of virtual memory pages while the physical block map in the Zephyr MM driver tracks + * the allocation of physical memory pages. + */ + +/* max number of allocations */ +#define VPAGE_MAX_ALLOCS CONFIG_SOF_VPAGE_MAX_ALLOCS + +/* + * Virtual memory allocation element - tracks allocated virtual page id and size + */ +struct valloc_elem { + unsigned short pages; /* number of 4kB pages allocated in contiguous block */ + unsigned short vpage; /* virtual page number from start of region */ +}; + +/* + * Virtual page allocator context + * + * This structure holds all information about virtual memory pages including the + * number of free and total pages, the virtual memory region, the block + * allocator for virtual pages and the allocations. + */ +struct vpage_context { + struct k_mutex lock; + unsigned int free_pages; /* number of free 4kB pages */ + unsigned int total_pages; /* total number of 4kB pages */ + + /* Virtual memory region information */ + const struct sys_mm_drv_region *virtual_region; + struct sys_mem_blocks vpage_blocks; + sys_bitarray_t bitmap; + + /* allocation elements to track page id to allocation size */ + unsigned int num_elems_in_use; /* number of allocated elements in use*/ + struct valloc_elem velems[VPAGE_MAX_ALLOCS]; +}; + +/* uncache persistent across all cores */ +static struct vpage_context vpage_ctx; + +/** + * @brief Allocate and map virtual memory pages + * + * Allocates memory pages from the virtual page allocator. + * Maps physical memory pages to the virtual region as needed. + * + * @param pages Number of 4kB pages to allocate. + * @param ptr Pointer to store the address of allocated pages. + * @retval 0 if successful. + */ +static int vpages_alloc_and_map(unsigned int pages, void **ptr) +{ + unsigned int i; + void *vaddr; + int ret; + + /* check for valid pages and ptr */ + if (!ptr) + return -EINVAL; + + if (!pages) { + *ptr = NULL; + return 0; + } + + /* quick check for enough free pages */ + if (vpage_ctx.free_pages < pages) { + LOG_ERR("error: not enough free pages %u for requested pages %u", + vpage_ctx.free_pages, pages); + return -ENOMEM; + } + + /* check for allocation elements */ + if (vpage_ctx.num_elems_in_use >= VPAGE_MAX_ALLOCS) { + LOG_ERR("error: max allocation elements reached"); + return -ENOMEM; + } + + /* allocate virtual continuous blocks */ + ret = sys_mem_blocks_alloc_contiguous(&vpage_ctx.vpage_blocks, pages, &vaddr); + if (ret < 0) { + LOG_ERR("error: failed to allocate %u continuous virtual pages, free %u", + pages, vpage_ctx.free_pages); + return ret; + } + + /* map the virtual blocks in virtual region to free physical blocks */ + ret = sys_mm_drv_map_region_safe(vpage_ctx.virtual_region, vaddr, + 0, pages * CONFIG_MM_DRV_PAGE_SIZE, SYS_MM_MEM_PERM_RW); + if (ret < 0) { + LOG_ERR("error: failed to map virtual region %p to physical region %p, error %d", + vaddr, vpage_ctx.virtual_region->addr, ret); + sys_mem_blocks_free(&vpage_ctx.vpage_blocks, pages, &vaddr); + return ret; + } + + /* success update the free pages */ + vpage_ctx.free_pages -= pages; + + /* store the size and virtual page number in first free alloc element, + * we have already checked for a free element before the mapping. + */ + for (i = 0; i < VPAGE_MAX_ALLOCS; i++) { + if (vpage_ctx.velems[i].pages == 0) { + vpage_ctx.velems[i].pages = pages; + vpage_ctx.velems[i].vpage = + (POINTER_TO_UINT(vaddr) - + POINTER_TO_UINT(vpage_ctx.virtual_region->addr)) / + CONFIG_MM_DRV_PAGE_SIZE; + vpage_ctx.num_elems_in_use++; + break; + } + } + + if (i == VPAGE_MAX_ALLOCS) { + /* + * The caller is holding a lock, so someone must have changed + * vpages state without taking it + */ + LOG_ERR("data corruption, check for races"); + return -EPROTO; + } + + /* return the virtual address */ + *ptr = vaddr; + return 0; +} + +/** + * @brief Allocate virtual memory pages + * + * Allocates virtual memory pages from the virtual page allocator. + * + * @param pages Number of 4kB pages to allocate. + * @retval NULL on allocation failure. + */ +void *vpage_alloc(unsigned int pages) +{ + void *ptr = NULL; + int ret; + + k_mutex_lock(&vpage_ctx.lock, K_FOREVER); + ret = vpages_alloc_and_map(pages, &ptr); + k_mutex_unlock(&vpage_ctx.lock); + if (ret < 0) + LOG_ERR("vpage_alloc failed %d for %d pages, total %d free %d", + ret, pages, vpage_ctx.total_pages, vpage_ctx.free_pages); + else + LOG_INF("vpage_alloc ptr %p pages %u free %u/%u", ptr, pages, vpage_ctx.free_pages, + vpage_ctx.total_pages); + return ptr; +} + +/** + * @brief Free and unmap virtual memory pages + * + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param ptr Pointer to the memory pages to free. + * @retval 0 if successful. + * @retval -EINVAL if ptr is invalid. + */ +static int vpages_free_and_unmap(uintptr_t *ptr) +{ + unsigned int alloc_idx; + unsigned int pages = 0; + int ret; + + /* check for valid ptr which must be page aligned */ + if (!IS_ALIGNED(ptr, CONFIG_MM_DRV_PAGE_SIZE)) { + LOG_ERR("error: invalid non aligned page pointer %p", ptr); + return -EINVAL; + } + + alloc_idx = (POINTER_TO_UINT(ptr) - POINTER_TO_UINT(vpage_ctx.virtual_region->addr)) / + CONFIG_MM_DRV_PAGE_SIZE; + + /* find the allocation element */ + for (int i = 0; i < VPAGE_MAX_ALLOCS; i++) { + if (vpage_ctx.velems[i].pages > 0 && vpage_ctx.velems[i].vpage == alloc_idx) { + pages = vpage_ctx.velems[i].pages; + + LOG_DBG("found allocation element %d pages %u vpage %u for ptr %p", + i, vpage_ctx.velems[i].pages, + vpage_ctx.velems[i].vpage, ptr); + + /* clear the element */ + vpage_ctx.velems[i].pages = 0; + vpage_ctx.velems[i].vpage = 0; + vpage_ctx.num_elems_in_use--; + break; + } + } + + /* check we found allocation element */ + if (pages == 0) { + LOG_ERR("error: invalid page pointer %p not found", ptr); + return -EINVAL; + } + + /* unmap the pages from virtual region */ + ret = sys_mm_drv_unmap_region((void *)ptr, pages * CONFIG_MM_DRV_PAGE_SIZE); + if (ret < 0) { + LOG_ERR("error: failed to unmap virtual region %p pages %u, error %d", + ptr, pages, ret); + return ret; + } + + /* free physical blocks */ + ret = sys_mem_blocks_free_contiguous(&vpage_ctx.vpage_blocks, ptr, pages); + if (ret < 0) { + LOG_ERR("error: failed to free %u continuous virtual page blocks at %p, error %d", + pages, ptr, ret); + return ret; + } + + /* success update the free pages */ + vpage_ctx.free_pages += pages; + return ret; +} + +/** + * @brief Free virtual pages + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param ptr + */ +void vpage_free(void *ptr) +{ + int ret; + + k_mutex_lock(&vpage_ctx.lock, K_FOREVER); + ret = vpages_free_and_unmap((uintptr_t *)ptr); + k_mutex_unlock(&vpage_ctx.lock); + + if (!ret) + LOG_INF("vptr %p free/total pages %d/%d", ptr, vpage_ctx.free_pages, + vpage_ctx.total_pages); +} + +/** + * @brief Initialize virtual page allocator + * + * Initializes a virtual page allocator that manages a virtual memory region + * using a page table and block structures. + * + * @retval 0 if successful. + * @retval -ENOMEM on creation failure. + */ +static int init_vpages(void) +{ + const struct sys_mm_drv_region *virtual_memory_regions; + const struct sys_mm_drv_region *region; + uint32_t *bundles = NULL; + size_t block_count, bitmap_num_bundles; + int ret; + + /* create the virtual memory region and add it to the system */ + ret = adsp_add_virtual_memory_region(adsp_mm_get_unused_l2_start_aligned() + + CONFIG_SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE, + CONFIG_SOF_ZEPHYR_VIRTUAL_PAGE_REGION_SIZE, + VIRTUAL_REGION_VPAGES_ATTR); + if (ret) + return ret; + + k_mutex_init(&vpage_ctx.lock); + + /* now find the virtual region in all memory regions */ + virtual_memory_regions = sys_mm_drv_query_memory_regions(); + SYS_MM_DRV_MEMORY_REGION_FOREACH(virtual_memory_regions, region) { + if (region->attr == VIRTUAL_REGION_VPAGES_ATTR) { + vpage_ctx.virtual_region = region; + break; + } + } + sys_mm_drv_query_memory_regions_free(virtual_memory_regions); + + /* check for a valid region */ + if (!vpage_ctx.virtual_region) { + LOG_ERR("error: no valid virtual region found"); + k_panic(); + } + + block_count = region->size / CONFIG_MM_DRV_PAGE_SIZE; + if (block_count == 0) { + LOG_ERR("error: virtual region too small %d", region->size); + k_panic(); + } + + vpage_ctx.total_pages = block_count; + vpage_ctx.free_pages = block_count; + vpage_ctx.num_elems_in_use = 0; + + /* bundles are uint32_t of bits */ + bitmap_num_bundles = SOF_DIV_ROUND_UP(block_count, 32); + + /* allocate memory for bitmap bundles */ + bundles = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + bitmap_num_bundles * sizeof(uint32_t)); + if (!bundles) { + LOG_ERR("error: virtual region bitmap alloc failed"); + k_panic(); + } + + /* Fill allocators data based on config and virtual region data */ + vpage_ctx.vpage_blocks.info.num_blocks = block_count; + vpage_ctx.vpage_blocks.info.blk_sz_shift = ilog2(CONFIG_MM_DRV_PAGE_SIZE); + /* buffer is the start of the virtual memory region */ + vpage_ctx.vpage_blocks.buffer = (uint8_t *)vpage_ctx.virtual_region->addr; + + /* initialize bitmap */ + vpage_ctx.bitmap.num_bits = block_count; + vpage_ctx.bitmap.num_bundles = bitmap_num_bundles; + vpage_ctx.bitmap.bundles = bundles; + vpage_ctx.vpage_blocks.bitmap = &vpage_ctx.bitmap; + + LOG_INF("vpage_init region %p size %#x pages %d", + (void *)vpage_ctx.virtual_region->addr, + (int)vpage_ctx.virtual_region->size, block_count); + + return 0; +} + +SYS_INIT(init_vpages, POST_KERNEL, 1); diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c new file mode 100644 index 000000000000..613ea8d29747 --- /dev/null +++ b/zephyr/lib/vregion.c @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. + * + * Author: Liam Girdwood + */ + +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(vregion, CONFIG_SOF_LOG_LEVEL); + +/* + * Pre Allocated Contiguous Virtual Memory Region Allocator + * + * This allocator manages a pre-allocated virtual memory region that uses the + * virtual page allocator to allocate and free memory pages. + * + * It is designed for use cases where a contiguous virtual memory region + * is required, such as for batched allocation of audio pipelines and modules. + * + * New pipelines will create a new virtual region and will specify the size of the region + * which can be divided into multiple areas for different allocation lifetimes, permissions + * and sharing requirements. + * + * Advantages: + * + * 1) Contiguous virtual memory region for easier management and tracking of + * pipeline & DP module memory. i.e. we just need to track the vregion pointer. + * 2) Reduced fragmentation and better cache utilization by using a simple linear + * allocator for lifetime objects. + * + * Note: Software must pass in the size of the region areas at pipeline creation time. + */ + +/** + * @brief virtual region memory region structure. + * + * This structure represents a virtual memory region, which includes + * information about the base address, size, and allocation status + * of the region. + * + * Currently the virtual region memory region can be partitioned into two areas + * on page-aligned boundaries: + * + * 1. Interim Heap: An interim memory area used for multiple temporary + * allocations and frees over the lifetime of the audio processing pipeline. + * E.g. for module kcontrol derived allocations. + * + * 2. Lifetime Allocator: A simple incrementing allocator used for long-term + * static allocations that persist for the lifetime of the audio processing + * pipeline. This allocator compresses allocations for better cache + * utilization. + * + * More types can be added in the future. + * + * * TODO: Pipeline/module reset() could reset the dynamic heap. + */ + + /* linear heap used for lifetime allocations */ +struct vlinear_heap { + uint8_t *base; /* base address of linear allocator */ + uint8_t *ptr; /* current alloc pointer */ + size_t size; /* size of linear allocator in bytes */ + size_t used; /* used bytes in linear allocator */ + int free_count; /* number of frees - tuning only */ +}; + +/* zephyr k_heap for interim allocations. TODO: make lockless for improved performance */ +struct zephyr_heap { + struct k_heap heap; +}; + +/* Main vregion context, see above intro for more details. + * TODO: Add support to flag which heaps should have their contexts saved and restored. + */ +struct vregion { + /* region context */ + uint8_t *base; /* base address of entire region */ + size_t size; /* size of whole region in bytes */ + size_t pages; /* size of whole region in pages */ + + /* interim heap */ + struct zephyr_heap interim; /* interim heap */ + + /* lifetime heap */ + struct vlinear_heap lifetime; /* lifetime linear heap */ +}; + +/** + * @brief Create a new virtual region instance. + * + * Create a new VIRTUAL REGION instance with specified static and dynamic + * sizes. Total size is their sum. + * + * @param[in] lifetime_size Size of the virtual region lifetime partition. + * @param[in] interim_size Size of the virtual region interim partition. + * @return struct vregion* Pointer to the new virtual region instance, or NULL on failure. + */ +struct vregion *vregion_create(size_t lifetime_size, size_t interim_size) +{ + struct vregion *vr; + unsigned int pages; + size_t total_size; + uint8_t *vregion_base; + + if (!lifetime_size || !interim_size) { + LOG_ERR("error: invalid vregion lifetime size %d or interim size %d", + lifetime_size, interim_size); + return NULL; + } + + /* + * Align up lifetime sizes and interim sizes to nearest page, the + * vregion structure is stored in lifetime area so account for its size too. + */ + lifetime_size += sizeof(*vr); + lifetime_size = ALIGN_UP(lifetime_size, CONFIG_MM_DRV_PAGE_SIZE); + interim_size = ALIGN_UP(interim_size, CONFIG_MM_DRV_PAGE_SIZE); + total_size = lifetime_size + interim_size; + + /* allocate pages for vregion */ + pages = total_size / CONFIG_MM_DRV_PAGE_SIZE; + vregion_base = vpage_alloc(pages); + if (!vregion_base) + return NULL; + + /* init vregion - place it at the start of the lifetime region */ + vr = (struct vregion *)(vregion_base + interim_size); + vr->base = vregion_base; + vr->size = total_size; + vr->pages = pages; + + /* set partition sizes */ + vr->interim.heap.heap.init_bytes = interim_size; + vr->lifetime.size = lifetime_size; + + /* set base addresses for partitions */ + vr->interim.heap.heap.init_mem = vr->base; + vr->lifetime.base = vr->base + interim_size; + + /* set alloc ptr addresses for lifetime linear partitions */ + vr->lifetime.ptr = vr->lifetime.base + sizeof(*vr); /* skip vregion struct */ + vr->lifetime.used = sizeof(*vr); + + /* init interim heaps */ + k_heap_init(&vr->interim.heap, vr->interim.heap.heap.init_mem, interim_size); + + /* log the new vregion */ + LOG_INF("new at base %p size %#x pages %d struct embedded at %p", + (void *)vr->base, total_size, pages, (void *)vr); + LOG_DBG(" interim size %#x at %p", interim_size, (void *)vr->interim.heap.heap.init_mem); + LOG_DBG(" lifetime size %#x at %p", lifetime_size, (void *)vr->lifetime.base); + + return vr; +} + +/** + * @brief Destroy a virtual region instance. + * + * @param[in] vr Pointer to the virtual region instance to destroy. + */ +void vregion_destroy(struct vregion *vr) +{ + if (!vr) + return; + + /* log the vregion being destroyed */ + LOG_DBG("destroy %p size %#x pages %d", (void *)vr->base, vr->size, vr->pages); + LOG_DBG(" lifetime used %d free count %d", vr->lifetime.used, vr->lifetime.free_count); + vpage_free(vr->base); +} + +/** + * @brief Allocate memory with alignment from the virtual region dynamic heap. + * + * @param[in] heap Pointer to the heap to use. + * @param[in] size Size of the allocation. + * @param[in] align Alignment of the allocation. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +static void *interim_alloc(struct zephyr_heap *heap, + size_t size, size_t align) +{ + void *ptr; + + ptr = k_heap_aligned_alloc(&heap->heap, align, size, K_NO_WAIT); + if (!ptr) + LOG_ERR("error: interim alloc failed for %d bytes align %d", + size, align); + + return ptr; +} + +/** + * @brief Free memory from the virtual region interim heap. + * + * @param[in] heap Pointer to the heap to use. + * @param[in] ptr Pointer to the memory to free. + */ +static void interim_free(struct zephyr_heap *heap, void *ptr) +{ + k_heap_free(&heap->heap, ptr); +} + +/** + * @brief Allocate memory from the virtual region lifetime allocator. + * + * @param[in] heap Pointer to the linear heap to use. + * @param[in] size Size of the allocation. + * @param[in] align Alignment of the allocation. + * + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +static void *lifetime_alloc(struct vlinear_heap *heap, + size_t size, size_t align) +{ + void *ptr; + uint8_t *aligned_ptr; + size_t heap_obj_size; + + /* align heap pointer to alignment requested */ + aligned_ptr = UINT_TO_POINTER(ALIGN_UP(POINTER_TO_UINT(heap->ptr), align)); + + /* also align up size to D$ bytes if asked - allocation head and tail aligned */ + if (align == CONFIG_DCACHE_LINE_SIZE) + size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); + + /* calculate new heap object size for object and alignments */ + heap_obj_size = aligned_ptr - heap->ptr + size; + + /* check we have enough lifetime space left */ + if (heap_obj_size + heap->used > heap->size) { + LOG_ERR("error: lifetime alloc failed for object %d heap %d bytes free %d", + size, heap_obj_size, heap->size - heap->used); + return NULL; + } + + /* allocate memory */ + ptr = aligned_ptr; + heap->ptr += heap_obj_size; + heap->used += heap_obj_size; + + return ptr; +} + +/** + * @brief Free memory from the virtual region lifetime allocator. + * + * @param[in] heap Pointer to the linear heap to use. + * @param[in] ptr Pointer to the memory to free. + */ +static void lifetime_free(struct vlinear_heap *heap, void *ptr) +{ + /* simple free, just increment free count, this is for tuning only */ + heap->free_count++; + + LOG_DBG("lifetime free %p count %d", ptr, heap->free_count); +} + +/** + * @brief Free memory from the virtual region. + * + * @param vr Pointer to the virtual region instance. + * @param ptr Pointer to the memory to free. + */ +void vregion_free(struct vregion *vr, void *ptr) +{ + if (!vr || !ptr) + return; + + /* check if pointer is in interim heap */ + if (ptr >= (void *)vr->interim.heap.heap.init_mem && + ptr < (void *)((uint8_t *)vr->interim.heap.heap.init_mem + + vr->interim.heap.heap.init_bytes)) { + interim_free(&vr->interim, ptr); + return; + } + + /* check if pointer is in lifetime heap */ + if (ptr >= (void *)vr->lifetime.base && + ptr < (void *)(vr->lifetime.base + vr->lifetime.size)) { + lifetime_free(&vr->lifetime, ptr); + return; + } + + LOG_ERR("error: vregion free invalid pointer %p", ptr); +} +EXPORT_SYMBOL(vregion_free); + +/** + * @brief Allocate memory type from the virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Memory type to allocate. + * @param[in] size Size of the allocation. + * @param[in] alignment Alignment of the allocation. + * + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment) +{ + if (!vr || !size) + return NULL; + + if (!alignment) + alignment = 4; /* default align 4 bytes */ + + switch (type) { + case VREGION_MEM_TYPE_INTERIM: + return interim_alloc(&vr->interim, size, alignment); + case VREGION_MEM_TYPE_LIFETIME: + return lifetime_alloc(&vr->lifetime, size, alignment); + default: + LOG_ERR("error: invalid memory type %d", type); + return NULL; + } +} +EXPORT_SYMBOL(vregion_alloc_align); + +/** + * @brief Allocate memory from the virtual region. + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Memory type to allocate. + * @param[in] size Size of the allocation. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size) +{ + return vregion_alloc_align(vr, type, size, 0); +} +EXPORT_SYMBOL(vregion_alloc); + +/** + * @brief Log virtual region memory usage. + * + * @param[in] vr Pointer to the virtual region instance. + */ +void vregion_info(struct vregion *vr) +{ + if (!vr) + return; + + LOG_INF("base %p size %#x pages %d", + (void *)vr->base, vr->size, vr->pages); + LOG_INF("lifetime used %#x free count %d", + vr->lifetime.used, vr->lifetime.free_count); +} +EXPORT_SYMBOL(vregion_info); diff --git a/zephyr/test/CMakeLists.txt b/zephyr/test/CMakeLists.txt index c5b66c83bbaa..ee23b892da31 100644 --- a/zephyr/test/CMakeLists.txt +++ b/zephyr/test/CMakeLists.txt @@ -2,6 +2,9 @@ if(CONFIG_SOF_BOOT_TEST) zephyr_library_sources_ifdef(CONFIG_VIRTUAL_HEAP vmh.c ) + zephyr_library_sources_ifdef(CONFIG_SOF_VREGIONS + vregion.c + ) zephyr_library_sources_ifdef(CONFIG_USERSPACE userspace/ksem.c ) diff --git a/zephyr/test/vregion.c b/zephyr/test/vregion.c new file mode 100644 index 000000000000..8aa96d0aeb1c --- /dev/null +++ b/zephyr/test/vregion.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2023 Intel Corporation. All rights reserved. + * + * Author: Guennadi Liakhovetski + */ + +#include +#include +#include + +#include +#include + +#include +#include + +LOG_MODULE_DECLARE(sof_boot_test, CONFIG_SOF_LOG_LEVEL); + +static struct vregion *test_vreg_create(void) +{ + struct vregion *vreg = vregion_create(CONFIG_MM_DRV_PAGE_SIZE - 100, + CONFIG_MM_DRV_PAGE_SIZE); + + zassert_not_null(vreg); + + return vreg; +} + +static void test_vreg_alloc_lifet(struct vregion *vreg) +{ + void *ptr = vregion_alloc(vreg, VREGION_MEM_TYPE_LIFETIME, 2000); + + zassert_not_null(ptr); + + void *ptr_align = vregion_alloc_align(vreg, VREGION_MEM_TYPE_LIFETIME, 2000, 16); + + zassert_not_null(ptr_align); + zassert_equal((uintptr_t)ptr_align & 15, 0); + + void *ptr_nomem = vregion_alloc(vreg, VREGION_MEM_TYPE_LIFETIME, 2000); + + zassert_is_null(ptr_nomem); + + vregion_free(vreg, ptr_align); + vregion_free(vreg, ptr); + + /* Freeing isn't possible with LIFETIME */ + ptr_nomem = vregion_alloc(vreg, VREGION_MEM_TYPE_LIFETIME, 2000); + + zassert_is_null(ptr_nomem); +} + +static void test_vreg_alloc_tmp(struct vregion *vreg) +{ + void *ptr = vregion_alloc(vreg, VREGION_MEM_TYPE_INTERIM, 20); + + zassert_not_null(ptr); + + void *ptr_align = vregion_alloc_align(vreg, VREGION_MEM_TYPE_INTERIM, 2000, 16); + + zassert_not_null(ptr_align); + zassert_equal((uintptr_t)ptr_align & 15, 0); + + void *ptr_nomem = vregion_alloc(vreg, VREGION_MEM_TYPE_INTERIM, 2000); + + zassert_is_null(ptr_nomem); + + vregion_free(vreg, ptr_align); + vregion_free(vreg, ptr); + + /* Should be possible to allocate again */ + ptr = vregion_alloc(vreg, VREGION_MEM_TYPE_INTERIM, 2000); + + zassert_not_null(ptr); +} + +static void test_vreg_destroy(struct vregion *vreg) +{ + vregion_info(vreg); + vregion_destroy(vreg); +} + +ZTEST(sof_boot, vregion) +{ + struct vregion *vreg = test_vreg_create(); + + /* Test interim allocations */ + test_vreg_alloc_tmp(vreg); + /* Test lifetime allocations */ + test_vreg_alloc_lifet(vreg); + + test_vreg_destroy(vreg); +}