diff --git a/posix/include/rtos/kernel.h b/posix/include/rtos/kernel.h
index 714fc6672a1d..9afb1d13d4ac 100644
--- a/posix/include/rtos/kernel.h
+++ b/posix/include/rtos/kernel.h
@@ -28,6 +28,12 @@ typedef struct {
 
 #define Z_TIMEOUT_MS(t) ((k_timeout_t) { .ticks = clock_ms_to_ticks(PLATFORM_DEFAULT_CLOCK, t) })
 
+struct k_thread;
+static inline bool thread_is_userspace(struct k_thread *thread)
+{
+	return false;
+}
+
 static inline void k_sleep(k_timeout_t timeout)
 {
 	wait_delay(timeout.ticks);
diff --git a/zephyr/include/rtos/kernel.h b/zephyr/include/rtos/kernel.h
index 17d388cbf8e8..dcf12454f096 100644
--- a/zephyr/include/rtos/kernel.h
+++ b/zephyr/include/rtos/kernel.h
@@ -8,4 +8,9 @@
 
 #include <zephyr/kernel.h>
 
+static inline bool thread_is_userspace(struct k_thread *thread)
+{
+	return !!(thread->base.user_options & K_USER);
+}
+
 #endif /* __ZEPHYR_RTOS_KERNEL_H__ */
diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c
index c681252ae27a..6ce61fa64601 100644
--- a/zephyr/lib/fast-get.c
+++ b/zephyr/lib/fast-get.c
@@ -28,9 +28,6 @@
 struct sof_fast_get_entry {
 	const void *dram_ptr;
 	void *sram_ptr;
-#if CONFIG_USERSPACE
-	struct k_mem_domain *mdom;
-#endif
 	size_t size;
 	unsigned int refcount;
 };
@@ -103,10 +100,8 @@ static struct sof_fast_get_entry *fast_get_find_entry(struct sof_fast_get_data *
 #endif
 
 #if CONFIG_USERSPACE
-static bool fast_get_partition_exists(struct k_thread *thread, void *start, size_t size)
+static bool fast_get_partition_exists(struct k_mem_domain *domain, void *start, size_t size)
 {
-	struct k_mem_domain *domain = thread->mem_domain_info.mem_domain;
-
 	for (unsigned int i = 0; i < domain->num_partitions; i++) {
 		struct k_mem_partition *dpart = &domain->partitions[i];
 
@@ -132,6 +127,9 @@ static int fast_get_access_grant(struct k_mem_domain *mdom, void *addr, size_t s
 
 const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size)
 {
+#if CONFIG_USERSPACE
+	bool current_is_userspace = thread_is_userspace(k_current_get());
+#endif
 	struct sof_fast_get_data *data = &fast_get_data;
 	uint32_t alloc_flags = SOF_MEM_FLAG_USER;
 	struct sof_fast_get_entry *entry;
@@ -189,13 +187,10 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size)
 		 * We only get there for large buffers, since small buffers with
 		 * enabled userspace don't create fast-get entries
 		 */
-		if (mdom->num_partitions > 1) {
-			/* A userspace thread makes the request */
-			if (mdom != entry->mdom &&
-			    !fast_get_partition_exists(k_current_get(), ret,
+		if (current_is_userspace) {
+			if (!fast_get_partition_exists(mdom, ret,
 						       ALIGN_UP(size, CONFIG_MM_DRV_PAGE_SIZE))) {
-				LOG_DBG("grant access to domain %p first was %p", mdom,
-					entry->mdom);
+				LOG_DBG("grant access to domain %p", mdom);
 
 				int err = fast_get_access_grant(mdom, ret, size);
 
@@ -234,14 +229,14 @@ const void *fast_get(struct k_heap *heap, const void *dram_ptr, size_t size)
 	dcache_writeback_region((__sparse_force void __sparse_cache *)entry->sram_ptr, size);
 
 #if CONFIG_USERSPACE
-	entry->mdom = k_current_get()->mem_domain_info.mem_domain;
-	if (size > FAST_GET_MAX_COPY_SIZE) {
+	if (size > FAST_GET_MAX_COPY_SIZE && current_is_userspace) {
 		/* Otherwise we've allocated on thread's heap, so it already has access */
-		int err = fast_get_access_grant(entry->mdom, ret, size);
+		int err = fast_get_access_grant(k_current_get()->mem_domain_info.mem_domain,
+						ret, size);
 
 		if (err < 0) {
 			LOG_ERR("failed to grant access err=%d", err);
-			sof_heap_free(NULL, ret);
+			sof_heap_free(heap, ret);
 			ret = NULL;
 			goto out;
 		}
@@ -296,11 +291,8 @@ void fast_put(struct k_heap *heap, struct k_mem_domain *mdom, const void *sram_p
 	 * For large buffers, each thread that called fast_get() has a partition
 	 * in its memory domain. Each thread must remove its own partition here
 	 * to prevent partition leaks.
-	 *
-	 * Order matters: free buffer first (needs partition for cache access),
-	 * then remove partition.
 	 */
-	if (entry->size > FAST_GET_MAX_COPY_SIZE && entry->mdom && mdom) {
+	if (entry->size > FAST_GET_MAX_COPY_SIZE && mdom) {
 		struct k_mem_partition part = {
 			.start = (uintptr_t)sram_ptr,
 			.size = ALIGN_UP(entry->size, CONFIG_MM_DRV_PAGE_SIZE),