diff --git a/drivers/rknpu/include/rknpu_mem.h b/drivers/rknpu/include/rknpu_mem.h
index bb7d94472e9a..d2f4757c05f9 100644
--- a/drivers/rknpu/include/rknpu_mem.h
+++ b/drivers/rknpu/include/rknpu_mem.h
@@ -33,6 +33,7 @@ struct rknpu_mem_object {
 	struct page **pages;
 	struct sg_table *sgt;
 	struct dma_buf *dmabuf;
+	struct dma_buf_attachment *attachment;
 	struct list_head head;
 	unsigned int owner;
 };
diff --git a/drivers/rknpu/rknpu_drv.c b/drivers/rknpu/rknpu_drv.c
index ec5b38b2a384..eea2e50caa02 100644
--- a/drivers/rknpu/rknpu_drv.c
+++ b/drivers/rknpu/rknpu_drv.c
@@ -301,7 +301,7 @@ static const struct rknpu_config rv1126b_rknpu_config = {
 	.max_submit_number = (1 << 16) - 1,
 	.core_mask = 0x1,
 	.amount_top = &rknpu_top_amount,
-	.amount_core = &rknpu_core_amount,
+	.amount_core = NULL,
 	.state_init = NULL,
 	.cache_sgt_init = NULL,
 };
@@ -569,8 +569,13 @@ static int rknpu_release(struct inode *inode, struct file *file)
 		vunmap(entry->kv_addr);
 		entry->kv_addr = NULL;
 
-		if (!entry->owner)
+		if (!entry->owner) {
 			dma_buf_put(entry->dmabuf);
+		} else {
+			dma_buf_unmap_attachment(entry->attachment, entry->sgt,
+						 DMA_BIDIRECTIONAL);
+			dma_buf_detach(entry->dmabuf, entry->attachment);
+		}
 
 		list_del(&entry->head);
 		kfree(entry);
diff --git a/drivers/rknpu/rknpu_mem.c b/drivers/rknpu/rknpu_mem.c
index 222fce4d76e8..70f363d7d21a 100644
--- a/drivers/rknpu/rknpu_mem.c
+++ b/drivers/rknpu/rknpu_mem.c
@@ -194,6 +194,7 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
 	rknpu_obj->size = PAGE_ALIGN(args.size);
 	rknpu_obj->dma_addr = phys;
 	rknpu_obj->sgt = table;
+	rknpu_obj->attachment = attachment;
 
 	args.size = rknpu_obj->size;
 	args.obj_addr = (__u64)(uintptr_t)rknpu_obj;
@@ -212,9 +213,6 @@ int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
 		goto err_unmap_kv_addr;
 	}
 
-	dma_buf_unmap_attachment(attachment, table, DMA_BIDIRECTIONAL);
-	dma_buf_detach(dmabuf, attachment);
-
 	spin_lock(&rknpu_dev->lock);
 
 	session = file->private_data;
@@ -300,9 +298,15 @@ int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
 		vunmap(rknpu_obj->kv_addr);
 		rknpu_obj->kv_addr = NULL;
 
-		if (!rknpu_obj->owner)
+		if (!rknpu_obj->owner) {
 			dma_buf_put(rknpu_obj->dmabuf);
-
+		} else {
+			dma_buf_unmap_attachment(rknpu_obj->attachment,
+						 rknpu_obj->sgt,
+						 DMA_BIDIRECTIONAL);
+			dma_buf_detach(rknpu_obj->dmabuf,
+				       rknpu_obj->attachment);
+		}
 		kfree(rknpu_obj);
 	}