diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index b9a451bd7d6502226fadf5847d103362925f42d0..44e1c0debb1a79b54b2bf70876c60fb0b24361cb 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -1186,6 +1186,7 @@ CONFIG_COMPACTION=y CONFIG_COMPACT_UNEVICTABLE_DEFAULT=1 CONFIG_PAGE_REPORTING=y CONFIG_MIGRATION=y +CONFIG_MIGRATE_PAGES_DMA_OFFLOADING=y CONFIG_DEVICE_MIGRATION=y CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION=y CONFIG_ARCH_ENABLE_THP_MIGRATION=y diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h index 0f577f932bb4c70069da374c7218fed7b9ca40e6..4c992bc665757d3a5bf5c3e8110069462b05491e 100644 --- a/include/linux/migrate_mode.h +++ b/include/linux/migrate_mode.h @@ -19,6 +19,8 @@ enum migrate_mode { MIGRATE_SYNC_NO_COPY, }; +#define MIGRATE_ASYNC_DMA_OFFLOADING (10) + enum migrate_reason { MR_COMPACTION, MR_MEMORY_FAILURE, diff --git a/include/linux/mm.h b/include/linux/mm.h index c0040a2014c45176fe1e9a575f78352e7fc9e6a5..036822cb1b9d666151bf1f5ef896745b059eb407 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1330,6 +1330,14 @@ void put_pages_list(struct list_head *pages); void split_page(struct page *page, unsigned int order); void folio_copy(struct folio *dst, struct folio *src); int folio_mc_copy(struct folio *dst, struct folio *src); +#ifdef CONFIG_MIGRATE_PAGES_DMA_OFFLOADING +int folio_dma_copy(struct folio *dst, struct folio *src); +#else +static inline int folio_dma_copy(struct folio *dst, struct folio *src) +{ + return -ENODEV; +} +#endif unsigned long nr_free_buffer_pages(void); diff --git a/mm/Kconfig b/mm/Kconfig index 4eb0642b71e5cb49292f0afa60bccac1842ad50b..9408a3ca7f4ce0e39ccb7133903b04549ffc101c 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -696,6 +696,13 @@ config MIGRATION pages as migration can relocate pages to satisfy a huge page allocation instead of reclaiming. +config MIGRATE_PAGES_DMA_OFFLOADING + bool "Support to use DMA channels for page migration" + depends on DMA_ENGINE + default y + help + Allows kernel to use dma offloading for page migraton. + config DEVICE_MIGRATION def_bool MIGRATION && ZONE_DEVICE diff --git a/mm/Makefile b/mm/Makefile index e45cdeda47b7cfef2d8cf32d7ce4a8a0a15cb607..dd81f8c33e03a62ef73c010f4e1eb5c983aecf4e 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -92,6 +92,7 @@ obj-$(CONFIG_FAILSLAB) += failslab.o obj-$(CONFIG_FAIL_PAGE_ALLOC) += fail_page_alloc.o obj-$(CONFIG_MEMTEST) += memtest.o obj-$(CONFIG_MIGRATION) += migrate.o +obj-$(CONFIG_MIGRATE_PAGES_DMA_OFFLOADING) += migrate_dma.o obj-$(CONFIG_NUMA) += memory-tiers.o obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o diff --git a/mm/migrate.c b/mm/migrate.c index 4edd29d9a041741a40a13173229bdf5da00c3394..8b1058b28175b4f0e1b6a28c55afdc0741ad2aba 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -583,6 +583,11 @@ static int folio_migrate_mc_copy(struct folio *dst, struct folio *src, if (mode == MIGRATE_SYNC_NO_COPY) return 0; + if (mode == MIGRATE_ASYNC_DMA_OFFLOADING && folio_test_hugetlb(src)) + /* if dma offloading fail, fallback */ + if (!folio_dma_copy(dst, src)) + return 0; + return folio_mc_copy(dst, src); } diff --git a/mm/migrate_dma.c b/mm/migrate_dma.c new file mode 100644 index 0000000000000000000000000000000000000000..c6ff6f1258aaeb5a00c115e7a8dd196007ca7a5e --- /dev/null +++ b/mm/migrate_dma.c @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Support to use DMA channels for page migration. + * + * Copyright (C) 2025 Huawei Limited + */ + +#include +#include + +/* DMA channel track its transfers done */ +struct dma_channel_work { + struct dma_chan *chan; + enum dma_status status; + struct completion done; +}; + +static void folios_dma_copy_completion_callback(void *param, + const struct dmaengine_result *result) +{ + struct dma_channel_work *chan_work = param; + + if (result) { + enum dmaengine_tx_result dma_res = result->result; + + if (dma_res == DMA_TRANS_NOERROR) + chan_work->status = DMA_COMPLETE; + else + chan_work->status = DMA_ERROR; + } + + complete(&chan_work->done); +} + +static int process_folio_dma_transfer(struct dma_channel_work *chan_work, + struct folio *src, struct folio *dst) +{ + struct dma_chan *chan = chan_work->chan; + struct device *dev = dmaengine_get_dma_device(chan); + enum dma_ctrl_flags flags = DMA_CTRL_ACK; + struct dma_async_tx_descriptor *tx; + dma_addr_t src_handle, dst_handle; + size_t size = folio_size(src); + int ret; + + flags |= DMA_PREP_INTERRUPT; + + src_handle = dma_map_page(dev, &src->page, 0, size, DMA_TO_DEVICE); + if (dma_mapping_error(dev, src_handle)) { + pr_err("map dma src page error.\n"); + return -ENOMEM; + } + + dst_handle = dma_map_page(dev, &dst->page, 0, size, DMA_FROM_DEVICE); + if (dma_mapping_error(dev, dst_handle)) { + pr_err("map dma dst page error.\n"); + ret = -ENOMEM; + goto out_unmap; + } + + tx = dmaengine_prep_dma_memcpy(chan, dst_handle, src_handle, + size, flags); + if (unlikely(!tx)) { + pr_err("prep dma memcpy error.\n"); + ret = -EBUSY; + goto out_unmap_all; + } + + tx->callback_result = folios_dma_copy_completion_callback; + tx->callback_param = chan_work; + init_completion(&chan_work->done); + chan_work->status = DMA_ERROR; + + if (dma_submit_error(dmaengine_submit(tx))) { + pr_err("dma submit error.\n"); + ret = -EINVAL; + goto out_unmap_all; + } + + dma_async_issue_pending(chan); + if (!wait_for_completion_timeout(&chan_work->done, + msecs_to_jiffies(1000))) { + ret = -ETIMEDOUT; + goto out_unmap_all; + } + + ret = (chan_work->status == DMA_COMPLETE) ? 0 : -EPROTO; + +out_unmap_all: + dma_unmap_page(dev, dst_handle, size, DMA_FROM_DEVICE); +out_unmap: + dma_unmap_page(dev, src_handle, size, DMA_TO_DEVICE); + + return ret; +} + +static bool folio_dma_chan_filter(struct dma_chan *chan, void *param) +{ + return !strcmp(dev_name(chan->device->dev), "ub_dma_device"); +} + +int folio_dma_copy(struct folio *dst, struct folio *src) +{ + struct dma_channel_work *chan_work; + struct dma_slave_config dma_cfg; + struct dma_chan *chan; + dma_cap_mask_t mask; + int ret = -ENODEV; + + dma_cap_zero(mask); + dma_cap_set(DMA_MEMCPY, mask); + chan = dma_request_channel(mask, folio_dma_chan_filter, NULL); + if (!chan) { + pr_err("failed to allocate dma channel.\n"); + return ret; + } + + memset(&dma_cfg, 0, sizeof(dma_cfg)); + dma_cfg.direction = DMA_MEM_TO_MEM; + ret = dmaengine_slave_config(chan, &dma_cfg); + if (ret) { + pr_err("failed to config dma channel.\n"); + goto out_release; + } + + chan_work = kmalloc(sizeof(*chan_work), GFP_KERNEL); + if (unlikely(!chan_work)) { + pr_err("failed to allocate memory for chan work.\n"); + goto out_release; + } + + chan_work->chan = chan; + ret = process_folio_dma_transfer(chan_work, src, dst); + if (unlikely(ret)) + pr_err("failed to process folio dma transfer.\n"); + + kfree(chan_work); +out_release: + dma_release_channel(chan); + + return ret; +} +EXPORT_SYMBOL(folio_dma_copy);