如何使用 O_DIRECT 将内核空间内存（物理地址）写入文件？

Question

我想将物理内存写入文件。内存本身不会再次被触及，因此我想使用O_DIRECT来获得最佳的写入性能。

我的第一个想法是打开 /dev/mem 并映射内存并将所有内容写入一个文件，该文件使用 O_DIRECT 打开。对 mmap 返回的内存地址的写入调用失败 (EFAULT)。如果我不使用O_DIRECT，则会生成memcpy。

#include <cstdint>
#include <iostream>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdio.h>
#include <errno.h>
#include <malloc.h>
#include <sys/mman.h>

#define PRINT_ERRNO_REASON(reason)  \
        case reason: { std::cout << #reason << std::endl; } break;

void write_page_aligned_buffer(int out_fd)
{
    const ssize_t PAGE_SIZE = getpagesize();

    void* page_aligned_buffer = memalign(PAGE_SIZE, PAGE_SIZE);
    if(!page_aligned_buffer)
    {
        std::cout << "Could not allocate page aligned buffer." << std::endl;
        return;
    }

    std::cout << "Allocated a buffer at address " << page_aligned_buffer << "." << std::endl;

    if(write(out_fd, page_aligned_buffer, PAGE_SIZE) < 0)
    {
        std::cout << "Could not write page-aligned user buffer to tmp-file. Quitting..." << std::endl;
        std::cout << "Reason of fail is ";

        switch(errno)
        {
            PRINT_ERRNO_REASON(EAGAIN);
            PRINT_ERRNO_REASON(EBADF);
            PRINT_ERRNO_REASON(EFAULT);
            PRINT_ERRNO_REASON(EFBIG);
            PRINT_ERRNO_REASON(EINTR);
            PRINT_ERRNO_REASON(EINVAL);
            PRINT_ERRNO_REASON(EIO);
            PRINT_ERRNO_REASON(ENOSPC);
            PRINT_ERRNO_REASON(EPIPE);
        default:
            std::cout << "Unknown" << std::endl;
        }
    }
    else
    {
        std::cout << "Successfully written user-page-aligned buffer." << std::endl;
    }

    free(page_aligned_buffer);
}

int main()
{
    const ssize_t PAGE_SIZE = getpagesize();

    // number of pages to copy
    const uint32_t PAGES_TO_COPY = 1;

    char* tmp_file_name = 0;
    int tmp_file_fd = -1;
    ssize_t bytes_copied = 0;

    std::cout << "Copying " << PAGES_TO_COPY << " pages with PAGE_SIZE = " << PAGE_SIZE << std::endl;
    std::cout << "Copying " << PAGES_TO_COPY * PAGE_SIZE / 1024 << " kBytes." << std::endl << std::endl;

    uid_t user_id = geteuid();
    if(user_id)
    {
        std::cout << "We need to be root. I am euid == " << user_id << ". Quitting..." << std::endl;
        return 0;
    }
    else
    {
        seteuid(0);
        setuid(0);
    }

    // get the file descriptor
    int mem_fd = open("/dev/mem", O_RDONLY);
    if(mem_fd < 0)
    {
        std::cout << "Could not open /dev/mem. Quitting..." << std::endl;
        std::cout << "Reason of fail is ";

        switch(errno)
        {
            PRINT_ERRNO_REASON(EACCES);
            PRINT_ERRNO_REASON(EEXIST);
            PRINT_ERRNO_REASON(EINTR);
            PRINT_ERRNO_REASON(EINVAL);
            PRINT_ERRNO_REASON(EIO);
            PRINT_ERRNO_REASON(EISDIR);
            PRINT_ERRNO_REASON(ELOOP);
            PRINT_ERRNO_REASON(EMFILE);
            PRINT_ERRNO_REASON(ENFILE);
            PRINT_ERRNO_REASON(ENOENT);
            PRINT_ERRNO_REASON(ENOSR);
            PRINT_ERRNO_REASON(ENOSPC);
            PRINT_ERRNO_REASON(ENOTDIR);
            PRINT_ERRNO_REASON(ENXIO);
            PRINT_ERRNO_REASON(EOVERFLOW);
            PRINT_ERRNO_REASON(EROFS);
            PRINT_ERRNO_REASON(EAGAIN);
            PRINT_ERRNO_REASON(ENAMETOOLONG);
            PRINT_ERRNO_REASON(ENOMEM);
            PRINT_ERRNO_REASON(ETXTBSY);
        default:
            std::cout << "Unknown" << std::endl;
        }
        return 0;
    }

    // get read pointer
    uint8_t* mem_ptr = static_cast<uint8_t*>(mmap(0,
            PAGE_SIZE,
            PROT_READ,
            MAP_SHARED,
            mem_fd,
            PAGE_SIZE));
    if(mem_ptr == MAP_FAILED)
    {
        std::cout << "Could not mmap /dev/mem. Quitting..." << std::endl;
        std::cout << "Reason of fail is ";

        switch(errno)
        {
            PRINT_ERRNO_REASON(EACCES);
            PRINT_ERRNO_REASON(EAGAIN);
            PRINT_ERRNO_REASON(EBADF);
            PRINT_ERRNO_REASON(EINVAL);
            PRINT_ERRNO_REASON(ENFILE);
            PRINT_ERRNO_REASON(ENODEV);
            PRINT_ERRNO_REASON(ENOMEM);
            PRINT_ERRNO_REASON(EPERM);
            PRINT_ERRNO_REASON(ETXTBSY);
        default:
            std::cout << "Unknown" << std::endl;
        }
        goto CLEANUP_FD_DEV_MEM;
    }

    tmp_file_name = tempnam("/tmp", "prefix");
    if(!tmp_file_name)
    {
        std::cout << "Could not get a free tmp-filename";
        goto CLEANUP_MMAP_DEV_MEM;
    }

    // if O_DIRECT is omitted the example will work
    tmp_file_fd = open(tmp_file_name,
            O_WRONLY | O_CREAT | O_DIRECT | O_TRUNC,
            S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
    if(tmp_file_fd < 0)
    {
        std::cout << "Could not create tmp file with O_DIRECT." << std::endl;
        goto CLEANUP_MMAP_DEV_MEM;
    }

    write_page_aligned_buffer(tmp_file_fd);

    // everything worked so lets start the copying
    for(uint i = 0; i < PAGES_TO_COPY; i++)
    {
        // check memory
        // snip
        for(int i = 0; i < PAGE_SIZE; i += 32)
        {
            printf("%02X%02X%02X%02X %02X%02X%02X%02X %02X%02X%02X%02X %02X%02X%02X%02X %02X%02X%02X%02X %02X%02X%02X%02X %02X%02X%02X%02X %02X%02X%02X%02X\n",
                    mem_ptr[i + 0], mem_ptr[i + 1], mem_ptr[i + 2], mem_ptr[i + 3],
                    mem_ptr[i + 4], mem_ptr[i + 5], mem_ptr[i + 6], mem_ptr[i + 7],
                    mem_ptr[i + 8], mem_ptr[i + 9], mem_ptr[i + 10], mem_ptr[i + 11],
                    mem_ptr[i + 12], mem_ptr[i + 13], mem_ptr[i + 14], mem_ptr[i + 15],
                    mem_ptr[i + 16], mem_ptr[i + 17], mem_ptr[i + 18], mem_ptr[i + 19],
                    mem_ptr[i + 20], mem_ptr[i + 21], mem_ptr[i + 22], mem_ptr[i + 23],
                    mem_ptr[i + 24], mem_ptr[i + 25], mem_ptr[i + 26], mem_ptr[i + 27],
                    mem_ptr[i + 28], mem_ptr[i + 29], mem_ptr[i + 30], mem_ptr[i + 31]);
        }
        std::cout << "\n";
        // endsnip

        bytes_copied = write(tmp_file_fd, mem_ptr, PAGE_SIZE);
        if(bytes_copied < 0)
        {
            std::cout << "Could not write to tmp-file. Quitting..." << std::endl;
            std::cout << "Reason of fail is ";

            switch(errno)
            {
                PRINT_ERRNO_REASON(EAGAIN);
                PRINT_ERRNO_REASON(EBADF);
                PRINT_ERRNO_REASON(EFAULT);
                PRINT_ERRNO_REASON(EFBIG);
                PRINT_ERRNO_REASON(EINTR);
                PRINT_ERRNO_REASON(EINVAL);
                PRINT_ERRNO_REASON(EIO);
                PRINT_ERRNO_REASON(ENOSPC);
                PRINT_ERRNO_REASON(EPIPE);
            default:
                std::cout << "Unknown" << std::endl;
            }
            goto CLEANUP_FD_TMP_FILE;
        }
    }

CLEANUP_FD_TMP_FILE:
    if(tmp_file_name)
    {
        if(close(tmp_file_fd))
        {
            std::cout << "Could close tmp-file " << tmp_file_name << "." << std::endl;
        }

        if(remove(tmp_file_name))
        {
            std::cout << "Could remove tmp-file " << tmp_file_name << "." << std::endl;
        }

        free(tmp_file_name);
    }

CLEANUP_MMAP_DEV_MEM:
    if(munmap(mem_ptr, PAGE_SIZE))
    {
        std::cout << "munmap failed." << std::endl;
    }

CLEANUP_FD_DEV_MEM:
    if(close(mem_fd))
    {
        std::cout << "Could not close /dev/mem filedescriptor." << std::endl;
    }

    return 0;
}

下一步是编写一个处理内存传输的字符设备或块设备。但是如何绕过copy_to_user呢？目标系统是嵌入式 PowerPC 架构，其缺点是将用户内存写入硬盘（使用 DMA 控制器）比从 RAM 到 RAM 的 memcpy 更快。因此我必须绕过页面缓存。

最好的问候

弗里德里希

Answer 1

你的问题看起来有点奇怪。由于您的编程非常接近硬件，因此您可能会考虑使用直接内存访问 (DMA)。这可能有点棘手，因为您需要了解分页和部分 I/O 机制。您可能想阅读：

http://www.linuxjournal.com/article/7104

(这只是简单的介绍，以了解这个想法。）

Answer 2

我今晚测试过，如果使用 O_DIRECT 你的写入将是有条件的。