多线程curl应用程序存在内存分配问题

Question

我正在开发一个 C++ 应用程序，该应用程序可以线程化并传递一堆线程 URL 供 cURL 并行下载。

我采用的方法应该可以安全地下载图像和视频等。我使用 memcpy 而不是假设数据是字符串或字符数组。

我向每个线程传递一个结构体 thread_status，用于许多事情。该结构让父进程知道线程已完成下载。它还存储 cURL 正在下载的数据，并在 cURL 返回更多写入缓冲区时跟踪其大小。

我传递一个 (void *) 指针，该指针指向在初始化时分配给执行下载的每个线程的每个结构。第一页已正确下载，之后我不断收到来自 realloc() 的错误。

这是说明我的问题的最简单的例子。该示例不是多线程的，但使用类似的结构来跟踪自身。

#include <string>
#include <assert.h>
#include <iostream>
#include <curl/curl.h>
#include <stdlib.h> 
#include <stdio.h>
#include <string.h>

#define NOT_READY   1
#define READY       0

using namespace std;

struct thread_status {
    int id;
  pthread_t *pid;
    int readyState;
    char *url;
    void *data;
    size_t bufferlen;
    size_t writepos;
    int initialized;
} ;


size_t static 
writefunction(  void *ptr, size_t size, 
                    size_t nmemb, void *userdata)
{
    size_t nbytes = size*nmemb;
        struct thread_status **this_status;
        this_status = (struct thread_status **) userdata;

        if (!(*this_status)->initialized){
                (*this_status)->data = (void *)malloc(1024);
                (*this_status)->bufferlen = 1024;
                (*this_status)->writepos = 0;
                (*this_status)->initialized = true;
        }

        if ((*this_status)->bufferlen < ((*this_status)->writepos + nbytes)){
            (*this_status)->bufferlen = (*this_status)->bufferlen + nbytes;
            (*this_status)->data = realloc((*this_status)->data, (size_t) ((*this_status)->writepos + nbytes));
        }

        assert((*this_status)->data != NULL);
        memcpy((*this_status)->data + (*this_status)->writepos, ptr, nbytes);
        (*this_status)->writepos += nbytes; 
    return nbytes;
}

void *pull_data (void *my_struct){

struct thread_status *this_struct;
this_struct = (struct thread_status *) my_struct;
this_struct->initialized = false;

cout<<(char *)this_struct->url<<"\n";

CURL *curl;
curl = curl_easy_init();
size_t rc = 0;

while(true){

    curl_easy_setopt(curl,
        CURLOPT_WRITEFUNCTION, writefunction);
    curl_easy_setopt(curl,
        CURLOPT_WRITEDATA, (void *) &this_struct);
    curl_easy_setopt(curl,
        CURLOPT_NOSIGNAL, true);
    curl_easy_setopt(curl,
        CURLOPT_URL, (char *)this_struct->url);

    if (curl_easy_perform(curl) != 0){
        cout<<"curl did not perform\n";
        exit(1);
    } else { 
    if (this_struct->data != NULL){
            // Use a binary write.
            rc = fwrite(this_struct->data, this_struct->writepos, 1, stdout);
            free(this_struct->data);
        } else {
            cout<<"Data is NULL\n";
        } 
    }

    // Tell the babysitter the thread is ready.
    this_struct->readyState = READY;
// This would pause the thread until the parent thread has processed the data in it.
//  while(this_struct->readyState == READY){;}

    // Now get ready for another round!
    this_struct->writepos = (size_t) 0;
    this_struct->initialized = false;
    this_struct->bufferlen = (size_t) 0; 

    break;
}

    curl_easy_cleanup(curl);
    return (void *)"a";
}

int main(){

    char *urls[] = { "http://www.example.com/", "http://www.google.com", "http://www.touspassagers.com/", "http://www.facebook.com/" };
    int i=0;
    struct thread_status mystatuses[4];
    for (i=0;i<4;i++){

        struct thread_status my_status; 
        char *data;

        my_status.id = i;
        my_status.readyState = NOT_READY;
        my_status.url = urls[i];
        my_status.data = data;
        my_status.bufferlen = 0;
        my_status.writepos = 0;
        my_status.initialized = false;

        mystatuses[i] = my_status;
    }

    for (i=0;i<4;i++){
        cout<<"pulling #"<<i<<"\n";
        pull_data((void *)&mystatuses[i]);
    }

}

如果有人能启发我关于我的错误的根源或补救措施，我将不胜感激。

Answer 1

您可以考虑使用 valgrind 来帮助查找内存问题的根源。

Answer 2

知道了！

显然 1KB 的内存不足以处理第一个 cURL 缓冲区。我把 1024 改成 nbytes 就可以了！

在放入缓冲区的内存 memcpy 跑过分配的内存之前，导致损坏。

如果有人想看看完整的实现，我写了一篇关于它的文章：
http://www.touspassagers.com/2011/ 01/a-working-curlopt_writefunction-function-for-libcurl/