HttpWebRequest FileUpload 导致 OutOfMemoryException

Question

当我执行以下源代码时，我总是收到此 OutOfMemoryException:

 System.OutOfMemoryException was not handled.
 Message=Exception of type 'System.OutOfMemoryException' was thrown.
 Source=System
 StackTrace:
    at System.Net.ScatterGatherBuffers.AllocateMemoryChunk(Int32 newSize)
    at System.Net.ScatterGatherBuffers.Write(Byte[] buffer, Int32 offset, Int32 count)
    at System.Net.ConnectStream.InternalWrite(Boolean async, Byte[] buffer, Int32 offset, Int32 size, AsyncCallback callback, Object state)
    at System.Net.ConnectStream.Write(Byte[] buffer, Int32 offset, Int32 size)
    …

... 并且内存使用量正在稳步增加（根据 Windows 任务管理器）：

Imports System.Net
Imports System.Text
Imports System.IO

Public Class HttpFileUploader

    Public Function uploadFile(ByVal containa As CookieContainer, ByVal uri As String, ByVal filePath As String, ByVal fileParameterName As String, ByVal contentType As String, ByVal otherParameters As Specialized.NameValueCollection) As String

        Dim boundary As String = "---------------------------" & DateTime.Now.Ticks.ToString("x")
        Dim newLine As String = System.Environment.NewLine
        Dim boundaryBytes As Byte() = System.Text.Encoding.ASCII.GetBytes(newLine & "--" & boundary & newLine)
        Dim request As Net.HttpWebRequest = Net.WebRequest.Create(uri)

        request.ContentType = "multipart/form-data; boundary=" & boundary
        request.Method = "POST"
        request.CookieContainer = containa
        request.AllowAutoRedirect = True
        request.Timeout = -1

        Using requestStream As IO.Stream = request.GetRequestStream()

            Dim formDataTemplate As String = "Content-Disposition: form-data; name=""{0}""{1}{1}{2}"

            For Each key As String In otherParameters.Keys

                requestStream.Write(boundaryBytes, 0, boundaryBytes.Length)
                Dim formItem As String = String.Format(formDataTemplate, key, newLine, otherParameters(key))
                Dim formItemBytes As Byte() = System.Text.Encoding.UTF8.GetBytes(formItem)
                requestStream.Write(formItemBytes, 0, formItemBytes.Length)

            Next key

            requestStream.Write(boundaryBytes, 0, boundaryBytes.Length)

            Dim headerTemplate As String = "Content-Disposition: form-data; name=""{0}""; filename=""{1}""{2}Content-Type: {3}{2}{2}"
            Dim header As String = String.Format(headerTemplate, fileParameterName, filePath, newLine, contentType)
            Dim headerBytes As Byte() = System.Text.Encoding.UTF8.GetBytes(header)
            requestStream.Write(headerBytes, 0, headerBytes.Length)

            Using fileStream As New IO.FileStream(filePath, IO.FileMode.Open, IO.FileAccess.Read)

                Dim buffer(4096) As Byte
                Dim bytesRead As Int32 = fileStream.Read(buffer, 0, buffer.Length)

                Do While (bytesRead > 0)
                    ' the exception is triggered by the following line:     
                    requestStream.Write(buffer, 0, bytesRead)
                    bytesRead = fileStream.Read(buffer, 0, buffer.Length)
                Loop

            End Using

            Dim trailer As Byte() = System.Text.Encoding.ASCII.GetBytes(newLine & "--" + boundary + "--" & newLine)
            requestStream.Write(trailer, 0, trailer.Length)
            requestStream.Close()

        End Using

        Dim response As Net.WebResponse = Nothing
        Dim responseText = ""

        Try

            response = request.GetResponse()

            Using responseStream As IO.Stream = response.GetResponseStream()

                Using responseReader As New IO.StreamReader(responseStream)

                    responseText = responseReader.ReadToEnd()

                End Using

            End Using

        Catch exception As Net.WebException

            MsgBox(exception.Message)

        Finally

            response.Close()
            response = Nothing
            request = Nothing

        End Try

        Return responseText

    End Function

End Class

我已用注释标记了触发异常的行。

我使用 10 个线程来上传不同的文件（大小最大为 250 MB）。

为什么我的这段代码内存不足？

Answer 1

以下代码正在运行：

Imports System.Net
Imports System.Text
Imports System.IO

Public Class HttpFileUploader

    Public Function uploadFile(ByVal containa As CookieContainer, ByVal uri As String, ByVal filePath As String, ByVal fileParameterName As String, ByVal contentType As String, ByVal otherParameters As Specialized.NameValueCollection) As String

        Dim boundary As String = "---------------------------" & DateTime.Now.Ticks.ToString("x")
        Dim newLine As String = System.Environment.NewLine
        Dim boundaryBytes As Byte() = System.Text.Encoding.ASCII.GetBytes(newLine & "--" & boundary & newLine)
        Dim request As Net.HttpWebRequest = Net.WebRequest.Create(uri)

        request.ContentType = "multipart/form-data; boundary=" & boundary
        request.Method = "POST"
        request.CookieContainer = containa
        request.AllowAutoRedirect = True
        request.Timeout = -1
        request.KeepAlive = True
        request.AllowWriteStreamBuffering = False

        Dim ms As New MemoryStream()
        Dim formDataTemplate As String = "Content-Disposition: form-data; name=""{0}""{1}{1}{2}"

        For Each key As String In otherParameters.Keys
            ms.Write(boundaryBytes, 0, boundaryBytes.Length)
            Dim formItem As String = String.Format(formDataTemplate, key, newLine, otherParameters(key))
            Dim formItemBytes As Byte() = System.Text.Encoding.UTF8.GetBytes(formItem)
            ms.Write(formItemBytes, 0, formItemBytes.Length)
        Next key

        ms.Write(boundaryBytes, 0, boundaryBytes.Length)

        Dim headerTemplate As String = "Content-Disposition: form-data; name=""{0}""; filename=""{1}""{2}Content-Type: {3}{2}{2}"
        Dim header As String = String.Format(headerTemplate, fileParameterName, filePath, newLine, contentType)
        Dim headerBytes As Byte() = System.Text.Encoding.UTF8.GetBytes(header)
        ms.Write(headerBytes, 0, headerBytes.Length)

        Dim length As Long = ms.Length
        length += New FileInfo(filePath).Length
        request.ContentLength = length

        Using requestStream As IO.Stream = request.GetRequestStream()
            Dim bheader() As Byte = ms.ToArray()
            requestStream.Write(bheader, 0, bheader.Length)
            Using fileStream As New IO.FileStream(filePath, IO.FileMode.Open, IO.FileAccess.Read)

                Dim buffer(4096) As Byte
                Dim bytesRead As Int32 = fileStream.Read(buffer, 0, buffer.Length)

                Do While (bytesRead > 0)
                    requestStream.Write(buffer, 0, bytesRead)
                    bytesRead = fileStream.Read(buffer, 0, buffer.Length)
                Loop
            End Using
            requestStream.Close()
        End Using

        Dim response As Net.WebResponse = Nothing
        Dim responseText = ""

        Try

            response = request.GetResponse()

            Using responseStream As IO.Stream = response.GetResponseStream()

                Using responseReader As New IO.StreamReader(responseStream)

                    responseText = responseReader.ReadToEnd()

                End Using

            End Using

        Catch exception As Net.WebException

            MsgBox(exception.Message)

        Finally

            response.Close()
            response = Nothing
            request = Nothing
        End Try

        Return responseText

    End Function

End Class

非常感谢 Tom 和来自 http://blogs.msdn.com/b/johan/archive/2006/11/15/are-you-getting-outofmemoryexceptions-when-uploading-large -files.aspx！！！

Answer 2

该线程似乎对大字节流有所启发：

http://social.msdn.microsoft.com/Forums/en-US/csharpgeneral/thread/1af59645-cdef-46a9-9eb1-616661babf90/

何时使用 10 个线程一次发送 250mb 文件，您实际上可能会耗尽内存！您使用什么规格的机器？

更新

我完全赞成尝试一下，所以这就是我所做的。我创建了一个rapidshare 帐户，但厌倦了阅读有关上传API 的复杂性。因此我实际上不确定这段代码是否有效。然而，它确实运行。我注意到运行上面的代码在文件读取期间似乎没有做任何事情。但是，一旦调用 GetResponseStream，它就会立即上传文件。嗯，我们希望在流读取过程中上传它。

因此，我尝试了 Jeffrey Richter 的博客 上的建议，并得到了快速浏览此处以及其他一些上传示例。现在我在上面说过，我并没有深入研究 RapidShares 上传 api 的复杂性，而是讨论流操作本身。

请参阅下面的代码。除了为标头信息引入 MemoryStream 之外，它是相同的，并且我还删除了页脚内容（以获得正确的字节计数）。

    Dim ms As New MemoryStream()
    Dim formDataTemplate As String = "Content-Disposition: form-data; name=""{0}""{1}{1}{2}"

    For Each key As String In otherParameters.Keys
        ms.Write(boundaryBytes, 0, boundaryBytes.Length)
        Dim formItem As String = String.Format(formDataTemplate, key, newLine, otherParameters(key))
        Dim formItemBytes As Byte() = System.Text.Encoding.UTF8.GetBytes(formItem)
        ms.Write(formItemBytes, 0, formItemBytes.Length)
    Next key

    ms.Write(boundaryBytes, 0, boundaryBytes.Length)

    Dim headerTemplate As String = "Content-Disposition: form-data; name=""{0}""; filename=""{1}""{2}Content-Type: {3}{2}{2}"
    Dim header As String = String.Format(headerTemplate, fileParameterName, filePath, newLine, contentType)
    Dim headerBytes As Byte() = System.Text.Encoding.UTF8.GetBytes(header)
    ms.Write(headerBytes, 0, headerBytes.Length)

    Dim length As Long = ms.Length
    length += New FileInfo(filePath).Length
    request.ContentLength = length

    Using requestStream As IO.Stream = request.GetRequestStream()
        Dim bheader() As Byte = ms.ToArray()
        requestStream.Write(bheader, 0, bheader.Length)
        Using fileStream As New IO.FileStream(filePath, IO.FileMode.Open, IO.FileAccess.Read)

            Dim buffer(4096) As Byte
            Dim bytesRead As Int32 = fileStream.Read(buffer, 0, buffer.Length)

            Do While (bytesRead > 0)
                requestStream.Write(buffer, 0, bytesRead)
                bytesRead = fileStream.Read(buffer, 0, buffer.Length)
            Loop
        End Using
        requestStream.Close()
    End Using

这样做的全部目的是预先分配 ContentLength 属性，以便它立即开始流式传输。它不是在 GetResponseStream 上执行工作，而是在 requestStream.Write 命令上执行工作。对 MemoryStream 中的字节进行计数，然后写入，然后将文件的大小添加到 length 变量并设置为 ContentLength。这行得通吗？！

Answer 3

Dim WebClient1 As New WebClient
WebClient1.UploadFile(...blablabla...)