Read a very large file by chunks and not line-by-line

c#, file

Solution

The problem with your solution is that you recreate the streams in each iteration. Try this version:

const int MAX_BUFFER = 33554432; //32MB
byte[] buffer = new byte[MAX_BUFFER];
int bytesRead;
StringBuilder currentLine = new StringBuilder();

using (FileStream fs = File.Open(filePath, FileMode.Open, FileAccess.Read))
using (BufferedStream bs = new BufferedStream(fs))
{
    string line;
    bool stop = false;
    var memoryStream = new MemoryStream(buffer);
    var stream = new StreamReader(memoryStream);
    while ((bytesRead = bs.Read(buffer, 0, MAX_BUFFER)) != 0)
    {
        memoryStream.Seek(0, SeekOrigin.Begin);

        while (!stream.EndOfStream)
        {
            line = ReadLineWithAccumulation(stream, currentLine);

            if (line != null)
            {
                //process line
            }
        }
    }
}

private string ReadLineWithAccumulation(StreamReader stream, StringBuilder currentLine)
{
    while (stream.Read(buffer, 0, 1) > 0)
    {
        if (charBuffer [0].Equals('\n'))
        {
            string result = currentLine.ToString();
            currentLine.Clear();

            if (result.Last() == '\r') //remove if newlines are single character
            {
                result = result.Substring(0, result.Length - 1);
            }

            return result;
        }
        else
        {
            currentLine.Append(charBuffer [0]);
        }
    }

    return null;  //line not complete yet
}

private char[] charBuffer = new char[1];

NOTE: This needs some tweaking if newlines are two characters long and you need newline characters to be contained in the result. The worst case would be newline pair `"\r\n"` split across two blocks. However since you were using `ReadLine` I assumed that you don't need this.

Also, the problem is that in case your whole data contains only one line, this will end up in an attempt to read the whole data into memory anyway.

Problem

I want to read a CSV file which can be at a size of hundreds of GBs and even TB. I got a limitation that I can only read the file in chunks of 32MB. My solution to the problem, not only does it work kinda slow, but it can also break a line in the middle of it. I wanted to ask if you know of a better solution: ``` const int MAX_BUFFER = 33554432; //32MB byte[] buffer = new byte[MAX_BUFFER]; int bytesRead; using (FileStream fs = File.Open(filePath, FileMode.Open, FileAccess.Read)) using (BufferedStream bs = new BufferedStream(fs)) { string line; bool stop = false; while ((bytesRead = bs.Read(buffer, 0, MAX_BUFFER)) != 0) //reading only 32mb chunks at a time { var stream = new StreamReader(new MemoryStream(buffer)); while ((line = stream.ReadLine()) != null) { //process line } } } ``` Please do not respond with a solution which reads the file line by line (for example `File.ReadLines` is NOT an acceptable solution). Why? Because I'm just searching for another solution...

Original source