lz4 compression c++ example

c++, compression

Solution

All working now thanks, here is the code for anyone that is interested

#include <fstream>
#include <iostream>

#include "lz4.h"

using namespace std;

int main()
{
char szSource[] = "2013-01-07 00:00:04,0.98644,0.98676 2013-01-07 00:01:19,0.98654,0.98676 2013-01-07 00:01:38,0.98644,0.98696";
int nInputSize = sizeof(szSource);

// compress szSource into pchCompressed
char* pchCompressed = new char[nInputSize];
int nCompressedSize = LZ4_compress((const char *)(&szSource), pchCompressed, nInputSize);

// write pachCompressed to binary lz4.dat
ofstream outBinaryFile("lz4.dat",ofstream::binary);
outBinaryFile.write(pchCompressed, nCompressedSize);
outBinaryFile.close();
delete[] pchCompressed;
pchCompressed = 0;

//read compressed binary file (assume we pass/encode nInputSize but don't know nCompressedSize)
ifstream infCompressedBinaryFile( "lz4.dat", ifstream::binary );

//Get compressed file size for buffer
infCompressedBinaryFile.seekg (0,infCompressedBinaryFile.end);
int nCompressedInputSize = infCompressedBinaryFile.tellg();
infCompressedBinaryFile.clear();
infCompressedBinaryFile.seekg(0,ios::beg);

//Read file into buffer
char* pchCompressedInput = new char[nCompressedInputSize];
infCompressedBinaryFile.read(pchCompressedInput,nCompressedSize);
infCompressedBinaryFile.close();

// Decompress buffer
char* pchDeCompressed = new char[nInputSize]; //(nCompressedInputSize *2) +8
LZ4_uncompress(pchCompressedInput, pchDeCompressed, nInputSize);
delete[] pchCompressedInput;
pchCompressedInput = 0;

// write decompressed pachUnCompressed to
ofstream outFile("lz4.txt");
outFile.write(pchDeCompressed, nInputSize);
outFile.close();

delete[] pchDeCompressed;
pchDeCompressed = 0;

return 0;
}

I am also working on a a simple CLI csv to binary I/O example here

Problem

In the process of writing a lz4 csv to compressed binary file converter (high volume forex tick data csv) in the hope of reducing the storage/disk bandwidth requirements on my tiny vps. self contained code to illustrate ``` #include <string> #include <fstream> #include <iostream> #include "lz4.h" using namespace std; int main() { char szString[] = "2013-01-07 00:00:04,0.98644,0.98676 2013-01-07 00:01:19,0.98654,0.98676 2013-01-07 00:01:38,0.98644,0.98696"; const char* pchSource = szString; int nInputSize = sizeof(szString); cout <<"- pchSource -" << endl << pchSource << endl; cout <<"nbytes = "<< nInputSize << endl << endl; ofstream source("pchSource.txt"); source << pchSource; int nbytesPassed = 0; int nMaxCompressedSize = LZ4_compressBound(nInputSize); char *pszDest = new char[nMaxCompressedSize]; nbytesPassed = LZ4_compress(pchSource, pszDest, nInputSize); cout <<"- pszDest Compressed-" << endl; cout <<"nbytesPassed = "<< nbytesPassed << endl; cout << pszDest << endl << endl; // pszDest garbage ? char *pszDestUnCompressed = new char[nInputSize]; LZ4_uncompress(pszDest, pszDestUnCompressed, nInputSize); cout <<"- pszDestUnCompressed -" << endl; cout <<"nbytesPassed = "<< nbytesPassed << endl; cout << pszDestUnCompressed << endl << endl; //pszDestUnCompressed is correct ? delete[] pszDestUnCompressed; pszDestUnCompressed = 0; // ok lets write compressed pszDest to pszDest.dat ofstream outCompressedFile("pszDest.dat",std::ofstream::binary); outCompressedFile.write (pszDest,nMaxCompressedSize); delete[] pszDest; pszDest = 0; //read it back in and try to uncompress it ifstream infile("pszDest.dat",std::ifstream::binary); infile.seekg (0,infile.end); int nCompressedInputSize = infile.tellg(); infile.seekg (0); char* buffer = new char[nCompressedInputSize]; infile.read (buffer,nCompressedInputSize); const char* pchbuffer = buffer; char* pszUnCompressedFile = new char[nInputSize]; nbytesPassed = LZ4_uncompress(pchbuffer, pszUnCompressedFile, nInputSize); cout <<"- pszUnCompressedFile -" << endl; cout <<"nbytesPassed = "<< nbytesPassed << endl; cout << pszUnCompressedFile << endl; //write uncompressed pszDest.dat to pszUnCompressedFile.txt ofstream outUncompressedSource("pszUnCompressedFile.txt"); outUncompressedSource << pszUnCompressedFile; // On my system 32bit ArchLinux 3.7.10-1 - gcc 4.7.2-4 // file contains random Garbage delete[] buffer; buffer = 0; delete[] pszUnCompressedFile; pszUnCompressedFile = 0; return 0; } CONSOLE OUTPUT : - pchSource - 2013-01-07 00:00:04,0.98644 ..... nbytes = 108 - pszDest Compressed- nbytesPassed = 63 �2013-01-07 00: - pszDestUnCompressed - nbytesPassed = 63 2013-01-07 00:00:04,0.98644 ..... - pszUnCompressedFile - nbytesPassed = -17 �W��W�-07 q Process returned 0 (0x0) execution time : 0.010 s Press ENTER to continue. ``` I'm obviously missing something, apart form the samples included in the source are there any-other usage examples ?

Original source