常见压缩算法性能和压缩率对比 LZ4 LZO ZSTD SNAPPY-海口c网

网传压缩算法对比表

算法	压缩率	压缩速度	解压速度	支持流式压缩	适用场景
LZ4	低	极快	极快	是	实时数据压缩、日志压缩、内存缓存等
Zstandard	高	快	快	是	文件压缩、网络传输、数据库备份等
Brotli	很高	中等	快	是	静态资源压缩（HTML、CSS、JS）等
LZO	低	极快	快	是	嵌入式系统、实时数据压缩
Snappy	中等	快	快	否	日志压缩、内存缓存、网络传输

性能实测

在这里插入图片描述

./compress_test syslog.txtLZ4 0 comp:   64.669 ms  decom:   15.282 ms  Size: 85.906 MB  Ratio: 17.186 %
LZ4 1 comp:   65.662 ms  decom:   15.046 ms  Size: 85.906 MB  Ratio: 17.186 %
LZO 0 comp:   65.834 ms  decom:   51.929 ms  Size: 85.906 MB  Ratio: 17.936 %
LZO 1 comp:   61.834 ms  decom:   50.462 ms  Size: 85.906 MB  Ratio: 17.936 %
Zstd 0 comp:   71.713 ms  decom:   31.642 ms  Size: 85.906 MB  Ratio: 9.230 %
Zstd 1 comp:   93.255 ms  decom:   47.457 ms  Size: 85.906 MB  Ratio: 9.230 %
Brotli 0 comp:  156.201 ms  decom:  115.879 ms  Size: 85.906 MB  Ratio: 13.189 %
Brotli 1 comp:  127.775 ms  decom:  104.258 ms  Size: 85.906 MB  Ratio: 13.189 %
Snappy 0 comp:   84.727 ms  decom:   52.499 ms  Size: 85.906 MB  Ratio: 18.220 %
Snappy 1 comp:   82.503 ms  decom:   52.898 ms  Size: 85.906 MB  Ratio: 18.220 %

对于一个系统日志文件，LZ4 LZO速度最快，压缩率zstd遥遥领先，综合性价比 Zstd完胜。

安装与使用

LZ4

编译

git clone https://github.com/lz4/lz4.git
cd lz4
make

使用

ckun@DESKTOP-PODAH1A:~/ws/compress/test$ rm -f log.lz4
ckun@DESKTOP-PODAH1A:~/ws/compress/test$ time ./lz4 syslog -T1 log.lz4
Compressed 3290808339 bytes into 567183342 bytes ==> 17.24%real    0m5.855s
user    0m4.636s
sys     0m1.910s

Zstd

git clone https://github.com/facebook/zstd.git
cd zstd
make

Brotli

git clone https://github.com/google/brotli.git
cd brotlimkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=Release ..
make

LZO

wget http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz
tar -xvzf lzo-2.10.tar.gz
cd lzo-2.10
mkdir build
cd build
cmake -DENABLE_SHARED=ON ..
make

Snappy

git clone https://github.com/google/snappy.git
cd snappy
git submodule update --init
mkdir build
cd build 
cmake ..
make

测试程序


#include <iostream>
#include <fstream>
#include <vector>
#include <chrono>
#include <cstring>
#include <lz4.h>
#include <lzo/lzo1x.h>
#include <zstd.h>
#include <brotli/encode.h>
#include <brotli/decode.h>
#include <snappy.h>#include <iomanip> 
// Helper function to calculate checksum (simple Adler-32 implementation)
uint32_t calculate_checksum(const std::vector<char>& data) {uint32_t a = 1, b = 0;for (char c : data) {a = (a + static_cast<uint8_t>(c)) % 65521;b = (b + a) % 65521;}return (b << 16) | a;
}// Read file into memory
std::vector<char> read_file(const std::string& filename) {std::ifstream file(filename, std::ios::binary | std::ios::ate);if (!file) {throw std::runtime_error("Failed to open file: " + filename);}std::streamsize size = file.tellg();file.seekg(0, std::ios::beg);std::vector<char> buffer(size);if (!file.read(buffer.data(), size)) {throw std::runtime_error("Failed to read file: " + filename);}return buffer;
}// Test LZ4 compression and decompression
void test_lz4(const std::vector<char>& input, int i) {//std::cout << "Testing LZ4..." << std::endl;// Allocate buffersint max_compressed_size = LZ4_compressBound(input.size());std::vector<char> compressed(max_compressed_size);std::vector<char> decompressed(input.size());// Compressauto start = std::chrono::high_resolution_clock::now();int compressed_size = LZ4_compress_default(input.data(), compressed.data(), input.size(), max_compressed_size);auto end = std::chrono::high_resolution_clock::now();if (compressed_size <= 0) {throw std::runtime_error("LZ4 compression failed");}auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Decompressstart = std::chrono::high_resolution_clock::now();int decompressed_size = LZ4_decompress_safe(compressed.data(), decompressed.data(), compressed_size, decompressed.size());end = std::chrono::high_resolution_clock::now();if (decompressed_size != input.size()) {throw std::runtime_error("LZ4 decompression failed");}auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Verify checksumuint32_t original_checksum = calculate_checksum(input);uint32_t decompressed_checksum = calculate_checksum(decompressed);if (original_checksum != decompressed_checksum) {throw std::runtime_error("LZ4 checksum mismatch");}// Output resultsprintf("LZ4 %d comp: %8.3lf ms  decom: %8.3lf ms  Size: %.3f MB  Ratio: %.3f %%\n",i, (compression_time / 1000.0), (decompression_time / 1000.0),(input.size() / (1024.0 * 1024.0)),(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}// Test LZO compression and decompression
void test_lzo(const std::vector<char>& input, int i) {//std::cout << "Testing LZO..." << std::endl;// Initialize LZOif (lzo_init() != LZO_E_OK) {throw std::runtime_error("LZO initialization failed");}// Allocate buffersstd::vector<char> compressed(input.size() + input.size() / 16 + 64 + 3);std::vector<char> decompressed(input.size());std::vector<unsigned char> wrkmem(LZO1X_1_MEM_COMPRESS);// Compressauto start = std::chrono::high_resolution_clock::now();lzo_uint compressed_size;int r = lzo1x_1_compress(reinterpret_cast<const unsigned char*>(input.data()), input.size(),reinterpret_cast<unsigned char*>(compressed.data()), &compressed_size, wrkmem.data());auto end = std::chrono::high_resolution_clock::now();if (r != LZO_E_OK) {throw std::runtime_error("LZO compression failed");}auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Decompressstart = std::chrono::high_resolution_clock::now();lzo_uint decompressed_size;r = lzo1x_decompress(reinterpret_cast<const unsigned char*>(compressed.data()), compressed_size,reinterpret_cast<unsigned char*>(decompressed.data()), &decompressed_size, nullptr);end = std::chrono::high_resolution_clock::now();if (r != LZO_E_OK || decompressed_size != input.size()) {throw std::runtime_error("LZO decompression failed");}auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Verify checksumuint32_t original_checksum = calculate_checksum(input);uint32_t decompressed_checksum = calculate_checksum(decompressed);if (original_checksum != decompressed_checksum) {throw std::runtime_error("LZO checksum mismatch");}// Output results// std::cout << "LZO " << i << " ";// std::cout << "comp: " << (compression_time / 1000.0) << " ms  "//           << "decom: " << (decompression_time / 1000.0) << " ms  "//           << "Size: " << (input.size() / (1024.0 * 1024.0)) << " MB  "//           << "Ratio: " << (static_cast<double>(compressed_size) / input.size()) * 100 << " %" //           << std::endl;printf("LZO %d comp: %8.3lf ms  decom: %8.3lf ms  Size: %.3f MB  Ratio: %.3f %%\n",i, (compression_time / 1000.0), (decompression_time / 1000.0),(input.size() / (1024.0 * 1024.0)),(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}void test_zstd(const std::vector<char>& input, int i) {// Allocate bufferssize_t max_compressed_size = ZSTD_compressBound(input.size());std::vector<char> compressed(max_compressed_size);std::vector<char> decompressed(input.size());// Compressauto start = std::chrono::high_resolution_clock::now();size_t compressed_size = ZSTD_compress(compressed.data(), max_compressed_size, input.data(), input.size(), 1);auto end = std::chrono::high_resolution_clock::now();if (ZSTD_isError(compressed_size)) {throw std::runtime_error("Zstd compression failed: " + std::string(ZSTD_getErrorName(compressed_size)));}auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Decompressstart = std::chrono::high_resolution_clock::now();size_t decompressed_size = ZSTD_decompress(decompressed.data(), decompressed.size(), compressed.data(), compressed_size);end = std::chrono::high_resolution_clock::now();if (ZSTD_isError(decompressed_size) || decompressed_size != input.size()) {throw std::runtime_error("Zstd decompression failed: " + std::string(ZSTD_getErrorName(decompressed_size)));}auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Verify checksumuint32_t original_checksum = calculate_checksum(input);uint32_t decompressed_checksum = calculate_checksum(decompressed);if (original_checksum != decompressed_checksum) {throw std::runtime_error("Zstd checksum mismatch");}// Output resultsprintf("Zstd %d comp: %8.3lf ms  decom: %8.3lf ms  Size: %.3f MB  Ratio: %.3f %%\n",i, (compression_time / 1000.0), (decompression_time / 1000.0),(input.size() / (1024.0 * 1024.0)),(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}void test_brotli(const std::vector<char>& input, int i) {// Allocate bufferssize_t max_compressed_size = BrotliEncoderMaxCompressedSize(input.size());std::vector<uint8_t> compressed(max_compressed_size);std::vector<uint8_t> decompressed(input.size());// Compressauto start = std::chrono::high_resolution_clock::now();size_t compressed_size = max_compressed_size;if (!BrotliEncoderCompress(0, BROTLI_DEFAULT_WINDOW, BROTLI_MODE_GENERIC,input.size(), reinterpret_cast<const uint8_t*>(input.data()),&compressed_size, compressed.data())) {throw std::runtime_error("Brotli compression failed");}auto end = std::chrono::high_resolution_clock::now();auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Decompressstart = std::chrono::high_resolution_clock::now();size_t decompressed_size = decompressed.size();if (BrotliDecoderDecompress(compressed_size, compressed.data(),&decompressed_size, decompressed.data()) != BROTLI_DECODER_RESULT_SUCCESS) {throw std::runtime_error("Brotli decompression failed");}end = std::chrono::high_resolution_clock::now();auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Verify checksumuint32_t original_checksum = calculate_checksum(input);uint32_t decompressed_checksum = calculate_checksum(std::vector<char>(decompressed.begin(), decompressed.end()));if (original_checksum != decompressed_checksum) {throw std::runtime_error("Brotli checksum mismatch");}// Output resultsprintf("Brotli %d comp: %8.3lf ms  decom: %8.3lf ms  Size: %.3f MB  Ratio: %.3f %%\n",i, (compression_time / 1000.0), (decompression_time / 1000.0),(input.size() / (1024.0 * 1024.0)),(double) (static_cast<double>(compressed_size) / input.size()) * 100);
}void test_snappy(const std::vector<char>& input, int i) {// Allocate buffersstd::string compressed;std::string decompressed;// Compressauto start = std::chrono::high_resolution_clock::now();snappy::Compress(input.data(), input.size(), &compressed);auto end = std::chrono::high_resolution_clock::now();auto compression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Decompressstart = std::chrono::high_resolution_clock::now();if (!snappy::Uncompress(compressed.data(), compressed.size(), &decompressed)) {throw std::runtime_error("Snappy decompression failed");}end = std::chrono::high_resolution_clock::now();auto decompression_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count();// Verify checksumuint32_t original_checksum = calculate_checksum(input);uint32_t decompressed_checksum = calculate_checksum(std::vector<char>(decompressed.begin(), decompressed.end()));if (original_checksum != decompressed_checksum) {throw std::runtime_error("Snappy checksum mismatch");}// Output resultsprintf("Snappy %d comp: %8.3lf ms  decom: %8.3lf ms  Size: %.3f MB  Ratio: %.3f %%\n",i, (compression_time / 1000.0), (decompression_time / 1000.0),(input.size() / (1024.0 * 1024.0)),(double) (static_cast<double>(compressed.size()) / input.size()) * 100);
}template <typename Func>
void run_test(Func func, const std::vector<char>& input, int times) {for (int i = 0; i < times; ++i) {func(input, i);}
}int main(int argc, char* argv[]) {if (argc != 2) {std::cerr << "Usage: " << argv[0] << " <file>" << std::endl;return 1;}std::cout << std::fixed << std::setprecision(3); try {// Read file into memorystd::vector<char> input = read_file(argv[1]);// Calculate checksumuint32_t checksum = calculate_checksum(input);std::cout << "Original Checksum: " << checksum << std::endl;// Test LZ4run_test(test_lz4, input, 2);// Test LZOrun_test(test_lzo, input, 2);run_test(test_zstd, input, 2);// Test Brotlirun_test(test_brotli, input, 2);// Test Snappyrun_test(test_snappy, input, 2);} catch (const std::exception& e) {std::cerr << "Error: " << e.what() << std::endl;return 1;}return 0;
}

Makefile

TARGET = compress_testSRCS = $(wildcard *.cpp)CC = g++CFLAGS += -g -O2 -I../lz4/lib -I../lzo-2.10/include -I../zstd/lib \-I../brotli/c/include -I../snappy LDFLAGS += -lyaml-cpp  -L../lz4/lib -llz4 \-L../lzo-2.10/build -llzo2 -lz \-L../zstd/lib -lzstd \-L../brotli/build -lbrotlicommon -lbrotlidec -lbrotlienc \-L../snappy/build -lsnappy
all: $(TARGET)$(TARGET): $(SRCS)$(CC) $(CFLAGS) $(SRCS) -o $@ $(LDFLAGS).PHONY: clean
clean:rm $(TARGET)

运行

export LD_LIBRARY_PATH=../lz4/lib:../lzo-2.10/build:../zstd/lib:../brotli/build/:../snappy/build$ ldd compress_testlinux-vdso.so.1 (0x00007ffdb71cf000)liblz4.so.1 => ../lz4/lib/liblz4.so.1 (0x00007fc26045a000)liblzo2.so.2 => ../lzo-2.10/build/liblzo2.so.2 (0x00007fc260427000)libzstd.so.1 => ../zstd/lib/libzstd.so.1 (0x00007fc260330000)libbrotlidec.so.1 => ../brotli/build/libbrotlidec.so.1 (0x00007fc260321000)libbrotlienc.so.1 => ../brotli/build/libbrotlienc.so.1 (0x00007fc260268000)libsnappy.so.1 => ../snappy/build/libsnappy.so.1 (0x00007fc260259000)libstdc++.so.6 => /lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007fc260026000)libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007fc260006000)libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007fc25fddd000)libbrotlicommon.so.1 => ../brotli/build/libbrotlicommon.so.1 (0x00007fc25fdb7000)libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007fc25fcd0000)/lib64/ld-linux-x86-64.so.2 (0x00007fc260498000)./compress_test file