Fix public issue #26: Take memory allocation and reallocation entirely out of the
Measure() loop. This gives all algorithms a small speed boost, except Snappy which
already didn't do reallocation (so the measurements were slightly biased in its
favor).
R=csilvers
DELTA=92 (69 added, 9 deleted, 14 changed)
Revision created by MOE tool push_codebase.
MOE_MIGRATION=1151
git-svn-id: https://snappy.googlecode.com/svn/trunk@24 03e5f5b5-db94-4691-08a0-1a8bf15f6143
diff --git a/snappy_unittest.cc b/snappy_unittest.cc
index e2bcf28..b2de246 100644
--- a/snappy_unittest.cc
+++ b/snappy_unittest.cc
@@ -128,13 +128,58 @@
"ZLIB", "LZO", "LIBLZF", "QUICKLZ", "FASTLZ", "SNAPPY",
};
-// Returns true if we successfully compressed, false otherwise
+static size_t MinimumRequiredOutputSpace(size_t input_size,
+ CompressorType comp) {
+ switch (comp) {
+#ifdef ZLIB_VERSION
+ case ZLIB:
+ return ZLib::MinCompressbufSize(input_size);
+#endif // ZLIB_VERSION
+
+#ifdef LZO_VERSION
+ case LZO:
+ return input_size + input_size/64 + 16 + 3;
+#endif // LZO_VERSION
+
+#ifdef LZF_VERSION
+ case LIBLZF:
+ return input_size;
+#endif // LZF_VERSION
+
+#ifdef QLZ_VERSION_MAJOR
+ case QUICKLZ:
+ return input_size + 36000; // 36000 is used for scratch.
+#endif // QLZ_VERSION_MAJOR
+
+#ifdef FASTLZ_VERSION
+ case FASTLZ:
+ return max(static_cast<int>(ceil(input_size * 1.05)), 66);
+#endif // FASTLZ_VERSION
+
+ case SNAPPY:
+ return snappy::MaxCompressedLength(input_size);
+
+ default:
+ LOG(FATAL) << "Unknown compression type number " << comp;
+ }
+}
+
+// Returns true if we successfully compressed, false otherwise.
+//
+// If compressed_is_preallocated is set, do not resize the compressed buffer.
+// This is typically what you want for a benchmark, in order to not spend
+// time in the memory allocator. If you do set this flag, however,
+// "compressed" must be preinitialized to at least MinCompressbufSize(comp)
+// number of bytes, and may contain junk bytes at the end after return.
static bool Compress(const char* input, size_t input_size, CompressorType comp,
- string* compressed) {
+ string* compressed, bool compressed_is_preallocated) {
+ if (!compressed_is_preallocated) {
+ compressed->resize(MinimumRequiredOutputSpace(input_size, comp));
+ }
+
switch (comp) {
#ifdef ZLIB_VERSION
case ZLIB: {
- compressed->resize(ZLib::MinCompressbufSize(input_size));
ZLib zlib;
uLongf destlen = compressed->size();
int ret = zlib.Compress(
@@ -143,14 +188,15 @@
reinterpret_cast<const Bytef*>(input),
input_size);
CHECK_EQ(Z_OK, ret);
- compressed->resize(destlen);
+ if (!compressed_is_preallocated) {
+ compressed->resize(destlen);
+ }
return true;
}
#endif // ZLIB_VERSION
#ifdef LZO_VERSION
case LZO: {
- compressed->resize(input_size + input_size/64 + 16 + 3);
unsigned char* mem = new unsigned char[LZO1X_1_15_MEM_COMPRESS];
lzo_uint destlen;
int ret = lzo1x_1_15_compress(
@@ -161,18 +207,19 @@
mem);
CHECK_EQ(LZO_E_OK, ret);
delete[] mem;
- compressed->resize(destlen);
+ if (!compressed_is_preallocated) {
+ compressed->resize(destlen);
+ }
break;
}
#endif // LZO_VERSION
#ifdef LZF_VERSION
case LIBLZF: {
- compressed->resize(input_size - 1);
int destlen = lzf_compress(input,
input_size,
string_as_array(compressed),
- input_size - 1);
+ input_size);
if (destlen == 0) {
// lzf *can* cause lots of blowup when compressing, so they
// recommend to limit outsize to insize, and just not compress
@@ -180,14 +227,15 @@
compressed->assign(input, input_size);
destlen = input_size;
}
- compressed->resize(destlen);
+ if (!compressed_is_preallocated) {
+ compressed->resize(destlen);
+ }
break;
}
#endif // LZF_VERSION
#ifdef QLZ_VERSION_MAJOR
case QUICKLZ: {
- compressed->resize(input_size + 36000); // 36000 is used for scratch
qlz_state_compress *state_compress = new qlz_state_compress;
int destlen = qlz_compress(input,
string_as_array(compressed),
@@ -195,23 +243,24 @@
state_compress);
delete state_compress;
CHECK_NE(0, destlen);
- compressed->resize(destlen);
+ if (!compressed_is_preallocated) {
+ compressed->resize(destlen);
+ }
break;
}
#endif // QLZ_VERSION_MAJOR
#ifdef FASTLZ_VERSION
case FASTLZ: {
- int compressed_size = max(static_cast<int>(ceil(input_size * 1.05)),
- 66);
- compressed->resize(compressed_size);
// Use level 1 compression since we mostly care about speed.
int destlen = fastlz_compress_level(
1,
input,
input_size,
string_as_array(compressed));
- compressed->resize(destlen);
+ if (!compressed_is_preallocated) {
+ compressed->resize(destlen);
+ }
CHECK_NE(destlen, 0);
break;
}
@@ -223,7 +272,9 @@
string_as_array(compressed),
&destlen);
CHECK_LE(destlen, snappy::MaxCompressedLength(input_size));
- compressed->resize(destlen);
+ if (!compressed_is_preallocated) {
+ compressed->resize(destlen);
+ }
break;
}
@@ -351,13 +402,12 @@
input[b] = data+input_start;
input_length[b] = input_limit-input_start;
- // Pre-grow the output buffer so we don't measure stupid string
- // append time
- compressed[b].resize(block_size * 2);
+ // Pre-grow the output buffer so we don't measure string append time.
+ compressed[b].resize(MinimumRequiredOutputSpace(block_size, comp));
}
// First, try one trial compression to make sure the code is compiled in
- if (!Compress(input[0], input_length[0], comp, &compressed[0])) {
+ if (!Compress(input[0], input_length[0], comp, &compressed[0], true)) {
LOG(WARNING) << "Skipping " << names[comp] << ": "
<< "library not compiled in";
return;
@@ -366,12 +416,23 @@
for (int run = 0; run < kRuns; run++) {
CycleTimer ctimer, utimer;
+ for (int b = 0; b < num_blocks; b++) {
+ // Pre-grow the output buffer so we don't measure string append time.
+ compressed[b].resize(MinimumRequiredOutputSpace(block_size, comp));
+ }
+
ctimer.Start();
for (int b = 0; b < num_blocks; b++)
for (int i = 0; i < repeats; i++)
- Compress(input[b], input_length[b], comp, &compressed[b]);
+ Compress(input[b], input_length[b], comp, &compressed[b], true);
ctimer.Stop();
+ // Compress once more, with resizing, so we don't leave junk
+ // at the end that will confuse the decompressor.
+ for (int b = 0; b < num_blocks; b++) {
+ Compress(input[b], input_length[b], comp, &compressed[b], false);
+ }
+
for (int b = 0; b < num_blocks; b++) {
output[b].resize(input_length[b]);
}
@@ -911,8 +972,7 @@
File::ReadFileToStringOrDie(fname, &fullinput);
string compressed;
- compressed.resize(fullinput.size() * 2);
- Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed);
+ Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false);
File::WriteStringToFileOrDie(compressed,
string(fname).append(".comp").c_str());
