From a95a21bc2b253b412d571bebb8f4f9d9a34f38ba Mon Sep 17 00:00:00 2001 From: Samuel Gomes Date: Sat, 22 Jun 2024 11:03:56 +0530 Subject: [PATCH 1/2] Handle unusual cases and optimize func__deflate() & func__inflate() --- Makefile | 2 - internal/c/parts/compression/build.mk | 9 ++- internal/c/parts/compression/compression.cpp | 68 ++++++++++---------- 3 files changed, 43 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 8d3887bfa..c88d0cb34 100644 --- a/Makefile +++ b/Makefile @@ -324,8 +324,6 @@ else endif ifneq ($(filter y,$(DEP_ZLIB)),) - EXE_LIBS += $(MINIZ_OBJS) $(COMPRESSION_OBJS) - LICENSE_IN_USE += miniz endif diff --git a/internal/c/parts/compression/build.mk b/internal/c/parts/compression/build.mk index 192c54c46..81c6020bb 100644 --- a/internal/c/parts/compression/build.mk +++ b/internal/c/parts/compression/build.mk @@ -15,4 +15,11 @@ $(PATH_INTERNAL_C)/parts/compression/%.o: $(PATH_INTERNAL_C)/parts/compression/% $(PATH_INTERNAL_C)/parts/compression/%.o: $(PATH_INTERNAL_C)/parts/compression/%.cpp $(CXX) -O2 $(CXXFLAGS) -DDEPENDENCY_CONSOLE_ONLY -Wall $< -c -o $@ -CLEAN_LIST += $(MINIZ_OBJS) $(COMPRESSION_OBJS) +COMPRESSION_LIB := $(PATH_INTERNAL_C)/parts/compression/compression.a + +$(COMPRESSION_LIB): $(MINIZ_OBJS) $(COMPRESSION_OBJS) + $(AR) rcs $@ $(MINIZ_OBJS) $(COMPRESSION_OBJS) + +EXE_LIBS += $(COMPRESSION_LIB) + +CLEAN_LIST += $(COMPRESSION_LIB) $(MINIZ_OBJS) $(COMPRESSION_OBJS) diff --git a/internal/c/parts/compression/compression.cpp b/internal/c/parts/compression/compression.cpp index 974f786f0..55f8784ec 100644 --- a/internal/c/parts/compression/compression.cpp +++ b/internal/c/parts/compression/compression.cpp @@ -7,52 +7,54 @@ #include "compression.h" #include "qbs.h" + #include "miniz.h" +#include + uint32_t func__adler32(qbs *text) { - if (!text->len) return 1; - return (uint32_t) adler32(1, text->chr, text->len); + if (!text->len) + return 1; + return (uint32_t)adler32(1, text->chr, text->len); } uint32_t func__crc32(qbs *text) { - if (!text->len) return 0; - return (uint32_t) crc32(0, text->chr, text->len); + if (!text->len) + return 0; + return (uint32_t)crc32(0, text->chr, text->len); } qbs *func__deflate(qbs *text) { - uLongf filesize = (uint32_t)text->len; // length of the text - uLongf compsize = compressBound(filesize); - unsigned char *dest = (unsigned char *)malloc(compsize); - int32_t result = compress(dest, &compsize, text->chr, filesize); - qbs *ret = qbs_new(compsize, 1); - memcpy(ret->chr, dest, compsize); - free(dest); - return ret; + auto fileSize = uLongf(text->len); + auto compSize = compressBound(fileSize); + auto dest = qbs_new(compSize, 1); // compressing directly to the qbs gives us a performance boost + compress(dest->chr, &compSize, text->chr, fileSize); // discard result because we do not do any error checking + qbs_left(dest, compSize); // discard the return value because it will return the same tmp qbs anyway + return dest; } -qbs *func__inflate(qbs *text, int64_t originalsize, int32_t passed) { - int32_t result = 0; - if (passed == 1) { - uLongf uncompsize = originalsize; - unsigned char *dest = (unsigned char *)malloc(originalsize); - int32_t result = uncompress(dest, &uncompsize, text->chr, text->len); - qbs *ret = qbs_new(uncompsize, 1); - memcpy(ret->chr, dest, uncompsize); - free(dest); - return ret; +qbs *func__inflate(qbs *text, int64_t originalSize, int32_t passed) { + if (passed) { + // Passing negative values can do bad things to qbs + if (originalSize > 0) { + auto uncompSize = uLongf(originalSize); + auto dest = qbs_new(uncompSize, 1); // decompressing directly to the qbs gives us a performance boost + uncompress(dest->chr, &uncompSize, text->chr, uLongf(text->len)); // discard result because we do not do any error checking + return dest; // no size adjustment is done assuming the exact original size was passed + } else { + return qbs_new(0, 1); // simply return an empty qbs if originalSize is zero or negative + } } else { - uLongf uncompsize = 0; - unsigned char *dest; + static const uLongf InflateChunkSize = 10 * 1024 * 1024; + std::vector dest; // to get rid of malloc() and free() + auto compSize = uLongf(text->len); + uLongf uncompSize = 0; do { - uncompsize = uncompsize + 10000000; // 10 mb original buffer, resized by 10 mb each pass until it's large enough to hold the uncompressed data. - dest = (unsigned char *)malloc(uncompsize); - result = uncompress(dest, &uncompsize, text->chr, text->len); - if (result == Z_BUF_ERROR) - free(dest); // if the buffer is too small, free the old buffer - } while (result == Z_BUF_ERROR); // and try again with a larger buffer - qbs *ret = qbs_new(uncompsize, 1); - memcpy(ret->chr, dest, uncompsize); - free(dest); + uncompSize += InflateChunkSize; // 10 mb original buffer, resized by 10 mb each pass until it's large enough to hold the uncompressed data. + dest.resize(uncompSize); + } while (uncompress(&dest[0], &uncompSize, text->chr, compSize) == Z_BUF_ERROR); // and try again with a larger buffer + auto ret = qbs_new(uncompSize, 1); + memcpy(ret->chr, &dest[0], uncompSize); return ret; } } From 6f0b8fe11ca18458fef7c8908616de64bdc1cc41 Mon Sep 17 00:00:00 2001 From: Samuel Gomes Date: Sat, 22 Jun 2024 12:34:20 +0530 Subject: [PATCH 2/2] Return value from qbs_left() --- internal/c/parts/compression/compression.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/c/parts/compression/compression.cpp b/internal/c/parts/compression/compression.cpp index 55f8784ec..e23f654a1 100644 --- a/internal/c/parts/compression/compression.cpp +++ b/internal/c/parts/compression/compression.cpp @@ -29,8 +29,7 @@ qbs *func__deflate(qbs *text) { auto compSize = compressBound(fileSize); auto dest = qbs_new(compSize, 1); // compressing directly to the qbs gives us a performance boost compress(dest->chr, &compSize, text->chr, fileSize); // discard result because we do not do any error checking - qbs_left(dest, compSize); // discard the return value because it will return the same tmp qbs anyway - return dest; + return qbs_left(dest, compSize); } qbs *func__inflate(qbs *text, int64_t originalSize, int32_t passed) {