Using various flags in huffman only and deflate compresssion/decompression

[rohitpreddy07]

#include <fstream>
#include <memory>
#include <vector>

#include "qpl/qpl.h"

constexpr size_t MB = 1024 * 1024;
constexpr size_t MAX_FILE_SIZE = 100 * MB; // 100 MB in bytes

auto main(int argc, char* argv[]) -> int {
    if (argc != 2) {
        std::cerr << "Usage: " << argv[0] << " <input_file.bin>\n";
        return 1;
    }

    const char* input_file = argv[1];

    // Open input file
    std::ifstream file(input_file, std::ios::binary | std::ios::ate);
    if (!file) {
        std::cerr << "Error opening file: " << input_file << "\n";
        return 1;
    }

    // Get file size
    size_t file_size = file.tellg();
    if (file_size > MAX_FILE_SIZE) {
        std::cerr << "File size exceeds maximum allowed size of 100 MB.\n";
        return 1;
    }
    file.seekg(0, std::ios::beg);

    // Read file into source vector
    std::vector<uint8_t> source(file_size);
    if (!file.read(reinterpret_cast<char*>(source.data()), file_size)) {
        std::cerr << "Error reading file: " << input_file << "\n";
        return 1;
    }
    file.close();

    // Prepare destination and reference vectors
    std::vector<uint8_t> destination(file_size);
    std::vector<uint8_t> reference(file_size);

    std::unique_ptr<uint8_t[]> job_buffer;
    uint32_t size = 0;

    // Job initialization
    qpl_status status = qpl_get_job_size(qpl_path_software, &size);
    if (status != QPL_STS_OK) {
        std::cout << "An error " << status << " occurred during job size getting.\n";
        return 1;
    }

    job_buffer   = std::make_unique<uint8_t[]>(size);
    qpl_job* job = reinterpret_cast<qpl_job*>(job_buffer.get());

    status = qpl_init_job(qpl_path_software, job);
    if (status != QPL_STS_OK) {
        std::cout << "An error " << status << " occurred during job initializing.\n";
        return 1;
    }

    // Performing a compression operation
    job->op            = qpl_op_compress;
    job->level         = qpl_default_level;
    job->next_in_ptr   = source.data();
    job->next_out_ptr  = destination.data();
    job->available_in  = static_cast<uint32_t>(source.size());
    job->available_out = static_cast<uint32_t>(destination.size());
    job->flags         = QPL_FLAG_FIRST | QPL_FLAG_LAST | QPL_FLAG_OMIT_VERIFY;
    job->huffman_table = NULL;

    // Compression
    status = qpl_execute_job(job);
    if (status != QPL_STS_OK) {
        std::cout << "An error " << status << " occurred during compression.\n";
        return 1;
    }

    const uint32_t compressed_size = job->total_out;

    // Performing a decompression operation
    job->op            = qpl_op_decompress;
    job->next_in_ptr   = destination.data();
    job->next_out_ptr  = reference.data();
    job->available_in  = compressed_size;
    job->available_out = static_cast<uint32_t>(reference.size());
    job->flags         = QPL_FLAG_FIRST | QPL_FLAG_LAST;
    job->huffman_table = NULL;

    // Decompression
    status = qpl_execute_job(job);
    if (status != QPL_STS_OK) {
        std::cout << "An error " << status << " occurred during decompression.\n";
        return 1;
    }

    // Freeing resources
    status = qpl_fini_job(job);
    if (status != QPL_STS_OK) {
        std::cout << "An error " << status << " occurred during job finalization.\n";
        return 1;
    }

    // Compare reference functions
    for (size_t i = 0; i < source.size(); i++) {
        if (source[i] != reference[i]) {
            std::cout << "Content wasn't successfully compressed and decompressed.\n";
            return 1;
        }
    }

    std::cout << "Content was successfully compressed and decompressed.\n";
    std::cout << "Input size: " << source.size() << " bytes (" << source.size() / MB << " MB)"
              << ", compressed size: " << compressed_size << " bytes (" << compressed_size / MB << " MB)"
              << ", compression ratio: " << (float)source.size() / (float)compressed_size << ".\n";

    return 0;
}

The following code reads a file input of binaries and does the compression/decompression similar to the deflate compression/decompression example. When I run the code without the QPL_FLAG_DYNAMIC_HUFFMAN flag II get error code 217. How can I run a fixed block encoding and decoding program as I seem to be using the right combination of flags.

[abdelrahim-hentabli]

Hey @rohitpreddy07 , could you share the binary file that you are using for this? The compression/decompression job seems to be set up correctly, I wasn't able to reproduce this issue (Error Code 217). Could you also share which version of QPL you are using?

[rohitpreddy07]

Hey @rohitpreddy07 , could you share the binary file that you are using for this? The compression/decompression job seems to be set up correctly, I wasn't able to reproduce this issue (Error Code 217). Could you also share which version of QPL you are using?

I am using QPL Version 1.6.0. Attached is the file that I am trying to execute the job on.
model.layers.0.input_layernorm.weight[4096].zip

[abdelrahim-hentabli]

I tried to run your example, with your input. the only issue I encountered was that the file was not compressible, and as a result the destination size was too small.

The compressed size of the binary was 8197 while the non compressed size was 8192. Once, I extended the destination size, the reproducer didn't generate the 217 error code.

[rohitpreddy07]

Thanks for the help @abdelrahim-hentabli . I was facing this issue for larger files of around 80MB but once I increased the destination size like you said it seems to have fixed the issue.

[mzhukova]

Closing as resolved, please let us know if you would encounter any other issues!