Skip to content

Commit

Permalink
Merge pull request #2559 from senhuang42/add_dict_regression_tests_ba…
Browse files Browse the repository at this point in the history 
…ckup

Add different dict modes to compression ratio regression test, update results.csv
  • Loading branch information
@senhuang42
senhuang42 authored Mar 25, 2021
2 parents 4fe2e7a + bbbd578 commit ab216bc
Show file tree
Hide file tree
Showing 7 changed files with 435 additions and 104 deletions.
2 changes: 1 addition & 1 deletion programs/util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -983,7 +983,7 @@ void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nb
}

FileNamesTable*
UTIL_createExpandedFNT(const char** inputNames, size_t nbIfns, int followLinks)
UTIL_createExpandedFNT(const char* const* inputNames, size_t nbIfns, int followLinks)
{
unsigned nbFiles;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
Expand Down
2 changes: 1 addition & 1 deletion programs/util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ void UTIL_refFilename(FileNamesTable* fnt, const char* filename);
* or NULL in case of error
*/
FileNamesTable*
UTIL_createExpandedFNT(const char** filenames, size_t nbFilenames, int followLinks);
UTIL_createExpandedFNT(const char* const* filenames, size_t nbFilenames, int followLinks);


/*-****************************************
Expand Down
78 changes: 63 additions & 15 deletions tests/regression/config.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,20 +28,68 @@
};

/* Define a config for each level we want to test with. */
#define LEVEL(x) \
param_value_t const level_##x##_param_values[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
}; \
config_t const level_##x = { \
.name = "level " #x, \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values), \
}; \
config_t const level_##x##_dict = { \
.name = "level " #x " with dict", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values), \
.use_dictionary = 1, \
#define LEVEL(x) \
param_value_t const level_##x##_param_values[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
}; \
param_value_t const level_##x##_param_values_dms[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \
}; \
param_value_t const level_##x##_param_values_dds[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 1}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceAttach}, \
}; \
param_value_t const level_##x##_param_values_dictcopy[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceCopy}, \
}; \
param_value_t const level_##x##_param_values_dictload[] = { \
{.param = ZSTD_c_compressionLevel, .value = x}, \
{.param = ZSTD_c_enableDedicatedDictSearch, .value = 0}, \
{.param = ZSTD_c_forceAttachDict, .value = ZSTD_dictForceLoad}, \
}; \
config_t const level_##x = { \
.name = "level " #x, \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values), \
}; \
config_t const level_##x##_dict = { \
.name = "level " #x " with dict", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values), \
.use_dictionary = 1, \
}; \
config_t const level_##x##_dict_dms = { \
.name = "level " #x " with dict dms", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values_dms), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
}; \
config_t const level_##x##_dict_dds = { \
.name = "level " #x " with dict dds", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values_dds), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
}; \
config_t const level_##x##_dict_copy = { \
.name = "level " #x " with dict copy", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values_dictcopy), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
}; \
config_t const level_##x##_dict_load = { \
.name = "level " #x " with dict load", \
.cli_args = "-" #x, \
.param_values = PARAM_VALUES(level_##x##_param_values_dictload), \
.use_dictionary = 1, \
.advanced_api_only = 1, \
};

#define PARAM_VALUES(pv) \
Expand Down Expand Up @@ -194,7 +242,7 @@ static config_t explicit_params = {
static config_t const* g_configs[] = {

#define FAST_LEVEL(x) &level_fast##x, &level_fast##x##_dict,
#define LEVEL(x) &level_##x, &level_##x##_dict,
#define LEVEL(x) &level_##x, &level_##x##_dict, &level_##x##_dict_dms, &level_##x##_dict_dds, &level_##x##_dict_copy, &level_##x##_dict_load,
#include "levels.h"
#undef LEVEL
#undef FAST_LEVEL
Expand Down
5 changes: 5 additions & 0 deletions tests/regression/config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ typedef struct {
* when the method allows it. Defaults to yes.
*/
int no_pledged_src_size;
/**
* Boolean parameter that says that this config should only be used
* for methods that use the advanced compression API
*/
int advanced_api_only;
} config_t;

/**
Expand Down
1 change: 1 addition & 0 deletions tests/regression/data.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h> /* free() */

#include <sys/stat.h>

Expand Down
15 changes: 14 additions & 1 deletion tests/regression/method.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@ static result_t simple_compress(method_state_t* base, config_t const* config) {
*/
if (base->data->type != data_type_file)
return result_error(result_error_skip);

if (config->advanced_api_only)
return result_error(result_error_skip);

if (config->use_dictionary || config->no_pledged_src_size)
return result_error(result_error_skip);
Expand Down Expand Up @@ -151,6 +154,9 @@ static result_t compress_cctx_compress(

if (base->data->type != data_type_dir)
return result_error(result_error_skip);

if (config->advanced_api_only)
return result_error(result_error_skip);

int const level = config_get_level(config);

Expand Down Expand Up @@ -254,6 +260,9 @@ static result_t cli_compress(method_state_t* state, config_t const* config) {
if (config->cli_args == NULL)
return result_error(result_error_skip);

if (config->advanced_api_only)
return result_error(result_error_skip);

/* We don't support no pledged source size with directories. Too slow. */
if (state->data->type == data_type_dir && config->no_pledged_src_size)
return result_error(result_error_skip);
Expand Down Expand Up @@ -523,6 +532,10 @@ static result_t old_streaming_compress_internal(
result = result_error(result_error_skip);
goto out;
}
if (config->advanced_api_only) {
result = result_error(result_error_skip);
goto out;
}
if (init_cstream(state, zcs, config, advanced, cdict ? &cd : NULL)) {
result = result_error(result_error_compression_error);
goto out;
Expand Down Expand Up @@ -651,7 +664,7 @@ method_t const old_streaming_advanced = {
};

method_t const old_streaming_cdict = {
.name = "old streaming cdcit",
.name = "old streaming cdict",
.create = buffer_state_create,
.compress = old_streaming_compress_cdict,
.destroy = buffer_state_destroy,
Expand Down
Loading

0 comments on commit ab216bc

Please sign in to comment.