Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FFS upstream #2820

Merged
merged 2 commits into from
Aug 10, 2021
Merged

FFS upstream #2820

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d3a5b2e
ffs 2021-08-10 (2887fb48)
Aug 10, 2021
7b2e439
Merge branch 'upstream-ffs' into FFSup
eisenhauer Aug 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 52 additions & 24 deletions thirdparty/ffs/ffs/ffs/ffs_conv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,37 @@ create_default_conversion(FMField iofield, void *default_value,
conv_ptr->conversions[conv_index].rc_swap = no_row_column_swap;
}

/*
* differs from find_field in fm in that it includes cur_field in search shortcut
*/
static long
find_field_for_conv(char *field_name, FMFieldList fields, int cur_field, void *search_help)
{
int i;
if (cur_field > 10) {
/* search close first */
for (i = cur_field; i > cur_field - 10; i--) {
if (strcmp(field_name, fields[i].field_name) == 0) {
return i;
}
}
for (i = cur_field + 1; i < cur_field + 10; i++) {
if (fields[i].field_name == NULL) break;
if (strcmp(field_name, fields[i].field_name) == 0) {
return i;
}
}
}
i = 0;
while (fields[i].field_name != NULL) {
if (strcmp(field_name, fields[i].field_name) == 0) {
return i;
}
i++;
}
return -1;
}

static
IOConversionPtr
create_conversion(src_ioformat, target_field_list, target_struct_size,
Expand Down Expand Up @@ -363,33 +394,30 @@ FMStructDescList target_list;
field_name_strip_get_default(&nfl_sort[i], tmp_field_name, &default_val);
search_name = tmp_field_name;
}
while (strcmp(search_name,
input_field_list[input_index].field_name) != 0) {
input_index++;
if (input_index >= src_ioformat->body->field_count) {
if(default_val){
if ((conv == buffer_and_convert) ||
(conv == copy_dynamic_portion)) {
input_index = -1; /* Basically invalidating input_index
Indication for using default_val */
break;
} else {
if (default_val) {
free(default_val);
default_val = NULL;
}
conv = buffer_and_convert;
if (tmp_field_name) free(tmp_field_name);
goto restart;
input_index = find_field_for_conv(search_name, input_field_list, i, NULL);
if (input_index == -1) {
if(default_val){
if ((conv == buffer_and_convert) ||
(conv == copy_dynamic_portion)) {
input_index = -1; /* Basically invalidating input_index
Indication for using default_val */
break;
} else {
if (default_val) {
free(default_val);
default_val = NULL;
}
conv = buffer_and_convert;
if (tmp_field_name) free(tmp_field_name);
goto restart;
}
fprintf(stderr,
"Requested field %s missing from input format\n",
nfl_sort[i].field_name);
FFSfree_conversion(conv_ptr);
if (tmp_field_name) free(tmp_field_name);
return NULL;
}
fprintf(stderr,
"Requested field %s missing from input format\n",
nfl_sort[i].field_name);
FFSfree_conversion(conv_ptr);
if (tmp_field_name) free(tmp_field_name);
return NULL;
}
if (tmp_field_name) free(tmp_field_name);
if(input_index == -1){
Expand Down
3 changes: 2 additions & 1 deletion thirdparty/ffs/ffs/fm/fm_formats.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2758,7 +2758,7 @@ const char *str;
return unknown_type;
}

long
static long
find_field(char *field_name, FMFieldList fields, int cur_field, void *search_help)
{
int i;
Expand All @@ -2770,6 +2770,7 @@ find_field(char *field_name, FMFieldList fields, int cur_field, void *search_hel
}
}
for (i = cur_field + 1; i < cur_field + 10; i++) {
if (fields[i].field_name == NULL) break;
if (strcmp(field_name, fields[i].field_name) == 0) {
return i;
}
Expand Down
49 changes: 43 additions & 6 deletions thirdparty/ffs/ffs/fm/tests/scale_test.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
#include "config.h"
#include <string.h>
#include <time.h>
#include <stdio.h>
#include "fm.h"
#include "fm_internal.h"
#include "ffs.h"

#ifdef HAVE_WINDOWS_H
#include <windows.h>
#define sleep(x) Sleep(1000*x)
Expand All @@ -23,6 +25,13 @@ char *gen_name(int i)
return strdup(tmp_name);
}

struct base_elem {
int64_t elem_count1;
int64_t elem_count2;
int64_t *array1;
int64_t *array2;
};

int
main(argc, argv)
int argc;
Expand All @@ -31,12 +40,9 @@ char **argv;

FMStructDescRec str_list[5];
struct timespec start, stop;

if (argc > 1) {
}

FMContext context = create_FMcontext(NULL);
int field_count = 2000000;
FMContext context;
int field_count = 20000;
field_count = ((field_count >> 2 ) << 2); // ensure field count is divisible by 4;
FMFieldList list = malloc(sizeof(struct _FMField) * (field_count + 1));
int cur_count = 0;
Expand Down Expand Up @@ -76,9 +82,40 @@ char **argv;
str_list[0].struct_size = sizeof(first_rec);
str_list[0].opt_info = NULL;
str_list[1].format_name = NULL;
FFSContext fc = create_FFSContext();
context = FMContext_from_FFS(fc);

FMFormat format = register_data_format(context, str_list);

clock_gettime(CLOCK_MONOTONIC, &stop);
double duration = (stop.tv_sec + 1.0e-9*stop.tv_nsec) - (start.tv_sec + 1.0e-9*start.tv_nsec);
printf("Registration took %g seconds\n", duration);

char * buf= malloc(sizeof(struct base_elem) * (field_count / 4));
int i;
for (i = 0; i < field_count ; i+= 4) {
struct base_elem *tmp = (struct base_elem *) (buf + i * sizeof(int64_t));
tmp->elem_count1 = 3;
tmp->elem_count2 = 3;
tmp->array1 = malloc(3*sizeof(tmp->array1[0]));
tmp->array2 = malloc(3*sizeof(tmp->array2[0]));
}

clock_gettime(CLOCK_MONOTONIC, &start);
FFSBuffer b = create_FFSBuffer();

int buf_size;
char *encode = FFSencode(b, format, buf, &buf_size);
FFSTypeHandle th = FFSTypeHandle_from_encode(fc, encode);
clock_gettime(CLOCK_MONOTONIC, &stop);
duration = (stop.tv_sec + 1.0e-9*stop.tv_nsec) - (start.tv_sec + 1.0e-9*start.tv_nsec);
printf("encode took %g seconds\n", duration);
clock_gettime(CLOCK_MONOTONIC, &start);
establish_conversion(fc, th, str_list);
clock_gettime(CLOCK_MONOTONIC, &stop);
duration = (stop.tv_sec + 1.0e-9*stop.tv_nsec) - (start.tv_sec + 1.0e-9*start.tv_nsec);
printf("establish took %g seconds\n", duration);

}