Skip to content

Commit

Permalink
Fixes and enhancements of SIMD raidz parity (2)
Browse files Browse the repository at this point in the history 
Enable use of original parity methods in [fastest] configuration.
New opaque original ops structure, representing native methods, is added
to supported raidz methods. Original parity methods are executed if selected
implementation has NULL fn pointer.

Signed-off-by: Gvozden Neskovic <neskovic@gmail.com>
  • Loading branch information
@ironMann
ironMann committed Jul 17, 2016
1 parent c3a7b00 commit 17f820c
Show file tree
Hide file tree
Showing 5 changed files with 83 additions and 76 deletions.
2 changes: 1 addition & 1 deletion include/sys/vdev_raidz.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ int vdev_raidz_reconstruct(struct raidz_map *, const int *, int);
void vdev_raidz_math_init(void);
void vdev_raidz_math_fini(void);
struct raidz_impl_ops * vdev_raidz_math_get_ops(void);
void vdev_raidz_math_generate(struct raidz_map *);
int vdev_raidz_math_generate(struct raidz_map *);
int vdev_raidz_math_reconstruct(struct raidz_map *,
const int *, const int *, const int);
int vdev_raidz_impl_set(const char *);
Expand Down
2 changes: 2 additions & 0 deletions include/sys/vdev_raidz_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,8 @@ typedef struct raidz_map {
raidz_col_t rm_col[1]; /* Flexible array of I/O columns */
} raidz_map_t;

#define RAIDZ_ORIGINAL_IMPL (INT_MAX)

extern const raidz_impl_ops_t vdev_raidz_scalar_impl;
#if defined(__x86_64) && defined(HAVE_SSE2) /* only x86_64 for now */
extern const raidz_impl_ops_t vdev_raidz_sse2_impl;
Expand Down
20 changes: 8 additions & 12 deletions module/zfs/vdev_raidz.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -611,10 +611,9 @@ vdev_raidz_generate_parity_pqr(raidz_map_t *rm)
void
vdev_raidz_generate_parity(raidz_map_t *rm)
{
if (rm->rm_ops) {
vdev_raidz_math_generate(rm);
/* Generate using the new math implementation */
if (vdev_raidz_math_generate(rm) != RAIDZ_ORIGINAL_IMPL)
return;
}

switch (rm->rm_firstdatacol) {
case 1:
Expand Down Expand Up @@ -1284,7 +1283,7 @@ vdev_raidz_reconstruct(raidz_map_t *rm, const int *t, int nt)
{
int tgts[VDEV_RAIDZ_MAXPARITY], *dt;
int ntgts;
int i, c;
int i, c, ret;
int code;
int nbadparity, nbaddata;
int parity_valid[VDEV_RAIDZ_MAXPARITY];
Expand Down Expand Up @@ -1322,14 +1321,11 @@ vdev_raidz_reconstruct(raidz_map_t *rm, const int *t, int nt)

dt = &tgts[nbadparity];

/*
* Reconstruct using the new math implementation if
* rm_ops is set.
*/
if (rm->rm_ops) {
return (vdev_raidz_math_reconstruct(rm, parity_valid, dt,
nbaddata));
}

/* Reconstruct using the new math implementation */
ret = vdev_raidz_math_reconstruct(rm, parity_valid, dt, nbaddata);
if (ret != RAIDZ_ORIGINAL_IMPL)
return (ret);

/*
* See if we can use any of our optimized reconstruction routines.
Expand Down
133 changes: 71 additions & 62 deletions module/zfs/vdev_raidz_math.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,22 @@
#include <sys/vdev_raidz.h>
#include <sys/vdev_raidz_impl.h>

extern boolean_t raidz_will_scalar_work(void);

/* Opaque implementation with NULL methods to represent original methods */
static const raidz_impl_ops_t vdev_raidz_original_impl = {
.name = "original",
.is_supported = raidz_will_scalar_work,
};

/* RAIDZ parity op that contain the fastest methods */
static raidz_impl_ops_t vdev_raidz_fastest_impl = {
.name = "fastest"
};

/* All compiled in implementations */
const raidz_impl_ops_t *raidz_all_maths[] = {
&vdev_raidz_original_impl,
&vdev_raidz_scalar_impl,
#if defined(__x86_64) && defined(HAVE_SSE2) /* only x86_64 for now */
&vdev_raidz_sse2_impl,
Expand All @@ -49,27 +63,19 @@ const raidz_impl_ops_t *raidz_all_maths[] = {
static boolean_t raidz_math_initialized = B_FALSE;

/* Select raidz implementation */
#define IMPL_FASTEST (UINT32_MAX - 1)
#define IMPL_ORIGINAL (UINT32_MAX - 2)
#define IMPL_CYCLE (UINT32_MAX - 3)
#define IMPL_SCALAR (0)
#define IMPL_FASTEST (UINT32_MAX)
#define IMPL_CYCLE (UINT32_MAX - 1)
#define IMPL_ORIGINAL (0)
#define IMPL_SCALAR (1)

#define RAIDZ_IMPL_READ(i) (*(volatile uint32_t *) &(i))

static uint32_t zfs_vdev_raidz_impl = IMPL_SCALAR;
static uint32_t user_sel_impl = IMPL_FASTEST;

/* RAIDZ op that contain the fastest routines */
static raidz_impl_ops_t vdev_raidz_fastest_impl = {
.name = "fastest"
};

/* Hold all supported implementations */
size_t raidz_supp_impl_cnt = 1;
raidz_impl_ops_t *raidz_supp_impl[ARRAY_SIZE(raidz_all_maths) + 1] = {
(raidz_impl_ops_t *) &vdev_raidz_scalar_impl, /* scalar is supported */
NULL
};
static size_t raidz_supp_impl_cnt = 0;
static raidz_impl_ops_t *raidz_supp_impl[ARRAY_SIZE(raidz_all_maths)];

/*
* kstats values for supported impl & original methods
Expand All @@ -95,38 +101,41 @@ vdev_raidz_math_get_ops()
ASSERT(raidz_math_initialized);
ops = &vdev_raidz_fastest_impl;
break;
case IMPL_ORIGINAL:
ops = NULL;
break;
#if !defined(_KERNEL)
case IMPL_CYCLE:
{
/*
* Cycle through all supported implementations
* note: raidz_supp_impl[raidz_supp_impl_cnt] == NULL, in which
* case the original implementation is used
*/
ASSERT(raidz_math_initialized);
ASSERT3U(raidz_supp_impl_cnt, >, 0);
/* Cycle through all supported implementations */
static size_t cycle_impl_idx = 0;
size_t idx = (++cycle_impl_idx) % (raidz_supp_impl_cnt + 1);
size_t idx = (++cycle_impl_idx) % raidz_supp_impl_cnt;
ops = raidz_supp_impl[idx];
}
break;
#endif
case IMPL_ORIGINAL:
ops = (raidz_impl_ops_t *) &vdev_raidz_original_impl;
break;
case IMPL_SCALAR:
ops = (raidz_impl_ops_t *) &vdev_raidz_scalar_impl;
break;
default:
ASSERT3U(impl, >=, 0);
ASSERT(raidz_math_initialized);
ASSERT3U(impl, <, raidz_supp_impl_cnt);
ASSERT3U(raidz_supp_impl_cnt, >, 0);
ops = raidz_supp_impl[impl];
break;
}

ASSERT3P(ops, !=, NULL);

return (ops);
}

/*
* Select parity generation method for raidz_map
*/
void
int
vdev_raidz_math_generate(raidz_map_t *rm)
{
raidz_gen_f gen_parity = NULL;
Expand All @@ -148,13 +157,17 @@ vdev_raidz_math_generate(raidz_map_t *rm)
break;
}

ASSERT(gen_parity != NULL);
/* if method is NULL execute the original implementation */
if (gen_parity == NULL)
return (RAIDZ_ORIGINAL_IMPL);

gen_parity(rm);

return (0);
}

static raidz_rec_f
_reconstruct_fun_raidz1(raidz_map_t *rm, const int *parity_valid,
reconstruct_fun_p_sel(raidz_map_t *rm, const int *parity_valid,
const int nbaddata)
{
if (nbaddata == 1 && parity_valid[CODE_P]) {
Expand All @@ -164,7 +177,7 @@ _reconstruct_fun_raidz1(raidz_map_t *rm, const int *parity_valid,
}

static raidz_rec_f
_reconstruct_fun_raidz2(raidz_map_t *rm, const int *parity_valid,
reconstruct_fun_pq_sel(raidz_map_t *rm, const int *parity_valid,
const int nbaddata)
{
if (nbaddata == 1) {
Expand All @@ -181,7 +194,7 @@ _reconstruct_fun_raidz2(raidz_map_t *rm, const int *parity_valid,
}

static raidz_rec_f
_reconstruct_fun_raidz3(raidz_map_t *rm, const int *parity_valid,
reconstruct_fun_pqr_sel(raidz_map_t *rm, const int *parity_valid,
const int nbaddata)
{
if (nbaddata == 1) {
Expand Down Expand Up @@ -221,27 +234,25 @@ vdev_raidz_math_reconstruct(raidz_map_t *rm, const int *parity_valid,
raidz_rec_f rec_data = NULL;

switch (raidz_parity(rm)) {
case 1:
rec_data = _reconstruct_fun_raidz1(rm, parity_valid,
nbaddata);
break;
case 2:
rec_data = _reconstruct_fun_raidz2(rm, parity_valid,
nbaddata);
break;
case 3:
rec_data = _reconstruct_fun_raidz3(rm, parity_valid,
nbaddata);
break;
default:
cmn_err(CE_PANIC, "invalid RAID-Z configuration %d",
raidz_parity(rm));
break;
case PARITY_P:
rec_data = reconstruct_fun_p_sel(rm, parity_valid, nbaddata);
break;
case PARITY_PQ:
rec_data = reconstruct_fun_pq_sel(rm, parity_valid, nbaddata);
break;
case PARITY_PQR:
rec_data = reconstruct_fun_pqr_sel(rm, parity_valid, nbaddata);
break;
default:
cmn_err(CE_PANIC, "invalid RAID-Z configuration %d",
raidz_parity(rm));
break;
}

ASSERT(rec_data != NULL);

return (rec_data(rm, dt));
if (rec_data == NULL)
return (RAIDZ_ORIGINAL_IMPL);
else
return (rec_data(rm, dt));
}

const char *raidz_gen_name[] = {
Expand Down Expand Up @@ -322,13 +333,10 @@ benchmark_raidz_impl(raidz_map_t *bench_rm, const int fn, benchmark_fn bench_fn)
uint64_t run_cnt, speed, best_speed = 0;
hrtime_t t_start, t_diff;
raidz_impl_ops_t *curr_impl;
raidz_impl_kstat_t * fstat = &raidz_impl_kstats[raidz_supp_impl_cnt];
int impl, i;

/*
* Use the sentinel (NULL) from the end of raidz_supp_impl_cnt
* to run "original" implementation (bench_rm->rm_ops = NULL)
*/
for (impl = 0; impl <= raidz_supp_impl_cnt; impl++) {
for (impl = 0; impl < raidz_supp_impl_cnt; impl++) {
/* set an implementation to benchmark */
curr_impl = raidz_supp_impl[impl];
bench_rm->rm_ops = curr_impl;
Expand All @@ -351,16 +359,19 @@ benchmark_raidz_impl(raidz_map_t *bench_rm, const int fn, benchmark_fn bench_fn)
else
raidz_impl_kstats[impl].rec[fn].value.ui64 = speed;

/* if curr_impl==NULL the original impl is benchmarked */
if (curr_impl != NULL && speed > best_speed) {
/* Update fastest implementation method */
if (speed > best_speed) {
best_speed = speed;

if (bench_fn == benchmark_gen_impl)
if (bench_fn == benchmark_gen_impl) {
vdev_raidz_fastest_impl.gen[fn] =
curr_impl->gen[fn];
else
fstat->gen[fn].value.ui64 = speed;
} else {
vdev_raidz_fastest_impl.rec[fn] =
curr_impl->rec[fn];
fstat->rec[fn].value.ui64 = speed;
}
}
}
}
Expand Down Expand Up @@ -389,12 +400,10 @@ vdev_raidz_math_init(void)
raidz_supp_impl[c++] = (raidz_impl_ops_t *) curr_impl;
}
}
raidz_supp_impl[c] = NULL; /* sentinel */
membar_producer(); /* complete raidz_supp_impl[] init */
membar_producer(); /* complete raidz_supp_impl[] init */
raidz_supp_impl_cnt = c; /* number of supported impl */

/* init kstat for original routines */
init_raidz_kstat(&(raidz_impl_kstats[raidz_supp_impl_cnt]), "original");
init_raidz_kstat(&(raidz_impl_kstats[raidz_supp_impl_cnt]), "fastest");

#if !defined(_KERNEL)
/* Skip benchmarking and use last implementation as fastest */
Expand Down Expand Up @@ -568,7 +577,7 @@ zfs_vdev_raidz_impl_get(char *buffer, struct kernel_param *kp)
ASSERT(raidz_math_initialized);

/* list mandatory options */
for (i = 0; i < ARRAY_SIZE(math_impl_opts) - 1; i++) {
for (i = 0; i < ARRAY_SIZE(math_impl_opts) - 2; i++) {
fmt = (impl == math_impl_opts[i].sel) ? "[%s] " : "%s ";
cnt += sprintf(buffer + cnt, fmt, math_impl_opts[i].name);
}
Expand Down
2 changes: 1 addition & 1 deletion module/zfs/vdev_raidz_math_scalar.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ static const struct {
DEFINE_GEN_METHODS(scalar);
DEFINE_REC_METHODS(scalar);

static boolean_t
boolean_t
raidz_will_scalar_work(void)
{
return (B_TRUE); /* always */
Expand Down

0 comments on commit 17f820c

Please sign in to comment.