Skip to content

Commit

Permalink
ABD kmap to kmap_atomic
Browse files Browse the repository at this point in the history 
Convert usage of kmap to kmap_atomic while correctly saving off
irq state.
  • Loading branch information
@behlendorf
Chunwei Chen authored and behlendorf committed Nov 29, 2016
1 parent 88cc235 commit 4f60152
Showing 1 changed file with 49 additions and 35 deletions.
84 changes: 49 additions & 35 deletions module/zfs/abd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,9 @@
#include <sys/zio.h>
#include <sys/zfs_context.h>
#include <sys/zfs_znode.h>
#ifdef _KERNEL
#include <linux/kmap_compat.h>
#endif

#ifndef KMC_NOTOUCH
#define KMC_NOTOUCH 0
Expand Down Expand Up @@ -187,22 +190,6 @@ abd_free_chunk(struct page *c)
__free_pages(c, 0);
}

static void *
abd_map_chunk(struct page *c)
{
/*
* Use of segkpm means we don't care if this is mapped S_READ or S_WRITE
* but S_WRITE is conceptually more accurate.
*/
return (kmap(c));
}

static void
abd_unmap_chunk(struct page *c)
{
kunmap(c);
}

void
abd_init(void)
{
Expand Down Expand Up @@ -230,11 +217,10 @@ struct page;
#define abd_alloc_chunk() \
((struct page *) umem_alloc_aligned(PAGESIZE, 64, KM_SLEEP))
#define abd_free_chunk(chunk) umem_free(chunk, PAGESIZE)
#define abd_map_chunk(chunk) ((void *)chunk)
static void
abd_unmap_chunk(struct page *c)
{
}
#define zfs_kmap_atomic(chunk, km) ((void *)chunk)
#define zfs_kunmap_atomic(addr, km) do { (void)(addr); } while (0)
#define local_irq_save(flags) do { (void)(flags); } while (0)
#define local_irq_restore(flags) do { (void)(flags); } while (0)

void
abd_init(void)
Expand Down Expand Up @@ -697,11 +683,28 @@ abd_release_ownership_of_buf(abd_t *abd)
ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size);
}

#ifndef HAVE_1ARG_KMAP_ATOMIC
#define NR_KM_TYPE (6)
#ifdef _KERNEL
int km_table[NR_KM_TYPE] = {
KM_USER0,
KM_USER1,
KM_BIO_SRC_IRQ,
KM_BIO_DST_IRQ,
KM_PTE0,
KM_PTE1,
};
#endif
#endif

struct abd_iter {
abd_t *iter_abd; /* ABD being iterated through */
size_t iter_pos; /* position (relative to abd_offset) */
void *iter_mapaddr; /* addr corresponding to iter_pos */
size_t iter_mapsize; /* length of data valid at mapaddr */
#ifndef HAVE_1ARG_KMAP_ATOMIC
int iter_km; /* KM_* for kmap_atomic */
#endif
};

static inline size_t
Expand All @@ -724,13 +727,17 @@ abd_iter_scatter_chunk_index(struct abd_iter *aiter)
* Initialize the abd_iter.
*/
static void
abd_iter_init(struct abd_iter *aiter, abd_t *abd)
abd_iter_init(struct abd_iter *aiter, abd_t *abd, int km_type)
{
abd_verify(abd);
aiter->iter_abd = abd;
aiter->iter_pos = 0;
aiter->iter_mapaddr = NULL;
aiter->iter_mapsize = 0;
#ifndef HAVE_1ARG_KMAP_ATOMIC
ASSERT3U(km_type, <, NR_KM_TYPE);
aiter->iter_km = km_type;
#endif
}

/*
Expand Down Expand Up @@ -779,8 +786,9 @@ abd_iter_map(struct abd_iter *aiter)
aiter->iter_mapsize = MIN(PAGESIZE - offset,
aiter->iter_abd->abd_size - aiter->iter_pos);

paddr = abd_map_chunk(
aiter->iter_abd->abd_u.abd_scatter.abd_chunks[index]);
paddr = zfs_kmap_atomic(
aiter->iter_abd->abd_u.abd_scatter.abd_chunks[index],
km_table[aiter->iter_km]);
}

aiter->iter_mapaddr = (char *)paddr + offset;
Expand All @@ -799,9 +807,9 @@ abd_iter_unmap(struct abd_iter *aiter)

if (!abd_is_linear(aiter->iter_abd)) {
/* LINTED E_FUNC_SET_NOT_USED */
size_t index = abd_iter_scatter_chunk_index(aiter);
abd_unmap_chunk(
aiter->iter_abd->abd_u.abd_scatter.abd_chunks[index]);
zfs_kunmap_atomic(aiter->iter_mapaddr -
abd_iter_scatter_chunk_offset(aiter),
km_table[aiter->iter_km]);
}

ASSERT3P(aiter->iter_mapaddr, !=, NULL);
Expand All @@ -821,7 +829,7 @@ abd_iterate_func(abd_t *abd, size_t off, size_t size,
abd_verify(abd);
ASSERT3U(off + size, <=, abd->abd_size);

abd_iter_init(&aiter, abd);
abd_iter_init(&aiter, abd, 0);
abd_iter_advance(&aiter, off);

while (size > 0) {
Expand Down Expand Up @@ -953,8 +961,8 @@ abd_iterate_func2(abd_t *dabd, abd_t *sabd, size_t doff, size_t soff,
ASSERT3U(doff + size, <=, dabd->abd_size);
ASSERT3U(soff + size, <=, sabd->abd_size);

abd_iter_init(&daiter, dabd);
abd_iter_init(&saiter, sabd);
abd_iter_init(&daiter, dabd, 0);
abd_iter_init(&saiter, sabd, 1);
abd_iter_advance(&daiter, doff);
abd_iter_advance(&saiter, soff);

Expand Down Expand Up @@ -1039,17 +1047,19 @@ abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
struct abd_iter caiters[3];
struct abd_iter daiter;
void *caddrs[3];
unsigned long flags;

ASSERT3U(parity, <=, 3);

for (i = 0; i < parity; i++)
abd_iter_init(&caiters[i], cabds[i]);
abd_iter_init(&caiters[i], cabds[i], i);

if (dabd)
abd_iter_init(&daiter, dabd);
abd_iter_init(&daiter, dabd, i);

ASSERT3S(dsize, >=, 0);

local_irq_save(flags);
while (csize > 0) {
len = csize;

Expand Down Expand Up @@ -1106,6 +1116,7 @@ abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
ASSERT3S(dsize, >=, 0);
ASSERT3S(csize, >=, 0);
}
local_irq_restore(flags);
}

/*
Expand All @@ -1130,14 +1141,16 @@ abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds,
struct abd_iter citers[3];
struct abd_iter xiters[3];
void *caddrs[3], *xaddrs[3];
unsigned long flags;

ASSERT3U(parity, <=, 3);

for (i = 0; i < parity; i++) {
abd_iter_init(&citers[i], cabds[i]);
abd_iter_init(&xiters[i], tabds[i]);
abd_iter_init(&citers[i], cabds[i], 2*i);
abd_iter_init(&xiters[i], tabds[i], 2*i+1);
}

local_irq_save(flags);
while (tsize > 0) {

for (i = 0; i < parity; i++) {
Expand Down Expand Up @@ -1179,6 +1192,7 @@ abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds,
tsize -= len;
ASSERT3S(tsize, >=, 0);
}
local_irq_restore(flags);
}

#if defined(_KERNEL) && defined(HAVE_SPL)
Expand Down Expand Up @@ -1215,7 +1229,7 @@ abd_scatter_bio_map_off(struct bio *bio, abd_t *abd,
ASSERT(!abd_is_linear(abd));
ASSERT3U(io_size, <=, abd->abd_size - off);

abd_iter_init(&aiter, abd);
abd_iter_init(&aiter, abd, 0);
abd_iter_advance(&aiter, off);

for (i = 0; i < bio->bi_max_vecs; i++) {
Expand Down

0 comments on commit 4f60152

Please sign in to comment.