summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--drivers/lightnvm/pblk-core.c28
-rw-r--r--drivers/lightnvm/pblk-gc.c2
-rw-r--r--drivers/lightnvm/pblk-init.c9
-rw-r--r--drivers/lightnvm/pblk-map.c4
-rw-r--r--drivers/lightnvm/pblk-rl.c6
-rw-r--r--drivers/lightnvm/pblk-write.c4
-rw-r--r--drivers/lightnvm/pblk.h6
7 files changed, 37 insertions, 22 deletions
diff --git a/drivers/lightnvm/pblk-core.c b/drivers/lightnvm/pblk-core.c
index ac3742b85f2e..5e44768ccffa 100644
--- a/drivers/lightnvm/pblk-core.c
+++ b/drivers/lightnvm/pblk-core.c
@@ -29,6 +29,7 @@ static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line,
pr_debug("pblk: erase failed: line:%d, pos:%d\n", line->id, pos);
atomic_long_inc(&pblk->erase_failed);
+ atomic_dec(&line->blk_in_line);
if (test_and_set_bit(pos, line->blk_bitmap))
pr_err("pblk: attempted to erase bb: line:%d, pos:%d\n",
line->id, pos);
@@ -832,21 +833,28 @@ int pblk_line_erase(struct pblk *pblk, struct pblk_line *line)
struct ppa_addr ppa;
int bit = -1;
- /* Erase one block at the time and only erase good blocks */
- while ((bit = find_next_zero_bit(line->erase_bitmap, lm->blk_per_line,
- bit + 1)) < lm->blk_per_line) {
+ /* Erase only good blocks, one at a time */
+ do {
+ spin_lock(&line->lock);
+ bit = find_next_zero_bit(line->erase_bitmap, lm->blk_per_line,
+ bit + 1);
+ if (bit >= lm->blk_per_line) {
+ spin_unlock(&line->lock);
+ break;
+ }
+
ppa = pblk->luns[bit].bppa; /* set ch and lun */
ppa.g.blk = line->id;
- /* If the erase fails, the block is bad and should be marked */
- line->left_eblks--;
+ atomic_dec(&line->left_eblks);
WARN_ON(test_and_set_bit(bit, line->erase_bitmap));
+ spin_unlock(&line->lock);
if (pblk_blk_erase_sync(pblk, ppa)) {
pr_err("pblk: failed to erase line %d\n", line->id);
return -ENOMEM;
}
- }
+ } while (1);
return 0;
}
@@ -1007,6 +1015,7 @@ retry_smeta:
static int pblk_line_prepare(struct pblk *pblk, struct pblk_line *line)
{
struct pblk_line_meta *lm = &pblk->lm;
+ int blk_in_line = atomic_read(&line->blk_in_line);
line->map_bitmap = mempool_alloc(pblk->line_meta_pool, GFP_ATOMIC);
if (!line->map_bitmap)
@@ -1030,12 +1039,13 @@ static int pblk_line_prepare(struct pblk *pblk, struct pblk_line *line)
return -EINTR;
}
line->state = PBLK_LINESTATE_OPEN;
+
+ atomic_set(&line->left_eblks, blk_in_line);
+ atomic_set(&line->left_seblks, blk_in_line);
spin_unlock(&line->lock);
/* Bad blocks do not need to be erased */
bitmap_copy(line->erase_bitmap, line->blk_bitmap, lm->blk_per_line);
- line->left_eblks = line->blk_in_line;
- atomic_set(&line->left_seblks, line->left_eblks);
kref_init(&line->ref);
@@ -1231,7 +1241,7 @@ retry_line:
left_seblks = atomic_read(&new->left_seblks);
if (left_seblks) {
/* If line is not fully erased, erase it */
- if (new->left_eblks) {
+ if (atomic_read(&new->left_eblks)) {
if (pblk_line_erase(pblk, new))
return NULL;
} else {
diff --git a/drivers/lightnvm/pblk-gc.c b/drivers/lightnvm/pblk-gc.c
index f173fd4ea947..eaf479c6b63c 100644
--- a/drivers/lightnvm/pblk-gc.c
+++ b/drivers/lightnvm/pblk-gc.c
@@ -332,7 +332,7 @@ next_gc_group:
}
line = list_first_entry(group_list, struct pblk_line, list);
- nr_blocks_free += line->blk_in_line;
+ nr_blocks_free += atomic_read(&line->blk_in_line);
spin_lock(&line->lock);
WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
diff --git a/drivers/lightnvm/pblk-init.c b/drivers/lightnvm/pblk-init.c
index 3996e4b8fb0e..15b2787c3ddc 100644
--- a/drivers/lightnvm/pblk-init.c
+++ b/drivers/lightnvm/pblk-init.c
@@ -678,6 +678,8 @@ add_emeta_page:
nr_free_blks = 0;
for (i = 0; i < l_mg->nr_lines; i++) {
+ int blk_in_line;
+
line = &pblk->lines[i];
line->pblk = pblk;
@@ -693,14 +695,15 @@ add_emeta_page:
goto fail_free_lines;
}
- line->blk_in_line = lm->blk_per_line - nr_bad_blks;
- if (line->blk_in_line < lm->min_blk_line) {
+ blk_in_line = lm->blk_per_line - nr_bad_blks;
+ if (blk_in_line < lm->min_blk_line) {
line->state = PBLK_LINESTATE_BAD;
list_add_tail(&line->list, &l_mg->bad_list);
continue;
}
- nr_free_blks += line->blk_in_line;
+ nr_free_blks += blk_in_line;
+ atomic_set(&line->blk_in_line, blk_in_line);
l_mg->nr_free_lines++;
list_add_tail(&line->list, &l_mg->free_list);
diff --git a/drivers/lightnvm/pblk-map.c b/drivers/lightnvm/pblk-map.c
index 3f8bab4c4d5c..17c16955284d 100644
--- a/drivers/lightnvm/pblk-map.c
+++ b/drivers/lightnvm/pblk-map.c
@@ -110,7 +110,7 @@ void pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd,
continue;
set_bit(erase_lun, e_line->erase_bitmap);
- e_line->left_eblks--;
+ atomic_dec(&e_line->left_eblks);
*erase_ppa = rqd->ppa_list[i];
erase_ppa->g.blk = e_line->id;
@@ -129,7 +129,7 @@ void pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd,
return;
set_bit(i, e_line->erase_bitmap);
- e_line->left_eblks--;
+ atomic_dec(&e_line->left_eblks);
*erase_ppa = pblk->luns[i].bppa; /* set ch and lun */
erase_ppa->g.blk = e_line->id;
}
diff --git a/drivers/lightnvm/pblk-rl.c b/drivers/lightnvm/pblk-rl.c
index 4042162ec9bc..ab7cbb144f3f 100644
--- a/drivers/lightnvm/pblk-rl.c
+++ b/drivers/lightnvm/pblk-rl.c
@@ -107,9 +107,10 @@ void pblk_rl_set_gc_rsc(struct pblk_rl *rl, int rsv)
void pblk_rl_free_lines_inc(struct pblk_rl *rl, struct pblk_line *line)
{
struct pblk *pblk = container_of(rl, struct pblk, rl);
+ int blk_in_line = atomic_read(&line->blk_in_line);
int ret;
- atomic_add(line->blk_in_line, &rl->free_blocks);
+ atomic_add(blk_in_line, &rl->free_blocks);
/* Rates will not change that often - no need to lock update */
ret = pblk_rl_update_rates(rl, rl->rb_budget);
@@ -122,9 +123,10 @@ void pblk_rl_free_lines_inc(struct pblk_rl *rl, struct pblk_line *line)
void pblk_rl_free_lines_dec(struct pblk_rl *rl, struct pblk_line *line)
{
struct pblk *pblk = container_of(rl, struct pblk, rl);
+ int blk_in_line = atomic_read(&line->blk_in_line);
int ret;
- atomic_sub(line->blk_in_line, &rl->free_blocks);
+ atomic_sub(blk_in_line, &rl->free_blocks);
/* Rates will not change that often - no need to lock update */
ret = pblk_rl_update_rates(rl, rl->rb_budget);
diff --git a/drivers/lightnvm/pblk-write.c b/drivers/lightnvm/pblk-write.c
index a896190c82f0..aef6fd7c4a0c 100644
--- a/drivers/lightnvm/pblk-write.c
+++ b/drivers/lightnvm/pblk-write.c
@@ -244,7 +244,7 @@ static int pblk_setup_w_rq(struct pblk *pblk, struct nvm_rq *rqd,
}
ppa_set_empty(&erase_ppa);
- if (likely(!e_line || !e_line->left_eblks))
+ if (likely(!e_line || !atomic_read(&e_line->left_eblks)))
pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, valid, 0);
else
pblk_map_erase_rq(pblk, rqd, c_ctx->sentry, lun_bitmap,
@@ -257,7 +257,7 @@ out:
struct nvm_geo *geo = &dev->geo;
int bit;
- e_line->left_eblks++;
+ atomic_inc(&e_line->left_eblks);
bit = erase_ppa.g.lun * geo->nr_chnls + erase_ppa.g.ch;
WARN_ON(!test_and_clear_bit(bit, e_line->erase_bitmap));
up(&pblk->erase_sem);
diff --git a/drivers/lightnvm/pblk.h b/drivers/lightnvm/pblk.h
index 11ed7d83f572..99f3186b5288 100644
--- a/drivers/lightnvm/pblk.h
+++ b/drivers/lightnvm/pblk.h
@@ -363,14 +363,14 @@ struct pblk_line {
unsigned int sec_in_line; /* Number of usable secs in line */
- unsigned int blk_in_line; /* Number of good blocks in line */
+ atomic_t blk_in_line; /* Number of good blocks in line */
unsigned long *blk_bitmap; /* Bitmap for valid/invalid blocks */
unsigned long *erase_bitmap; /* Bitmap for erased blocks */
unsigned long *map_bitmap; /* Bitmap for mapped sectors in line */
unsigned long *invalid_bitmap; /* Bitmap for invalid sectors in line */
- int left_eblks; /* Blocks left for erasing */
+ atomic_t left_eblks; /* Blocks left for erasing */
atomic_t left_seblks; /* Blocks left for sync erasing */
int left_msecs; /* Sectors left for mapping */
@@ -383,7 +383,7 @@ struct pblk_line {
spinlock_t lock; /* Necessary for invalid_bitmap only */
};
-#define PBLK_DATA_LINES 2
+#define PBLK_DATA_LINES 4
enum{
PBLK_KMALLOC_META = 1,