[stgt] [PATCH RFC] save reservations through power loss
Alexander Gordeev
agordeev at parallels.com
Tue Oct 8 13:12:35 CEST 2013
This patch adds support for storing persistent registrations and
reservation on a persistent storage so they survive a power loss. This
support is enabled by setting new target option called 'pr_dir' before
adding LUNs. The option value should be set to a valid existing
directory. Then for each added LUN a file in this directory named after
the LUN's id is opened, already stored reservations for this LUN are
read if any, and new reservations are stored.
I wanted to store new reservations asynchronously to the main loop. AIO
doesn't offer fsync/fdatasync-like functions (that are really working
as it is expected) so I had to use a separate thread for it. I think
having a thread per LUN is too much, so there is a single thread at the
moment with a single queue of requests, shared by all LUNs.
The final implmentation should honor APTPL bit. This is not done at the
moment.
This implementation is not completely finished. I tested it, but surely
more testing is necessary. I hope to get some feedback on the overall
design here. There are some things yet to do:
1. support preempt
2. check that LUN with PR store is destroyed the right way
3. remove mutexes (they can block), move requests through pipes instead
(?) 4. store initiator names instead of full transport ids (?)
5. honor APTPL bit
6. add crc32 sum to each stored registration
7. update man pages (pr_dir option, ?)
8. check how it can be set up through targets.conf
9. use AIO for writes (?), use the separate thread to open, read, flush
and close only
Signed-off-by: Alexander Gordeev <agordeev at parallels.com>
---
usr/Makefile | 2 +-
usr/mgmt.c | 2 +
usr/prstore.c | 649
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
usr/prstore.h | 30 +++ usr/spc.c | 185 +++++++++++------
usr/target.c | 18 ++
usr/target.h | 2 +
usr/tgtd.c | 5 +
usr/tgtd.h | 20 +-
9 files changed, 850 insertions(+), 63 deletions(-)
create mode 100644 usr/prstore.c
create mode 100644 usr/prstore.h
diff --git a/usr/Makefile b/usr/Makefile
index 60bff6d..623b924 100644
--- a/usr/Makefile
+++ b/usr/Makefile
@@ -46,7 +46,7 @@ PROGRAMS += tgtd tgtadm tgtimg
TGTD_OBJS += tgtd.o mgmt.o target.o scsi.o log.o driver.o util.o
work.o \ concat_buf.o parser.o spc.o sbc.o mmc.o osd.o scc.o smc.o \
ssc.o libssc.o bs_rdwr.o bs_ssc.o \
- bs_null.o bs_sg.o bs.o libcrc32c.o bs_sheepdog.o
+ bs_null.o bs_sg.o bs.o libcrc32c.o bs_sheepdog.o
prstore.o
TGTD_DEP = $(TGTD_OBJS:.o=.d)
diff --git a/usr/mgmt.c b/usr/mgmt.c
index c0bf7be..5dc0e4d 100644
--- a/usr/mgmt.c
+++ b/usr/mgmt.c
@@ -171,6 +171,8 @@ static tgtadm_err target_mgmt(int lld_no, struct
mgmt_task *mtask)
if (!strcmp(mtask->req_buf, "state")) {
adm_err = tgt_set_target_state(req->tid, p);
+ } else if (!strcmp(mtask->req_buf, "pr_dir")) {
+ adm_err = tgt_set_target_pr_dir(req->tid, p);
} else if (tgt_drivers[lld_no]->update)
adm_err =
tgt_drivers[lld_no]->update(req->mode, req->op, req->tid, req->sid,
req->lun, diff --git a/usr/prstore.c b/usr/prstore.c
new file mode 100644
index 0000000..37398a6
--- /dev/null
+++ b/usr/prstore.c
@@ -0,0 +1,649 @@
+/*
+ * PR store routines
+ *
+ * Copyright (C) 2013 Alexander Gordeev <agordeev at parallels.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation, version 2 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ */
+#include <errno.h>
+#include <string.h>
+#include <inttypes.h>
+#include <pthread.h>
+#include <unistd.h>
+#include <sys/epoll.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "list.h"
+#include "util.h"
+#include "tgtd.h"
+#include "target.h"
+#include "scsi.h"
+#include "prstore.h"
+
+enum pr_req_type {
+ PR_REQ_OPEN,
+ PR_REQ_UPDATE,
+ PR_REQ_DELETE,
+ PR_REQ_CLOSE
+};
+
+struct pr_request {
+ enum pr_req_type type;
+ struct list_head list;
+ union {
+ struct {
+ struct scsi_lu *lu;
+ } open;
+
+ struct {
+ struct registration *reg;
+ struct scsi_cmd *cmd;
+ } update;
+
+ struct {
+ int slot;
+ struct scsi_cmd *cmd;
+ } del;
+
+ struct {
+ struct scsi_lu *lu;
+ } close;
+ };
+};
+
+typedef void (*reqhandler_t)(struct pr_request *req);
+
+static LIST_HEAD(incoming_list);
+static pthread_mutex_t incoming_lock;
+static pthread_cond_t incoming_cond;
+
+static LIST_HEAD(done_list);
+static pthread_mutex_t done_lock;
+static int done_fd[2];
+
+static pthread_t pr_thread;
+static int pr_thread_run;
+
+#define PR_INIT_SLOTS 16
+#define BITS_PER_LONG (sizeof(long) * 8)
+
+/* TODO:
+ * 1. support preempt
+ * 2. ensure that LUN with PR store is destroyed the right way
+ * 3. remove mutexes (they can block), move requests through pipes
instead (?)
+ * 4. store initiator names instead of full transport ids (?)
+ * 5. honor APTPL bit
+ * 6. add crc32 sum to each stored registration
+ * 7. fix man
+ * 8. check if it can be setup through targets.conf
+ */
+
+
+/* Bitset ops */
+
+static inline int bitset_slots_to_size(int nslots)
+{
+ return (nslots + BITS_PER_LONG - 1) / BITS_PER_LONG;
+}
+
+static inline int bitset_slots_to_bytes(int nslots)
+{
+ return (nslots + BITS_PER_LONG - 1) / 8;
+}
+
+static int bitset_ffs(long *bs, int slots)
+{
+ int w;
+
+ for (w = 0; bs[w] == 0; ++w)
+ if (w >= bitset_slots_to_size(slots))
+ return -1;
+ return w * sizeof(long) * 8 + ffsl(bs[w]) - 1;
+}
+
+static inline void bitset_set(long *bs, int index)
+{
+ bs[index / (sizeof(long) * 8)] |= 1L << (index % (sizeof(long)
* 8)); +}
+
+static inline void bitset_unset(long *bs, int index)
+{
+ bs[index / (sizeof(long) * 8)] &= ~(1L << (index %
(sizeof(long) * 8))); +}
+
+
+/* Safe write and read */
+
+/* returns sz if ok or -errno if error */
+static int swrite(int fd, void *buf, int sz)
+{
+ int w = sz;
+
+ while (w) {
+ int n = write(fd, buf, w);
+ if (n < 0) {
+ if (errno == EAGAIN || errno == EINTR)
+ continue;
+ return -errno;
+ }
+ buf += n;
+ w -= n;
+ }
+ return sz;
+}
+
+/* returns number of bytes read or -errno if error */
+static int sread(int fd, void *buf, int sz)
+{
+ int r = 0;
+ while (sz) {
+ int n = read(fd, buf, sz);
+ if (n < 0) {
+ if (errno == EAGAIN || errno == EINTR)
+ continue;
+ return -errno;
+ }
+ if (n == 0)
+ break;
+ buf += n;
+ r += n;
+ sz -= n;
+ }
+ return r;
+}
+
+
+/* Callbacks that will be executed after completion in the main thread
*/ +
+static void done_open_req(struct pr_request *req)
+{
+}
+
+static void done_update_req(struct pr_request *req)
+{
+ dprintf("back to tgtd, %p\n", req->update.cmd);
+ target_cmd_io_done(req->update.cmd,
scsi_get_result(req->update.cmd)); +}
+
+static void done_delete_req(struct pr_request *req)
+{
+ dprintf("back to tgtd, %p\n", req->del.cmd);
+ target_cmd_io_done(req->del.cmd,
scsi_get_result(req->del.cmd)); +}
+
+static void done_close_req(struct pr_request *req)
+{
+}
+
+reqhandler_t done_handlers[] = {
+ [PR_REQ_OPEN] = done_open_req,
+ [PR_REQ_UPDATE] = done_update_req,
+ [PR_REQ_DELETE] = done_delete_req,
+ [PR_REQ_CLOSE] = done_close_req
+};
+
+static void pr_request_done(int fd, int events, void *data)
+{
+ struct pr_request *req;
+ int ack, ret;
+
+ ret = read(done_fd[0], &ack, sizeof(ack));
+ if (ret < 0) {
+ eprintf("wrong wakeup\n");
+ return;
+ }
+
+ pthread_mutex_lock(&done_lock);
+ if (list_empty(&done_list)) {
+ pthread_mutex_unlock(&done_lock);
+ return;
+ }
+
+ req = list_first_entry(&done_list,
+ struct pr_request, list);
+ list_del(&req->list);
+ pthread_mutex_unlock(&done_lock);
+
+ done_handlers[req->type](req);
+ free(req);
+}
+
+
+/* PR thread logic */
+
+static void clear_regs(struct list_head *list)
+{
+ struct registration *reg;
+
+ while (!list_empty(list)) {
+ reg = list_first_entry(list, struct registration,
registration_siblings);
+ list_del(®->registration_siblings);
+ free(reg);
+ }
+}
+
+static int enlarge_store(struct pr_store *prs, int nslots)
+{
+ off_t filesz;
+ int bsz_old, bsz_new;
+ void *ptr;
+ int i;
+
+ if (prs->nslots >= nslots)
+ return 0;
+
+ filesz = (off_t)nslots * sizeof(struct stored_registration);
+ if (posix_fallocate(prs->fd, 0, filesz) < 0) {
+ eprintf("failed to preallocate PR file: %s\n",
strerror(errno));
+ return -1;
+ }
+
+ bsz_old = bitset_slots_to_bytes(prs->nslots);
+ bsz_new = bitset_slots_to_bytes(nslots);
+ ptr = realloc(prs->free_slot_mask, bsz_new);
+ if (!ptr) {
+ eprintf("memory allocation failed\n");
+ return -1;
+ }
+ prs->free_slot_mask = ptr;
+ memset(ptr + bsz_old, 0, bsz_new - bsz_old);
+
+ for (i = prs->nslots; i < nslots; ++i)
+ bitset_set(prs->free_slot_mask, i);
+ prs->nslots = nslots;
+
+ return 0;
+}
+
+static int check_reservation(struct scsi_lu *lu, struct registration
*reg) +{
+ if (!reg->st.pr_type && !reg->st.pr_scope)
+ return 0;
+
+ switch (reg->st.pr_type) {
+ case PR_TYPE_WRITE_EXCLUSIVE:
+ case PR_TYPE_EXCLUSIVE_ACCESS:
+ case PR_TYPE_WRITE_EXCLUSIVE_REGONLY:
+ case PR_TYPE_EXCLUSIVE_ACCESS_REGONLY:
+ case PR_TYPE_WRITE_EXCLUSIVE_ALLREG:
+ case PR_TYPE_EXCLUSIVE_ACCESS_ALLREG:
+ break;
+ default:
+ eprintf("invalid PR type: %u\n", reg->st.pr_type);
+ return -1;
+ }
+
+ if (reg->st.pr_scope != PR_LU_SCOPE) {
+ eprintf("invalid PR scope: %u\n", reg->st.pr_scope);
+ return -1;
+ }
+
+ if (!lu->pr_holder)
+ lu->pr_holder = reg;
+
+ return 0;
+}
+
+static tgtadm_err open_store(struct scsi_lu *lu)
+{
+ /* safe to use pr_dir here because we
+ * checked that it is set already */
+ char path[strlen(lu->tgt->pr_dir) + 18];
+ struct pr_store *prs = &lu->prs;
+ struct stat st;
+ struct registration *reg = NULL;
+ LIST_HEAD(tmplist);
+ ssize_t len;
+ int i;
+
+ if (snprintf(path, sizeof(path), "%s/%016llx", lu->tgt->pr_dir,
+ (unsigned long long)lu->lun) !=
sizeof(path) - 1) {
+ eprintf("failed to snprintf PR path\n");
+ return TGTADM_UNKNOWN_ERR;
+ }
+
+ prs->fd = open(path, O_RDWR | O_CLOEXEC | O_CREAT, 0644);
+ if (prs->fd < 0) {
+ eprintf("failed to open PR file: %s\n",
strerror(errno));
+ return TGTADM_UNKNOWN_ERR;
+ }
+
+ if (fstat(prs->fd, &st) < 0) {
+ eprintf("failed to stat PR file: %s\n",
strerror(errno));
+ goto fail_close;
+ }
+
+ prs->nslots = st.st_size / sizeof(struct stored_registration);
+ prs->free_slot_mask =
zalloc(bitset_slots_to_bytes(prs->nslots));
+ if (!prs->free_slot_mask) {
+ eprintf("memory allocation failed\n");
+ goto fail_close;
+ }
+
+ for (i = 0; i < prs->nslots; ++i) {
+ if (!reg) {
+ reg = zalloc(sizeof(struct registration));
+ if (!reg) {
+ eprintf("memory allocation failed\n");
+ goto fail_free_regs;
+ }
+ }
+ len = sread(prs->fd, ®->st, sizeof(struct
stored_registration));
+ if (len != sizeof(struct stored_registration)) {
+ if (len < 0)
+ eprintf("failed to read PR file:
%s\n", strerror(errno));
+ else if (len > 0)
+ eprintf("partial read from PR file\n");
+ else
+ eprintf("unexpected end of PR file\n");
+ free(reg);
+ goto fail_free_regs;
+ }
+
+ if (reg->st.key && (check_reservation(lu, reg) == 0)) {
+ reg->slot = i;
+ list_add_tail(®->registration_siblings,
&tmplist);
+ reg = NULL;
+ } else
+ bitset_set(prs->free_slot_mask, i);
+ }
+
+ if (enlarge_store(prs, PR_INIT_SLOTS) < 0)
+ goto fail_free_regs;
+
+ list_splice_init(&tmplist, &lu->registration_list);
+
+ return TGTADM_SUCCESS;
+
+fail_free_regs:
+ clear_regs(&tmplist);
+ free(prs->free_slot_mask);
+ prs->free_slot_mask = NULL;
+fail_close:
+ prs->nslots = 0;
+ close(prs->fd);
+ prs->fd = -1;
+ prs->enable = 0;
+ return TGTADM_UNKNOWN_ERR;
+}
+
+static void handle_open_req(struct pr_request *req)
+{
+ open_store(req->open.lu);
+}
+
+static int flush_reg(int fd, struct stored_registration *reg, off_t
slot) +{
+ if (lseek(fd, slot * sizeof(struct stored_registration),
SEEK_SET) < 0) {
+ eprintf("failed to lseek in PR file: %s\n",
strerror(errno));
+ return -1;
+ }
+ if (swrite(fd, reg, sizeof(struct stored_registration)) < 0) {
+ eprintf("failed to write to PR file: %s\n",
strerror(errno));
+ return -1;
+ }
+ if (fdatasync(fd) < 0) {
+ eprintf("failed to fdatasync PR file: %s\n",
strerror(errno));
+ return -1;
+ }
+ return 0;
+}
+
+static void handle_update_req(struct pr_request *req)
+{
+ struct pr_store *prs = &req->update.cmd->dev->prs;
+ struct registration *reg = req->update.reg;
+
+ if (!prs->enable)
+ return;
+
+ if (reg->slot == -1) {
+ reg->slot = bitset_ffs(prs->free_slot_mask,
prs->nslots);
+ if (reg->slot == -1) {
+ if (enlarge_store(prs, prs->nslots * 2) < 0) {
+ scsi_set_result(req->update.cmd,
SAM_STAT_CHECK_CONDITION);
+ return;
+ }
+ reg->slot = bitset_ffs(prs->free_slot_mask,
prs->nslots);
+ }
+ bitset_unset(prs->free_slot_mask, reg->slot);
+ }
+
+ if (flush_reg(prs->fd, ®->st, reg->slot) < 0)
+ scsi_set_result(req->update.cmd,
SAM_STAT_CHECK_CONDITION); +}
+
+static void handle_delete_req(struct pr_request *req)
+{
+ struct pr_store *prs = &req->del.cmd->dev->prs;
+ struct stored_registration buf;
+
+ if (!prs->enable)
+ return;
+
+ bitset_set(prs->free_slot_mask, req->del.slot);
+ memset(&buf, 0, sizeof(struct stored_registration));
+ if (flush_reg(prs->fd, &buf, req->del.slot) < 0)
+ scsi_set_result(req->del.cmd,
SAM_STAT_CHECK_CONDITION); +}
+
+static void handle_close_req(struct pr_request *req)
+{
+ struct scsi_lu *lu = req->close.lu;
+ struct pr_store *prs = &lu->prs;
+
+ if (!prs->enable)
+ return;
+
+ free(prs->free_slot_mask);
+ prs->free_slot_mask = NULL;
+ prs->nslots = 0;
+ if (close(lu->prs.fd) < 0)
+ eprintf("error when closing PR file:%s\n",
strerror(errno));
+ lu->prs.fd = -1;
+}
+
+reqhandler_t incoming_handlers[] = {
+ [PR_REQ_OPEN] = handle_open_req,
+ [PR_REQ_UPDATE] = handle_update_req,
+ [PR_REQ_DELETE] = handle_delete_req,
+ [PR_REQ_CLOSE] = handle_close_req
+};
+
+static void *pr_thread_fn(void *arg)
+{
+ int nr = 1;
+ struct pr_request *req;
+
+ while (pr_thread_run) {
+ pthread_mutex_lock(&incoming_lock);
+ while (list_empty(&incoming_list))
+ pthread_cond_wait(&incoming_cond,
&incoming_lock); +
+ req = list_first_entry(&incoming_list,
+ struct pr_request, list);
+ list_del(&req->list);
+ pthread_mutex_unlock(&incoming_lock);
+
+ incoming_handlers[req->type](req);
+
+ pthread_mutex_lock(&done_lock);
+ list_add_tail(&req->list, &done_list);
+ pthread_mutex_unlock(&done_lock);
+
+ if (swrite(done_fd[1], &nr, sizeof(nr)) < 0)
+ eprintf("can't ack tgtd\n");
+ }
+
+ pthread_exit(NULL);
+}
+
+
+/* user API */
+
+int prstore_engine_init(void)
+{
+ int ret;
+
+ pthread_cond_init(&incoming_cond, NULL);
+ pthread_mutex_init(&incoming_lock, NULL);
+ pthread_mutex_init(&done_lock, NULL);
+
+ ret = pipe(done_fd);
+ if (ret) {
+ eprintf("failed to create done pipe, %m\n");
+ goto destroy_cond_mutex;
+ }
+
+ ret = tgt_event_add(done_fd[0], EPOLLIN, pr_request_done,
NULL);
+ if (ret) {
+ eprintf("failed to add epoll event\n");
+ goto close_done_fd;
+ }
+
+ pr_thread_run = 1;
+ ret = pthread_create(&pr_thread, NULL, pr_thread_fn, NULL);
+ if (ret) {
+ eprintf("failed to create prstore pr thread, %s\n",
strerror(ret));
+ goto event_del;
+ }
+
+ return 0;
+
+event_del:
+ tgt_event_del(done_fd[0]);
+close_done_fd:
+ close(done_fd[0]);
+ close(done_fd[1]);
+destroy_cond_mutex:
+ pthread_cond_destroy(&incoming_cond);
+ pthread_mutex_destroy(&incoming_lock);
+ pthread_mutex_destroy(&done_lock);
+
+ return 1;
+}
+
+static void prstore_enqueue(struct pr_request *req)
+{
+ pthread_mutex_lock(&incoming_lock);
+ list_add_tail(&req->list, &incoming_list);
+ pthread_cond_signal(&incoming_cond);
+ pthread_mutex_unlock(&incoming_lock);
+}
+
+int prstore_open(struct scsi_lu *lu)
+{
+ struct pr_request *req;
+
+ if (!lu->tgt->pr_dir)
+ return 0;
+
+ if (lu->prs.enable)
+ /* store is already open */
+ return 0;
+
+ req = zalloc(sizeof(struct pr_request));
+ if (!req) {
+ eprintf("memory allocation failed\n");
+ return 1;
+ }
+
+ lu->prs.enable = 1;
+
+ req->type = PR_REQ_OPEN;
+ req->open.lu = lu;
+
+ prstore_enqueue(req);
+ return 0;
+}
+
+int prstore_close(struct scsi_lu *lu)
+{
+ struct pr_request *req;
+
+ if (!lu->prs.enable)
+ return 0;
+
+ req = zalloc(sizeof(struct pr_request));
+ if (!req) {
+ eprintf("memory allocation failed\n");
+ return 1;
+ }
+
+ req->type = PR_REQ_CLOSE;
+ req->close.lu = lu;
+
+ prstore_enqueue(req);
+ return 0;
+}
+
+int prstore_update(struct registration *reg, struct scsi_cmd *cmd)
+{
+ struct pr_request *req;
+
+ if (!cmd->dev->prs.enable)
+ return 0;
+
+ req = zalloc(sizeof(struct pr_request));
+ if (!req) {
+ eprintf("memory allocation failed\n");
+ return 1;
+ }
+
+ req->type = PR_REQ_UPDATE;
+ req->update.reg = reg;
+ req->update.cmd = cmd;
+
+ /* must flush reg before notifying client */
+ set_cmd_async(cmd);
+
+ prstore_enqueue(req);
+ return 0;
+}
+
+int prstore_delete(int slot, struct scsi_cmd *cmd)
+{
+ struct pr_request *req;
+
+ if (!cmd->dev->prs.enable)
+ return 0;
+
+ req = zalloc(sizeof(struct pr_request));
+ if (!req) {
+ eprintf("memory allocation failed\n");
+ return 1;
+ }
+
+ req->type = PR_REQ_DELETE;
+ req->del.slot = slot;
+ req->del.cmd = cmd;
+
+ /* must flush reg before notifying client */
+ set_cmd_async(cmd);
+
+ prstore_enqueue(req);
+ return 0;
+}
+
+void prstore_engine_deinit(void)
+{
+ pr_thread_run = 0;
+ pthread_join(pr_thread, NULL);
+}
diff --git a/usr/prstore.h b/usr/prstore.h
new file mode 100644
index 0000000..cbc5eb7
--- /dev/null
+++ b/usr/prstore.h
@@ -0,0 +1,30 @@
+#ifndef __PRSTORE_H
+#define __PRSTORE_H
+
+#include "list.h"
+#include "tgtd.h"
+
+struct registration;
+
+struct pr_store {
+ int enable;
+ int fd;
+ int nslots;
+ long *free_slot_mask;
+};
+
+/* start prstore thread */
+int prstore_engine_init(void);
+/* stop prstore thread and flush queues */
+void prstore_engine_deinit(void);
+
+/* open store for a LUN */
+int prstore_open(struct scsi_lu *lu);
+/* close store for a LUN */
+int prstore_close(struct scsi_lu *lu);
+/* update slot in a store */
+int prstore_update(struct registration *reg, struct scsi_cmd *cmd);
+/* free slot in a store */
+int prstore_delete(int slot, struct scsi_cmd *cmd);
+
+#endif
diff --git a/usr/spc.c b/usr/spc.c
index 00e0dd7..b12c9e4 100644
--- a/usr/spc.c
+++ b/usr/spc.c
@@ -35,6 +35,7 @@
#include "tgtadm_error.h"
#include "scsi.h"
#include "spc.h"
+#include "prstore.h"
#define INQUIRY_EVPD 0x01
@@ -972,8 +973,8 @@ int spc_service_action(int host_no, struct scsi_cmd
*cmd)
static int is_pr_holder(struct scsi_lu *lu, struct registration *reg)
{
- if (lu->pr_holder->pr_type == PR_TYPE_WRITE_EXCLUSIVE_ALLREG ||
- lu->pr_holder->pr_type == PR_TYPE_EXCLUSIVE_ACCESS_ALLREG)
+ if (lu->pr_holder->st.pr_type ==
PR_TYPE_WRITE_EXCLUSIVE_ALLREG ||
+ lu->pr_holder->st.pr_type ==
PR_TYPE_EXCLUSIVE_ACCESS_ALLREG) return 1;
if (lu->pr_holder == reg)
@@ -1008,7 +1009,7 @@ static int spc_pr_read_keys(int host_no, struct
scsi_cmd *cmd) list_for_each_entry(reg, &cmd->dev->registration_list,
registration_siblings) {
- reg_key = __cpu_to_be64(reg->key);
+ reg_key = __cpu_to_be64(reg->st.key);
actual_len += spc_memcpy(&buf[actual_len], &remain_len,
(uint8_t *)®_key, 8);
avail_len += 8;
@@ -1052,16 +1053,16 @@ static int spc_pr_read_reservation(int host_no,
struct scsi_cmd *cmd) put_unaligned_be32(add_len, &buf[4]); /*
additional length */
if (reg) {
- if (reg->pr_type == PR_TYPE_WRITE_EXCLUSIVE_ALLREG ||
- reg->pr_type == PR_TYPE_EXCLUSIVE_ACCESS_ALLREG)
+ if (reg->st.pr_type == PR_TYPE_WRITE_EXCLUSIVE_ALLREG
||
+ reg->st.pr_type == PR_TYPE_EXCLUSIVE_ACCESS_ALLREG)
res_key = 0;
else
- res_key = reg->key;
+ res_key = reg->st.key;
put_unaligned_be64(res_key, &buf[8]);
- buf[21] = (reg->pr_scope << 4) & 0xf0;
- buf[21] |= reg->pr_type & 0x0f;
+ buf[21] = (reg->st.pr_scope << 4) & 0xf0;
+ buf[21] |= reg->st.pr_type & 0x0f;
}
data = scsi_get_in_buffer(cmd);
@@ -1094,7 +1095,10 @@ static int spc_pr_report_capabilities(int
host_no, struct scsi_cmd *cmd)
put_unaligned_be16(avail_len, &buf[0]); /* length */
- /* we don't set any capability for now */
+ /* set only PTPL capabilities for now */
+ buf[2] |= 0x1; /* PTPL_C */
+ if (cmd->dev->prs.enable)
+ buf[3] |= 0x1; /* PTPL_A */
/* Persistent Reservation Type Mask format */
buf[3] |= 0x80; /* Type Mask Valid (TMV) */
@@ -1138,22 +1142,61 @@ static struct registration
*lookup_registration_by_nexus(struct scsi_lu *lu, return NULL;
}
+static struct registration *lookup_registration_by_iname(struct
scsi_lu *lu,
+ char *iname, unsigned int ilen)
+{
+ struct registration *reg;
+
+ list_for_each_entry(reg, &lu->registration_list,
registration_siblings) {
+ if (reg->st.ilen == ilen &&
+ memcmp(reg->st.iname, iname, ilen) == 0)
+ return reg;
+ }
+
+ return NULL;
+}
+
+static struct registration
*lookup_registration_by_nexus_or_iname(struct scsi_lu *lu,
+ struct it_nexus *itn)
+{
+ int (*id)(int, uint64_t, char *, int);
+ struct registration *reg;
+
+ id = tgt_drivers[lu->tgt->lid]->transportid;
+
+ reg = lookup_registration_by_nexus(lu, itn);
+ if (!reg && id) {
+ char buf[223];
+ int len;
+ len = id(lu->tgt->tid, itn->itn_id, buf, sizeof(buf));
+ reg = lookup_registration_by_iname(lu, buf, len);
+ if (reg) {
+ reg->nexus_id = itn->itn_id;
+ reg->ctime = itn->ctime;
+ }
+ }
+
+ return reg;
+}
+
static int check_registration_key_exists(struct scsi_lu *lu, uint64_t
key) {
struct registration *reg;
list_for_each_entry(reg, &lu->registration_list,
registration_siblings) {
- if (reg->key == key)
+ if (reg->st.key == key)
return 0;
}
return 1;
}
-static void __unregister(struct scsi_lu *lu, struct registration *reg)
+static void __unregister(struct scsi_cmd *cmd, struct registration
*reg) {
list_del(®->registration_siblings);
+ /* cleanup slot */
+ //prstore_delete(reg->slot, cmd);
free(reg);
}
@@ -1165,6 +1208,8 @@ static void __unregister_and_clean(struct
scsi_cmd *cmd,
/* if reservation owner goes away then so does reservation */
list_del(®->registration_siblings);
+ /* cleanup slot */
+ prstore_delete(reg->slot, cmd);
holder = cmd->dev->pr_holder;
if (!holder) {
@@ -1177,14 +1222,14 @@ static void __unregister_and_clean(struct
scsi_cmd *cmd, return;
}
- if (((holder->pr_type != PR_TYPE_WRITE_EXCLUSIVE_ALLREG) &&
- (holder->pr_type != PR_TYPE_EXCLUSIVE_ACCESS_ALLREG)) ||
+ if (((holder->st.pr_type != PR_TYPE_WRITE_EXCLUSIVE_ALLREG) &&
+ (holder->st.pr_type != PR_TYPE_EXCLUSIVE_ACCESS_ALLREG))
|| list_empty(&cmd->dev->registration_list)) {
/* not all-registrants or no more registrants */
- holder->pr_scope = 0;
- holder->pr_type = 0;
+ holder->st.pr_scope = 0;
+ holder->st.pr_type = 0;
cmd->dev->pr_holder = NULL;
/* tell other registrants the reservation went away */
@@ -1209,8 +1254,8 @@ static void __unregister_and_clean(struct
scsi_cmd *cmd, if (registrant != reg) {
/* give resvn to any sibling */
cmd->dev->pr_holder = registrant;
- registrant->pr_scope =
holder->pr_scope;
- registrant->pr_type = holder->pr_type;
+ registrant->st.pr_scope =
holder->st.pr_scope;
+ registrant->st.pr_type =
holder->st.pr_type; break;
}
}
@@ -1269,18 +1314,23 @@ static int spc_pr_register(int host_no, struct
scsi_cmd *cmd) res_key = get_unaligned_be64(buf);
sa_res_key = get_unaligned_be64(buf + 8);
- reg = lookup_registration_by_nexus(cmd->dev, cmd->it_nexus);
+ reg = lookup_registration_by_nexus_or_iname(cmd->dev,
cmd->it_nexus); if (reg) {
- if (force || reg->key == res_key) {
- if (sa_res_key)
- reg->key = sa_res_key;
- else
+ if (force || reg->st.key == res_key) {
+ if (sa_res_key) {
+ reg->st.key = sa_res_key;
+ /* will make cmd async */
+ prstore_update(reg, cmd);
+ } else
__unregister_and_clean(cmd, reg);
} else
return SAM_STAT_RESERVATION_CONFLICT;
} else {
if (force || !res_key) {
if (sa_res_key) {
+ int (*id)(int, uint64_t, char *, int);
+ id =
tgt_drivers[cmd->dev->tgt->lid]->transportid; +
reg = zalloc(sizeof(*reg));
if (!reg) {
key = ILLEGAL_REQUEST;
@@ -1288,11 +1338,21 @@ static int spc_pr_register(int host_no, struct
scsi_cmd *cmd) goto sense;
}
- reg->key = sa_res_key;
+ if (id)
+ reg->st.ilen =
id(cmd->dev->tgt->tid,
+
cmd->cmd_itn_id,
+ reg->st.iname,
+
sizeof(reg->st.iname)); +
+ reg->st.key = sa_res_key;
reg->nexus_id = cmd->cmd_itn_id;
reg->ctime = cmd->it_nexus->ctime;
+ reg->slot = -1;
list_add_tail(®->registration_siblings,
&cmd->dev->registration_list);
+
+ /* will make cmd async */
+ prstore_update(reg, cmd);
} else
; /* do nothing */
} else
@@ -1338,7 +1398,7 @@ static int spc_pr_reserve(int host_no, struct
scsi_cmd *cmd) if (pr_scope != PR_LU_SCOPE)
goto sense;
- reg = lookup_registration_by_nexus(cmd->dev, cmd->it_nexus);
+ reg = lookup_registration_by_nexus_or_iname(cmd->dev,
cmd->it_nexus); if (!reg)
return SAM_STAT_RESERVATION_CONFLICT;
@@ -1347,16 +1407,18 @@ static int spc_pr_reserve(int host_no, struct
scsi_cmd *cmd) if (!is_pr_holder(cmd->dev, reg))
return SAM_STAT_RESERVATION_CONFLICT;
- if (holder->pr_type != pr_type ||
- holder->pr_scope != pr_scope)
+ if (holder->st.pr_type != pr_type ||
+ holder->st.pr_scope != pr_scope)
return SAM_STAT_RESERVATION_CONFLICT;
return SAM_STAT_GOOD;
}
- reg->pr_scope = pr_scope;
- reg->pr_type = pr_type;
+ reg->st.pr_scope = pr_scope;
+ reg->st.pr_type = pr_type;
cmd->dev->pr_holder = reg;
+ /* will make cmd async */
+ prstore_update(reg, cmd);
return SAM_STAT_GOOD;
sense:
@@ -1386,7 +1448,7 @@ static int spc_pr_release(int host_no, struct
scsi_cmd *cmd)
res_key = get_unaligned_be64(buf);
- reg = lookup_registration_by_nexus(cmd->dev, cmd->it_nexus);
+ reg = lookup_registration_by_nexus_or_iname(cmd->dev,
cmd->it_nexus); if (!reg)
return SAM_STAT_RESERVATION_CONFLICT;
@@ -1397,17 +1459,19 @@ static int spc_pr_release(int host_no, struct
scsi_cmd *cmd) if (!is_pr_holder(cmd->dev, reg))
return SAM_STAT_GOOD;
- if (res_key != reg->key)
+ if (res_key != reg->st.key)
return SAM_STAT_RESERVATION_CONFLICT;
- if (holder->pr_scope != pr_scope || holder->pr_type !=
pr_type) {
+ if (holder->st.pr_scope != pr_scope || holder->st.pr_type !=
pr_type) { asc = ASC_INVALID_RELEASE_OF_PERSISTENT_RESERVATION;
goto sense;
}
cmd->dev->pr_holder = NULL;
- reg->pr_scope = 0;
- reg->pr_type = 0;
+ reg->st.pr_scope = 0;
+ reg->st.pr_type = 0;
+ /* will make cmd async */
+ prstore_update(reg, cmd);
switch (pr_type) {
case PR_TYPE_WRITE_EXCLUSIVE_REGONLY:
@@ -1453,17 +1517,17 @@ static int spc_pr_clear(int host_no, struct
scsi_cmd *cmd)
res_key = get_unaligned_be64(buf);
- reg = lookup_registration_by_nexus(cmd->dev, cmd->it_nexus);
+ reg = lookup_registration_by_nexus_or_iname(cmd->dev,
cmd->it_nexus); if (!reg)
return SAM_STAT_RESERVATION_CONFLICT;
- if (reg->key != res_key)
+ if (reg->st.key != res_key)
return SAM_STAT_RESERVATION_CONFLICT;
holder = cmd->dev->pr_holder;
if (holder) {
- holder->pr_scope = 0;
- holder->pr_type = 0;
+ holder->st.pr_scope = 0;
+ holder->st.pr_type = 0;
cmd->dev->pr_holder = NULL;
}
@@ -1508,11 +1572,11 @@ static int spc_pr_preempt(int host_no, struct
scsi_cmd *cmd) res_key = get_unaligned_be64(buf);
sa_res_key = get_unaligned_be64(buf + 8);
- reg = lookup_registration_by_nexus(cmd->dev, cmd->it_nexus);
+ reg = lookup_registration_by_nexus_or_iname(cmd->dev,
cmd->it_nexus); if (!reg)
return SAM_STAT_RESERVATION_CONFLICT;
- if (reg->key != res_key)
+ if (reg->st.key != res_key)
return SAM_STAT_RESERVATION_CONFLICT;
if (sa_res_key) {
@@ -1524,25 +1588,25 @@ static int spc_pr_preempt(int host_no, struct
scsi_cmd *cmd) holder = cmd->dev->pr_holder;
if (holder) {
- if (holder->pr_type == PR_TYPE_WRITE_EXCLUSIVE_ALLREG
||
- holder->pr_type ==
PR_TYPE_EXCLUSIVE_ACCESS_ALLREG) {
+ if (holder->st.pr_type ==
PR_TYPE_WRITE_EXCLUSIVE_ALLREG ||
+ holder->st.pr_type ==
PR_TYPE_EXCLUSIVE_ACCESS_ALLREG) {
if (!sa_res_key) {
- if (pr_type != holder->pr_type ||
- pr_scope != holder->pr_scope)
+ if (pr_type != holder->st.pr_type ||
+ pr_scope !=
holder->st.pr_scope) res_released = 1;
- reg->pr_type = pr_type;
- reg->pr_scope = pr_scope;
+ reg->st.pr_type = pr_type;
+ reg->st.pr_scope = pr_scope;
cmd->dev->pr_holder = reg;
remove_all_reg = 1;
}
} else {
- if (holder->key == sa_res_key) {
- if ((pr_type != holder->pr_type) ||
- (pr_scope != holder->pr_scope))
+ if (holder->st.key == sa_res_key) {
+ if ((pr_type != holder->st.pr_type) ||
+ (pr_scope !=
holder->st.pr_scope)) res_released = 1;
- reg->pr_type = pr_type;
- reg->pr_scope = pr_scope;
+ reg->st.pr_type = pr_type;
+ reg->st.pr_scope = pr_scope;
cmd->dev->pr_holder = reg;
} else {
if (!sa_res_key)
@@ -1556,10 +1620,10 @@ static int spc_pr_preempt(int host_no, struct
scsi_cmd *cmd) if (sibling == reg)
continue;
- if (sibling->key == sa_res_key || remove_all_reg) {
+ if (sibling->st.key == sa_res_key || remove_all_reg) {
ua_sense_add_it_nexus(sibling->nexus_id,
cmd->dev, ASC_RESERVATIONS_PREEMPTED);
- __unregister(cmd->dev, sibling);
+ __unregister(cmd, sibling);
} else {
if (res_released)
ua_sense_add_it_nexus(sibling->nexus_id,
@@ -1612,7 +1676,7 @@ static int spc_pr_register_and_move(int host_no,
struct scsi_cmd *cmd) if (param_list_len - 24 < tpid_data_len)
goto sense;
- reg = lookup_registration_by_nexus(cmd->dev, cmd->it_nexus);
+ reg = lookup_registration_by_nexus_or_iname(cmd->dev,
cmd->it_nexus); if (!reg) {
if (cmd->dev->pr_holder)
return SAM_STAT_RESERVATION_CONFLICT;
@@ -1623,7 +1687,7 @@ static int spc_pr_register_and_move(int host_no,
struct scsi_cmd *cmd) if (!is_pr_holder(cmd->dev, reg))
return SAM_STAT_GOOD;
- if (reg->key != res_key)
+ if (reg->st.key != res_key)
return SAM_STAT_RESERVATION_CONFLICT;
if (!sa_res_key)
@@ -1631,7 +1695,7 @@ static int spc_pr_register_and_move(int host_no,
struct scsi_cmd *cmd)
list_for_each_entry(dst, &cmd->dev->registration_list,
registration_siblings)
- if (dst->key == sa_res_key)
+ if (dst->st.key == sa_res_key)
goto found;
/* we can't find the destination */
@@ -1648,7 +1712,7 @@ found:
cmd->dev->pr_holder = dst;
if (unreg)
- __unregister(cmd->dev, reg);
+ __unregister(cmd, reg);
cmd->dev->prgeneration++;
return SAM_STAT_GOOD;
@@ -1681,12 +1745,12 @@ int spc_access_check(struct scsi_cmd *cmd)
if (!cmd->dev->pr_holder)
return 0;
- reg = lookup_registration_by_nexus(cmd->dev, cmd->it_nexus);
+ reg = lookup_registration_by_nexus_or_iname(cmd->dev,
cmd->it_nexus);
if (reg && is_pr_holder(cmd->dev, reg))
return 0;
- pr_type = cmd->dev->pr_holder->pr_type;
+ pr_type = cmd->dev->pr_holder->st.pr_type;
bits = cmd->dev->dev_type_template.ops[op].pr_conflict_bits;
if (pr_type == PR_TYPE_EXCLUSIVE_ACCESS)
@@ -2078,6 +2142,9 @@ int spc_lu_init(struct scsi_lu *lu)
lu->dev_type_template.lu_offline(lu);
+ if (prstore_open(lu))
+ return -ENOMEM;
+
return 0;
}
@@ -2098,4 +2165,6 @@ void spc_lu_exit(struct scsi_lu *lu)
list_del(&pg->mode_pg_siblings);
free(pg);
}
+
+ prstore_close(lu);
}
diff --git a/usr/target.c b/usr/target.c
index b1729b3..8f73433 100644
--- a/usr/target.c
+++ b/usr/target.c
@@ -587,6 +587,7 @@ tgtadm_err tgt_device_create(int tid, int dev_type,
uint64_t lun, char *params, INIT_LIST_HEAD(&lu->mode_pages);
lu->prgeneration = 0;
lu->pr_holder = NULL;
+ lu->prs.fd = -1;
lu->cmd_perform = &target_cmd_perform;
lu->cmd_done = &__cmd_done;
@@ -1935,6 +1936,21 @@ tgtadm_err tgt_set_target_state(int tid, char
*str) return adm_err;
}
+tgtadm_err tgt_set_target_pr_dir(int tid, char *str)
+{
+ struct target *target;
+
+ target = target_lookup(tid);
+ if (!target)
+ return TGTADM_NO_TARGET;
+
+ target->pr_dir = strdup(str);
+ if (!target->pr_dir)
+ return TGTADM_NOMEM;
+
+ return TGTADM_SUCCESS;
+}
+
static char *print_disksize(uint64_t size)
{
static char buf[64];
@@ -2228,6 +2244,8 @@ tgtadm_err tgt_target_destroy(int lld_no, int
tid, int force)
list_del(&target->lld_siblings);
+ if (target->pr_dir)
+ free(target->pr_dir);
free(target->account.in_aids);
free(target->name);
free(target);
diff --git a/usr/target.h b/usr/target.h
index 1750d4c..69d4b20 100644
--- a/usr/target.h
+++ b/usr/target.h
@@ -43,6 +43,8 @@ struct target {
struct tgt_account account;
struct list_head lld_siblings;
+
+ char *pr_dir;
};
struct it_nexus {
diff --git a/usr/tgtd.c b/usr/tgtd.c
index f985510..52d4c14 100644
--- a/usr/tgtd.c
+++ b/usr/tgtd.c
@@ -42,6 +42,7 @@
#include "driver.h"
#include "work.h"
#include "util.h"
+#include "prstore.h"
unsigned long pagesize, pageshift;
@@ -608,8 +609,12 @@ int main(int argc, char **argv)
bs_init();
+ prstore_engine_init();
+
event_loop();
+ prstore_engine_deinit();
+
lld_exit();
work_timer_stop();
diff --git a/usr/tgtd.h b/usr/tgtd.h
index 484e6e9..120bb72 100644
--- a/usr/tgtd.h
+++ b/usr/tgtd.h
@@ -4,6 +4,7 @@
#include "log.h"
#include "scsi_cmnd.h"
#include "tgtadm_error.h"
+#include "prstore.h"
struct concat_buf;
@@ -177,14 +178,23 @@ struct mode_pg {
uint8_t mode_data[0]; /* Rest of mode page info */
};
-struct registration {
+struct stored_registration {
uint64_t key;
- uint64_t nexus_id;
- long ctime;
- struct list_head registration_siblings;
+ /* iSCSI names must be 223 bytes or less
+ * according to RFC 3270 section 3.2.6.1 */
+ char iname[240];
+ uint8_t ilen;
uint8_t pr_scope;
uint8_t pr_type;
+} __attribute__ ((aligned(256)));
+
+struct registration {
+ struct stored_registration st;
+ int slot;
+ uint64_t nexus_id;
+ long ctime;
+ struct list_head registration_siblings;
};
struct scsi_lu {
@@ -220,6 +230,7 @@ struct scsi_lu {
struct list_head registration_list;
uint32_t prgeneration;
struct registration *pr_holder;
+ struct pr_store prs;
/* A pointer for each modules private use.
* Currently used by ssc, smc and mmc modules.
@@ -333,6 +344,7 @@ extern const unsigned char
*get_scsi_cdb_usage_data(unsigned char op,
extern enum scsi_target_state tgt_get_target_state(int tid);
extern tgtadm_err tgt_set_target_state(int tid, char *str);
+extern tgtadm_err tgt_set_target_pr_dir(int tid, char *str);
extern tgtadm_err acl_add(int tid, char *address);
extern tgtadm_err acl_del(int tid, char *address);
--
1.8.4.rc3
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo at vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
More information about the stgt
mailing list