[Stgt-devel] uSpace Transport Patch

Tom Tucker tom
Tue Aug 22 14:39:53 CEST 2006


On Tue, 2006-08-22 at 08:11 +0900, FUJITA Tomonori wrote:
> From: Tom Tucker <tom at opengridcomputing.com>
> Subject: [Stgt-devel] uSpace Transport Patch
> Date: Mon, 21 Aug 2006 16:29:40 -0500
> 
> > Tomo:
> > 
> > Enclosed is a patch that allows you to plug in multiple transports. It
> > has a few benefits over the last approach:
> 
> Thanks.
> 
> 
> > 1. The TCP side can remain exactly the same. i.e. user-mode connection
> > management and login send/recv.
> > 
> > 2. The stgtd implementation still uses pollfd to receive I/O events. The
> >    iser side will provide an fd that can be polled.
> > 
> > 
> > I have built and run this patch with the current code and connected with
> > a iscsi initiator over TCP. I did encounter problems, however, trying to
> > do disk i/o.
> 
> The write path code is broken. I will fix it if the kernel-mode
> approach would likely be accepted into mainline.
> 
> The user-space mode code should work better. As I said in the previous
> mail, I can do mkfs, extract linux kernel tar, etc with the open-iscsi
> default configuration.
> 
> 
> > This is not done, it is a proof-of-concept/design proposal. The netlink
> > stuff will obviously change as I flesh out the iSER side. Please let me
> > know what you think, I'd like your opinion before I get to far down this
> > road.

> I see. I will read this soon. Can you post iSER part code too?

Yes, I'm working on that now, I just didn't want to get too far down a
road that lead somewhere we didn't want to go. Basically, the iSER code
is implemented in the kernel. The user-mode interface is a file
descriptor (used for polling), and a either a) set of methods
implemented via read/write for listening for, accepting and
send/receiving, or b) using the netlink interface. I'm leaning towards
the former actually.

Tom

> 
> 
> > Thanks, 
> > Tom
> > 
> > Index: usr/iscsi/session.c
> > ===================================================================
> > --- usr/iscsi/session.c	(revision 532)
> > +++ usr/iscsi/session.c	(working copy)
> > @@ -112,12 +112,13 @@
> >  
> >  	log_debug("session_create: %#" PRIx64, sid);
> >  
> > -	ki->create_session(thandle, conn->exp_cmd_sn, &session->ksid,
> > +	ki->create_session(conn->kth, conn->exp_cmd_sn, &session->ksid,
> >  			   &session->hostno);
> >  
> >  	list_add(&session->hlist, &sessions_list);
> >  }
> >  
> > +#if 0
> >  void session_remove(struct session *session)
> >  {
> >  	uint64_t sid = sid64(session->isid, session->tsih);
> > @@ -140,3 +141,4 @@
> >  	free(session->initiator);
> >  	free(session);
> >  }
> > +#endif
> > Index: usr/iscsi/iscsid.h
> > ===================================================================
> > --- usr/iscsi/iscsid.h	(revision 532)
> > +++ usr/iscsi/iscsid.h	(working copy)
> > @@ -9,13 +9,14 @@
> >  
> >  #include <sys/types.h>
> >  #include <linux/types.h>
> > -
> > +#include <sys/socket.h>
> > +#include <linux/socket.h>
> >  #include "types.h"
> >  #include "iscsi_if.h"
> >  #include "list.h"
> >  #include "param.h"
> >  #include "log.h"
> > -
> > +#include "iscsi_uspace_transport.h"
> >  #include <scsi/iscsi_proto.h>
> >  
> >  #define ISCSI_NAME_LEN 255
> > @@ -25,7 +26,6 @@
> >  #define DIGEST_NONE		(1 << 0)
> >  #define DIGEST_CRC32C           (1 << 1)
> >  
> > -extern uint64_t thandle;
> >  extern int nl_fd;
> >  
> >  #define sid64(isid, tsih)					\
> > @@ -69,6 +69,7 @@
> >  	int state;
> >  	int iostate;
> >  	int fd;
> > +	uint64_t kth;
> >  
> >  	struct list_head clist;
> >  	struct session *session;
> > Index: usr/iscsi/iscsi_uspace_transport.c
> > ===================================================================
> > --- usr/iscsi/iscsi_uspace_transport.c	(revision 0)
> > +++ usr/iscsi/iscsi_uspace_transport.c	(revision 0)
> > @@ -0,0 +1,118 @@
> > +#include <stdio.h>
> > +#include <stdint.h>
> > +#include <stddef.h>
> > +#include <stdlib.h>
> > +#include <errno.h>
> > +
> > +#include "list.h"
> > +#include "iscsi_uspace_transport.h"
> > +
> > +struct iut_el {
> > +	struct list_head list;
> > +	struct iut *transport;
> > +};
> > +
> > +static LIST_HEAD(iscsi_transport_list);
> > +
> > +extern struct iut iscsi_tcp;
> > +
> > +#if 0
> > +struct iut iscsi_iser = {
> > +	.name = "iser",
> > +	.rdma = 1,
> > +	.init = iser_transport_init,
> > +	.poll_init = iser_poll_init,
> > +	.ep_accept = iser_ep_accept,
> > +	.ep_close = iser_ep_close,
> > +};
> > +#endif
> > +
> > +void iut_init(void)
> > +{
> > +	struct iut_el *el;
> > +
> > +	el = malloc(sizeof(*el));
> > +	el->transport = &iscsi_tcp;
> > +	list_add(&(el->list),  &iscsi_transport_list);
> > +#if 0
> > +	el = malloc(sizeof(*el));
> > +	el->transport = &iscsi_iser;
> > +	list_add(&(el->list),  &iscsi_transport_list);
> > +#endif
> > +	list_for_each_entry(el, &iscsi_transport_list, list)
> > +		el->transport->init(el->transport);
> > +}
> > +
> > +int iut_lookup_handles(int fd, 
> > +		       iut_handle_t *pth,
> > +		       iut_ep_handle_t* peh)
> > +{
> > +	struct iut_el *el;
> > +	iut_ep_handle_t eh;
> > +	int found = 0;
> > +
> > +	list_for_each_entry(el, &iscsi_transport_list, list) {
> > +		eh = iut_ep_lookup(el->transport, fd);
> > +		if (eh) {
> > +			dprintf("found handle = %p for fd = %d\n", 
> > +				eh, fd);
> > +			*pth = el->transport;
> > +			*peh = eh;
> > +			found = 1;
> > +			break;
> > +		}
> > +	}
> > +
> > +	return found;
> > +}
> > +
> > +int iut_poll_init(struct pollfd *pfds, int listen_max)
> > +{
> > +	struct iut_el *el;
> > +	int listen_count = 0;
> > +
> > +	list_for_each_entry(el, &iscsi_transport_list, list) 
> > +		listen_count += el->transport->poll_init(el->transport, 
> > +							 pfds + listen_count, 
> > +							 listen_max - listen_count);
> > +
> > +	return listen_count;
> > +}
> > +
> > +int iut_ep_close(iut_handle_t th, iut_ep_handle_t ep_h)
> > +{
> > +	return th->ep_close(th, ep_h);
> > +}
> > +
> > +int iut_ep_accept(iut_handle_t th, iut_ep_handle_t ep_h,
> > +			 iut_ep_handle_t *new_ep_h,
> > +			 struct sockaddr *addr, socklen_t *addrlen)
> > +{
> > +	return th->ep_accept(th, ep_h, new_ep_h, addr, addrlen);
> > +}
> > +
> > +iut_ep_handle_t iut_ep_lookup(iut_handle_t th, int fd)
> > +{
> > +	return th->ep_lookup(th, fd);
> > +}
> > +
> > +int iut_fd_write(int fd, const void *buf, int buflen, int cork)
> > +{
> > +	iut_handle_t th;
> > +	iut_ep_handle_t ep_h;
> > +	if (iut_lookup_handles(fd, &th, &ep_h))
> > +		return th->ep_write(th, ep_h, buf, buflen, cork);
> > +
> > +	return -ENOENT;
> > +}
> > +
> > +int iut_fd_read(int fd, void *buf, int buflen)
> > +{
> > +	iut_handle_t th;
> > +	iut_ep_handle_t ep_h;
> > +	if (iut_lookup_handles(fd, &th, &ep_h))
> > +		return th->ep_read(th, ep_h, buf, buflen);
> > +
> > +	return -ENOENT;
> > +}
> > +
> > Index: usr/iscsi/istgt.c
> > ===================================================================
> > --- usr/iscsi/istgt.c	(revision 532)
> > +++ usr/iscsi/istgt.c	(working copy)
> > @@ -33,9 +33,8 @@
> >  #include <arpa/inet.h>
> >  
> >  #include "iscsid.h"
> > +#include "iscsi_uspace_transport.h"
> >  
> > -#define ISCSI_LISTEN_PORT	3260
> > -
> >  #define LISTEN_MAX	4
> >  #define INCOMING_MAX	32
> >  
> > @@ -47,92 +46,29 @@
> >  };
> >  
> >  static struct connection *incoming[INCOMING_MAX];
> > -uint64_t thandle;
> >  int nl_fd;
> >  
> > -static void set_non_blocking(int fd)
> > -{
> > -	int res = fcntl(fd, F_GETFL);
> > -
> > -	if (res != -1) {
> > -		res = fcntl(fd, F_SETFL, res | O_NONBLOCK);
> > -		if (res)
> > -			dprintf("unable to set fd flags (%s)!\n", strerror(errno));
> > -	} else
> > -		dprintf("unable to get fd flags (%s)!\n", strerror(errno));
> > -}
> > -
> > -static void listen_socket_create(struct pollfd *pfds)
> > -{
> > -	struct addrinfo hints, *res, *res0;
> > -	char servname[64];
> > -	int i, sock, opt;
> > -
> > -	memset(servname, 0, sizeof(servname));
> > -	snprintf(servname, sizeof(servname), "%d", ISCSI_LISTEN_PORT);
> > -
> > -	memset(&hints, 0, sizeof(hints));
> > -	hints.ai_socktype = SOCK_STREAM;
> > -	hints.ai_flags = AI_PASSIVE;
> > -
> > -	if (getaddrinfo(NULL, servname, &hints, &res0)) {
> > -		eprintf("unable to get address info (%s)!\n", strerror(errno));
> > -		exit(1);
> > -	}
> > -
> > -	for (i = 0, res = res0; res && i < LISTEN_MAX; i++, res = res->ai_next) {
> > -		sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
> > -		if (sock < 0) {
> > -			eprintf("unable to create server socket (%s) %d %d %d!\n",
> > -				  strerror(errno), res->ai_family,
> > -				  res->ai_socktype, res->ai_protocol);
> > -			continue;
> > -		}
> > -
> > -		opt = 1;
> > -		if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)))
> > -			dprintf("unable to set SO_REUSEADDR on server socket (%s)!\n",
> > -				    strerror(errno));
> > -		opt = 1;
> > -		if (res->ai_family == AF_INET6 &&
> > -		    setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt)))
> > -			continue;
> > -
> > -		if (bind(sock, res->ai_addr, res->ai_addrlen)) {
> > -			eprintf("unable to bind server socket (%s)!\n", strerror(errno));
> > -			continue;
> > -		}
> > -
> > -		if (listen(sock, INCOMING_MAX)) {
> > -			eprintf("unable to listen to server socket (%s)!\n", strerror(errno));
> > -			continue;
> > -		}
> > -
> > -		set_non_blocking(sock);
> > -
> > -		pfds[i].fd = sock;
> > -		pfds[i].events = POLLIN;
> > -	}
> > -
> > -	freeaddrinfo(res0);
> > -}
> > -
> >  static void accept_connection(struct pollfd *pfds, int afd)
> >  {
> > -	struct sockaddr_storage from;
> > -	socklen_t namesize;
> > -	struct pollfd *pfd;
> > +	struct sockaddr addr;
> > +	socklen_t addrlen;
> > +	iut_handle_t th;
> > +	iut_ep_handle_t l_eh;
> > +	iut_ep_handle_t new_eh;
> >  	struct connection *conn;
> > -	int fd, i;
> > +	struct pollfd *pfd;
> > +	int i, fd;
> > +	
> > +	if (!iut_lookup_handles(afd, &th, &l_eh)) {
> > +		eprintf("could not find transport handles "
> > +			"for specified fd=%d\n",
> > +		       afd);
> > +		return;
> > +	}
> >  
> > -	eprintf("%d\n", afd);
> > -
> > -	namesize = sizeof(from);
> > -	if ((fd = accept(afd, (struct sockaddr *) &from, &namesize)) < 0) {
> > -		if (errno != EINTR && errno != EAGAIN) {
> > -			eprintf("accept(incoming_socket)\n");
> > -			exit(1);
> > -		}
> > +	fd = iut_ep_accept(th, l_eh, &new_eh, &addr, &addrlen); 
> > +	if (fd < 0) {
> > +		printf("failed to accept incoming connection request\n");
> >  		return;
> >  	}
> >  
> > @@ -150,11 +86,11 @@
> >  		eprintf("fail to allocate conn\n");
> >  		goto out;
> >  	}
> > +	conn->kth = th->kernel_handle;
> >  	conn->fd = fd;
> >  	incoming[i] = conn;
> >  	conn_read_pdu(conn);
> >  
> > -	set_non_blocking(fd);
> >  	pfd = &pfds[POLL_INCOMING + i];
> >  	pfd->fd = fd;
> >  	pfd->events = POLLIN;
> > @@ -162,7 +98,7 @@
> >  
> >  	return;
> >  out:
> > -	close(fd);
> > +	iut_ep_close(th, new_eh);
> >  	return;
> >  }
> >  
> > @@ -170,7 +106,7 @@
> >  {
> >  	struct connection *conn;
> >  	struct pollfd *pfd;
> > -	int i, res, opt;
> > +	int i, res;
> >  
> >  	for (i = 0; i < LISTEN_MAX; i++) {
> >  		if (pfds[POLL_LISTEN + i].revents)
> > @@ -189,14 +125,15 @@
> >  		case IOSTATE_READ_BHS:
> >  		case IOSTATE_READ_AHS_DATA:
> >  		read_again:
> > -			res = read(pfd->fd, conn->buffer, conn->rwsize);
> > +			res = iut_fd_read(pfd->fd, conn->buffer, conn->rwsize);
> >  			if (res <= 0) {
> >  				if (res == 0 || (errno != EINTR && errno != EAGAIN))
> >  					conn->state = STATE_CLOSE;
> >  				else if (errno == EINTR)
> >  					goto read_again;
> >  				break;
> > -			}
> > +			} else
> > +				printf("read %d bytes: \"%s\"\n", res, conn->buffer);
> >  			conn->rwsize -= res;
> >  			conn->buffer += res;
> >  			if (conn->rwsize)
> > @@ -233,9 +170,8 @@
> >  		case IOSTATE_WRITE_AHS:
> >  		case IOSTATE_WRITE_DATA:
> >  		write_again:
> > -			opt = 1;
> > -			setsockopt(pfd->fd, SOL_TCP, TCP_CORK, &opt, sizeof(opt));
> > -			res = write(pfd->fd, conn->buffer, conn->rwsize);
> > +			res = iut_fd_write(pfd->fd, conn->buffer, conn->rwsize, 1);
> > +			printf("wrote %d bytes: \"%s\"\n", conn->rwsize, conn->buffer);
> >  			if (res < 0) {
> >  				if (errno != EINTR && errno != EAGAIN)
> >  					conn->state = STATE_CLOSE;
> > @@ -272,8 +208,7 @@
> >  					goto write_again;
> >  				}
> >  			case IOSTATE_WRITE_DATA:
> > -				opt = 0;
> > -				setsockopt(pfd->fd, SOL_TCP, TCP_CORK, &opt, sizeof(opt));
> > +				iut_fd_write(pfd->fd, NULL, 0, 0);
> >  				cmnd_finish(conn);
> >  
> >  				switch (conn->state) {
> > @@ -312,11 +247,13 @@
> >  int iscsi_poll_init(struct pollfd *pfd)
> >  {
> >  	int i;
> > +	int listeners;
> >  
> >  	pfd[POLL_NL].fd = nl_fd;
> >  	pfd[POLL_NL].events = POLLIN;
> >  
> > -	listen_socket_create(pfd + POLL_LISTEN);
> > +	listeners = iut_poll_init(pfd + POLL_LISTEN, LISTEN_MAX);
> > +	dprintf("%d listeners\n", listeners);
> >  
> >  	for (i = 0; i < INCOMING_MAX; i++) {
> >  		pfd[POLL_INCOMING + i].fd = -1;
> > @@ -332,5 +269,7 @@
> >  	iscsi_nl_init();
> >  	*npfd = POLL_MAX;
> >  
> > +	iut_init();
> > +
> >  	return 0;
> >  }
> > Index: usr/iscsi/iscsi_uspace_transport.h
> > ===================================================================
> > --- usr/iscsi/iscsi_uspace_transport.h	(revision 0)
> > +++ usr/iscsi/iscsi_uspace_transport.h	(revision 0)
> > @@ -0,0 +1,60 @@
> > +#ifndef __ISCSI_USPACE_TRANSPORT_H
> > +#define __ISCSI_USPACE_TRANSPORT_H
> > +
> > +#include <sys/poll.h>
> > +#include <sys/socket.h>
> > +#include "list.h"
> > +
> > +#define ISCSI_LISTEN_PORT	3260
> > +
> > +typedef struct iut_ep_handle {
> > +	uint64_t kernel_handle;
> > +	struct iut *transport;
> > +} *iut_ep_handle_t;
> > +
> > +typedef struct iut *iut_handle_t;
> > +struct iut {
> > +	uint64_t kernel_handle;
> > +	const char *name;
> > +	int rdma;
> > +
> > +	int (*init)(iut_handle_t);
> > +	int (*poll_init)(iut_handle_t th, struct pollfd *, int listen_max);
> > +	int (*ep_close)(iut_handle_t th, iut_ep_handle_t ep_h);
> > +	int (*ep_accept)(iut_handle_t th, iut_ep_handle_t ep_h,
> > +			 iut_ep_handle_t *new_ep_h,
> > +			 struct sockaddr *addr, socklen_t *addrlen);
> > +	iut_ep_handle_t (*ep_lookup)(iut_handle_t th, int fd);
> > +	int (*ep_write)(iut_handle_t th, iut_ep_handle_t ep_h,
> > +			const void *buf, int buflen, int cork);
> > +	int (*ep_read)(iut_handle_t th, iut_ep_handle_t ep_h,
> > +		       void *buf, int buflen);
> > +};
> > +
> > +/* Transport independent functions */
> > +extern int iut_poll_init(struct pollfd *pfds, int);
> > +extern void iut_init(void);
> > +extern int iut_lookup_handles(int fd, iut_handle_t *th, iut_ep_handle_t *eh);
> > +extern int iut_ep_close(iut_handle_t th, iut_ep_handle_t ep_h);
> > +extern int iut_ep_accept(iut_handle_t th, iut_ep_handle_t ep_h,
> > +			 iut_ep_handle_t *new_ep_h,
> > +			 struct sockaddr *addr, socklen_t *addrlen);
> > +extern iut_ep_handle_t iut_ep_lookup(iut_handle_t th, int fd);
> > +extern int iut_fd_write(int fd, const void *buf, int buflen, int cork);
> > +extern int iut_fd_read(int fd, void *buf, int buflen);
> > +
> > +/* TCP Transport Functions */
> > +int tcp_transport_init(iut_handle_t th);
> > +int tcp_poll_init(iut_handle_t, struct pollfd *, int listen_max);
> > +int tcp_get_poll_fd(iut_handle_t th, iut_ep_handle_t ep_h);
> > +int tcp_ep_close(iut_handle_t th, iut_ep_handle_t ep_h);
> > +int tcp_ep_accept(iut_handle_t th, iut_ep_handle_t ep_h,
> > +		  iut_ep_handle_t *new_ep_h,
> > +		  struct sockaddr *addr, socklen_t *addrlen);
> > +iut_ep_handle_t tcp_ep_lookup(iut_handle_t th, int fd);
> > +int tcp_ep_write(iut_handle_t th, iut_ep_handle_t ep_h,
> > +		 const void *buf, int buflen, int cork);
> > +int tcp_ep_read(iut_handle_t th, iut_ep_handle_t ep_h,
> > +		void *buf, int buflen);
> > +
> > +#endif
> > Index: usr/iscsi/iscsi_transport_tcp.c
> > ===================================================================
> > --- usr/iscsi/iscsi_transport_tcp.c	(revision 0)
> > +++ usr/iscsi/iscsi_transport_tcp.c	(revision 0)
> > @@ -0,0 +1,281 @@
> > +#include <ctype.h>
> > +#include <errno.h>
> > +#include <fcntl.h>
> > +#include <stdio.h>
> > +#include <stdlib.h>
> > +#include <string.h>
> > +#include <unistd.h>
> > +#include <getopt.h>
> > +#include <netdb.h>
> > +
> > +#include <sys/poll.h>
> > +#include <sys/socket.h>
> > +#include <sys/stat.h>
> > +#include <sys/types.h>
> > +#include <sys/un.h>
> > +
> > +#include <netinet/in.h>
> > +#include <netinet/tcp.h>
> > +#include <netinet/ip.h>
> > +#include <arpa/inet.h>
> > +
> > +#include "iscsid.h"
> > +#include "list.h"
> > +#include "iscsi_uspace_transport.h"
> > +
> > +struct iut iscsi_tcp = {
> > +	.name = "tcp",
> > +	.rdma = 0,
> > +	.init = tcp_transport_init,
> > +	.poll_init = tcp_poll_init,
> > +	.ep_lookup = tcp_ep_lookup,
> > +	.ep_accept = tcp_ep_accept,
> > +	.ep_close = tcp_ep_close,
> > +	.ep_write = tcp_ep_write,
> > +	.ep_read = tcp_ep_read,
> > +};
> > +
> > +struct tcp_ep_dir {
> > +	int size;
> > +	iut_ep_handle_t dir[0];
> > +};
> > +static struct tcp_ep_dir *tcp_ep_dir;
> > +
> > +static iut_ep_handle_t tcp_ep_create(int fd);
> > +static void tcp_ep_destroy(iut_ep_handle_t ep);
> > +
> > +#define TCP_EP_DIR_INITIAL_SIZE 256
> > +#define TCP_EP_DIR_BUMP		16
> > +static struct tcp_ep_dir* tcp_create_ep_dir(int size)
> > +{
> > +	struct tcp_ep_dir* dir;
> > +	int i;
> > +
> > +	dir = malloc(sizeof(struct tcp_ep_dir) + (sizeof(iut_ep_handle_t)*size));
> > +	if (!dir)
> > +		return NULL;
> > +
> > +	dir->size = size;
> > +	for (i=0; i < size; i++)
> > +		dir->dir[i] = NULL;
> > +
> > +	return dir;
> > +}		
> > +
> > +static void tcp_copy_ep_dir(struct tcp_ep_dir *src, struct tcp_ep_dir *dest)
> > +{
> > +	int i;
> > +	for (i=0; i < dest->size; i++)
> > +		if (i < src->size)
> > +			dest->dir[i] = src->dir[i];
> > +}		
> > +
> > +static void tcp_destroy_ep_dir(struct tcp_ep_dir *dir)
> > +{
> > +	free(dir);
> > +}		
> > +
> > +static struct tcp_ep_dir* tcp_resize_ep_dir(struct tcp_ep_dir* old_dir, int size)
> > +{
> > +	struct tcp_ep_dir* new_dir;
> > +
> > +	new_dir = tcp_create_ep_dir(size);
> > +	if (!new_dir)
> > +		return NULL;
> > +
> > +	tcp_copy_ep_dir(old_dir, new_dir);
> > +	tcp_destroy_ep_dir(old_dir);
> > +	return new_dir;
> > +}		
> > +
> > +int tcp_transport_init(iut_handle_t t)
> > +{
> > +	int fd, err;
> > +	char buf[64];
> > +	uint64_t thandle;
> > +
> > +	/* Initialize the endpoint mapping cache */
> > +	tcp_ep_dir = tcp_create_ep_dir(TCP_EP_DIR_INITIAL_SIZE);
> > +
> > +	/* Get the tcp kernel driver's handle */
> > +	fd = open("/sys/class/iscsi_transport/iscsi_tcp_tgt/handle", O_RDONLY);
> > +	if (fd < 0)
> > +		return fd;
> > +	err = read(fd, buf, sizeof(buf));
> > +	if (err < 0)
> > +		goto out;
> > +	thandle = strtoull(buf, NULL, 10);
> > +	iscsi_tcp.kernel_handle = thandle;
> > +	printf("%s: transport handle = %" PRIx64 "\n", __FUNCTION__, thandle);
> > +	err = 0;
> > +out:
> > +	close(fd);
> > +	return err;
> > +}
> > +
> > +static void set_non_blocking(int fd)
> > +{
> > +	int res = fcntl(fd, F_GETFL);
> > +
> > +	if (res != -1) {
> > +		res = fcntl(fd, F_SETFL, res | O_NONBLOCK);
> > +		if (res)
> > +			dprintf("unable to set fd flags (%s)!\n", strerror(errno));
> > +	} else
> > +		dprintf("unable to get fd flags (%s)!\n", strerror(errno));
> > +}
> > +
> > +int tcp_poll_init(iut_handle_t th, struct pollfd *pfds, int listen_max)
> > +{
> > +	struct addrinfo hints, *res, *res0;
> > +	char servname[64];
> > +	int i, sock, opt;
> > +
> > +	memset(servname, 0, sizeof(servname));
> > +	snprintf(servname, sizeof(servname), "%d", ISCSI_LISTEN_PORT);
> > +
> > +	memset(&hints, 0, sizeof(hints));
> > +	hints.ai_socktype = SOCK_STREAM;
> > +	hints.ai_flags = AI_PASSIVE;
> > +
> > +	if (getaddrinfo(NULL, servname, &hints, &res0)) {
> > +		eprintf("unable to get address info (%s)!\n", strerror(errno));
> > +		exit(1);
> > +	}
> > +
> > +	for (i = 0, res = res0; res && i < listen_max; res = res->ai_next) {
> > +		sock = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
> > +		if (sock < 0) {
> > +			eprintf("unable to create server socket (%s) %d %d %d!\n",
> > +				  strerror(errno), res->ai_family,
> > +				  res->ai_socktype, res->ai_protocol);
> > +			continue;
> > +		}
> > +
> > +		opt = 1;
> > +		if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)))
> > +			dprintf("unable to set SO_REUSEADDR on server socket (%s)!\n",
> > +				    strerror(errno));
> > +		opt = 1;
> > +		if (res->ai_family == AF_INET6 &&
> > +		    setsockopt(sock, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt)))
> > +			continue;
> > +
> > +		if (bind(sock, res->ai_addr, res->ai_addrlen)) {
> > +			eprintf("unable to bind server socket (%s)!\n", strerror(errno));
> > +			continue;
> > +		}
> > +
> > +		if (listen(sock, 10)) {
> > +			eprintf("unable to listen to server socket (%s)!\n", strerror(errno));
> > +			continue;
> > +		}
> > +
> > +		set_non_blocking(sock);
> > +
> > +		(void)tcp_ep_create(sock);
> > +
> > +		pfds[i].fd = sock;
> > +		pfds[i].events = POLLIN;
> > +		i++;
> > +	}
> > +
> > +	freeaddrinfo(res0);
> > +	return i;
> > +}
> > +
> > +int tcp_get_poll_fd(iut_handle_t th, 
> > +			   iut_ep_handle_t ep_h)
> > +{
> > +	return (int)(ep_h->kernel_handle);
> > +}
> > +
> > +int tcp_ep_close(iut_handle_t th, iut_ep_handle_t ep_h)
> > +{
> > +	int fd = (int)ep_h->kernel_handle;
> > +	tcp_ep_destroy(ep_h);
> > +	return close(fd);
> > +}
> > +
> > +int tcp_ep_write(iut_handle_t th, iut_ep_handle_t ep_h, const void *buf, int len, int cork)
> > +{
> > +	int rc = 0;
> > +	int opt = cork;
> > +	setsockopt((int)ep_h->kernel_handle, SOL_TCP, TCP_CORK, 
> > +		   &opt, sizeof(opt));
> > +
> > +	if (len)
> > +		rc = write((int)ep_h->kernel_handle, buf, len);
> > +
> > +	return rc;
> > +}
> > +
> > +int tcp_ep_read(iut_handle_t th, iut_ep_handle_t ep_h, void *buf, int len)
> > +{
> > +	return read((int)ep_h->kernel_handle, buf, len);
> > +}
> > +
> > +iut_ep_handle_t tcp_ep_lookup(iut_handle_t th, int fd)
> > +{
> > +	iut_ep_handle_t eh;
> > +	if (fd > tcp_ep_dir->size)
> > +		return NULL;
> > +
> > +	eh = tcp_ep_dir->dir[fd];
> > +	return eh;
> > +}
> > +
> > +static iut_ep_handle_t tcp_ep_create(int fd)
> > +{
> > +	iut_ep_handle_t ep = malloc(sizeof(*ep));
> > +	if (!ep)
> > +		return NULL;
> > +
> > +	if (fd >= tcp_ep_dir->size) {
> > +		struct tcp_ep_dir* new_dir;
> > +		new_dir = tcp_resize_ep_dir(tcp_ep_dir, 
> > +					    tcp_ep_dir->size+TCP_EP_DIR_BUMP);
> > +		if (new_dir == NULL) {
> > +			printf("Failure expanding size of TCP endpoint directory\n");
> > +			free(ep);
> > +			return NULL;
> > +		}
> > +		tcp_ep_dir = new_dir;
> > +	}
> > +
> > +	ep->kernel_handle = (uint64_t)fd;
> > +	ep->transport = &iscsi_tcp;
> > +	tcp_ep_dir->dir[fd] = ep;
> > +
> > +	return ep;
> > +}
> > +
> > +static void tcp_ep_destroy(iut_ep_handle_t ep)
> > +{
> > +	tcp_ep_dir->dir[(int)ep->kernel_handle] = NULL;
> > +	free(ep);
> > +}
> > +
> > +int tcp_ep_accept(iut_handle_t th, iut_ep_handle_t ep_h, 
> > +		  iut_ep_handle_t *new_ep_h,
> > +		  struct sockaddr *addr, socklen_t *addrlen)
> > +{
> > +	iut_ep_handle_t new_h;
> > +	int new_fd = accept((int)ep_h->kernel_handle, 
> > +			    addr, addrlen);
> > +
> > +	if (new_fd < 0) 
> > +		return new_fd;
> > +
> > +	new_h = tcp_ep_create(new_fd);
> > +	if (!new_h) {
> > +		close(new_fd);
> > +		return -ENOMEM;
> > +	}
> > +			
> > +	*new_ep_h = new_h;
> > +	return new_fd;
> > +}
> > +
> > +
> > +
> > Index: usr/iscsi/Makefile
> > ===================================================================
> > --- usr/iscsi/Makefile	(revision 532)
> > +++ usr/iscsi/Makefile	(working copy)
> > @@ -2,7 +2,7 @@
> >  LIBS = -lcrypto
> >  DAEMON = iscsi.o
> >  
> > -$(DAEMON): istgt.o conn.o param.o session.o iscsid.o target.o chap.o netlink.o
> > +$(DAEMON): istgt.o conn.o param.o session.o iscsid.o target.o chap.o netlink.o iscsi_uspace_transport.o iscsi_transport_tcp.o 
> >  	$(CC) -o $@ $^ $(LIBS)
> >  
> >  clean:
> > Index: usr/iscsi/conn.c
> > ===================================================================
> > --- usr/iscsi/conn.c	(revision 532)
> > +++ usr/iscsi/conn.c	(working copy)
> > @@ -11,7 +11,6 @@
> >  #include <string.h>
> >  #include <errno.h>
> >  #include <sys/stat.h>
> > -
> >  #include "iscsid.h"
> >  
> >  #define ISCSI_CONN_NEW		1
> > @@ -74,7 +73,7 @@
> >  
> >  	conn->session->conn_cnt++;
> >  
> > -	err = ki->create_conn(thandle, conn->session->ksid, conn->kcid,
> > +	err = ki->create_conn(conn->kth, conn->session->ksid, conn->kcid,
> >  			      &conn->kcid);
> >  	if (err) {
> >  		eprintf("%d %d %u %u %u %" PRIx64,
> > @@ -86,20 +85,20 @@
> >  		/* FIXME */
> >  		if (i == ISCSI_PARAM_DATADGST_EN || i == ISCSI_PARAM_HDRDGST_EN)
> >  			continue;
> > -		if (ki->set_param(thandle, conn->session->ksid, conn->kcid, i,
> > +		if (ki->set_param(conn->kth, conn->session->ksid, conn->kcid, i,
> >  				  &conn->session_param[i].val,
> >  				  sizeof(uint32_t), &err) || err) {
> >  			break;
> >  		}
> >  	}
> >  
> > -	if (ki->bind_conn(thandle, conn->session->ksid, conn->kcid, fd, 1, &err) || err) {
> > +	if (ki->bind_conn(conn->kth, conn->session->ksid, conn->kcid, fd, 1, &err) || err) {
> >  		eprintf("%d %d %u %u %u %" PRIx64,
> >  			fd, err, conn->cid, conn->stat_sn, conn->exp_stat_sn, sid);
> >  		goto out;
> >  	}
> >  
> > -	if (ki->start_conn(thandle, conn->session->ksid, conn->kcid, &err) || err) {
> > +	if (ki->start_conn(conn->kth, conn->session->ksid, conn->kcid, &err) || err) {
> >  		eprintf("%d %d %u %u %u %" PRIx64,
> >  			fd, err, conn->cid, conn->stat_sn, conn->exp_stat_sn, sid);
> >  		goto out;
> > Index: usr/Makefile
> > ===================================================================
> > --- usr/Makefile	(revision 532)
> > +++ usr/Makefile	(working copy)
> > @@ -1,4 +1,4 @@
> > -CFLAGS += -O2 -fno-inline -Wall -Wstrict-prototypes -fPIC -I$(KERNELSRC)/include -I../istgt/include -I../include -I. -D_LARGEFILE64_SOURCE
> > +CFLAGS += -g -fno-inline -Wall -Wstrict-prototypes -fPIC -I$(KERNELSRC)/include -I../istgt/include -I../include -I. -D_LARGEFILE64_SOURCE
> >  PROGRAMS = tgtd tgtadm
> >  TGTD_OBJS = tgtd.o tgtif.o mgmt.o target.o scsi.o log.o driver.o util.o
> >  
> > @@ -9,7 +9,7 @@
> >  
> >  ifneq ($(ISCSI),)
> >  CFLAGS += -DISCSI
> > -TGTD_OBJS += $(addprefix iscsi/, istgt.o conn.o param.o session.o iscsid.o target.o chap.o netlink.o)
> > +TGTD_OBJS += $(addprefix iscsi/, istgt.o conn.o param.o session.o iscsid.o target.o chap.o netlink.o iscsi_uspace_transport.o iscsi_transport_tcp.o)
> >  LIBS = -lcrypto
> >  endif
> >  
> > Index: istgt/include/iscsi_if.h
> > ===================================================================
> > --- istgt/include/iscsi_if.h	(revision 532)
> > +++ istgt/include/iscsi_if.h	(working copy)
> > @@ -46,11 +46,14 @@
> >  	ISCSI_UEVENT_TRANSPORT_EP_CONNECT	= UEVENT_BASE + 12,
> >  	ISCSI_UEVENT_TRANSPORT_EP_POLL		= UEVENT_BASE + 13,
> >  	ISCSI_UEVENT_TRANSPORT_EP_DISCONNECT	= UEVENT_BASE + 14,
> > +	ISCSI_UEVENT_TRANSPORT_CREATE_LISTEN	= UEVENT_BASE + 15,
> > +	ISCSI_UEVENT_TRANSPORT_DESTROY_LISTEN	= UEVENT_BASE + 16,
> >  
> >  	/* up events */
> >  	ISCSI_KEVENT_RECV_PDU		= KEVENT_BASE + 1,
> >  	ISCSI_KEVENT_CONN_ERROR		= KEVENT_BASE + 2,
> >  	ISCSI_KEVENT_IF_ERROR		= KEVENT_BASE + 3,
> > +	ISCSI_KEVENT_NEW_CONN		= KEVENT_BASE + 4,
> >  };
> >  
> >  struct iscsi_uevent {
> > @@ -116,6 +119,14 @@
> >  		struct msg_transport_disconnect {
> >  			uint64_t	ep_handle;
> >  		} ep_disconnect;
> > +		struct msg_create_listen {
> > +			uint32_t	backlog;
> > +			struct sockaddr local_addr;
> > +			int		local_addr_len;
> > +		} c_listen;
> > +		struct msg_destroy_listen {
> > +			uint32_t	ep_handle;
> > +		} d_listen;
> >  	} u;
> >  	union {
> >  		/* messages k -> u */
> > @@ -141,6 +152,16 @@
> >  		struct msg_transport_connect_ret {
> >  			uint64_t	handle;
> >  		} ep_connect_ret;
> > +		struct msg_ep_new_conn {
> > +			uint64_t	ep_handle;
> > +			struct sockaddr	local_addr;
> > +			int local_addr_len;
> > +			struct sockaddr	remote_addr;
> > +			int remote_addr_len;
> > +		} ep_new_conn;
> > +		struct msg_create_listen_ret {
> > +			uint64_t	ep_handle;
> > +		} c_listen_ret;
> >  	} r;
> >  } __attribute__ ((aligned (sizeof(uint64_t))));
> >  
> > Index: usr/iscsi/netlink.c
> > ===================================================================
> > --- usr/iscsi/netlink.c	(revision 532)
> > +++ usr/iscsi/netlink.c	(working copy)
> > @@ -298,6 +298,7 @@
> >  	return 0;
> >  }
> >  
> > +#if 0
> >  static int transport_handle_init(void)
> >  {
> >  	int fd, err;
> > @@ -316,15 +317,17 @@
> >  	close(fd);
> >  	return err;
> >  }
> > +#endif
> >  
> >  int iscsi_nl_init(void)
> >  {
> >  	int err, rsize = 256 * 1024;
> >  
> > +#if 0
> >  	err = transport_handle_init();
> >  	if (err)
> >  		return err;
> > -
> > +#endif
> >  	nl_fd = socket(PF_NETLINK, SOCK_RAW, NETLINK_ISCSI);
> >  	if (nl_fd < 0) {
> >  		eprintf("Fail to create the netlink socket %d\n", errno);
> > @@ -358,6 +361,93 @@
> >  	return err;
> >  }
> >  
> > +int ktransport_create_listen(uint64_t transport_handle, 
> > +			     uint64_t user_context,
> > +			     struct sockaddr *sa, int backlog,
> > +			     uint64_t *out_ep_handle)
> > +{
> > +	int rc;
> > +	struct iscsi_uevent ev;
> > +
> > +	dprintf("%"PRIx64 " %p %p %d\n",
> > +		transport_handle, out_ep_handle, sa, backlog);
> > +
> > +	ev.type = ISCSI_UEVENT_TRANSPORT_CREATE_LISTEN;
> > +	ev.transport_handle = transport_handle;
> > +	// ev.u.c_listen.user_context = user_context;
> > +	ev.u.c_listen.backlog = backlog;
> > +	ev.u.c_listen.local_addr = *sa;
> > +
> > +	if ((rc = __kipc_call(&ev, sizeof(ev))) < 0) {
> > +		return rc;
> > +	}
> > +
> > +	*out_ep_handle = ev.r.c_listen_ret.ep_handle;
> > +
> > +	return ev.r.retcode;
> > +}
> > +
> > +int ktransport_destroy_listen(uint64_t transport_handle, uint64_t ep_handle)
> > +{
> > +	int rc;
> > +	struct iscsi_uevent ev;
> > +
> > +	dprintf("%" PRIx64 " %" PRIx64 "\n",
> > +		transport_handle, ep_handle);
> > +
> > +	ev.type = ISCSI_UEVENT_TRANSPORT_DESTROY_LISTEN;
> > +	ev.transport_handle = transport_handle;
> > +	ev.u.d_listen.ep_handle = ep_handle;
> > +
> > +	if ((rc = __kipc_call(&ev, sizeof(ev))) < 0) {
> > +		return rc;
> > +	}
> > +
> > +	return ev.r.retcode;
> > +}
> > +
> > +#if 0
> > +static int ktransport_accept(uint64_t transport_handle, 
> > +		   uint64_t ep_handle, uint64_t ep_context)
> > +{
> > +	int rc;
> > +	struct iscsi_uevent ev;
> > +
> > +	dprintf("%" PRIx64 " %" PRIx64 " %" PRIx64 "\n",
> > +		transport_handle, ep_handle, ep_context);
> > +
> > +	ev.type = ISCSI_UEVENT_EP_ACCEPT;
> > +	ev.transport_handle = transport_handle;
> > +	ev.u.ep_accept.ep_handle = ep_handle;
> > +	ev.u.ep_accept.ep_context = ep_context;
> > +
> > +	if ((rc = __kipc_call(&ev, sizeof(ev))) < 0) {
> > +		return rc;
> > +	}
> > +
> > +	return ev.r.retcode;
> > +}
> > +
> > +static int ktransport_reject(uint64_t transport_handle, uint64_t ep_handle)
> > +{
> > +	int rc;
> > +	struct iscsi_uevent ev;
> > +
> > +	dprintf("%" PRIx64 " %" PRIx64 "\n",
> > +		transport_handle, ep_handle);
> > +
> > +	ev.type = ISCSI_UEVENT_EP_REJECT;
> > +	ev.transport_handle = transport_handle;
> > +	ev.u.ep_reject.ep_handle = ep_handle;
> > +
> > +	if ((rc = __kipc_call(&ev, sizeof(ev))) < 0) {
> > +		return rc;
> > +	}
> > +
> > +	return ev.r.retcode;
> > +}
> > +#endif
> > +
> >  struct iscsi_kernel_interface nl_ki = {
> >  	.create_session		= kcreate_session,
> >  	.destroy_session	= kdestroy_session,
> > 
> > 
> > _______________________________________________
> > Stgt-devel mailing list
> > Stgt-devel at lists.berlios.de
> > https://lists.berlios.de/mailman/listinfo/stgt-devel
> > 




More information about the stgt mailing list