[sheepdog] [PATCH v2] sockfd_cache: close socket always when node is crashed

[MORITA Kazutaka]

From: MORITA Kazutaka <morita.kazutaka at lab.ntt.co.jp>

When timeout happens, we cannot reuse the socket for other transfers
because data may remain in its send (or recv) buffer.  We have to
close the socket before making the sockfd_cache reusable.

Signed-off-by: MORITA Kazutaka <morita.kazutaka at lab.ntt.co.jp>
---
 sheep/sockfd_cache.c |   31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/sheep/sockfd_cache.c b/sheep/sockfd_cache.c
index 45d9d04..c1a0e0d 100644
--- a/sheep/sockfd_cache.c
+++ b/sheep/sockfd_cache.c
@@ -428,6 +428,27 @@ static void sockfd_cache_put(const struct node_id *nid, int idx)
 	pthread_rwlock_unlock(&sockfd_cache.lock);
 }
 
+static void sockfd_cache_close(const struct node_id *nid, int idx)
+{
+	bool use_io = nid->io_port ? true : false;
+	const uint8_t *addr = use_io ? nid->io_addr : nid->addr;
+	int port = use_io ? nid->io_port : nid->port;
+	struct sockfd_cache_entry *entry;
+	char name[INET6_ADDRSTRLEN];
+
+	addr_to_str(name, sizeof(name), addr, 0);
+	sd_dprintf("%s:%d idx %d", name, port, idx);
+
+	pthread_rwlock_wrlock(&sockfd_cache.lock);
+	entry = sockfd_cache_search(nid);
+	if (entry) {
+		close(entry->fds[idx].fd);
+		entry->fds[idx].fd = -1;
+		uatomic_set_false(&entry->fds[idx].in_use);
+	}
+	pthread_rwlock_unlock(&sockfd_cache.lock);
+}
+
 /*
  * Return a sockfd connected to the vnode to the caller
  *
@@ -485,9 +506,10 @@ void sheep_put_sockfd(const struct node_id *nid, struct sockfd *sfd)
 /*
  * Delete a sockfd connected to the vnode, when vnode is crashed.
  *
- * If it is a long FD, de-refcount it and tres to destroy all the cached FDs of
- * this vnode in the cache.
- * If it is a short FD, just close it.
+ * This closes the specified socket whether it is a long FD or not.
+ * It is because data may remain in its send (or recv) buffer in the
+ * case of timeout and we cannot reuse the sockfd for other transfers
+ * any more.
  */
 void sheep_del_sockfd(const struct node_id *nid, struct sockfd *sfd)
 {
@@ -498,8 +520,7 @@ void sheep_del_sockfd(const struct node_id *nid, struct sockfd *sfd)
 		return;
 	}
 
-	sockfd_cache_put(nid, sfd->idx);
-	sockfd_cache_del(nid);
+	sockfd_cache_close(nid, sfd->idx);
 	free(sfd);
 }
 
-- 
1.7.9.5