FS-9775: A bunch of stuff related to chaining multiple jobs, bug fixes, few other changes

libs/libks/src/dht/ks_dht-int.h

@@ -22,11 +22,11 @@ KS_BEGIN_EXTERN_C
 KS_DECLARE(ks_status_t) ks_dht_autoroute_check(ks_dht_t *dht, const ks_sockaddr_t *raddr, ks_dht_endpoint_t **endpoint);
 
 /**
- * Called internally to expire search data.
- * Handles completing and purging of finished searches.
+ * Called internally to expire or reannounce storage item data.
+ * Handles reannouncing and purging of expiring storage items.
  * @param dht pointer to the dht instance
  */
-KS_DECLARE(void) ks_dht_pulse_searches(ks_dht_t *dht);
+KS_DECLARE(void) ks_dht_pulse_storageitems(ks_dht_t *dht);
 
 /**
  * Called internally to process job state machine.
@@ -276,19 +276,18 @@ KS_DECLARE(void) ks_dht_datagram_destroy(ks_dht_datagram_t **datagram);
 KS_DECLARE(ks_status_t) ks_dht_job_create(ks_dht_job_t **job,
 										  ks_pool_t *pool,
 										  const ks_sockaddr_t *raddr,
-										  int32_t attempts);
+										  int32_t attempts,
+										  void *data);
 KS_DECLARE(void) ks_dht_job_build_ping(ks_dht_job_t *job, ks_dht_job_callback_t query_callback, ks_dht_job_callback_t finish_callback);
 KS_DECLARE(void) ks_dht_job_build_findnode(ks_dht_job_t *job,
-										   ks_dht_search_t *search,
 										   ks_dht_job_callback_t query_callback,
 										   ks_dht_job_callback_t finish_callback,
 										   ks_dht_nodeid_t *target);
 KS_DECLARE(void) ks_dht_job_build_get(ks_dht_job_t *job,
-									  ks_dht_search_t *search,
 									  ks_dht_job_callback_t query_callback,
 									  ks_dht_job_callback_t finish_callback,
 									  ks_dht_nodeid_t *target,
-									  uint8_t *salt,
+									  const uint8_t *salt,
 									  ks_size_t salt_length);
 KS_DECLARE(void) ks_dht_job_build_put(ks_dht_job_t *job,
 									  ks_dht_job_callback_t query_callback,
@@ -296,6 +295,9 @@ KS_DECLARE(void) ks_dht_job_build_put(ks_dht_job_t *job,
 									  ks_dht_token_t *token,
 									  int64_t cas,
 									  ks_dht_storageitem_t *item);
+KS_DECLARE(void) ks_dht_job_build_search(ks_dht_job_t *job,
+										 ks_dht_job_callback_t query_callback,
+										 ks_dht_job_callback_t finish_callback);
 KS_DECLARE(void) ks_dht_job_build_search_findnode(ks_dht_job_t *job,
 									ks_dht_nodeid_t *target,
 									uint32_t family,
@@ -344,9 +346,35 @@ KS_DECLARE(ks_status_t) ks_dht_message_response(ks_dht_message_t *message,
 /**
  *
  */
-KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t *pool, const ks_dht_nodeid_t *target, ks_dht_search_callback_t callback);
+KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search,
+											 ks_pool_t *pool,
+											 ks_dhtrt_routetable_t *table,
+											 const ks_dht_nodeid_t *target,
+											 ks_dht_job_callback_t callback,
+											 void *data);
 KS_DECLARE(void) ks_dht_search_destroy(ks_dht_search_t **search);
 
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_publish_create(ks_dht_publish_t **publish,
+											  ks_pool_t *pool,
+											  ks_dht_job_callback_t callback,
+											  void *data,
+											  int64_t cas,
+											  ks_dht_storageitem_t *item);
+KS_DECLARE(void) ks_dht_publish_destroy(ks_dht_publish_t **publish);
+
+/**
+ *
+ */
+KS_DECLARE(ks_status_t) ks_dht_distribute_create(ks_dht_distribute_t **distribute,
+												 ks_pool_t *pool,
+												 ks_dht_storageitem_callback_t callback,
+												 void *data,
+												 int64_t cas,
+												 ks_dht_storageitem_t *item);
+KS_DECLARE(void) ks_dht_distribute_destroy(ks_dht_distribute_t **distribute);
 
 /**
  *

libs/libks/src/dht/ks_dht.h

@@ -31,7 +31,6 @@ KS_BEGIN_EXTERN_C
 #define KS_DHT_TRANSACTION_EXPIRATION 10
 #define KS_DHT_TRANSACTIONS_PULSE 1
 
-#define KS_DHT_SEARCH_EXPIRATION 10
 #define KS_DHT_SEARCH_RESULTS_MAX_SIZE 8 // @todo replace with KS_DHTRT_BUCKET_SIZE
 
 #define KS_DHT_STORAGEITEM_PKEY_SIZE crypto_sign_PUBLICKEYBYTES
@@ -39,6 +38,8 @@ KS_BEGIN_EXTERN_C
 #define KS_DHT_STORAGEITEM_SALT_MAX_SIZE 64
 #define KS_DHT_STORAGEITEM_SIGNATURE_SIZE crypto_sign_BYTES
 #define KS_DHT_STORAGEITEM_EXPIRATION 7200
+#define KS_DHT_STORAGEITEM_KEEPALIVE 300
+#define KS_DHT_STORAGEITEMS_PULSE 10
 
 #define KS_DHT_TOKEN_SIZE SHA_DIGEST_LENGTH
 #define KS_DHT_TOKEN_EXPIRATION 300
@@ -58,7 +59,8 @@ typedef struct ks_dht_message_s ks_dht_message_t;
 typedef struct ks_dht_endpoint_s ks_dht_endpoint_t;
 typedef struct ks_dht_transaction_s ks_dht_transaction_t;
 typedef struct ks_dht_search_s ks_dht_search_t;
-typedef struct ks_dht_search_pending_s ks_dht_search_pending_t;
+typedef struct ks_dht_publish_s ks_dht_publish_t;
+typedef struct ks_dht_distribute_s ks_dht_distribute_t;
 typedef struct ks_dht_node_s ks_dht_node_t;
 typedef struct ks_dhtrt_routetable_s ks_dhtrt_routetable_t;
 typedef struct ks_dhtrt_querynodes_s ks_dhtrt_querynodes_t;
@@ -67,7 +69,9 @@ typedef struct ks_dht_storageitem_s ks_dht_storageitem_t;
 
 typedef ks_status_t (*ks_dht_job_callback_t)(ks_dht_t *dht, ks_dht_job_t *job);
 typedef ks_status_t (*ks_dht_message_callback_t)(ks_dht_t *dht, ks_dht_message_t *message);
-typedef ks_status_t (*ks_dht_search_callback_t)(ks_dht_t *dht, ks_dht_search_t *search);
+//typedef ks_status_t (*ks_dht_search_callback_t)(ks_dht_t *dht, ks_dht_search_t *search);
+typedef ks_status_t (*ks_dht_storageitem_callback_t)(ks_dht_t *dht, ks_dht_storageitem_t *item);
+
 
 struct ks_dht_datagram_s {
 	ks_pool_t *pool;
@@ -115,11 +119,12 @@ enum ks_dht_job_state_t {
 	KS_DHT_JOB_STATE_COMPLETING,
 };
 
-//enum ks_dht_job_type_t {
-//	KS_DHT_JOB_TYPE_NONE = 0,
-//	KS_DHT_JOB_TYPE_PING,
-//	KS_DHT_JOB_TYPE_FINDNODE,
-//};
+enum ks_dht_job_result_t {
+	KS_DHT_JOB_RESULT_SUCCESS = 0,
+	KS_DHT_JOB_RESULT_EXPIRED,
+	KS_DHT_JOB_RESULT_ERROR,
+	KS_DHT_JOB_RESULT_FAILURE,
+};
 
 struct ks_dht_job_s {
 	ks_pool_t *pool;
@@ -127,8 +132,7 @@ struct ks_dht_job_s {
 	ks_dht_job_t *next;
 
 	enum ks_dht_job_state_t state;
-
-	ks_dht_search_t *search;
+	enum ks_dht_job_result_t result;
 
 	ks_sockaddr_t raddr; // will obtain local endpoint node id when creating message using raddr
 	int32_t attempts;
@@ -137,6 +141,7 @@ struct ks_dht_job_s {
 	ks_dht_job_callback_t query_callback;
 	ks_dht_job_callback_t finish_callback;
 
+	void *data;
 	ks_dht_message_t *response;
 
 	// job specific query parameters
@@ -145,15 +150,22 @@ struct ks_dht_job_s {
 	int64_t query_cas;
 	ks_dht_token_t query_token;
 	ks_dht_storageitem_t *query_storageitem;
-    uint32_t query_family;
+    int32_t query_family;
 
+	// error response parameters
+	int64_t error_code;
+	struct bencode *error_description;
+	
 	// job specific response parameters
-	ks_dht_nodeid_t response_id;
+	ks_dht_node_t *response_id;
 	ks_dht_node_t *response_nodes[KS_DHT_RESPONSE_NODES_MAX_SIZE];
 	ks_size_t response_nodes_count;
 	ks_dht_node_t *response_nodes6[KS_DHT_RESPONSE_NODES_MAX_SIZE];
 	ks_size_t response_nodes6_count;
+	
 	ks_dht_token_t response_token;
+	int64_t response_seq;
+	ks_bool_t response_hasitem;
 	ks_dht_storageitem_t *response_storageitem;
 };
 
@@ -219,25 +231,48 @@ struct ks_dht_transaction_s {
 
 struct ks_dht_search_s {
 	ks_pool_t *pool;
-	ks_dht_search_t *next;
+	ks_dhtrt_routetable_t *table;
 	ks_dht_nodeid_t target;
-	ks_dht_search_callback_t callback;
+	ks_dht_job_callback_t callback;
+	void *data;
 	ks_mutex_t *mutex;
 	ks_hash_t *searched;
-	ks_hash_t *searching;
+	int32_t searching;
 	ks_dht_node_t *results[KS_DHT_SEARCH_RESULTS_MAX_SIZE];
 	ks_dht_nodeid_t distances[KS_DHT_SEARCH_RESULTS_MAX_SIZE];
 	ks_size_t results_length;
 };
 
+struct ks_dht_publish_s {
+	ks_pool_t *pool;
+	ks_dht_job_callback_t callback;
+	void *data;
+	int64_t cas;
+	ks_dht_storageitem_t *item;
+};
+
+struct ks_dht_distribute_s {
+	ks_pool_t *pool;
+	ks_dht_storageitem_callback_t callback;
+	void *data;
+	ks_mutex_t *mutex;
+	int32_t publishing;
+	int64_t cas;
+	ks_dht_storageitem_t *item;
+};
+
 struct ks_dht_storageitem_s {
 	ks_pool_t *pool;
 	ks_dht_nodeid_t id;
 	ks_time_t expiration;
+	ks_time_t keepalive;
 	struct bencode *v;
-	
-	ks_bool_t mutable;
+
 	ks_mutex_t *mutex;
+	volatile int32_t refc;
+	ks_dht_storageitem_callback_t callback;
+
+	ks_bool_t mutable;
 	ks_dht_storageitem_pkey_t pk;
 	ks_dht_storageitem_skey_t sk;
 	struct bencode *salt;
@@ -282,15 +317,12 @@ struct ks_dht_s {
 	ks_dhtrt_routetable_t *rt_ipv4;
 	ks_dhtrt_routetable_t *rt_ipv6;
 
-	ks_mutex_t *searches_mutex;
-	ks_dht_search_t *searches_first;
-	ks_dht_search_t *searches_last;
-
 	ks_time_t tokens_pulse;
 	volatile uint32_t token_secret_current;
 	volatile uint32_t token_secret_previous;
 	ks_time_t token_secret_expiration;
 
+	ks_time_t storageitems_pulse;
 	ks_hash_t *storageitems_hash;
 };
 
@@ -395,7 +427,22 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_signature_generate(ks_dht_storageitem
 															  int64_t sequence,
 															  const uint8_t *value,
 															  ks_size_t value_length);
-												
+
+/**
+ *
+ */
+KS_DECLARE(void) ks_dht_storageitem_reference(ks_dht_storageitem_t *item);
+
+/**
+ *
+ */
+KS_DECLARE(void) ks_dht_storageitem_dereference(ks_dht_storageitem_t *item);
+
+/**
+ *
+ */
+KS_DECLARE(void) ks_dht_storageitem_callback(ks_dht_storageitem_t *item, ks_dht_storageitem_callback_t callback);
+						
 /**
  *
  */
@@ -429,37 +476,38 @@ KS_DECLARE(ks_status_t) ks_dht_storageitems_insert(ks_dht_t *dht, ks_dht_storage
 /**
  *
  */
-KS_DECLARE(ks_status_t) ks_dht_ping(ks_dht_t *dht, const ks_sockaddr_t *raddr, ks_dht_job_callback_t callback);
+KS_DECLARE(void) ks_dht_ping(ks_dht_t *dht, const ks_sockaddr_t *raddr, ks_dht_job_callback_t callback, void *data);
 
 /**
  *
  */
-KS_DECLARE(ks_status_t) ks_dht_findnode(ks_dht_t *dht,
-										ks_dht_search_t *search,
-										const ks_sockaddr_t *raddr,
-										ks_dht_job_callback_t callback,
-										ks_dht_nodeid_t *target);
+KS_DECLARE(void) ks_dht_findnode(ks_dht_t *dht,
+								 const ks_sockaddr_t *raddr,
+								 ks_dht_job_callback_t callback,
+								 void *data,
+								 ks_dht_nodeid_t *target);
 
 /**
  *
  */
-KS_DECLARE(ks_status_t) ks_dht_get(ks_dht_t *dht,
-								   ks_dht_search_t *search,
-								   const ks_sockaddr_t *raddr,
-								   ks_dht_job_callback_t callback,
-								   ks_dht_nodeid_t *target,
-								   uint8_t *salt,
-								   ks_size_t salt_length);
+KS_DECLARE(void) ks_dht_get(ks_dht_t *dht,
+							const ks_sockaddr_t *raddr,
+							ks_dht_job_callback_t callback,
+							void *data,
+							ks_dht_nodeid_t *target,
+							const uint8_t *salt,
+							ks_size_t salt_length);
 
 /**
  *
  */
-KS_DECLARE(ks_status_t) ks_dht_put(ks_dht_t *dht,
-								   const ks_sockaddr_t *raddr,
-								   ks_dht_job_callback_t callback,
-								   ks_dht_token_t *token,
-								   int64_t cas,
-								   ks_dht_storageitem_t *item);
+KS_DECLARE(void) ks_dht_put(ks_dht_t *dht,
+							const ks_sockaddr_t *raddr,
+							ks_dht_job_callback_t callback,
+							void *data,
+							ks_dht_token_t *token,
+							int64_t cas,
+							ks_dht_storageitem_t *item);
 						
 /**
  * Create a network search of the closest nodes to a target.
@@ -468,27 +516,29 @@ KS_DECLARE(ks_status_t) ks_dht_put(ks_dht_t *dht,
  * @param target pointer to the nodeid for the target to be searched
  * @param callback an optional callback to add to the search when it is finished
  * @param search dereferenced out pointer to the allocated search, may be NULL to ignore search output
- * @return The ks_status_t result: KS_STATUS_SUCCESS, KS_STATUS_FAIL
  * @see ks_dht_search_create
- * @see ks_dht_search_callback_add
  * @see ks_hash_insert
- * @see ks_dht_search_pending_create
- * @see ks_dht_send_findnode
+ * @see ks_dht_findnode
  */
-KS_DECLARE(ks_status_t) ks_dht_search_findnode(ks_dht_t *dht,
-											   int32_t family,
-											   ks_dht_nodeid_t *target,
-											   ks_dht_search_callback_t callback,
-											   ks_dht_search_t **search);
-
-KS_DECLARE(ks_status_t) ks_dht_queue_search_findnode(ks_dht_t* dht,
-											ks_dhtrt_routetable_t *rt,
-											ks_dht_nodeid_t *target,
-											ks_dht_job_callback_t callback);
-										
-KS_DECLARE(ks_status_t) ks_dht_exec_search_findnode(ks_dht_t *dht, ks_dht_job_t *job);
+KS_DECLARE(void) ks_dht_search(ks_dht_t *dht,
+							   ks_dht_job_callback_t callback,
+							   void *data,
+							   ks_dhtrt_routetable_t *table,
+							   ks_dht_nodeid_t *target);
 
+KS_DECLARE(void) ks_dht_publish(ks_dht_t *dht,
+								const ks_sockaddr_t *raddr,
+								ks_dht_job_callback_t callback,
+								void *data,
+								int64_t cas,
+								ks_dht_storageitem_t *item);
 
+KS_DECLARE(void) ks_dht_distribute(ks_dht_t *dht,
+								   ks_dht_storageitem_callback_t callback,
+								   void *data,
+								   ks_dhtrt_routetable_t *table,
+								   int64_t cas,
+								   ks_dht_storageitem_t *item);
 
 /**
  * route table methods

libs/libks/src/dht/ks_dht_bucket.c

@@ -1673,7 +1673,7 @@ void ks_dhtrt_ping(ks_dhtrt_internal_t *internal, ks_dhtrt_bucket_entry_t *entry
 #endif
 	ks_dht_node_t* node = entry->gptr;
 	ks_log(KS_LOG_DEBUG, "Node addr %s %d\n", node->addr.host, node->addr.port);
-	ks_dht_ping(internal->dht, &node->addr, NULL);
+	ks_dht_ping(internal->dht, &node->addr, NULL, NULL);
 
 	return;
 }
@@ -1683,7 +1683,7 @@ void ks_dhtrt_find(ks_dhtrt_routetable_t *table, ks_dhtrt_internal_t *internal,
 
 	char buf[100];
 	ks_log(KS_LOG_DEBUG, "Find queued for target %s\n", ks_dhtrt_printableid(target->id, buf));
-	ks_dht_queue_search_findnode(internal->dht, table, target, NULL);
+	ks_dht_search(internal->dht, NULL, NULL, table, target);
     return;
 }
 

libs/libks/src/dht/ks_dht_distribute.c

@@ -0,0 +1,65 @@
+#include "ks_dht.h"
+#include "ks_dht-int.h"
+#include "sodium.h"
+
+KS_DECLARE(ks_status_t) ks_dht_distribute_create(ks_dht_distribute_t **distribute,
+												 ks_pool_t *pool,
+												 ks_dht_storageitem_callback_t callback,
+												 void *data,
+												 int64_t cas,
+												 ks_dht_storageitem_t *item)
+{
+	ks_dht_distribute_t *d;
+	ks_status_t ret = KS_STATUS_SUCCESS;
+
+	ks_assert(distribute);
+	ks_assert(pool);
+	ks_assert(cas >= 0);
+	ks_assert(item);
+
+	*distribute = d = ks_pool_alloc(pool, sizeof(ks_dht_distribute_t));
+	ks_assert(d);
+
+	d->pool = pool;
+
+	d->callback = callback;
+	d->data = data;
+	ks_mutex_create(&d->mutex, KS_MUTEX_FLAG_DEFAULT, d->pool);
+	ks_assert(d->mutex);
+	d->cas = cas;
+	d->item = item;
+	
+	ks_dht_storageitem_reference(d->item);
+
+	// done:
+	if (ret != KS_STATUS_SUCCESS) {
+		if (d) ks_dht_distribute_destroy(distribute);
+	}
+	return ret;
+}
+
+KS_DECLARE(void) ks_dht_distribute_destroy(ks_dht_distribute_t **distribute)
+{
+	ks_dht_distribute_t *d;
+
+	ks_assert(distribute);
+	ks_assert(*distribute);
+
+	d = *distribute;
+
+	if (d->mutex) ks_mutex_destroy(&d->mutex);
+	ks_dht_storageitem_dereference(d->item);
+	
+	ks_pool_free(d->pool, distribute);
+}
+
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */

libs/libks/src/dht/ks_dht_job.c

@@ -4,7 +4,8 @@
 KS_DECLARE(ks_status_t) ks_dht_job_create(ks_dht_job_t **job,
 										  ks_pool_t *pool,
 										  const ks_sockaddr_t *raddr,
-										  int32_t attempts)
+										  int32_t attempts,
+										  void *data)
 {
 	ks_dht_job_t *j;
 	ks_status_t ret = KS_STATUS_SUCCESS;
@@ -12,7 +13,6 @@ KS_DECLARE(ks_status_t) ks_dht_job_create(ks_dht_job_t **job,
 	ks_assert(job);
 	ks_assert(pool);
 	//ks_assert(dht);
-	ks_assert(raddr);
 	ks_assert(attempts > 0 && attempts <= 10);
 
 	*job = j = ks_pool_alloc(pool, sizeof(ks_dht_job_t));
@@ -20,13 +20,9 @@ KS_DECLARE(ks_status_t) ks_dht_job_create(ks_dht_job_t **job,
 
 	j->pool = pool;
 	j->state = KS_DHT_JOB_STATE_QUERYING;
-	j->raddr = *raddr;
+	if (raddr) j->raddr = *raddr;
 	j->attempts = attempts;
-
-	//ks_mutex_lock(dht->jobs_mutex);
-	//if (dht->jobs_last) dht->jobs_last = dht->jobs_last->next = j;
-	//else dht->jobs_first = dht->jobs_last = j;
-	//ks_mutex_unlock(dht->jobs_mutex);
+	j->data = data;
 
 	// done:
 	if (ret != KS_STATUS_SUCCESS) {
@@ -45,7 +41,6 @@ KS_DECLARE(void) ks_dht_job_build_ping(ks_dht_job_t *job, ks_dht_job_callback_t
 }
 
 KS_DECLARE(void) ks_dht_job_build_findnode(ks_dht_job_t *job,
-										   ks_dht_search_t *search,
 										   ks_dht_job_callback_t query_callback,
 										   ks_dht_job_callback_t finish_callback,
 										   ks_dht_nodeid_t *target)
@@ -54,25 +49,22 @@ KS_DECLARE(void) ks_dht_job_build_findnode(ks_dht_job_t *job,
 	ks_assert(query_callback);
 	ks_assert(target);
 
-	job->search = search;
 	job->query_callback = query_callback;
 	job->finish_callback = finish_callback;
 	job->query_target = *target;
 }
 
 KS_DECLARE(void) ks_dht_job_build_get(ks_dht_job_t *job,
-									  ks_dht_search_t *search,
 									  ks_dht_job_callback_t query_callback,
 									  ks_dht_job_callback_t finish_callback,
 									  ks_dht_nodeid_t *target,
-									  uint8_t *salt,
+									  const uint8_t *salt,
 									  ks_size_t salt_length)
 {
 	ks_assert(job);
 	ks_assert(query_callback);
 	ks_assert(target);
 
-	job->search = search;
 	job->query_callback = query_callback;
 	job->finish_callback = finish_callback;
 	job->query_target = *target;
@@ -96,23 +88,18 @@ KS_DECLARE(void) ks_dht_job_build_put(ks_dht_job_t *job,
 	job->query_token = *token;
 	job->query_cas = cas;
 	job->query_storageitem = item;
+	ks_dht_storageitem_reference(job->query_storageitem);
 }
 
-KS_DECLARE(void) ks_dht_job_build_search_findnode(ks_dht_job_t *job,
-										   ks_dht_nodeid_t *target,
-										   uint32_t family,
-                                           ks_dht_job_callback_t query_callback,
-                                           ks_dht_job_callback_t finish_callback)
+KS_DECLARE(void) ks_dht_job_build_search(ks_dht_job_t *job,
+										 ks_dht_job_callback_t query_callback,
+										 ks_dht_job_callback_t finish_callback)
 {
 	ks_assert(job);
-	ks_assert(target);
-	ks_assert(family);
+	ks_assert(query_callback);
 
-	job->search = NULL;
 	job->query_callback = query_callback;
 	job->finish_callback = finish_callback;
-	job->query_target = *target;
-	job->query_family = family;
 }
 
 KS_DECLARE(void) ks_dht_job_destroy(ks_dht_job_t **job)
@@ -125,9 +112,15 @@ KS_DECLARE(void) ks_dht_job_destroy(ks_dht_job_t **job)
 	j = *job;
 
 	if (j->query_salt) ben_free(j->query_salt);
+	if (j->response_id) ks_dhtrt_release_node(j->response_id);
 	for (int32_t i = 0; i < j->response_nodes_count; ++i) ks_dhtrt_release_node(j->response_nodes[i]);
 	for (int32_t i = 0; i < j->response_nodes6_count; ++i) ks_dhtrt_release_node(j->response_nodes6[i]);
 
+	if (j->query_storageitem) ks_dht_storageitem_dereference(j->query_storageitem);
+	if (j->response_storageitem) ks_dht_storageitem_dereference(j->response_storageitem);
+
+	if (j->error_description) ben_free(j->error_description);
+
 	ks_pool_free(j->pool, job);
 }
 

libs/libks/src/dht/ks_dht_publish.c

@@ -0,0 +1,62 @@
+#include "ks_dht.h"
+#include "ks_dht-int.h"
+#include "sodium.h"
+
+KS_DECLARE(ks_status_t) ks_dht_publish_create(ks_dht_publish_t **publish,
+											  ks_pool_t *pool,
+											  ks_dht_job_callback_t callback,
+											  void *data,
+											  int64_t cas,
+											  ks_dht_storageitem_t *item)
+{
+	ks_dht_publish_t *p;
+	ks_status_t ret = KS_STATUS_SUCCESS;
+
+	ks_assert(publish);
+	ks_assert(pool);
+	ks_assert(cas >= 0);
+	ks_assert(item);
+
+	*publish = p = ks_pool_alloc(pool, sizeof(ks_dht_publish_t));
+	ks_assert(p);
+
+	p->pool = pool;
+
+	p->callback = callback;
+	p->data = data;
+	p->cas = cas;
+	p->item = item;
+	
+	ks_dht_storageitem_reference(p->item);
+
+	// done:
+	if (ret != KS_STATUS_SUCCESS) {
+		if (p) ks_dht_publish_destroy(publish);
+	}
+	return ret;
+}
+
+KS_DECLARE(void) ks_dht_publish_destroy(ks_dht_publish_t **publish)
+{
+	ks_dht_publish_t *p;
+
+	ks_assert(publish);
+	ks_assert(*publish);
+
+	p = *publish;
+
+	ks_dht_storageitem_dereference(p->item);
+	
+	ks_pool_free(p->pool, publish);
+}
+
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */

libs/libks/src/dht/ks_dht_search.c

@@ -2,13 +2,19 @@
 #include "ks_dht-int.h"
 #include "sodium.h"
 
-KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t *pool, const ks_dht_nodeid_t *target, ks_dht_search_callback_t callback)
+KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search,
+											 ks_pool_t *pool,
+											 ks_dhtrt_routetable_t *table,
+											 const ks_dht_nodeid_t *target,
+											 ks_dht_job_callback_t callback,
+											 void *data)
 {
 	ks_dht_search_t *s;
 	ks_status_t ret = KS_STATUS_SUCCESS;
 
 	ks_assert(search);
 	ks_assert(pool);
+	ks_assert(table);
 	ks_assert(target);
 
 	*search = s = ks_pool_alloc(pool, sizeof(ks_dht_search_t));
@@ -19,17 +25,17 @@ KS_DECLARE(ks_status_t) ks_dht_search_create(ks_dht_search_t **search, ks_pool_t
 	ks_mutex_create(&s->mutex, KS_MUTEX_FLAG_DEFAULT, s->pool);
 	ks_assert(s->mutex);
 
+	s->table = table;
 	memcpy(s->target.id, target->id, KS_DHT_NODEID_SIZE);
 
 	s->callback = callback;
+	s->data = data;
 
 	ks_hash_create(&s->searched, KS_HASH_MODE_ARBITRARY, KS_HASH_FLAG_NOLOCK | KS_HASH_FLAG_DUP_CHECK, s->pool);
 	ks_assert(s->searched);
 	ks_hash_set_keysize(s->searched, KS_DHT_NODEID_SIZE);
 
-	ks_hash_create(&s->searching, KS_HASH_MODE_ARBITRARY, KS_HASH_FLAG_NOLOCK | KS_HASH_FLAG_DUP_CHECK, s->pool);
-	ks_assert(s->searching);
-	ks_hash_set_keysize(s->searching, KS_DHT_NODEID_SIZE);
+	s->searching = 0;
 
 	// done:
 	if (ret != KS_STATUS_SUCCESS) {
@@ -47,7 +53,6 @@ KS_DECLARE(void) ks_dht_search_destroy(ks_dht_search_t **search)
 
 	s = *search;
 
-	if (s->searching) ks_hash_destroy(&s->searching);
 	if (s->searched) ks_hash_destroy(&s->searched);
 	if (s->mutex) ks_mutex_destroy(&s->mutex);
 

libs/libks/src/dht/ks_dht_storageitem.c

@@ -23,9 +23,12 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_immutable_internal(ks_dht_stor
 	si->id = *target;
 	si->mutable = KS_FALSE;
 	si->expiration = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_EXPIRATION * KS_USEC_PER_SEC);
+	si->keepalive = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_KEEPALIVE * KS_USEC_PER_SEC);
 	si->v = clone_v ? ben_clone(v) : v;
 	ks_assert(si->v);
 
+	si->refc = 1;
+
 	// done:
 	if (ret != KS_STATUS_SUCCESS) {
 		if (si) ks_dht_storageitem_destroy(item);
@@ -81,9 +84,12 @@ KS_DECLARE(ks_status_t) ks_dht_storageitem_create_mutable_internal(ks_dht_storag
 	si->id = *target;
 	si->mutable = KS_TRUE;
 	si->expiration = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_EXPIRATION * KS_USEC_PER_SEC);
+	si->keepalive = ks_time_now() + ((ks_time_t)KS_DHT_STORAGEITEM_KEEPALIVE * KS_USEC_PER_SEC);
 	si->v = clone_v ? ben_clone(v) : v;
 	ks_assert(si->v);
 
+	si->refc = 1;
+
 	ks_mutex_create(&si->mutex, KS_MUTEX_FLAG_DEFAULT, si->pool);
 	ks_assert(si->mutex);
 	
@@ -169,6 +175,33 @@ KS_DECLARE(void) ks_dht_storageitem_destroy(ks_dht_storageitem_t **item)
 	ks_pool_free(si->pool, item);
 }
 
+KS_DECLARE(void) ks_dht_storageitem_reference(ks_dht_storageitem_t *item)
+{
+	ks_assert(item);
+
+	ks_mutex_lock(item->mutex);
+	item->refc++;
+	ks_mutex_unlock(item->mutex);
+}
+
+KS_DECLARE(void) ks_dht_storageitem_dereference(ks_dht_storageitem_t *item)
+{
+	ks_assert(item);
+
+	ks_mutex_lock(item->mutex);
+	item->refc--;
+	ks_mutex_unlock(item->mutex);
+
+	ks_assert(item->refc >= 0);
+}
+
+KS_DECLARE(void) ks_dht_storageitem_callback(ks_dht_storageitem_t *item, ks_dht_storageitem_callback_t callback)
+{
+	ks_assert(item);
+
+	item->callback = callback;
+}
+
 /* For Emacs:
  * Local Variables:
  * mode:c

libs/libks/src/ks_thread_pool.c

@@ -102,10 +102,10 @@ static int check_queue(ks_thread_pool_t *tp, ks_bool_t adding)
 		
 		need--;
 	}
-
+	/*
 	ks_log(KS_LOG_DEBUG, "WORKER check: adding %d need %d running %d dying %d total %d max %d\n", 
 		   adding, need, tp->running_thread_count, tp->dying_thread_count, tp->thread_count, tp->max);
-
+	*/
 	return need;
 }
 
@@ -129,10 +129,10 @@ static void *worker_thread(ks_thread_t *thread, void *data)
 		ks_status_t status;
 		
 		status = ks_q_pop_timeout(tp->q, &pop, 1000);
-
+		/*
 		ks_log(KS_LOG_DEBUG, "WORKER %d idle_sec %d running %d dying %d total %d max %d\n", 
 			   my_id, idle_sec, tp->running_thread_count, tp->dying_thread_count, tp->thread_count, tp->max);		
-		
+		*/		
 		check_queue(tp, KS_FALSE);
 		
 		if (status == KS_STATUS_TIMEOUT) {