Try to send data using writev() to reduce copying and system calls.

send queue, and read queue.

CMakeLists.txt

@@ -27,6 +27,7 @@ find_package(Sqlite3)
 
 include(TestBigEndian)
 include(CheckSymbolExists)
+include(CheckFunctionExists)
 include(CheckIncludeFile)
 include(CheckTypeSize)
 
@@ -69,9 +70,19 @@ check_include_file(sys/types.h HAVE_SYS_TYPES_H)
 if (HAVE_SYS_TYPES_H)
 set (CMAKE_EXTRA_INCLUDE_FILES ${CMAKE_EXTRA_INCLUDE_FILES} "sys/types.h")
 endif()
-check_type_size( ssize_t SSIZE_T )
+
+check_include_file(sys/uio.h HAVE_SYS_UIO_H)
+if (HAVE_SYS_UIO_H)
+set (CMAKE_EXTRA_INCLUDE_FILES ${CMAKE_EXTRA_INCLUDE_FILES} "sys/uio.h")
+endif()
+
+check_type_size(ssize_t SSIZE_T)
 check_symbol_exists(memmem string.h HAVE_MEMMEM)
 check_symbol_exists(strndup string.h HAVE_STRNDUP)
+check_function_exists(writev HAVE_FUNC_WRITEV)
+check_function_exists(pwritev HAVE_FUNC_PWRITEV)
+check_function_exists(readv HAVE_FUNC_WRITEV)
+check_function_exists(preadv HAVE_FUNC_PWRITEV)
 
 include_directories("${PROJECT_SOURCE_DIR}")
 include_directories(${SQLITE3_INCLUDE_DIRS})

autotest/test.c

@@ -706,6 +706,34 @@ int main(int argc, char** argv)
 	exotic_add_test(&handle, &exotic_test_utf8_valid_10, "utf8_valid_10");
 	exotic_add_test(&handle, &exotic_test_utf8_valid_11, "utf8_valid_11");
 	exotic_add_test(&handle, &exotic_test_utf8_valid_12, "utf8_valid_12");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_13, "utf8_valid_13");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_14, "utf8_valid_14");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_15, "utf8_valid_15");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_16, "utf8_valid_16");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_17, "utf8_valid_17");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_18, "utf8_valid_18");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_19, "utf8_valid_19");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_20, "utf8_valid_20");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_21, "utf8_valid_21");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_22, "utf8_valid_22");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_23, "utf8_valid_23");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_24, "utf8_valid_24");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_25, "utf8_valid_25");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_26, "utf8_valid_26");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_27, "utf8_valid_27");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_28, "utf8_valid_28");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_29, "utf8_valid_29");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_30, "utf8_valid_30");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_31, "utf8_valid_31");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_32, "utf8_valid_32");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_33, "utf8_valid_33");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_34, "utf8_valid_34");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_35, "utf8_valid_35");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_36, "utf8_valid_36");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_37, "utf8_valid_37");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_38, "utf8_valid_38");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_39, "utf8_valid_39");
+	exotic_add_test(&handle, &exotic_test_utf8_valid_40, "utf8_valid_40");
 	exotic_add_test(&handle, &exotic_test_rbtree_create_destroy, "rbtree_create_destroy");
 	exotic_add_test(&handle, &exotic_test_rbtree_create_1, "rbtree_create_1");
 	exotic_add_test(&handle, &exotic_test_rbtree_size_0, "rbtree_size_0");

autotest/test_misc.tcc

@@ -107,6 +107,7 @@ static const char test_utf_seq_6[] = { 0xE2, 0x82, 0xAC, 0x00}; // valid
 static const char test_utf_seq_7[] = { 0xC2, 0x32, 0x00}; // invalid
 static const char test_utf_seq_8[] = { 0xE2, 0x82, 0x32, 0x00}; // invalid
 static const char test_utf_seq_9[] = { 0xE2, 0x32, 0x82, 0x00}; // invalid
+static const char test_utf_seq_10[] = { 0xF0, 0x9F, 0x98, 0x81, 0x00}; // valid
 
 EXO_TEST(utf8_valid_4, { return is_valid_utf8(test_utf_seq_1); });
 EXO_TEST(utf8_valid_5, { return !is_valid_utf8(test_utf_seq_2); });
@@ -117,5 +118,61 @@ EXO_TEST(utf8_valid_9, { return is_valid_utf8(test_utf_seq_6); });
 EXO_TEST(utf8_valid_10, { return !is_valid_utf8(test_utf_seq_7); });
 EXO_TEST(utf8_valid_11, { return !is_valid_utf8(test_utf_seq_8); });
 EXO_TEST(utf8_valid_12, { return !is_valid_utf8(test_utf_seq_9); });
+EXO_TEST(utf8_valid_13, { return is_valid_utf8(test_utf_seq_10); });
 
+// Limits of utf-8
+static const char test_utf_seq_11[] = { 0x7F, 0x00 }; // valid last 7-bit character
+static const char test_utf_seq_12[] = { 0x80, 0x00 }; // invalid truncated string
+static const char test_utf_seq_13[] = { 0xBF, 0x00 }; // invalid truncated string
+static const char test_utf_seq_14[] = { 0xC0, 0x80, 0x00 }; // invalid out of 2 bytes range
+static const char test_utf_seq_15[] = { 0xC1, 0x7F, 0x00 }; // invalid out of 2 bytes range
+static const char test_utf_seq_16[] = { 0xC2, 0x00 }; // invalid truncated string
+static const char test_utf_seq_17[] = { 0xC2, 0x80, 0x00 }; // valid
+static const char test_utf_seq_18[] = { 0xDF, 0xBF, 0x00 }; // valid
+static const char test_utf_seq_19[] = { 0xE0, 0x80, 0x80, 0x00 }; // invalid out of 3 bytes range
+static const char test_utf_seq_20[] = { 0xE0, 0x9F, 0xBF, 0x00 }; // invalid out of 3 bytes range
+static const char test_utf_seq_21[] = { 0xE0, 0x00 }; // invalid truncated string
+static const char test_utf_seq_22[] = { 0xE0, 0xA0, 0x00 }; // invalid truncated string
+static const char test_utf_seq_23[] = { 0xE0, 0xA0, 0x80, 0x00 }; // valid
+static const char test_utf_seq_24[] = { 0xEC, 0x9F, 0xBF, 0x00 }; // valid
+static const char test_utf_seq_25[] = { 0xED, 0xA0, 0x80, 0x00 }; // invalid surrogate
+static const char test_utf_seq_26[] = { 0xED, 0xBF, 0xBF, 0x00 }; // invalid surrogate
+static const char test_utf_seq_27[] = { 0xEF, 0x80, 0x80, 0x00 }; // valid
+static const char test_utf_seq_28[] = { 0xEF, 0xBF, 0xBF, 0x00 }; // valid
+static const char test_utf_seq_29[] = { 0xF0, 0x80, 0x80, 0x80, 0x00 }; // invalid out of 4 bytes range
+static const char test_utf_seq_30[] = { 0xF0, 0x8F, 0xBF, 0xBF, 0x00 }; // invalid out of 4 bytes range
+static const char test_utf_seq_31[] = { 0xF0, 0x00 }; // invalid truncated string
+static const char test_utf_seq_32[] = { 0xF0, 0x90, 0x00 }; // invalid truncated string
+static const char test_utf_seq_33[] = { 0xF0, 0x90, 0x80, 0x00 }; // invalid truncated string
+static const char test_utf_seq_34[] = { 0xF0, 0x90, 0x80, 0x80, 0x00 }; // valid
+static const char test_utf_seq_35[] = { 0xF4, 0x8F, 0xBF, 0xBF, 0x00 }; // valid
+static const char test_utf_seq_36[] = { 0xF4, 0x90, 0x80, 0x80, 0x00 }; // invalid out of 4 bytes range
+static const char test_utf_seq_37[] = { 0xFF, 0xBF, 0xBF, 0xBF, 0x00 }; // invalid out of 4 bytes range
 
+EXO_TEST(utf8_valid_14, { return is_valid_utf8(test_utf_seq_11); });
+EXO_TEST(utf8_valid_15, { return !is_valid_utf8(test_utf_seq_12); });
+EXO_TEST(utf8_valid_16, { return !is_valid_utf8(test_utf_seq_13); });
+EXO_TEST(utf8_valid_17, { return !is_valid_utf8(test_utf_seq_14); });
+EXO_TEST(utf8_valid_18, { return !is_valid_utf8(test_utf_seq_15); });
+EXO_TEST(utf8_valid_19, { return !is_valid_utf8(test_utf_seq_16); });
+EXO_TEST(utf8_valid_20, { return is_valid_utf8(test_utf_seq_17); });
+EXO_TEST(utf8_valid_21, { return is_valid_utf8(test_utf_seq_18); });
+EXO_TEST(utf8_valid_22, { return !is_valid_utf8(test_utf_seq_19); });
+EXO_TEST(utf8_valid_23, { return !is_valid_utf8(test_utf_seq_20); });
+EXO_TEST(utf8_valid_24, { return !is_valid_utf8(test_utf_seq_21); });
+EXO_TEST(utf8_valid_25, { return !is_valid_utf8(test_utf_seq_22); });
+EXO_TEST(utf8_valid_26, { return is_valid_utf8(test_utf_seq_23); });
+EXO_TEST(utf8_valid_27, { return is_valid_utf8(test_utf_seq_24); });
+EXO_TEST(utf8_valid_28, { return !is_valid_utf8(test_utf_seq_25); });
+EXO_TEST(utf8_valid_29, { return !is_valid_utf8(test_utf_seq_26); });
+EXO_TEST(utf8_valid_30, { return is_valid_utf8(test_utf_seq_27); });
+EXO_TEST(utf8_valid_31, { return is_valid_utf8(test_utf_seq_28); });
+EXO_TEST(utf8_valid_32, { return !is_valid_utf8(test_utf_seq_29); });
+EXO_TEST(utf8_valid_33, { return !is_valid_utf8(test_utf_seq_30); });
+EXO_TEST(utf8_valid_34, { return !is_valid_utf8(test_utf_seq_31); });
+EXO_TEST(utf8_valid_35, { return !is_valid_utf8(test_utf_seq_32); });
+EXO_TEST(utf8_valid_36, { return !is_valid_utf8(test_utf_seq_33); });
+EXO_TEST(utf8_valid_37, { return is_valid_utf8(test_utf_seq_34); });
+EXO_TEST(utf8_valid_38, { return is_valid_utf8(test_utf_seq_35); });
+EXO_TEST(utf8_valid_39, { return !is_valid_utf8(test_utf_seq_36); });
+EXO_TEST(utf8_valid_40, { return !is_valid_utf8(test_utf_seq_37); });

autotest/travis/build-and-test.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 
 set -x
+set -e
 
 export CFLAGS="$(dpkg-buildflags --get CFLAGS) $(dpkg-buildflags --get CPPFLAGS)"
 export LDFLAGS="$(dpkg-buildflags --get LDFLAGS) -Wl,--as-needed"
@@ -33,6 +34,10 @@ cmake ${CMAKEOPTS} \
 make VERBOSE=1
 
 
+make VERBOSE=1 test
+./test
+
+
 sudo make install
 du -shc /etc/uhub/ /usr/bin/uhub* /usr/lib/uhub/
 

src/adc/adcconst.h

@@ -156,9 +156,10 @@ typedef uint32_t fourcc_t;
 #define ADC_SUP_FLAG_ADD                "AD"
 #define ADC_SUP_FLAG_REMOVE             "RM"
 
-#define ADC_CLIENT_TYPE_BOT             "1"
+#define ADC_CLIENT_TYPE_BOT              "1"
-#define ADC_CLIENT_TYPE_REGISTERED_USER "2"
+#define ADC_CLIENT_TYPE_REGISTERED_USER  "2"
-#define ADC_CLIENT_TYPE_OPERATOR        "4"
+#define ADC_CLIENT_TYPE_OPERATOR         "4"
+#define ADC_CLIENT_TYPE_HUBBOT           "5"  /* 1 + 4 */
 #define ADC_CLIENT_TYPE_SUPER_USER      "12"  /* 8 + 4 */
 #define ADC_CLIENT_TYPE_ADMIN           "20"  /* 16 + 4 = hub owner */
 #define ADC_CLIENT_TYPE_HUB             "32"  /* the hub itself */

src/adc/message.c

@@ -24,7 +24,6 @@
 	uhub_assert(X); \
 	uhub_assert(X->cache); \
 	uhub_assert(X->capacity); \
-	uhub_assert(X->length); \
 	uhub_assert(X->length <= X->capacity); \
 	uhub_assert(X->references > 0); \
 	uhub_assert(X->length == strlen(X->cache));
@@ -311,7 +310,7 @@ struct adc_message* adc_msg_parse_verify(struct hub_user* u, const char* line, s
 	if (!command)
 		return 0;
 
-	if (command->source && (!u || command->source != u->id.sid))
+	if (command->source && (!u || (command->source != u->id.sid && !auth_cred_is_unrestricted(u->credentials))))
 	{
 		LOG_DEBUG("Command does not match user's SID (command->source=%d, user->id.sid=%d)", command->source, (u ? u->id.sid : 0));
 		adc_msg_free(command);

src/core/auth.c

@@ -170,6 +170,9 @@ static int acl_parse_line(char* line, int line_count, void* ptr_data)
 	LOG_DEBUG("acl_parse_line: '%s'", line);
 
 	ACL_ADD_USER("bot",        handle->users, auth_cred_bot);
+	ACL_ADD_USER("ubot",        handle->users, auth_cred_ubot);
+	ACL_ADD_USER("opbot",        handle->users, auth_cred_opbot);
+	ACL_ADD_USER("opubot",        handle->users, auth_cred_opubot);
 	ACL_ADD_USER("user_admin", handle->users, auth_cred_admin);
 	ACL_ADD_USER("user_super", handle->users, auth_cred_super);
 	ACL_ADD_USER("user_op",    handle->users, auth_cred_operator);

src/core/config.py

@@ -84,7 +84,7 @@ class Option(object):
 
 class SourceGenerator(object):
 	def __init__(self, filename, cppStyle = True):
-		print "Generating %s..." % filename
+		print ("Generating %s..." % filename)
 		self.f = open(filename, 'w');
 
 	def write_header(self, Comment = True):

src/core/hub.c

@@ -23,7 +23,7 @@ struct hub_info* g_hub = 0;
 
 /* FIXME: Flood control should be done in a plugin! */
 #define CHECK_FLOOD(TYPE, WARN) \
-	if (flood_control_check(&u->flood_ ## TYPE , hub->config->flood_ctl_  ## TYPE, hub->config->flood_ctl_interval, net_get_time())) \
+	if (flood_control_check(&u->flood_ ## TYPE , hub->config->flood_ctl_  ## TYPE, hub->config->flood_ctl_interval, net_get_time()) &&  !auth_cred_is_unrestricted(u->credentials)) \
 	{ \
 		if (WARN) \
 		{ \

src/core/inf.c

@@ -571,6 +571,10 @@ static int set_credentials(struct hub_info* hub, struct hub_user* user, struct a
 			adc_msg_add_argument(cmd, ADC_INF_FLAG_CLIENT_TYPE ADC_CLIENT_TYPE_BOT);
 			break;
 
+		case auth_cred_ubot:
+			adc_msg_add_argument(cmd, ADC_INF_FLAG_CLIENT_TYPE ADC_CLIENT_TYPE_BOT);
+			break;
+
 		case auth_cred_guest:
 			/* Nothing to be added to the info message */
 			break;
@@ -583,6 +587,14 @@ static int set_credentials(struct hub_info* hub, struct hub_user* user, struct a
 			adc_msg_add_argument(cmd, ADC_INF_FLAG_CLIENT_TYPE ADC_CLIENT_TYPE_OPERATOR);
 			break;
 
+		case auth_cred_opbot:
+			adc_msg_add_argument(cmd, ADC_INF_FLAG_CLIENT_TYPE ADC_CLIENT_TYPE_HUBBOT);
+			break;
+
+		case auth_cred_opubot:
+			adc_msg_add_argument(cmd, ADC_INF_FLAG_CLIENT_TYPE ADC_CLIENT_TYPE_HUBBOT);
+			break;
+
 		case auth_cred_super:
 			adc_msg_add_argument(cmd, ADC_INF_FLAG_CLIENT_TYPE ADC_CLIENT_TYPE_SUPER_USER);
 			break;

src/core/ioqueue.c

@@ -19,6 +19,23 @@
 
 #include "uhub.h"
 
+struct ioq_send
+{
+  size_t               size;      /** Size of send queue (in bytes, not messages) */
+  size_t               offset;    /** Queue byte offset in the first message. Should be 0 unless a partial write. */
+#ifdef SSL_SUPPORT
+  size_t               last_send; /** When using SSL, one have to send the exact same buffer and length if a write cannot complete. */
+#endif
+  struct linked_list*  queue;     /** List of queued messages (struct adc_message) */
+};
+
+struct ioq_recv
+{
+  char* buf;
+  size_t size;
+};
+
+
 #ifdef DEBUG_SENDQ
 static void debug_msg(const char* prefix, struct adc_message* msg)
 {
@@ -143,10 +160,46 @@ static void ioq_send_remove(struct ioq_send* q, struct adc_message* msg)
 
 int ioq_send_send(struct ioq_send* q, struct net_connection* con)
 {
-	int ret;
+  int ret;
-	struct adc_message* msg = list_get_first(q->queue);
+  struct adc_message* msg = list_get_first(q->queue);
-	if (!msg) return 0;
+  if (!msg) return 0;
-	uhub_assert(msg->cache && *msg->cache);
+  uhub_assert(msg->cache && *msg->cache);
+
+#ifdef HAVE_FUNC_WRITEV
+#define MAX_IOVEC 32
+  struct iovec vec[MAX_IOVEC];
+  size_t n = 0;
+  vec[0].iov_base = msg->cache + q->offset;
+  vec[0].iov_len = msg->length - q->offset;
+
+  for (struct adc_message* tmp = (struct adc_message*) list_get_next(q->queue), n = 1; tmp && n < MAX_IOVEC; tmp = (struct adc_message*) list_get_next(q->queue), n++)
+  {
+    vec[n].iov_base = msg->cache;
+    vec[n].iov_len = msg->length;
+  }
+
+  ret = net_con_writev(con, &iovec, n);
+
+  if (ret > 0)
+  {
+    while (ret > 0)
+    {
+      if (ret >= (msg->length - q->offset))
+      {
+        ret -= (msg->length - q->offset);
+        q->offset = 0;
+        ioq_send_remove(q, msg);
+      }
+      else
+      {
+        q->offset += ret;
+        return 0;
+      }
+    }
+    return 1;
+  }
+  return ret;
+#else
 	ret = net_con_send(con, msg->cache + q->offset, msg->length - q->offset);
 
 	if (ret > 0)
@@ -159,8 +212,10 @@ int ioq_send_send(struct ioq_send* q, struct net_connection* con)
 		return 1;
 	}
 	return ret;
+#endif
 }
 
+
 int ioq_send_is_empty(struct ioq_send* q)
 {
 	return (q->size - q->offset) == 0;

src/core/ioqueue.h

@@ -21,25 +21,8 @@
 #define HAVE_UHUB_IO_QUEUE_H
 
 struct adc_message;
-struct linked_list;
+struct ioq_send;
-typedef int (*ioq_write)(void* desc, const void* buf, size_t len);
+struct ioq_recv;
-typedef int (*ioq_read)(void* desc, void* buf, size_t len);
-
-struct ioq_send
-{
-	size_t               size;      /** Size of send queue (in bytes, not messages) */
-	size_t               offset;    /** Queue byte offset in the first message. Should be 0 unless a partial write. */
-#ifdef SSL_SUPPORT
-	size_t               last_send; /** When using SSL, one have to send the exact same buffer and length if a write cannot complete. */
-#endif
-	struct linked_list*  queue;     /** List of queued messages (struct adc_message) */
-};
-
-struct ioq_recv
-{
-	char* buf;
-	size_t size;
-};
 
 /**
  * Create a send queue

src/core/route.c

@@ -80,13 +80,13 @@ static int check_send_queue(struct hub_info* hub, struct hub_user* user, struct
 	if (user_flag_get(user, flag_user_list))
 		return 1;
 
-	if ((user->send_queue->size + msg->length) > get_max_send_queue(hub))
+	if ((ioq_send_get_bytes(user->send_queue) + msg->length) > get_max_send_queue(hub))
 	{
 		LOG_WARN("send queue overflowed, message discarded.");
 		return -1;
 	}
 
-	if (user->send_queue->size > get_max_send_queue_soft(hub))
+	if (ioq_send_get_bytes(user->send_queue) > get_max_send_queue_soft(hub))
 	{
 		LOG_WARN("send queue soft overflowed.");
 		return 0;

src/network/connection.c

@@ -57,6 +57,48 @@ ssize_t net_con_send(struct net_connection* con, const void* buf, size_t len)
 	return ret;
 }
 
+#ifdef HAVE_FUNC_WRITEV
+ssize_t net_con_writev(struct net_connection* con, const struct iovec* iov, size_t iocnt)
+{
+  int ret;
+#ifdef SSL_SUPPORT
+  if (!con->ssl)
+  {
+#endif
+    ret = writev(con->sd, iovec, (int) iocnt);
+    if (ret == -1)
+    {
+      if (is_blocked_or_interrupted())
+        return 0;
+      return -1;
+    }
+#ifdef SSL_SUPPORT
+  }
+  else
+  {
+    ssize_t total = 0;
+    ret = 0;
+    while (iocnt--)
+    {
+      ret = net_ssl_send(con, buf, len);
+      if (ret >= 0)
+        total += ret;
+      else
+      {
+        break;
+      }
+    }
+
+    if (total == 0 && !is_blocked_or_interrupted())
+        return -1;
+
+    return total;
+  }
+#endif /* SSL_SUPPORT */
+  return ret;
+}
+#endif
+
 ssize_t net_con_recv(struct net_connection* con, void* buf, size_t len)
 {
 	int ret;

src/network/connection.h

@@ -92,6 +92,11 @@ extern void net_con_close(struct net_connection* con);
  */
 extern ssize_t net_con_send(struct net_connection* con, const void* buf, size_t len);
 
+
+#ifdef HAVE_FUNC_WRITEV
+extern ssize_t net_con_writev(struct net_connection* con, const struct iovec* iov, size_t iocnt);
+#endif
+
 /**
  * Receive data
  *

src/plugins/mod_chat_history_sqlite.c

@@ -49,7 +49,7 @@ static int null_callback(void* ptr, int argc, char **argv, char **colName) { ret
 
 static const char* sql_escape_string(const char* str)
 {
-	static char out[1024];
+	static char out[MAX_HISTORY_SIZE];
 	size_t i = 0;
 	size_t n = 0;
 	for (; n < strlen(str); n++)
@@ -65,7 +65,7 @@ static const char* sql_escape_string(const char* str)
 static int sql_execute(struct chat_history_data* sql, int (*callback)(void* ptr, int argc, char **argv, char **colName), void* ptr, const char* sql_fmt, ...)
 {
 	va_list args;
-	char query[1024];
+	char query[MAX_HISTORY_SIZE];
 	char* errMsg;
 	int rc;
 
@@ -226,6 +226,8 @@ static int command_historycleanup(struct plugin_handle* plugin, struct plugin_us
 	plugin->hub.send_message(plugin, user, cbuf_get(buf));
 	cbuf_destroy(buf);
 
+	sql_execute(data, null_callback, NULL, "VACUUM;");
+
 	return 0;
 }
 

src/system.h.in

@@ -88,6 +88,11 @@
 #cmakedefine HAVE_STRNDUP
 #cmakedefine HAVE_MEMMEM
 
+#cmakedefine HAVE_SYS_UIO_H
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/uio.h>
+#endif
+
 /* printf support for size_t and uint64_t */
 #if defined(WIN32)
 #define PRINTF_SIZE_T "%Iu"

src/tools/uhub-passwd.c

@@ -101,7 +101,19 @@ static const char* validate_cred(const char* cred_str)
 	if (!strcmp(cred_str, "user"))
 		return "user";
 
-	fprintf(stderr, "Invalid user credentials. Must be one of: 'admin', 'super', 'op' or 'user'\n");
+	if (!strcmp(cred_str, "bot"))
+		return "bot";
+
+	if (!strcmp(cred_str, "ubot"))
+		return "ubot";
+
+	if (!strcmp(cred_str, "opbot"))
+		return "opbot";
+
+	if (!strcmp(cred_str, "opubot"))
+		return "opubot";
+
+	fprintf(stderr, "Invalid user credentials. Must be one of: 'bot', 'ubot', 'opbot', 'opubot', 'admin', 'super', 'op' or 'user'\n");
 	exit(1);
 }
 

src/util/credentials.c

@@ -19,11 +19,33 @@
 
 #include "uhub.h"
 
+/**
+ * Returns 1 if a user is unrestricted.
+ * Unrestricted users override the limits of flood and can send messages in
+ * the name of other users.
+ * This is useful for amongst other external chatrooms.
+ */
+int auth_cred_is_unrestricted(enum auth_credentials cred)
+{
+	switch (cred)
+	{
+		case auth_cred_ubot:
+		case auth_cred_opubot:
+			return 1;
+		default:
+			break;
+	}
+	return 0;
+}
+
 int auth_cred_is_protected(enum auth_credentials cred)
 {
 	switch (cred)
 	{
 		case auth_cred_bot:
+		case auth_cred_ubot:
+		case auth_cred_opbot:
+		case auth_cred_opubot:
 		case auth_cred_operator:
 		case auth_cred_super:
 		case auth_cred_admin:
@@ -45,6 +67,9 @@ int auth_cred_is_registered(enum auth_credentials cred)
 	switch (cred)
 	{
 		case auth_cred_bot:
+		case auth_cred_ubot:
+		case auth_cred_opbot:
+		case auth_cred_opubot:
 		case auth_cred_user:
 		case auth_cred_operator:
 		case auth_cred_super:
@@ -64,6 +89,9 @@ const char* auth_cred_to_string(enum auth_credentials cred)
 	{
 		case auth_cred_none:         return "none";
 		case auth_cred_bot:          return "bot";
+		case auth_cred_ubot:         return "ubot";
+		case auth_cred_opbot:        return "opbot";
+		case auth_cred_opubot:       return "opubot";
 		case auth_cred_guest:        return "guest";
 		case auth_cred_user:         return "user";
 		case auth_cred_operator:     return "operator";
@@ -95,14 +123,20 @@ int auth_string_to_cred(const char* str, enum auth_credentials* out)
 			if (!strcasecmp(str, "none")) { *out = auth_cred_none; return 1; }
 			if (!strcasecmp(str, "user")) { *out = auth_cred_user; return 1; }
 			if (!strcasecmp(str, "link")) { *out = auth_cred_link; return 1; }
+			if (!strcasecmp(str, "ubot")) { *out = auth_cred_ubot; return 1; }
 			return 0;
 
 		case 5:
 			if (!strcasecmp(str, "admin")) { *out = auth_cred_admin; return 1; }
 			if (!strcasecmp(str, "super")) { *out = auth_cred_super; return 1; }
+			if (!strcasecmp(str, "opbot")) { *out = auth_cred_opbot; return 1; }
 			if (!strcasecmp(str, "guest")) { *out = auth_cred_guest; return 1; }
 			return 0;
 
+		case 6:
+			if (!strcasecmp(str, "opubot")) { *out = auth_cred_opubot; return 1; }
+			return 0;
+
 		case 8:
 			if (!strcasecmp(str, "operator")) { *out = auth_cred_operator; return 1; }
 			return 0;

src/util/credentials.h

@@ -23,15 +23,24 @@
 enum auth_credentials
 {
 	auth_cred_none,                 /**<<< "User has no credentials (not yet logged in)" */
-	auth_cred_bot,                  /**<<< "User is a robot" */
 	auth_cred_guest,                /**<<< "User is a guest (unregistered user)" */
 	auth_cred_user,                 /**<<< "User is identified as a registered user" */
+	auth_cred_bot,                  /**<<< "User is a robot" */
+	auth_cred_ubot,                 /**<<< "User is an unrestricted robot" */
 	auth_cred_operator,             /**<<< "User is identified as a hub operator" */
+	auth_cred_opbot,                /**<<< "User is a operator robot" */
+	auth_cred_opubot,               /**<<< "User is an unrestricted operator robot" */
 	auth_cred_super,                /**<<< "User is a super user" (not used) */
 	auth_cred_link,                 /**<<< "User is a link (not used currently)" */
 	auth_cred_admin,                /**<<< "User is identified as a hub administrator/owner" */
 };
 
+/**
+ * Returns 1 if the credentials means that a user is unrestricted.
+ * Returns 0 otherwise.
+ */
+int auth_cred_is_unrestricted(enum auth_credentials cred);
+
 /**
  * Returns 1 if the credentials means that a user is protected.
  * Returns 0 otherwise.

src/util/misc.c

@@ -63,7 +63,7 @@ char* strip_white_space(char* string)
 	return string;
 }
 
-static int is_valid_utf8_str(const char* string, size_t length)
+static int is_valid_utf8_str(const unsigned char* string, size_t length)
 {
 	int expect = 0;
 	char div = 0;
@@ -82,12 +82,32 @@ static int is_valid_utf8_str(const char* string, size_t length)
 		{
 			if (string[pos] & 0x80)
 			{
-				for (div = 0x40; div > 0x10; div /= 2)
+				for (div = 0x40; div > 0x08; div /= 2)
 				{
 					if (string[pos] & div) expect++;
 					else break;
 				}
 				if ((string[pos] & div) || (pos+expect >= length)) return 0;
+				switch (expect) {
+					case 0:
+						return 0;
+					case 1:
+						/* Out of range */
+						if (string[pos] < 0xC2) return 0;
+						break;
+					case 2:
+						/* Out of range */
+						if ((string[pos] == 0xE0) && (string[pos+1] < 0xA0 )) return 0;
+						/* Surrogates */
+						if ((string[pos] == 0xED) && (string[pos+1] > 0x9F )) return 0;
+						break;
+					case 3:
+						/* Out of range */
+						if ((string[pos] == 0xF0) && (string[pos+1] < 0x90 )) return 0;
+						if (string[pos] > 0xF4) return 0;
+						if ((string[pos] == 0xF4) && (string[pos+1] > 0x8F )) return 0;
+						break;
+				}
 			}
 		}
 	}

tools/fh2uh_regimport.pl

@@ -0,0 +1,114 @@
+#!/usr/bin/perl
+
+# A simple tool for importing FlexHub users to uhub sqlite database
+#
+# Usage: ./fh2uh_regimport.pl <flexhub_userlist> <uhub_database>
+#
+# Note: uhub database has to be created before running this script.
+
+use File::Slurp;
+use Data::Dumper;
+use DBI;
+
+my @uhubaccounts;
+my $text = read_file $ARGV[0];
+my $dbfile = $ARGV[1];
+
+sub convertprofile
+{
+	$flexprofile = $_[0];
+	if($flexprofile >= 0 && $flexprofile <= 3)
+	{
+		return 'user';
+	}
+	elsif($flexprofile >= 4 && $flexprofile <= 6)
+	{
+		return 'operator';
+	}
+	elsif($flexprofile >= 7 && $flexprofile <= 8)
+	{
+		return 'super';
+	}
+	elsif($flexprofile >= 9 && $flexprofile <= 10)
+	{
+		return 'admin';
+	}
+
+	return 'unknown';
+}
+
+sub parseinfo
+{
+	my @info = split('\n', $_[0]);
+
+	for my $line (@info)
+	{
+		chop $line;
+		my %reginfo;
+		if ($line =~ /\["sNick"\]\s*=\s*\S+/)
+		{
+			my @nick = split(/\["sNick"\]\s*=\s*"(\S+)"/, $line);
+			$reginfo->{'nickname'} = $nick[1];
+		}
+		elsif ($line =~ /\["sPassword"\]\s*=\s*\S+/)
+		{
+			my @password = split(/\["sPassword"\]\s*=\s*"(\S+)"/, $line);
+			$reginfo->{'password'} = $password[1];
+		}
+		elsif ($line =~ /\["iLevel"\]\s*=\s*\S+/)
+		{
+			my @level = split(/\["iLevel"\]\s*=\s*(\d+)/, $line);
+			$reginfo->{'credentials'} = convertprofile $level[1];
+		}
+		elsif ($line =~ /\["iRegDate"\]\s*=\s*\S+/)
+		{
+			my @created = split(/\["iRegDate"\]\s*=\s*(\d+)/, $line);
+			$reginfo->{'created'} = $created[1];
+		}
+		elsif ($line =~ /\["iLastLogin"\]\s*=\s*\S+/)
+		{
+			my @activity = split(/\["iLastLogin"\]\s*=\s*(\d+)/, $line);
+			$reginfo->{'activity'} = $activity[1];
+		}
+	}
+
+	return %{$reginfo};
+}
+
+sub dbimport
+{
+	my @arr = @_;
+	my $db = DBI->connect("dbi:SQLite:dbname=$dbfile", "", "", {RaiseError => 1, AutoCommit => 1});
+
+	for my $import (@arr)
+	{
+		if ($import->{'credentials'} ne 'unknown')
+		{
+			$db->do("INSERT OR IGNORE INTO users (nickname,password,credentials,created,activity) VALUES('$import->{'nickname'}','$import->{'password'}','$import->{'credentials'}',datetime($import->{'created'}, 'unixepoch'),datetime($import->{'activity'}, 'unixepoch'));");
+		}
+	}
+
+	$db->disconnect();
+}
+
+
+if ($text =~ /tAccounts = {/)
+{
+	$text =~ s/^(?:.*\n){1}/},\n/;
+	my @flexaccounts = split('},.*\n.*\[".+"\] = {', $text);
+
+	shift(@flexaccounts);
+
+	for my $account (@flexaccounts)
+	{
+		my %info = parseinfo $account;
+		push(@uhubaccounts, \%info);
+	}
+
+	dbimport @uhubaccounts;
+
+}
+else
+{
+	print "Provided file is not valid FlexHub userlist.\n";
+}

tools/px2uh_regimport.pl

@@ -0,0 +1,62 @@
+#!/usr/bin/perl
+
+# A simple tool for importing PtokaX (< 0.5.0.0) users to uhub sqlite database.
+# Userlist MUST be in xml format.
+#
+# Usage: ./px2uh_regimport.pl <ptokax_userlist.xml> <uhub_database.db>
+#
+# Note: uhub database has to be created before running this script.
+
+use XML::Simple;
+use DBI;
+
+# create xml object
+my $xml = new XML::Simple;
+
+# read XML file
+my $regdata = $xml->XMLin($ARGV[0], ForceArray => [ 'RegisteredUser' ]);
+
+my $dbfile = $ARGV[1];
+my @pxaccounts = @{$regdata->{'RegisteredUser'}};
+
+sub convertprofile
+{
+	$pxprofile = $_[0];
+	if($pxprofile == 2 || $pxprofile == 3)
+	{
+		return 'user';
+	}
+	elsif($pxprofile == 1)
+	{
+		return 'operator';
+	}
+	elsif($pxprofile == 0)
+	{
+		return 'admin';
+	}
+
+	return 'unknown';
+}
+
+sub dbimport
+{
+	my @arr = @_;
+	my $db = DBI->connect("dbi:SQLite:dbname=$dbfile", "", "", {RaiseError => 1, AutoCommit => 1});
+
+	for my $import (@arr)
+	{
+		if ($import->{'credentials'} ne 'unknown')
+		{
+			$db->do("INSERT OR IGNORE INTO users (nickname,password,credentials) VALUES('$import->{'Nick'}','$import->{'Password'}','$import->{'credentials'}');");
+		}
+	}
+
+	$db->disconnect();
+}
+
+for my $account (@pxaccounts)
+{
+	$account->{'credentials'} = convertprofile $account->{'Profile'};
+}
+
+dbimport @pxaccounts;