[Feature] Issue#44 Thread pool

Description:
------------
Supply a new thread handing strategy that is pool-of-thread,
one-thread-per-connection strategy will create many threads to service connections that
clients requested. Not only it has low efficiency, but also cost much CPU time on context switch.

Pool-of-thread only create limited threads to service all connections，and keep the throughput stable
no matter how many connection requests.

include/my_sys.h

@@ -52,6 +52,9 @@ typedef struct my_aio_result {
 
 #define MY_INIT(name)   { my_progname= name; my_init(); }
 
+extern ulonglong my_interval_timer(void);
+#define microsecond_interval_timer()  (my_interval_timer()/1000)
+
 /**
   Max length of an error message generated by mysys utilities.
   Some mysys functions produce error messages. These mostly go

include/mysql/plugin_audit.h.pp

@@ -44,7 +44,8 @@
   THD_WAIT_BINLOG= 8,
   THD_WAIT_GROUP_COMMIT= 9,
   THD_WAIT_SYNC= 10,
-  THD_WAIT_LAST= 11
+  THD_WAIT_NET= 11,
+  THD_WAIT_LAST= 12
 } thd_wait_type;
 extern struct thd_wait_service_st {
   void (*thd_wait_begin_func)(void*, int);

include/mysql/plugin_auth.h.pp

@@ -44,7 +44,8 @@
   THD_WAIT_BINLOG= 8,
   THD_WAIT_GROUP_COMMIT= 9,
   THD_WAIT_SYNC= 10,
-  THD_WAIT_LAST= 11
+  THD_WAIT_NET= 11,
+  THD_WAIT_LAST= 12
 } thd_wait_type;
 extern struct thd_wait_service_st {
   void (*thd_wait_begin_func)(void*, int);

include/mysql/plugin_ftparser.h.pp

@@ -44,7 +44,8 @@
   THD_WAIT_BINLOG= 8,
   THD_WAIT_GROUP_COMMIT= 9,
   THD_WAIT_SYNC= 10,
-  THD_WAIT_LAST= 11
+  THD_WAIT_NET= 11,
+  THD_WAIT_LAST= 12
 } thd_wait_type;
 extern struct thd_wait_service_st {
   void (*thd_wait_begin_func)(void*, int);

include/mysql/service_thd_wait.h

@@ -74,7 +74,8 @@ typedef enum _thd_wait_type_e {
   THD_WAIT_BINLOG= 8,
   THD_WAIT_GROUP_COMMIT= 9,
   THD_WAIT_SYNC= 10,
-  THD_WAIT_LAST= 11
+  THD_WAIT_NET= 11,
+  THD_WAIT_LAST= 12
 } thd_wait_type;
 
 extern struct thd_wait_service_st {

include/mysql/thread_pool_priv.h

@@ -107,7 +107,7 @@ void end_connection(THD *thd);
 /* Release resources of the THD object */
 void thd_release_resources(THD *thd);
 /* Decrement connection counter */
-void dec_connection_count();
+void dec_connection_count(THD *thd);
 /* Destroy THD object */
 void destroy_thd(THD *thd);
 /* Remove the THD from the set of global threads. */

include/violite.h

@@ -74,6 +74,7 @@ Vio* vio_new_win32shared_memory(HANDLE handle_file_map,
 
 void    vio_delete(Vio* vio);
 int vio_shutdown(Vio* vio);
+int vio_cancel(Vio* vio, int how);
 my_bool vio_reset(Vio* vio, enum enum_vio_type type,
                   my_socket sd, void *ssl, uint flags);
 size_t  vio_read(Vio *vio, uchar *	buf, size_t size);
@@ -105,6 +106,8 @@ ssize_t vio_pending(Vio *vio);
 #endif
 /* Set timeout for a network operation. */
 int vio_timeout(Vio *vio, uint which, int timeout_sec);
+extern void vio_set_wait_callback(void (*before_wait)(void),
+                                  void (*after_wait)(void));
 /* Connect to a peer. */
 my_bool vio_socket_connect(Vio *vio, struct sockaddr *addr, socklen_t len,
                            int timeout);
@@ -190,6 +193,7 @@ void vio_end(void);
 #define vio_keepalive(vio, set_keep_alive)  (vio)->viokeepalive(vio, set_keep_alive)
 #define vio_should_retry(vio)                   (vio)->should_retry(vio)
 #define vio_was_timeout(vio)                    (vio)->was_timeout(vio)
+#define vio_cancel(vio, how)                    ((vio)->viocancel)(vio, how)
 #define vio_shutdown(vio)                       ((vio)->vioshutdown)(vio)
 #define vio_peer_addr(vio, buf, prt, buflen)    (vio)->peer_addr(vio, buf, prt, buflen)
 #define vio_io_wait(vio, event, timeout)        (vio)->io_wait(vio, event, timeout)
@@ -256,6 +260,7 @@ struct st_vio
      descriptors, handles can remain valid after a shutdown.
   */
   int     (*vioshutdown)(Vio*);
+  int     (*viocancel)(Vio*, int);
   my_bool (*is_connected)(Vio*);
   my_bool (*has_data) (Vio*);
   int (*io_wait)(Vio*, enum enum_vio_io_event, int);

mysql-test/include/have_pool_of_threads.inc

@@ -0,0 +1,4 @@
+-- require r/have_pool_of_threads.require
+disable_query_log;
+show variables like 'thread_handling';
+enable_query_log;

mysql-test/r/have_pool_of_threads.require

@@ -0,0 +1,2 @@
+Variable_name	Value
+thread_handling	pool-of-threads

mysql-test/r/information_schema-big.result

@@ -43,6 +43,7 @@ SCHEMATA	SCHEMA_NAME
 SCHEMA_PRIVILEGES	TABLE_SCHEMA
 SESSION_STATUS	VARIABLE_NAME
 SESSION_VARIABLES	VARIABLE_NAME
+SQL_FILTER_INFO	TYPE
 STATISTICS	TABLE_SCHEMA
 TABLES	TABLE_SCHEMA
 TABLESPACES	TABLESPACE_NAME
@@ -51,6 +52,15 @@ TABLE_PRIVILEGES	TABLE_SCHEMA
 TRIGGERS	TRIGGER_SCHEMA
 USER_PRIVILEGES	GRANTEE
 VIEWS	TABLE_SCHEMA
+TABLE_STATISTICS	TABLE_SCHEMA
+INDEX_STATISTICS	TABLE_SCHEMA
+THREAD_GROUP_STATUS	ID
+TokuDB_file_map	table_schema
+TokuDB_trx	trx_id
+TokuDB_locks	locks_table_schema
+TokuDB_lock_waits	lock_waits_table_schema
+TokuDB_fractal_tree_block_map	table_schema
+TokuDB_fractal_tree_info	table_schema
 SELECT t.table_name, c1.column_name
 FROM information_schema.tables t
 INNER JOIN
@@ -91,6 +101,7 @@ SCHEMATA	SCHEMA_NAME
 SCHEMA_PRIVILEGES	TABLE_SCHEMA
 SESSION_STATUS	VARIABLE_NAME
 SESSION_VARIABLES	VARIABLE_NAME
+SQL_FILTER_INFO	TYPE
 STATISTICS	TABLE_SCHEMA
 TABLES	TABLE_SCHEMA
 TABLESPACES	TABLESPACE_NAME
@@ -99,3 +110,12 @@ TABLE_PRIVILEGES	TABLE_SCHEMA
 TRIGGERS	TRIGGER_SCHEMA
 USER_PRIVILEGES	GRANTEE
 VIEWS	TABLE_SCHEMA
+TABLE_STATISTICS	TABLE_SCHEMA
+INDEX_STATISTICS	TABLE_SCHEMA
+THREAD_GROUP_STATUS	ID
+TokuDB_file_map	table_schema
+TokuDB_trx	trx_id
+TokuDB_locks	locks_table_schema
+TokuDB_lock_waits	lock_waits_table_schema
+TokuDB_fractal_tree_block_map	table_schema
+TokuDB_fractal_tree_info	table_schema

mysql-test/r/information_schema.result

@@ -87,6 +87,7 @@ USER_PRIVILEGES
 VIEWS
 TABLE_STATISTICS
 INDEX_STATISTICS
+THREAD_GROUP_STATUS
 TokuDB_file_map
 TokuDB_trx
 TokuDB_locks
@@ -134,6 +135,7 @@ TABLE_CONSTRAINTS	TABLE_CONSTRAINTS
 TABLE_PRIVILEGES	TABLE_PRIVILEGES
 TRIGGERS	TRIGGERS
 TABLE_STATISTICS	TABLE_STATISTICS
+THREAD_GROUP_STATUS	THREAD_GROUP_STATUS
 TokuDB_file_map	TokuDB_file_map
 TokuDB_trx	TokuDB_trx
 TokuDB_locks	TokuDB_locks
@@ -161,6 +163,7 @@ TABLE_CONSTRAINTS	TABLE_CONSTRAINTS
 TABLE_PRIVILEGES	TABLE_PRIVILEGES
 TRIGGERS	TRIGGERS
 TABLE_STATISTICS	TABLE_STATISTICS
+THREAD_GROUP_STATUS	THREAD_GROUP_STATUS
 TokuDB_file_map	TokuDB_file_map
 TokuDB_trx	TokuDB_trx
 TokuDB_locks	TokuDB_locks
@@ -188,6 +191,7 @@ TABLE_CONSTRAINTS	TABLE_CONSTRAINTS
 TABLE_PRIVILEGES	TABLE_PRIVILEGES
 TRIGGERS	TRIGGERS
 TABLE_STATISTICS	TABLE_STATISTICS
+THREAD_GROUP_STATUS	THREAD_GROUP_STATUS
 TokuDB_file_map	TokuDB_file_map
 TokuDB_trx	TokuDB_trx
 TokuDB_locks	TokuDB_locks
@@ -688,6 +692,7 @@ TABLE_CONSTRAINTS
 TABLE_PRIVILEGES
 TRIGGERS
 TABLE_STATISTICS
+THREAD_GROUP_STATUS
 TokuDB_file_map
 TokuDB_trx
 TokuDB_locks
@@ -705,6 +710,7 @@ TABLE_CONSTRAINTS	SYSTEM VIEW
 TABLE_PRIVILEGES	SYSTEM VIEW
 TRIGGERS	SYSTEM VIEW
 TABLE_STATISTICS	SYSTEM VIEW
+THREAD_GROUP_STATUS	SYSTEM VIEW
 TokuDB_file_map	SYSTEM VIEW
 TokuDB_trx	SYSTEM VIEW
 TokuDB_locks	SYSTEM VIEW
@@ -725,6 +731,7 @@ TABLE_CONSTRAINTS
 TABLE_PRIVILEGES
 TRIGGERS
 TABLE_STATISTICS
+THREAD_GROUP_STATUS
 TokuDB_file_map
 TokuDB_trx
 TokuDB_locks
@@ -951,7 +958,7 @@ table_schema IN ('mysql', 'INFORMATION_SCHEMA', 'test', 'mysqltest')
 AND table_name not like 'ndb%' AND table_name not like 'innodb_%'
 GROUP BY TABLE_SCHEMA;
 table_schema	count(*)
-information_schema	40
+information_schema	41
 mysql	25
 create table t1 (i int, j int);
 create trigger trg1 before insert on t1 for each row
@@ -1422,6 +1429,7 @@ TABLESPACES	information_schema.TABLESPACES	1
 TABLE_CONSTRAINTS	information_schema.TABLE_CONSTRAINTS	1
 TABLE_PRIVILEGES	information_schema.TABLE_PRIVILEGES	1
 TABLE_STATISTICS	information_schema.TABLE_STATISTICS	1
+THREAD_GROUP_STATUS	information_schema.THREAD_GROUP_STATUS	1
 TokuDB_file_map	information_schema.TokuDB_file_map	1
 TokuDB_fractal_tree_block_map	information_schema.TokuDB_fractal_tree_block_map	1
 TokuDB_fractal_tree_info	information_schema.TokuDB_fractal_tree_info	1

mysql-test/r/information_schema_db.result

@@ -39,6 +39,7 @@ USER_PRIVILEGES
 VIEWS
 TABLE_STATISTICS
 INDEX_STATISTICS
+THREAD_GROUP_STATUS
 TokuDB_file_map
 TokuDB_trx
 TokuDB_locks
@@ -53,6 +54,7 @@ TABLE_CONSTRAINTS
 TABLE_PRIVILEGES
 TRIGGERS
 TABLE_STATISTICS
+THREAD_GROUP_STATUS
 TokuDB_file_map
 TokuDB_trx
 TokuDB_locks

mysql-test/r/mysqld--help-notwin.result

@@ -242,6 +242,11 @@ The following options may be given as the first argument:
  With this option enabled you can run myisamchk to test
  (not repair) tables while the MySQL server is running.
  Disable with --skip-external-locking.
+ --extra-max-connections=# 
+ The number of connections on extra-port
+ --extra-port=#      Extra port number to use for tcp connections in a
+ one-thread-per-connection manner. 0 means don't use
+ another port
  --flush             Flush MyISAM tables to disk between SQL commands
  --flush-time=#      A dedicated thread is created to flush all tables at the
  given interval
@@ -1047,8 +1052,46 @@ The following options may be given as the first argument:
  How many threads we should keep in a cache for reuse
  --thread-handling=name 
  Define threads usage for handling queries, one of
- one-thread-per-connection, no-threads, loaded-dynamically
+ one-thread-per-connection, no-threads, pool-of-threads
+ --thread-pool-high-prio-mode=name 
+ High priority queue mode: one of 'transactions',
+ 'statements' or 'none'. In the 'transactions' mode the
+ thread pool uses both high- and low-priority queues
+ depending on whether an event is generated by an already
+ started transaction and whether it has any high priority
+ tickets (see thread_pool_high_prio_tickets). In the
+ 'statements' mode all events (i.e. individual statements)
+ always go to the high priority queue, regardless of the
+ current transaction state and high priority tickets.
+ 'none' is the opposite of 'statements', i.e. disables the
+ high priority queue completely.
+ --thread-pool-high-prio-tickets=# 
+ Number of tickets to enter the high priority event queue
+ for each transaction.
+ --thread-pool-idle-timeout=# 
+ Timeout in seconds for an idle thread in the thread
+ pool.Worker thread will be shutdown after timeout
+ --thread-pool-max-threads=# 
+ Maximus allowed number of worker threads in the thread
+ pool
+ --thread-pool-oversubscribe=# 
+ How many additional active worker threads in a group are
+ allowed.
+ --thread-pool-size=# 
+ Number of thread groups in the pool. This parameter is
+ roughly equivalent to maximum number of concurrently
+ executing threads (threads in a waiting state do not
+ count as executing).
+ --thread-pool-stall-limit=# 
+ Maximum query execution time in milliseconds,before an
+ executing non-yielding thread is considered stalled.If a
+ worker thread is stalled, additional worker thread may be
+ created to handle remaining clients.
  --thread-stack=#    The stack size for each thread
+ --threadpool-workaround-epoll-bug 
+ Workaround Linux kernel bug: missing events in epoll, if
+ EPOLL_CTL_MOD is used.The bug is fixed in the newest 3.x
+ kernel series
  --time-format=name  The TIME format (ignored)
  --timed-mutexes     Specify whether to time mutexes. Deprecated, has no
  effect.
@@ -1268,6 +1311,8 @@ event-scheduler OFF
 expire-logs-days 0
 explicit-defaults-for-timestamp FALSE
 external-locking FALSE
+extra-max-connections 10000
+extra-port 0
 flush FALSE
 flush-time 0
 ft-boolean-syntax + -><()~*:""&|
@@ -1515,7 +1560,15 @@ table-open-cache-instances 1
 tc-heuristic-recover COMMIT
 thread-cache-size 9
 thread-handling one-thread-per-connection
+thread-pool-high-prio-mode transactions
+thread-pool-high-prio-tickets -1
+thread-pool-idle-timeout 60
+thread-pool-max-threads 100000
+thread-pool-oversubscribe 3
+thread-pool-size 24
+thread-pool-stall-limit 10
 thread-stack 262144
+threadpool-workaround-epoll-bug FALSE
 time-format %H:%i:%s
 timed-mutexes FALSE
 tmp-table-size 16777216

mysql-test/r/mysqlshow.result

@@ -113,6 +113,7 @@ Database: information_schema
 | VIEWS                                 |
 | TABLE_STATISTICS                      |
 | INDEX_STATISTICS                      |
+| THREAD_GROUP_STATUS                   |
 | TokuDB_file_map                       |
 | TokuDB_trx                            |
 | INNODB_SYS_DATAFILES                  |
@@ -187,6 +188,7 @@ Database: INFORMATION_SCHEMA
 | VIEWS                                 |
 | TABLE_STATISTICS                      |
 | INDEX_STATISTICS                      |
+| THREAD_GROUP_STATUS                   |
 | TokuDB_file_map                       |
 | TokuDB_trx                            |
 | INNODB_SYS_DATAFILES                  |