Skip to content
This repository was archived by the owner on Apr 24, 2022. It is now read-only.

Boost::asio async reads hammering socket and network driver #1059

Closed
AndreaLanfranchi opened this issue May 3, 2018 · 3 comments
Closed

Boost::asio async reads hammering socket and network driver #1059

AndreaLanfranchi opened this issue May 3, 2018 · 3 comments
Labels
invalid

Comments

@AndreaLanfranchi
Copy link
Collaborator

AndreaLanfranchi commented May 3, 2018 •
edited
Loading

I just performed an strace over network communications for both ethminer and claymore

Claymore first :

9545  18:51:45.549111 sendto(138, "{\"id\":6,\"jsonrpc\":\"2.0\",\"method\":\"eth_submitHashrate\",\"params\":[\"0x51d10a7\", \"0x000000000000000000000000000000000000000000000000000000000b767c02\"]}\n", 148, MSG_NOSIGNAL, NULL, 0) = 148
9545  18:51:45.550319 sendto(138, "{\"worker\": \"\", \"jsonrpc\": \"2.0\", \"params\": [], \"id\": 3, \"method\": \"eth_getWork\"}\n", 81, MSG_NOSIGNAL, NULL, 0) = 81
9545  18:51:45.714395 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0x5f4b5ef9d667711a32a3db17f9d058afd8b45c2809dec119f604be5cace1d5f5\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:51:50.405278 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0x08cf4c7615dc32bbe49ba37f21a2a19bc1f5c516d88c6f4e99b026558bae6b12\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:51:55.551469 sendto(138, "{\"worker\": \"\", \"jsonrpc\": \"2.0\", \"params\": [], \"id\": 3, \"method\": \"eth_getWork\"}\n", 81, MSG_NOSIGNAL, NULL, 0) = 81
9545  18:51:55.615130 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0x08cf4c7615dc32bbe49ba37f21a2a19bc1f5c516d88c6f4e99b026558bae6b12\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:51:58.002485 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0xfe4af961882c76ea71e8fd87ba2fd901462d152cd1c64ac08bb6d77d89f0cdc5\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:52:04.629169 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0xccde7f074c300af06d51ba6b285691b01eeba3cb0d6a4b6d5a8aa0c672667fbc\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:52:05.550750 sendto(138, "{\"id\":6,\"jsonrpc\":\"2.0\",\"method\":\"eth_submitHashrate\",\"params\":[\"0x51d0b86\", \"0x000000000000000000000000000000000000000000000000000000000b767c02\"]}\n", 148, MSG_NOSIGNAL, NULL, 0) = 148
9545  18:52:05.555174 sendto(138, "{\"worker\": \"\", \"jsonrpc\": \"2.0\", \"params\": [], \"id\": 3, \"method\": \"eth_getWork\"}\n", 81, MSG_NOSIGNAL, NULL, 0) = 81
9545  18:52:05.715074 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0xccde7f074c300af06d51ba6b285691b01eeba3cb0d6a4b6d5a8aa0c672667fbc\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:52:15.555828 sendto(138, "{\"worker\": \"\", \"jsonrpc\": \"2.0\", \"params\": [], \"id\": 3, \"method\": \"eth_getWork\"}\n", 81, MSG_NOSIGNAL, NULL, 0) = 81
9545  18:52:15.613852 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0xccde7f074c300af06d51ba6b285691b01eeba3cb0d6a4b6d5a8aa0c672667fbc\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:52:18.870710 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0xa67b4edcf4a502970d7c504555328100ad816297c590519239eefe01621ff5c3\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:52:25.552176 sendto(138, "{\"id\":6,\"jsonrpc\":\"2.0\",\"method\":\"eth_submitHashrate\",\"params\":[\"0x51ca8e1\", \"0x000000000000000000000000000000000000000000000000000000000b767c02\"]}\n", 148, MSG_NOSIGNAL, NULL, 0) = 148
9545  18:52:25.557709 sendto(138, "{\"worker\": \"\", \"jsonrpc\": \"2.0\", \"params\": [], \"id\": 3, \"method\": \"eth_getWork\"}\n", 81, MSG_NOSIGNAL, NULL, 0) = 81
9545  18:52:25.717288 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0xa67b4edcf4a502970d7c504555328100ad816297c590519239eefe01621ff5c3\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:52:34.369118 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0x81c706766eb03683ac8bb4aee6dbcce43eddb29199941362143eaa65c9740344\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:52:35.558789 sendto(138, "{\"worker\": \"\", \"jsonrpc\": \"2.0\", \"params\": [], \"id\": 3, \"method\": \"eth_getWork\"}\n", 81, MSG_NOSIGNAL, NULL, 0) = 81
9545  18:52:35.618940 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0x81c706766eb03683ac8bb4aee6dbcce43eddb29199941362143eaa65c9740344\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:52:45.553705 sendto(138, "{\"id\":6,\"jsonrpc\":\"2.0\",\"method\":\"eth_submitHashrate\",\"params\":[\"0x51d67a2\", \"0x000000000000000000000000000000000000000000000000000000000b767c02\"]}\n", 148, MSG_NOSIGNAL, NULL, 0) = 148
9545  18:52:45.560267 sendto(138, "{\"worker\": \"\", \"jsonrpc\": \"2.0\", \"params\": [], \"id\": 3, \"method\": \"eth_getWork\"}\n", 81, MSG_NOSIGNAL, NULL, 0) = 81
9545  18:52:45.711185 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0x81c706766eb03683ac8bb4aee6dbcce43eddb29199941362143eaa65c9740344\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:52:52.985316 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0x6d38803fffb145641fa5ce7ff86cb973bdc58ea37bd4e36b16beb24edbf0849a\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243
9545  18:52:55.561693 sendto(138, "{\"worker\": \"\", \"jsonrpc\": \"2.0\", \"params\": [], \"id\": 3, \"method\": \"eth_getWork\"}\n", 81, MSG_NOSIGNAL, NULL, 0) = 81
9545  18:52:55.625927 recvfrom(138, "{\"jsonrpc\":\"2.0\",\"id\":0,\"result\":[\"0x6d38803fffb145641fa5ce7ff86cb973bdc58ea37bd4e36b16beb24edbf0849a\",\"0xad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0x000000006df37f675ef6eadf5ab9a2072d44268d97df837e6748956e5c6c2116\"]}\n", 243, 0, NULL, NULL) = 243

Nothing special and really expected.

Now ethminer

3883  18:46:53.928269 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:53.930865 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)

[cut 253 identical rows in less than 1 second]

3883  18:46:54.588125 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3890  18:46:54.589874 sendmsg(51, {msg_name(0)=NULL, msg_iov(1)=[{"{\"id\":4,\"jsonrpc\":\"2.0\",\"method\":\"mining.submit\",\"params\":[\"0x4813aEEE0c30C584C559fa8Dc7424481E2e9Fc91.BG01R01\",\"f1247fc5c48962cea16221be235de1020643218257d490093ee262ae750f1616\",\"0x2682f115787dd122\",\"0xf1247fc5c48962cea16221be235de1020643218257d490093ee262ae750f1616\",\"0x7a159032565b9bb8085d49bde0451e1bbc0688471957066ea4dd9c5883a97835\"],\"worker\":\"BG01R01\"}\n", 359}], msg_controllen=0, msg_flags=0}, MSG_NOSIGNAL) = 359
3883  18:46:54.590712 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.593297 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.595880 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.598460 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.601034 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.603609 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.606183 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.609143 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.611737 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.614339 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.616936 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.619524 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.622107 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.624682 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.627258 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.629833 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3886  18:46:54.632166 recvmsg(51, {msg_name(0)=NULL, msg_iov(1)=[{"{\"id\":4,\"jsonrpc\":\"2.0\",\"result\":true}\n", 1024}], msg_controllen=0, msg_flags=0}, 0) = 39
3886  18:46:54.632284 recvmsg(51, 0x7f2b6b472240, 0) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.632409 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:54.634988 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)

[cut 360 identical rows in less than 1 second]

3883  18:46:55.493760 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:55.496347 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:55.498934 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3886  18:46:55.499474 recvmsg(51, {msg_name(0)=NULL, msg_iov(1)=[{"{\"id\":6,\"jsonrpc\":\"2.0\",\"method\":\"mining.notify\",\"params\":[\"cbaa1a4d2d3b1e86484e17319a97320d4f469d0aefbe9c8b463b6923adff7f18\",\"cbaa1a4d2d3b1e86484e17319a97320d4f469d0aefbe9c8b463b6923adff7f18\",\"ad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0000000112e0be826d694b2e62d01511f12a6061fbaec8bc02357593e70e52ba\",false]}\n", 1024}], msg_controllen=0, msg_flags=0}, 0) = 335
3886  18:46:55.499645 recvmsg(51, 0x7f2b6b472240, 0) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:55.501514 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:46:55.504098 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)

[cut 3530 identical rows in less than 10 seconds]

3883  18:47:04.656530 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:47:04.659108 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3886  18:47:04.661301 recvmsg(51, {msg_name(0)=NULL, msg_iov(1)=[{"{\"id\":6,\"jsonrpc\":\"2.0\",\"method\":\"mining.notify\",\"params\":[\"06cf9c1a96c15709e160c4c8b3af8c43fc5d19da938a77f41b7d88d37a4b37c6\",\"06cf9c1a96c15709e160c4c8b3af8c43fc5d19da938a77f41b7d88d37a4b37c6\",\"ad15d04b13b18ecbb6bc1c05cefa1e952fe584f2c79fb5f3dbc48656704b0f95\",\"0000000112e0be826d694b2e62d01511f12a6061fbaec8bc02357593e70e52ba\",false]}\n", 1024}], msg_controllen=0, msg_flags=0}, 0) = 335
3886  18:47:04.661457 recvmsg(51, 0x7f2b6b472240, 0) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:47:04.661687 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:47:04.664266 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:47:04.666849 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)
3883  18:47:04.669435 accept(50, 0x7f2b6cc75ec0, 0x7f2b6cc75eac) = -1 EAGAIN (Resource temporarily unavailable)

Apparently Boost::asio is hammering with polls the socket in search for data and I have the suspect that this behavior may lead to cases like #936 where weak CPUs may record high usage and eventually cause segfaults on nic driver.

I think (but I might be well wrong) async_read_until (as it's a combined operation of multiple async_read_some) keeps polling io_service=>nic in search of small chuncks of data causing exit with error EAGAIN continuosly preventing locking.
Probably keeping the listening socket on a separate lockable and sync thread would mitigate this issue.

@chfast
@smurfy
@jean-m-cyr
@MariusVanDerWijden

Any help or hint appreciated.
@chfast
Copy link
Contributor

chfast commented May 3, 2018

I have little knowledge about network and sockets in general. Maybe we should reach out to boost?

@AndreaLanfranchi
Copy link
Collaborator Author

Would be helpful.
Does anyone how to get in touch with them ?

@AndreaLanfranchi
Copy link
Collaborator Author

It's API server causing this issue.

AndreaLanfranchi added a commit that referenced this issue Jun 2, 2018
@AndreaLanfranchi
Api Server Revised

Unverified

This commit is not signed, but one or more authors requires that any commit attributed to them is signed.
Learn about vigilant mode
41ee9ca 
Got rid of libjson-rpc-cpp which hammers network driver (#1059)
Built API server entirely on boost::asio

Starting point to share http and rpc on same port
@AndreaLanfranchi AndreaLanfranchi mentioned this issue Jun 2, 2018
Closed
AndreaLanfranchi added a commit that referenced this issue Jun 3, 2018
@AndreaLanfranchi
New Api server based on Boost

Unverified

This commit is not signed, but one or more authors requires that any commit attributed to them is signed.
Learn about vigilant mode
d4229f5 
- Got rid of jsonrpccpp server which hammers socket (#1059)
- Some cnotes pushed in log to detect API operations
- Socket(s) are persistent till disconnection (a client can issue
multiple requests)
- Added miner_shuffle method to re-rand nonce scrambler at runtime (it
may give you the chance to abandon a low performance batch)
@AndreaLanfranchi AndreaLanfranchi mentioned this issue Jun 3, 2018
Merged
Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Assignees
No one assigned
Labels
invalid
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
2 participants
@chfast @AndreaLanfranchi