Winsock non-blocking overlapped IO still blocks - sockets

Question

I am trying to write a simple winsock client that sends data to a
winsock server using non-blocking overlapped IO. The problem is that the WSASend call is blocking. The client code looks like this
// Initialize Winsock
WSAStartup(MAKEWORD(2, 2), &wsaData);
// Create a SOCKET for connecting to server
ConnectSocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
// Connect to server.
WSAConnect(ConnectSocket, ai_addr, ai_addrlen, 0, 0, 0, 0);
// Make the socket non-blocking
u_long iMode = 1;
ioctlsocket(ConnectSocket, FIONBIO, &iMode);
// Send the data
WSAOVERLAPPED SendOverlapped{};
SendOverlapped.hEvent = WSACreateEvent();
WSASend(ConnectSocket, &DataBuf, 1, &SendBytes, 0, &SendOverlapped, 0);
I have made the socket non-blocking through the ioctlsocket function and I have provided WSA_FLAG_OVERLAPPED flag to the WSASocket function. I have also provided the lpOverlapped parameter to the WSASend function. However, the call to WSASend is still blocking. Am I missing something here?
Please excuse the lack of error checking in the above code, it is only to keep it simple for the purposes of this question.
Clarification: The reason why it seems to me the call is blocking is this - I called the WSASend function 2560 times in a loop, sending 4MB each time in the loop. The loop completed in 16 secs after all the data was transferred to the server. If it was non-blocking, I would have expected the loop to complete much sooner. As expected, the WSASend function does return with ERROR_IO_PENDING though.
Here is the full client code
// Client.cpp
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <stdio.h>
#include <windows.h>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <iostream>
#include <ctime>
#pragma comment (lib, "Ws2_32.lib")
#pragma comment (lib, "Mswsock.lib")
#pragma comment (lib, "AdvApi32.lib")
#define DEFAULT_BUFLEN 4 * 1024 * 1024
#define DEFAULT_PORT "27015"
int __cdecl main(int argc, char **argv)
{
// Initialize Winsock
WSADATA wsaData;
int iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
if (iResult != 0) {
printf("WSAStartup failed with error: %d\n", iResult);
return 1;
}
// Resolve the server address and port
addrinfo *result = NULL;
addrinfo hints{};
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
iResult = getaddrinfo("127.0.0.1", DEFAULT_PORT, &hints, &result);
if (iResult != 0) {
printf("getaddrinfo failed with error: %d\n", iResult);
WSACleanup();
return 1;
}
// Create a SOCKET for connecting to server
SOCKET ConnectSocket = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
if (ConnectSocket == INVALID_SOCKET) {
printf("socket failed with error: %ld\n", WSAGetLastError());
WSACleanup();
return 1;
}
// Connect to server.
iResult = WSAConnect(ConnectSocket, result->ai_addr, result->ai_addrlen, 0, 0, 0, 0);
freeaddrinfo(result);
if (ConnectSocket == INVALID_SOCKET) {
printf("Unable to connect to server!\n");
WSACleanup();
return 1;
}
// Make the socket non-blocking
u_long iMode = 1;
iResult = ioctlsocket(ConnectSocket, FIONBIO, &iMode);
if (iResult != NO_ERROR) {
printf("ioctlsocket failed with error: %ld\n", iResult);
WSACleanup();
return 1;
}
// Prepare the buffer
char *sendbuf = new char[DEFAULT_BUFLEN];
for (int i = 0; i < DEFAULT_BUFLEN; ++i)
sendbuf[i] = 'a';
WSABUF DataBuf;
DWORD SendBytes = 0;
DataBuf.buf = sendbuf;
DataBuf.len = DEFAULT_BUFLEN;
// Send the buffer in a loop
int loopCount = 2560;
WSAOVERLAPPED* SendOverlapped = (WSAOVERLAPPED*)calloc(loopCount, sizeof(WSAOVERLAPPED));
clock_t start = clock();
for (int i = 0; i < loopCount; ++i)
{
SendOverlapped[i].hEvent = WSACreateEvent();
iResult = WSASend(ConnectSocket, &DataBuf, 1, &SendBytes, 0, SendOverlapped + i, 0);
if (iResult == SOCKET_ERROR)
{
if (ERROR_IO_PENDING == WSAGetLastError())
{
continue;
}
printf("send failed with error: %d\n", WSAGetLastError());
closesocket(ConnectSocket);
WSACleanup();
return 1;
}
}
std::cout << "initiating send data took " << clock() - start << " ms" << std::endl;
// Wait for all the events to be signalled
for (int i = 0; i < loopCount; ++i)
{
iResult = WSAWaitForMultipleEvents(1, &SendOverlapped[i].hEvent, TRUE, INFINITE, TRUE);
if (iResult == WSA_WAIT_FAILED) {
printf("WSAWaitForMultipleEvents failed with error: %d\n", WSAGetLastError());
closesocket(ConnectSocket);
WSACleanup();
return 1;
}
DWORD Flags = 0;
BOOL result = WSAGetOverlappedResult(ConnectSocket, SendOverlapped + i, &SendBytes, FALSE, &Flags);
if (result == FALSE) {
printf("WSASend failed with error: %d\n", WSAGetLastError());
break;
}
}
std::cout << "actual send data took " << clock() - start << " ms";
// shutdown the connection since no more data will be sent
iResult = shutdown(ConnectSocket, SD_SEND);
if (iResult == SOCKET_ERROR) {
printf("shutdown failed with error: %d\n", WSAGetLastError());
closesocket(ConnectSocket);
WSACleanup();
return 1;
}
// cleanup
closesocket(ConnectSocket);
WSACleanup();
////free(SendOverlapped);
return 0;
}
And here is the server side code
// Server.cpp
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <stdio.h>
#include <windows.h>
#include <winsock2.h>
#include <ws2tcpip.h>
#include <iostream>
#pragma comment (lib, "Ws2_32.lib")
#define DEFAULT_BUFLEN 4 * 1024 * 1024
#define DEFAULT_PORT "27015"
int __cdecl main(void)
{
// Initialize Winsock
WSADATA wsaData;
int iResult = WSAStartup(MAKEWORD(2, 2), &wsaData);
if (iResult != 0) {
printf("WSAStartup failed with error: %d\n", iResult);
return 1;
}
// Resolve the server address and port
addrinfo hints{};
hints.ai_family = AF_INET;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
hints.ai_flags = AI_PASSIVE;
addrinfo *result = NULL;
iResult = getaddrinfo(NULL, DEFAULT_PORT, &hints, &result);
if (iResult != 0) {
printf("getaddrinfo failed with error: %d\n", iResult);
WSACleanup();
return 1;
}
// Create a SOCKET for connecting to server
SOCKET ListenSocket = socket(result->ai_family, result->ai_socktype, result->ai_protocol);
if (ListenSocket == INVALID_SOCKET) {
printf("socket failed with error: %ld\n", WSAGetLastError());
freeaddrinfo(result);
WSACleanup();
return 1;
}
// Setup the TCP listening socket
iResult = bind(ListenSocket, result->ai_addr, (int)result->ai_addrlen);
if (iResult == SOCKET_ERROR) {
printf("bind failed with error: %d\n", WSAGetLastError());
freeaddrinfo(result);
closesocket(ListenSocket);
WSACleanup();
return 1;
}
freeaddrinfo(result);
iResult = listen(ListenSocket, SOMAXCONN);
if (iResult == SOCKET_ERROR) {
printf("listen failed with error: %d\n", WSAGetLastError());
closesocket(ListenSocket);
WSACleanup();
return 1;
}
// Accept a client socket
SOCKET ClientSocket = accept(ListenSocket, NULL, NULL);
if (ClientSocket == INVALID_SOCKET) {
printf("accept failed with error: %d\n", WSAGetLastError());
closesocket(ListenSocket);
WSACleanup();
return 1;
}
// No longer need server socket
closesocket(ListenSocket);
// Receive until the peer shuts down the connection
char* recvbuf = new char[DEFAULT_BUFLEN];
do {
iResult = recv(ClientSocket, recvbuf, DEFAULT_BUFLEN, 0);
if (iResult > 0) {
printf("Bytes received: %d\n", iResult);
}
else if (iResult == 0)
printf("Connection closing...\n");
else {
printf("recv failed with error: %d\n", WSAGetLastError());
closesocket(ClientSocket);
WSACleanup();
return 1;
}
} while (iResult > 0);
// shutdown the connection since we're done
iResult = shutdown(ClientSocket, SD_SEND);
if (iResult == SOCKET_ERROR) {
printf("shutdown failed with error: %d\n", WSAGetLastError());
closesocket(ClientSocket);
WSACleanup();
return 1;
}
// cleanup
closesocket(ClientSocket);
WSACleanup();
return 0;
}

Answer 1

The WSASend call was blocking because it was completing too quickly. One would think a 4MB transfer would be sufficiently large to call for async processing but that was not the case. When I inserted a sleep of 1 ms on the server side, I could see the client processing asynchronously, and the loop ended much before the data transfer.