I have a program which connects to a bunch of hosts and checks if they are "socket reflectors". Basically, it is scanning a bunch of ips and doing this:
Connect, check - if there is data, is that data the same as what I am sending? Yes, return true, no return false. No data, return false.
For some reason, asyncio is not reliably closing TCP connections after they time out. I attribute this to the fact that a lot of these hosts I am connecting to are god knows what, and maybe just buggy servers. Be that as it may, there must be a way to make this force timeout? When I run this, it hangs after a while. Out of 12,978 hosts, about 12,768 of them complete. Then I end up with a bunch of open ESTABLISHED connections! Why does this happen?
I need it close the connection if nothing happens during the given timeout period.
async def tcp_echo_client(message, host_port, loop, connection_timeout=10):
"""
Asyncio TCP echo client
:param message: data to send
:param host_port: host and port to connect to
:param loop: asyncio loop
"""
host_port_ = host_port.split(':')
try:
host = host_port_[0]
port = host_port_[1]
except IndexError:
pass
else:
fut = asyncio.open_connection(host, port, loop=loop)
try:
reader, writer = await asyncio.wait_for(fut, timeout=connection_timeout)
except asyncio.TimeoutError:
print('[t] Connection Timeout')
return 1
except Exception:
return 1
else:
if args.verbosity >= 1:
print('[~] Send: %r' % message)
writer.write(message.encode())
writer.drain()
data = await reader.read(1024)
await asyncio.sleep(1)
if data:
if args.verbosity >= 1:
print(f'[~] Host: {host} Received: %r' % data.decode())
if data.decode() == message:
honeypots.append(host_port)
writer.close()
return 0
else:
filtered_list.append(host_port)
print(f'[~] Received: {data.decode()}')
writer.close()
return 1
else:
filtered_list.append(host_port)
writer.close()
if args.verbosity > 1:
print(f'[~] No data received for {host}')
return 1
What am I doing wrong?
Related
I need to set timeout on python's socket recv method. How to do it?
The typical approach is to use select() to wait until data is available or until the timeout occurs. Only call recv() when data is actually available. To be safe, we also set the socket to non-blocking mode to guarantee that recv() will never block indefinitely. select() can also be used to wait on more than one socket at a time.
import select
mysocket.setblocking(0)
ready = select.select([mysocket], [], [], timeout_in_seconds)
if ready[0]:
data = mysocket.recv(4096)
If you have a lot of open file descriptors, poll() is a more efficient alternative to select().
Another option is to set a timeout for all operations on the socket using socket.settimeout(), but I see that you've explicitly rejected that solution in another answer.
there's socket.settimeout()
As mentioned both select.select() and socket.settimeout() will work.
Note you might need to call settimeout twice for your needs, e.g.
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("",0))
sock.listen(1)
# accept can throw socket.timeout
sock.settimeout(5.0)
conn, addr = sock.accept()
# recv can throw socket.timeout
conn.settimeout(5.0)
conn.recv(1024)
You could set timeout before receiving the response and after having received the response set it back to None:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5.0)
data = sock.recv(1024)
sock.settimeout(None)
The timeout that you are looking for is the connection socket's timeout not the primary socket's, if you implement the server side. In other words, there is another timeout for the connection socket object, which is the output of socket.accept() method. Therefore:
sock.listen(1)
connection, client_address = sock.accept()
connection.settimeout(5) # This is the one that affects recv() method.
connection.gettimeout() # This should result 5
sock.gettimeout() # This outputs None when not set previously, if I remember correctly.
If you implement the client side, it would be simple.
sock.connect(server_address)
sock.settimeout(3)
Got a bit confused from the top answers so I've wrote a small gist with examples for better understanding.
Option #1 - socket.settimeout()
Will raise an exception in case the sock.recv() waits for more than the defined timeout.
import socket
sock = socket.create_connection(('neverssl.com', 80))
timeout_seconds = 2
sock.settimeout(timeout_seconds)
sock.send(b'GET / HTTP/1.1\r\nHost: neverssl.com\r\n\r\n')
data = sock.recv(4096)
data = sock.recv(4096) # <- will raise a socket.timeout exception here
Option #2 - select.select()
Waits until data is sent until the timeout is reached. I've tweaked Daniel's answer so it will raise an exception
import select
import socket
def recv_timeout(sock, bytes_to_read, timeout_seconds):
sock.setblocking(0)
ready = select.select([sock], [], [], timeout_seconds)
if ready[0]:
return sock.recv(bytes_to_read)
raise socket.timeout()
sock = socket.create_connection(('neverssl.com', 80))
timeout_seconds = 2
sock.send(b'GET / HTTP/1.1\r\nHost: neverssl.com\r\n\r\n')
data = recv_timeout(sock, 4096, timeout_seconds)
data = recv_timeout(sock, 4096, timeout_seconds) # <- will raise a socket.timeout exception here
You can use socket.settimeout() which accepts a integer argument representing number of seconds. For example, socket.settimeout(1) will set the timeout to 1 second
try this it uses the underlying C.
timeval = struct.pack('ll', 2, 100)
s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, timeval)
As mentioned in previous replies, you can use something like: .settimeout()
For example:
import socket
s = socket.socket()
s.settimeout(1) # Sets the socket to timeout after 1 second of no activity
host, port = "somehost", 4444
s.connect((host, port))
s.send("Hello World!\r\n")
try:
rec = s.recv(100) # try to receive 100 bytes
except socket.timeout: # fail after 1 second of no activity
print("Didn't receive data! [Timeout]")
finally:
s.close()
I hope this helps!!
#! /usr/bin/python3.6
# -*- coding: utf-8 -*-
import socket
import time
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
s.settimeout(5)
PORT = 10801
s.bind(('', PORT))
print('Listening for broadcast at ', s.getsockname())
BUFFER_SIZE = 4096
while True:
try:
data, address = s.recvfrom(BUFFER_SIZE)
except socket.timeout:
print("Didn't receive data! [Timeout 5s]")
continue
Shout out to: https://boltons.readthedocs.io/en/latest/socketutils.html
It provides a buffered socket, this provides a lot of very useful functionality such as:
.recv_until() #recv until occurrence of bytes
.recv_closed() #recv until close
.peek() #peek at buffer but don't pop values
.settimeout() #configure timeout (including recv timeout)
I use Raspberry Pi Pico with ESP8266 WiFi module, and I am trying to write a TCP client. Rasp Pi Pico is able to send AT commands and receive responses and send data through UART. Also the TCP client is able to send data to the TCP server, which runs in my laptop. However the problem is that the client is not able to connect to the server after some point.
Let me first show the server-side code. In server, I am trying to receive data basically. ConnectionResetError was a problem for me so I wrote the following except block. I am not sure it is buggy or not, since I'm kind of a noob in this area.
import socket
HOST = ""
PORT = 8080
mysocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
mysocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
mysocket.bind((HOST, PORT))
mysocket.listen(1)
while True:
print("Waiting...")
conn, addr = mysocket.accept()
print('[SERVER] - Connected from: %s' % str(addr))
while True:
try:
request = conn.recv(1024)
if not request:
break
conn.sendall(request.upper())
print('[SERVER] - Received Data: %s' % str(request))
except ConnectionResetError as cr_err:
break
conn.close()
print("[SERVER] - Disconnected")
Here is my client-side code. In client, I wrote two helper classes called ESP8266 and Sensor, in which I control the WiFi module and read analog value from a sensor. Pico first tries to start WiFi module, afterwards it tries to connect to TCP server and send data. After some point it does not connect to the TCP server, so it restart the WiFi module and reconnects.
class EndDevice:
def __init__(self, sensor_id):
self.__wifi_module = ESP8266(UART_PIN, BAUDRATE)
self.__sensor = Sensor(sensor_id, SENSOR_PIN)
def start(self):
self.__wifi_module.start()
self.__wifi_module.set_mode(STATION_MODE)
self.__wifi_module.join_access_point(AP_NAME, AP_PWD)
def reconnect(self):
self.__wifi_module.restart()
self.__wifi_module.set_mode(STATION_MODE)
self.__wifi_module.join_access_point(AP_NAME, AP_PWD)
def run(self):
retry_count = 0
while True:
if self.__wifi_module.start_connection("TCP", SERVER_HOST, SERVER_PORT):
self.__wifi_module.send_data(
str(self.__sensor.generate_package()))
self.__wifi_module.close_connection()
else:
retry_count += 1
if retry_count == MAX_RETRY:
break
if __name__ == "__main__":
pico = EndDevice("SM-0")
pico.start()
while True:
pico.run()
pico.reconnect()
Finally I will share some of the methods in classes ESP8266 and UARTHandler (which is used in ESP8266), so you can see if I do anything non-sense.
start_connection method in ESP8266 is as follows. In this method, I tried to send the corresponding AT command to connect to a TCP server. In the method self.__uart_handler.send_receive_cmd timeout duration is 2000ms, and other parameters are AT command, connection_type (TCP), server IP address and server port, in order.
def start_connection(self, conn_type, remote_ip, remote_port):
conn_type, remote_ip = "\"{}\"".format(conn_type), "\"{}\"".format(remote_ip)
response = self.__uart_handler.send_receive_cmd(2000, CONN_START, conn_type, remote_ip, str(remote_port))
if "OK" in response:
self.__log("Connected to {} at port {}.".format(remote_ip, remote_port))
return True
else:
self.__log("Failed to create a connection with {} at port {}.".format(remote_ip, remote_port))
return False
send_receive_cmd method in UARTHandler is as follows. In this method I use lots of helper methods as you can see, however they are just formatting and writing to UART or reading from UART. I also insert a timeout between UART-read and UART-write
def __generate_cmd(self, cmd, *args):
if len(args) != 0:
cmd += "="
for idx, each in enumerate(args):
cmd += str(each)
if idx != len(args)-1:
cmd += ","
cmd += "\r\n"
return cmd
def __send_cmd(self, cmd, *args):
sent_cmd = self.__generate_cmd(cmd, *args)
self.__uart.write(sent_cmd)
def __receive_response(self, cmd):
response = self.__uart.read()
try: return response.decode('utf-8')
except: return response
def send_receive_cmd(self, timeout, cmd, *args):
self.__send_cmd(cmd, *args)
utime.sleep_ms(timeout)
return self.__receive_response(self.__generate_cmd(cmd, *args))
Let me ask my question again. This codes are working properly in starting-restarting and sending data for (let me say) 3 connections. However, after some connect-disconnect later, TCP client is not able to make a connection with TCP server. Again after some failed connection attempt, WiFi module is restarted and TCP connection is made and working properly again.
I'm trying to write a very simple Python 2.7 function to listen for a socket connection on a localhost port.
Question is: If something bad happens when is my code responsible for killing the socket so that is not left dangling out there? In what cases will a socket error close the socket for me so that I don't need to call 'close()'?
I want to avoid having to start my testing all over with a different port because the old one was left open.
Python code sample:
def getconn(PORT,TIMEOUT_SECS):
conn = ''
try:
HOST = ''
s1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s1.settimeout(TIMEOUT_SECS)
s1.bind((HOST, PORT))
s1.listen(1)
conn, addr = s1.accept()
print('Connected by', addr)
except socket.timeout as e:
conn = 'socket timeout'
except socket.error as e:
conn = 'other socket error'
except:
conn = 'deaddead'
if s1:
s1.close()
return conn
And used here (say PORT argument is 19111)...
def runtest01_normalpoll(PORT):
conn = getconn(PORT, 10.0)
if conn == 'socket timeout' or conn == 'other socket error' or conn == 'deaddead':
print 'could not establish a connection...why you ask?', conn
else:
conn.close()
print 'done...'
It seems to work but am I missing any edge cases that would leave the socket open or cause anything else bad to happen?
I've been working on a client for this chat server but I am running into a bit of a challenge. The server uses Python's 3.4RC1 asyncio module.
Behavior:
My client connects. My second client connects. Either can send messages to the server BUT, the server is not broadcasting them as it should in a normal public chat room.
User1: Hello. Presses Enter.
User2 does not see it.
User2: Anyone there? Presses Enter.
User2 sees User1: Hello. and User2: Anyone there?
Just... strange. Not sure what I'm missing.
Here are the files. Give it a try.
Server:
from socket import socket, SO_REUSEADDR, SOL_SOCKET
from asyncio import Task, coroutine, get_event_loop
class Peer(object):
def __init__(self, server, sock, name):
self.loop = server.loop
self.name = name
self._sock = sock
self._server = server
Task(self._peer_handler())
def send(self, data):
return self.loop.sock_send(self._sock, data.encode('utf-8'))
#coroutine
def _peer_handler(self):
try:
yield from self._peer_loop()
except IOError:
pass
finally:
self._server.remove(self)
#coroutine
def _peer_loop(self):
while True:
buf = yield from self.loop.sock_recv(self._sock, 1024)
if buf == b'':
break
self._server.broadcast('%s: %s' % (self.name, buf.decode('utf-8')))
class Server(object):
def __init__(self, loop, port):
self.loop = loop
self._serv_sock = socket()
self._serv_sock.setblocking(0)
self._serv_sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
self._serv_sock.bind(('',port))
self._serv_sock.listen(5)
self._peers = []
Task(self._server())
def remove(self, peer):
self._peers.remove(peer)
self.broadcast('Peer %s quit!' % (peer.name,))
def broadcast(self, message):
for peer in self._peers:
peer.send(message)
#coroutine
def _server(self):
while True:
peer_sock, peer_name = yield from self.loop.sock_accept(self._serv_sock)
peer_sock.setblocking(0)
peer = Peer(self, peer_sock, peer_name)
self._peers.append(peer)
self.broadcast('Peer %s connected!' % (peer.name,))
def main():
loop = get_event_loop()
Server(loop, 1234)
loop.run_forever()
if __name__ == '__main__':
main()
Client:
# import socket
from socket import *
# form socket import socket, bind, listen, recv, send
HOST = 'localhost' #localhost / 192.168.1.1
# LAN - 192.168.1.1
PORT = 1234
s = socket(AF_INET, SOCK_STREAM)# 98% of all socket programming will use AF_INET and SOCK_STREAM
s.connect((HOST, PORT))
while True:
message = input("Your Message: ")
encoded_msg = message.encode('utf-8')
s.send(encoded_msg)
print('Awaiting Reply..')
reply = s.recv(1024)
decoded_reply = reply.decode('utf-8')
decoded_reply = repr(decoded_reply)
print('Received ', decoded_reply)
s.close()
Here's the non threaded server code I wrote. works great but ONLY between 2 people. How could this code be updated to broadcast every message received to all clients connected?
# import socket
from socket import *
# form socket import socket, bind, listen, recv, send
HOST = 'localhost' #localhost / 192.168.1.1
# LAN - 192.168.1.1
PORT = 1234
s = socket(AF_INET, SOCK_STREAM) # 98% of all socket programming will use AF_INET and SOCK_STREAM
s.bind((HOST, PORT))
s.listen(5) # how many connections it can receive at one time
conn, addr = s.accept() # accept the connection
print('Connected by', addr) # print the address of the person connected
while True:
data = conn.recv(1024)
decoded_data = data.decode('utf-8')
data = repr(decoded_data)
print('Received ', decoded_data)
reply = input("Reply: ")
encoded_reply = reply.encode('utf-8')
conn.sendall(encoded_reply)
print('Server Started')
conn.close()
Okay, let’s think about what your client does. You ask for a message to send, blocking for user input. Then you send that message and receive whatever there is at the server. Afterwards, you block again, waiting for another message.
So when client A sends a text, client B is likely blocking for user input. As such, B won’t actually check if the server sent anything. It will only display what’s there after you have sent something.
Obviously, in a chat, you don’t want to block on user input. You want to continue receiving new messages from the server even if the user isn’t sending messages. So you need to separate those, and run both asynchronously.
I haven’t really done much with asyncio yet, so I don’t really know if this can be nicely done with it, but you essentially just need to put the reading and sending into two separate concurrent tasks, e.g. using threads or concurrent.futures.
A quick example of what you could do, using threading:
from socket import *
from threading import Thread
HOST = 'localhost'
PORT = 1234
s = socket(AF_INET, SOCK_STREAM)
s.connect((HOST, PORT))
def keepReading ():
try:
while True:
reply = s.recv(1024).decode()
print('Received ', reply)
except ConnectionAbortedError:
pass
t = Thread(target=keepReading)
t.start()
try:
while True:
message = input('')
s.send(message.encode())
except EOFError:
pass
finally:
s.close()
So I am working on an iPhone app that requires a socket to handle multiple clients for online gaming. I have tried Twisted, and with much effort, I have failed to get a bunch of info to be sent at once, which is why I am now going to attempt socket.
My question is, using the code below, how would you be able to have multiple clients connected? I've tried lists, but I just can't figure out the format for that. How can this be accomplished where multiple clients are connected at once and I am able to send a message to a specific client?
Thank you!
#!/usr/bin/python # This is server.py file
import socket # Import socket module
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
port = 50000 # Reserve a port for your service.
print 'Server started!'
print 'Waiting for clients...'
s.bind((host, port)) # Bind to the port
s.listen(5) # Now wait for client connection.
c, addr = s.accept() # Establish connection with client.
print 'Got connection from', addr
while True:
msg = c.recv(1024)
print addr, ' >> ', msg
msg = raw_input('SERVER >> ')
c.send(msg);
#c.close() # Close the connection
Based on your question:
My question is, using the code below, how would you be able to have multiple clients connected? I've tried lists, but I just can't figure out the format for that. How can this be accomplished where multiple clients are connected at once and I am able to send a message to a specific client?
Using the code you gave, you can do this:
#!/usr/bin/python # This is server.py file
import socket # Import socket module
import thread
def on_new_client(clientsocket,addr):
while True:
msg = clientsocket.recv(1024)
#do some checks and if msg == someWeirdSignal: break:
print addr, ' >> ', msg
msg = raw_input('SERVER >> ')
#Maybe some code to compute the last digit of PI, play game or anything else can go here and when you are done.
clientsocket.send(msg)
clientsocket.close()
s = socket.socket() # Create a socket object
host = socket.gethostname() # Get local machine name
port = 50000 # Reserve a port for your service.
print 'Server started!'
print 'Waiting for clients...'
s.bind((host, port)) # Bind to the port
s.listen(5) # Now wait for client connection.
print 'Got connection from', addr
while True:
c, addr = s.accept() # Establish connection with client.
thread.start_new_thread(on_new_client,(c,addr))
#Note it's (addr,) not (addr) because second parameter is a tuple
#Edit: (c,addr)
#that's how you pass arguments to functions when creating new threads using thread module.
s.close()
As Eli Bendersky mentioned, you can use processes instead of threads, you can also check python threading module or other async sockets framework. Note: checks are left for you to implement how you want and this is just a basic framework.
accept can continuously provide new client connections. However, note that it, and other socket calls are usually blocking. Therefore you have a few options at this point:
Open new threads to handle clients, while the main thread goes back to accepting new clients
As above but with processes, instead of threads
Use asynchronous socket frameworks like Twisted, or a plethora of others
Here is the example from the SocketServer documentation which would make an excellent starting point
import SocketServer
class MyTCPHandler(SocketServer.BaseRequestHandler):
"""
The RequestHandler class for our server.
It is instantiated once per connection to the server, and must
override the handle() method to implement communication to the
client.
"""
def handle(self):
# self.request is the TCP socket connected to the client
self.data = self.request.recv(1024).strip()
print "{} wrote:".format(self.client_address[0])
print self.data
# just send back the same data, but upper-cased
self.request.sendall(self.data.upper())
if __name__ == "__main__":
HOST, PORT = "localhost", 9999
# Create the server, binding to localhost on port 9999
server = SocketServer.TCPServer((HOST, PORT), MyTCPHandler)
# Activate the server; this will keep running until you
# interrupt the program with Ctrl-C
server.serve_forever()
Try it from a terminal like this
$ telnet localhost 9999
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
Hello
HELLOConnection closed by foreign host.
$ telnet localhost 9999
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
Sausage
SAUSAGEConnection closed by foreign host.
You'll probably need to use A Forking or Threading Mixin too
This program will open 26 sockets where you would be able to connect a lot of TCP clients to it.
#!usr/bin/python
from thread import *
import socket
import sys
def clientthread(conn):
buffer=""
while True:
data = conn.recv(8192)
buffer+=data
print buffer
#conn.sendall(reply)
conn.close()
def main():
try:
host = '192.168.1.3'
port = 6666
tot_socket = 26
list_sock = []
for i in range(tot_socket):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
s.bind((host, port+i))
s.listen(10)
list_sock.append(s)
print "[*] Server listening on %s %d" %(host, (port+i))
while 1:
for j in range(len(list_sock)):
conn, addr = list_sock[j].accept()
print '[*] Connected with ' + addr[0] + ':' + str(addr[1])
start_new_thread(clientthread ,(conn,))
s.close()
except KeyboardInterrupt as msg:
sys.exit(0)
if __name__ == "__main__":
main()
def get_clients():
first_run = True
startMainMenu = False
while True:
if first_run:
global done
done = False
Thread(target=animate, args=("Waiting For Connection",)).start()
Client, address = objSocket.accept()
global menuIsOn
if menuIsOn:
menuIsOn = False # will stop main menu
startMainMenu = True
done = True
# Get Current Directory in Client Machine
current_client_directory = Client.recv(1024).decode("utf-8", errors="ignore")
# beep on connection
beep()
print(f"{bcolors.OKBLUE}\n***** Incoming Connection *****{bcolors.OKGREEN}")
print('* Connected to: ' + address[0] + ':' + str(address[1]))
try:
get_client_info(Client, first_run)
except Exception as e:
print("Error data received is not a json!")
print(e)
now = datetime.now()
current_time = now.strftime("%D %H:%M:%S")
print("* Current Time =", current_time)
print("* Current Folder in Client: " + current_client_directory + bcolors.WARNING)
connections.append(Client)
addresses.append(address)
if first_run:
Thread(target=threaded_main_menu, daemon=True).start()
first_run = False
else:
print(f"{bcolors.OKBLUE}* Hit Enter To Continue.{bcolors.WARNING}\n#>", end="")
if startMainMenu == True:
Thread(target=threaded_main_menu, daemon=True).start()
startMainMenu = False
#!/usr/bin/python
import sys
import os
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
port = 50000
try:
s.bind((socket.gethostname() , port))
except socket.error as msg:
print(str(msg))
s.listen(10)
conn, addr = s.accept()
print 'Got connection from'+addr[0]+':'+str(addr[1]))
while 1:
msg = s.recv(1024)
print +addr[0]+, ' >> ', msg
msg = raw_input('SERVER >>'),host
s.send(msg)
s.close()