this is the code I am currently using. But I would like it to be able to send serialized data using pickle. I have been tinkering with it for a few hours now with no luck. Perhaps if someone here has better networking experience, quick insight would be appreciated so I don't have to rewrite everything, I think it's confusing as it is already.
import select
def send(sdef, data, slen):
sdef.setblocking(0)
sdef.sendall(str(len(str(data))).encode("utf-8").zfill(slen))
sdef.sendall(str(data).encode("utf-8"))
def receive(sdef, slen):
sdef.setblocking(0)
ready = select.select([sdef], [], [], 60)
if ready[0]:
data = int(sdef.recv(slen)) # receive length
# print "To receive: "+str(data)
else:
raise RuntimeError("Socket timeout")
chunks = []
bytes_recd = 0
while bytes_recd < data:
ready = select.select([sdef], [], [], 60)
if ready[0]:
chunk = sdef.recv(min(data - bytes_recd, 2048))
if chunk == b'':
raise RuntimeError("Socket connection broken")
chunks.append(chunk)
bytes_recd = bytes_recd + len(chunk)
else:
raise RuntimeError("Socket timeout")
segments = b''.join(chunks).decode("utf-8")
# print "Received segments: "+str(segments)
return segments
Simplest implementation:
import select, pickle
def send(sdef, data, slen):
sdef.setblocking(0)
sdef.sendall(str(len(str(pickle.dumps(data)))).encode("utf-8").zfill(slen))
sdef.sendall(str(pickle.dumps(data)).encode("utf-8"))
def receive(sdef, slen):
sdef.setblocking(0)
ready = select.select([sdef], [], [], 60)
if ready[0]:
data = int(sdef.recv(slen)) # receive length
# print "To receive: "+str(data)
else:
raise RuntimeError("Socket timeout")
chunks = []
bytes_recd = 0
while bytes_recd < data:
ready = select.select([sdef], [], [], 60)
if ready[0]:
chunk = sdef.recv(min(data - bytes_recd, 2048))
if chunk == b'':
raise RuntimeError("Socket connection broken")
chunks.append(chunk)
bytes_recd = bytes_recd + len(chunk)
else:
raise RuntimeError("Socket timeout")
segments = b''.join(chunks).decode("utf-8")
# print "Received segments: "+str(segments)
return segments
and I cannot deserialize "segments" anymore, because it is a string
This is the way to get it, literaleval should work under all circumstances on serialized data
pickle.loads(ast.literal_eval(segments))
Related
What are the intentions of this program:
I want to send some commands from a client to a server using sockets, the server then send these command to an Arduino using serial. And another thing that I want the server to do in the future is that periodically sends other commands to the Arduino without getting any input from the client, so the sockets needs to be non-blocking or there needs to be another way to run the code separately from the sockets code.
The problem is that the part that is supposed to send the command to the Arduino only runs once.
What I have come up with after playing with the debugger in Pycharm, is that the problem is that the following line blocks after a connection has been established, and thus not allowing the rest of the code to be run.
conn, addr = s.accept()
Is this correct, or is there something else wrong?
I have tried to set the socket to non-blocking but when I do this I get an error.
"BlockingIOError: [WinError 10035] A non-blocking socket operation could not be completed immediately"
I have some basic knowledge of C/C++ and C# and am new to Python.
server.py
import socket
import serial
import sys
from _thread import *
import threading
import queue
# command that the client sends are "ON" and "OFF"
class serialConnect:
comPort =' '
baudrate = 115200
myserial = serial.Serial('COM5', baudrate)
def serialstart(self):
# self.comPort = input('Comport: ')
try:
self.myserial.open()
except IOError:
print('Port is already open!')
def serialRead(self):
data = self.myserial.read(16)
data.decode('UTF-8')
return data
def serialWrite(self, data):
data += '\n' #the arduino needs a \n after each command.
databytes = data.encode('UTF-8')
self.myserial.write(databytes)
print('send data: ', databytes)
def threaded_client(conn, dataqueue):
data = {bytes}
conn.send(str.encode('welcome, type your info \n'))
while True:
data = conn.recv(2048)
if not data:
break
reply = 'server output: ' + data.decode('UTF-8')
dataqueue.put(data.decode('UTF-8'))
print("Items in queue: ",dataqueue.qsize())
#conn.sendall(str.encode(reply))
print("Recieved data in threaded_client: ", data.decode('UTF-8') + '\n')
conn.close()
def Main():
ser = serialConnect()
host = ''
port = 5555
dataRecieved = 'hello'
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2)
s.setblocking(1) #when set to non-blocking error occurs : "BlockingIOError: [WinError 10035] A non-blocking socket operation could not be completed immediately"
workQueue = queue.Queue(10)
try:
s.bind((host,port))
except socket.error as e:
print(str(e))
s.listen(5)
print('waiting for a connection')
while True:
try:
conn, addr = s.accept() #once connection is established it blocks?
print('connected to: ' + addr[0] + ':' + str())
t = threading.Thread(target=threaded_client, args=(conn, workQueue))
t.daemon = True
t.start()
except:
e = sys.exc_info()
print('Error:', e)
# This section of code is only run once, doesn't matter if put inside try block or not. :(
dataRecieved = workQueue.get()
print('The recieved data: ', dataRecieved)
ser.serialstart()
ser.serialWrite(dataRecieved)
if __name__ == '__main__':
Main()
client.py
import socket
def Main():
host = '127.0.0.1'
port = 5555
message = "<,R,G,B,>"
mySocket = socket.socket()
mySocket.connect((host, port))
while message != 'q':
message = input(" -> ")
mySocket.send(message.encode())
mySocket.close()
if __name__ == '__main__':
Main()
Arduino Code
String inputString = ""; // a string to hold incoming data
boolean stringComplete = false; // whether the string is complete
int LEDpin = 10;
// the setup function runs once when you press reset or power the board
void setup() {
// initialize digital pin 13 as an output.
pinMode(10, OUTPUT);
Serial.begin(19200);
}
// the loop function runs over and over again forever
void loop() {
serialEvent();
if(stringComplete){
Serial.println(inputString);
if(inputString == "ON\n"){
digitalWrite(LEDpin, HIGH); // turn the LED on (HIGH is the voltage level)
}
if(inputString == "OFF\n"){
digitalWrite(LEDpin, LOW); // turn the LED off by making the voltage LOW
}
inputString = "";
stringComplete = false;
}
}
void serialEvent()
{
while (Serial.available()) {
// get the new byte:
char inChar = (char)Serial.read();
// add it to the inputString:
inputString += inChar;
// if the incoming character is a newline, set a flag
// so the main loop can do something about it:
if (inChar == '\n') {
stringComplete = true;
}
}
}
Refactored server code for anyone that is interested in it.
I am not sure if this is up to standard, but it is working.
import serial
import socket
import queue
import sys
import threading
class serialConnect:
comPort = 'COM5'
baudrate = 115200
myserial = serial.Serial(comPort, baudrate)
def serial_run(self):
# self.comPort = input('Comport: ')
try:
if not self.myserial.isOpen():
self.myserial.open()
else:
print('Port is already open!')
except IOError as e:
print('Error: ', e)
def serial_read(self):
data = self.myserial.read(16)
data.decode('UTF-8')
return data
def serial_write(self, data):
data += '\n' #the arduino needs a \n after each command.
databytes = data.encode('UTF-8')
self.myserial.write(databytes)
print('send data: ', databytes)
class socketServer:
host = ''
port = 5555
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
soc.setblocking(1)
data_queue = queue.Queue(1)
def __init__(self):
try:
self.soc.bind((self.host, self.port))
except:
print('Bind error: ', sys.exc_info())
self.soc.listen(5)
def socket_accept_thread(self):
while True:
try:
print('Waiting for a new connection')
conn, addr = self.soc.accept()
client_thread = threading.Thread(target=self.threaded_client, args=(conn, self.data_queue))
client_thread.daemon = True
client_thread.start()
except:
print('Accept thread Error: ', sys.exc_info())
def threaded_client(self, conn, data_queue):
# conn.send(str.encode('welcome, type your info \n'))
try:
while True:
data = conn.recv(2048)
if not data:
break
# reply = 'server output: ' + data.decode('UTF-8')
data_queue.put(data.decode('UTF-8'))
print("Items in queue: ", data_queue.qsize())
# conn.sendall(str.encode(reply))
print("Received data in threaded_client: ", data.decode('UTF-8'))
except:
print("Error: ", sys.exc_info())
conn.close()
def get_data(self):
data = self.data_queue.get()
return data
def Main():
server = socketServer()
arduino_conn = serialConnect()
accept_thread = threading.Thread(target=server.socket_accept_thread)
data_received = 'Nothing received'
while True:
if not accept_thread.is_alive():
accept_thread.daemon = True
accept_thread.start()
arduino_conn.serial_run()
data_received = server.get_data()
arduino_conn.serial_write(data_received)
if __name__ == '__main__':
Main()
I'm programming a basic chat program. The goal of this is first to set up the server by launching the server.py script then leave that running in the background. Then a user launches the client.py script and chooses a name then starts to type a message and send it. The problem is when I attempt to send it it returns this error.
Traceback (most recent call last):
File "C:\Users\Hello\AppData\Local\Programs\Python\Python35-32\client.py", line 38, in <module>
s.sendto(alias + ': ' + message.encode() , server)
TypeError: Can't convert 'bytes' object to str implicitly
Here's the server.py script ---
import socket
import time
host = '127.0.0.1'
port = 47015
clients = []
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind((host, port))
s.setblocking(0)
qutting = False
print("Server Started")
while not qutting:
try:
data, addr = s.recvfrom(1024)
if 'Quit' in str(data):
qutting = True
if addr not in clients:
clients.append(addr)
print(time.ctime(time.time()) + str(addr) + ": :" + str(data))
for client in clients:
s.sendto(data, client)
except:
pass
s.close()
then the client.py script----
import socket
import threading
import time
tLock = threading.Lock()
shutdown = False
def recieving(name, sock):
while not shutdown:
try:
tLock.acquire()
while True:
data, addr = sock.recvfrom(1024)
data.decode()
print(str(data))
except:
pass
finally:
tLock.release()
host = '127.0.0.1'
port = 0
server = ("127.0.0.1", 47015)
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind((host, port))
s.setblocking(0)
rT = threading.Thread(target=recieving, args=('RecvThread', s))
rT.start()
alias = input("Name: ")
message = input(alias + "->")
while message != 'q':
if message != '':
s.sendto(alias + ': ' + message.encode() , server)
tLock.acquire()
message = input(alias + '->')
tLock.release()
time.sleep(0.2)
shutdown = True
rT.join()
s.close()
So if you have any idea how to fix this it would be greatly appreciated also a side question is would there be a way to make it so that the receiving thread runs all the time so it actively updates the chat?
The problem is in this line:
s.sendto(alias + ': ' + message.encode() , server)
You are converting message to a bytes, but then you are adding unconverted strings to the bytes object. This is an invalid operation, as the error message is telling you, because the bytes can not be unambiguously converted to a string. Try to encode the whole thing:
s.sendto((alias + ': ' + message).encode() , server)
I have this piece of code for server to handle clients. it properly receive data but when i want to send received data to clients nothing happens.
server
import socket
from _thread import *
class GameServer:
def __init__(self):
# Game parameters
board = [None] * 9
turn = 1
# TCP parameters specifying
self.tcp_ip = socket.gethostname()
self.tcp_port = 9999
self.buffer_size = 2048
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.s.bind((self.tcp_ip, self.tcp_port))
except:
print("socket error, Please try again! ")
self.s.listen(5)
print('Waiting for a connection...')
def messaging(self, conn):
while True:
data = conn.recv(self.buffer_size)
if not data:
break
print("This data from client:", data)
conn.send(data)
def thread_run(self):
while True:
conn, addr = self.s.accept()
print('connected to: ' + addr[0] + " : " + str(addr[1]))
start_new_thread(self.messaging, (conn,))
def main():
gameserver = GameServer()
gameserver.thread_run()
if __name__ == '__main__':
main()
'
I want to if data received completely send to clients by retrieve the address of sender and send it to other clients by means of conn.send() but seems there is no way to do this with 'send()' method.
The piece of client side code
'
def receive_parser(self):
global turn
rcv_data = self.s.recv(4096)
rcv_data.decode()
if rcv_data[:2] == 'c2':
message = rcv_data[2:]
if message[:3] == 'trn':
temp = message[3]
if temp == 2:
turn = -1
elif temp ==1:
turn = 1
elif message[:3] == 'num':
self.set_text(message[3])
elif message[:3] == 'txt':
self.plainTextEdit_4.appendPlainText('client1: ' + message[3:])
else:
print(rcv_data)
'
the receiver method does not receive any data.
I modified your code a little(as I have python 2.7) and conn.send() seems to work fine. You can also try conn.sendall(). Here is the code I ran:
Server code:
import socket
from thread import *
class GameServer:
def __init__(self):
# Game parameters
board = [None] * 9
turn = 1
# TCP parameters specifying
self.tcp_ip = "127.0.0.1"#socket.gethostname()
self.tcp_port = 9999
self.buffer_size = 2048
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.s.bind((self.tcp_ip, self.tcp_port))
except:
print("socket error, Please try again! ")
self.s.listen(5)
print('Waiting for a connection...')
def messaging(self, conn):
while True:
data = conn.recv(self.buffer_size)
if not data:
break
print("This data from client:", data)
conn.send(data)
def thread_run(self):
while True:
conn, addr = self.s.accept()
print('connected to: ' + addr[0] + " : " + str(addr[1]))
start_new_thread(self.messaging, (conn,))
def main():
gameserver = GameServer()
gameserver.thread_run()
main()
Client code:
import socket
s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("127.0.0.1", 9999))
def receive_parser():
#global turn
s.sendall("hello world")
rcv_data = s.recv(4096)
# rcv_data.decode()
# if rcv_data[:2] == 'c2':
# message = rcv_data[2:]
# if message[:3] == 'trn':
# temp = message[3]
# if temp == 2:
# turn = -1
# elif temp ==1:
# turn = 1
# elif message[:3] == 'num':
# self.set_text(message[3])
# elif message[:3] == 'txt':
# self.plainTextEdit_4.appendPlainText('client1: ' + message[3:])
# else:
print(rcv_data)
receive_parser()
i am losing messages in my tornado chat and i do not known how to detect when the message wasn't sent and to send the message again
there is any way to detect when the conexion get lost? and when the conexión restart send the message
this is my code
def get(self):
try:
json.dumps(MessageMixin.cache)
except KeyError:
raise tornado.web.HTTPError(404)
class MessageMixin(object):
waiters = {}
cache = {}
cache_size = 200
def wait_for_messages(self,cursor=None):
t = self.section_slug
waiters = self.waiters.setdefault(t, [])
result_future = Future()
waiters.append(result_future)
return result_future
def cancel_wait(self, future):
t = self.section_slug
waiters = self.waiters.setdefault(t, [])
waiters.remove(future)
# Set an empty result to unblock any coroutines waiting.
future.set_result([])
def new_messages(self, message):
t = self.section_slug
#cache = self.cache.setdefault(t, [])
#print t
#print self.waiters.setdefault(t, [])
waiters = self.waiters.setdefault(t, [])
for future in waiters:
try:
if message is not None:
future.set_result(message)
except Exception:
logging.error("Error in waiter callback", exc_info=True)
waiters = []
#self.cache.extend(message)
#if len(self.cache) > self.cache_size:
#self.cache = self.cache[-self.cache_size:]
class MessageNewHandler(MainHandler, MessageMixin):
def post(self, section_slug):
self.section_slug = section_slug
post = self.get_argument("html")
idThread = self.get_argument("idThread")
isOpPost = self.get_argument("isOpPost")
arg_not = self.get_argument("arg")
type_not = self.get_argument("type")
redirect_to = self.get_argument("next", None)
message= {"posts": [post],"idThread": idThread,"isOpPost": isOpPost,
"type": type_not,"arg_not": arg_not}
if redirect_to:
self.redirect(redirect_to)
else:
self.write(post)
self.new_messages(message)
class MessageUpdatesHandler(MainHandler, MessageMixin):
#gen.coroutine
def post(self, section_slug):
self.section_slug = section_slug
try:
self.future = self.wait_for_messages(cursor=self.get_argument("cursor", None))
data = yield self.future
if self.request.connection.stream.closed():
return
self.write(data)
except Exception:
raise tornado.web.HTTPError(404)
def on_connection_close(self):
self.cancel_wait(self.future)
class Application(tornado.web.Application):
def __init__(self):
handlers = [
(r"/api/1\.0/stream/(\w+)", MessageUpdatesHandler),
(r"/api/1\.0/streamp/(\w+)", MessageNewHandler)
]
tornado.web.Application.__init__(self, handlers)
def main():
tornado.options.parse_command_line()
app = Application()
port = int(os.environ.get("PORT", 5000))
app.listen(port)
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main()
In the original chatdemo, this is what the cursor parameter to wait_for_messages is for: the browser tells you the last message it got, so you can send it every message since then. You need to buffer messages and potentially re-send them in wait_for_messages. The code you've quoted here will only send messages to those clients that are connected at the time the message came in (and remember that in long-polling, sending a message puts the client out of the "waiting" state for the duration of the network round-trip, so even when things are working normally clients will constantly enter and leave the waiting state)
I have the following script, which I found online, in Python. What it does is tries to connect to a MineCraft server, first by sending a 'handshake', then sending a login request. The protocol specs can be found here: http://wiki.vg/Protocol
Anyway, the python script works fine. However, I think the second packet is encoded wrong, as when it is sent, nothing appears on the server console. The player isn't connected or anything. It just eventually times out and closes the connection due to the 'client' not logging in in time.
Basically, anyway who has experience with struct.pack() should be able to help me here. I have commented the line where I am unsure of whether I have encoded everything right. The detailed information on packing the data is shown in the link above.
Any help would be greatly appreciated, I'm pretty clueless with encoding/packing data. :(
Here's the code
import struct
import socket
import time
import urllib
import urllib2
host = str(raw_input('What is the host ip: '))
port = int(raw_input('What is the server port: '))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
usrnm = str(raw_input('What is your username: '))
psswrd = str(raw_input('What is your password: '))
logindata = {'user':usrnm, 'password':psswrd, 'version':'12'}
data = urllib.urlencode(logindata)
print('Sending data to login.minecraft.net...')
req = urllib2.Request('https://login.minecraft.net', data)
response = urllib2.urlopen(req)
returndata = response.read()
returndata = returndata.split(":")
mcsessionid = returndata[3]
del req
del returndata
print("Session ID: " + mcsessionid)
data = {'user':usrnm,'host':host,'port':port}
enc_user = data['user'].encode('utf-16BE')
packfmt = '>bih{}shiibBB'.format(len(enc_user))
packetbytes = struct.pack(packfmt, 1, 23, len(data['user']), enc_user, 0, 0, 0, 0, 0, 0)
stringfmt = u'%(user)s;%(host)s:%(port)d'
string = stringfmt % data
structfmt = '>bh'
packetbytes = struct.pack(structfmt, 2, len(string))+string.encode('utf-16BE')
s.send(packetbytes)
connhash = s.recv(1024)
print("Connection Hash: " + connhash)
print('Sending data to http://session.minecraft.net/game/joinserver.jsp?user=' + usrnm + '&sessionId=' + mcsessionid + '&serverId=' + connhash + '...')
req = urllib.urlopen('http://session.minecraft.net/game/joinserver.jsp?user=' + usrnm + '&sessionId=' + mcsessionid + '&serverId=' + connhash)
returndata = req.read()
if(returndata == 'OK'):
print('session.minecraft.net says everything is okay, proceeding to send data to server.')
else:
print('Oops, something went wrong.')
time.sleep(5)
# All above here works perfectly.
enc_user = data['user'].encode('utf-16BE')
packfmt = '>bih{}shiibBB'.format(len(enc_user))
packetbytes = struct.pack(packfmt, 1, 23, len(data['user']), enc_user, 0, 0, 0, 0, 0, 0)
#This line is probably where something's going wrong:
packetbytes = struct.pack('>bih', 1, 23, len(data['user'])) + data['user'].encode('utf-16BE') + struct.pack('>hiibBB', 2,0,0,0,0,0)
print(len(packetbytes))
print('Sending ' + packetbytes + ' to server.')
s.send(packetbytes)
while True:
data = s.recv(1024)
if data:
print(data)
Yeah, you're sending the length of the string, which is the number of characters. Instead, you should be sending the number of bytes in the encoded string. Also, you should use "!" instead of ">" for clarity's sake, as "!" is used to indicate "network order", which this is. So this...
structfmt = '>bh'
packetbytes = struct.pack(structfmt, 2, len(string))+string.encode('utf-16BE')
... gets changed to this...
structfmt = '!bh'
encoded = string.encode('utf-16BE')
packetbytes = struct.pack(structfmt, 2, len(encoded))+encoded