Use JSON for serialising requests/responses, rather than pickle, to make C integration easier

bmspy/client.py

@@ -2,7 +2,7 @@
 # Library with socket client for use by consumers
 #
 import atexit, os, sys
-import pickle, struct
+import struct, json
 import socket
 
 
@@ -55,26 +55,35 @@ def socket_comms(socket_path, request_data, debug=0):
     # Send request
     if debug > 2:
         print('socket client: sending {!r}'.format(request_data))
-    request = pickle.dumps(request_data)
+    request = bytes()
-    # add length to the start of the pickled bytes, so we know how much to read on the other end
+    try:
-    length = struct.pack('!I', len(request))
+        request = json.dumps(request_data).encode()
-    if debug > 3:
+        # add length to the start of the json string, so we know how much to read on the other end
-        print("socket client: outgoing request length: {}, encoded as {}".format(len(request), length))
+        length = struct.pack('!I', len(request))
-    request = length + request
+        if debug > 3:
-    if debug > 4:
+            print("socket client: outgoing request length: {}, encoded as {}".format(len(request), length))
-        print("socket client: outgoing request: {}".format(request))
+        request = length + request
+        if debug > 4:
+            print("socket client: outgoing request: {}".format(request))
+    except:
+        print("socket client ERROR: unable to encode request")
+        sys.exit(1)
     sock.sendall(request)
   
-    # get length of expected pickled bytes
+    # get length of expected json string
     response = sock.recv(struct.calcsize('!I'))
-    length = struct.unpack('!I', response)[0]
+    try:
-    if debug > 4:
+        length = struct.unpack('!I', response)[0]
-        print("socket client: incoming length: {}, encoded as {}".format(length, response))
+        if debug > 4:
-    # read length bytes
+            print("socket client: incoming length: {}, encoded as {}".format(length, response))
-    response = sock.recv(length)
+        # read length bytes
-    if debug > 3:
+        response = sock.recv(length)
-        print("socket client: incoming response: {}".format(response))
+        if debug > 3:
-    response_data = pickle.loads(response)
+            print("socket client: incoming response: {}".format(response))
+        response_data = json.loads(response)
+    except:
+        print("socket client ERROR: unable to decode response")
+        sys.exit(1)
     if debug > 2:
         print('socket client: received {!r}'.format(response_data))
 

bmspy/influxdb.py

@@ -65,7 +65,7 @@ def influxdb_create_snapshot(data, debug=0):
 
     for kind, contains in data.items():
         # discard bmspy metadata
-        if kind == 'client':
+        if kind == 'client' or kind == 'timestamp':
             break
 
         helpmsg = ''

bmspy/server.py

@@ -8,7 +8,7 @@ import os, sys, stat, time
 import json
 import atexit, signal
 import serial, serial.rs485
-import pickle, struct
+import struct, json
 import pprint
 
 connected_clients = list()
@@ -527,17 +527,30 @@ def collect_data(ser, debug=0):
 
 
 def read_request(connection, debug=0):
-    # get length of expected pickle bytes
+    # get length of expected json string
-    request = connection.recv(struct.calcsize('!I'))
+    request = bytes()
-    length = struct.unpack('!I', request)[0]
+    try:
+        request = connection.recv(struct.calcsize('!I'))
+    except Exception as e:
+        raise OSError("unable to read request length from socket: {}".format(e))
+    try:
+       length = struct.unpack('!I', request)[0]
+    except Exception as e:
+        raise Exception("unable to determine request length: {}".format(e))
     if debug > 4:
         print("socket: incoming length: {}, encoded as {}".format(length, request))
 
     # read length bytes
-    request = connection.recv(length)
+    try:
+        request = connection.recv(length)
+    except Exception as e:
+        raise OSError("unable to read socket: {}".format(e))
     if debug > 3:
         print("socket: incoming request: {}".format(request))
-    request_data = pickle.loads(request)
+    try:
+        request_data = json.loads(request)
+    except Exception as e:
+        raise Exception("unable to read incoming request: {}".format(e))
     if debug > 2:
         print('socket: received {!r}'.format(request_data))
 
@@ -552,15 +565,18 @@ def send_response(connection, response_data, debug=0):
 
     if debug > 2:
         print('socket: sending {!r}'.format(response_data))
-    response = pickle.dumps(response_data)
+    try:
-    # add length to the start of the pickled bytes, so we know how much to read on the other end
+        response = json.dumps(response_data).encode()
-    length = struct.pack('!I', len(response))
+        # add length to the start of the json string, so we know how much to read on the other end
-    if debug > 4:
+        length = struct.pack('!I', len(response))
-        print('socket: sending {} data of length: {}'.format(client, length))
+        if debug > 4:
-    response = length + response
+            print('socket: sending {} data of length: {}'.format(client, length))
-    if debug > 3:
+        response = length + response
-        print("socket: outgoing response: {}".format(response))
+        if debug > 3:
-    return connection.sendall(response)
+            print("socket: outgoing response: {}".format(response))
+        return connection.sendall(response)
+    except Exception as e:
+        raise OSError("unable to encode response: {}".format(e))
 
 
 def main():
@@ -664,7 +680,12 @@ def main():
             connection, client_address = sock.accept()
 
             request_data = dict()
-            request_data = read_request(connection, debug)
+            try:
+                request_data = read_request(connection, debug)
+            except Exception as e:
+                print("socket ERROR: {}".format(e))
+                continue
+
             client = request_data['client'] or 'unknown'
 
             match request_data['command']:
@@ -682,6 +703,9 @@ def main():
                     send_response(connection, {'status': 'DEREGISTERED', 'client': client}, debug)
 
                 case 'GET':
+                    timestamp = 0
+                    if bool(current_data) is True:
+                        timestamp = current_data.get('timestamp', 0)
                     print("reading data, current timestamp is {}, time is {}".format(timestamp, time.time()))
                     # only get new data five seconds after the last read
                     if timestamp <= time.time() - 5:
@@ -690,7 +714,7 @@ def main():
                         while bool(current_data) is False:
                             current_data = collect_data(ser, debug)
                             time.sleep(1)
-                        timestamp = time.time()
+                        current_data['timestamp'] = time.time()
                         current_data['client'] = client
   
                     send_response(connection, current_data, debug)

pyproject.toml

@@ -25,4 +25,4 @@ bmspy = "bmspy:main"
 bmspy-server = "bmspy.server:main"
 bmspy-influxdb = "bmspy.influxdb:main"
 bmspy-prometheus = "bmspy.prometheus:main"
-#bmspy-usbd = "bmspy.usbhid:main"
+bmspy-usbd = "bmspy.usbhid:main"