From b7a174513ded02759d0cfbd627f602bf724c24b2 Mon Sep 17 00:00:00 2001 From: Mingcan Li Date: Sun, 5 May 2024 23:45:35 -0500 Subject: [PATCH 1/3] add frontend visual serial port selector and remove unnecessary prints --- Backend/.gitignore | 5 +- Backend/core/comms.py | 63 +++++++++++++-- Backend/core/core_api.py | 26 +++++- Backend/file_sync | 2 +- Backend/main.py | 2 +- Backend/setup.py | 2 +- .../Components/Communication/Communication.js | 11 +-- .../src/Components/Dashboard/Dashboard.js | 13 +++ .../SerialSelector/SerialSelector.js | 81 +++++++++++++++++++ 9 files changed, 190 insertions(+), 15 deletions(-) create mode 100644 Frontend/src/Components/SerialSelector/SerialSelector.js diff --git a/Backend/.gitignore b/Backend/.gitignore index 0d36319c..def57c14 100644 --- a/Backend/.gitignore +++ b/Backend/.gitignore @@ -7,4 +7,7 @@ recordedData/sessions/*.bin recordedData/processedData/*.csv # Python -*.egg-info \ No newline at end of file +*.egg-info + +# File sync +Downloaded*/ \ No newline at end of file diff --git a/Backend/core/comms.py b/Backend/core/comms.py index 80c3e3ee..ae318c16 100644 --- a/Backend/core/comms.py +++ b/Backend/core/comms.py @@ -4,6 +4,7 @@ import aiohttp import asyncio import config +import serial import signal import sys @@ -17,10 +18,11 @@ byte_length = 0 properties = [] frontend_data = {} -solar_car_connection = {'lte': False, 'udp': False} +solar_car_connection = {'lte': False, 'udp': False, 'serial': False} # Convert dataformat to format string for struct conversion # Docs: https://docs.python.org/3/library/struct.html types = {'bool': '?', 'float': 'f', 'char': 'c', 'uint8': 'B', 'uint16': 'H', 'uint64': 'Q'} +serial_port = {"device": "", 'baud': 115200} # shared object with core_api for setting serial device from frontend def set_format(file_path: str): global format_string, byte_length, properties @@ -45,7 +47,7 @@ def unpack_data(data): class Telemetry: - __tmp_data = {'tcp': b'', 'lte': b'', 'udp': b'', 'file_sync': b''} + __tmp_data = {'tcp': b'', 'lte': b'', 'udp': b'', 'file_sync': b'', 'serial': b''} latest_tstamp = 0 def listen_udp(self, port: int): @@ -140,6 +142,56 @@ def listen_tcp(self, server_addr: str, port: int): solar_car_connection['tcp'] = False break + def serial_read(self): + global frontend_data, serial_port + latest_tstamp = 0 + while True: + curr_device = serial_port['device'] + curr_baud = serial_port['baud'] + print('serial', curr_device) + if(curr_device): + # Establish a serial connection) + ser = serial.Serial(curr_device, curr_baud) + # if device has been updated then exit loop and connect to new device + while curr_device == serial_port['device']: + print('read serial') + # Read data from serial port + try: + data = b'' + if(ser.in_waiting > 0): + data = ser.read(ser.in_waiting) + else: + time.sleep(0.1) + if not data: + # No data received, continue listening + continue + print('read data') + print('data:', data) + packets = self.parse_packets(data, 'serial') + for packet in packets: + if len(packet) == byte_length: + d = unpack_data(packet) + latest_tstamp = time.time() + try: + frontend_data = d.copy() + db.insert_data(d) + except Exception as e: + print(traceback.format_exc()) + continue + solar_car_connection['serial'] = True + if time.time() - latest_tstamp / 1000 > 5: + solar_car_connection['lte'] = False + break + except Exception: + print(traceback.format_exc()) + solar_car_connection['serial'] = False + serial_port['device'] = "" + break + else: + solar_car_connection['serial'] = False + # wait before retry + time.sleep(1) + async def fetch(self, session, url): try: async with session.get(url, timeout=2) as response: @@ -255,11 +307,12 @@ def sigint_handler(signal, frame): def start_comms(): # start file sync p.start() - - + # Start two live comm channels vps_thread = threading.Thread(target=lambda : asyncio.run(telemetry.remote_db_fetch(config.VPS_URL))) vps_thread.start() - socket_thread = threading.Thread(target=lambda: telemetry.listen_udp(config.UDP_PORT)) + #socket_thread = threading.Thread(target=lambda: telemetry.listen_udp(config.UDP_PORT)) + #socket_thread.start() + socket_thread = threading.Thread(target=lambda: telemetry.serial_read()) socket_thread.start() diff --git a/Backend/core/core_api.py b/Backend/core/core_api.py index ce90abee..a397ba5d 100644 --- a/Backend/core/core_api.py +++ b/Backend/core/core_api.py @@ -1,14 +1,18 @@ from fastapi import APIRouter +import serial.tools.list_ports from . import comms +from pydantic import BaseModel + router = APIRouter() @router.get("/single-values") async def single_values(): - if comms.solar_car_connection['udp'] or comms.solar_car_connection['lte']: + if comms.solar_car_connection['udp'] or comms.solar_car_connection['lte'] or comms.solar_car_connection['serial']: latest_data = comms.frontend_data latest_data['solar_car_connection'] = True latest_data['udp_status'] = comms.solar_car_connection['udp'] latest_data['lte_status'] = comms.solar_car_connection['lte'] + latest_data['serial_status'] = comms.solar_car_connection['serial'] latest_data['timestamps'] = f'{latest_data["tstamp_hr"]:02d}:{latest_data["tstamp_mn"]:02d}:' \ f'{latest_data["tstamp_sc"]:02d}.{latest_data["tstamp_ms"]}' format_data = {} @@ -17,3 +21,23 @@ async def single_values(): json_data = {'response': format_data} return json_data return {'response': None} + + +@router.get("/serial-info") +async def list_serial_ports(): + """return currently connected device and all available serial device""" + ports = serial.tools.list_ports.comports() + # Extract the device name from each port object + return {'connected_device': {'device': comms.serial_port['device'], 'baud': comms.serial_port['baud']}, + 'all_devices': [port.device for port in sorted(ports, key=lambda port: port.device)] + } + +class SerialDevice(BaseModel): + device: str + baud: int + +@router.post("/connect-device") +async def dev_conn(serial_device: SerialDevice): + """Connect to serial port, pass in empty device name for disconnect""" + comms.serial_port['device'] = serial_device.device + comms.serial_port['baud'] = serial_device.baud \ No newline at end of file diff --git a/Backend/file_sync b/Backend/file_sync index 14768c06..924590d6 160000 --- a/Backend/file_sync +++ b/Backend/file_sync @@ -1 +1 @@ -Subproject commit 14768c06144f148f230b97dbd3e6b03cb45514df +Subproject commit 924590d67594cbcf00f7d6bcd04c21fe5567d5ae diff --git a/Backend/main.py b/Backend/main.py index a8455534..497f1469 100644 --- a/Backend/main.py +++ b/Backend/main.py @@ -11,6 +11,6 @@ async def startup(): process.start_processes() if __name__ == '__main__': - uvicorn.run(app='main:app', host="0.0.0.0", port=config.HOST_PORT) + uvicorn.run(app='main:app', host="0.0.0.0", port=config.HOST_PORT, log_level='critical') \ No newline at end of file diff --git a/Backend/setup.py b/Backend/setup.py index 2040ce1d..9b553a0e 100644 --- a/Backend/setup.py +++ b/Backend/setup.py @@ -9,5 +9,5 @@ author='Badger Solar Racing Software Team', author_email='', description='', - install_requires=['uvicorn','fastapi','redis', 'requests', 'numpy', 'XlsxWriter', 'pandas', 'aiohttp'] + install_requires=['uvicorn','fastapi','redis', 'requests', 'numpy', 'XlsxWriter', 'pandas', 'aiohttp', 'pyserial'] ) diff --git a/Frontend/src/Components/Communication/Communication.js b/Frontend/src/Components/Communication/Communication.js index 1a7193b0..8ba62744 100644 --- a/Frontend/src/Components/Communication/Communication.js +++ b/Frontend/src/Components/Communication/Communication.js @@ -38,12 +38,8 @@ export default function Communication(props) { - -  Packet Delay: +  Packet Delay: {_getFormattedPacketDelay()} + {switchDataView(dataView4)} + + + { + fetch('/serial-info') + .then((response) => { + if (response.ok) { + return response.json(); + } else { + throw new Error(`Error fetching serial port with code ${response.status}`); + } + }) + .then((body) => { + setSelectedDevice(body['connected_device']['device']); // Set default device + setSelectedBaud(body['connected_device']['baud']); + setAllDevices(body['all_devices']); + }).catch(error => console.error('Fetch error:', error)); + }; + + useInterval(refresh, 3000); + + useEffect(()=> { + fetch("/connect-device", { + method: "POST", + body: JSON.stringify({ + device: selectedDevice, + baud: selectedBaud + }), + headers: { + "Content-type": "application/json" + } + }); + + }, [selectedBaud, selectedDevice]) + + const getSerialPort = () => { + return ( + + ); + }; + + const getBaud = () => { + const defaultBaud = [4800, 9600, 14400, 19200, 38400, 57600, 115200, 128000, 256000]; + return ( + + ) + } + + return ( + +

Serial Device:

+ {getSerialPort()} +

Baud:

+ {getBaud()} + + ); +} From 12af3e331d9189cf12b43268f4f7bb5a2324c5bf Mon Sep 17 00:00:00 2001 From: Mingcan Li Date: Mon, 6 May 2024 11:36:29 -0500 Subject: [PATCH 2/3] Make time out work --- Backend/core/comms.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Backend/core/comms.py b/Backend/core/comms.py index ae318c16..f95e3da1 100644 --- a/Backend/core/comms.py +++ b/Backend/core/comms.py @@ -148,13 +148,13 @@ def serial_read(self): while True: curr_device = serial_port['device'] curr_baud = serial_port['baud'] - print('serial', curr_device) if(curr_device): # Establish a serial connection) ser = serial.Serial(curr_device, curr_baud) # if device has been updated then exit loop and connect to new device - while curr_device == serial_port['device']: - print('read serial') + while curr_device == serial_port['device'] and curr_baud == serial_port['baud']: + if time.time() - latest_tstamp > 5: + solar_car_connection['serial'] = False # Read data from serial port try: data = b'' @@ -165,8 +165,6 @@ def serial_read(self): if not data: # No data received, continue listening continue - print('read data') - print('data:', data) packets = self.parse_packets(data, 'serial') for packet in packets: if len(packet) == byte_length: @@ -179,9 +177,6 @@ def serial_read(self): print(traceback.format_exc()) continue solar_car_connection['serial'] = True - if time.time() - latest_tstamp / 1000 > 5: - solar_car_connection['lte'] = False - break except Exception: print(traceback.format_exc()) solar_car_connection['serial'] = False @@ -270,7 +265,7 @@ def parse_packets(self, new_data: bytes, tmp_source: str): # If the remaining data is longer than the expected packet length, # there might be an incomplete packet, so log a warning. if len(self.__tmp_data[tmp_source]) >= byte_length: - print("Warning: Incomplete or malformed packet ------------------------------------") + print(f"Source: {tmp_source}: Warning: Incomplete or malformed packet ------------------------------------") self.__tmp_data[tmp_source] = b'' return packets From b9426c5d78d0e565e741247a74e75ad179a3073d Mon Sep 17 00:00:00 2001 From: Mingcan Li Date: Fri, 17 May 2024 12:37:02 -0500 Subject: [PATCH 3/3] optimize parse data for lossy telemetry data --- Backend/core/comms.py | 52 ++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/Backend/core/comms.py b/Backend/core/comms.py index f95e3da1..4e57b4f1 100644 --- a/Backend/core/comms.py +++ b/Backend/core/comms.py @@ -13,6 +13,7 @@ from multiprocessing.managers import BaseManager from . import db from file_sync.file_sync_down.main import * +import re format_string = '<' # little-endian byte_length = 0 @@ -241,35 +242,40 @@ async def remote_db_fetch(self, server_url: str): def parse_packets(self, new_data: bytes, tmp_source: str): """ - Parse and check the length of each packet - :param new_data: Newly received bytes from the comm channel - :param tmp_source: Name of tmp data source, put comm channel name here e.g. tcp, lte + Parse and check the length of each packet. + + :param new_data: Newly received bytes from the comm channel. + :param tmp_source: Name of tmp data source, put comm channel name here e.g. tcp, lte. """ - header = b'' - footer = b'' + header = b"" + footer = b" and tags + pattern = re.compile(b'(.*?)', re.DOTALL) + packets = [] while True: - # Search for the next complete data packet - try: - start_index = self.__tmp_data[tmp_source].index(header) - end_index = self.__tmp_data[tmp_source].index(footer) - except ValueError: + match = pattern.search(self.__tmp_data[tmp_source]) + if not match: break + # Extract the packet data + packet = match.group(1) + #remove headers and footers + packets.append(packet) - # Extract a complete data packet - packets.append(self.__tmp_data[tmp_source][start_index + len(header):end_index]) - # Update the remaining data to exclude the processed packet - self.__tmp_data[tmp_source] = self.__tmp_data[tmp_source][end_index + len(footer):] - - # If the remaining data is longer than the expected packet length, - # there might be an incomplete packet, so log a warning. - if len(self.__tmp_data[tmp_source]) >= byte_length: - print(f"Source: {tmp_source}: Warning: Incomplete or malformed packet ------------------------------------") - self.__tmp_data[tmp_source] = b'' + if match.start(0) != 0: + print(f"skipping {match.start(0)} bytes") + # Remove the processed packet from the temporary buffer + self.__tmp_data[tmp_source] = self.__tmp_data[tmp_source][match.end():] return packets + def fs_down_callback(self, data): # copied from listen_upd() if not data: @@ -301,11 +307,11 @@ def sigint_handler(signal, frame): def start_comms(): # start file sync - p.start() + # p.start() # Start two live comm channels - vps_thread = threading.Thread(target=lambda : asyncio.run(telemetry.remote_db_fetch(config.VPS_URL))) - vps_thread.start() + #vps_thread = threading.Thread(target=lambda : asyncio.run(telemetry.remote_db_fetch(config.VPS_URL))) + #vps_thread.start() #socket_thread = threading.Thread(target=lambda: telemetry.listen_udp(config.UDP_PORT)) #socket_thread.start() socket_thread = threading.Thread(target=lambda: telemetry.serial_read())