diff --git a/doc/architecture/formatting_scripts.adoc b/doc/architecture/formatting_scripts.adoc index 9b1a26857..a953b3e13 100644 --- a/doc/architecture/formatting_scripts.adoc +++ b/doc/architecture/formatting_scripts.adoc @@ -7,28 +7,6 @@ endif::[] The OSI repository contains Python scripts for converting trace files from one format to another. The formatting scripts are stored in `open-simulation-interface/format/` -**txt2osi.py** - -`txt2osi.py` converts plain-text trace files to binary `.osi` trace files. -This script takes the following parameters: - -`--data`, `-d`:: -String containing the path to the file with serialized data. - -`--type`, `-t`:: -Optional string describing the message type used to serialize data. -`'SensorView'`, `'GroundTruth'`, or `'SensorData'` are permitted values. -The default value is `'SensorView'`. - -`--output`, `-o`:: -Optional string containing the name of the output file. -The default value is `'converted.osi'`. - -`--compress`, `-c`:: -Optional Boolean controlling whether to compress the output to an lzma file. -`True`, or `False` are permitted values. -The default value is `False`. - **osi2read.py** `osi2read.py` converts trace files to human-readable `.txth` trace files. diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index 0ed0a4515..60f2d2a4b 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -5,7 +5,7 @@ endif::[] [#top-osi_trace_file_formats] = OSI trace file formats -There are multiple formats for storing multiple serialized OSI messages in one trace file. +There are two formats for storing multiple serialized OSI messages in one trace file. *.osi:: Binary trace file. @@ -13,11 +13,12 @@ Messages are separated by a length specification before each message. The length is represented by a four-byte, little-endian, unsigned integer. The length does not include the integer itself. -*.txt:: -Plain-text trace file. -Messages are separated by `$$__$$`. - *.txth:: Human-readable plain-text trace file. Messages are separated by newlines. + +NOTE: Previous releases of OSI also supported a so-called plain-text trace file format, with file extension `.txt`. +This legacy format did not contain plain-text, but rather binary protobuf messages separated by a special separator. +For obvious reasons the format was deprecated and fully replaced with the `.osi` binary file format. +This release no longer contains any support for the legacy `.txt` file format. These files may be used for manual checks. diff --git a/format/OSITrace.py b/format/OSITrace.py index d8d2c246a..cc92f06c7 100644 --- a/format/OSITrace.py +++ b/format/OSITrace.py @@ -13,8 +13,6 @@ warnings.simplefilter("default") -SEPARATOR = b"$$__$$" -SEPARATOR_LENGTH = len(SEPARATOR) BUFFER_SIZE = 1000000 @@ -47,7 +45,7 @@ def __init__(self, path=None, type_name="SensorView"): self.retrieved_scenario_size = 0 self._int_length = len(struct.pack("= scenario_size: - break - self.message_offsets.append(message_offset) - - if eof: - self.retrieved_scenario_size = scenario_size - else: - self.retrieved_scenario_size = self.message_offsets[-1] - self.message_offsets.pop() - - return len(self.message_offsets) + self.timestep_count = self.retrieve_message() def retrieve_message(self): scenario_size = get_size_from_file_stream(self.scenario_file) @@ -180,42 +116,21 @@ def get_messages_in_index_range(self, begin, end): for abs_message_offset in self.message_offsets[begin:end] ] - if self.format_type == "separated": - message_sequence_len = abs_last_offset - abs_first_offset - SEPARATOR_LENGTH - serialized_messages_extract = self.scenario_file.read(message_sequence_len) - - for rel_index, rel_message_offset in enumerate(rel_message_offsets): - rel_begin = rel_message_offset - rel_end = ( - rel_message_offsets[rel_index + 1] - SEPARATOR_LENGTH - if rel_index + 1 < len(rel_message_offsets) - else message_sequence_len - ) - message = MESSAGES_TYPE[self.type_name]() - serialized_message = serialized_messages_extract[rel_begin:rel_end] - message.ParseFromString(serialized_message) - yield message - - elif self.format_type is None: - message_sequence_len = abs_last_offset - abs_first_offset - serialized_messages_extract = self.scenario_file.read(message_sequence_len) - - for rel_index, rel_message_offset in enumerate(rel_message_offsets): - rel_begin = rel_message_offset + self._int_length - rel_end = ( - rel_message_offsets[rel_index + 1] - if rel_index + 1 < len(rel_message_offsets) - else message_sequence_len - ) - - message = MESSAGES_TYPE[self.type_name]() - serialized_message = serialized_messages_extract[rel_begin:rel_end] - message.ParseFromString(serialized_message) - yield message + message_sequence_len = abs_last_offset - abs_first_offset + serialized_messages_extract = self.scenario_file.read(message_sequence_len) - else: - self.scenario_file.close() - raise Exception(f"The defined format {self.format_type} does not exist.") + for rel_index, rel_message_offset in enumerate(rel_message_offsets): + rel_begin = rel_message_offset + self._int_length + rel_end = ( + rel_message_offsets[rel_index + 1] + if rel_index + 1 < len(rel_message_offsets) + else message_sequence_len + ) + + message = MESSAGES_TYPE[self.type_name]() + serialized_message = serialized_messages_extract[rel_begin:rel_end] + message.ParseFromString(serialized_message) + yield message def make_readable(self, name, interval=None, index=None): self.scenario_file.seek(0) diff --git a/format/osi2read.py b/format/osi2read.py index 90eb88a4d..240507177 100644 --- a/format/osi2read.py +++ b/format/osi2read.py @@ -1,9 +1,8 @@ """ -This program converts serialized txt/osi trace files into a human readable txth file. +This program converts serialized osi trace files into a human readable txth file. Example usage: python3 osi2read.py -d trace.osi -o myreadableosifile - python3 osi2read.py -d trace.txt -f separated -o myreadableosifile """ from OSITrace import OSITrace @@ -19,7 +18,7 @@ def command_line_arguments(): dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) parser = argparse.ArgumentParser( - description="Convert a serialized osi/txt trace file to a readable txth output.", + description="Convert a serialized osi trace file to a readable txth output.", prog="osi2read converter", ) parser.add_argument( @@ -42,15 +41,6 @@ def command_line_arguments(): type=str, required=False, ) - parser.add_argument( - "--format", - "-f", - help="Set the format type of the trace.", - choices=["separated", None], - default=None, - type=str, - required=False, - ) return parser.parse_args() @@ -61,7 +51,7 @@ def main(): # Initialize the OSI trace class trace = OSITrace() - trace.from_file(path=args.data, type_name=args.type, format_type=args.format) + trace.from_file(path=args.data, type_name=args.type) args.output = args.output.split(".", 1)[0] + ".txth" diff --git a/format/txt2osi.py b/format/txt2osi.py deleted file mode 100644 index 13eb8e788..000000000 --- a/format/txt2osi.py +++ /dev/null @@ -1,88 +0,0 @@ -""" -This program converts txt trace files separated with $$__$$ to OSI trace files which are defined by the length of each OSI message. - -Example usage: - python3 txt2osi.py -d trace.txt - python3 txt2osi.py -d trace.txt -o myfile - python3 txt2osi.py -d trace.txt -o myfile -c - python3 txt2osi.py -d trace.txt.lzma -c - python3 txt2osi.py -d trace.txt.lzma -""" - -from OSITrace import OSITrace -import struct -import lzma -import argparse -import os - - -def command_line_arguments(): - """Define and handle command line interface""" - - dir_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - - parser = argparse.ArgumentParser( - description="Convert txt trace file to osi trace files.", - prog="txt2osi converter", - ) - parser.add_argument( - "--data", "-d", help="Path to the file with serialized data.", type=str - ) - parser.add_argument( - "--type", - "-t", - help="Name of the type used to serialize data.", - choices=["SensorView", "GroundTruth", "SensorData"], - default="SensorView", - type=str, - required=False, - ) - parser.add_argument( - "--output", - "-o", - help="Output name of the file.", - default="converted.osi", - type=str, - required=False, - ) - parser.add_argument( - "--compress", - "-c", - help="Compress the output to a lzma file.", - default=False, - required=False, - action="store_true", - ) - - return parser.parse_args() - - -def main(): - # Handling of command line arguments - args = command_line_arguments() - - # Initialize the OSI trace class - trace = OSITrace() - trace.from_file(path=args.data, type_name=args.type, format_type="separated") - sv = trace.get_messages() # Create an iterator for messages - - args.output = args.output.split(".", 1)[0] + ".osi" - - if args.output == "converted.osi": - args.output = args.data.split(".", 1)[0] + ".osi" - - if args.compress: - f = lzma.open(args.output + ".lzma", "ab") - else: - f = open(args.output, "ab") - - for message in sv: - byte_buffer = message.SerializeToString() - f.write(struct.pack("