[Java] Support arbitrary char arrays in PBTs without data after null.

I spotted a failure in the DTO round-trip test locally where the DTO's
(and flyweight codec's) String-typed accessor would return `""` if the
first character in an array was the `null` character. Therefore, it
would not round-trip the "hidden bytes" after the `null`.

In this commit, I've added a generation mode that disables the
generation of strings with this format. Given the representation of
char[N] in DTOs it would not be sensible to support round-tripping these
"hidden bytes".

Author: ZachBray

sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/DtosPropertyTest.java

@@ -31,7 +31,6 @@
 import uk.co.real_logic.sbe.properties.arbitraries.SbeArbitraries;
 import uk.co.real_logic.sbe.properties.utils.InMemoryOutputManager;
 import org.agrona.*;
-import org.agrona.concurrent.UnsafeBuffer;
 import org.agrona.io.DirectBufferInputStream;
 
 import java.io.IOException;
@@ -115,10 +114,7 @@ void javaDtoEncodeShouldBeTheInverseOfDtoDecode(
                     encodedMessage.buffer(), MessageHeaderDecoder.ENCODED_LENGTH, blockLength, actingVersion);
                 outputBuffer.setMemory(0, outputBuffer.capacity(), (byte)0);
                 final int outputLength = (int)encodeWith.invoke(null, dto, outputBuffer, 0);
-                if (!areEqual(inputBuffer, inputLength, outputBuffer, outputLength))
-                {
-                    fail("Input and output differ");
-                }
+                assertEqual(inputBuffer, inputLength, outputBuffer, outputLength);
             }
         }
         catch (final Throwable throwable)
@@ -319,9 +315,9 @@ private static void execute(
     @Provide
     Arbitrary<SbeArbitraries.EncodedMessage> encodedMessage()
     {
-        final SbeArbitraries.CharGenerationMode mode =
-            SbeArbitraries.CharGenerationMode.JSON_PRINTER_COMPATIBLE;
-        return SbeArbitraries.encodedMessage(mode);
+        final SbeArbitraries.CharGenerationConfig config =
+            SbeArbitraries.CharGenerationConfig.jsonPrinterCompatibleAndNullTerminates();
+        return SbeArbitraries.encodedMessage(config);
     }
 
     private static void copyResourceToFile(
@@ -346,13 +342,33 @@ private static void copyResourceToFile(
         }
     }
 
-    private boolean areEqual(
+    private void assertEqual(
         final ExpandableArrayBuffer inputBuffer,
         final int inputLength,
         final ExpandableArrayBuffer outputBuffer,
         final int outputLength)
     {
-        return new UnsafeBuffer(inputBuffer, 0, inputLength).equals(new UnsafeBuffer(outputBuffer, 0, outputLength));
+        final boolean lengthsDiffer = inputLength != outputLength;
+        final int minLength = Math.min(inputLength, outputLength);
+
+        for (int i = 0; i < minLength; i++)
+        {
+            if (inputBuffer.getByte(i) != outputBuffer.getByte(i))
+            {
+                throw new AssertionError(
+                    "Input and output differ at byte " + i + ".\n" +
+                        "Input length: " + inputLength + ", Output length: " + outputLength + "\n" +
+                        "Input: " + inputBuffer.getByte(i) + ", Output: " + outputBuffer.getByte(i) +
+                        (lengthsDiffer ? "\nLengths differ." : ""));
+            }
+        }
+
+        if (lengthsDiffer)
+        {
+            throw new AssertionError(
+                "Input and output differ in length.\n" +
+                    "Input length: " + inputLength + ", Output length: " + outputLength);
+        }
     }
 
     private void addGeneratedSourcesFootnotes(

sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/JsonPropertyTest.java

@@ -46,8 +46,8 @@ void shouldGenerateValidJson(@ForAll("encodedMessage") final SbeArbitraries.Enco
     @Provide
     Arbitrary<SbeArbitraries.EncodedMessage> encodedMessage()
     {
-        final SbeArbitraries.CharGenerationMode mode =
-            SbeArbitraries.CharGenerationMode.JSON_PRINTER_COMPATIBLE;
-        return SbeArbitraries.encodedMessage(mode);
+        final SbeArbitraries.CharGenerationConfig config =
+            SbeArbitraries.CharGenerationConfig.jsonPrinterCompatibleAndNullTerminates();
+        return SbeArbitraries.encodedMessage(config);
     }
 }

sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java

@@ -114,10 +114,38 @@ private static Arbitrary<EncodedDataTypeSchema> encodedDataTypeSchema()
         ).as(EncodedDataTypeSchema::new);
     }
 
-    public enum CharGenerationMode
+    public static class CharGenerationConfig
     {
-        UNRESTRICTED,
-        JSON_PRINTER_COMPATIBLE
+        private final boolean alphaOnly;
+        private final boolean nullCharTerminatesData;
+
+        public CharGenerationConfig(
+            final boolean alphaOnly,
+            final boolean nullCharTerminatesData)
+        {
+            this.alphaOnly = alphaOnly;
+            this.nullCharTerminatesData = nullCharTerminatesData;
+        }
+
+        boolean alphaOnly()
+        {
+            return alphaOnly;
+        }
+
+        boolean nullCharTerminatesData()
+        {
+            return nullCharTerminatesData;
+        }
+
+        public static CharGenerationConfig unrestricted()
+        {
+            return new CharGenerationConfig(false, false);
+        }
+
+        public static CharGenerationConfig jsonPrinterCompatibleAndNullTerminates()
+        {
+            return new CharGenerationConfig(true, true);
+        }
     }
 
     private static Arbitrary<EnumTypeSchema> enumTypeSchema()
@@ -343,27 +371,24 @@ private static Arbitrary<Encoder> combineArbitraryEncoders(final List<Arbitrary<
         }
     }
 
-    public static CharacterArbitrary chars(final CharGenerationMode mode)
+    public static CharacterArbitrary chars(final CharGenerationConfig config)
     {
-        switch (mode)
+        if (config.alphaOnly())
         {
-            case UNRESTRICTED:
-                return Arbitraries.chars();
-
-            case JSON_PRINTER_COMPATIBLE:
-                return Arbitraries.chars().alpha();
-
-            default:
-                throw new IllegalArgumentException("Unsupported mode: " + mode);
+            return Arbitraries.chars().alpha();
+        }
+        else
+        {
+            return Arbitraries.chars();
         }
     }
 
     private static Arbitrary<Encoder> encodedTypeEncoder(
         final Encoding encoding,
         final boolean isOptional,
-        final CharGenerationMode charGenerationMode)
+        final CharGenerationConfig charGenerationConfig)
     {
-        final Arbitrary<Encoder> inRangeEncoder = encodedTypeEncoder(encoding, charGenerationMode);
+        final Arbitrary<Encoder> inRangeEncoder = encodedTypeEncoder(encoding, charGenerationConfig);
 
         if (isOptional)
         {
@@ -378,7 +403,7 @@ private static Arbitrary<Encoder> encodedTypeEncoder(
 
     private static Arbitrary<Encoder> encodedTypeEncoder(
         final Encoding encoding,
-        final CharGenerationMode charGenerationMode)
+        final CharGenerationConfig charGenerationConfig)
     {
         final PrimitiveValue minValue = encoding.applicableMinValue();
         final PrimitiveValue maxValue = encoding.applicableMaxValue();
@@ -387,7 +412,7 @@ private static Arbitrary<Encoder> encodedTypeEncoder(
         {
             case CHAR:
                 assert minValue.longValue() <= maxValue.longValue();
-                return chars(charGenerationMode).map(c ->
+                return chars(charGenerationConfig).map(c ->
                     (builder, buffer, offset, limit) ->
                     {
                         builder.appendLine().append(c).append(" @ ").append(offset)
@@ -558,13 +583,13 @@ private static Arbitrary<Encoder> encodedTypeEncoder(
         final int offset,
         final Token memberToken,
         final Token typeToken,
-        final CharGenerationMode charGenerationMode)
+        final CharGenerationConfig charGenerationConfig)
     {
         final Encoding encoding = typeToken.encoding();
         final Arbitrary<Encoder> arbEncoder = encodedTypeEncoder(
             encoding,
             memberToken.isOptionalEncoding(),
-            charGenerationMode
+            charGenerationConfig
         );
 
         if (typeToken.arrayLength() == 1)
@@ -577,11 +602,28 @@ private static Arbitrary<Encoder> encodedTypeEncoder(
             return arbEncoder.list().ofSize(typeToken.arrayLength())
                 .map(encoders -> (builder, buffer, bufferOffset, limit) ->
                 {
+                    boolean hasNullTerminated = false;
+
                     for (int i = 0; i < typeToken.arrayLength(); i++)
                     {
                         builder.beginScope("[" + i + "]");
                         final int elementOffset = bufferOffset + offset + i * encoding.primitiveType().size();
-                        encoders.get(i).encode(builder, buffer, elementOffset, limit);
+                        if (hasNullTerminated)
+                        {
+                            buffer.putByte(elementOffset, (byte)0);
+                        }
+                        else
+                        {
+                            encoders.get(i).encode(builder, buffer, elementOffset, limit);
+                        }
+
+                        if (encoding.primitiveType() == PrimitiveType.CHAR &&
+                            charGenerationConfig.nullCharTerminatesData() &&
+                            buffer.getByte(elementOffset) == 0)
+                        {
+                            hasNullTerminated = true;
+                        }
+
                         builder.endScope();
                     }
                 });
@@ -786,7 +828,7 @@ private static Arbitrary<Encoder> fieldsEncoder(
         final MutableInteger cursor,
         final int endIdxInclusive,
         final boolean expectFields,
-        final CharGenerationMode charGenerationMode)
+        final CharGenerationConfig charGenerationConfig)
     {
         final List<Arbitrary<Encoder>> encoders = new ArrayList<>();
         while (cursor.get() <= endIdxInclusive)
@@ -820,7 +862,7 @@ else if (expectFields)
                         final int endCompositeTokenCount = 1;
                         final int lastMemberIdx = nextFieldIdx - endCompositeTokenCount - endFieldTokenCount - 1;
                         final Arbitrary<Encoder> encoder = fieldsEncoder(
-                            tokens, cursor, lastMemberIdx, false, charGenerationMode);
+                            tokens, cursor, lastMemberIdx, false, charGenerationConfig);
                         fieldEncoder = encoder.map(e ->
                             (builder, buffer, bufferOffset, limit) ->
                                 e.encode(builder, buffer, bufferOffset + offset, limit));
@@ -839,7 +881,7 @@ else if (expectFields)
                         break;
 
                     case ENCODING:
-                        fieldEncoder = encodedTypeEncoder(offset, memberToken, typeToken, charGenerationMode);
+                        fieldEncoder = encodedTypeEncoder(offset, memberToken, typeToken, charGenerationConfig);
                         break;
 
                     default:
@@ -868,7 +910,7 @@ private static Arbitrary<Encoder> groupsEncoder(
         final List<Token> tokens,
         final MutableInteger cursor,
         final int endIdxInclusive,
-        final CharGenerationMode charGenerationMode)
+        final CharGenerationConfig charGenerationConfig)
     {
         final List<Arbitrary<Encoder>> encoders = new ArrayList<>();
 
@@ -894,9 +936,9 @@ private static Arbitrary<Encoder> groupsEncoder(
 
 
             final Arbitrary<Encoder> groupElement = Combinators.combine(
-                fieldsEncoder(tokens, cursor, nextFieldIdx - 1, true, charGenerationMode),
-                groupsEncoder(tokens, cursor, nextFieldIdx - 1, charGenerationMode),
-                varDataEncoder(tokens, cursor, nextFieldIdx - 1, charGenerationMode)
+                fieldsEncoder(tokens, cursor, nextFieldIdx - 1, true, charGenerationConfig),
+                groupsEncoder(tokens, cursor, nextFieldIdx - 1, charGenerationConfig),
+                varDataEncoder(tokens, cursor, nextFieldIdx - 1, charGenerationConfig)
             ).as((fieldsEncoder, groupsEncoder, varDataEncoder) ->
                 (builder, buffer, ignored, limit) ->
                 {
@@ -946,7 +988,7 @@ private static Arbitrary<Encoder> varDataEncoder(
         final List<Token> tokens,
         final MutableInteger cursor,
         final int endIdxInclusive,
-        final CharGenerationMode charGenerationMode)
+        final CharGenerationConfig charGenerationConfig)
     {
         final List<Arbitrary<Encoder>> encoders = new ArrayList<>();
 
@@ -969,7 +1011,7 @@ private static Arbitrary<Encoder> varDataEncoder(
             final String characterEncoding = varDataToken.encoding().characterEncoding();
             final Arbitrary<Byte> arbitraryByte = null == characterEncoding ?
                 Arbitraries.bytes() :
-                chars(charGenerationMode).map(c -> (byte)c.charValue());
+                chars(charGenerationConfig).map(c -> (byte)c.charValue());
             encoders.add(arbitraryByte.list()
                 .ofMaxSize((int)Math.min(lengthToken.encoding().applicableMaxValue().longValue(), 260L))
                 .map(bytes -> (builder, buffer, ignored, limit) ->
@@ -1003,7 +1045,7 @@ private static Arbitrary<Encoder> varDataEncoder(
     private static Arbitrary<Encoder> messageValueEncoder(
         final Ir ir,
         final short messageId,
-        final CharGenerationMode charGenerationMode)
+        final CharGenerationConfig charGenerationConfig)
     {
         final List<Token> tokens = ir.getMessage(messageId);
         final MutableInteger cursor = new MutableInteger(1);
@@ -1015,11 +1057,11 @@ private static Arbitrary<Encoder> messageValueEncoder(
         }
 
         final Arbitrary<Encoder> fieldsEncoder = fieldsEncoder(
-            tokens, cursor, tokens.size() - 1, true, charGenerationMode);
+            tokens, cursor, tokens.size() - 1, true, charGenerationConfig);
         final Arbitrary<Encoder> groupsEncoder = groupsEncoder(
-            tokens, cursor, tokens.size() - 1, charGenerationMode);
+            tokens, cursor, tokens.size() - 1, charGenerationConfig);
         final Arbitrary<Encoder> varDataEncoder = varDataEncoder(
-            tokens, cursor, tokens.size() - 1, charGenerationMode);
+            tokens, cursor, tokens.size() - 1, charGenerationConfig);
         return Combinators.combine(fieldsEncoder, groupsEncoder, varDataEncoder)
             .as((fields, groups, varData) -> (builder, buffer, offset, limit) ->
             {
@@ -1087,7 +1129,7 @@ public String encodingLog()
         }
     }
 
-    public static Arbitrary<EncodedMessage> encodedMessage(final CharGenerationMode mode)
+    public static Arbitrary<EncodedMessage> encodedMessage(final CharGenerationConfig charGenerationConfig)
     {
         return SbeArbitraries.messageSchema().flatMap(testSchema ->
         {
@@ -1101,7 +1143,7 @@ public static Arbitrary<EncodedMessage> encodedMessage(final CharGenerationMode
                     .build();
                 final uk.co.real_logic.sbe.xml.MessageSchema parsedSchema = parse(in, options);
                 final Ir ir = new IrGenerator().generate(parsedSchema);
-                return SbeArbitraries.messageValueEncoder(ir, testSchema.templateId(), mode)
+                return SbeArbitraries.messageValueEncoder(ir, testSchema.templateId(), charGenerationConfig)
                     .map(encoder ->
                     {
                         final EncodingLogger logger = new EncodingLogger();