golang backend phase2

This release is mostly functionally complete but see gocode/README.md for next steps. Note, there are API Changes and added methods.

Handle extension logic so that schemas may be extended.

Implement Deprecated().

General code improvements/refactoring.

Vastly improve group handling code.

Add helper methods to choices.

Add much more testing with bug fixes.

Properly introduce raneg checking on encode/decode so we can better check for correctness.

Improve the documentation/design doc.

Improve the build environment (but see README.md)

Add some initial benchmark cases before beginning performance work.

Fix go fmt - generated code is now clean.

Author: billsegall

.gitignore

@@ -70,15 +70,18 @@ gocode/src/composite/*.go
 !gocode/src/composite/*_test.go
 gocode/src/composite_elements/*.go
 !gocode/src/composite_elements/*_test.go
-gocode/src/group_with_data/*.go
-!gocode/src/group_with_data/*_test.go
+gocode/src/group_with_data*/*.go
+!gocode/src/group_with_data*/*_test.go
 gocode/src/mktdata/*.go
 !gocode/src/mktdata/*_test.go
 gocode/src/simple/*.go
 !gocode/src/simple/*_test.go
 gocode/src/*/*/*.go
 !gocode/src/*/*/*_test.go
-gocode/src/example-schema/example-schema
+gocode/src/example-schema/example-schema*
+gocode/src/example-schema/cpu.out
+gocode/src/extension
+gocode/src/extension2
 
 # Mac
 .DS_Store

config/suppress.xml

@@ -4,4 +4,5 @@
         "http://www.puppycrawl.com/dtds/suppressions_1_0.dtd">
 <suppressions>
     <suppress files=".*generated.*" checks="."/>
+    <suppress files="GolangGenerator.java" checks="MethodLength"/>
 </suppressions>

gocode/Makefile

@@ -1,30 +1,22 @@
 ROOT=..
-SBE_TOOL_VERSION=`cat $(ROOT)/version.txt`
+SBE_TOOL_VERSION=$(shell cat $(ROOT)/version.txt)
 SBE_JAR=$(ROOT)/sbe-all/build/libs/sbe-all-${SBE_TOOL_VERSION}.jar
 
-SCHEMA_DIR=$(ROOT)/sbe-samples/src/main/resources
-EXAMPLES_SCHEMA=$(SCHEMA_DIR)/example-schema.xml
-EXAMPLES_OUTPUT=EXAMPLES_OUTPUT
-
 #JAVA=/usr/lib/jvm/java-8-openjdk-amd64/bin/java
 JAVA=java
 
 TESTDIR=$(ROOT)/sbe-tool/src/test/golang
 
 JAVA_FILES="-jar ${SBE_JAR}"
 
-# During development we can speed up a build by selective building and
-# using this but it needs periodic refresh for agrona version
-#JAVA_FILES="-cp ${$CP}"
-#CP=$(ROOT)/sbe-all/build/classes/main:$(ROOT)/sbe-all/build/resources/main:$(ROOT)/sbe-tool/build/libs/sbe-tool-$(SBE_TOOL_VERSION).jar:/home/bill/.gradle/caches/modules-2/files-2.1/org.agrona/Agrona/0.5.5/5a576d17c57c5c85b9a1ce19f030746e214e281b/Agrona-0.5.5.jar
-
 # FIXME: Go doesn't like relative paths so us a gnumake extension for now
 GOPATH=$(realpath $(ROOT)/gocode)
 OUTPUTDIR=$(GOPATH)/src
+XMLLOCAL=$(GOPATH)/resources
 
 # Local examples to get started
-SIMPLE=$(GOPATH)/resources/simple.xml
-COMPOSITE=$(GOPATH)/resources/example-composite.xml
+SIMPLE=$(XMLLOCAL)/simple.xml
+COMPOSITE=$(XMLLOCAL)/example-composite.xml
 
 # Existing test cases
 XMLTESTDIR=$(ROOT)/sbe-tool/src/test/resources
@@ -35,12 +27,27 @@ COMPOSITE4=$(XMLTESTDIR)/composite-elements-schema-rc4.xml
 NESTED_GROUP=$(XMLTESTDIR)/group-with-data-schema.xml
 FIX_BINARY=$(XMLTESTDIR)/FixBinary.xml
 
+SCHEMA_DIR=$(ROOT)/sbe-samples/src/main/resources
+EXAMPLES_SCHEMA=$(SCHEMA_DIR)/example-schema.xml
+EXTENSION_SCHEMA=$(SCHEMA_DIR)/example-extension-schema.xml
+EXTENSION2_SCHEMA=$(XMLLOCAL)/example-extension-2-schema.xml
+GROUP_EXTENSION=$(XMLLOCAL)/group-with-data-extension-schema.xml
+
 # Convenience during development
 #SAVE_FORMAT=mkdir -p fmt && cp *.go fmt &&
 SAVE_FORMAT=
 
+$(OUTPUTDIR)/example-schema/example-schema: $(SBE_JAR) $(OUTPUTDIR)/baseline/Car.go $(OUTPUTDIR)/extension/Car.go $(OUTPUTDIR)/extension2/Car.go $(OUTPUTDIR)/example-schema/CarExample.go
+	(export GOPATH=$(GOPATH) && \
+	cd $(OUTPUTDIR)/example-schema && \
+	go build && \
+	go fmt && \
+	./example-schema)
 
-GeneratedStubExample: $(STUB_EXAMPLE)
+clean:
+	rm -f $(OUTPUTDIR)/baseline/Car.go $(OUTPUTDIR)/extension/Car.go $(OUTPUTDIR)/extension2/Car.go
+
+$(OUTPUTDIR)/baseline/Car.go: $(SBE_JAR)
 	$(JAVA) \
 	-Dsbe.output.dir=$(OUTPUTDIR) \
 	-Dsbe.generate.ir="false" \
@@ -50,13 +57,41 @@ GeneratedStubExample: $(STUB_EXAMPLE)
 	(export GOPATH=$(GOPATH) && \
         cd $(OUTPUTDIR)/baseline && \
 	go build && \
+	go fmt && \
 	go test && \
-	go install && \
-	cd $(OUTPUTDIR)/example-schema && \
+	go install)
+
+$(OUTPUTDIR)/extension/Car.go: $(SBE_JAR)
+	$(JAVA) \
+	-Dsbe.output.dir=$(OUTPUTDIR) \
+	-Dsbe.generate.ir="false" \
+	-Dsbe.target.language="golang" \
+	-jar ${SBE_JAR} \
+	$(EXTENSION_SCHEMA) && \
+	(export GOPATH=$(GOPATH) && \
+        cd $(OUTPUTDIR)/extension && \
 	go build && \
-	./example-schema)
+	go install)
 
-test: fix-binary nested-group basic-vardata basic-group basic-types simple composite composite-elements
+$(OUTPUTDIR)/extension2/Car.go: $(SBE_JAR)
+	$(JAVA) \
+	-Dsbe.output.dir=$(OUTPUTDIR) \
+	-Dsbe.generate.ir="false" \
+	-Dsbe.target.language="golang" \
+	-jar ${SBE_JAR} \
+	$(EXTENSION2_SCHEMA) && \
+	(export GOPATH=$(GOPATH) && \
+        cd $(OUTPUTDIR)/extension2 && \
+	go build && \
+	go install)
+
+bench: $(SBE_JAR) $(OUTPUTDIR)/example-schema/example-schema
+	(export GOPATH=$(GOPATH) && \
+        cd $(OUTPUTDIR)/example-schema && \
+	go test --bench . -cpuprofile=cpu.out && \
+	go tool pprof -text example-schema cpu.out)
+
+test: fix-binary nested-group nested-group-extension basic-vardata basic-group basic-types simple composite composite-elements
 
 fix-binary:
 	$(JAVA) \
@@ -68,12 +103,14 @@ fix-binary:
 	-Duser.variant \
 	-jar ${SBE_JAR} \
 	$(FIX_BINARY)
-	(cd $(OUTPUTDIR)/mktdata && \
+	(export GOPATH=$(GOPATH) && \
+	cd $(OUTPUTDIR)/mktdata && \
 	go build && \
 	$(SAVE_FORMAT) \
 	go fmt && echo && \
 	go test)
 
+# Note this one installs so it can be used in the extension below
 nested-group:
 	$(JAVA) \
 	-Dsbe.output.dir=$(OUTPUTDIR) \
@@ -84,7 +121,26 @@ nested-group:
 	-Duser.variant \
 	-jar ${SBE_JAR} \
 	$(NESTED_GROUP)
-	(cd $(OUTPUTDIR)/group_with_data && \
+	(export GOPATH=$(GOPATH) && \
+	cd $(OUTPUTDIR)/group_with_data && \
+	go build && \
+	$(SAVE_FORMAT) \
+	go fmt && echo && \
+	go install && \
+	go test)
+
+nested-group-extension:
+	$(JAVA) \
+	-Dsbe.output.dir=$(OUTPUTDIR) \
+	-Dsbe.target.language=golang \
+	-Dfile.encoding=US-ASCII \
+	-Duser.country=US \
+	-Duser.langimuage=en \
+	-Duser.variant \
+	-jar ${SBE_JAR} \
+	$(GROUP_EXTENSION)
+	(export GOPATH=$(GOPATH) && \
+	cd $(OUTPUTDIR)/group_with_data_extension && \
 	go build && \
 	$(SAVE_FORMAT) \
 	go fmt && echo && \
@@ -100,7 +156,8 @@ composite-elements:
 	-Duser.variant \
 	-jar ${SBE_JAR} \
 	$(COMPOSITE4)
-	(cd $(OUTPUTDIR)/composite_elements && \
+	(export GOPATH=$(GOPATH) && \
+	cd $(OUTPUTDIR)/composite_elements && \
 	go build && \
 	$(SAVE_FORMAT) \
 	go fmt && echo && \
@@ -116,7 +173,8 @@ basic-vardata:
 	-Duser.variant \
 	-jar ${SBE_JAR} \
 	$(BASIC_VARDATA)
-	(cd $(OUTPUTDIR)/vardata/'SBE tests' && \
+	(export GOPATH=$(GOPATH)/vardata && \
+	cd $(OUTPUTDIR)/vardata/'SBE tests' && \
 	go build && \
 	$(SAVE_FORMAT) \
 	go fmt && echo && \
@@ -132,7 +190,8 @@ basic-group:
 	-Duser.variant \
 	-jar ${SBE_JAR} \
 	$(BASIC_GROUP)
-	(cd $(OUTPUTDIR)/group/'SBE tests' && \
+	(export GOPATH=$(GOPATH)/group && \
+	cd $(OUTPUTDIR)/group/'SBE tests' && \
 	go build && \
 	$(SAVE_FORMAT) \
 	go fmt && echo && \
@@ -148,7 +207,8 @@ basic-types:
 	-Duser.variant \
 	-jar ${SBE_JAR} \
 	$(BASIC_TYPES)
-	(cd $(OUTPUTDIR)/basic/'SBE tests' && \
+	(export GOPATH=$(GOPATH)/basic && \
+	cd $(OUTPUTDIR)/basic/'SBE tests' && \
 	go build && \
 	$(SAVE_FORMAT) \
 	go fmt && echo && \
@@ -165,8 +225,8 @@ composite:
 	-Duser.variant \
 	-jar ${SBE_JAR} \
 	$(COMPOSITE)
-	(cd $(OUTPUTDIR)/composite && \
-	go build && \
+	(export GOPATH=$(GOPATH) && \
+	cd $(OUTPUTDIR)/composite && \
 	$(SAVE_FORMAT) \
 	go fmt && echo && \
 	go test)
@@ -181,7 +241,8 @@ simple:
 	-Duser.variant \
 	-jar ${SBE_JAR} \
 	$(SIMPLE)
-	(cd $(OUTPUTDIR)/simple && \
+	(export GOPATH=$(GOPATH) && \
+	cd $(OUTPUTDIR)/simple && \
 	go build && \
 	$(SAVE_FORMAT) \
 	go fmt && echo && \

gocode/README.md

@@ -14,27 +14,58 @@ Getter/Setter method but is instead performed in the Encode and Decode
 methods. In general the Encode methods attempt to be strictly
 enforcing to ensure only valid SBE is emitted. Where it remains within
 the standard the Decode methods attempt to be forgiving in their
-interpretation and range checking can be disabled via the the use of
-the 'strict' parameter.
+interpretation.
+
+To maintain consistency of datastreams, `Encode()` will fail without
+writing to the stream, and `Decode()` will fail but will have read (and
+where possible decoded) an entire message from the stream.
 
 Secondly, constant values are not set when an object is created. As
 golang does not provide constructors, we instead have each support
-type Foo provide a FooInit() function which will fill out constant
-values. Decoding does fill out constant values.
+type `Foo` provide a `FooInit()` function which will fill out constant
+values. Decoding automatically fills out constant values.
 
 Some design decisions are based around the structure of sbe-tool
 itself. sbe-tool parses the XML into an internal representation (IR)
 and then passes this to the language specific generator. As a result
 some information on the xml structure has been lost.
 
-An examples of this include the use of the <ref> tag which if used to
+An examples of this include the use of the `<ref>` tag which if used to
 the same underlying type in a composite will create two differently
 named definitions of the type in the IR.
 
 A second example occurs where a field references an aliased primitive
-type definition in which case we lose the aliaed type name in the IR
+type definition in which case we lose the aliased type name in the IR
 as it is unreferenced.
 
+Code Layout
+-----------
+To match golang's requirement on directory structure and file layout,
+the build environment, example and test code are structured into a top
+level gocode directory hierarchy.
+
+Code is generated into this structure with pre-existing test code in place.
+
+For the example, the code is generated into a library and the matching
+example code lives in it's own directory at the same level. For
+example, the example-schema generates the baseline library code into
+`gocode/src/baseline` and example code lives in `gocode/src/example-schema`.
+
+To use this layout you should `set GOPATH=/path/to/gocode` or use the
+supplied Makefile which does this for you. For the tests you will need
+to not have `GOPATH` set or use the supplied Makefile which does this
+for you.
+
+Alpha Code Warning
+------------------
+The golang generator is still under development and the provided APIs
+are still subject to change. Some design decisions remain
+open. Currently the Car example (and extension) works but this is not
+an exhaustive set of features.
+
+
+Design Notes
+============
 
 Primitive Types
 ---------------
@@ -57,59 +88,47 @@ as a type and a variable with the known constant values is
 provided. To preserve name uniqueness within a message the const
 values contain the enum type name.
 
+Range Checking of enumeration values is only performed when the
+schemaVersion is equal to or greater than the actingVersion so that
+new enumeration values are allowed to propogate to older decoders.
+
 Messages and Composites
 -----------------------
 Messages and composites are both represented as golang structures with
-Encode() and Decode() methods amongst others.
+`Encode()` and `Decode()` methods amongst others.
 
 Groups and VarData
 ------------------
 Groups and VarData are represented as arrays with their NumInGroup
-being implicit in len()
+being implicit in `len()`
 
 Semantic Types
 --------------
+The golang generator provides no special support for semantic
+types (as for Java and C++).
 
-The Boolean semantic type is not represented as a golang bool as the
+The `Boolean` semantic type is not represented as a golang `bool` as the
 spec requires that discrepancies between SinceVersion and
-ActingVersion can be resolved using the Null Value.
+ActingVersion can be resolved using the NullValue.
 
-The String semantic type is simply represented as a byte array as
+The `String` semantic type is simply represented as a byte array as
 golang Strings are immutable and are not useful in structures.
 
-[FIXME: insert more here]
-
-
-Code Layout
------------
-To match golang's requirement on directory structure and file layout,
-the build environment, example and test code are structured into a top
-level gocode directory hierarchy.
-
-Code is generated into this structure with pre-existing test code in place.
-
-For the example, the code is generated into a library and the matching
-example code lives in it's own directory at the same level. For
-example the example-schema generates the baseline library code into
-gocode/src/baseline and example code lives in gocode/src/example-schema.
-
-To use this layout you will need to set GOPATH=/path/to/gocode or use
-the supplied Makefile which does this for you.
+The `UTCTimestamp`, `UTCTimeOnly`, `UTCDateOnly` and `MktTime` semantic types
+are not represented as a golang `Time` type as none of these semantic
+types contain a location. Mapping to a golang `Time` type will be
+relatively easy for the application developer who will know the
+location and can set it accordingly (null is UTC). They can also
+decide what to with golang `Time`'s nanosecond timer when converting
+(probably treating it as zero).
 
 Roadmap
--------
- * Deprecated support
- * Support full Range checking on both Encode and Decode
- * Car example-extension-schema works
+=======
  * Examples documented
- * Restructure recursive group generation as it's currently somewhat broken
- * Chase up Ennn error message meanings with Real Logic and support?
- * go format fixes
  * Windows developer support (currently tested on Linux/MacOS)
  * Enhance Design/Rationale document (this one)
  * presence=optional
  * Unnecessary code removal (e.g., GroupSizeEncoding)
- * Unicode
+ * Further Unicode support
  * Testing/Bug fixes
- * Semantic types (perhaps just date/time, possibly more)
  * Benchmarking & Performance Enhancements