update protoc and related (#552)

* update protoc and related

updated protoc to v25.2
updated protoc-gen-go to v1.32.0
added protoc-gen-go-grpc at v1.3.0

Modified the makefile to use "go install" to install tools.
Use extended regular expressions in "sed" command.

Fixes build on arm OSX host

* fix github actions

Always run "make build_go" on github actions

Always update the timesptamp of spec.md/csi.proto to make sure lib/go is built on CI

* remove github.com/golang/protobuf usage

* simplify Makefile

- use git to check for out-of-date generated files, avoids all the temp files and manual diff in the Makefile.
- not specify version for protoc-gen-go, it will use the version from go.mod
- do not copy over the protoc binary. Call it directly from unzipped path. So it can find the includes automatically, and we don't need the -I or --go_opts=Mxxx flags.
- use paths=source_relative to generate go files directly into destination, avoid the copy.
- simplify the sed command used to extract "csi.proto" file.

Author: huww98

.github/workflows/build.yaml

@@ -22,7 +22,6 @@ jobs:
 
     - name: Build Test
       run: |
+        touch spec.md  # Ensure its timestamp is newer than any generated files
         make
-        make -C lib/go protoc
-        make -C lib/go protoc-gen-go
-        make build_go
+        git diff --exit-code || (echo "Generated files are out of date. Please run 'make' and commit the changes." && exit 1)

Makefile

@@ -4,50 +4,22 @@ CSI_SPEC := spec.md
 CSI_PROTO := csi.proto
 ## Build go language bindings
 CSI_A := csi.a
-CSI_GO := lib/go/csi/csi.pb.go
 CSI_PKG := lib/go/csi
 
-# This is the target for building the temporary CSI protobuf file.
-#
-# The temporary file is not versioned, and thus will always be
-# built on GitHub Actions.
-$(CSI_PROTO).tmp: $(CSI_SPEC) Makefile
-	echo "// Code generated by make; DO NOT EDIT." > "$@"
-	cat $< | sed -n -e '/```protobuf$$/,/^```$$/ p' | sed '/^```/d' >> "$@"
-
 # This is the target for building the CSI protobuf file.
-#
-# This target depends on its temp file, which is not versioned.
-# Therefore when built on GitHub Actions the temp file will always
-# be built and trigger this target. On GitHub Actions the temp file
-# is compared with the real file, and if they differ the build
-# will fail.
-#
-# Locally the temp file is simply copied over the real file.
-$(CSI_PROTO): $(CSI_PROTO).tmp
-ifeq (true,$(GITHUB_ACTIONS))
-	diff "$@" "$?"
-else
-	diff "$@" "$?" > /dev/null 2>&1 || cp -f "$?" "$@"
-endif
-
-build: check
-
-# If this is not running on GitHub Actions then for sake of convenience
-# go ahead and update the language bindings as well.
-ifneq (true,$(GITHUB_ACTIONS))
-build: build_cpp build_go
-endif
+$(CSI_PROTO): $(CSI_SPEC) Makefile
+	echo "// Code generated by make; DO NOT EDIT." > "$@"
+	sed -n -e '/```protobuf$$/,/^```$$/ {//!p;}' $< >> "$@"
 
+build: check build_cpp build_go
 
 build_cpp:
 	$(MAKE) -C lib/cxx
 
-# The file exists, but could be out-of-date.
-$(CSI_GO): $(CSI_PROTO)
-	$(MAKE) -C lib/go csi/csi.pb.go
+$(CSI_PKG)/%.pb.go: $(CSI_PROTO)
+	$(MAKE) -C lib/go
 
-$(CSI_A): $(CSI_GO)
+$(CSI_A): $(CSI_PKG)/*.go
 	go mod download
 	go install ./$(CSI_PKG)
 	go build -o "$@" ./$(CSI_PKG)
@@ -60,10 +32,10 @@ clean:
 
 clobber: clean
 	$(MAKE) -C lib/go $@
-	rm -f $(CSI_PROTO) $(CSI_PROTO).tmp
+	rm -f $(CSI_PROTO)
 
 # check generated files for violation of standards
 check: $(CSI_PROTO)
 	awk '{ if (length > 72) print NR, $$0 }' $? | diff - /dev/null
 
-.PHONY: clean clobber check
+.PHONY: clean clobber check build_go build_cpp

csi.proto

@@ -6,7 +6,8 @@ import "google/protobuf/descriptor.proto";
 import "google/protobuf/timestamp.proto";
 import "google/protobuf/wrappers.proto";
 
-option go_package = "csi";
+option go_package = 
+  "github.com/container-storage-interface/spec/lib/go/csi";
 
 extend google.protobuf.EnumOptions {
   // Indicates that this enum is OPTIONAL and part of an experimental

go.mod

@@ -3,14 +3,14 @@ module github.com/container-storage-interface/spec
 go 1.18
 
 require (
-	github.com/golang/protobuf v1.5.3
 	google.golang.org/grpc v1.57.0
+	google.golang.org/protobuf v1.32.0
 )
 
 require (
+	github.com/golang/protobuf v1.5.3 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
 )

go.sum

@@ -16,5 +16,5 @@ google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
 google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=

lib/go/Makefile

@@ -14,14 +14,19 @@ GOPATH := $(word 1,$(subst :, ,$(GOPATH)))
 endif
 export GOPATH
 
+GOBIN := $(shell go env GOBIN)
+ifeq (,$(strip $(GOBIN)))
+GOBIN := $(GOPATH)/bin
+endif
+
 
 ########################################################################
 ##                             PROTOC                                 ##
 ########################################################################
 
 # Only set PROTOC_VER if it has an empty value.
 ifeq (,$(strip $(PROTOC_VER)))
-PROTOC_VER := 3.9.1
+PROTOC_VER := 25.2
 endif
 
 PROTOC_OS := $(shell uname -s)
@@ -32,95 +37,60 @@ endif
 PROTOC_ARCH := $(shell uname -m)
 ifeq (i386,$(PROTOC_ARCH))
 PROTOC_ARCH := x86_32
+else ifeq (arm64,$(PROTOC_ARCH))
+PROTOC_ARCH := aarch_64
 endif
 
-PROTOC := ./protoc
 PROTOC_ZIP := protoc-$(PROTOC_VER)-$(PROTOC_OS)-$(PROTOC_ARCH).zip
-PROTOC_URL := https://github.com/google/protobuf/releases/download/v$(PROTOC_VER)/$(PROTOC_ZIP)
+PROTOC_URL := https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VER)/$(PROTOC_ZIP)
 PROTOC_TMP_DIR := .protoc
-PROTOC_TMP_BIN := $(PROTOC_TMP_DIR)/bin/protoc
+PROTOC := $(PROTOC_TMP_DIR)/bin/protoc
+
+$(GOBIN)/protoc-gen-go: ../../go.mod
+	go install google.golang.org/protobuf/cmd/protoc-gen-go
+$(GOBIN)/protoc-gen-go-grpc:
+	go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.3.0
 
 $(PROTOC):
 	-mkdir -p "$(PROTOC_TMP_DIR)" && \
 	  curl -L $(PROTOC_URL) -o "$(PROTOC_TMP_DIR)/$(PROTOC_ZIP)" && \
 	  unzip "$(PROTOC_TMP_DIR)/$(PROTOC_ZIP)" -d "$(PROTOC_TMP_DIR)" && \
-	  chmod 0755 "$(PROTOC_TMP_BIN)" && \
-	  cp -f "$(PROTOC_TMP_BIN)" "$@"
+	  chmod 0755 "$@"
 	stat "$@" > /dev/null 2>&1
 
-
-########################################################################
-##                          PROTOC-GEN-GO                             ##
-########################################################################
-
-# This is the recipe for getting and installing the go plug-in
-# for protoc
-PROTOC_GEN_GO_PKG := github.com/golang/protobuf/protoc-gen-go
-PROTOC_GEN_GO := protoc-gen-go
-$(PROTOC_GEN_GO): PROTOBUF_PKG := $(dir $(PROTOC_GEN_GO_PKG))
-$(PROTOC_GEN_GO): PROTOBUF_VERSION := v1.3.2
-$(PROTOC_GEN_GO):
-	mkdir -p $(dir $(GOPATH)/src/$(PROTOBUF_PKG))
-	test -d $(GOPATH)/src/$(PROTOBUF_PKG)/.git || git clone https://$(PROTOBUF_PKG) $(GOPATH)/src/$(PROTOBUF_PKG)
-	(cd $(GOPATH)/src/$(PROTOBUF_PKG) && \
-		(test "$$(git describe --tags | head -1)" = "$(PROTOBUF_VERSION)" || \
-			(git fetch && git checkout tags/$(PROTOBUF_VERSION))))
-	(cd $(GOPATH)/src/$(PROTOBUF_PKG) && go get -v -d $$(go list -f '{{ .ImportPath }}' ./...)) && \
-	go build -o "$@" $(PROTOC_GEN_GO_PKG)
-
+PROTOC_ALL := $(GOBIN)/protoc-gen-go $(GOBIN)/protoc-gen-go-grpc $(PROTOC)
 
 ########################################################################
 ##                              PATH                                  ##
 ########################################################################
 
-# Update PATH with the current directory. This enables the protoc
-# binary to discover the protoc-gen-go binary, built inside this
-# directory.
-export PATH := $(shell pwd):$(PATH)
+# Update PATH with GOBIN. This enables the protoc binary to discover
+# the protoc-gen-go binary
+export PATH := $(GOBIN):$(PATH)
 
 
 ########################################################################
 ##                              BUILD                                 ##
 ########################################################################
 CSI_PROTO := ../../csi.proto
-CSI_PKG_ROOT := github.com/container-storage-interface/spec
-CSI_PKG_SUB := $(shell cat $(CSI_PROTO) | sed -n -e 's/^package.\([^;]*\).v[0-9]\+;$$/\1/p'|tr '.' '/')
-CSI_BUILD := $(CSI_PKG_SUB)/.build
+CSI_PKG_SUB := csi
 CSI_GO := $(CSI_PKG_SUB)/csi.pb.go
-CSI_GO_TMP := $(CSI_BUILD)/$(CSI_PKG_ROOT)/csi.pb.go
-
-# This recipe generates the go language bindings to a temp area.
-$(CSI_GO_TMP): HERE := $(shell pwd)
-$(CSI_GO_TMP): PTYPES_PKG := github.com/golang/protobuf/ptypes
-$(CSI_GO_TMP): GO_OUT := plugins=grpc
-$(CSI_GO_TMP): GO_OUT := $(GO_OUT),Mgoogle/protobuf/descriptor.proto=github.com/golang/protobuf/protoc-gen-go/descriptor
-$(CSI_GO_TMP): GO_OUT := $(GO_OUT),Mgoogle/protobuf/wrappers.proto=$(PTYPES_PKG)/wrappers
-$(CSI_GO_TMP): GO_OUT := $(GO_OUT):"$(HERE)/$(CSI_BUILD)"
-$(CSI_GO_TMP): INCLUDE := -I$(GOPATH)/src -I$(HERE)/$(PROTOC_TMP_DIR)/include
-$(CSI_GO_TMP): $(CSI_PROTO) | $(PROTOC) $(PROTOC_GEN_GO)
-	@mkdir -p "$(@D)"
-	(cd "$(GOPATH)/src" && \
-		$(HERE)/$(PROTOC) $(INCLUDE) --go_out=$(GO_OUT) "$(CSI_PKG_ROOT)/$(<F)")
-
-# The temp language bindings are compared to the ones that are
-# versioned. If they are different then it means the language
-# bindings were not updated prior to being committed.
-$(CSI_GO): $(CSI_GO_TMP)
-ifeq (true,$(GITHUB_ACTIONS))
-	diff "$@" "$?"
-else
-	@mkdir -p "$(@D)"
-	diff "$@" "$?" > /dev/null 2>&1 || cp -f "$?" "$@"
-endif
+CSI_GRPC := $(CSI_PKG_SUB)/csi_grpc.pb.go
 
+# This recipe generates the go language bindings
+$(CSI_GO) $(CSI_GRPC): $(CSI_PROTO) $(PROTOC_ALL)
+	@mkdir -p "$(@D)"
+	$(PROTOC) -I../.. --go-grpc_out=$(CSI_PKG_SUB) --go_out=$(CSI_PKG_SUB) \
+		--go_opt=paths=source_relative --go-grpc_opt=paths=source_relative \
+		"$(<F)"
 
-build: $(CSI_GO)
+build: $(CSI_GO) $(CSI_GRPC)
 
 clean:
 	go clean -i ./...
-	rm -rf "$(CSI_GO)" "$(CSI_BUILD)"
+	rm -rf "$(CSI_PKG_SUB)"
 
 clobber: clean
-	rm -fr "$(PROTOC)" "$(PROTOC_GEN_GO)" "$(CSI_PKG_SUB)"
+	rm -fr "$(PROTOC_TMP_DIR)"
 
 .PHONY: clean clobber