all: support Go 1.16 in the Go Playground

This makes a number of changes in order to support modules being on by
default in Go 1.16.

- Change default Go version in Dockerfile to go1.16
- Change default Go version in sandbox/Dockerfile to go1.16
- Build playground web server with GO_VERSION instead of
GO_BOOTSTRAP_VERSION (for golang/go#40319)
- Drop code related to non-module codepaths. This should keep us
future-proof for later versions of Go.

This works for me testing locally with gVisor's runsc.

Fixes golang/go#44389
Fixes golang/go#40319

Change-Id: Ib99510e662658b75c401567314c3e8ed612002a0
Reviewed-on: https://go-review.googlesource.com/c/playground/+/295649
Trust: Alexander Rakoczy <alex@golang.org>
Run-TryBot: Alexander Rakoczy <alex@golang.org>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>

Author: toothrot

Dockerfile

@@ -2,48 +2,74 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
 
-ARG GO_VERSION=go1.14.1
+# The playground builds Go from a bootstrap version for two reasons:
+# - The playground deployment is triggered before the artifacts are
+#   published for the latest version of Go.
+# - The sandbox builds the Go standard library with a custom build
+#   flag called faketime.
 
-FROM debian:buster AS go-faketime
+# GO_VERSION is provided by Cloud Build, and is set to the latest
+# version of Go. See the configuration in the deploy directory.
+ARG GO_VERSION=go1.16
+
+############################################################################
+# Build Go at GO_VERSION, and build faketime standard library.
+FROM debian:buster AS build-go
 LABEL maintainer="golang-dev@googlegroups.com"
 
 ENV BUILD_DEPS 'curl git gcc patch libc6-dev ca-certificates'
 RUN apt-get update && apt-get install -y ${BUILD_DEPS} --no-install-recommends
 
 ENV GOPATH /go
-ENV PATH /usr/local/go/bin:$GOPATH/bin:$PATH
-ENV GO_BOOTSTRAP_VERSION go1.14.1
+ENV GOROOT_BOOTSTRAP=/usr/local/go-bootstrap
+ENV GO_BOOTSTRAP_VERSION go1.16
 ARG GO_VERSION
 ENV GO_VERSION ${GO_VERSION}
 
-# Get a version of Go for building the playground
+# Get a bootstrap version of Go for building GO_VERSION. At the time
+# of this Dockerfile being built, GO_VERSION's artifacts may not yet
+# be published.
 RUN curl -sSL https://dl.google.com/go/$GO_BOOTSTRAP_VERSION.linux-amd64.tar.gz -o /tmp/go.tar.gz
 RUN curl -sSL https://dl.google.com/go/$GO_BOOTSTRAP_VERSION.linux-amd64.tar.gz.sha256 -o /tmp/go.tar.gz.sha256
 RUN echo "$(cat /tmp/go.tar.gz.sha256) /tmp/go.tar.gz" | sha256sum -c -
-RUN mkdir -p /usr/local/go
-RUN tar --strip=1 -C /usr/local/go -vxzf /tmp/go.tar.gz
+RUN mkdir -p $GOROOT_BOOTSTRAP
+RUN tar --strip=1 -C $GOROOT_BOOTSTRAP -vxzf /tmp/go.tar.gz
 
 RUN mkdir /gocache
 ENV GOCACHE /gocache
 ENV GO111MODULE on
 ENV GOPROXY=https://proxy.golang.org
 
-# Compile Go at target sandbox version and install standard library with --tags=faketime.
+# Compile Go at target version in /usr/local/go.
 WORKDIR /usr/local
-RUN git clone https://go.googlesource.com/go go-faketime && cd go-faketime && git reset --hard $GO_VERSION
-WORKDIR /usr/local/go-faketime/src
+RUN git clone https://go.googlesource.com/go go && cd go && git reset --hard $GO_VERSION
+WORKDIR /usr/local/go/src
 RUN ./make.bash
+
+# Make a copy in /usr/local/go-faketime where the standard library
+# is installed with -tags=faketime.
+RUN cp -R /usr/local/go /usr/local/go-faketime
 ENV GOROOT /usr/local/go-faketime
+WORKDIR /usr/local/go-faketime/src
 RUN ../bin/go install --tags=faketime std
 
-FROM golang:1.14 as build-playground
+############################################################################
+# Build playground web server.
+FROM debian:buster as build-playground
 
+RUN apt-get update && apt-get install -y ca-certificates --no-install-recommends
+# Build playground from Go built at GO_VERSION.
+COPY --from=build-go /usr/local/go /usr/local/go
+ENV GOROOT /usr/local/go
+ENV GOPATH /go
+ENV PATH="/go/bin:/usr/local/go/bin:${PATH}"
+# Cache dependencies for efficient Dockerfile building.
 COPY go.mod /go/src/playground/go.mod
 COPY go.sum /go/src/playground/go.sum
 WORKDIR /go/src/playground
 RUN go mod download
 
-# Add and compile playground daemon
+# Add and compile playground daemon.
 COPY . /go/src/playground/
 RUN go install
 
@@ -53,43 +79,19 @@ FROM debian:buster
 
 RUN apt-get update && apt-get install -y git ca-certificates --no-install-recommends
 
-COPY --from=go-faketime /usr/local/go-faketime /usr/local/go-faketime
+COPY --from=build-go /usr/local/go-faketime /usr/local/go-faketime
 
 ARG GO_VERSION
 ENV GO_VERSION ${GO_VERSION}
 ENV GOPATH /go
-ENV PATH /usr/local/go-faketime/bin:$GOPATH/bin:$PATH
-
-# Add and compile tour packages
-RUN go get \
-    golang.org/x/tour/pic \
-    golang.org/x/tour/reader \
-    golang.org/x/tour/tree \
-    golang.org/x/tour/wc \
-    golang.org/x/talks/content/2016/applicative/google && \
-    rm -rf $GOPATH/src/golang.org/x/tour/.git && \
-    rm -rf $GOPATH/src/golang.org/x/talks/.git
-
-# Add tour packages under their old import paths (so old snippets still work)
-RUN mkdir -p $GOPATH/src/code.google.com/p/go-tour && \
-    cp -R $GOPATH/src/golang.org/x/tour/* $GOPATH/src/code.google.com/p/go-tour/ && \
-    sed -i 's_// import_// public import_' $(find $GOPATH/src/code.google.com/p/go-tour/ -name *.go) && \
-    go install \
-    code.google.com/p/go-tour/pic \
-    code.google.com/p/go-tour/reader \
-    code.google.com/p/go-tour/tree \
-    code.google.com/p/go-tour/wc
+ENV PATH="/go/bin:/usr/local/go-faketime/bin:${PATH}"
 
 RUN mkdir /app
-
 COPY --from=build-playground /go/bin/playground /app
 COPY edit.html /app
 COPY static /app/static
 COPY examples /app/examples
 WORKDIR /app
 
-# Whether we allow third-party imports via proxy.golang.org:
-ENV ALLOW_PLAY_MODULE_DOWNLOADS true
-
 EXPOSE 8080
 ENTRYPOINT ["/app/playground"]

sandbox.go

@@ -396,8 +396,6 @@ type buildResult struct {
 	goPath string
 	// exePath is the path to the built binary.
 	exePath string
-	// useModules is true if the binary was built with module support.
-	useModules bool
 	// testParam is set if tests should be run when running the binary.
 	testParam string
 	// errorMessage is an error message string to be returned to the user.
@@ -408,7 +406,7 @@ type buildResult struct {
 
 // cleanup cleans up the temporary goPath created when building with module support.
 func (b *buildResult) cleanup() error {
-	if b.useModules && b.goPath != "" {
+	if b.goPath != "" {
 		return os.RemoveAll(b.goPath)
 	}
 	return nil
@@ -435,8 +433,7 @@ func sandboxBuild(ctx context.Context, tmpDir string, in []byte, vet bool) (*bui
 		}
 	}
 
-	br.useModules = allowModuleDownloads(files)
-	if !files.Contains("go.mod") && br.useModules {
+	if !files.Contains("go.mod") {
 		files.AddFile("go.mod", []byte("module play\n"))
 	}
 
@@ -471,20 +468,16 @@ func sandboxBuild(ctx context.Context, tmpDir string, in []byte, vet bool) (*bui
 	cmd.Dir = tmpDir
 	cmd.Env = []string{"GOOS=linux", "GOARCH=amd64", "GOROOT=/usr/local/go-faketime"}
 	cmd.Env = append(cmd.Env, "GOCACHE="+goCache)
-	if br.useModules {
-		// Create a GOPATH just for modules to be downloaded
-		// into GOPATH/pkg/mod.
-		cmd.Args = append(cmd.Args, "-modcacherw")
-		br.goPath, err = ioutil.TempDir("", "gopath")
-		if err != nil {
-			log.Printf("error creating temp directory: %v", err)
-			return nil, fmt.Errorf("error creating temp directory: %v", err)
-		}
-		cmd.Env = append(cmd.Env, "GO111MODULE=on", "GOPROXY="+playgroundGoproxy())
-	} else {
-		br.goPath = os.Getenv("GOPATH")              // contains old code.google.com/p/go-tour, etc
-		cmd.Env = append(cmd.Env, "GO111MODULE=off") // in case it becomes on by default later
+	// Create a GOPATH just for modules to be downloaded
+	// into GOPATH/pkg/mod.
+	cmd.Args = append(cmd.Args, "-modcacherw")
+	cmd.Args = append(cmd.Args, "-mod=mod")
+	br.goPath, err = ioutil.TempDir("", "gopath")
+	if err != nil {
+		log.Printf("error creating temp directory: %v", err)
+		return nil, fmt.Errorf("error creating temp directory: %v", err)
 	}
+	cmd.Env = append(cmd.Env, "GO111MODULE=on", "GOPROXY="+playgroundGoproxy())
 	cmd.Args = append(cmd.Args, buildPkgArg)
 	cmd.Env = append(cmd.Env, "GOPATH="+br.goPath)
 	out := &bytes.Buffer{}
@@ -521,7 +514,7 @@ func sandboxBuild(ctx context.Context, tmpDir string, in []byte, vet bool) (*bui
 	}
 	if vet {
 		// TODO: do this concurrently with the execution to reduce latency.
-		br.vetOut, err = vetCheckInDir(tmpDir, br.goPath, br.useModules)
+		br.vetOut, err = vetCheckInDir(tmpDir, br.goPath)
 		if err != nil {
 			return nil, fmt.Errorf("running vet: %v", err)
 		}
@@ -567,21 +560,6 @@ func sandboxRun(ctx context.Context, exePath string, testParam string) (sandboxt
 	return execRes, nil
 }
 
-// allowModuleDownloads reports whether the code snippet in src should be allowed
-// to download modules.
-func allowModuleDownloads(files *fileSet) bool {
-	if files.Num() == 1 && bytes.Contains(files.Data(progName), []byte(`"code.google.com/p/go-tour/`)) {
-		// This domain doesn't exist anymore but we want old snippets using
-		// these packages to still run, so the Dockerfile adds these packages
-		// at this name in $GOPATH. Any snippets using this old name wouldn't
-		// have expected (or been able to use) third-party packages anyway,
-		// so disabling modules and proxy fetches is acceptable.
-		return false
-	}
-	v, _ := strconv.ParseBool(os.Getenv("ALLOW_PLAY_MODULE_DOWNLOADS"))
-	return v
-}
-
 // playgroundGoproxy returns the GOPROXY environment config the playground should use.
 // It is fetched from the environment variable PLAY_GOPROXY. A missing or empty
 // value for PLAY_GOPROXY returns the default value of https://proxy.golang.org.

sandbox/Dockerfile

@@ -4,7 +4,7 @@
 # environment so the play-sandbox server can connect to the host's
 # docker daemon, which has the gvisor "runsc" runtime available.
 
-FROM golang:1.14 AS build
+FROM golang:1.16 AS build
 
 COPY go.mod /go/src/playground/go.mod
 COPY go.sum /go/src/playground/go.sum

server_test.go

@@ -299,37 +299,6 @@ func TestCommandHandler(t *testing.T) {
 	}
 }
 
-func TestAllowModuleDownloads(t *testing.T) {
-	const envKey = "ALLOW_PLAY_MODULE_DOWNLOADS"
-	defer func(old string) { os.Setenv(envKey, old) }(os.Getenv(envKey))
-
-	tests := []struct {
-		src  string
-		env  string
-		want bool
-	}{
-		{src: "package main", want: true},
-		{src: "package main", env: "false", want: false},
-		{src: `import "code.google.com/p/go-tour/"`, want: false},
-	}
-	for i, tt := range tests {
-		if tt.env != "" {
-			os.Setenv(envKey, tt.env)
-		} else {
-			os.Setenv(envKey, "true")
-		}
-		files, err := splitFiles([]byte(tt.src))
-		if err != nil {
-			t.Errorf("%d. splitFiles = %v", i, err)
-			continue
-		}
-		got := allowModuleDownloads(files)
-		if got != tt.want {
-			t.Errorf("%d. allow = %v; want %v; files:\n%s", i, got, tt.want, filesAsString(files))
-		}
-	}
-}
-
 func TestPlaygroundGoproxy(t *testing.T) {
 	const envKey = "PLAY_GOPROXY"
 	defer os.Setenv(envKey, os.Getenv(envKey))

tests.go

@@ -41,10 +41,6 @@ func (s *server) runTests() {
 		stdlog.Fatal(err)
 	}
 
-	// Enable module downloads for testing:
-	defer func(old string) { os.Setenv("ALLOW_PLAY_MODULE_DOWNLOADS", old) }(os.Getenv("ALLOW_PLAY_MODULE_DOWNLOADS"))
-	os.Setenv("ALLOW_PLAY_MODULE_DOWNLOADS", "true")
-
 	failed := false
 	for i, t := range tests {
 		stdlog.Printf("testing case %d (%q)...\n", i, t.name)
@@ -178,30 +174,6 @@ func main() {
 	}
 }
 `, want: "timers fired as expected"},
-
-	{
-		name: "old_tour_pkgs_in_gopath",
-		prog: `
-package main
-
-import (
-	"code.google.com/p/go-tour/pic"
-	"code.google.com/p/go-tour/reader"
-	"code.google.com/p/go-tour/tree"
-	"code.google.com/p/go-tour/wc"
-)
-
-var (
-	_ = pic.Show
-	_ = reader.Validate
-	_ = tree.New
-	_ = wc.Test
-)
-
-func main() {
-	println("ok")
-}
-`, want: "ok"},
 	{
 		name: "must_be_package_main",
 		prog: `

vet.go

@@ -21,7 +21,7 @@ import (
 // Deprecated: this is the handler for the legacy /vet endpoint; use
 // the /compile (compileAndRun) handler instead with the WithVet
 // boolean set. This code path doesn't support modules and only exists
-// as a temporary compatiblity bridge to older javascript clients.
+// as a temporary compatibility bridge to older javascript clients.
 func vetCheck(ctx context.Context, req *request) (*response, error) {
 	tmpDir, err := ioutil.TempDir("", "vet")
 	if err != nil {
@@ -33,8 +33,7 @@ func vetCheck(ctx context.Context, req *request) (*response, error) {
 	if err := ioutil.WriteFile(in, []byte(req.Body), 0400); err != nil {
 		return nil, fmt.Errorf("error creating temp file %q: %v", in, err)
 	}
-	const useModules = false // legacy handler; no modules (see func comment)
-	vetOutput, err := vetCheckInDir(tmpDir, os.Getenv("GOPATH"), useModules)
+	vetOutput, err := vetCheckInDir(tmpDir, os.Getenv("GOPATH"))
 	if err != nil {
 		// This is about errors running vet, not vet returning output.
 		return nil, err
@@ -43,27 +42,21 @@ func vetCheck(ctx context.Context, req *request) (*response, error) {
 }
 
 // vetCheckInDir runs go vet in the provided directory, using the
-// provided GOPATH value, and whether modules are enabled. The
-// returned error is only about whether go vet was able to run, not
-// whether vet reported problem. The returned value is ("", nil) if
-// vet successfully found nothing, and (non-empty, nil) if vet ran and
-// found issues.
-func vetCheckInDir(dir, goPath string, modules bool) (output string, execErr error) {
+// provided GOPATH value. The returned error is only about whether
+// go vet was able to run, not whether vet reported problem. The
+// returned value is ("", nil) if vet successfully found nothing,
+// and (non-empty, nil) if vet ran and found issues.
+func vetCheckInDir(dir, goPath string) (output string, execErr error) {
 	cmd := exec.Command("go", "vet")
-	if !modules {
-		cmd.Args = append(cmd.Args, progName)
-	}
 	cmd.Dir = dir
 	// Linux go binary is not built with CGO_ENABLED=0.
 	// Prevent vet to compile packages in cgo mode.
 	// See #26307.
 	cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOPATH="+goPath)
-	if modules {
-		cmd.Env = append(cmd.Env,
-			"GO111MODULE=on",
-			"GOPROXY="+playgroundGoproxy(),
-		)
-	}
+	cmd.Env = append(cmd.Env,
+		"GO111MODULE=on",
+		"GOPROXY="+playgroundGoproxy(),
+	)
 	out, err := cmd.CombinedOutput()
 	if err == nil {
 		return "", nil