feat(engine): provide DLE logs in UI (#378)

- create a new sub-route (`/admin`) and endpoint (`/instance/logs`) to serve container logs
- create a middleware to authorize admin (in the future) users. Authorization is not enabled yet.
- support one-time tokens to authorize web-socket connections (`/admin/ws-auth`)
- enable Web-socket connections 
- collect logs from the DLE container and de-multiplex log entries
- encode log entries and send them to the UI client side
- create a prototype of the UI client side to display log entries

Author: agneum

engine/cmd/database-lab/main.go

@@ -32,6 +32,7 @@ import (
 	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/cont"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/srv"
+	"gitlab.com/postgres-ai/database-lab/v3/internal/srv/ws"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/telemetry"
 	"gitlab.com/postgres-ai/database-lab/v3/pkg/config"
 	"gitlab.com/postgres-ai/database-lab/v3/pkg/config/global"
@@ -125,6 +126,13 @@ func main() {
 		log.Errf(errors.WithMessage(err, `error in the "provision" section of the config`).Error())
 	}
 
+	tokenHolder, err := ws.NewTokenKeeper()
+	if err != nil {
+		log.Errf(errors.WithMessage(err, `failed to init WebSockets Token Manager`).Error())
+	}
+
+	go tokenHolder.RunCleaningUp(ctx)
+
 	observingChan := make(chan string, 1)
 
 	emergencyShutdown := func() {
@@ -154,7 +162,8 @@ func main() {
 	})
 
 	embeddedUI := embeddedui.New(cfg.EmbeddedUI, engProps, runner, docker)
-	server := srv.NewServer(&cfg.Server, &cfg.Global, engProps, docker, cloningSvc, provisioner, retrievalSvc, platformSvc, obs, est, pm, tm)
+	server := srv.NewServer(&cfg.Server, &cfg.Global, engProps, docker, cloningSvc, provisioner, retrievalSvc, platformSvc,
+		obs, est, pm, tm, tokenHolder, embeddedUI)
 	shutdownCh := setShutdownListener()
 
 	go setReloadListener(ctx, provisioner, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server)

engine/go.mod

@@ -4,6 +4,7 @@ go 1.17
 
 require (
 	github.com/AlekSi/pointer v1.1.0
+	github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26
 	github.com/araddon/dateparse v0.0.0-20210207001429-0eec95c9db7e
 	github.com/aws/aws-sdk-go v1.33.8
 	github.com/docker/cli v20.10.13+incompatible
@@ -27,6 +28,7 @@ require (
 	github.com/testcontainers/testcontainers-go v0.12.0
 	github.com/urfave/cli/v2 v2.1.1
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
+	golang.org/x/mod v0.5.1
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -42,6 +44,7 @@ require (
 	github.com/docker/distribution v2.8.0+incompatible // indirect
 	github.com/go-ole/go-ole v1.2.5 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
@@ -66,7 +69,6 @@ require (
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/stretchr/objx v0.2.0 // indirect
-	golang.org/x/mod v0.5.1 // indirect
 	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
 	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
 	golang.org/x/text v0.3.7 // indirect

engine/go.sum

@@ -78,6 +78,8 @@ github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDO
 github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
 github.com/agneum/distribution v2.8.1-0.20220215080619-a3a6b67e8f8d+incompatible h1:vAvi61+QBpyxUHPbRNJTV9caJv2rWzzInrjCh9nFtz4=
 github.com/agneum/distribution v2.8.1-0.20220215080619-a3a6b67e8f8d+incompatible/go.mod h1:f1dYlUrYWo0taNTDy5JzZMM5RUfE5BuL/Ot+4vxuFZs=
+github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:pzStYMLAXM7CNQjS/Wn+zK9MUxDhSUNfVvnHsyQyjs0=
+github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:ilK+u7u1HoqaDk0mjhh27QJB7PyWMreGffEvOCoEKiY=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -267,6 +269,8 @@ github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx
 github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
+github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

engine/internal/embeddedui/embedded_ui.go

@@ -8,6 +8,8 @@ package embeddedui
 import (
 	"context"
 	"fmt"
+	"net"
+	"net/url"
 	"strconv"
 	"time"
 
@@ -151,7 +153,7 @@ func (ui *UIManager) Restart(ctx context.Context) error {
 	return nil
 }
 
-// Stop removes a embedded UI container.
+// Stop removes an embedded UI container.
 func (ui *UIManager) Stop(ctx context.Context) {
 	tools.RemoveContainer(ctx, ui.docker, getEmbeddedUIName(ui.engProps.InstanceID), cont.StopTimeout)
 }
@@ -170,3 +172,25 @@ func reportLaunching(cfg Config) {
 
 	log.Msg(fmt.Sprintf("Embedded UI has started successfully on %s:%d.", host, cfg.Port))
 }
+
+// IsEnabled reports if the embedded UI container is enabled.
+func (ui *UIManager) IsEnabled() bool {
+	return ui.cfg.Enabled
+}
+
+// GetHost provides a host name from the UI container configuration.
+func (ui *UIManager) GetHost() string {
+	return ui.cfg.Host
+}
+
+// OriginURL reports the URL of the embedded UI container.
+func (ui *UIManager) OriginURL() string {
+	uiHost := ui.cfg.Host
+	if uiHost == "" || uiHost == engine.DefaultListenerHost {
+		uiHost = engine.Localhost
+	}
+
+	localURL := url.URL{Scheme: engine.HTTPScheme, Host: net.JoinHostPort(uiHost, strconv.Itoa(ui.cfg.Port))}
+
+	return localURL.String()
+}

engine/internal/embeddedui/embedded_ui_test.go

@@ -0,0 +1,59 @@
+package embeddedui
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUIManagerOriginURL(t *testing.T) {
+	testCases := []struct {
+		host   string
+		port   int
+		result string
+	}{
+		{
+			host:   "example.com",
+			port:   2345,
+			result: "http://example.com:2345",
+		},
+		{
+			host:   "example.com",
+			result: "http://example.com:0",
+		},
+		{
+			host:   "container_host",
+			port:   2345,
+			result: "http://container_host:2345",
+		},
+		{
+			host:   "172.168.1.1",
+			port:   2345,
+			result: "http://172.168.1.1:2345",
+		},
+		{
+			host:   "0.0.0.0",
+			port:   2345,
+			result: "http://127.0.0.1:2345",
+		},
+		{
+			host:   "",
+			port:   2345,
+			result: "http://127.0.0.1:2345",
+		},
+	}
+
+	for _, tc := range testCases {
+		uiManager := UIManager{
+			cfg: Config{
+				Enabled: true,
+				Host:    tc.host,
+				Port:    tc.port,
+			},
+		}
+
+		require.True(t, uiManager.IsEnabled())
+		require.Equal(t, tc.result, uiManager.OriginURL())
+
+	}
+}

engine/internal/platform/platform.go

@@ -7,6 +7,7 @@ package platform
 
 import (
 	"context"
+	"net/url"
 
 	"github.com/pkg/errors"
 
@@ -99,3 +100,15 @@ func (s *Service) IsPersonalTokenEnabled() bool {
 func (s *Service) isAllowedOrganization(organizationID uint) bool {
 	return organizationID != 0 && organizationID == s.organizationID
 }
+
+// OriginURL reports the origin Platform hostname.
+func (s *Service) OriginURL() string {
+	parsedURL, err := url.Parse(s.cfg.URL)
+	if err != nil {
+		log.Dbg("Cannot parse Platform URL")
+	}
+
+	platformURL := url.URL{Scheme: parsedURL.Scheme, Host: parsedURL.Host}
+
+	return platformURL.String()
+}

engine/internal/platform/platform_test.go

@@ -26,3 +26,13 @@ func TestIfOrganizationIsAllowed(t *testing.T) {
 	assert.Equal(t, s.isAllowedOrganization(0), false)
 	assert.Equal(t, s.isAllowedOrganization(1), true)
 }
+
+func TestOriginURL(t *testing.T) {
+	s := Service{
+		cfg: Config{
+			URL: "https://example.com:2345/api/path",
+		},
+	}
+
+	assert.Equal(t, "https://example.com:2345", s.OriginURL())
+}

engine/internal/retrieval/engine/postgres/logical/dump.go

@@ -125,7 +125,7 @@ type Connection struct {
 	Port     int    `yaml:"port"`
 	DBName   string `yaml:"dbname"`
 	Username string `yaml:"username"`
-	Password string `yaml:"password"`
+	Password string `yaml:"password" json:"-"`
 }
 
 // ImmediateRestore contains options for direct data restore without saving the dump file on disk.

engine/internal/srv/config/config.go

@@ -7,7 +7,7 @@ package config
 
 // Config provides configuration for an HTTP server of the Database Lab.
 type Config struct {
-	VerificationToken string `yaml:"verificationToken"`
+	VerificationToken string `yaml:"verificationToken" json:"-"`
 	Host              string `yaml:"host"`
 	Port              uint   `yaml:"port"`
 }

engine/internal/srv/mw/auth.go

@@ -12,11 +12,15 @@ import (
 
 	"gitlab.com/postgres-ai/database-lab/v3/internal/platform"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/srv/api"
+	"gitlab.com/postgres-ai/database-lab/v3/internal/srv/ws"
 )
 
-// VerificationTokenHeader defines a verification token name that should be passed in request headers.
+// VerificationTokenHeader defines the verification token name that should be passed in request headers.
 const VerificationTokenHeader = "Verification-Token"
 
+// wsTokenKey defines the name of web-sockets token parameter that should be passed in query string.
+const wsTokenKey = "token"
+
 // Auth defines an authorization middleware of the Database Lab HTTP server.
 type Auth struct {
 	verificationToken     string
@@ -41,6 +45,20 @@ func (a *Auth) Authorized(h http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
+// AdminMW checks if the user has permission to access to admin sub-route.
+// TODO: check admin permissions.
+func (a *Auth) AdminMW(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		token := r.Header.Get(VerificationTokenHeader)
+		if !a.isAccessAllowed(r.Context(), token) {
+			api.SendUnauthorizedError(w, r)
+			return
+		}
+
+		h.ServeHTTP(w, r)
+	})
+}
+
 func (a *Auth) isAccessAllowed(ctx context.Context, token string) bool {
 	if a.verificationToken == "" {
 		return true
@@ -57,3 +75,29 @@ func (a *Auth) isAccessAllowed(ctx context.Context, token string) bool {
 
 	return false
 }
+
+// WebSocketsMW checks if the user has a token to access to web-socket handlers.
+func (a *Auth) WebSocketsMW(holder *ws.TokenKeeper, h http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		authToken := r.URL.Query().Get(wsTokenKey)
+		if authToken == "" {
+			api.SendUnauthorizedError(w, r)
+
+			return
+		}
+
+		if err := holder.ValidateToken(authToken); err != nil {
+			api.SendUnauthorizedError(w, r)
+
+			return
+		}
+
+		if err := holder.ExpendToken(authToken); err != nil {
+			api.SendUnauthorizedError(w, r)
+
+			return
+		}
+
+		h(w, r)
+	}
+}