common, docker, test: Send outgoing email via SMTP2Go

Sending email using a reputable commercial service, rather than
having to maintain our own mail servers, seems like the way to go.

Author: justinclift

common/config.go

@@ -150,10 +150,18 @@ func ReadConfig() error {
 		Conf.Event.EmailQueueProcessingDelay = 10
 	}
 
-	// Warn if the email queue directory isn't set in the config file
-	if Conf.Event.EmailQueueDir == "" {
-		log.Printf("WARN: Email queue directory isn't set in the config file. Defaulting to /tmp.")
-		Conf.Event.EmailQueueDir = "/tmp"
+	// If an SMTP2Go environment variable is already set, don't mess with it.
+	tempString = os.Getenv("SMTP2GO_API_KEY")
+	if tempString != "" {
+		Conf.Event.Smtp2GoKey = tempString
+	} else {
+		// If this is a production environment, and the SMTP2Go env variable wasn't set, we'd better
+		// warn when the key isn't in the config file either
+		if Conf.Event.Smtp2GoKey == "" && Conf.Environment.Environment == "production" {
+			log.Printf("WARN: SMTP2Go API key isn't set in the config file.  Event emails won't be sent.")
+		} else {
+			os.Setenv("SMTP2GO_API_KEY", Conf.Event.Smtp2GoKey)
+		}
 	}
 
 	// Check cache directory exists

common/cypress.go

@@ -3,10 +3,12 @@ package common
 /* Shared backend code we need that is specific to testing with Cypress */
 
 import (
+	"fmt"
 	"log"
 	"net/http"
 	"os"
 	"path"
+	"strings"
 	"time"
 )
 
@@ -27,6 +29,14 @@ func CypressSeed(w http.ResponseWriter, r *http.Request) {
 	// Switch to the default user
 	Conf.Environment.UserOverride = "default"
 
+	// Change the email address of the default user to match the local server
+	serverName := strings.Split(Conf.Web.ServerName, ":")
+	err := SetUserPreferences("default", 10, "Default system user", fmt.Sprintf("default@%s", serverName[0]))
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
 	// Add test SQLite databases
 	testDB, err := os.Open(path.Join(Conf.Web.BaseDir, "cypress", "test_data", "Assembly Election 2017.sqlite"))
 	if err != nil {
@@ -58,17 +68,17 @@ func CypressSeed(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Add some test users
-	err = AddUser("auth0first", "first", RandomString(32), "first@example.org", "First test user", "")
+	err = AddUser("auth0first", "first", RandomString(32), fmt.Sprintf("first@%s", serverName[0]), "First test user", "")
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	err = AddUser("auth0second", "second", RandomString(32), "second@example.org", "Second test user", "")
+	err = AddUser("auth0second", "second", RandomString(32), fmt.Sprintf("second@%s", serverName[0]), "Second test user", "")
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
-	err = AddUser("auth0third", "third", RandomString(32), "third@example.org", "Third test user", "")
+	err = AddUser("auth0third", "third", RandomString(32), fmt.Sprintf("third@%s", serverName[0]), "Third test user", "")
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return

common/postgresql.go

@@ -14,9 +14,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/hectane/hectane/email"
-	"github.com/hectane/hectane/queue"
+	"github.com/aquilax/truncate"
 	"github.com/jackc/pgx"
+	smtp2go "github.com/smtp2go-oss/smtp2go-go"
 	gfm "github.com/sqlitebrowser/github_flavored_markdown"
 	"golang.org/x/crypto/bcrypt"
 )
@@ -2981,18 +2981,10 @@ func SaveDBSettings(userName, dbFolder, dbName, oneLineDesc, fullDesc, defaultTa
 
 // SendEmails sends status update emails to people watching databases
 func SendEmails() {
-	// Create Hectane email queue
-	cfg := &queue.Config{
-		Directory:              Conf.Event.EmailQueueDir,
-		DisableSSLVerification: true,
-	}
-	q, err := queue.NewQueue(cfg)
-	if err != nil {
-		log.Printf("Couldn't start Hectane queue: %s", err.Error())
+	// If the SMTP2Go API key hasn't been configured, there's no use in trying to send emails
+	if Conf.Event.Smtp2GoKey == "" && os.Getenv("SMTP2GO_API_KEY") == "" {
 		return
 	}
-	log.Printf("Created Hectane email queue in '%s'.  Queue processing loop refreshes every %d seconds",
-		Conf.Event.EmailQueueDir, Conf.Event.EmailQueueProcessingDelay)
 
 	for {
 		// Retrieve unsent emails from the email_queue
@@ -3026,22 +3018,22 @@ func SendEmails() {
 
 		// Send emails
 		for _, j := range emailList {
-			e := &email.Email{
-				From:    "updates@dbhub.io",
-				To:      []string{j.Address},
-				Subject: j.Subject,
-				Text:    j.Body,
+			e := smtp2go.Email{
+				From:     "updates@dbhub.io",
+				To:       []string{j.Address},
+				Subject:  j.Subject,
+				TextBody: j.Body,
+				HtmlBody: j.Body,
 			}
-			msgs, err := e.Messages(q.Storage)
+			_, err = smtp2go.Send(&e)
 			if err != nil {
-				log.Printf("Queuing email in Hectane failed: %v", err.Error())
-				return // Abort, as we don't want to continuously resend the same emails
-			}
-			for _, m := range msgs {
-				q.Deliver(m)
+				log.Println(err)
 			}
 
-			// Mark message as sent
+			log.Printf("Email with subject '%v' sent to '%v'",
+				truncate.Truncate(j.Subject, 35, "...", truncate.PositionEnd), j.Address)
+
+			// We only attempt delivery via smtp2go once (retries are handled on their end), so mark message as sent
 			dbQuery := `
 				UPDATE email_queue
 				SET sent = true, sent_timestamp = now()
@@ -3199,7 +3191,7 @@ func StatusUpdatesLoop() {
 		// For each event, add a status update to the status_updates list for each watcher it's for
 		for id, ev := range evList {
 			// Retrieve the list of watchers for the database the event occurred on
-			dbQuery := `
+			dbQuery = `
 				SELECT user_id
 				FROM watchers
 				WHERE db_id = $1`
@@ -3228,19 +3220,26 @@ func StatusUpdatesLoop() {
 			for _, u := range users {
 				// Retrieve the current status updates list for the user
 				var eml pgx.NullString
-				dbQuery := `
+				dbQuery = `
 					SELECT user_name, email, status_updates
 					FROM users
 					WHERE user_id = $1`
 				userEvents := make(map[string][]StatusUpdateEntry)
 				var userName string
-				err := tx.QueryRow(dbQuery, u).Scan(&userName, &eml, &userEvents)
+				err = tx.QueryRow(dbQuery, u).Scan(&userName, &eml, &userEvents)
 				if err != nil {
 					log.Printf("Database query failed: %v\n", err)
 					tx.Rollback()
 					continue
 				}
 
+				// If the user generated this event themselves, skip them
+				if userName == ev.details.UserName {
+					log.Printf("User '%v' generated this event (id: %v), so not adding it to their event list",
+						userName, ev.eventID)
+					continue
+				}
+
 				// * Add the new event to the users status updates list *
 
 				// Group the status updates by database, and coalesce multiple updates for the same discussion or MR
@@ -3281,7 +3280,7 @@ func StatusUpdatesLoop() {
 				}
 				if numRows := commandTag.RowsAffected(); numRows != 1 {
 					log.Printf("Wrong number of rows affected (%v) when adding status update for database ID "+
-						"'%d' to user '%s'", numRows, ev.dbID, u)
+						"'%d' to user '%v'", numRows, ev.dbID, u)
 					tx.Rollback()
 					continue
 				}
@@ -3324,19 +3323,27 @@ func StatusUpdatesLoop() {
 					log.Printf("Unknown message type when creating email message")
 				}
 				if eml.Valid {
-					// TODO: Check if the email is username@thisserver, which indicates a non-functional email address
+					// If the email address is of the form username@this_server (which indicates a non-functional email address), then skip it
+					serverName := strings.Split(Conf.Web.ServerName, ":")
+					if strings.HasSuffix(eml.String, serverName[0]) {
+						log.Printf("Skipping email '%v' to destination '%v', as it ends in '%v'",
+							truncate.Truncate(subj, 35, "...", truncate.PositionEnd), eml.String, serverName[0])
+						continue
+					}
+
+					// Add the email to the queue
 					dbQuery = `
 						INSERT INTO email_queue (mail_to, subject, body)
 						VALUES ($1, $2, $3)`
 					commandTag, err = tx.Exec(dbQuery, eml.String, subj, msg)
 					if err != nil {
-						log.Printf("Adding status update to email queue for user '%s' failed: %v", u, err)
+						log.Printf("Adding status update to email queue for user '%v' failed: %v", u, err)
 						tx.Rollback()
 						continue
 					}
 					if numRows := commandTag.RowsAffected(); numRows != 1 {
 						log.Printf("Wrong number of rows affected (%v) when adding status update to email"+
-							"queue for user '%s'", numRows, u)
+							"queue for user '%v'", numRows, u)
 						tx.Rollback()
 						continue
 					}

common/types.go

@@ -140,8 +140,8 @@ type EnvInfo struct {
 // EventProcessingInfo hold configuration for the event processing loop
 type EventProcessingInfo struct {
 	Delay                     time.Duration `toml:"delay"`
-	EmailQueueDir             string        `toml:"email_queue_dir"`
 	EmailQueueProcessingDelay time.Duration `toml:"email_queue_processing_delay"`
+	Smtp2GoKey                string        `toml:"smtp2go_key"` // The SMTP2GO API key
 }
 
 // LicenceDir holds the path to the licence files

cypress/e2e/1-webui/user-preferences.cy.js

@@ -22,7 +22,7 @@ describe('logged-in user preferences page', () => {
 
   // Change user email address
   it('change user email address', () => {
-    cy.get('[data-cy="email"]').should('have.attr', 'value', 'default@dbhub.io')
+    cy.get('[data-cy="email"]').should('have.attr', 'value', 'default@docker-dev.dbhub.io')
     cy.get('[data-cy="email"]').type('{selectall}{backspace}').type('test@example.org').should('have.value', 'test@example.org')
     cy.get('[data-cy="updatebtn"]').click()
     cy.visit('pref')

cypress/e2e/2-api/api.cy.js

@@ -85,7 +85,7 @@ describe('api tests', () => {
         let temp = JSON.parse(response.body)
         let jsonBody = temp[Object.keys(temp)[0]]
 
-        expect(jsonBody).to.have.property('author_email', 'default@dbhub.io')
+        expect(jsonBody).to.have.property('author_email', 'default@docker-dev.dbhub.io')
         expect(jsonBody).to.have.property('author_name', 'Default system user')
         expect(jsonBody).to.have.property('committer_email', '')
         expect(jsonBody).to.have.property('committer_name', '')
@@ -344,7 +344,7 @@ describe('api tests', () => {
         // Test the "commits" structure
         let commitID = Object.keys(jsonBody.commits)
         let commitData = jsonBody.commits[commitID]
-        expect(commitData).to.have.property('author_email', 'default@dbhub.io')
+        expect(commitData).to.have.property('author_email', 'default@docker-dev.dbhub.io')
         expect(commitData).to.have.property('author_name', 'Default system user')
         expect(commitData).to.have.property('committer_email', '')
         expect(commitData).to.have.property('committer_name', '')
@@ -437,7 +437,7 @@ describe('api tests', () => {
         expect(jsonBody).to.have.property('Some release name')
         expect(jsonBody['Some release name']).to.include.keys(['commit', 'date'])
         expect(jsonBody['Some release name']).to.have.property('description', 'Some release description')
-        expect(jsonBody['Some release name']).to.have.property('email', 'default@dbhub.io')
+        expect(jsonBody['Some release name']).to.have.property('email', 'default@docker-dev.dbhub.io')
         expect(jsonBody['Some release name']).to.have.property('name', 'Default system user')
         expect(jsonBody['Some release name']).to.have.property('size', 73728)
       }
@@ -509,7 +509,7 @@ describe('api tests', () => {
         expect(jsonBody).to.have.property('Some tag name')
         expect(jsonBody['Some tag name']).to.include.keys(['commit', 'date'])
         expect(jsonBody['Some tag name']).to.have.property('description', 'Some tag description')
-        expect(jsonBody['Some tag name']).to.have.property('email', 'default@dbhub.io')
+        expect(jsonBody['Some tag name']).to.have.property('email', 'default@docker-dev.dbhub.io')
         expect(jsonBody['Some tag name']).to.have.property('name', 'Default system user')
       }
     )

docker/config.toml

@@ -15,7 +15,7 @@ user_override = "default"
 [event]
 delay = 2
 email_queue_processing_delay = 5
-email_queue_dir = "/home/dbhub/.dbhub/email_queue"
+smtp2go_key = ""
 
 [licence]
 licence_dir = "/dbhub.io/default_licences"

go.mod

@@ -31,6 +31,7 @@ require (
 
 require (
 	github.com/Freeaqingme/dkim v0.0.0-20160606200812-57418d2fbdcd // indirect
+	github.com/aquilax/truncate v1.0.0 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/go-ini/ini v1.56.0 // indirect
 	github.com/go-playground/locales v0.13.0 // indirect
@@ -64,6 +65,7 @@ require (
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.6.0 // indirect
 	github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect
+	github.com/smtp2go-oss/smtp2go-go v1.0.2 // indirect
 	github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d // indirect
 	github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e // indirect
 	github.com/sqlitebrowser/blackfriday v9.0.0+incompatible // indirect

go.sum

@@ -5,6 +5,8 @@ github.com/Freeaqingme/dkim v0.0.0-20160606200812-57418d2fbdcd h1:OXAdgeC8/sNLxo
 github.com/Freeaqingme/dkim v0.0.0-20160606200812-57418d2fbdcd/go.mod h1:2jaO1ZDxGuwzrfRhIe1QKvws/FY3SJ9f1EMNLQLTa0g=
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
+github.com/aquilax/truncate v1.0.0 h1:UgIGS8U/aZ4JyOJ2h3xcF5cSQ06+gGBnjxH2RUHJe0U=
+github.com/aquilax/truncate v1.0.0/go.mod h1:BeMESIDMlvlS3bmg4BVvBbbZUNwWtS8uzYPAKXwwhLw=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
@@ -128,6 +130,8 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/smtp2go-oss/smtp2go-go v1.0.2 h1:vXkqx9kyoQIuetyV3nm40b+OZevihhgb78X4vA/u2fs=
+github.com/smtp2go-oss/smtp2go-go v1.0.2/go.mod h1:lkv36awQXRBWAvnd517FFESKvne8465KCu90lPThcEY=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d h1:yKm7XZV6j9Ev6lojP2XaIshpT4ymkqhMeSghO5Ps00E=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
 github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e h1:qpG93cPwA5f7s/ZPBJnGOYQNK/vKsaDaseuKT5Asee8=