feat(aws): Implement optional Amazon KMS (Key Management Service) based encryption (#1052)

Signed-off-by: Akhil Repala <arepala@blinklabs.io>

Author: arepala-uml

database/plugin/blob/aws/commit_timestamp.go

@@ -16,20 +16,48 @@ package aws
 
 import (
 	"context"
+	"encoding/json"
 	"math/big"
+
+	dingosops "github.com/blinklabs-io/dingo/database/sops"
 )
 
 const commitTimestampBlobKey = "metadata_commit_timestamp"
 
 func (b *BlobStoreS3) GetCommitTimestamp(ctx context.Context) (int64, error) {
-	data, err := b.Get(ctx, commitTimestampBlobKey)
+	ciphertext, err := b.Get(ctx, commitTimestampBlobKey)
+	if err != nil {
+		return 0, err
+	}
+	plaintext, err := dingosops.Decrypt(ciphertext)
 	if err != nil {
+		if !json.Valid(ciphertext) {
+			ts := new(big.Int).SetBytes(ciphertext).Int64()
+			b.logger.Warningf(
+				"commit timestamp stored plaintext in S3, migrating to SOPS encryption: %v",
+				err,
+			)
+			if migrateErr := b.SetCommitTimestamp(ctx, ts); migrateErr != nil {
+				b.logger.Errorf("failed to migrate plaintext commit timestamp: %v", migrateErr)
+			}
+			return ts, nil
+		}
+		b.logger.Errorf("failed to decrypt commit timestamp: %v", err)
 		return 0, err
 	}
-	return new(big.Int).SetBytes(data).Int64(), nil
+	return new(big.Int).SetBytes(plaintext).Int64(), nil
 }
 
 func (b *BlobStoreS3) SetCommitTimestamp(ctx context.Context, ts int64) error {
 	raw := new(big.Int).SetInt64(ts).Bytes()
-	return b.Put(ctx, commitTimestampBlobKey, raw)
+	ciphertext, err := dingosops.Encrypt(raw)
+	if err != nil {
+		b.logger.Errorf("failed to encrypt commit timestamp: %v", err)
+		return err
+	}
+	if err := b.Put(ctx, commitTimestampBlobKey, ciphertext); err != nil {
+		return err
+	}
+	b.logger.Infof("commit timestamp %d written to S3", ts)
+	return nil
 }

database/plugin/blob/gcs/commit_timestamp.go

@@ -16,6 +16,7 @@ package gcs
 
 import (
 	"context"
+	"encoding/json"
 	"io"
 	"math/big"
 
@@ -40,6 +41,17 @@ func (b *BlobStoreGCS) GetCommitTimestamp(ctx context.Context) (int64, error) {
 
 	plaintext, err := dingosops.Decrypt(ciphertext)
 	if err != nil {
+		if !json.Valid(ciphertext) && len(ciphertext) <= 8 {
+			ts := new(big.Int).SetBytes(ciphertext).Int64()
+			b.logger.Warningf(
+				"commit timestamp stored plaintext in GCS, migrating to SOPS encryption: %v",
+				err,
+			)
+			if migrateErr := b.SetCommitTimestamp(ctx, ts); migrateErr != nil {
+				b.logger.Errorf("failed to migrate plaintext commit timestamp: %v", migrateErr)
+			}
+			return ts, nil
+		}
 		b.logger.Errorf("failed to decrypt commit timestamp: %v", err)
 		return 0, err
 	}

database/sops/sops.go

@@ -22,24 +22,27 @@ import (
 	sopsapi "github.com/getsops/sops/v3"
 	"github.com/getsops/sops/v3/aes"
 	scommon "github.com/getsops/sops/v3/cmd/sops/common"
+	"github.com/getsops/sops/v3/config"
 	"github.com/getsops/sops/v3/decrypt"
 	"github.com/getsops/sops/v3/gcpkms"
 	skeys "github.com/getsops/sops/v3/keys"
+	awskms "github.com/getsops/sops/v3/kms"
 	jsonstore "github.com/getsops/sops/v3/stores/json"
 	"github.com/getsops/sops/v3/version"
 )
 
 func Decrypt(data []byte) ([]byte, error) {
-	ret, err := decrypt.Data(data, "json")
+	ret, err := decrypt.Data(data, "binary")
 	if err != nil {
 		return nil, err
 	}
 	return ret, nil
 }
 
 func Encrypt(data []byte) ([]byte, error) {
-	input := &jsonstore.Store{}
-	output := &jsonstore.Store{}
+	storeConfig := &config.JSONBinaryStoreConfig{}
+	input := jsonstore.NewBinaryStore(storeConfig)
+	output := jsonstore.NewBinaryStore(storeConfig)
 
 	// prevent double encryption
 	branches, err := input.LoadPlainFile(data)
@@ -56,20 +59,12 @@ func Encrypt(data []byte) ([]byte, error) {
 
 	// create tree and encrypt
 	tree := sopsapi.Tree{Branches: branches}
-
-	// Configure Google KMS from env to encrypt
-	rid := os.Getenv("DINGO_GCP_KMS_RESOURCE_ID")
-	if rid == "" {
-		return nil, errors.New(
-			"DINGO_GCP_KMS_RESOURCE_ID not set: SOPS requires at least one master key to encrypt",
-		)
-	}
-	keys := []skeys.MasterKey{}
-	for _, k := range gcpkms.MasterKeysFromResourceIDString(rid) {
-		keys = append(keys, k)
+	keyGroups, err := getMasterKeyGroupsFromEnv()
+	if err != nil {
+		return nil, err
 	}
 	tree.Metadata = sopsapi.Metadata{
-		KeyGroups: []sopsapi.KeyGroup{keys},
+		KeyGroups: keyGroups,
 		Version:   version.Version,
 	}
 
@@ -91,3 +86,38 @@ func Encrypt(data []byte) ([]byte, error) {
 	}
 	return encrypted, nil
 }
+
+func getMasterKeyGroupsFromEnv() ([]sopsapi.KeyGroup, error) {
+	keyGroups := []sopsapi.KeyGroup{}
+
+	// Configure Google KMS from env to encrypt
+	if rid := os.Getenv("DINGO_GCP_KMS_RESOURCE_ID"); rid != "" {
+		keys := []skeys.MasterKey{}
+		for _, k := range gcpkms.MasterKeysFromResourceIDString(rid) {
+			keys = append(keys, k)
+		}
+		if len(keys) > 0 {
+			keyGroups = append(keyGroups, keys)
+		}
+	}
+
+	// Configure AWS KMS from env to encrypt
+	if arns := os.Getenv("DINGO_AWS_KMS_KEY_ARNS"); arns != "" {
+		keys := []skeys.MasterKey{}
+		profile := os.Getenv("DINGO_AWS_KMS_PROFILE")
+		for _, k := range awskms.MasterKeysFromArnString(arns, nil, profile) {
+			keys = append(keys, k)
+		}
+		if len(keys) > 0 {
+			keyGroups = append(keyGroups, keys)
+		}
+	}
+
+	if len(keyGroups) == 0 {
+		return nil, errors.New(
+			"SOPS requires at least one master key to encrypt: set DINGO_GCP_KMS_RESOURCE_ID and/or DINGO_AWS_KMS_KEY_ARNS",
+		)
+	}
+
+	return keyGroups, nil
+}