MQE-1919: MFTF AWS Secrets Manager - CI Use

jilu1 · jilu1 · commit 93e221c4a99d · 2020-01-21T11:14:56.000-06:00
diff --git a/docs/credentials.md b/docs/credentials.md
@@ -144,32 +144,54 @@ AWS Secrets Manager offers secret management that supports:
 - Audit secret rotation centrally for resources in the AWS Cloud, third-party services, and on-premises
 
 ### Prerequisites
-- AWS account
-- AWS Secrets Manger is created and configured
+
+#### Use AWS Secrets Manager from your own AWS account
+
+- AWS account with Secrets Manager service available
 - IAM User or Role is created with appropriate AWS Secrets Manger access permission
 
+#### Use AWS Secrets Manager from other AWS account
+
+- AWS account ID where the AWS Secrets Manager service is hosted
+- IAM User or Role with appropriate access permission
+
 ### Store secrets in AWS Secrets Manager
 
+
 #### Secrets format
-`Secret Name`, `Secret Key`, `Secret Value` are three key pieces of information to construct an AWS Secret. 
-`Secret Key` and `Secret Value` can be any content you want to secure, `Secret Name` must follow the format:
+
+`Secret Name` and `Secret Value` are two key pieces of information for creating a secret. 
+
+`Secret Value` can be either plaintext or key/value pairs in JSON format.  
+
+`Secrets Name` must use the following format:
 
 ```conf
-mftf/<VENDOR>/<SECRET_KEY>
+mftf/<VENDOR>/<YOUR/SECRET/KEY>
 ```
 
-```conf
-# Secret name for carriers_usps_userid
-mftf/magento/carriers_usps_userid
+`Secrets Value` in plaintext format can be any content you want to secure. `Secrets Value` in key/value pairs format, however, the `key` must be same as the `Secret Name` with `mftf/<VENDOR>/` part removed. 
+e.g. in above example, `key` should be `<YOUR/SECRET/KEY>`
+
+##### Create Secrets using AWS CLI
 
-# Secret key for carriers_usps_userid
-carriers_usps_userid
+```bash
+aws secretsmanager create-secret --name "mftf/magento/shipping/carriers_usps_userid" --description "Carriers USPS user id" --secret-string "1234567"
+```
+
+##### Create Secrets using AWS Console
+
+To save the same secret in key/value JSON format, you should use
+
+```conf
+# Secret Name
+mftf/magento/shipping/carriers_usps_userid
 
-# Secret name for carriers_usps_password
-mftf/magento/carriers_usps_password
+# Secret Key
+shipping/carriers_usps_userid
 
-# Secret key for carriers_usps_password
-carriers_usps_password
+# Secret Value
+1234567
 ```
 
 ### Setup MFTF to use AWS Secrets Manager
@@ -186,6 +208,16 @@ CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1
 CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default
 ```
 
+### Optionally set CREDENTIAL_AWS_ACCOUNT_ID environment variable
+ 
+Full AWS KMS ([Key Management Service][]) key ARN ([Amazon Resource Name][]) is required when accessing secrets stored in other AWS account.
+If this is the case, you will also need to set `CREDENTIAL_AWS_ACCOUNT_ID` environment variable so that MFTF can construct the full ARN. 
+This is also commonly used in CI system.
+
+```bash
+export CREDENTIAL_AWS_ACCOUNT_ID=<Your_12_Digits_AWS_Account_ID>
+```
+
 ## Configure multiple credential storage
 
 It is possible and sometimes useful to setup and use multiple credential storage at the same time.
@@ -239,4 +271,6 @@ The MFTF tests delivered with Magento application do not use credentials and do
 [`CREDENTIAL_VAULT_SECRET_BASE_PATH`]: configuration.md#credential_vault_secret_base_path
 [credential chain]: https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials.html
 [`CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE`]: configuration.md#credential_aws_secrets_manager_profile
-[`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`]: configuration.md#credential_aws_secrets_manager_region
+[`CREDENTIAL_AWS_SECRETS_MANAGER_REGION`]: configuration.md#credential_aws_secrets_manager_region
+[Key Management Service]: https://aws.amazon.com/kms/
+[Amazon Resource Name]: https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
diff --git a/etc/config/.env.example b/etc/config/.env.example
@@ -37,8 +37,6 @@ BROWSER=chrome
 #*** To use AWS Secrets Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***#
 #CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE=default
 #CREDENTIAL_AWS_SECRETS_MANAGER_REGION=us-east-1
-#*** If using non-default AWS account ***#
-#CREDENTIAL_AWS_ACCOUNT_ID=
 
 #*** Uncomment these properties to set up a dev environment with symlinked projects ***#
 #TESTS_BP=
diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/CredentialStore.php
@@ -57,45 +57,10 @@ public static function getInstance()
      */
     private function __construct()
     {
-        // Initialize file storage
-        try {
-            $this->credStorage[self::ARRAY_KEY_FOR_FILE]  = new FileStorage();
-        } catch (TestFrameworkException $e) {
-        }
-
-        // Initialize vault storage
-        $cvAddress = getenv('CREDENTIAL_VAULT_ADDRESS');
-        $cvSecretPath = getenv('CREDENTIAL_VAULT_SECRET_BASE_PATH');
-        if ($cvAddress !== false && $cvSecretPath !== false) {
-            try {
-                $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage(
-                    UrlFormatter::format($cvAddress, false),
-                    '/' . trim($cvSecretPath, '/')
-                );
-            } catch (TestFrameworkException $e) {
-            }
-        }
-
-        // Initialize AWS Secrets Manager storage
-        $awsRegion = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_REGION');
-        $awsProfile = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE');
-        $awsId = getenv('CREDENTIAL_AWS_ACCOUNT_ID');
-        if ($awsRegion !== false) {
-            if ($awsProfile === false) {
-                $awsProfile = null;
-            }
-            if ($awsId === false) {
-                $awsId = null;
-            }
-            try {
-                $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER] = new AwsSecretsManagerStorage(
-                    $awsRegion,
-                    $awsProfile,
-                    $awsId
-                );
-            } catch (TestFrameworkException $e) {
-            }
-        }
+        // Initialize credential storage by defined order of precedence as the following
+        $this->initializeFileStorage();
+        $this->initializeVaultStorage();
+        $this->initializeAwsSecretsManagerStorage();
 
         if (empty($this->credStorage)) {
             throw new TestFrameworkException(
@@ -155,4 +120,68 @@ public function decryptAllSecretsInString($string)
             return $storage->getAllDecryptedValuesInString($string);
         }
     }
+
+    /**
+     * Initialize file storage
+     *
+     * @return void
+     */
+    private function initializeFileStorage()
+    {
+        // Initialize file storage
+        try {
+            $this->credStorage[self::ARRAY_KEY_FOR_FILE]  = new FileStorage();
+        } catch (TestFrameworkException $e) {
+        }
+    }
+
+    /**
+     * Initialize Vault storage
+     *
+     * @return void
+     */
+    private function initializeVaultStorage()
+    {
+        // Initialize vault storage
+        $cvAddress = getenv('CREDENTIAL_VAULT_ADDRESS');
+        $cvSecretPath = getenv('CREDENTIAL_VAULT_SECRET_BASE_PATH');
+        if ($cvAddress !== false && $cvSecretPath !== false) {
+            try {
+                $this->credStorage[self::ARRAY_KEY_FOR_VAULT] = new VaultStorage(
+                    UrlFormatter::format($cvAddress, false),
+                    '/' . trim($cvSecretPath, '/')
+                );
+            } catch (TestFrameworkException $e) {
+            }
+        }
+    }
+
+    /**
+     * Initialize AWS Secrets Manager storage
+     *
+     * @return void
+     */
+    private function initializeAwsSecretsManagerStorage()
+    {
+        // Initialize AWS Secrets Manager storage
+        $awsRegion = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_REGION');
+        $awsProfile = getenv('CREDENTIAL_AWS_SECRETS_MANAGER_PROFILE');
+        $awsId = getenv('CREDENTIAL_AWS_ACCOUNT_ID');
+        if (!empty($awsRegion)) {
+            if (empty($awsProfile)) {
+                $awsProfile = null;
+            }
+            if (empty($awsId)) {
+                $awsId = null;
+            }
+            try {
+                $this->credStorage[self::ARRAY_KEY_FOR_AWS_SECRETS_MANAGER] = new AwsSecretsManagerStorage(
+                    $awsRegion,
+                    $awsProfile,
+                    $awsId
+                );
+            } catch (TestFrameworkException $e) {
+            }
+        }
+    }
 }
diff --git a/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php b/src/Magento/FunctionalTestingFramework/DataGenerator/Handlers/SecretStorage/AwsSecretsManagerStorage.php
@@ -115,17 +115,18 @@ public function getEncryptedValue($key)
             $reValue = openssl_encrypt($value, parent::ENCRYPTION_ALGO, parent::$encodedKey, 0, parent::$iv);
             parent::$cachedSecretData[$key] = $reValue;
         } catch (AwsException $e) {
-            $error = $e->getAwsErrorCode();
+            $errMessage = "\nAWS Exception:\n" . $e->getAwsErrorMessage()
+                . "\nUnable to read value for key {$key} from AWS Secrets Manager\n";
+            print_r($errMessage);
             if (MftfApplicationConfig::getConfig()->verboseEnabled()) {
-                LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug(
-                    "AWS error code: {$error}. Unable to read value for key {$key} from AWS Secrets Manager"
-                );
+                LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug($errMessage);
             }
         } catch (\Exception $e) {
+            $errMessage = "\nException:\n" . $e->getMessage()
+                . "\nUnable to read value for key {$key} from AWS Secrets Manager\n";
+            print_r($errMessage);
             if (MftfApplicationConfig::getConfig()->verboseEnabled()) {
-                LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug(
-                    "Unable to read value for key {$key} from AWS Secrets Manager"
-                );
+                LoggingUtil::getInstance()->getLogger(AwsSecretsManagerStorage::class)->debug($errMessage);
             }
         }
         return $reValue;
@@ -145,13 +146,22 @@ private function parseAwsSecretResult($awsResult, $key)
         if (isset($awsResult['SecretString'])) {
             $rawSecret = $awsResult['SecretString'];
         } else {
-            throw new TestFrameworkException("Error parsing result from AWS Secrets Manager");
+            throw new TestFrameworkException(
+                "'SecretString' field is not set in AWS Result. Error parsing result from AWS Secrets Manager"
+            );
         }
+
+        // Secrets are saved as JSON structures of key/value pairs if using AWS Secrets Manager console, and
+        // Secrets are saved as plain text if using AWS CLI. We need to handle both cases.
         $secret = json_decode($rawSecret, true);
         if (isset($secret[$key])) {
             return $secret[$key];
+        } elseif (is_string($rawSecret)) {
+            return $rawSecret;
         }
-        throw new TestFrameworkException("Error parsing result from AWS Secrets Manager");
+        throw new TestFrameworkException(
+            "$key not found or value is not string . Error parsing result from AWS Secrets Manager"
+        );
     }
 
     /**
@@ -169,13 +179,17 @@ private function createAwsSecretsManagerClient($region, $profile)
             return;
         }
 
-        // Create AWS Secrets Manager client
-        $this->client = new SecretsManagerClient([
-            'profile' => $profile,
+        $options = [
             'region' => $region,
-            'version' => self::LATEST_VERSION
-        ]);
+            'version' => self::LATEST_VERSION,
+        ];
 
+        if (!empty($profile)) {
+            $options['profile'] = $profile;
+        }
+
+        // Create AWS Secrets Manager client
+        $this->client = new SecretsManagerClient($options);
         if ($this->client === null) {
             throw new TestFrameworkException("Unable to create AWS Secrets Manager client");
         }
