MQE-1918: MFTF AWS Secrets Manager - Local Use

Author: jilu1

composer.json

@@ -11,7 +11,10 @@
     "require": {
         "php": "7.0.2||7.0.4||~7.0.6||~7.1.0||~7.2.0||~7.3.0",
         "ext-curl": "*",
+        "ext-json": "*",
+        "ext-openssl": "*",
         "allure-framework/allure-codeception": "~1.3.0",
+        "aws/aws-sdk-php": "^3.132",
         "codeception/codeception": "~2.4.5",
         "composer/composer": "^1.4",
         "consolidation/robo": "^1.0.0",

composer.lock

@@ -0,0 +1,65 @@
+<?php
+/**
+ * Copyright © Magento, Inc. All rights reserved.
+ * See COPYING.txt for license details.
+ */
+
+namespace tests\unit\Magento\FunctionalTestFramework\DataGenerator\Handlers\SecretStorage;
+
+use Aws\SecretsManager\SecretsManagerClient;
+use Magento\FunctionalTestingFramework\DataGenerator\Handlers\SecretStorage\AwsSecretManagerStorage;
+use Aws\Result;
+use Magento\FunctionalTestingFramework\Util\MagentoTestCase;
+use ReflectionClass;
+
+class AwsSecretManagerStorageTest extends MagentoTestCase
+{
+    /**
+     * Test encryption/decryption functionality in AwsSecretManagerStorage class.
+     */
+    public function testEncryptAndDecrypt()
+    {
+        // Setup test data
+        $testProfile = 'profile';
+        $testRegion = 'region';
+        $testLongKey = 'magento/myKey';
+        $testShortKey = 'myKey';
+        $testValue = 'myValue';
+        $data = [
+            'Name' => 'mftf/magento/' . $testShortKey,
+            'SecretString' => json_encode([$testShortKey => $testValue])
+        ];
+        /** @var Result */
+        $result = new Result($data);
+
+        $mockClient = $this->getMockBuilder(SecretsManagerClient::class)
+            ->disableOriginalConstructor()
+            ->setMethods(['__call'])
+            ->getMock();
+
+        $mockClient->expects($this->once())
+            ->method('__call')
+            ->willReturnCallback(function($name, $args) use ($result) {
+                return $result;
+            });
+
+        /** @var SecretsManagerClient */
+        $credentialStorage = new AwsSecretManagerStorage($testRegion, $testProfile);
+        $reflection = new ReflectionClass($credentialStorage);
+        $reflection_property = $reflection->getProperty('client');
+        $reflection_property->setAccessible(true);
+        $reflection_property->setValue($credentialStorage, $mockClient);
+
+        // Test getEncryptedValue()
+        $encryptedCred = $credentialStorage->getEncryptedValue($testLongKey);
+
+        // Assert the value we've gotten is in fact not identical to our test value
+        $this->assertNotEquals($testValue, $encryptedCred);
+
+        // Test getDecryptedValue()
+        $actualValue = $credentialStorage->getDecryptedValue($encryptedCred);
+
+        // Assert that we are able to successfully decrypt our secret value
+        $this->assertEquals($testValue, $actualValue);
+    }
+}

dev/tests/unit/Magento/FunctionalTestFramework/DataGenerator/Handlers/SecretStorage/AwsSecretManagerStorageTest.php

@@ -277,6 +277,28 @@ Example:
 CREDENTIAL_VAULT_SECRET_BASE_PATH=secret
 ```
 
+### CREDENTIAL_AWS_SECRET_MANAGER_REGION
+
+The region that Aws Secret Manager is located.
+
+Example:
+
+```conf
+# Region of Aws Secret Manager
+CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1
+```
+
+### CREDENTIAL_AWS_SECRET_MANAGER_PROFILE
+
+The profile used to connect to Aws Secret Manager.
+
+Example:
+
+```conf
+# Profile used to connect to Aws Secret Manager.
+CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default
+```
+
 ### ENABLE_BROWSER_LOG
 
 Enables addition of browser logs to Allure steps

docs/configuration.md

@@ -3,10 +3,11 @@
 When you test functionality that involves external services such as UPS, FedEx, PayPal, or SignifyD,
 use the MFTF credentials feature to hide sensitive [data][] like integration tokens and API keys.
 
-Currently the MFTF supports two types of credential storage:
+Currently the MFTF supports three types of credential storage:
 
 -  **.credentials file**
--  **HashiCorp vault**
+-  **HashiCorp Vault**
+-  **Aws Secret Manager**
 
 ## Configure File Storage
 
@@ -135,11 +136,64 @@ CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200
 CREDENTIAL_VAULT_SECRET_BASE_PATH=secret
 ```
 
-## Configure both File Storage and Vault Storage
+## Configure Aws Secret Manager
 
-It is possible and sometimes useful to setup and use both `.credentials` file and vault for secret storage at the same time.
-In this case, the MFTF tests are able to read secret data at runtime from both storage options, but the local `.credentials` file will take precedence.
+Aws Secrets Manager offers secret management that supports:
+- Secret rotation with built-in integration for Amazon RDS, Amazon Redshift, and Amazon DocumentDB
+- Fine-grained policies and permissions
+- Audit secret rotation centrally for resources in the AWS Cloud, third-party services, and on-premises
 
+### Prerequisites
+- AWS account
+- AWS Secret Manger is created and configured
+- IAM User or Role is created
+
+### Store secrets in Aws Secret Manager
+
+#### Secrets format
+`Secret Name`, `Secret Key`, `Secret Value` are three key pieces of information to construct an Aws Secret. 
+`Secret Key` and `Secret Value` can be any content you want to secure, `Secret Name` must follow the format:
+
+```conf
+mftf/<VENDOR>/<SECRET_KEY>
+```
+
+```conf
+# Secret name for carriers_usps_userid
+mftf/magento/carriers_usps_userid
+
+# Secret key for carriers_usps_userid
+carriers_usps_userid
+
+# Secret name for carriers_usps_password
+mftf/magento/carriers_usps_password
+
+# Secret key for carriers_usps_password
+carriers_usps_password
+```
+
+### Setup MFTF to use Aws Secret Manager
+
+To use Aws Secret Manager, the Aws region to connect to is required. You can set it through environment variable [`CREDENTIAL_AWS_SECRET_MANAGER_REGION`][] in `.env`.
+
+MFTF uses the recommended [Default Credential Provider Chain][credential chain] to establish connection to Aws Secret Manager service. 
+You can setup credentials according to [Default Credential Provider Chain][credential chain] and there is no MFTF specific setup required. 
+Optionally, however, you can explicitly set Aws profile through environment variable [`CREDENTIAL_AWS_SECRET_MANAGER_PROFILE`][] in `.env`.
+
+```conf
+# Sample Aws Secret Manager configuration
+CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1
+CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default
+```
+
+## Configure multiple credential storage
+
+It is possible and sometimes useful to setup and use multiple credential storage at the same time.
+In this case, the MFTF tests are able to read secret data at runtime from all storage options, in this case MFTF use the following precedence:
+
+```
+.credentials File > HashiCorp Vault > Aws Secret Manager
+```
 <!-- {% raw %} -->
 
 ## Use credentials in a test
@@ -183,3 +237,6 @@ The MFTF tests delivered with Magento application do not use credentials and do
 [Vault KV2]: https://www.vaultproject.io/docs/secrets/kv/kv-v2.html
 [`CREDENTIAL_VAULT_ADDRESS`]: configuration.md#credential_vault_address
 [`CREDENTIAL_VAULT_SECRET_BASE_PATH`]: configuration.md#credential_vault_secret_base_path
+[credential chain]: https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/guide_credentials.html
+[`CREDENTIAL_AWS_SECRET_MANAGER_PROFILE`]: configuration.md#credential_aws_secret_manager_profile
+[`CREDENTIAL_AWS_SECRET_MANAGER_REGION`]: configuration.md#credential_aws_secret_manager_region

docs/credentials.md

@@ -30,10 +30,14 @@ BROWSER=chrome
 #MAGENTO_RESTAPI_SERVER_PORT=8080
 #MAGENTO_RESTAPI_SERVER_PROTOCOL=https
 
-#*** Uncomment and set vault address and secret base path if you want to use vault to manage _CREDS secrets ***#
+#*** To use HashiCorp Vault to manage _CREDS secrets, uncomment and set vault address and secret base path ***#
 #CREDENTIAL_VAULT_ADDRESS=http://127.0.0.1:8200
 #CREDENTIAL_VAULT_SECRET_BASE_PATH=secret
 
+#*** To use AWS Secret Manager to manage _CREDS secrets, uncomment and set region, profile is optional, when omitted, AWS default credential provider chain will be used ***#
+#CREDENTIAL_AWS_SECRET_MANAGER_PROFILE=default
+#CREDENTIAL_AWS_SECRET_MANAGER_REGION=us-east-1
+
 #*** Uncomment these properties to set up a dev environment with symlinked projects ***#
 #TESTS_BP=
 #FW_BP=