diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 0000000..2f7efbe --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-minimal \ No newline at end of file diff --git a/docs/docs/.pages b/docs/docs/.pages index 73c79c0..75b7389 100644 --- a/docs/docs/.pages +++ b/docs/docs/.pages @@ -1,7 +1,7 @@ arrange: - Overview: 'index.md' - [Installation]: 'Installation.md' - - PluginDevelopment : 'PluginDevelopment.md' - Discovery: 'Discovery.md' - Linking: 'Linking.md' - Provisioning: 'Provisioning.md' + diff --git a/docs/docs/Discovery.md b/docs/docs/Discovery.md index 0b5a76f..79f99ae 100644 --- a/docs/docs/Discovery.md +++ b/docs/docs/Discovery.md @@ -5,33 +5,27 @@ Environment discovery is a process that enables the Couchbase Plugin to determin . Whenever there is any change (installing a new database home) to an already set up environment in the Delphix application, we need to perform an environment refresh. -Prerequisites -============= +## Prerequisites -- A source environment must be added to the Delphix Engine. +- A stagine or target environment must be added to the Delphix Engine. - Installation of the Couchbase Plugin is required before the Discovery. - Environment variable `$COUCHBASE_PATH ` should set on staging/target host, which contains the binary path of Couchbase. -Refreshing an Environment -========================= -Environment refresh will update the metadata associated with that environment and send a new Plugin to the host. +### Configuring a staging environment -1. Login to the **Delphix Management** application. -2. Click **Manage**. -3. Select **Environments**. -4. In the Environments panel, click the name of the environment you want to refresh. -5. Select the **Refresh** icon. -6. In the Refresh confirmation dialog select **Refresh**. +Environments contain repositories, and each environment may have any number of repositories associated with it. +Couchbase is limiting number of Couchbase binaries installed on server to one, and in that case each environment will contain +a single repository with current Couchbase installation. + +Repository contains database instances and in each repository any number of source cluster can be configured. +Please keep in mind that only one dSource can be enabled simiultaniusly on a given staging server. -![Screenshot](/couchbase-plugin/image/image9.png) +For next step go to XDCR Setup or Backup Setup sections below +### XDCR Setup -XDCR Setup -=================== -Environments exist to contain `repositories`, and each environment may have any number of repositories associated with it. -`Repository` contains database instances and in each repository any number of `SourceConfig` objects, which represent known database instances. Source config is not generated automatically in - Couchbase plugin. Therefore, we need to add `SourceConfig` object through which can create a dSource. +By default Couchbase cluster are not discovered and has to be added manually using the following steps: 1. Login to the **Delphix Management** application. 2. Click **Manage**. @@ -40,40 +34,46 @@ Environments exist to contain `repositories`, and each environment may have any 5. Click on **+** icon (Shown in next image). -![Screenshot](/couchbase-plugin/image/image10.png) +![Screenshot](./image/add_repository.png) 6. Add required details in the `Add database` section. - Enter port number in **Source Couchbase port** section. - - Enter source host address in section **Source Host**. + - Enter source host address in section **Source Host** ( this will be used as Source cluster for XDCR replication ) - Enter unique name for the staging database in **identify field** section. - Enter Couchbase data path of staging host in **DB data path** section. -![Screenshot](/couchbase-plugin/image/image11.png) - +![Screenshot](./image/add_sourceconfig_xdcr.png) -CBBACKUPMGR Setup -================= +### Backup Setup -The steps to add source config remain the same as we saw in XDCR setup. In this approach, we don't connect to source environment as this is zero-touch production approach. -We can enter any random or dummy value in this field of source host name when we choose CBBACKUPMGR option for data ingestion. +By default Couchbase clusters are not discovered and has to be added manually using the following steps: 1. Login to the **Delphix Management** application. 2. Click **Manage**. 3. Select **Environments**. 4. Select the repository. -5. Click on **+** icon (Shown in next image). -![Screenshot](/couchbase-plugin/image/image10.png) +5. Click on **+** icon (Shown in next image). -6. In the **Add Database** section enter the following information: - - `Source Couchbase port`: This is the port number to be used by Couchbase services. - - `Source Host`: Leave this field as blank. - - `identity field`: Provide unique name for staging database. - - `DB data path`: Leave this field as blank. +![Screenshot](./image/add_repository.png) +6. Add required details in the `Add database` section. + - Enter port number in **Source Couchbase port** section ( this can be any dummy number - this field is not used for backup based ingestion ) + - Enter source host address in section **Source Host** ( this can be any dummy number - this field is not used for backup based ingestion ) + - Enter unique name for the staging database in **identify field** section. + - Enter Couchbase data path of staging host in **DB data path** section. -![Screenshot](/couchbase-plugin/image/image11.png) +## Refreshing an Environment +Environment refresh will update the metadata associated with that environment and send a new Plugin to the host. + +1. Login to the **Delphix Management** application. +2. Click **Manage**. +3. Select **Environments**. +4. In the Environments panel, click the name of the environment you want to refresh. +5. Select the **Refresh** icon. +6. In the Refresh confirmation dialog select **Refresh**. +![Screenshot](./image/add_sourceconfig_backup.png) diff --git a/docs/docs/Installation.md b/docs/docs/Installation.md index 5b9bc07..dc4bb84 100644 --- a/docs/docs/Installation.md +++ b/docs/docs/Installation.md @@ -14,22 +14,22 @@ Method1: Using GUI ------------------ 1. Click on **Manage** (present on top-left of this page) and then **Plugins**. -![Screenshot](/couchbase-plugin/image/image2.png) +![Screenshot](./image/image2.png) 2. Click on **+** icon. -![Screenshot](/couchbase-plugin/image/image3.png) +![Screenshot](./image/image3.png) 3. Click on **Upload** a plugin. -![Screenshot](/couchbase-plugin/image/image4.png) +![Screenshot](./image/image4.png) 4. Select the `build(artifacts.json)` from your device. -![Screenshot](/couchbase-plugin/image/image5.png) +![Screenshot](./image/image5.png) 5. Click on **close** button. -![Screenshot](/couchbase-plugin/image/image6.png) +![Screenshot](./image/image6.png) 6. See the plugin version in **Plugins** section. -![Screenshot](/couchbase-plugin/image/image7.png) +![Screenshot](./image/image7.png) Method2: Using dvp command diff --git a/docs/docs/Linking.md b/docs/docs/Linking.md index 2bb90e3..6556a28 100644 --- a/docs/docs/Linking.md +++ b/docs/docs/Linking.md @@ -4,90 +4,140 @@ Linking a data source will create a dSource object on the engine and allow Delph -Prerequisites -============= +## Prerequisites -Discovery and source config object should be created on the staging host before proceeding to link Couchbase dataset. - - -Creating dSource -============== +Staging environment created and source database configured in discovered repository +## Creating dSource using XDCR 1. Login to **Delphix Management** application. 2. Click **Manage** > **Datasets**. 3. Select **Add dSource**. 4. In the Add dSource wizard, select the Couchbase source configuration which is created on the staging host. 5. Enter the Couchbase-specific parameters for your dSource configuration. -6. Select the dSource approach from the drop-down (XDCR and Couchbase Backup Manager) available on dSource wizard. +6. Select the dSource type XDCR from the drop-down available on dSource wizard. 7. Based on approach selection, follow the steps either for XDCR or Couchbase Backup Manager method. The Description of both methods is below. - - - Method1: XDCR --------------- -Cross datacenter replication allows data to be replicated across clusters that are potentially located in different data centers. - - -1. Enter the details for **Staging Couchbase host** - FQDN or IP address recommended. -2. Enter the details for **Staging Port Number** available on the staging host. The default port for couchbase is 8091. -3. Enter the details for **Mount Path** available on the staging host. This empty folder acts as a base for NFS mounts. -4. Enter the details for **Staging Cluster Name** to setup new cluster on the staging host. -5. Enter the configuration details for your staging cluster as per resource availability on the staging host. +8. Enter the details for **Staging Couchbase host** - FQDN or IP address recommended. +9. Enter the details for **Staging Port Number** available on the staging host. The default port for couchbase is 8091. +10. Enter the details for **Mount Path** available on the staging host. This empty folder acts as a base for NFS mounts. +11. Enter the details for **Staging Cluster Name** to setup new cluster on the staging host. +12. Enter the configuration details for your staging cluster as per resource availability on the staging host. - Cluster RAM Size - Cluster Index RAM Size - Cluster FTS RAM Size - - Cluster Eventing RAM Size - - Cluster Analysis RAM Size -![Screenshot](/couchbase-plugin/image/image12.png) + - Cluster Eventing RAM Size - this should be 0 + - Cluster Analysis RAM Size - this should be 0 -6. Click on **+** plus symbol to modify configuration settings. Mention bucket list for which cross datacenter replication (XDCR) only be enabled. -![Screenshot](/couchbase-plugin/image/image14.png) + ![Screenshot](./image/add_dsource_1.png) -7. Enter the details of **Bucket Name** to be part of XDCR. Then click on **Next** button -![Screenshot](/couchbase-plugin/image/image15.png) +13. Enter the details for **Staging Cluster Admin User** and **Staging Cluster Admin Password** +14. Enter the details for **Source Cluster Admin User** and **Source Cluster Admin Password** -8. Provide the details for **dSource Name** and **Target group** on the dSource configuration page. -![Screenshot](/couchbase-plugin/image/image16.png) + ![Screenshot](./image/add_dsource_2.png) -9. On the **Data management** page, select the following: +15. If not all buckets needs to be replicated, click on **+** plus symbol to modify configuration settings. Mention bucket list for which cross datacenter replication (XDCR) only be enabled. + ![Screenshot](./image/image14.png) + +16. Enter the details of **Bucket Name** to be part of XDCR. Then click on **Next** button + ![Screenshot](./image/image15.png) + +17. Provide the details for **dSource Name** and **Target group** on the dSource configuration page. + ![Screenshot](./image/add_dsource_3.png) + +18. On the **Data management** page, select the following: - Staging Environment: This will be your staging host where source config was created. - User: Database OS user with required privileges for linking the dataset. -10. On the next section, review the configuration and click on **Next** button to view the summary. -11. Click the **Submit** button which will initiate the linking process. -![Screenshot](/couchbase-plugin/image/image17.png) -12. Once dSource is created successfully, you can review the datasets on **Manage** > **Datasets** > **dSource Name**. -![Screenshot](/couchbase-plugin/image/image19.png) + ![Screenshot](./image/add_dsource_4.png) + +19. On the next screens, configure a policy, hooks and review the configuration and click on **Next** button to view the summary. + + ![Screenshot](./image/add_dsource_5.png) -Method2: Couchbase Backup Manager ---------------------------------------- -**Note**: Follow the instructions below before creating dSource to avoid source/production server dependency. + ![Screenshot](./image/add_dsource_6.png) -- Provide source server buckets related information in a file: */tmp/couchbase_src_bucket_info.cfg*. - `/opt/couchbase/bin/couchbase-cli bucket-list --cluster :8091 --username $username --password $password` -- **Backup Repository**: This file will be required at the time of dSource creation using CBBACKUPMGR. - `/opt/couchbase/bin/cbbackupmgr config --archive /u01/couchbase_backup --repo delphix` +20. Click the **Submit** button which will initiate the linking process. -- **Backup Location**: Get data from source host in backup directory of staging host. -`/opt/couchbase/bin/cbbackupmgr backup -a /u01/couchbase_backup -r delphix -c couchbase:// -u user -p password` + ![Screenshot](./image/add_dsource_7.png) -**Procedure**: +21. Once dSource is created successfully, you can review the datasets on **Manage** > **Datasets** > **dSource Name**. + + + ![Screenshot](./image/dsource_ingested.png) + + +## Creating dSource using backup + +Prerequisites: +Access to production backup with those ex. values: + +/backup - archive + +PROD - repository + +``` + /opt/couchbase/bin/cbbackupmgr info -a /backup -r PROD + Name | Size | # Backups | + PROD | 58.03MB | 1 | + + Backup | Size | Type | Source | Cluster UUID | Range | Events | Aliases | Complete | + + 2022-01-10T11_29_26.465860528-05_00 | 58.03MB | FULL | http://couchbasesrc.dlpxdc.co:8091 | 08f7937a26b2d20178a5ed16d7a2dd1c | N/A | 0 | 0 | true | +``` 1. Login to **Delphix Management** application. 2. Click **Manage** > **Datasets**. 3. Select **Add dSource**. -4. In the **Add dSource wizard**, select the Couchbase source configuration you created on the staging host. +4. In the Add dSource wizard, select the Couchbase source configuration which is created on the staging host. 5. Enter the Couchbase-specific parameters for your dSource configuration. -6. Select the dSource type from the drop-down available on dSource wizard. -7. When we select CBBACKUPMGR as dSource Type, the following fields on dSource wizard are mandatory. - - Enter the details for **Backup Location** where the backup files generated through CBBACKUPMGR are present on the staging host. - - Enter the details for **Backup Repository** that contains a backup configuration of staging host. -8. The remaining steps for CBBACKUPMGR ingestion are similar to XDCR. Use steps from the second point mentioned in XDCR method. +6. Select the dSource type Couchbase Backup Manager from the drop-down available on dSource wizard. +7. Based on approach selection, follow the steps either for XDCR or Couchbase Backup Manager method. The Description of both methods is below. +8. Enter the details for **Staging Couchbase host** - FQDN or IP address recommended. +9. Enter the details for **Staging Port Number** available on the staging host. The default port for couchbase is 8091. +10. Enter the details for **Backup Location** available on the staging host. +11. Enter the details for **Backup repository** +12. Enter the details for **Mount Path** available on the staging host. This empty folder acts as a base for NFS mounts. +13. Enter the details for **Staging Cluster Name** to setup new cluster on the staging host. +14. Enter the configuration details for your staging cluster as per resource availability on the staging host. + - Cluster RAM Size + - Cluster Index RAM Size + - Cluster FTS RAM Size + - Cluster Eventing RAM Size - this should be 0 + - Cluster Analysis RAM Size - this should be 0 + + ![Screenshot](./image/add_dsource_1backup.png) + +15. Enter the details for **Staging Cluster Admin User** and **Staging Cluster Admin Password** +16. Enter dummy values for **Source Cluster Admin User** and **Source Cluster Admin Password** - they are not used + + ![Screenshot](./image/add_dsource_2backup.png) + +17. Then click on **Next** button + +17. Provide the details for **dSource Name** and **Target group** on the dSource configuration page. + ![Screenshot](./image/add_dsource_3.png) + +18. On the **Data management** page, select the following: + - Staging Environment: This will be your staging host where source config was created. + - User: Database OS user with required privileges for linking the dataset. + + ![Screenshot](./image/add_dsource_4.png) + +19. On the next screens, configure a policy, hooks and review the configuration and click on **Next** button to view the summary. + + ![Screenshot](./image/add_dsource_5.png) + + ![Screenshot](./image/add_dsource_6.png) + + +20. Click the **Submit** button which will initiate the linking process. + + ![Screenshot](./image/add_dsource_7backup.png) + + +21. Once dSource is created successfully, you can review the datasets on **Manage** > **Datasets** > **dSource Name**. -Note: When we select dSource type as Couchbase Backup Manager, we do not require any details for the `Staging Couchbase Host` field. -![Screenshot](/couchbase-plugin/image/image22.png) + ![Screenshot](./image/dsource_ingested.png) diff --git a/docs/docs/Provisioning.md b/docs/docs/Provisioning.md index b99e8e2..8d7d1e5 100644 --- a/docs/docs/Provisioning.md +++ b/docs/docs/Provisioning.md @@ -2,44 +2,120 @@ Virtual databases are a virtualized copies of dSource. -Prerequisites -============= +## Prerequisites - Required a linked dSource from a source host. -- Added compatible target environment on Delphix Engine. +- All prerequisites configured on target environments +- Added compatible target environment on Delphix Engine -Provisioning a VDB -================== +## Provisioning a VDB + +1. Start a provisioning wizard -1. Click on the icon highlighted in red color. -![Screenshot](/couchbase-plugin/image/image24.png) 2. Select the target host from the dropdown on which VDB needs to be created. -![Screenshot](/couchbase-plugin/image/image25.png) + ![Screenshot](./image/provision_1.png) 3. Enter the following values for the target configuration: - - `Target Port Number`: Port number on which Couchbase services will be started. - - `Mount Path`: NFS mount path where dSource snapshot will be mounted by Engine. - - `Target Cluster name`: Cluster name which is required to be set up on the target host. - - `Cluster Ram Size` - - `Cluster Index Ram Size` - - `Cluster FTS Ram Size` - - `Cluster Eventing Ram Size` - - `Cluster Analytics Ram Size` - - `Target couchbase Admin User` - - `Target couchbase Admin password` -![Screenshot](/couchbase-plugin/image/image26.png) + - **Target Port Number**: Port number on which Couchbase services will be started. ( ex. 8091 ) + - **Mount Path**: NFS mount path where dSource snapshot will be mounted by Engine ( ex. /mnt/provision/targetdemo ) + - **Target Cluster name**: Cluster name which is required to be set up on the target host. ( ex. targetdemo ) + - **Cluster Ram Size**: Whole Cluster memory + - **Cluster Index Ram Size**: Cluster indexer memory + - **Cluster FTS Ram Size**: Cluster FTS memory ( if needed and FTS service will be configured ) + - **Cluster Eventing Ram Size**: Cluster Eventing memory ( if needed and Eventing service will be configured ) + - **Cluster Analytics Ram Size**: Cluster Analytics memory ( if needed and Analytics service will be configured ) + + ![Screenshot](./image/provision_2.png) + +4. Enter the following values for the target configuration: + - **Target couchbase Admin User**: Target Cluster admin username + - **Target couchbase Admin password**: Target Cluster admin password + - Select services needed on the target cluster ( FTS, Eventing, Analytics ) + + ![Screenshot](./image/provision_3.png) -4. Provision vFiles: Add VDB name and target group. -![Screenshot](/couchbase-plugin/image/image27.png) +5. Provision plugin based VDB. Enter the follwing value: + - **VDB Name**: Delphix Target Cluster name + - **Target group**: Delphix Target Cluster group + ![Screenshot](./image/provision_4.png) -5. No need to add Policies, select **Next**. +5. Select a policy for VDB, select **Next**. + ![Screenshot](./image/provision_5.png) -6. No need to add Masking, select **Next**. +6. Select masking for VDB if needed, select **Next** + ![Screenshot](./image/provision_6.png) -7. No need to add Hooks, select **Next**. +7. Add hooks for VDB if needed, select **Next** + ![Screenshot](./image/provision_7.png) -8. Preview the summary and select **Submit**. +8. Preview the summary and select **Submit** + ![Screenshot](./image/provision_8.png) 9. Once the VDB is created successfully, you can review the datasets on **Manage** > **Datasets** > **vdb Name**. + ![Screenshot](./image/provision_9.png) + + +## Provisioning a Multinode VDB + +1. Start a provisioning wizard + + +2. Select the target host from the dropdown on which VDB needs to be created. + ![Screenshot](./image/provision_1.png) + +3. Enter the following values for the target configuration: + - **Target Port Number**: Port number on which Couchbase services will be started. ( ex. 8091 ) + - **Mount Path**: NFS mount path where dSource snapshot will be mounted by Engine ( ex. /mnt/provision/targetdemo ) + - **Target Cluster name**: Cluster name which is required to be set up on the target host. ( ex. targetdemo ) + - **Cluster Ram Size**: Whole Cluster memory + - **Cluster Index Ram Size**: Cluster indexer memory + - **Cluster FTS Ram Size**: Cluster FTS memory ( if needed and FTS service will be configured ) + - **Cluster Eventing Ram Size**: Cluster Eventing memory ( if needed and Eventing service will be configured ) + - **Cluster Analytics Ram Size**: Cluster Analytics memory ( if needed and Analytics service will be configured ) + + ![Screenshot](./image/provision_2.png) + +4. Enter the following values for the target configuration: + - **Target couchbase Admin User**: Target Cluster admin username + - **Target couchbase Admin password**: Target Cluster admin password + - Select services needed on the first node of the cluster ( FTS, Eventing, Analytics ) + ![Screenshot](./image/provision_3.png) + - Click Add buton to open a dialog box for additional node. If you need more nodes, click Add button again to add more nodes + ![Screenshot](./image/provision_3_mt1.png) + + - Enter the following values for each node: + - **Delphix Environment Name**: Select an additional node environment from drop-down menu + - **Delphix Environment User**: Select an environment user + - **Node hostname / IP**: Enter a hostname or IP of the new node - it will be used as a server name in Couchbase configuration + - Select services needed on the additional node of the cluster ( FTS, Eventing, Analytics ) + ![Screenshot](./image/provision_3_mt2.png) + +5. Provision plugin based VDB. Enter the follwing value: + - **VDB Name**: Delphix Target Cluster name + - **Target group**: Delphix Target Cluster group + ![Screenshot](./image/provision_4.png) + +5. Select a policy for VDB, select **Next**. + ![Screenshot](./image/provision_5.png) + +6. Select masking for VDB if needed, select **Next** + ![Screenshot](./image/provision_6.png) + +7. Add hooks for VDB if needed, select **Next** + ![Screenshot](./image/provision_7.png) + +8. Preview the summary and select **Submit** + ![Screenshot](./image/provision_8.png) + +9. Once the VDB is created successfully, you can review the datasets on **Manage** > **Datasets** > **vdb Name**. + ![Screenshot](./image/provision_9.png) + + +## Accesssing Target VDB Cluster + +Use a IP / Hostname of the target environment and VDB port defined above to access Target Cluster VDB. +Admin user name and password are defined based on input from point 4. + + diff --git a/docs/docs/Troubleshooting.md b/docs/docs/Troubleshooting.md new file mode 100644 index 0000000..4a883b6 --- /dev/null +++ b/docs/docs/Troubleshooting.md @@ -0,0 +1,61 @@ +As the first step for analysis, we would request the support team to fetch the following custom plugin log files from the customer environment and upload them to the same case along with the support bundle. + +# Plugin Logs +Download the Plugin logs using the following methods: + + * Using dvp + + `dvp download-logs -c plugin_config.yml -e FQDN_of_engine -u admin --password` + + * Using GUI + + Help --> Supports Logs --> Plugin Logs --> Download + +# Couchbase logs +Couchbase logs are usually located at this path: `/opt/couchbase/var/couchbase/lib/logs` +and it should be zipped and uploaded together with a support bundle + + +# Typical issues + + 1. Couchbase services not disabled on the new server start - VDB not configured + + After installation or reboot OS may start a Couchbase server without storage allocated from Delphix Engine + and Couchbase will create an empty configuration file and allow end user to configure it. + If Delphix VDB provisioning will be started, Delphix won't be able to kill not configured Couchbase process + and VDB creation till timeout after 3600 sec. + + Solution: + + - Disable Couchbase services using systemctl, + - Kill all Couchbase processes, + - Create a VDB using Delphix + + 2. Couchbase services not disabled on server start but VDB configured + + After reboot OS may start a Couchbase server without storage allocated from Delphix Engine and Couchbase will see + an empty data directory. Couchbase server won't report this as an error but rather will recreate all buckets without data. + + Solution: + + - Disable Couchbase services using systemctl, + - Kill all Couchbase processes, + - Disable force VDB in Delphix Engine, + - Enable VDB + + + 3. Not enough memory on server to restore bucket + + dSource initial ingestion or snapshot is faling with an cbbackmgr transaction error. + Check in the Couchbae logs, if this error is realted to a memory allocated for the bucket. + + ``` + 2022-01-10T15:39:58.421+01:00 WARN: (Pool) (XXX) Failed to send key 'CL-BAG-C618641009-2' due to error 'server is out of memory | {"status_code":130,"bucket":"XXX","error_name":"ENOMEM","error_description":"No memory available to store item. Add memory or remove some items and try later"... + ``` + + If this is a case there are to options: + + - For an exiting staging serer, login to staging server Couchbase GUI and change bucket memory configuration + - For a new ingestion, set a higher memory for all bucket - there is a limitation there as for know Plugin + allows to overwrite a memory setting only for all buckets together + diff --git a/docs/docs/image/add_dsource_1.png b/docs/docs/image/add_dsource_1.png new file mode 100644 index 0000000..acbf905 Binary files /dev/null and b/docs/docs/image/add_dsource_1.png differ diff --git a/docs/docs/image/add_dsource_1backup.png b/docs/docs/image/add_dsource_1backup.png new file mode 100644 index 0000000..5dfb07a Binary files /dev/null and b/docs/docs/image/add_dsource_1backup.png differ diff --git a/docs/docs/image/add_dsource_2.png b/docs/docs/image/add_dsource_2.png new file mode 100644 index 0000000..19b36cb Binary files /dev/null and b/docs/docs/image/add_dsource_2.png differ diff --git a/docs/docs/image/add_dsource_2backup.png b/docs/docs/image/add_dsource_2backup.png new file mode 100644 index 0000000..a80f365 Binary files /dev/null and b/docs/docs/image/add_dsource_2backup.png differ diff --git a/docs/docs/image/add_dsource_3.png b/docs/docs/image/add_dsource_3.png new file mode 100644 index 0000000..39c568e Binary files /dev/null and b/docs/docs/image/add_dsource_3.png differ diff --git a/docs/docs/image/add_dsource_4.png b/docs/docs/image/add_dsource_4.png new file mode 100644 index 0000000..5ea7902 Binary files /dev/null and b/docs/docs/image/add_dsource_4.png differ diff --git a/docs/docs/image/add_dsource_5.png b/docs/docs/image/add_dsource_5.png new file mode 100644 index 0000000..b35bdb7 Binary files /dev/null and b/docs/docs/image/add_dsource_5.png differ diff --git a/docs/docs/image/add_dsource_6.png b/docs/docs/image/add_dsource_6.png new file mode 100644 index 0000000..591281e Binary files /dev/null and b/docs/docs/image/add_dsource_6.png differ diff --git a/docs/docs/image/add_dsource_7.png b/docs/docs/image/add_dsource_7.png new file mode 100644 index 0000000..20f8631 Binary files /dev/null and b/docs/docs/image/add_dsource_7.png differ diff --git a/docs/docs/image/add_dsource_7backup.png b/docs/docs/image/add_dsource_7backup.png new file mode 100644 index 0000000..e555895 Binary files /dev/null and b/docs/docs/image/add_dsource_7backup.png differ diff --git a/docs/docs/image/add_repository.png b/docs/docs/image/add_repository.png new file mode 100644 index 0000000..9424bdf Binary files /dev/null and b/docs/docs/image/add_repository.png differ diff --git a/docs/docs/image/add_sourceconfig_backup.png b/docs/docs/image/add_sourceconfig_backup.png new file mode 100644 index 0000000..69d2295 Binary files /dev/null and b/docs/docs/image/add_sourceconfig_backup.png differ diff --git a/docs/docs/image/add_sourceconfig_xdcr.png b/docs/docs/image/add_sourceconfig_xdcr.png new file mode 100644 index 0000000..25339e5 Binary files /dev/null and b/docs/docs/image/add_sourceconfig_xdcr.png differ diff --git a/docs/docs/image/architecture_backup.png b/docs/docs/image/architecture_backup.png new file mode 100644 index 0000000..82b7229 Binary files /dev/null and b/docs/docs/image/architecture_backup.png differ diff --git a/docs/docs/image/architecture_xdcr.png b/docs/docs/image/architecture_xdcr.png new file mode 100644 index 0000000..192b3f7 Binary files /dev/null and b/docs/docs/image/architecture_xdcr.png differ diff --git a/docs/docs/image/compability_couchbase.png b/docs/docs/image/compability_couchbase.png new file mode 100644 index 0000000..18cb738 Binary files /dev/null and b/docs/docs/image/compability_couchbase.png differ diff --git a/docs/docs/image/compability_os.png b/docs/docs/image/compability_os.png new file mode 100644 index 0000000..79c16c2 Binary files /dev/null and b/docs/docs/image/compability_os.png differ diff --git a/docs/docs/image/couchbase_target.png b/docs/docs/image/couchbase_target.png new file mode 100644 index 0000000..db729fb Binary files /dev/null and b/docs/docs/image/couchbase_target.png differ diff --git a/docs/docs/image/dsource_ingested.png b/docs/docs/image/dsource_ingested.png new file mode 100644 index 0000000..991c3f7 Binary files /dev/null and b/docs/docs/image/dsource_ingested.png differ diff --git a/docs/docs/image/dsource_prod_view.png b/docs/docs/image/dsource_prod_view.png new file mode 100644 index 0000000..f246d14 Binary files /dev/null and b/docs/docs/image/dsource_prod_view.png differ diff --git a/docs/docs/image/dsource_staging_view.png b/docs/docs/image/dsource_staging_view.png new file mode 100644 index 0000000..4456ae5 Binary files /dev/null and b/docs/docs/image/dsource_staging_view.png differ diff --git a/docs/docs/image/env_with_sourceconfig.png b/docs/docs/image/env_with_sourceconfig.png new file mode 100644 index 0000000..f371a06 Binary files /dev/null and b/docs/docs/image/env_with_sourceconfig.png differ diff --git a/docs/docs/image/ports_backup.png b/docs/docs/image/ports_backup.png new file mode 100644 index 0000000..9502ad3 Binary files /dev/null and b/docs/docs/image/ports_backup.png differ diff --git a/docs/docs/image/ports_xdcr.png b/docs/docs/image/ports_xdcr.png new file mode 100644 index 0000000..fb7d92c Binary files /dev/null and b/docs/docs/image/ports_xdcr.png differ diff --git a/docs/docs/image/provision_1.png b/docs/docs/image/provision_1.png new file mode 100644 index 0000000..5bfffda Binary files /dev/null and b/docs/docs/image/provision_1.png differ diff --git a/docs/docs/image/provision_2.png b/docs/docs/image/provision_2.png new file mode 100644 index 0000000..5debb29 Binary files /dev/null and b/docs/docs/image/provision_2.png differ diff --git a/docs/docs/image/provision_3.png b/docs/docs/image/provision_3.png new file mode 100644 index 0000000..e32a459 Binary files /dev/null and b/docs/docs/image/provision_3.png differ diff --git a/docs/docs/image/provision_3_mt1.png b/docs/docs/image/provision_3_mt1.png new file mode 100644 index 0000000..1e525bd Binary files /dev/null and b/docs/docs/image/provision_3_mt1.png differ diff --git a/docs/docs/image/provision_3_mt2.png b/docs/docs/image/provision_3_mt2.png new file mode 100644 index 0000000..717a113 Binary files /dev/null and b/docs/docs/image/provision_3_mt2.png differ diff --git a/docs/docs/image/provision_4.png b/docs/docs/image/provision_4.png new file mode 100644 index 0000000..0f463ff Binary files /dev/null and b/docs/docs/image/provision_4.png differ diff --git a/docs/docs/image/provision_5.png b/docs/docs/image/provision_5.png new file mode 100644 index 0000000..9030378 Binary files /dev/null and b/docs/docs/image/provision_5.png differ diff --git a/docs/docs/image/provision_6.png b/docs/docs/image/provision_6.png new file mode 100644 index 0000000..b67e0f4 Binary files /dev/null and b/docs/docs/image/provision_6.png differ diff --git a/docs/docs/image/provision_7.png b/docs/docs/image/provision_7.png new file mode 100644 index 0000000..2beeb5e Binary files /dev/null and b/docs/docs/image/provision_7.png differ diff --git a/docs/docs/image/provision_8.png b/docs/docs/image/provision_8.png new file mode 100644 index 0000000..aa03c4f Binary files /dev/null and b/docs/docs/image/provision_8.png differ diff --git a/docs/docs/image/provision_9.png b/docs/docs/image/provision_9.png new file mode 100644 index 0000000..5fc76f8 Binary files /dev/null and b/docs/docs/image/provision_9.png differ diff --git a/docs/docs/image/vdb_config.png b/docs/docs/image/vdb_config.png new file mode 100644 index 0000000..580685d Binary files /dev/null and b/docs/docs/image/vdb_config.png differ diff --git a/docs/docs/index.md b/docs/docs/index.md index 7843367..cd11961 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -1,56 +1,163 @@ # Overview -Couchbase plugin is developed to virtualize Couchbase data source leveraging the following built-in couchbase technologies: +Couchbase plugin is developed to virtualize Couchbase data source. The ingestion (synchronization process) leveraging the following built-in Couchbase technologies +depends on the Couchbase Edition: +- Community Edition: + - Cross Data Center Replication (XDCR) -- Cross Data Center Replication (XDCR) allows data to be replicated across clusters that are potentially located in different data centers. -- Cbbackupmgr allows data to be restored on staging host of Couchbase Server. +- Enterprise Edition: + - existing backup manged by `cbbackupmgr` tool (Zero production touch ingestion) + - Cross Data Center Replication (XDCR) -Ingesting Couchbase ----------------- -1. Couchbase cluster/single instance using (XDCR ingestion mechanism). -2. Couchbase backup piece using (CBBACKUPMGR ingestion mechanism) - Zero Touch Production. +### Cross Data Center Replication (XDCR) -### Prerequisites -**Source Requirements:** Couchbase database user with the following privileges: +Cross Data Center Replication is a method to replicate date between source and target bucket. Plugin is automatically setting up an one way replication +from a production Couchbase cluster to a staging Couchbase cluster created by Delphix Enging during a dSource ingestion. +Replication to staging server will be added to the existing list of replication and it will be managed by plugin itself. + +[link to official XDCR documentation](https://docs.couchbase.com/server/current/learn/clusters-and-availability/xdcr-overview.html) + + +### Exiting backup ingestion ( `cbbackupmgr` ) + +Couchbase Enterprise Edition is providing an additional tool called `cbbackupmgr`. +This tool can be leveraged to protect a production Couchbase cluster and an existing backup +will be used to create a staging server. This method allow cloning a production Couchbase cluster +without touching a production server by Delphix Engine nor staging server. In zero production touch +setup staging server has to have access to cbbackupmgr archive folder and repository. + +[link to official cbbackupmgr documentation](https://docs.couchbase.com/server/6.6/backup-restore/backup-restore.html) + + +# Architecture diagrams + +### Ingestion using XDCR + + +![Architecture for XDRC](./image/architecture_xdcr.png) + +### Ingestion using backup + + +![Architecture for cbbackupmgr](./image/architecture_backup.png) + +# Support Matrix + +![Support matrix Couchbase](./image/compability_couchbase.png) + +![Support matrix OS](./image/compability_os.png) + + +# Prerequisites + +## Staging environment + +1. Couchbase binaries installed and configured: + * disable a auto start using OS services + + `systemctl disable couchbase-server.service` + `systemctl stop couchbase-server.service` + +2. Regular o/s user - ex. `delphix_os` +3. Add OS user to `couchbase` OS group +4. Empty folder on host to hold delphix toolkit [ approximate 2GB free space ]. +5. Empty folder on host to mount nfs filesystem. This is just an empty folder with no space requirements and acts as a base folder for NFS mounts. +6. sudo privileges for mount and umount. See sample below assuming `delphix_os` is used as delphix user. + + ``` + Defaults:delphixos !requiretty + delphixos ALL=NOPASSWD: \ + /bin/mount, /bin/umount + ``` + +7. If Couchbase service is installed using `couchbase` user, Delphix OS user ex. `delphix_os` has to be able to run any command as `couchbase` using sudo + + ``` + delphix_os ALL=(couchbase) NOPASSWD: ALL + ``` + + +## Additional prerequisites for XDCR ingestion + +Production Couchbase database user with the following provileges * XDCR_ADMIN * DATA_MONITOR -**Staging Requirements**: O/S user with the following privileges: - -1. Regular o/s user. -2. Execute access on couchbase binaries [ chmod -R 775 /opt/couchbase ]. -3. Empty folder on host to hold delphix toolkit [ approximate 2GB free space ]. -4. Empty folder on host to mount nfs filesystem. This is just an empty folder with no space requirements and acts as a base folder for NFS mounts. -5. sudo privileges for mount and umount. See sample below assuming `delphix_os` is used as delphix user. - ```shell - Defaults:delphixos !requiretty - delphixos ALL=NOPASSWD: \ - /bin/mount, /bin/umount - ``` -6. Customers who intend to use CBBACKUPMGR (Couchbase backup manager ingestion) must follow the instructions to avoid source/production server dependency. - - * Provide all source server buckets related information (using the command below) in a file and place where `/couchbase_src_bucket_info.cfg`: - `/opt/couchbase/bin/couchbase-cli bucket-list --cluster :8091 --username $username --password $password` - - * Create config file using the following command. This file will be required at the time of dSource creation using CBBACKUPMGR. - `/opt/couchbase/bin/cbbackupmgr config --archive /u01/couchbase_backup --repo delphix` - - * Get data from source host in backup directory of staging host - `/opt/couchbase/bin/cbbackupmgr backup -a /u01/couchbase_backup -r delphix -c couchbase:// -u user -p password` +If source is a Community Edition, a production Couchbase database user has to be a full admin privilege +to be able to create and monitor XDCR replication + +## Additional prerequisites for backup ingestion + +Access to an existing backup on the staging server. +If access is provided over NFS to an existing backup (ex. mount path is /u01/couchbase_backup ) and repository is called PROD the following command have to succeed: + + `/opt/couchbase/bin/cbbackupmgr config --archive /u01/couchbase_backup --repo PROD` + +If a new backup will configured on the staging host, the following commands should be executed. Ex. backup location `/u01/couchbase_backup`, repository name `delphix` + +* Create config file using the following command. This file will be required at the time of dSource creation using CBBACKUPMGR. + `/opt/couchbase/bin/cbbackupmgr config --archive /u01/couchbase_backup --repo delphix` + +* Bring backup of the production system: + * Copy an existing backup + * Backup from source host into backup directory of staging host + `/opt/couchbase/bin/cbbackupmgr backup -a /u01/couchbase_backup -r delphix -c couchbase:// -u user -p password` + + + +## Target environment + +1. Couchbase binaries installed and configured: + * disable a auto start using OS services + + `systemctl disable couchbase-server.service` + `systemctl stop couchbase-server.service` + +2. Regular o/s user - ex. `delphix_os` +3. Add OS user to `couchbase` OS group +4. Empty folder on host to hold delphix toolkit [ approximate 2GB free space ]. +5. Empty folder on host to mount nfs filesystem. This is just an empty folder with no space requirements and acts as a base folder for NFS mounts. +6. sudo privileges for mount and umount. See sample below assuming `delphix_os` is used as delphix user. + + ```bash + Defaults:delphixos !requiretty + delphixos ALL=NOPASSWD: \ + /bin/mount, /bin/umount + ``` + +7. If Couchbase service is installed using `couchbase` user, Delphix OS user ex. `delphix_os` has to be able to run any command as `couchbase` using sudo + ```shell + delphix_os ALL=(couchbase) NOPASSWD: ALL + ``` + + +## Ports + +### Ports for XDCR + +![Ports for XDCR](./image/ports_xdcr.png) + +### Ports for Backup + +![Ports for Backup](./image/ports_backup.png) + + + +# Limitations + +* Multi-node VDB can't be cloned. +* V2P is not supported +* Point in time recovery is not supported. Provision / refresh / rewind time is based on snapshot only + + + + -**Target Requirements**: O/S user with the following privileges: -1. Regular o/s user. -2. Execute access on couchbase binaries `[ chmod -R 775 /opt/couchbase ]`. -3. Empty folder on host to hold Delphix toolkit `[ approximate 2GB free space ]`. -4. Empty folder on host to mount nfs filesystem. This is just an empty folder with no space requirements and act as a base folder for NFS mounts. -5. sudo privileges for mount and umount. See sample below assuming `delphix_os` is used as delphix user. - `Defaults:delphixos !requiretty` - `delphixos ALL=NOPASSWD: /bin/mount, /bin/umount` diff --git a/docs/docs/stylesheets/extra.css b/docs/docs/stylesheets/extra.css new file mode 100644 index 0000000..ec6904f --- /dev/null +++ b/docs/docs/stylesheets/extra.css @@ -0,0 +1,9 @@ + +.md-header-nav__button.md-logo img { + display: block; + width: 157px; + height: 15px; + fill: currentColor; + margin-top: 5px; +} + diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 975473f..8d9a266 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,12 +1,10 @@ -site_name: Couchbase Plugin +site_name: "Couchbase Plugin" theme: name: material - custom_dir: 'material/' palette: - primary: + primary: light blue accent: logo: 'image/delphix-logo-white.png' - favicon: 'image/logo.png' font: text: Helvetica Neue code: Ubuntu Mono @@ -20,33 +18,26 @@ google_analytics: repo_url: https://github.com/delphix/couchbase-plugin repo_name: Github use_directory_urls: false -pages: - - Overview: 'index.md' - - Installation: 'Installation.md' - - Plugin Development : 'PluginDevelopment.md' - - Discovery: 'Discovery.md' - - Linking: 'Linking.md' - - Provisioning: 'Provisioning.md' + extra: social: - - type: sitemap - link: https://www.delphix.com/ - - type: facebook + - icon: fontawesome/brands/facebook link: https://www.facebook.com/DelphixCorp/ - - type: twitter + - icon: fontawesome/brands/twitter link: https://twitter.com/delphix - - type: linkedin + - icon: fontawesome/brands/linkedin link: https://www.linkedin.com/company/delphix - - type: github + - icon: fontawesome/brands/github link: https://github.com/delphix extra_css: - - 'stylesheets/extra.css' + - stylesheets/extra.css markdown_extensions: - toc: permalink: true + baselevel: 3 - admonition - codehilite: guess_lang: false @@ -54,3 +45,11 @@ markdown_extensions: plugins: - search + +nav: + - 'index.md' + - 'Installation.md' + - 'Discovery.md' + - 'Linking.md' + - 'Provisioning.md' + - 'Troubleshooting.md' \ No newline at end of file diff --git a/plugin_config.yml b/plugin_config.yml index 9278d98..57144f0 100644 --- a/plugin_config.yml +++ b/plugin_config.yml @@ -1,6 +1,7 @@ id: 18f4ff11-b758-4bf2-9a37-719a22f5a4b8 name: sc:couchbase -buildNumber: 1.1.0 +externalVersion: "1.1.3" +buildNumber: 1.1.3.0 language: PYTHON27 hostTypes: - UNIX diff --git a/schema.json b/schema.json index 048498c..dee5536 100644 --- a/schema.json +++ b/schema.json @@ -28,6 +28,16 @@ "type": "string", "prettyName": "Identity Name", "description": "Pretty name of this repository" + }, + "uid": { + "type": "integer", + "prettyName": "Couchbase User ID", + "description": "Couchbase User Identifier" + }, + "gid": { + "type": "integer", + "prettyName": "Couchbase Group ID", + "description": "Couchbase Group ID" } }, "nameField": "prettyName", @@ -106,9 +116,7 @@ "bucketEvictionPolicy", "couchbaseAdmin", "couchbaseAdminPassword", - "fts_service", - "analytics_service", - "eventing_service"], + "node_list"], "properties" : { "couchbasePort": { "type": "integer", @@ -180,23 +188,73 @@ "default": "" }, "fts_service": { - "default": true, - "type": "boolean", - "prettyName": "FTS Service", - "description": "" - }, - "analytics_service": { - "default": true, - "type": "boolean", - "prettyName": "Analytics Service", - "description": "" - }, - "eventing_service": { - "default": true, - "type": "boolean", - "prettyName": "Eventing Service", - "description": "" - } + "default": true, + "type": "boolean", + "prettyName": "FTS Service", + "description": "" + }, + "analytics_service": { + "default": true, + "type": "boolean", + "prettyName": "Analytics Service", + "description": "" + }, + "eventing_service": { + "default": true, + "type": "boolean", + "prettyName": "Eventing Service", + "description": "" + }, + "node_list": { + "type": "array", + "prettyName": "Additional Nodes", + "items": { + "type": "object", + "required": ["environment", "environmentUser", "node_addr"], + "ordering": ["environment", "environmentUser", "node_addr", "fts_service", "analytics_service", "eventing_service"], + "properties": { + "environment": { + "type": "string", + "format": "reference", + "referenceType": "UnixHostEnvironment", + "prettyName": "Delphix Environment name", + "description": "" + }, + "environmentUser": { + "type": "string", + "format": "reference", + "referenceType": "EnvironmentUser", + "prettyName": "Delphix Environment User", + "description": "", + "matches": "environment" + }, + "node_addr": { + "type": "string", + "prettyName": "Node hostname / IP", + "description": "", + "default": "" + }, + "fts_service": { + "default": false, + "type": "boolean", + "prettyName": "FTS Service", + "description": "" + }, + "analytics_service": { + "default": false, + "type": "boolean", + "prettyName": "Analytics Service", + "description": "" + }, + "eventing_service": { + "default": false, + "type": "boolean", + "prettyName": "Eventing Service", + "description": "" + } + } + } + } } }, "linkedSourceDefinition": { @@ -240,7 +298,6 @@ "xdcrAdmin", "xdcrAdminPassword", "fts_service", - "analytics_service", "eventing_service", "config_settings_prov" ], @@ -361,23 +418,23 @@ "default": "" }, "fts_service": { - "default": true, + "default": false, "type": "boolean", "prettyName": "FTS Service", "description": "" }, - "analytics_service": { - "default": true, - "type": "boolean", - "prettyName": "Analytics Service", - "description": "" - }, "eventing_service": { - "default": true, + "default": false, "type": "boolean", "prettyName": "Eventing Service", "description": "" }, + "analytics_service": { + "default": true, + "type": "boolean", + "prettyName": "Analytics Service", + "description": "" + }, "config_settings_prov" : { "type": "array", "prettyName": "Config Settings", @@ -436,6 +493,26 @@ }, "bucketList": { "type": "string" + }, + "snapshotPassword": { + "type": "string", + "format": "password" + }, + "indexes": { + "type": "array", + "items": [ + {"type": "string"} + ] + }, + "couchbaseAdmin": { + "type": "string", + "prettyName": "Source Couchbase Admin User", + "description": "" + }, + "couchbaseAdminPassword": { + "type": "string", + "format": "password", + "prettyName": "Source Couchbase Admin Password" } } }, diff --git a/src/controller/couchbase_lib/_bucket.py b/src/controller/couchbase_lib/_bucket.py index ae296f1..e90c1a1 100644 --- a/src/controller/couchbase_lib/_bucket.py +++ b/src/controller/couchbase_lib/_bucket.py @@ -11,8 +11,10 @@ import logging from utils import utilities import json +from os.path import join from internal_exceptions.database_exceptions import BucketOperationError from controller import helper_lib +from controller.helper_lib import remap_bucket_json from controller.couchbase_lib._mixin_interface import MixinInterface from controller.resource_builder import Resource from db_commands.commands import CommandFactory @@ -39,6 +41,7 @@ def bucket_edit(self, bucket_name, flush_value=1): env = _BucketMixin.generate_environment_map(self) command = CommandFactory.bucket_edit(bucket_name=bucket_name, flush_value=flush_value, **env) kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} + logger.debug("edit bucket {}".format(command)) return utilities.execute_bash(self.connection, command, **kwargs) def bucket_edit_ramquota(self, bucket_name, _ramsize): @@ -53,6 +56,7 @@ def bucket_edit_ramquota(self, bucket_name, _ramsize): env = _BucketMixin.generate_environment_map(self) command = CommandFactory.bucket_edit_ramquota(bucket_name=bucket_name, ramsize=_ramsize, **env) kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} + logger.debug("edit ram bucket {}".format(command)) return utilities.execute_bash(self.connection, command, **kwargs) def bucket_delete(self, bucket_name): @@ -62,6 +66,7 @@ def bucket_delete(self, bucket_name): env = _BucketMixin.generate_environment_map(self) command = CommandFactory.bucket_delete(bucket_name=bucket_name, **env) kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} + logger.debug("delete bucket {}".format(command)) return utilities.execute_bash(self.connection, command, **kwargs) def bucket_flush(self, bucket_name): @@ -71,6 +76,7 @@ def bucket_flush(self, bucket_name): env = _BucketMixin.generate_environment_map(self) command = CommandFactory.bucket_flush(bucket_name=bucket_name, **env) kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} + logger.debug("flush bucket {}".format(command)) return utilities.execute_bash(self.connection, command, **kwargs) def bucket_remove(self, bucket_name): @@ -81,31 +87,86 @@ def bucket_remove(self, bucket_name): self.bucket_delete(bucket_name) helper_lib.sleepForSecond(2) - def bucket_create(self, bucket_name, ram_size=0): + def bucket_create(self, bucket_name, ram_size, bucket_type, bucket_compression): logger.debug("Creating bucket: {} ".format(bucket_name)) # To create the bucket with given ram size self.__validate_bucket_name(bucket_name) if ram_size is None: logger.debug("Needed ramsize for bucket_create. Currently it is: {}".format(ram_size)) return + + + if bucket_type == 'membase': + # API return different type + bucket_type = 'couchbase' + + if bucket_compression is not None: + bucket_compression = '--compression-mode {}'.format(bucket_compression) + else: + bucket_compression = '' + policy = self.parameters.bucket_eviction_policy env = _BucketMixin.generate_environment_map(self) - command = CommandFactory.bucket_create(bucket_name=bucket_name, ramsize=ram_size, evictionpolicy=policy, **env) + command = CommandFactory.bucket_create(bucket_name=bucket_name, ramsize=ram_size, evictionpolicy=policy, bucket_type=bucket_type, bucket_compression=bucket_compression, **env) kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} + logger.debug("create bucket {}".format(command)) output, error, exit_code = utilities.execute_bash(self.connection, command, **kwargs) + logger.debug("create bucket output: {} {} {}".format(output, error, exit_code)) helper_lib.sleepForSecond(2) + def bucket_list(self, return_type=list): - # See the all bucket. It will return also other information like ramused, ramsize etc + # See the all bucket. + # It will return also other information like ramused, ramsize etc logger.debug("Finding staged bucket list") env = _BucketMixin.generate_environment_map(self) command = CommandFactory.bucket_list(**env) kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} + logger.debug("list bucket {}".format(command)) bucket_list, error, exit_code = utilities.execute_bash(self.connection, command, **kwargs) + logger.debug("list bucket output{}".format(bucket_list)) if return_type == list: - bucket_list = bucket_list.split("\n") + #bucket_list = bucket_list.split("\n") + if bucket_list == "[]" or bucket_list is None: + logger.debug("empty list") + return [] + else: + logger.debug("clean up json") + bucket_list = bucket_list.replace("u'","'") + bucket_list = bucket_list.replace("'", "\"") + bucket_list = bucket_list.replace("True", "\"True\"") + bucket_list = bucket_list.replace("False", "\"False\"") + logger.debug("parse json") + bucket_list_dict = json.loads(bucket_list) + logger.debug("remap json") + bucket_list_dict = map(helper_lib.remap_bucket_json, bucket_list_dict) logger.debug("Bucket details in staged environment: {}".format(bucket_list)) - return bucket_list + return bucket_list_dict + + + def move_bucket(self, bucket_name, direction): + logger.debug("Rename folder") + + + + if direction == 'save': + src = join(self.virtual_source.parameters.mount_path,'data',bucket_name) + dst = join(self.virtual_source.parameters.mount_path,'data',".{}.delphix".format(bucket_name)) + command = CommandFactory.os_mv(src, dst, self.need_sudo, self.uid) + logger.debug("rename command: {}".format(command)) + stdout, error, exit_code = utilities.execute_bash(self.connection, command) + elif direction == 'restore': + dst = join(self.virtual_source.parameters.mount_path,'data',bucket_name) + src = join(self.virtual_source.parameters.mount_path,'data',".{}.delphix".format(bucket_name)) + command = CommandFactory.delete_dir(dst, self.need_sudo, self.uid) + logger.debug("delete command: {}".format(command)) + stdout, error, exit_code = utilities.execute_bash(self.connection, command) + command = CommandFactory.os_mv(src, dst, self.need_sudo, self.uid) + logger.debug("rename command: {}".format(command)) + stdout, error, exit_code = utilities.execute_bash(self.connection, command) + + + def monitor_bucket(self, bucket_name, staging_UUID): # To monitor the replication diff --git a/src/controller/couchbase_lib/_cb_backup.py b/src/controller/couchbase_lib/_cb_backup.py index 4abb7d2..22a6d3d 100644 --- a/src/controller/couchbase_lib/_cb_backup.py +++ b/src/controller/couchbase_lib/_cb_backup.py @@ -14,6 +14,7 @@ from controller.resource_builder import Resource from db_commands.constants import ENV_VAR_KEY from db_commands.commands import CommandFactory +from dlpx.virtualization.platform.exceptions import UserError logger = logging.getLogger(__name__) @@ -32,32 +33,44 @@ def generate_environment_map(self): # MixinInterface.read_map(env) return env - # Defined for future updates - def get_indexes_name(self, index_name): - logger.debug("Finding indexes....") - env = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} - env = _CBBackupMixin.generate_environment_map(self) - cmd = CommandFactory.get_indexes_name(**env) - logger.debug("env detail is : ".format(env)) - command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd, **env) - logger.debug("Indexes are {}".format(command_output)) - return command_output - - # Defined for future updates - def build_index(self, index_name): - logger.debug("Building indexes....") - env = _CBBackupMixin.generate_environment_map(self) - cmd = CommandFactory.build_index(index_name=index_name, **env) - command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd, **env) - logger.debug("command_output is ".format(command_output)) - return command_output + def cb_backup_full(self, csv_bucket): logger.debug("Starting Restore via Backup file...") logger.debug("csv_bucket_list: {}".format(csv_bucket)) - kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} - env = _CBBackupMixin.generate_environment_map(self) - cmd = CommandFactory.cb_backup_full(backup_location=self.parameters.couchbase_bak_loc, - csv_bucket_list=csv_bucket, - backup_repo=self.parameters.couchbase_bak_repo, **env) - utilities.execute_bash(self.connection, cmd, **kwargs) + + skip = '--disable-analytics' + + if self.parameters.fts_service != True: + skip = skip + ' {} {} '.format('--disable-ft-indexes','--disable-ft-alias') + + if self.parameters.eventing_service != True: + skip = skip + ' {} '.format('--disable-eventing') + + + logger.debug("skip backup is set to: {}".format(skip)) + + # kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} + # env = _CBBackupMixin.generate_environment_map(self) + + # cmd = CommandFactory.cb_backup_full(backup_location=self.parameters.couchbase_bak_loc, + # csv_bucket_list=csv_bucket, + # backup_repo=self.parameters.couchbase_bak_repo, + # need_sudo=self.need_sudo, uid=self.uid, + # skip=skip, + # **env) + # logger.debug("Backup restore: {}".format(cmd)) + # utilities.execute_bash(self.connection, cmd, **kwargs) + + + stdout, stderr, exit_code = self.run_couchbase_command(couchbase_command='cb_backup_full', + backup_location=self.parameters.couchbase_bak_loc, + csv_bucket_list=csv_bucket, + backup_repo=self.parameters.couchbase_bak_repo, + skip=skip, + base_path=helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path) + ) + + if exit_code != 0: + raise UserError("Problem with restoring backup using cbbackupmgr", "Check if repo and all privileges are correct", + "stdout: {}, stderr: {}, exit_code: {}".format(stdout, stderr, exit_code)) diff --git a/src/controller/couchbase_lib/_cluster.py b/src/controller/couchbase_lib/_cluster.py index 89a8461..d105d89 100644 --- a/src/controller/couchbase_lib/_cluster.py +++ b/src/controller/couchbase_lib/_cluster.py @@ -11,6 +11,7 @@ import logging import re from utils import utilities +from controller.helper_lib import sleepForSecond from db_commands.commands import CommandFactory from controller.couchbase_lib._mixin_interface import MixinInterface from db_commands.constants import ENV_VAR_KEY @@ -52,16 +53,16 @@ def cluster_init(self): # Cluster initialization logger.debug("Cluster Initialization started") fts_service = self.parameters.fts_service - analytics_service = self.parameters.analytics_service + #analytics_service = self.parameters.analytics_service eventing_service = self.parameters.eventing_service cluster_name = self._get_cluster_name() kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} additional_service = "query" - if fts_service: + if fts_service == True: additional_service = additional_service + ",fts" - if analytics_service: - additional_service = additional_service + ",analytics" - if eventing_service: + # if analytics_service: + # additional_service = additional_service + ",analytics" + if eventing_service == True: additional_service = additional_service + ",eventing" logger.debug("additional services : {}".format(additional_service)) @@ -69,6 +70,7 @@ def cluster_init(self): env = _ClusterMixin.generate_environment_map(self) env['additional_services'] = additional_service cmd = CommandFactory.cluster_init(cluster_name=cluster_name, **env) + logger.debug("Cluster init: {}".format(cmd)) stdout, stderr, exit_code = utilities.execute_bash(self.connection, command_name=cmd, callback_func=lambda_expr, **kwargs) if re.search(r"ERROR", str(stdout)): @@ -80,6 +82,9 @@ def cluster_init(self): raise Exception(stdout) else: logger.debug("Cluster init succeeded") + + # here we should wait for indexer to start + sleepForSecond(10) return [stdout, stderr, exit_code] def cluster_setting(self): diff --git a/src/controller/couchbase_lib/_replication.py b/src/controller/couchbase_lib/_replication.py deleted file mode 100644 index f23a1aa..0000000 --- a/src/controller/couchbase_lib/_replication.py +++ /dev/null @@ -1,169 +0,0 @@ -# -# Copyright (c) 2020 by Delphix. All rights reserved. -# -####################################################################################################################### -""" -This class contains methods for replication related operations -This is child class of Resource and parent class of CouchbaseOperation -""" -####################################################################################################################### - -from utils import utilities -import re -import logging -from db_commands.commands import CommandFactory -from controller.couchbase_lib._mixin_interface import MixinInterface -from db_commands.constants import ENV_VAR_KEY -from controller.resource_builder import Resource -logger = logging.getLogger(__name__) - - -class _ReplicationMixin(Resource, MixinInterface): - - def __init__(self, builder): - super(_ReplicationMixin, self).__init__(builder) - - @MixinInterface.check_attribute_error - def generate_environment_map(self): - env = {'shell_path': self.repository.cb_shell_path, 'source_hostname': self.source_config.couchbase_src_host, - 'source_port': self.source_config.couchbase_src_port, 'source_username': self.parameters.xdcr_admin} - # MixinInterface.read_map(env) - return env - - def get_replication_uuid(self): - # False for string - logger.debug("Finding the replication uuid through host name") - is_ip_or_string = False - kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}} - cluster_name = self.parameters.stg_cluster_name - env = _ReplicationMixin.generate_environment_map(self) - cmd = CommandFactory.get_replication_uuid(**env) - try: - - stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs) - if stdout is None or stdout == "": - logger.debug("No Replication ID identified") - return None, None - logger.debug("xdcr remote references : {}".format(stdout)) - hostname = self.connection.environment.host.name - logger.debug("Environment hostname {}".format(hostname)) - host_ip = "" - if not re.search(r"{}".format(hostname), stdout): - logger.debug("cluster for hostname {} doesn't exist".format(hostname)) - logger.debug("Finding the ip for this host") - host_ip = self.get_ip() - logger.debug("Finding the replication uuid through host ip") - if not re.search(r"{}".format(host_ip), stdout): - logger.debug("cluster for host_ip {} doesn't exist".format(hostname)) - return None, None - else: - is_ip_or_string = True - logger.debug("cluster for host_ip {} exist".format(host_ip)) - else: - logger.debug("cluster for hostname {} exist".format(host_ip)) - if is_ip_or_string == False: - uuid = re.search(r"uuid:.*(?=\s.*{})".format(hostname), stdout).group() - else: - uuid = re.search(r"uuid:.*(?=\s.*{})".format(host_ip), stdout).group() - - uuid = uuid.split(":")[1].strip() - cluster_name_staging = re.search(r"cluster name:.*(?=\s.*{})".format(uuid), stdout).group() - cluster_name_staging = cluster_name_staging.split(":")[1].strip() - logger.debug("uuid for {} cluster : {}".format(uuid, cluster_name_staging)) - if cluster_name_staging == cluster_name: - return uuid, cluster_name - else: - return uuid, cluster_name_staging - except Exception as err: - logger.warn("Error identified: {} ".format(err.message)) - logger.warn("UUID is None. Not able to find any cluster") - return None, None - - def get_stream_id(self): - logger.debug("Finding the stream id for provided cluster name") - kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}} - uuid, cluster_name = self.get_replication_uuid() - if uuid is None: - return None, None - env = _ReplicationMixin.generate_environment_map(self) - cmd = CommandFactory.get_stream_id(cluster_name=cluster_name, **env) - stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs) - if stdout is None or stdout == "": - logger.debug("No stream ID identified") - return None, None - else: - stream_id = re.findall(r"(?<=stream id: ){}.*".format(uuid), stdout) - logger.debug("Stream id found: {}".format(stream_id)) - return stream_id, cluster_name - - def pause_replication(self): - logger.debug("Pausing replication ...") - stream_id, cluster_name = self.get_stream_id() - kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}} - env = _ReplicationMixin.generate_environment_map(self) - for replication_id in stream_id: - cmd = CommandFactory.pause_replication(cluster_name=cluster_name, id=replication_id, **env) - stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs) - logger.debug(stdout) - - def resume_replication(self): - logger.debug("Resuming replication ...") - stream_id, cluster_name = self.get_stream_id() - kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}} - env = _ReplicationMixin.generate_environment_map(self) - for s_id in stream_id: - cmd = CommandFactory.resume_replication(cluster_name=cluster_name, id=s_id , **env) - stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs) - logger.debug(stdout) - - def delete_replication(self): - logger.debug("Deleting replication...") - stream_id, cluster_name = self.get_stream_id() - logger.debug("stream_id: {} and cluster_name : {}".format(stream_id, cluster_name)) - if stream_id is None or stream_id == "": - logger.debug("No Replication is found to delete.") - return False, cluster_name - kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}} - env = _ReplicationMixin.generate_environment_map(self) - - for id in stream_id: - cmd = CommandFactory.delete_replication(cluster_name=cluster_name, id=id, **env) - stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs) - if exit_code != 0: - logger.warn("stream_id: {} deletion failed".format(id)) - else: - logger.debug("stream_id: {} deletion succeeded".format(id)) - return True, cluster_name - - def get_ip(self): - cmd = CommandFactory.get_ip_of_hostname() - stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd) - logger.debug("IP is {}".format(stdout)) - return stdout - - def check_duplicate_replication(self, cluster_name): - logger.debug("Searching cluster name") - kwargs = {ENV_VAR_KEY: {'source_password': self.parameters.xdcr_admin_password}} - env = _ReplicationMixin.generate_environment_map(self) - cmd = CommandFactory.get_replication_uuid(**env) - try: - stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd, **kwargs) - all_clusters = re.findall(r'cluster name:.*', stdout) - stream_id, cluster = self.get_stream_id() - logger.debug("stream_id:{} and cluster s:{} ".format(stream_id, cluster)) - if stream_id: - # cluster is already set up between these nodes# No setup and no mis match - logger.debug("Already XDCR set up have been between source and staging server") - return True, False - logger.debug("No XDCR for staging host. Now validating the cluster name... ") - for each_cluster_pair in all_clusters: - each_cluster = each_cluster_pair.split(':')[1].strip() - logger.debug("Listed cluster: {} and input is:{} ".format(each_cluster, cluster_name)) - if each_cluster == cluster_name: - logger.debug("Duplicate cluster name issue identified ") - # no setup but mismatch - return False, True - return False, False - - except Exception as err: - logger.debug("Failed to verify the duplicate name: {} ".format(err.message)) diff --git a/src/controller/couchbase_lib/_xdcr.py b/src/controller/couchbase_lib/_xdcr.py index 6effcb2..8b181e7 100644 --- a/src/controller/couchbase_lib/_xdcr.py +++ b/src/controller/couchbase_lib/_xdcr.py @@ -15,7 +15,7 @@ from db_commands.commands import CommandFactory from controller.couchbase_lib._mixin_interface import MixinInterface from controller.resource_builder import Resource - +from dlpx.virtualization.platform.exceptions import UserError from db_commands.constants import ENV_VAR_KEY logger = logging.getLogger(__name__) @@ -80,3 +80,185 @@ def xdcr_replicate(self, src, tgt): helper_lib.sleepForSecond(2) except Exception as e: logger.debug("XDCR error {}".format(e.message)) + + + def get_replication_uuid(self): + # False for string + logger.debug("Finding the replication uuid through host name") + is_ip_or_string = False + kwargs = {ENV_VAR_KEY: {}} + cluster_name = self.parameters.stg_cluster_name + + stdout, stderr, exit_code = self.run_couchbase_command('get_replication_uuid', + source_hostname=self.source_config.couchbase_src_host, + source_port=self.source_config.couchbase_src_port, + source_username=self.parameters.xdcr_admin, + source_password=self.parameters.xdcr_admin_password + ) + + + if exit_code != 0 or stdout is None or stdout == "": + logger.debug("No Replication ID identified") + return None + + try: + + logger.debug("xdcr remote references : {}".format(stdout)) + stg_hostname = self.connection.environment.host.name + logger.debug("Environment hostname {}".format(stg_hostname)) + # it can have more than single IP address + host_ips = self.get_ip() + # conver output into variables + clusters = {} + l = stdout.split("\n") + while l: + line = l.pop(0) + g = re.match(r"\s*cluster name:\s(\S*)", line) + if g: + xdrc_cluster_name = g.group(1) + uuid = re.match(r"\s*uuid:\s(\S*)", l.pop(0)).group(1) + hostname = re.match(r"\s*host name:\s(\S*):(\d*)", l.pop(0)).group(1) + user_name = l.pop(0) + uri = l.pop(0) + clusters[xdrc_cluster_name.lower()] = { + "hostname": hostname, + "uuid": uuid + } + + # check if a cluster name is really connected to staging - just in case + + if cluster_name.lower() in clusters: + logger.debug("Cluster {} found in xdrc-setup output".format(cluster_name)) + # check if hostname returned from source match hostname or IP's of staging server + + logger.debug(stg_hostname) + logger.debug(clusters[cluster_name.lower()]["hostname"]) + + if stg_hostname == clusters[cluster_name.lower()]["hostname"]: + # hostname matched + logger.debug("Cluster {} hostname {} is matching staging server hostname".format(cluster_name, stg_hostname)) + uuid = clusters[cluster_name.lower()]["uuid"] + else: + # check for IP's + logger.debug("Checking for IP match") + + logger.debug(clusters[cluster_name.lower()]) + + + if clusters[cluster_name.lower()]["hostname"] in host_ips: + # ip matched + logger.debug("Cluster {} IP {} is matching staging server IPs {}".format(cluster_name, clusters[cluster_name.lower()]["hostname"], host_ips)) + uuid = clusters[cluster_name.lower()]["uuid"] + else: + logger.debug("Can't confirm that xdrc-setup is matching staging") + raise UserError("XDRC Remote cluster {} on the source server is not pointed to staging server".format(cluster_name), + "Please check and delete remote cluster definition", clusters[cluster_name.lower()]) + + else: + logger.debug("Cluster {} configuration not found in XDCR of source".format(cluster_name)) + return None + + + logger.debug("uuid for {} cluster : {}".format(uuid, cluster_name)) + return uuid + + except UserError: + raise + except Exception as err: + logger.warn("Error identified: {} ".format(err.message)) + logger.warn("UUID is None. Not able to find any cluster") + return None + + + def get_stream_id(self): + logger.debug("Finding the stream id for provided cluster name") + uuid = self.get_replication_uuid() + if uuid is None: + return None + cluster_name = self.parameters.stg_cluster_name + + stdout, stderr, exit_code = self.run_couchbase_command('get_stream_id', + source_hostname=self.source_config.couchbase_src_host, + source_port=self.source_config.couchbase_src_port, + source_username=self.parameters.xdcr_admin, + source_password=self.parameters.xdcr_admin_password, + cluster_name=cluster_name + ) + + logger.debug(stdout) + logger.debug(uuid) + if exit_code != 0 or stdout is None or stdout == "": + logger.debug("No stream ID identified") + return None + else: + stream_id = re.findall(r"(?<=stream id:\s){}.*".format(uuid), stdout) + logger.debug("Stream id found: {}".format(stream_id)) + return stream_id + + + def delete_replication(self): + logger.debug("Deleting replication...") + stream_id = self.get_stream_id() + cluster_name = self.parameters.stg_cluster_name + logger.debug("stream_id: {} and cluster_name : {}".format(stream_id, cluster_name)) + if stream_id is None or stream_id == "": + logger.debug("No Replication is found to delete.") + return False, cluster_name + + for id in stream_id: + stdout, stderr, exit_code = self.run_couchbase_command('delete_replication', + source_hostname=self.source_config.couchbase_src_host, + source_port=self.source_config.couchbase_src_port, + source_username=self.parameters.xdcr_admin, + source_password=self.parameters.xdcr_admin_password, + cluster_name=cluster_name, + id=id + ) + + if exit_code != 0: + logger.warn("stream_id: {} deletion failed".format(id)) + else: + logger.debug("stream_id: {} deletion succeeded".format(id)) + return True, cluster_name + + def get_ip(self): + cmd = CommandFactory.get_ip_of_hostname() + stdout, stderr, exit_code = utilities.execute_bash(self.connection, cmd) + logger.debug("IP is {}".format(stdout)) + return stdout.split() + + + def setup_replication(self): + uuid = self.get_replication_uuid() + + if uuid is None: + logger.info("Setting up XDRC remote cluster") + self.xdcr_setup() + + + streams_id = self.get_stream_id() + + if streams_id is not None: + alredy_replicated_buckets = [ m.group(1) for m in ( re.match(r'\S*/(\S*)/\S*', x) for x in streams_id ) if m ] + else: + alredy_replicated_buckets = [] + + config_setting = self.staged_source.parameters.config_settings_prov + + if len(config_setting) > 0: + bucket_list = [ config_bucket["bucketName"] for config_bucket in config_setting ] + else: + bucket_details_source = self.source_bucket_list() + bucket_list = helper_lib.filter_bucket_name_from_json(bucket_details_source) + + logger.debug("Bucket list to create replication for") + logger.debug(bucket_list) + logger.debug("Already replicated buckets") + logger.debug(alredy_replicated_buckets) + + for bkt_name in bucket_list: + if bkt_name not in alredy_replicated_buckets: + logger.debug("Creating replication for {}".format(bkt_name)) + self.xdcr_replicate(bkt_name, bkt_name) + else: + logger.debug("Bucket {} replication already configured".format(bkt_name)) \ No newline at end of file diff --git a/src/controller/couchbase_operation.py b/src/controller/couchbase_operation.py index c300ade..124a95a 100644 --- a/src/controller/couchbase_operation.py +++ b/src/controller/couchbase_operation.py @@ -1,10 +1,10 @@ # -# Copyright (c) 2020 by Delphix. All rights reserved. +# Copyright (c) 2020,2021 by Delphix. All rights reserved. # ####################################################################################################################### """ -This class defines methods for couchbase operations. Parent classes are: _BucketMixin, _ClusterMixin, _ReplicationMixin, +This class defines methods for couchbase operations. Parent classes are: _BucketMixin, _ClusterMixin, _XDCrMixin, _CBBackupMixin. Modules name is explaining about the operations for which module is created for. The constructor of this class expects a `builder` on which each database operation will be performed Commands are defined for each method in module commands.py. To perform any delphix operation we need to create @@ -16,6 +16,8 @@ import logging import os import sys +import json +import inspect from dlpx.virtualization.platform import Status @@ -25,46 +27,140 @@ from controller import helper_lib from controller.couchbase_lib._bucket import _BucketMixin from controller.couchbase_lib._cluster import _ClusterMixin -from controller.couchbase_lib._replication import _ReplicationMixin from controller.couchbase_lib._xdcr import _XDCrMixin from controller.couchbase_lib._cb_backup import _CBBackupMixin from db_commands.commands import CommandFactory from db_commands.constants import ENV_VAR_KEY, StatusIsActive, DELPHIX_HIDDEN_FOLDER, CONFIG_FILE_NAME +from controller.helper_lib import remap_bucket_json import time +from db_commands import constants + +from dlpx.virtualization.platform.exceptions import UserError logger = logging.getLogger(__name__) -class CouchbaseOperation(_BucketMixin, _ClusterMixin, _ReplicationMixin, _XDCrMixin, _CBBackupMixin): +class CouchbaseOperation(_BucketMixin, _ClusterMixin, _XDCrMixin, _CBBackupMixin): - def __init__(self, builder): + def __init__(self, builder, node_connection=None): """ Main class through which other modules can run databases operations on provided parameters :param builder: builder object which contains all necessary parameters on which db methods will be executed + :param node_connection: connection to node, if this is not a default one """ + logger.debug("Object initialization") # Initializing the parent class constructor super(CouchbaseOperation, self).__init__(builder) - def restart_couchbase(self): + if node_connection is not None: + self.connection = node_connection + + self.__need_sudo = helper_lib.need_sudo(self.connection, self.repository.uid, self.repository.gid) + self.__uid = self.repository.uid + self.__gid = self.repository.gid + + + @property + def need_sudo(self): + return self.__need_sudo + + @property + def uid(self): + return self.__uid + + @property + def gid(self): + return self.__gid + + + def run_couchbase_command(self, couchbase_command, **kwargs): + logger.debug('run_couchbase_command') + logger.debug('couchbase_command: {}'.format(couchbase_command)) + if "password" in kwargs: + password = kwargs.pop('password') + else: + password = self.parameters.couchbase_admin_password + + if "username" in kwargs: + username = kwargs.pop('username') + else: + username = self.parameters.couchbase_admin + + if "hostname" in kwargs: + hostname = kwargs.pop('hostname') + else: + hostname = self.connection.environment.host.name + + env = {"password": password} + + if "newpass" in kwargs: + # for setting a new password + env["newpass"] = kwargs.pop('newpass') + + if "source_password" in kwargs: + env["source_password"] = kwargs.pop('source_password') + + autoparams = [ "shell_path", "install_path", "username", "port", "sudo", "uid", "hostname"] + + new_kwargs = {k: v for k, v in kwargs.items() if k not in autoparams} + + method_to_call = getattr(CommandFactory, couchbase_command) + command = method_to_call(shell_path=self.repository.cb_shell_path, + install_path=self.repository.cb_install_path, + username=username, + port=self.parameters.couchbase_port, + sudo=self.need_sudo, + uid=self.uid, + hostname=hostname, + **new_kwargs) + + logger.debug("couchbase command to run: {}".format(command)) + stdout, stderr, exit_code = utilities.execute_bash(self.connection, command, environment_vars=env) + return [stdout.encode('utf-8'), stderr.encode('utf-8'), exit_code] + + + def run_os_command(self, os_command, **kwargs): + + + + method_to_call = getattr(CommandFactory, os_command) + command = method_to_call(sudo=self.need_sudo, + uid=self.uid, + **kwargs) + + logger.debug("os command to run: {}".format(command)) + stdout, stderr, exit_code = utilities.execute_bash(self.connection, command) + return [stdout.encode('utf-8'), stderr.encode('utf-8'), exit_code] + + + def restart_couchbase(self, provision=False): """stop the couchbase service and then start again""" self.stop_couchbase() - self.start_couchbase() + self.start_couchbase(provision) - def start_couchbase(self): + def start_couchbase(self, provision=False, no_wait=False): """ start the couchbase service""" logger.debug("Starting couchbase services") - command = CommandFactory.start_couchbase(self.repository.cb_install_path) - utilities.execute_bash(self.connection, command) + + self.run_couchbase_command('start_couchbase') server_status = Status.INACTIVE + helper_lib.sleepForSecond(10) + + if no_wait: + logger.debug("no wait - leaving start procedure") + return + #Waiting for one minute to start the server - end_time = time.time() + 60 + # for prox to investigate + end_time = time.time() + 3660 #break the loop either end_time is exceeding from 1 minute or server is successfully started while time.time() < end_time and server_status == Status.INACTIVE: helper_lib.sleepForSecond(1) # waiting for 1 second - server_status = self.status() # fetching status + server_status = self.status(provision) # fetching status + logger.debug("server status {}".format(server_status)) # if the server is not running even in 60 seconds, then stop the further execution if server_status == Status.INACTIVE: @@ -75,65 +171,227 @@ def stop_couchbase(self): """ stop the couchbase service""" try: logger.debug("Stopping couchbase services") - command = CommandFactory.stop_couchbase(self.repository.cb_install_path) - utilities.execute_bash(self.connection, command) + self.run_couchbase_command('stop_couchbase') + end_time = time.time() + 60 server_status = Status.ACTIVE while time.time() < end_time and server_status == Status.ACTIVE: helper_lib.sleepForSecond(1) # waiting for 1 second server_status = self.status() # fetching status + + + logger.debug("Leaving stop loop") if server_status == Status.ACTIVE: + logger.debug("Have failed to stop couchbase server") raise CouchbaseServicesError("Have failed to stop couchbase server") except CouchbaseServicesError as err: + logger.debug("Error: {}".format(err)) raise err except Exception as err: + logger.debug("Exception Error: {}".format(err)) if self.status() == Status.INACTIVE: logger.debug("Seems like couchbase service is not running. {}".format(err.message)) else: raise CouchbaseServicesError(err.message) - def status(self): - """Check the server status. Healthy or Warmup could be one status if the server is running""" + def ip_file_name(self): + + ip_file = "{}/../var/lib/couchbase/ip".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + + # check_file_command = CommandFactory.check_file(ip_file, sudo=self.need_sudo, uid=self.uid) + # check_ip_file, check_ip_file_err, exit_code = utilities.execute_bash(self.connection, check_file_command, callback_func=self.ignore_err) + + + check_ip_file, check_ip_file_err, exit_code = self.run_os_command( + os_command='check_file', + file_path=ip_file + ) + + if not (exit_code == 0 and "Found" in check_ip_file): + ip_file = "{}/../var/lib/couchbase/ip_start".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + + + logger.debug("IP file is {}".format(ip_file)) + return ip_file + + + def staging_bootstrap_status(self): + logger.debug("staging_bootstrap_status") + try: - command = CommandFactory.server_info(self.repository.cb_shell_path, self.connection.environment.host.name, - self.parameters.couchbase_port, self.parameters.couchbase_admin) - kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} - server_info, std_err, exit_code = utilities.execute_bash(self.connection, command, **kwargs) - status = helper_lib.get_value_of_key_from_json(server_info, 'status') + server_info_out, std_err, exit_code = self.run_couchbase_command( + couchbase_command='couchbase_server_info', + hostname='127.0.0.1' + ) + + + # kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} + # username = self.parameters.couchbase_admin + + + # command = CommandFactory.server_info(self.repository.cb_shell_path, '127.0.0.1', + # self.parameters.couchbase_port, username) + # logger.debug("Status command {}".format(command)) + # server_info, std_err, exit_code = utilities.execute_bash(self.connection, command, **kwargs) + + #logger.debug("Status output: {}".format(server_info)) + + status = helper_lib.get_value_of_key_from_json(server_info_out, 'status') if status.strip() == StatusIsActive: logger.debug("Server status is: {}".format("ACTIVE")) return Status.ACTIVE else: logger.debug("Server status is: {}".format("INACTIVE")) return Status.INACTIVE + + + + except Exception as error: + # TODO + # rewrite it + logger.debug("Exception: {}".format(str(error))) + if re.search("Unable to connect to host at", error.message): + logger.debug("Couchbase service is not running") + return Status.INACTIVE + + def status(self, provision=False): + """Check the server status. Healthy or Warmup could be one status if the server is running""" + + logger.debug("checking status") + logger.debug(self.connection) + try: + + if provision==True: + username = self.snapshot.couchbase_admin + password = self.snapshot.couchbase_admin_password + + else: + password = self.parameters.couchbase_admin_password + username = self.parameters.couchbase_admin + + + #TODO + # Check if there is a mount point - even a started Couchbase without mountpoint means VDB + # is down or corrupted + # Couchbase with config file can start and recreate empty buckets if there is no mount point + # for future version - maybe whole /opt/couchbase/var directory should be virtualized like for Docker + # to avoid problems + + logger.debug("Checking for mount points") + mount_point_state = helper_lib.check_server_is_used(self.connection, self.parameters.mount_path) + logger.debug("Status of mount point {}".format(mount_point_state)) + + if mount_point_state == Status.INACTIVE: + logger.error("There is no mount point VDB is down regardless Couchbase status") + return Status.INACTIVE + + + ip_file = self.ip_file_name() + + # read_file_command = CommandFactory.cat(ip_file, sudo=self.need_sudo, uid=self.uid) + # logger.debug("read file command {}".format(read_file_command)) + + # read_ip_file, std_err, exit_code = utilities.execute_bash(self.connection, read_file_command) + # logger.debug("read file {}".format(read_ip_file)) + + + read_ip_file, std_err, exit_code = self.run_os_command( + os_command='cat', + path=ip_file + ) + + server_info, std_err, exit_code = self.run_couchbase_command( + couchbase_command='get_server_list', + hostname='127.0.0.1', + username=username, + password=password) + + #status = helper_lib.get_value_of_key_from_json(server_info, 'status') + + if self.dSource == False and self.parameters.node_list is not None and len(self.parameters.node_list) > 0: + multinode = True + else: + multinode = False + + + for line in server_info.split("\n"): + logger.debug("Checking line: {}".format(line)) + if read_ip_file in line: + logger.debug("Checking IP: {}".format(read_ip_file)) + if "unhealthy" in line: + logger.error("We have unhealthy active node") + return Status.INACTIVE + if "healthy" in line: + logger.debug("We have healthy active node") + return Status.ACTIVE + + if multinode and "warmup" in line: + logger.debug("We have starting mode in multinode cluster") + return Status.ACTIVE + + + return Status.INACTIVE + except Exception as error: + # TODO + # rewrite it + logger.debug("Exception: {}".format(str(error))) if re.search("Unable to connect to host at", error.message): logger.debug("Couchbase service is not running") return Status.INACTIVE - def make_directory(self, directory_path): + def make_directory(self, directory_path, force_env_user=False): """ Create a directory and set the permission level 775 :param directory_path: The directory path :return: None """ + + #TODO + # add error handling for OS errors + logger.debug("Creating Directory {} ".format(directory_path)) - env = {'directory_path': directory_path} - command = CommandFactory.make_directory(directory_path) - utilities.execute_bash(self.connection, command) + if force_env_user: + need_sudo = False + else: + need_sudo = self.need_sudo + + + command_output, std_err, exit_code = self.run_os_command( + os_command='make_directory', + directory_path=directory_path + ) + + # command = CommandFactory.make_directory(directory_path, need_sudo, self.uid) + # utilities.execute_bash(self.connection, command) + logger.debug("Changing permission of directory path {}".format(directory_path)) - command = CommandFactory.change_permission(directory_path) - utilities.execute_bash(self.connection, command) + + # command = CommandFactory.change_permission(directory_path, need_sudo, self.uid) + # utilities.execute_bash(self.connection, command) + + command_output, std_err, exit_code = self.run_os_command( + os_command='change_permission', + path=directory_path + ) + logger.debug("Changed the permission of directory") def create_config_dir(self): """create and return the hidden folder directory with name 'delphix'""" + + #TODO + # clean up error handling + logger.debug("Finding toolkit Path...") - command = CommandFactory.get_dlpx_bin() - bin_directory, std_err, exit_code = utilities.execute_bash(self.connection, command) + bin_directory, std_err, exit_code = self.run_os_command( + os_command='get_dlpx_bin' + ) + + if bin_directory is None or bin_directory == "": raise Exception("Failed to find the toolkit directory") # Toolkit directory tested on linux x86_64Bit is 6 level below jq path @@ -143,8 +401,9 @@ def create_config_dir(self): loop_var = loop_var - 1 dir_name = bin_directory + "/" + DELPHIX_HIDDEN_FOLDER if not helper_lib.check_dir_present(self.connection, dir_name): - self.make_directory(dir_name) + self.make_directory(dir_name, force_env_user=True) return dir_name + def source_bucket_list(self): """ @@ -154,19 +413,48 @@ def source_bucket_list(self): """ # See the bucket list on source server logger.debug("Collecting bucket list information present on source server ") - env = {ENV_VAR_KEY: {'password': self.staged_source.parameters.xdcr_admin_password}} - command = CommandFactory.get_source_bucket_list(self.repository.cb_shell_path, - self.source_config.couchbase_src_host, - self.source_config.couchbase_src_port, - self.staged_source.parameters.xdcr_admin) - bucket_list, error, exit_code = utilities.execute_bash(self.connection, command_name=command, **env) - if bucket_list == "" or bucket_list is None: + + # env = {ENV_VAR_KEY: {'password': self.staged_source.parameters.xdcr_admin_password}} + # command = CommandFactory.get_source_bucket_list(self.repository.cb_shell_path, + # self.source_config.couchbase_src_host, + # self.source_config.couchbase_src_port, + # self.staged_source.parameters.xdcr_admin) + # bucket_list, error, exit_code = utilities.execute_bash(self.connection, command_name=command, **env) + + bucket_list, error, exit_code = self.run_couchbase_command( + couchbase_command='get_source_bucket_list', + source_hostname=self.source_config.couchbase_src_host, + source_port=self.source_config.couchbase_src_port, + source_username=self.staged_source.parameters.xdcr_admin, + password=self.staged_source.parameters.xdcr_admin_password + ) + + + if bucket_list == "[]" or bucket_list is None: return [] - bucket_list = bucket_list.split("\n") - logger.debug("Source Bucket Information {}".format(bucket_list)) - return bucket_list + else: + logger.debug("clean up json") + bucket_list = bucket_list.replace("u'","'") + bucket_list = bucket_list.replace("'", "\"") + bucket_list = bucket_list.replace("True", "\"True\"") + bucket_list = bucket_list.replace("False", "\"False\"") + logger.debug("parse json") + bucket_list_dict = json.loads(bucket_list) + bucket_list_dict = map(helper_lib.remap_bucket_json, bucket_list_dict) - def source_bucket_list_offline(self, filename): + logger.debug("Source Bucket Information {}".format(bucket_list_dict)) + return bucket_list_dict + + + def get_backup_date(self, x): + w = x.replace('{}/{}'.format(self.parameters.couchbase_bak_loc, self.parameters.couchbase_bak_repo),'') + g = re.match(r'/(.+?)/.*',w) + if g: + return g.group(1) + else: + return '' + + def source_bucket_list_offline(self): """ This function will be used in CB backup manager. It will return the same output as by source_bucket_list method. To avoid source/production server dependency this function will be used. @@ -180,25 +468,77 @@ def source_bucket_list_offline(self, filename): :param filename: filename(couchbase_src_bucket_info.cfg) where bucket information is kept. :return: bucket list information """ - logger.debug( - "Reading bucket list information of source server from {} ".format(filename)) - command = CommandFactory.read_file(filename) - bucket_list, error, exit_code = utilities.execute_bash(self.connection, command) - if bucket_list == "" or bucket_list is None: - return [] - bucket_list = bucket_list.split("\n") - return bucket_list - def node_init(self): + + + logger.debug(self.parameters.couchbase_bak_loc) + logger.debug(self.parameters.couchbase_bak_repo) + + + # command = CommandFactory.get_backup_bucket_list(os.path.join(self.parameters.couchbase_bak_loc, self.parameters.couchbase_bak_repo), self.need_sudo, self.uid) + # logger.debug("Bucket search command: {}".format(command)) + # bucket_list, error, exit_code = utilities.execute_bash(self.connection, command_name=command, callback_func=self.ignore_err) + + + bucket_list, error, exit_code = self.run_os_command( + os_command='get_backup_bucket_list', + path=os.path.join(self.parameters.couchbase_bak_loc, self.parameters.couchbase_bak_repo) + ) + + + backup_list = bucket_list.split('\n') + logger.debug("Bucket search output: {}".format(backup_list)) + date_list = map(self.get_backup_date, backup_list) + date_list.sort() + logger.debug("date list: {}".format(date_list)) + files_to_process = [ x for x in backup_list if date_list[-1] in x ] + + logger.debug(files_to_process) + + bucket_list_dict = [] + + for f in files_to_process: + # command = CommandFactory.cat(f, self.need_sudo, self.uid) + # logger.debug("cat command: {}".format(command)) + # bucket_file_content, error, exit_code = utilities.execute_bash(self.connection, command_name=command) + + bucket_file_content, error, exit_code = self.run_os_command( + os_command='cat', + path=f + ) + + + logger.debug(bucket_file_content) + bucket_json = json.loads(bucket_file_content) + bucket_list_dict.append(remap_bucket_json(bucket_json)) + + + # command = CommandFactory.read_file(filename) + # bucket_list, error, exit_code = utilities.execute_bash(self.connection, command) + # if bucket_list == "" or bucket_list is None: + # return [] + # bucket_list = bucket_list.split("\n") + logger.debug("Bucket search output: {}".format(bucket_list_dict)) + return bucket_list_dict + + def node_init(self, nodeno=1): """ This method initializes couchbase server node. Where user sets different required paths :return: None """ logger.debug("Initializing the NODE") - kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} - command = CommandFactory.node_init(self.repository.cb_shell_path, self.parameters.couchbase_port, - self.parameters.couchbase_admin, self.parameters.mount_path) - command_output, std_err, exit_code = utilities.execute_bash(self.connection, command, **kwargs) + + command_output, std_err, exit_code = self.run_couchbase_command( + couchbase_command='node_init', + data_path="{}/data_{}".format(self.parameters.mount_path, nodeno) + ) + + # kwargs = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} + # command = CommandFactory.node_init(self.repository.cb_shell_path, self.parameters.couchbase_port, + # self.parameters.couchbase_admin, ) + # logger.debug("Node init: {}".format(command)) + # command_output, std_err, exit_code = utilities.execute_bash(self.connection, command, **kwargs) + logger.debug("Command Output {} ".format(command_output)) def get_config_directory(self): @@ -220,6 +560,513 @@ def get_config_file_path(self): return config_file_path + # Defined for future updates + def get_indexes_definition(self): + # by default take from staging but later take from source + logger.debug("Finding indexes....") + password = self.parameters.couchbase_admin_password + user = self.parameters.couchbase_admin + port = self.parameters.couchbase_port + if self.dSource: + if self.parameters.d_source_type == constants.CBBKPMGR: + hostname = self.parameters.couchbase_host + else: + port = self.source_config.couchbase_src_port + user = self.staged_source.parameters.xdcr_admin + password = self.staged_source.parameters.xdcr_admin_password + hostname = self.source_config.couchbase_src_host + else: + hostname = self.connection.environment.host.name + + # cmd = CommandFactory.get_indexes_name(hostname, port, user) + # logger.debug("command for indexes is : {}".format(cmd)) + # command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd, **env) + + command_output, std_err, exit_code = self.run_couchbase_command( + couchbase_command='get_indexes_name', + hostname=hostname, + username=user, + port=port, + password=password + ) + + + logger.debug("Indexes are {}".format(command_output)) + indexes_raw = json.loads(command_output) + indexes = [] + + logger.debug("dSource type for indexes: {}".format(self.parameters.d_source_type)) + + if self.parameters.d_source_type == constants.CBBKPMGR: + logger.debug("Only build for backup ingestion") + + buckets = {} + for i in indexes_raw['indexes']: + if i['bucket'] in buckets: + buckets[i['bucket']].append(i['indexName']) + else: + buckets[i['bucket']] = [ i['indexName'] ] + + for buc, ind in buckets.items(): + ind_def = 'build index on `{}` (`{}`)'.format(buc, '`,`'.join(ind)) + indexes.append(ind_def) + + else: + # full definition for replication + + for i in indexes_raw['indexes']: + indexes.append(i['definition'].replace('defer_build":true','defer_build":false')) + return indexes + + # Defined for future updates + def build_index(self, index_def): + command_output, std_err, exit_code = self.run_couchbase_command( + couchbase_command='build_index', + base_path=helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path), + index_def=index_def + ) + + + # env = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} + # cmd = CommandFactory.build_index(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path),self.connection.environment.host.name, self.parameters.couchbase_port, self.parameters.couchbase_admin, index_def) + # logger.debug("building index cmd: {}".format(cmd)) + # command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd, **env) + + logger.debug("command_output is {}".format(command_output)) + return command_output + + + def check_index_build(self): + # env = {ENV_VAR_KEY: {'password': self.parameters.couchbase_admin_password}} + # cmd = CommandFactory.check_index_build(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path),self.connection.environment.host.name, self.parameters.couchbase_port, self.parameters.couchbase_admin) + # logger.debug("check_index_build cmd: {}".format(cmd)) + + # set timeout to 12 hours + end_time = time.time() + 3660*12 + + tobuild = 1 + + #break the loop either end_time is exceeding from 1 minute or server is successfully started + while time.time() < end_time and tobuild <> 0: + + command_output, std_err, exit_code = self.run_couchbase_command( + couchbase_command='check_index_build', + base_path=helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path) + ) + + + #command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd, **env) + logger.debug("command_output is {}".format(command_output)) + logger.debug("std_err is {}".format(std_err)) + logger.debug("exit_code is {}".format(exit_code)) + try: + command_output_dict = json.loads(command_output) + logger.debug("dict {}".format(command_output_dict)) + tobuild = command_output_dict['results'][0]['unbuilt'] + logger.debug("to_build is {}".format(tobuild)) + helper_lib.sleepForSecond(30) # waiting for 1 second + except Exception as e: + logger.debug(str(e)) + + + + + def save_config(self, what, nodeno=1): + + # TODO + # Error handling + + logger.debug("start save_config") + + targetdir = self.get_config_directory() + target_config_filename = os.path.join(targetdir,"config.dat_{}".format(nodeno)) + target_local_filename = os.path.join(targetdir,"local.ini_{}".format(nodeno)) + target_encryption_filename = os.path.join(targetdir,"encrypted_data_keys_{}".format(nodeno)) + + if nodeno == 1: + ip_file = "{}/../var/lib/couchbase/ip".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + target_ip_filename = os.path.join(targetdir,"ip_{}".format(nodeno)) + else: + ip_file = "{}/../var/lib/couchbase/ip_start".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + target_ip_filename = os.path.join(targetdir,"ip_start_{}".format(nodeno)) + + + filename = "{}/../var/lib/couchbase/config/config.dat".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + + command_output, std_err, exit_code = self.run_os_command( + os_command='os_cp', + srcname=filename, + trgname=target_config_filename + ) + + logger.debug("save config.dat cp - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + if exit_code != 0: + raise UserError("Error saving configuration file: config.dat", "Check sudo or user privileges to read Couchbase config.dat file", std_err) + + + # encryption data keys may not exist on Community edition + + filename = "{}/../var/lib/couchbase/config/encrypted_data_keys".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + + check_encrypted_data_keys, check_ip_file_err, exit_code = self.run_os_command( + os_command='check_file', + file_path=filename + ) + + if exit_code == 0 and "Found" in check_encrypted_data_keys: + # cmd = CommandFactory.os_cp(filename, target_encryption_filename, self.need_sudo, self.uid) + # logger.debug("save encrypted_data_keys cp: {}".format(cmd)) + # command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd) + command_output, std_err, exit_code = self.run_os_command( + os_command='os_cp', + srcname=filename, + trgname=target_encryption_filename + ) + + logger.debug("save encrypted_data_keys.dat cp - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + if exit_code != 0: + raise UserError("Error saving configuration file: encrypted_data_keys", "Check sudo or user privileges to read Couchbase encrypted_data_keys file", std_err) + + + + + + command_output, std_err, exit_code = self.run_os_command( + os_command='os_cp', + srcname=ip_file, + trgname=target_ip_filename + ) + + + logger.debug("save {} - exit_code: {} stdout: {} std_err: {}".format(ip_file, exit_code, command_output, std_err)) + + if exit_code != 0: + raise UserError("Error saving configuration file: {}".format(ip_file), "Check sudo or user privileges to read Couchbase {} file".format(ip_file), std_err) + + + filename = "{}/../etc/couchdb/local.ini".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + + command_output, std_err, exit_code = self.run_os_command( + os_command='os_cp', + srcname=filename, + trgname=target_local_filename + ) + + logger.debug("save local.ini cp - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + if exit_code != 0: + raise UserError("Error saving configuration file: local.ini", "Check sudo or user privileges to read Couchbase local.ini file", std_err) + + + + def check_cluster_notconfigured(self): + + logger.debug("check_cluster") + + command_output, std_err, exit_code = self.run_couchbase_command( + couchbase_command='get_server_list', + hostname=self.connection.environment.host.name) + + if "unknown pool" in command_output: + return True + else: + return False + + + def check_cluster_configured(self): + + logger.debug("check_cluster configured") + + command_output, std_err, exit_code = self.run_couchbase_command( + couchbase_command='get_server_list', + hostname=self.connection.environment.host.name) + + if "healthy active" in command_output: + return True + else: + return False + + + + def check_config(self): + + filename = os.path.join(self.get_config_directory(),"config.dat") + cmd = CommandFactory.check_file(filename) + logger.debug("check file cmd: {}".format(cmd)) + command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd, callback_func=self.ignore_err) + + if exit_code == 0 and "Found" in command_output: + return True + else: + return False + + def restore_config(self, what, nodeno=1): + + # TODO + # Error handling + + logger.debug("start restore_config") + + sourcedir = self.get_config_directory() + + source_config_file = os.path.join(sourcedir,"config.dat_{}".format(nodeno)) + source_local_filename = os.path.join(sourcedir,"local.ini_{}".format(nodeno)) + source_encryption_keys = os.path.join(sourcedir,"encrypted_data_keys_{}".format(nodeno)) + + + source_ip_file = os.path.join(sourcedir,"ip_{}".format(nodeno)) + target_ip_file = "{}/../var/lib/couchbase/ip".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + delete_ip_file = "{}/../var/lib/couchbase/ip_start".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + + # check_file_command = CommandFactory.check_file(source_ip_file, sudo=self.need_sudo, uid=self.uid) + # check_ip_file, check_ip_file_err, exit_code = utilities.execute_bash(self.connection, check_file_command, callback_func=self.ignore_err) + + check_ip_file, check_ip_file_err, exit_code = self.run_os_command( + os_command='check_file', + file_path=source_ip_file + ) + + + if not (exit_code == 0 and "Found" in check_ip_file): + source_ip_file = os.path.join(sourcedir,"ip_start_{}".format(nodeno)) + target_ip_file = "{}/../var/lib/couchbase/ip_start".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + delete_ip_file = "{}/../var/lib/couchbase/ip".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + + logger.debug("IP file is {}".format(source_ip_file)) + + ip_filename = os.path.join(sourcedir,"ip_{}".format(source_ip_file)) + + targetfile = "{}/../var/lib/couchbase/config/config.dat".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + + command_output, std_err, exit_code = self.run_os_command( + os_command='os_cp', + srcname=source_config_file, + trgname=targetfile + ) + + logger.debug("config.dat restore - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + check_encrypted_data_keys, check_ip_file_err, exit_code = self.run_os_command( + os_command='check_file', + file_path=source_encryption_keys + ) + + logger.debug("Check check_encrypted_data_keys - exit_code: {} stdout: {}".format(exit_code, check_encrypted_data_keys)) + + if exit_code == 0 and "Found" in check_encrypted_data_keys: + targetfile = "{}/../var/lib/couchbase/config/encrypted_data_keys".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + command_output, std_err, exit_code = self.run_os_command( + os_command='os_cp', + srcname=source_encryption_keys, + trgname=targetfile + ) + + logger.debug("encrypted_data_keys restore - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + + check_ip_delete_file, check_ip_delete_file, check_ip_exit_code = self.run_os_command( + os_command='check_file', + file_path=delete_ip_file + ) + + logger.debug("Check delete old ip_file - exit_code: {} stdout: {}".format(check_ip_exit_code, check_ip_delete_file)) + + command_output, std_err, exit_code = self.run_os_command( + os_command='os_mv', + srcname=delete_ip_file, + trgname="{}.bak".format(delete_ip_file) + ) + + logger.debug("ipfile delete - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + + command_output, std_err, exit_code = self.run_os_command( + os_command='os_cp', + srcname=source_ip_file, + trgname=target_ip_file + ) + + logger.debug("ipfile restore - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + targetfile = "{}/../etc/couchdb/local.ini".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + + command_output, std_err, exit_code = self.run_os_command( + os_command='os_cp', + srcname=source_local_filename, + trgname=targetfile + ) + + logger.debug("local.ini restore - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + if what == 'parent': + #local.ini needs to have a proper entry + filename = "{}/../etc/couchdb/local.ini".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + newpath = "{}/data_{}".format(self.parameters.mount_path, nodeno) + cmd = CommandFactory.sed(filename, 's|view_index_dir.*|view_index_dir={}|'.format(newpath), self.need_sudo, self.uid) + logger.debug("sed config cmd: {}".format(cmd)) + command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd) + logger.debug("setting index paths - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + cmd = CommandFactory.sed(filename, 's|database_dir.*|database_dir={}|'.format(newpath), self.need_sudo, self.uid) + logger.debug("sed config cmd: {}".format(cmd)) + command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd) + logger.debug("setting data paths - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + + + def delete_config(self): + + # TODO: + # error handling + + logger.debug("start delete_config") + + filename = "{}/../var/lib/couchbase/config/config.dat".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + + cmd = CommandFactory.check_file(filename, self.need_sudo, self.uid) + logger.debug("check file cmd: {}".format(cmd)) + command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd, callback_func=self.ignore_err) + + if exit_code == 0 and "Found" in command_output: + cmd = CommandFactory.os_mv(filename, "{}.bak".format(filename), self.need_sudo, self.uid) + logger.debug("rename config cmd: {}".format(cmd)) + command_output, std_err, exit_code = utilities.execute_bash(self.connection, command_name=cmd) + logger.debug("rename config.dat to bak - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + filename = "{}/../etc/couchdb/local.ini".format(helper_lib.get_base_directory_of_given_path(self.repository.cb_shell_path)) + command_output, std_err, exit_code = self.run_os_command( + os_command='sed', + filename=filename, + regex='s/view_index_dir.*//' + ) + + logger.debug("clean local.ini index - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + command_output, std_err, exit_code = self.run_os_command( + os_command='sed', + filename=filename, + regex='s/database_dir.*//' + ) + + logger.debug("clean local.ini data - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + command_output, std_err, exit_code = self.run_os_command( + os_command='change_permission', + path=filename + ) + + logger.debug("fix local.ini permission - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + def ignore_err(self, input): + return True + + + def rename_cluster(self): + """Rename cluster based on user entries""" + + logger.debug("start rename_cluster") + + command_output, std_err, exit_code = self.run_couchbase_command( + couchbase_command='rename_cluster', + username=self.snapshot.couchbase_admin, + password=self.snapshot.couchbase_admin_password, + newuser=self.parameters.couchbase_admin, + newpass=self.parameters.couchbase_admin_password, + newname=self.parameters.tgt_cluster_name + ) + + logger.debug("rename cluster - exit_code: {} stdout: {} std_err: {}".format(exit_code, command_output, std_err)) + + + + + + def start_node_bootstrap(self): + logger.debug("start start_node_bootstrap") + self.start_couchbase(no_wait=True) + end_time = time.time() + 3660 + server_status = Status.INACTIVE + + #break the loop either end_time is exceeding from 1 minute or server is successfully started + while time.time() < end_time and server_status<>Status.ACTIVE: + helper_lib.sleepForSecond(1) # waiting for 1 second + server_status = self.staging_bootstrap_status() # fetching status + logger.debug("server status {}".format(server_status)) + + + + def addnode(self, nodeno, node_def): + logger.debug("start addnode") + + + self.delete_config() + + self.start_node_bootstrap() + + self.node_init(nodeno) + + + helper_lib.sleepForSecond(10) + + services = [ 'data', 'index', 'query' ] + + if "fts_service" in node_def and node_def["fts_service"] == True: + services.append('fts') + + if "eventing_service" in node_def and node_def["eventing_service"] == True: + services.append('eventing') + + if "analytics_service" in node_def and node_def["analytics_service"] == True: + services.append('analytics') + + logger.debug("services to add: {}".format(services)) + + + # hostip_command = CommandFactory.get_ip_of_hostname() + # logger.debug("host ip command: {}".format(hostip_command)) + # host_ip_output, std_err, exit_code = utilities.execute_bash(self.connection, hostip_command) + # logger.debug("host ip Output {} ".format(host_ip_output)) + + + logger.debug("node host name / IP: {}".format(node_def["node_addr"])) + + resolve_name_command = CommandFactory.resolve_name(hostname=node_def["node_addr"]) + logger.debug("resolve_name_command command: {}".format(resolve_name_command)) + resolve_name_output, std_err, exit_code = utilities.execute_bash(self.connection, resolve_name_command) + logger.debug("resolve_name_command Output {} ".format(resolve_name_output)) + + command_output, std_err, exit_code = self.run_couchbase_command( + couchbase_command='server_add', + hostname=self.connection.environment.host.name, + newhost=resolve_name_output, + services=','.join(services) + ) + + + logger.debug("Add node Output {} stderr: {} exit_code: {} ".format(command_output, std_err, exit_code)) + + if exit_code != 0: + logger.debug("Adding node error") + raise UserError("Problem with adding node", "Check an output and fix problem before retrying to provision a VDB", "stdout: {} stderr:{}".format(command_output, std_err)) + + + + + command_output, std_err, exit_code = self.run_couchbase_command( + couchbase_command='rebalance', + hostname=self.connection.environment.host.name + ) + + + + logger.debug("Rebalance Output {} stderr: {} exit_code: {} ".format(command_output, std_err, exit_code)) + + if exit_code != 0: + logger.debug("Rebalancing error") + raise UserError("Problem with rebalancing cluster", "Check an output and fix problem before retrying to provision a VDB", "stdout: {} stderr:{}".format(command_output, std_err)) + + + + if __name__ == "__main__": print "Checking Couchbase Class" test_object = CouchbaseOperation(Resource.ObjectBuilder.set_dsource(True).build()) diff --git a/src/controller/db_exception_handler.py b/src/controller/db_exception_handler.py index ee73936..146700e 100644 --- a/src/controller/db_exception_handler.py +++ b/src/controller/db_exception_handler.py @@ -5,12 +5,16 @@ import types import re import logging +import traceback +import sys + from db_commands.constants import CLUSTER_ALREADY_PRESENT, BUCKET_NAME_ALREADY_EXIST, MULTIPLE_VDB_ERROR, \ SHUTDOWN_FAILED, ALREADY_CLUSTER_INIT, ALREADY_CLUSTER_FOR_BUCKET from controller import helper_lib from internal_exceptions.base_exceptions import GenericUserError from internal_exceptions.plugin_exceptions import ERR_RESPONSE_DATA +from dlpx.virtualization.platform.exceptions import UserError logger = logging.getLogger(__name__) @@ -80,8 +84,22 @@ def wrapper_function(*args, **kwargs): try: output_list = function_name(*args, **kwargs) return output_list + + except UserError as ue: + logger.debug("User Error found") + ttype, value, traceb = sys.exc_info() + logger.debug("type: {}, value: {}".format(ttype, value)) + logger.debug("trackback") + logger.debug(traceback.format_exc()) + raise + except Exception as error: logger.debug("Caught Exception : {}".format(error.message)) + logger.debug("pioro") + ttype, value, traceb = sys.exc_info() + logger.debug("type: {}, value: {}".format(ttype, value)) + logger.debug("trackback") + logger.debug(traceback.format_exc()) mcs._exception_generator_factory(error.message) return wrapper_function diff --git a/src/controller/helper_lib.py b/src/controller/helper_lib.py index dfe0f19..90e7820 100644 --- a/src/controller/helper_lib.py +++ b/src/controller/helper_lib.py @@ -24,6 +24,9 @@ import db_commands.constants from db_commands.commands import CommandFactory from db_commands.constants import DEFAULT_CB_BIN_PATH +from dlpx.virtualization.platform.exceptions import UserError + +from dlpx.virtualization.platform import Status from internal_exceptions.plugin_exceptions import RepositoryDiscoveryError, SourceConfigDiscoveryError, FileIOError, \ UnmountFileSystemError @@ -91,6 +94,45 @@ def find_version(source_connection, install_path): return version +def find_ids(source_connection, install_path): + """ return the couchbase uid and gid""" + std_out, std_err, exit_code = utilities.execute_bash(source_connection, + CommandFactory.get_ids(install_path)) + logger.debug("find ids output: {}".format(std_out)) + ids = re.search(r"[-rwx.]+\s\d\s([\d]+)\s([\d]+).*", std_out) + if ids: + uid = int(ids.group(1)) + gid = int(ids.group(2)) + else: + uid = -1 + gid = -1 + logger.debug("Couchbase user uid {} gid {}".format(uid, gid)) + return (uid, gid) + +def find_whoami(source_connection): + """ return the user env id""" + std_out, std_err, exit_code = utilities.execute_bash(source_connection, + CommandFactory.whoami()) + logger.debug("find whoami output: {}".format(std_out)) + ids = re.search(r"uid=([\d]+).*gid=([\d]+)", std_out) + if ids: + uid = int(ids.group(1)) + gid = int(ids.group(2)) + else: + uid = -1 + gid = -1 + logger.debug("Delphix user uid {} gid {}".format(uid, gid)) + return (uid, gid) + + +def need_sudo(source_connection, couchbase_uid, couchbase_gid): + (uid, gid) = find_whoami(source_connection) + if uid != couchbase_uid or gid != couchbase_gid: + return True + else: + return False + + def is_instance_present_of_gosecrets(source_connection): """ check couchbase server is running or not""" instance, stderr, exit_code = utilities.execute_bash(source_connection, CommandFactory.get_process()) @@ -119,9 +161,32 @@ def get_base_directory_of_given_path(binary_path): return path -def get_all_bucket_list_with_size(bucket_output, bucket=None): - """ Return bucket name with ramUsed( adjust ramused value ) from bucket_output""" - logger.debug("bucket_output: {}".format(bucket_output)) +def remap_bucket_json(bucket): + output = {} + if 'bucketType' in bucket: + output['bucketType'] = bucket['bucketType'] + if 'name' in bucket: + output['name'] = bucket['name'] + if 'quota' in bucket and 'ram' in bucket['quota']: + output['ram'] = bucket['quota']['ram'] + elif 'ramQuota' in bucket: + # this is in MB + output['ram'] = int(bucket['ramQuota']) * 1024 * 1024 + else: + logger.debug('No memory in bucket - setting to default') + output['ram'] = 1024000 + if 'compressionMode' in bucket: + output['compressionMode'] = bucket['compressionMode'] + else: + output['compressionMode'] = None + return output + +def get_all_bucket_list_with_size(bucket_output): + """ + Return bucket name with ramUsed( adjust ramused value ) + from bucket_output + """ + additional_buffer = 10 min_size = 104857600 all_bucket_list = "" @@ -160,16 +225,51 @@ def get_stg_all_bucket_list_with_ramquota_size(bucket_output): return all_bucket_list.split(":") -def filter_bucket_name_from_output(bucket_output): +def filter_bucket_name_from_json(bucket_output): """ Filter bucket name from bucket_output. Return list of bucket names present in bucket_output""" - output = filter(lambda bucket: bucket.find(":") == -1, bucket_output) + output = [ x['name'] for x in bucket_output if x['ram'] > 0] + logger.debug("Bucket list: {}".format(output)) + return output + +def filter_bucket_name_from_output(bucket_output): + """ + Filter bucket name from bucket_output. + Return list of bucket names present in bucket_output + """ + output = [] + logger.debug("filter input: {}".format(bucket_output)) + logger.debug("filter input: {}".format(len(bucket_output))) + if bucket_output != []: + output = map(lambda x: x["name"], bucket_output) logger.debug("Bucket list: {}".format(output)) return output +def get_bucket_object(bucket_output, bucket): + """ + Return bucket dict + from bucket_output string for bucket(passed in argument) + """ + output = filter(lambda x: x['name'] == bucket, bucket_output) + if len(output) != 1: + ret = None + else: + ret = output[-1] + logger.debug("For Bucket {} detail is : {}".format(bucket, ret)) + return ret + def get_bucket_name_with_size(bucket_output, bucket): - """ Return `bucket_name:ramUsed` as output from bucket_output string for bucket(passed in argument) """ - output = get_all_bucket_list_with_size(bucket_output, bucket) + """ + Return `bucket_name:ramUsed` + as output from bucket_output string for bucket(passed in argument) + """ + + logger.debug("HUHU") + logger.debug(bucket_output) + + output = get_all_bucket_list_with_size(bucket_output) + logger.debug("HAHA") + logger.debug(output) output = ":".join(output) bucket_info = re.search(r"{},\d+".format(bucket), output).group() logger.debug("For Bucket {} detail is : {}".format(bucket, bucket_info)) @@ -188,8 +288,9 @@ def get_bucketlist_to_namesize_list(bucket_output, bucket_list): def sleepForSecond(sec): # Sleep/Pause the execution for given seconds + logger.debug("sleeping for {}".format(sec)) time.sleep(sec) - + logger.debug("sleeping is over") def current_time(): @@ -288,3 +389,64 @@ def get_sync_lock_file_name(dsource_type, dsource_name): striped_dsource_name = dsource_name.replace(" ", "") sync_filename = str(striped_dsource_name) + str(sync_filename) return sync_filename + + +def check_stale_mountpoint(connection, path): + + + + output, stderr, exit_code = utilities.execute_bash(connection, CommandFactory.df(path)) + if exit_code != 0: + if "No such file or directory" in stderr: + # this is actually OK + return False + else: + logger.error("df retured error - stale mount point or other error") + logger.error("stdout: {} stderr: {} exit_code: {}".format(output.encode('utf-8'), stderr.encode('utf-8'), exit_code)) + return True + else: + return False + + +def check_server_is_used(connection, path): + + ret = Status.INACTIVE + + output, stderr, exit_code = utilities.execute_bash(connection, CommandFactory.mount()) + if exit_code != 0: + logger.error("mount retured error") + logger.error("stdout: {} stderr: {} exit_code: {}".format(output.encode('utf-8'), stderr.encode('utf-8'), exit_code)) + raise UserError("Problem with reading mounted file systems", "Ask OS admin to check mount", stderr) + else: + # parse a mount output to find another Delphix mount points + fs_re = re.compile(r'(\S*)\son\s(\S*)\stype\s(\S*)') + for i in output.split("\n"): + match = re.search(fs_re, i) + if match is not None: + groups = match.groups() + if groups[2] == 'nfs': + if path == groups[1]: + # this is our mount point - skip it + ret = Status.ACTIVE + continue + if "domain0" in groups[0] and "timeflow" in groups[0]: + # this is a delphix mount point but it's not ours + # raise an exception + raise UserError("Another database (VDB or staging) is using this server.", "Disable another one to provision or enable this one", "{} {}".format(groups[0], groups[1])) + + + return ret + + + +def clean_stale_mountpoint(connection, path): + + + + umount_std, umount_stderr, umount_exit_code = utilities.execute_bash(connection, CommandFactory.unmount_file_system(mount_path=path, options='-lf')) + if umount_exit_code != 0: + logger.error("Problem with cleaning mount path") + logger.error("stderr {}".format(umount_stderr)) + raise UserError("Problem with cleaning mount path", "Ask OS admin to check mount points", umount_stderr) + + diff --git a/src/db_commands/commands.py b/src/db_commands/commands.py index 1a4b707..91dc57d 100644 --- a/src/db_commands/commands.py +++ b/src/db_commands/commands.py @@ -14,20 +14,26 @@ ####################################################################################################################### +import inspect +import logging + + +logger = logging.getLogger(__name__) + class OSCommand(object): def __init__(self): pass @staticmethod - def find_binary_path(): + def find_binary_path(**kwargs): return "echo $COUCHBASE_PATH" @staticmethod - def find_install_path(binary_path): + def find_install_path(binary_path, **kwargs): return "find {binary_path} -name couchbase-server".format(binary_path=binary_path) @staticmethod - def find_shell_path(binary_path): + def find_shell_path(binary_path, **kwargs): return "find {binary_path} -name couchbase-cli".format(binary_path=binary_path) @staticmethod @@ -35,70 +41,156 @@ def get_process(): return "ps -ef" @staticmethod - def make_directory(directory_path): - return "mkdir -p {directory_path}".format(directory_path=directory_path) + def make_directory(directory_path,sudo=False, uid=None, **kwargs): + if sudo: + return "sudo -u \#{uid} mkdir -p {directory_path}".format(uid=uid, directory_path=directory_path) + else: + return "mkdir -p {directory_path}".format(directory_path=directory_path) @staticmethod - def change_permission(directory_path): - return "chmod -R 775 {directory_path}".format(directory_path=directory_path) + def change_permission(path,sudo=False, uid=None, **kwargs): + if sudo: + return "sudo -u \#{uid} chmod -R 775 {path}".format(uid=uid, path=path) + else: + return "chmod -R 775 {path}".format(path=path) @staticmethod - def get_config_directory(mount_path): + def get_config_directory(mount_path, **kwargs): return "{mount_path}/.delphix".format(mount_path=mount_path) @staticmethod - def read_file(filename): + def read_file(filename, **kwargs): return "cat {filename}".format(filename=filename) @staticmethod - def check_file(file_path): - return "[ -f {file_path} ] && echo 'Found'".format(file_path=file_path) + def check_file(file_path, sudo=False, uid=None, **kwargs): + if sudo: + return "sudo -u \#{uid} [ -f {file_path} ] && echo 'Found'".format(file_path=file_path, uid=uid) + else: + return "[ -f {file_path} ] && echo 'Found'".format(file_path=file_path) @staticmethod - def write_file(filename, data): + def write_file(filename, data, **kwargs): return "echo {data} > {filename}".format(filename=filename, data=data) @staticmethod - def get_ip_of_hostname(): - return "hostname -i" + def get_ip_of_hostname(**kwargs): + return "hostname -I" @staticmethod - def check_directory(dir_path): - return "[ -d {dir_path} ] && echo 'Found'".format(dir_path=dir_path) + def check_directory(dir_path, sudo=False, uid=None, **kwargs): + if sudo: + return "sudo -u \#{uid} [ -d {dir_path} ] && echo 'Found'".format(dir_path=dir_path, uid=uid) + else: + return "[ -d {dir_path} ] && echo 'Found'".format(dir_path=dir_path) @staticmethod - def delete_file(filename): + def delete_file(filename, **kwargs): return "rm -f {filename}".format(filename=filename) @staticmethod - def get_dlpx_bin(): + def delete_dir(dirname, sudo=False, uid=None, **kwargs): + if sudo: + return "sudo -u \#{uid} rm -rf {dirname}".format(dirname=dirname, uid=uid) + else: + return "rm -rf {dirname}".format(dirname=dirname) + + @staticmethod + def os_mv(srcname, trgname, sudo=False, uid=None, **kwargs): + if sudo: + return "sudo -u \#{uid} mv {srcname} {trgname}".format(srcname=srcname, trgname=trgname, uid=uid) + else: + return "mv {srcname} {trgname}".format(srcname=srcname, trgname=trgname) + + @staticmethod + def os_cp(srcname, trgname, sudo=False, uid=None, **kwargs): + if sudo: + return "sudo -u \#{uid} cp {srcname} {trgname}".format(srcname=srcname, trgname=trgname, uid=uid) + else: + return "cp {srcname} {trgname}".format(srcname=srcname, trgname=trgname, uid=uid) + + @staticmethod + def get_dlpx_bin(**kwargs): return "echo $DLPX_BIN_JQ" @staticmethod - def unmount_file_system(mount_path): - return "sudo /bin/umount {mount_path}".format(mount_path=mount_path) + def unmount_file_system(mount_path, **kwargs): + if "options" in kwargs: + options = kwargs.pop('options') + else: + options = "" + return "sudo /bin/umount {options} {mount_path}".format(mount_path=mount_path, options=options) + + @staticmethod + def whoami(**kwargs): + # uid=1003(delphix) gid=1003(delphix) groups=1003(delphix) context=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 + return "id" + + + @staticmethod + def sed(filename, regex, sudo=False, uid=None, **kwargs): + if sudo: + return 'sudo -u \#{uid} sed -i -e "{regex}" {filename}'.format(regex=regex, filename=filename, uid=uid) + else: + return 'sed -i -e "{}" {}'.format(regex, filename) + @staticmethod + def cat(path, sudo=False, uid=None, **kwargs): + if sudo: + return "sudo -u \#{uid} cat {path}".format( + path=path, uid=uid + ) + else: + return "cat {path}".format( + path=path + ) + + @staticmethod + def df(mount_path, **kwargs): + return "df -h {mount_path}".format(mount_path=mount_path) + + @staticmethod + def mount(**kwargs): + return "mount" + + + @staticmethod + def resolve_name(hostname, **kwargs): + return "getent ahostsv4 {hostname} | grep STREAM | head -n 1 | cut -d ' ' -f 1".format(hostname=hostname) + class DatabaseCommand(object): def __init__(self): pass @staticmethod - def start_couchbase(install_path): - return "{install_path} \-- -noinput -detached .".format(install_path=install_path) + def start_couchbase(install_path, sudo=False, uid=None, **kwargs): + if sudo: + return "sudo -u \#{} {install_path} \-- -noinput -detached .".format(uid, install_path=install_path) + else: + return "{install_path} \-- -noinput -detached .".format(install_path=install_path) @staticmethod def get_version(install_path): return "{install_path} --version".format(install_path=install_path) + @staticmethod + def get_ids(install_path): + #-rwxr-xr-x. 1 996 993 514 Jan 30 2020 /opt/couchbase/bin/couchbase-cli + return "ls -n {install_path}".format(install_path=install_path) + + @staticmethod def get_data_directory(couchbase_base_dir): return "cat {couchbase_base_dir}/etc/couchbase/static_config|grep path_config_datadir".format( couchbase_base_dir=couchbase_base_dir) @staticmethod - def stop_couchbase(install_path): - return "{install_path} -k".format(install_path=install_path) + def stop_couchbase(install_path, sudo=False, uid=None, **kwargs): + if sudo: + return "sudo -u \#{} {install_path} -k".format(uid, install_path=install_path) + else: + return "{install_path} -k".format(install_path=install_path) @staticmethod def cluster_init(shell_path, @@ -111,7 +203,8 @@ def cluster_init(shell_path, cluster_fts_ramsize, cluster_eventing_ramsize, cluster_analytics_ramsize, - additional_services + additional_services, + **kwargs ): return "{shell_path} cluster-init --cluster {hostname}:{port} --cluster-username {username} --cluster-password $password --cluster-ramsize {cluster_ramsize} --cluster-name {cluster_name} --cluster-index-ramsize {cluster_index_ramsize} --cluster-fts-ramsize {cluster_fts_ramsize} --cluster-eventing-ramsize {cluster_eventing_ramsize} --cluster-analytics-ramsize {cluster_analytics_ramsize} --services data,index,{additional_services}".format( shell_path=shell_path, @@ -129,7 +222,7 @@ def cluster_init(shell_path, @staticmethod def cluster_setting(shell_path, hostname, port, username, cluster_ramsize, cluster_name, cluster_index_ramsize, - cluster_fts_ramsize, cluster_eventing_ramsize, cluster_analytics_ramsize): + cluster_fts_ramsize, cluster_eventing_ramsize, cluster_analytics_ramsize, **kwargs): return "{shell_path} setting-cluster -c {hostname}:{port} -u {username} -p $password --cluster-ramsize {cluster_ramsize} --cluster-name {cluster_name} --cluster-index-ramsize {cluster_index_ramsize} --cluster-fts-ramsize {cluster_fts_ramsize} --cluster-eventing-ramsize {cluster_eventing_ramsize} --cluster-analytics-ramsize {cluster_analytics_ramsize}".format( shell_path=shell_path, hostname=hostname, @@ -143,8 +236,18 @@ def cluster_setting(shell_path, hostname, port, username, cluster_ramsize, clust cluster_analytics_ramsize=cluster_analytics_ramsize ) + # @staticmethod + # def cluster_check(shell_path, hostname, port, username, **kwargs): + # return "{shell_path} server-list --cluster {hostname}:{port} --username {username} --password $password".format( + # shell_path=shell_path, + # hostname=hostname, + # port=port, + # username=username + # ) + + @staticmethod - def xdcr_setup(shell_path, source_hostname, source_port, source_username, hostname, port, username, cluster_name): + def xdcr_setup(shell_path, source_hostname, source_port, source_username, hostname, port, username, cluster_name, **kwargs): return "{shell_path} xdcr-setup --cluster {source_hostname}:{source_port} --username {source_username} --password $source_password --create --xdcr-hostname {hostname}:{port} --xdcr-username {username} --xdcr-password $password --xdcr-cluster-name {cluster_name}".format( shell_path=shell_path, source_hostname=source_hostname, @@ -157,7 +260,7 @@ def xdcr_setup(shell_path, source_hostname, source_port, source_username, hostna ) @staticmethod - def xdcr_replicate(shell_path, source_hostname, source_port, source_username, source_bucket_name, target_bucket_name, cluster_name, hostname, port, username): + def xdcr_replicate(shell_path, source_hostname, source_port, source_username, source_bucket_name, target_bucket_name, cluster_name, hostname, port, username, **kwargs): return "{shell_path} xdcr-replicate --cluster {source_hostname}:{source_port} --username {source_username} --password $source_password --create --xdcr-from-bucket {source_bucket_name} --xdcr-to-bucket {target_bucket_name} --xdcr-cluster-name {cluster_name}".format( shell_path=shell_path, source_hostname=source_hostname, @@ -169,7 +272,7 @@ def xdcr_replicate(shell_path, source_hostname, source_port, source_username, so ) @staticmethod - def get_replication_uuid(shell_path, source_hostname, source_port, source_username): + def get_replication_uuid(shell_path, source_hostname, source_port, source_username, **kwargs): return "{shell_path} xdcr-setup --cluster {source_hostname}:{source_port} --username {source_username} --password $source_password --list".format( shell_path=shell_path, source_hostname=source_hostname, @@ -177,8 +280,12 @@ def get_replication_uuid(shell_path, source_hostname, source_port, source_userna source_username=source_username, ) + + + + @staticmethod - def get_stream_id(shell_path, source_hostname, source_port, source_username, cluster_name): + def get_stream_id(shell_path, source_hostname, source_port, source_username, cluster_name, **kwargs): return "{shell_path} xdcr-replicate --cluster {source_hostname}:{source_port} --username {source_username} --password $source_password --xdcr-cluster-name {cluster_name} --list".format( shell_path=shell_path, source_hostname=source_hostname, @@ -188,7 +295,7 @@ def get_stream_id(shell_path, source_hostname, source_port, source_username, clu ) @staticmethod - def pause_replication(shell_path, source_hostname, source_port, source_username, cluster_name, id): + def pause_replication(shell_path, source_hostname, source_port, source_username, cluster_name, id, **kwargs): return "{shell_path} xdcr-replicate --cluster {source_hostname}:{source_port} --username {source_username} --password $source_password --xdcr-cluster-name {cluster_name} --pause --xdcr-replicator={id}".format( shell_path=shell_path, source_hostname=source_hostname, @@ -199,7 +306,7 @@ def pause_replication(shell_path, source_hostname, source_port, source_username, ) @staticmethod - def resume_replication(shell_path, source_hostname, source_port, source_username, cluster_name, id): + def resume_replication(shell_path, source_hostname, source_port, source_username, cluster_name, id, **kwargs): return "{shell_path} xdcr-replicate --cluster {source_hostname}:{source_port} --username {source_username} --password $source_password --xdcr-cluster-name {cluster_name} --resume --xdcr-replicator={id}".format( shell_path=shell_path, source_hostname=source_hostname, @@ -210,7 +317,7 @@ def resume_replication(shell_path, source_hostname, source_port, source_username ) @staticmethod - def delete_replication(shell_path, source_hostname, source_port, source_username, id, cluster_name): + def delete_replication(shell_path, source_hostname, source_port, source_username, id, cluster_name, **kwargs): return "{shell_path} xdcr-replicate --cluster {source_hostname}:{source_port} --username {source_username} --password $source_password --delete --xdcr-replicator {id} --xdcr-cluster-name {cluster_name}".format( shell_path=shell_path, source_hostname=source_hostname, @@ -221,7 +328,7 @@ def delete_replication(shell_path, source_hostname, source_port, source_username ) @staticmethod - def xdcr_delete(shell_path, source_hostname, source_port, source_username, hostname, port, username, cluster_name): + def xdcr_delete(shell_path, source_hostname, source_port, source_username, hostname, port, username, cluster_name, **kwargs): return "{shell_path} xdcr-setup --cluster {source_hostname}:{source_port} --username {source_username} --password $source_password --delete --xdcr-hostname {hostname}:{port} --xdcr-username {username} --xdcr-password $password --xdcr-cluster-name {cluster_name}".format( shell_path=shell_path, source_hostname=source_hostname, @@ -234,8 +341,8 @@ def xdcr_delete(shell_path, source_hostname, source_port, source_username, hostn ) @staticmethod - def get_source_bucket_list(shell_path, source_hostname, source_port, source_username): - return "{shell_path} bucket-list --cluster {source_hostname}:{source_port} --username {source_username} --password $password".format( + def get_source_bucket_list(shell_path, source_hostname, source_port, source_username, **kwargs): + return "{shell_path} bucket-list --cluster {source_hostname}:{source_port} --username {source_username} --password $password -o json".format( shell_path=shell_path, source_hostname=source_hostname, source_port=source_port, @@ -243,88 +350,120 @@ def get_source_bucket_list(shell_path, source_hostname, source_port, source_user ) @staticmethod - def get_status(shell_path, hostname, port, username): - return "{shell_path} server-info --cluster {hostname}:{port} --username {username} --password $password".format( + def get_server_list(shell_path, hostname, port, username, **kwargs): + return "{shell_path} server-list --cluster {hostname}:{port} --username {username} --password $password".format( shell_path=shell_path, hostname=hostname, port=port, username=username ) @staticmethod - def node_init(shell_path, port, username, mount_path): - return "{shell_path} node-init --cluster 127.0.0.1:{port} --username {username} --password $password --node-init-data-path {mount_path}/data --node-init-index-path {mount_path}/data --node-init-analytics-path {mount_path}/data --node-init-hostname 127.0.0.1".format( - shell_path=shell_path, port=port, username=username, mount_path=mount_path + def node_init(shell_path, port, username, data_path, **kwargs): + return "{shell_path} node-init --cluster 127.0.0.1:{port} --username {username} --password $password --node-init-data-path {data_path} --node-init-index-path {data_path} --node-init-analytics-path {data_path} --node-init-hostname 127.0.0.1".format( + shell_path=shell_path, port=port, username=username, data_path=data_path ) @staticmethod - def bucket_edit(shell_path, hostname, port, username, bucket_name, flush_value): + def bucket_edit(shell_path, hostname, port, username, bucket_name, flush_value, **kwargs): return "{shell_path} bucket-edit --cluster {hostname}:{port} --username {username} --password $password --bucket={bucket_name} --enable-flush {flush_value}".format( shell_path=shell_path, hostname=hostname, port=port, username=username, bucket_name=bucket_name, flush_value=flush_value ) @staticmethod - def bucket_edit_ramquota(shell_path, hostname, port, username, bucket_name, ramsize): + def bucket_edit_ramquota(shell_path, hostname, port, username, bucket_name, ramsize, **kwargs): return "{shell_path} bucket-edit --cluster {hostname}:{port} --username {username} --password $password --bucket={bucket_name} --bucket-ramsize {ramsize}".format( shell_path=shell_path, hostname=hostname, port=port, username=username, bucket_name=bucket_name, ramsize=ramsize ) @staticmethod - def bucket_delete(shell_path, hostname, port, username, bucket_name): + def bucket_delete(shell_path, hostname, port, username, bucket_name, **kwargs): return "{shell_path} bucket-delete --cluster {hostname}:{port} --username {username} --password $password --bucket={bucket_name}".format( shell_path=shell_path, hostname=hostname, port=port, username=username, bucket_name=bucket_name ) @staticmethod - def bucket_flush(shell_path, hostname, port, username, bucket_name): + def bucket_flush(shell_path, hostname, port, username, bucket_name, **kwargs): return "echo 'Yes' | {shell_path} bucket-flush --cluster {hostname}:{port} --username {username} --password $password --bucket={bucket_name}".format( shell_path=shell_path, hostname=hostname, port=port, username=username, bucket_name=bucket_name ) @staticmethod - def bucket_create(shell_path, hostname, port, username, bucket_name, ramsize, evictionpolicy): - return "{shell_path} bucket-create --cluster 127.0.0.1:{port} --username {username} --password $password --bucket {bucket_name} --bucket-type couchbase --bucket-ramsize {ramsize} --bucket-replica 0 --bucket-eviction-policy {evictionpolicy} --compression-mode passive --conflict-resolution sequence --wait".format( + def bucket_create(shell_path, hostname, port, username, bucket_name, ramsize, evictionpolicy, bucket_type, bucket_compression, **kwargs): + return "{shell_path} bucket-create --cluster 127.0.0.1:{port} --username {username} --password $password --bucket {bucket_name} --bucket-type {bucket_type} --bucket-ramsize {ramsize} --bucket-replica 0 --bucket-eviction-policy {evictionpolicy} {bucket_compression} --conflict-resolution sequence --wait".format( shell_path=shell_path, port=port, username=username, - bucket_name=bucket_name, ramsize=ramsize, evictionpolicy=evictionpolicy + bucket_name=bucket_name, ramsize=ramsize, evictionpolicy=evictionpolicy, + bucket_type=bucket_type, bucket_compression=bucket_compression ) @staticmethod - def bucket_list(shell_path, hostname, port, username): - return "{shell_path} bucket-list --cluster {hostname}:{port} --username {username} --password $password".format( + def bucket_list(shell_path, hostname, port, username, **kwargs): + return "{shell_path} bucket-list --cluster {hostname}:{port} --username {username} --password $password -o json".format( shell_path=shell_path, hostname=hostname, port=port, username=username, ) @staticmethod - def get_indexes_name(base_path, hostname, port, username, index): - return "{base_path}/cbq -e {hostname}:{port} -u {username} -p $password -q=true -s=\"SELECT name FROM system:indexes where keyspace_id = {index} AND state = 'deferred'\"".format( - base_path=base_path, hostname=hostname, port=port, username=username, index=index + def get_indexes_name(hostname, port, username, **kwargs): + return "curl {username}:$password@{hostname}:{port}/indexStatus".format( + hostname=hostname, port=port, username=username ) @staticmethod - def build_index(base_path, hostname, port, username, index_name): - return "{base_path}/cbq -e {hostname}:{port} -u {username} -p $password -q=true -s=echo \"BUILD INDEX ON {index_name}\" {index_name}".format( - base_path=base_path, hostname=hostname, port=port, username=username, index_name=index_name - ) + def get_backup_bucket_list(path, sudo=False, uid=None, **kwargs): + if sudo: + return "sudo -u \#{uid} find {path} -name bucket-config.json".format( + path=path, uid=uid + ) + else: + return "find {path} -name bucket-config.json".format( + path=path + ) + + + + @staticmethod - def is_build_completed(base_path, hostname, port, username, index): - return "{base_path}/cbq -e {hostname}:{port} -u {username} -p $password -q=true -s=\"SELECT COUNT(*) as unbuilt FROM system:indexes WHERE keyspace_id ={index} AND state <> 'online'".format( - base_path=base_path, hostname=hostname, port=port, username=username, index=index + def build_index(base_path, hostname, port, username, index_def, **kwargs): + return "{base_path}/cbq -e {hostname}:{port} -u {username} -p $password -q=true -s='{index_def}'".format( + base_path=base_path, hostname=hostname, port=port, username=username, index_def=index_def ) @staticmethod - def cb_backup_full(base_path, backup_location, backup_repo, hostname, port, username, csv_bucket_list): - return "{base_path}/cbbackupmgr restore --archive {backup_location} --repo {backup_repo} --cluster couchbase://{hostname}:{port} --username {username} --password $password --force-updates --no-progress-bar --include-buckets {csv_bucket_list}".format( - base_path=base_path, - backup_location=backup_location, - backup_repo=backup_repo, - hostname=hostname, - port=port, - username=username, - csv_bucket_list=csv_bucket_list + def check_index_build(base_path, hostname, port, username, **kwargs): + return "{base_path}/cbq -e {hostname}:{port} -u {username} -p $password -q=true -s=\"SELECT COUNT(*) as unbuilt FROM system:indexes WHERE state <> 'online'\"".format( + base_path=base_path, hostname=hostname, port=port, username=username ) @staticmethod - def monitor_replication(source_username, source_hostname, source_port, bucket_name, uuid): + def cb_backup_full(base_path, backup_location, backup_repo, hostname, port, username, csv_bucket_list, sudo, uid, skip, **kwargs): + if sudo: + return "sudo -u \#{uid} {base_path}/cbbackupmgr restore --archive {backup_location} --repo {backup_repo} --cluster couchbase://{hostname}:{port} --username {username} --password $password \ + --force-updates {skip} --no-progress-bar --include-buckets {csv_bucket_list}".format( + base_path=base_path, + backup_location=backup_location, + backup_repo=backup_repo, + hostname=hostname, + port=port, + username=username, + csv_bucket_list=csv_bucket_list, + uid=uid, + skip=skip + ) + else: + return "{base_path}/cbbackupmgr restore --archive {backup_location} --repo {backup_repo} --cluster couchbase://{hostname}:{port} --username {username} --password $password \ + --force-updates {skip} --no-progress-bar --include-buckets {csv_bucket_list}".format( + base_path=base_path, + backup_location=backup_location, + backup_repo=backup_repo, + hostname=hostname, + port=port, + username=username, + csv_bucket_list=csv_bucket_list, + skip=skip + ) + + @staticmethod + def monitor_replication(source_username, source_hostname, source_port, bucket_name, uuid, **kwargs): return "curl --silent -u {source_username}:$password http://{source_hostname}:{source_port}/pools/default/buckets/{bucket_name}/stats/replications%2F{uuid}%2F{bucket_name}%2F{bucket_name}%2Fchanges_left".format( source_username=source_username, source_hostname=source_hostname, @@ -333,12 +472,42 @@ def monitor_replication(source_username, source_hostname, source_port, bucket_na uuid=uuid, ) + # @staticmethod + # def couchbase_server_info(shell_path, hostname, port, username, **kwargs): + # return "{shell_path} server-info --cluster {hostname}:{port} --username {username} --password $password ".format( + # shell_path=shell_path, hostname=hostname, port=port, username=username + # ) + @staticmethod - def server_info(shell_path, hostname, port, username): - return "{shell_path} server-info --cluster {hostname}:{port} --username {username} --password $password ".format( + def couchbase_server_info(shell_path, hostname, username, port, **kwargs): + return "{shell_path} server-info --cluster {hostname}:{port} --username {username} --password $password".format( shell_path=shell_path, hostname=hostname, port=port, username=username ) + #return("{shell_path}".format(shell_path=shell_path)) + @staticmethod + def rename_cluster(shell_path, hostname, port, username, newuser, newname, **kwargs): + return "{shell_path} setting-cluster --cluster {hostname}:{port} --username {username} --password $password --cluster-username {newuser} --cluster-password $newpass --cluster-name {newname}".format( + shell_path=shell_path, hostname=hostname, port=port, username=username, + newuser=newuser, newname=newname + ) + + + @staticmethod + def server_add(shell_path, hostname, port, username, newhost, services, **kwargs): + return "{shell_path} server-add --cluster {hostname}:{port} --username {username} --password $password \ + --server-add http://{newhost}:{port} --server-add-username {username} --server-add-password $password \ + --services {services}".format( + shell_path=shell_path, hostname=hostname, port=port, username=username, services=services, newhost=newhost + ) + + + @staticmethod + def rebalance(shell_path, hostname, port, username, **kwargs): + return "{shell_path} rebalance --cluster {hostname}:{port} --username {username} --password $password \ + --no-progress-bar".format( + shell_path=shell_path, hostname=hostname, port=port, username=username + ) class CommandFactory(DatabaseCommand, OSCommand): def __init__(self): diff --git a/src/internal_exceptions/plugin_exceptions.py b/src/internal_exceptions/plugin_exceptions.py index 6e1018e..cf4cf71 100644 --- a/src/internal_exceptions/plugin_exceptions.py +++ b/src/internal_exceptions/plugin_exceptions.py @@ -9,7 +9,9 @@ ####################################################################################################################### from internal_exceptions.base_exceptions import PluginException +import logging +logger = logging.getLogger(__name__) class RepositoryDiscoveryError(PluginException): def __init__(self, message=""): @@ -45,10 +47,11 @@ def __init__(self, message=""): class MultipleSnapSyncError(PluginException): - def __init__(self, message=""): + def __init__(self, message="", filename=""): + logger.debug("Exception MultipleSnapSyncError file: {}".format(filename)) message = "SnapSync is running for any other dSource " + message super(MultipleSnapSyncError, self).__init__(message, - "Please wait while the other operation completes and try again ", + "Please wait while the other operation completes and try again or delete a lock file {}".format(filename), "Staging host already in use for SNAP-SYNC. Only Serial operations supported for couchbase") diff --git a/src/operations/discovery.py b/src/operations/discovery.py index 01bb4eb..0ac37f8 100644 --- a/src/operations/discovery.py +++ b/src/operations/discovery.py @@ -33,9 +33,10 @@ def find_repos(source_connection): install_path = helper_lib.find_install_path(source_connection, binary_path) shell_path = helper_lib.find_shell_path(source_connection, binary_path) version = helper_lib.find_version(source_connection, install_path) + (uid, gid) = helper_lib.find_ids(source_connection, install_path) pretty_name = "Couchbase ({})".format(version) repository_definition = RepositoryDefinition(cb_install_path=install_path, cb_shell_path=shell_path, - version=version, pretty_name=pretty_name) + version=version, pretty_name=pretty_name, uid=uid, gid=gid) repositories.append(repository_definition) return repositories diff --git a/src/operations/link_cbbkpmgr.py b/src/operations/link_cbbkpmgr.py index 88e5ef3..40cb195 100644 --- a/src/operations/link_cbbkpmgr.py +++ b/src/operations/link_cbbkpmgr.py @@ -7,6 +7,7 @@ import logging import os +import json from dlpx.virtualization.platform import Status @@ -18,185 +19,76 @@ from generated.definitions import SnapshotDefinition from internal_exceptions.plugin_exceptions import MultipleSyncError, MultipleSnapSyncError from operations import config +from operations import linking logger = logging.getLogger(__name__) def resync_cbbkpmgr(staged_source, repository, source_config, input_parameters): dsource_type = input_parameters.d_source_type + dsource_name = source_config.pretty_name + couchbase_host = input_parameters.couchbase_host bucket_size = staged_source.parameters.bucket_size rx_connection = staged_source.staged_connection resync_process = CouchbaseOperation( Resource.ObjectBuilder.set_staged_source(staged_source).set_repository(repository).set_source_config( source_config).build()) - config_dir = resync_process.create_config_dir() - config.SYNC_FILE_NAME = config_dir + "/" + get_sync_lock_file_name(dsource_type, source_config.pretty_name) - src_bucket_info_filename = db_commands.constants.SRC_BUCKET_INFO_FILENAME - src_bucket_info_filename = os.path.dirname(config_dir) + "/" + src_bucket_info_filename - logger.debug("src_bucket_info_filename = {}".format(src_bucket_info_filename)) - - if helper_lib.check_file_present(rx_connection, config.SYNC_FILE_NAME): - logger.debug("Sync file is already created by other process") - config.SYNC_FLAG_TO_USE_CLEANUP_ONLY_IF_CURRENT_JOB_CREATED = False - raise MultipleSyncError("Sync file is already created by other process") - else: - # creating sync file - msg = db_commands.constants.RESYNCE_OR_SNAPSYNC_FOR_OTHER_OBJECT_IN_PROGRESS.format(source_config.pretty_name, - input_parameters.couchbase_host) - helper_lib.write_file(rx_connection, msg, config.SYNC_FILE_NAME) - - resync_process.restart_couchbase() - resync_process.node_init() - resync_process.cluster_init() + + linking.check_for_concurrent(resync_process, dsource_type, dsource_name, couchbase_host) + + # validate if this works as well for backup + linking.configure_cluster(resync_process) + + logger.debug("Finding source and staging bucket list") - bucket_details_source = resync_process.source_bucket_list_offline(filename=src_bucket_info_filename) - bucket_details_staged = resync_process.bucket_list() - - config_setting = staged_source.parameters.config_settings_prov - logger.debug("Bucket names passed for configuration: {}".format(config_setting)) - - bucket_configured_staged = [] - if len(config_setting) > 0: - logger.debug("Getting bucket information from config") - for config_bucket in config_setting: - bucket_configured_staged.append(config_bucket["bucketName"]) - logger.debug("Filtering bucket name with size only from above output") - bkt_name_size = helper_lib.get_bucket_name_with_size(bucket_details_source, config_bucket["bucketName"]) - bkt_size_mb = get_bucket_size_in_MB(bucket_size, bkt_name_size.split(",")[1]) - - if config_bucket["bucketName"] not in bucket_details_staged: - resync_process.bucket_create(config_bucket["bucketName"], bkt_size_mb) - else: - logger.debug("Bucket {} already present in staged environment. Recreating bucket ".format( - config_bucket["bucketName"])) - resync_process.bucket_remove(config_bucket["bucketName"]) - resync_process.bucket_create(config_bucket["bucketName"], bkt_size_mb) - - logger.debug("Finding buckets present at staged server") - bucket_details_staged = resync_process.bucket_list() - filter_bucket_list = helper_lib.filter_bucket_name_from_output(bucket_details_staged) - extra_bucket = list(set(filter_bucket_list) - set(bucket_configured_staged)) - - logger.debug("Extra bucket found to delete:{} ".format(extra_bucket)) - for bucket in extra_bucket: - resync_process.bucket_remove(bucket) - else: - logger.debug("Finding buckets present at staged server with size") - all_bkt_list_with_size = helper_lib.get_all_bucket_list_with_size(bucket_details_source) - logger.debug("Filtering bucket name with size only from above output") - filter_source_bucket = helper_lib.filter_bucket_name_from_output(bucket_details_source) - for items in all_bkt_list_with_size: - if items: - logger.debug("Running bucket operations for {}".format(items)) - bkt_name, bkt_size = items.split(',') - - bkt_size_mb = get_bucket_size_in_MB(bucket_size, bkt_size) - if bkt_name not in bucket_details_staged: - resync_process.bucket_create(bkt_name, bkt_size_mb) - else: - logger.debug( - "Bucket {} already present in staged environment. Recreating bucket ".format(bkt_name)) - resync_process.bucket_remove(bkt_name) - resync_process.bucket_create(bkt_name, bkt_size_mb) - - bucket_details_staged = resync_process.bucket_list() - filter_staged_bucket = helper_lib.filter_bucket_name_from_output(bucket_details_staged) - extra_bucket = list(set(filter_staged_bucket) - set(filter_source_bucket)) - logger.info("Extra bucket found to delete:{}".format(extra_bucket)) - for bucket in extra_bucket: - resync_process.bucket_remove(bucket) - - bucket_details_staged = resync_process.bucket_list() - filter_bucket_list = helper_lib.filter_bucket_name_from_output(bucket_details_staged) - csv_bucket_list = ",".join(filter_bucket_list) + bucket_details_source = resync_process.source_bucket_list_offline() + bucket_details_staged = helper_lib.filter_bucket_name_from_output(resync_process.bucket_list()) + + buckets_toprocess = linking.buckets_precreation(resync_process, bucket_details_source, bucket_details_staged) + + csv_bucket_list = ",".join(buckets_toprocess) logger.debug("Started CB backup manager") + helper_lib.sleepForSecond(30) resync_process.cb_backup_full(csv_bucket_list) + helper_lib.sleepForSecond(30) + linking.build_indexes(resync_process) + logger.info("Stopping Couchbase") + resync_process.stop_couchbase() + resync_process.save_config('parent') def pre_snapshot_cbbkpmgr(staged_source, repository, source_config, input_parameters): + + + # this is for normal snapshot + #logger.info("Do nothing version Couchbase") + pre_snapshot_process = CouchbaseOperation( Resource.ObjectBuilder.set_staged_source(staged_source).set_repository(repository).set_source_config( source_config).build()) bucket_size = input_parameters.bucket_size rx_connection = staged_source.staged_connection - config_dir = pre_snapshot_process.create_config_dir() - config.SNAP_SYNC_FILE_NAME = config_dir + "/" + db_commands.constants.LOCK_SNAPSYNC_OPERATION - src_bucket_info_filename = db_commands.constants.SRC_BUCKET_INFO_FILENAME - src_bucket_info_filename = os.path.dirname(config_dir) + "/" + src_bucket_info_filename - - if helper_lib.check_file_present(rx_connection, config.SNAP_SYNC_FILE_NAME): - logger.debug("File path is already created {}".format(config.SNAP_SYNC_FILE_NAME)) - config.SNAP_SYNC_FLAG_TO_USE_CLEANUP_ONLY_IF_CURRENT_JOB_CREATED = False - raise MultipleSnapSyncError("SnapSync file is already created by other process") - else: - logger.debug("Creating lock file...") - msg = "dSource Creation / Snapsync for dSource {} is in progress. Same staging server {} cannot be used for other operations".format( - source_config.pretty_name, input_parameters.couchbase_host) - helper_lib.write_file(rx_connection, msg, config.SNAP_SYNC_FILE_NAME) - logger.debug("Re-ingesting from latest backup...") - pre_snapshot_process.start_couchbase() - pre_snapshot_process.node_init() - pre_snapshot_process.cluster_init() - bucket_details_source = pre_snapshot_process.source_bucket_list_offline( - filename=src_bucket_info_filename) - bucket_details_staged = pre_snapshot_process.bucket_list() - config_setting = staged_source.parameters.config_settings_prov - logger.debug("Buckets name passed for configuration: {}".format(config_setting)) - bucket_configured_staged = [] - if len(config_setting) != 0: - logger.debug("Inside config") - for config_bucket in config_setting: - logger.debug("Adding bucket names provided in config settings") - bucket_configured_staged.append(config_bucket["bucketName"]) - bkt_name_size = helper_lib.get_bucket_name_with_size(bucket_details_source, - config_bucket["bucketName"]) - bkt_size_mb = get_bucket_size_in_MB(bucket_size, bkt_name_size.split(",")[1]) - - if config_bucket["bucketName"] not in bucket_details_staged: - pre_snapshot_process.bucket_create(config_bucket["bucketName"], bkt_size_mb) - else: - pre_snapshot_process.bucket_remove(config_bucket["bucketName"]) - pre_snapshot_process.bucket_create(config_bucket["bucketName"], bkt_size_mb) - bucket_details_staged = pre_snapshot_process.bucket_list() - filter_bucket_list = helper_lib.filter_bucket_name_from_output(bucket_details_staged) - extra_bucket = list(set(filter_bucket_list) - set(bucket_configured_staged)) - logger.debug("Extra bucket found :{}".format(extra_bucket)) - for bucket in extra_bucket: - logger.debug("Deleting bucket {}".format(bucket)) - pre_snapshot_process.bucket_remove(bucket) - else: - all_bkt_list_with_size = helper_lib.get_all_bucket_list_with_size(bucket_details_source) - filter_source_bucket = helper_lib.filter_bucket_name_from_output(bucket_details_source) - logger.info("Creating the buckets") - for items in all_bkt_list_with_size: - if items: - bkt_name, bkt_size = items.split(',') - bkt_size_mb = get_bucket_size_in_MB(bucket_size, bkt_size) - if bkt_name not in bucket_details_staged: - pre_snapshot_process.bucket_create(bkt_name, bkt_size_mb) - else: - logger.info( - "Bucket {} already present in staged environment. Recreating bucket ".format( - bkt_name)) - pre_snapshot_process.bucket_remove(bkt_name) - pre_snapshot_process.bucket_create(bkt_name, bkt_size_mb) - - bucket_details_staged = pre_snapshot_process.bucket_list() - filter_staged_bucket = helper_lib.filter_bucket_name_from_output(bucket_details_staged) - extra_bucket = list(set(filter_staged_bucket) - set(filter_source_bucket)) - logger.info("Extra bucket found :{}".format(extra_bucket)) - for bucket in extra_bucket: - pre_snapshot_process.bucket_remove(bucket) - - bucket_details_staged = pre_snapshot_process.bucket_list() - filter_bucket_list = helper_lib.filter_bucket_name_from_output(bucket_details_staged) - csv_bucket_list = ",".join(filter_bucket_list) - pre_snapshot_process.cb_backup_full(csv_bucket_list) - logger.info("Re-ingesting from latest backup complete.") + dsource_type = input_parameters.d_source_type + dsource_name = source_config.pretty_name + couchbase_host = input_parameters.couchbase_host + linking.check_for_concurrent(pre_snapshot_process, dsource_type, dsource_name, couchbase_host) + + logger.debug("Finding source and staging bucket list") + bucket_details_source = pre_snapshot_process.source_bucket_list_offline() + bucket_details_staged = helper_lib.filter_bucket_name_from_output(pre_snapshot_process.bucket_list()) + + bucket_details_staged = pre_snapshot_process.bucket_list() + filter_bucket_list = helper_lib.filter_bucket_name_from_output(bucket_details_staged) + csv_bucket_list = ",".join(filter_bucket_list) + pre_snapshot_process.cb_backup_full(csv_bucket_list) + logger.info("Re-ingesting from latest backup complete.") + + linking.build_indexes(pre_snapshot_process) logger.info("Stopping Couchbase") pre_snapshot_process.stop_couchbase() + pre_snapshot_process.save_config('parent') def post_snapshot_cbbkpmgr(staged_source, repository, source_config, dsource_type): @@ -210,26 +102,36 @@ def post_snapshot_cbbkpmgr(staged_source, repository, source_config, dsource_typ bucket_list = [] bucket_details = post_snapshot_process.bucket_list() - if len(staged_source.parameters.config_settings_prov) != 0: - bucket_list = [] - for config_setting in staged_source.parameters.config_settings_prov: - bucket_list.append(helper_lib.get_bucket_name_with_size(bucket_details, config_setting["bucketName"])) - else: - bucket_list = helper_lib.get_stg_all_bucket_list_with_ramquota_size(bucket_details) + # if len(staged_source.parameters.config_settings_prov) != 0: + # bucket_list = [] + # for config_setting in staged_source.parameters.config_settings_prov: + # bucket_list.append(helper_lib.get_bucket_name_with_size(bucket_details, config_setting["bucketName"])) + # else: + # bucket_list = helper_lib.get_stg_all_bucket_list_with_ramquota_size(bucket_details) + + # extract index + + ind = post_snapshot_process.get_indexes_definition() + logger.debug("indexes definition : {}".format(ind)) + + snapshot.indexes = ind snapshot.db_path = staged_source.parameters.mount_path snapshot.couchbase_port = source_config.couchbase_src_port snapshot.couchbase_host = source_config.couchbase_src_host - snapshot.bucket_list = ":".join(bucket_list) + snapshot.bucket_list = json.dumps(bucket_details) snapshot.time_stamp = helper_lib.current_time() snapshot.snapshot_id = str(helper_lib.get_snapshot_id()) - logger.debug("snapshot schema: {}".format(snapshot)) + snapshot.couchbase_admin = post_snapshot_process.parameters.couchbase_admin + snapshot.couchbase_admin_password = post_snapshot_process.parameters.couchbase_admin_password + #logger.debug("snapshot schema: {}".format(snapshot)) logger.debug("Deleting the lock files") helper_lib.delete_file(rx_connection, config.SNAP_SYNC_FILE_NAME) helper_lib.delete_file(rx_connection, config.SYNC_FILE_NAME) - post_snapshot_process.stop_couchbase() - helper_lib.unmount_file_system(rx_connection, staged_source.parameters.mount_path) - logger.debug("Un mounting completed") + # for Prox investigation + #post_snapshot_process.stop_couchbase() + #helper_lib.unmount_file_system(rx_connection, staged_source.parameters.mount_path) + #logger.debug("Un mounting completed") return snapshot @@ -237,6 +139,10 @@ def start_staging_cbbkpmgr(staged_source, repository, source_config): start_staging = CouchbaseOperation( Resource.ObjectBuilder.set_staged_source(staged_source).set_repository(repository).set_source_config( source_config).build()) + + start_staging.delete_config() + # TODO error handling + start_staging.restore_config(what='current') start_staging.start_couchbase() @@ -245,12 +151,20 @@ def stop_staging_cbbkpmgr(staged_source, repository, source_config): Resource.ObjectBuilder.set_staged_source(staged_source).set_repository(repository).set_source_config( source_config).build()) stop_staging.stop_couchbase() + stop_staging.save_config(what='current') + stop_staging.delete_config() def d_source_status_cbbkpmgr(staged_source, repository, source_config): - if helper_lib.check_dir_present(staged_source.staged_connection, staged_source.parameters.couchbase_bak_loc): - return Status.ACTIVE - return Status.INACTIVE + # if helper_lib.check_dir_present(staged_source.staged_connection, staged_source.parameters.couchbase_bak_loc): + # return Status.ACTIVE + # return Status.INACTIVE + status_obj = CouchbaseOperation( + Resource.ObjectBuilder.set_staged_source(staged_source).set_repository(repository).set_source_config( + source_config).build()) + logger.debug("Checking status for D_SOURCE: {}".format(source_config.pretty_name)) + return status_obj.status() + def unmount_file_system_in_error_case(staged_source, repository, source_config): diff --git a/src/operations/link_xdcr.py b/src/operations/link_xdcr.py index e3a3e67..681884b 100644 --- a/src/operations/link_xdcr.py +++ b/src/operations/link_xdcr.py @@ -7,6 +7,8 @@ import logging import os +import json +import re from generated.definitions import SnapshotDefinition import db_commands.constants @@ -17,11 +19,14 @@ from internal_exceptions.database_exceptions import DuplicateClusterError from internal_exceptions.plugin_exceptions import MultipleSyncError, MultipleXDCRSyncError from operations import config +from operations import linking +from dlpx.virtualization.platform.exceptions import UserError logger = logging.getLogger(__name__) def resync_xdcr(staged_source, repository, source_config, input_parameters): + logger.debug("START resync_xdcr") dsource_type = input_parameters.d_source_type dsource_name = source_config.pretty_name bucket_size = staged_source.parameters.bucket_size @@ -29,100 +34,49 @@ def resync_xdcr(staged_source, repository, source_config, input_parameters): resync_process = CouchbaseOperation( Resource.ObjectBuilder.set_staged_source(staged_source).set_repository(repository).set_source_config( source_config).build()) - config_dir = resync_process.create_config_dir() - config.SYNC_FILE_NAME = config_dir + "/" + helper_lib.get_sync_lock_file_name(dsource_type, dsource_name) - if not verify_sync_lock_file_for_this_job(rx_connection, config.SYNC_FILE_NAME): - config.SYNC_FLAG_TO_USE_CLEANUP_ONLY_IF_CURRENT_JOB_CREATED = False - logger.debug("Sync file is already created by other dSource") - raise MultipleXDCRSyncError("Sync file is already created by other dSource") - else: - # creating sync file - msg = db_commands.constants.RESYNCE_OR_SNAPSYNC_FOR_OTHER_OBJECT_IN_PROGRESS.format(dsource_name, - input_parameters.couchbase_host) - helper_lib.write_file(rx_connection, msg, config.SYNC_FILE_NAME) - - resync_process.restart_couchbase() - resync_process.node_init() - resync_process.cluster_init() - already_set_up_done, name_conflict = resync_process.check_duplicate_replication( - resync_process.parameters.stg_cluster_name) - if already_set_up_done: - logger.info("No need to XDCR setup again") - elif name_conflict: - raise DuplicateClusterError("Already cluster is present") - else: - logger.info("First time XDCR set up") - resync_process.xdcr_setup() + couchbase_host = input_parameters.couchbase_host + + + linking.check_for_concurrent(resync_process, dsource_type, dsource_name, couchbase_host) + + linking.configure_cluster(resync_process) + + + + # common steps for both XDCR & CB back up - logger.debug("Finding source and staging bucket list") + bucket_details_source = resync_process.source_bucket_list() bucket_details_staged = resync_process.bucket_list() - config_setting = staged_source.parameters.config_settings_prov - logger.debug("Bucket names passed for configuration: {}".format(config_setting)) - bucket_configured_staged = [] - if len(config_setting) > 0: - logger.debug("Getting bucket information from config") - for config_bucket in config_setting: - bucket_configured_staged.append(config_bucket["bucketName"]) - logger.debug("Filtering bucket name with size only from above output") - bkt_name_size = helper_lib.get_bucket_name_with_size(bucket_details_source, config_bucket["bucketName"]) - bkt_size_mb = helper_lib.get_bucket_size_in_MB(bucket_size, bkt_name_size.split(",")[1]) - - if config_bucket["bucketName"] not in bucket_details_staged: - resync_process.bucket_create(config_bucket["bucketName"], bkt_size_mb) - else: - logger.debug("Bucket {} already present in staged environment. Recreating bucket ".format( - config_bucket["bucketName"])) - resync_process.bucket_remove(config_bucket["bucketName"]) - resync_process.bucket_create(config_bucket["bucketName"], bkt_size_mb) - resync_process.xdcr_replicate(config_bucket["bucketName"], config_bucket["bucketName"]) - - logger.debug("Finding buckets present at staged server") - bucket_details_staged = resync_process.bucket_list() - filter_bucket_list = helper_lib.filter_bucket_name_from_output(bucket_details_staged) - extra_bucket = list(set(filter_bucket_list) - set(bucket_configured_staged)) - - logger.debug("Extra bucket found to delete:{} ".format(extra_bucket)) - for bucket in extra_bucket: - resync_process.bucket_remove(bucket) - else: - logger.debug("Finding buckets present at staged server with size") - all_bkt_list_with_size = helper_lib.get_all_bucket_list_with_size(bucket_details_source) - logger.debug("Filtering bucket name with size only from above output") - filter_source_bucket = helper_lib.filter_bucket_name_from_output(bucket_details_source) - for items in all_bkt_list_with_size: - if items: - logger.debug("Running bucket operations for {}".format(items)) - bkt_name, bkt_size = items.split(',') - - bkt_size_mb = helper_lib.get_bucket_size_in_MB(bucket_size, bkt_size) - if bkt_name not in bucket_details_staged: - resync_process.bucket_create(bkt_name, bkt_size_mb) - else: - logger.debug( - "Bucket {} already present in staged environment. Recreating bucket ".format(bkt_name)) - resync_process.bucket_remove(bkt_name) - resync_process.bucket_create(bkt_name, bkt_size_mb) - resync_process.xdcr_replicate(bkt_name, bkt_name) - - bucket_details_staged = resync_process.bucket_list() - filter_staged_bucket = helper_lib.filter_bucket_name_from_output(bucket_details_staged) - extra_bucket = list(set(filter_staged_bucket) - set(filter_source_bucket)) - logger.info("Extra bucket found to delete:{}".format(extra_bucket)) - for bucket in extra_bucket: - resync_process.bucket_remove(bucket) + buckets_toprocess = linking.buckets_precreation(resync_process, bucket_details_source, bucket_details_staged) + + # run this for all buckets + resync_process.setup_replication() logger.debug("Finding staging_uuid & cluster_name on staging") - staging_uuid, cluster_name_staging = resync_process.get_replication_uuid() - bucket_details_staged = resync_process.bucket_list() - logger.debug("Filtering bucket name from output") - filter_bucket_list = helper_lib.filter_bucket_name_from_output(bucket_details_staged) - for bkt in filter_bucket_list: + staging_uuid = resync_process.get_replication_uuid() + + + if staging_uuid is None: + logger.debug("Can't find a replication UUID after setting it up") + raise UserError("Can't find a replication UUID after setting it up") + + # bucket_details_staged = resync_process.bucket_list() + # logger.debug("Filtering bucket name from output") + # filter_bucket_list = helper_lib.filter_bucket_name_from_output(bucket_details_staged) + for bkt in buckets_toprocess: resync_process.monitor_bucket(bkt, staging_uuid) + linking.build_indexes(resync_process) + + logger.info("Stopping Couchbase") + resync_process.stop_couchbase() + resync_process.save_config('parent') + + def pre_snapshot_xdcr(staged_source, repository, source_config, input_parameters): logger.info("In Pre snapshot...") pre_snapshot_process = CouchbaseOperation( @@ -140,6 +94,7 @@ def pre_snapshot_xdcr(staged_source, repository, source_config, input_parameters helper_lib.write_file(staged_source.staged_connection, msg, config.SNAP_SYNC_FILE_NAME) logger.info("Stopping Couchbase") pre_snapshot_process.stop_couchbase() + pre_snapshot_process.save_config('parent') def post_snapshot_xdcr(staged_source, repository, source_config, dsource_type): @@ -148,24 +103,35 @@ def post_snapshot_xdcr(staged_source, repository, source_config, dsource_type): Resource.ObjectBuilder.set_staged_source(staged_source).set_repository(repository).set_source_config( source_config).build()) + + # post_snapshot_process.save_config() post_snapshot_process.start_couchbase() snapshot = SnapshotDefinition(validate=False) bucket_details = post_snapshot_process.bucket_list() - if len(staged_source.parameters.config_settings_prov) != 0: - bucket_list = [] - for config_setting in staged_source.parameters.config_settings_prov: - bucket_list.append(helper_lib.get_bucket_name_with_size(bucket_details, config_setting["bucketName"])) - else: - bucket_list = helper_lib.get_stg_all_bucket_list_with_ramquota_size(bucket_details) + # if len(staged_source.parameters.config_settings_prov) != 0: + # bucket_list = [] + # for config_setting in staged_source.parameters.config_settings_prov: + # bucket_list.append(helper_lib.get_bucket_name_with_size(bucket_details, config_setting["bucketName"])) + # else: + # bucket_list = helper_lib.get_stg_all_bucket_list_with_ramquota_size(bucket_details) + + ind = post_snapshot_process.get_indexes_definition() + logger.debug("indexes definition : {}".format(ind)) + + snapshot.indexes = ind + + bucket_details = post_snapshot_process.bucket_list() snapshot.db_path = staged_source.parameters.mount_path snapshot.couchbase_port = source_config.couchbase_src_port snapshot.couchbase_host = source_config.couchbase_src_host - snapshot.bucket_list = ":".join(bucket_list) + snapshot.bucket_list = json.dumps(bucket_details) snapshot.time_stamp = helper_lib.current_time() snapshot.snapshot_id = str(helper_lib.get_snapshot_id()) - logger.debug("snapshot schema: {}".format(snapshot)) + snapshot.couchbase_admin = post_snapshot_process.parameters.couchbase_admin + snapshot.couchbase_admin_password = post_snapshot_process.parameters.couchbase_admin_password + #logger.debug("snapshot schema: {}".format(snapshot)) logger.debug("Deleting the snap sync lock file {}".format(config.SNAP_SYNC_FILE_NAME)) helper_lib.delete_file(staged_source.staged_connection, config.SNAP_SYNC_FILE_NAME) return snapshot @@ -179,37 +145,35 @@ def start_staging_xdcr(staged_source, repository, source_config): logger.debug("Enabling the D_SOURCE:{}".format(source_config.pretty_name)) dsource_type = staged_source.parameters.d_source_type rx_connection = staged_source.staged_connection + + + start_staging.stop_couchbase() + start_staging.delete_config() + # TODO error handling + start_staging.restore_config(what='current') start_staging.start_couchbase() - already_set_up_done, name_conflict = start_staging.check_duplicate_replication( - start_staging.parameters.stg_cluster_name) - if already_set_up_done: - logger.info("No need to XDCR setup again") - elif name_conflict: - raise DuplicateClusterError("Already cluster is present") - else: - logger.info("First time XDCR set up") - start_staging.xdcr_setup() + # already_set_up_done, name_conflict = start_staging.check_duplicate_replication( + # start_staging.parameters.stg_cluster_name) + # if already_set_up_done: + # logger.info("No need to XDCR setup again") + # elif name_conflict: + # raise DuplicateClusterError("Already cluster is present") + # else: + # logger.info("First time XDCR set up") + # start_staging.xdcr_setup() - config_setting = staged_source.parameters.config_settings_prov - if len(config_setting) > 0: - for config_bucket in config_setting: - logger.debug("Creating replication for {}".format(config_bucket["bucketName"])) - start_staging.xdcr_replicate(config_bucket["bucketName"], config_bucket["bucketName"]) - else: - bucket_details_source = start_staging.source_bucket_list() - all_bkt_list_with_size = helper_lib.get_all_bucket_list_with_size(bucket_details_source) - for items in all_bkt_list_with_size: - bkt_name, bkt_size = items.split(',') - logger.debug("Creating replication for {}".format(bkt_name)) - start_staging.xdcr_replicate(bkt_name, bkt_name) - - config_dir = start_staging.create_config_dir() - msg = "dSource Creation / Snapsync for dSource {} is in progress".format(source_config.pretty_name) - helper_lib.write_file(rx_connection, msg, - config_dir + "/" + helper_lib.get_sync_lock_file_name(dsource_type, - source_config.pretty_name)) + start_staging.setup_replication() + + + + + config_dir = start_staging.create_config_dir() + msg = "dSource Creation / Snapsync for dSource {} is in progress".format(source_config.pretty_name) + helper_lib.write_file(rx_connection, msg, + config_dir + "/" + helper_lib.get_sync_lock_file_name(dsource_type, + source_config.pretty_name)) logger.debug("D_SOURCE:{} enabled".format(source_config.pretty_name)) @@ -231,26 +195,12 @@ def stop_staging_xdcr(staged_source, repository, source_config): config_dir + "/" + helper_lib.get_sync_lock_file_name(dsource_type, source_config.pretty_name)) stop_staging.stop_couchbase() + stop_staging.save_config(what='current') + stop_staging.delete_config() logger.debug("D_SOURCE:{} disabled".format(source_config.pretty_name)) -def d_source_status_xdcr(staged_source, repository, source_config): - status_obj = CouchbaseOperation( - Resource.ObjectBuilder.set_staged_source(staged_source).set_repository(repository).set_source_config( - source_config).build()) - logger.debug("Checking status for D_SOURCE: {}".format(source_config.pretty_name)) - return status_obj.status() - - -def verify_sync_lock_file_for_this_job(rx_connection, sync_filename): - if helper_lib.check_file_present(rx_connection, sync_filename): - logger.debug("Sync File Present: {}".format(sync_filename)) - return True - config_dir = os.path.dirname(sync_filename) - - possible_sync_filename = "/*" + db_commands.constants.LOCK_SYNC_OPERATION - possible_sync_filename = config_dir + possible_sync_filename - logger.debug("Checking for {}".format(possible_sync_filename)) - if helper_lib.check_file_present(rx_connection, possible_sync_filename): - return False - return True + + + + diff --git a/src/operations/linked.py b/src/operations/linked.py index a7246e1..08fac6d 100644 --- a/src/operations/linked.py +++ b/src/operations/linked.py @@ -10,14 +10,16 @@ import config import db_commands +import linking from controller import helper_lib from controller.couchbase_operation import CouchbaseOperation from controller.resource_builder import Resource from controller.helper_lib import delete_file from db_commands import constants from internal_exceptions.base_exceptions import PluginException, DatabaseException, GenericUserError -from internal_exceptions.plugin_exceptions import MountPathError +from internal_exceptions.plugin_exceptions import MountPathError, MultipleSnapSyncError from operations import link_cbbkpmgr, link_xdcr +from dlpx.virtualization.platform.exceptions import UserError logger = logging.getLogger(__name__) @@ -29,7 +31,13 @@ def resync(staged_source, repository, source_config, input_parameters): link_cbbkpmgr.resync_cbbkpmgr(staged_source, repository, source_config, input_parameters) elif input_parameters.d_source_type == constants.XDCR: link_xdcr.resync_xdcr(staged_source, repository, source_config, input_parameters) + + logger.debug("Completed resynchronization") + except UserError: + raise + except Exception as ex_obj: + logger.debug(str(ex_obj)) logger.debug("Caught exception {}".format(ex_obj.message)) _cleanup_in_exception_case(staged_source.staged_connection, True, False) if input_parameters.d_source_type == constants.CBBKPMGR: @@ -37,7 +45,7 @@ def resync(staged_source, repository, source_config, input_parameters): if isinstance(ex_obj, PluginException) or isinstance(ex_obj, DatabaseException) or isinstance(ex_obj, GenericUserError): raise ex_obj.to_user_error(), None, sys.exc_info()[2] raise - logger.debug("Completed resynchronization") + def pre_snapshot(staged_source, repository, source_config, input_parameters): @@ -47,6 +55,9 @@ def pre_snapshot(staged_source, repository, source_config, input_parameters): link_cbbkpmgr.pre_snapshot_cbbkpmgr(staged_source, repository, source_config, input_parameters) elif input_parameters.d_source_type == constants.XDCR: link_xdcr.pre_snapshot_xdcr(staged_source, repository, source_config, input_parameters) + logger.debug("Completed Pre-snapshot") + except UserError: + raise except Exception as ex_obj: logger.debug("Caught exception: {}".format(ex_obj.message)) _cleanup_in_exception_case(staged_source.staged_connection, True, True) @@ -55,7 +66,7 @@ def pre_snapshot(staged_source, repository, source_config, input_parameters): if isinstance(ex_obj, PluginException) or isinstance(ex_obj, DatabaseException) or isinstance(ex_obj, GenericUserError): raise ex_obj.to_user_error(), None, sys.exc_info()[2] raise - logger.debug("Completed Pre-snapshot") + def post_snapshot(staged_source, repository, source_config, dsource_type): @@ -65,13 +76,16 @@ def post_snapshot(staged_source, repository, source_config, dsource_type): return link_cbbkpmgr.post_snapshot_cbbkpmgr(staged_source, repository, source_config, dsource_type) elif dsource_type == constants.XDCR: return link_xdcr.post_snapshot_xdcr(staged_source, repository, source_config, dsource_type) + logger.debug("Completed Post-snapshot") + except UserError: + raise except Exception as err: logger.debug("Caught exception in post snapshot: {}".format(err.message)) _cleanup_in_exception_case(staged_source.staged_connection, True, True) if dsource_type == constants.CBBKPMGR: link_cbbkpmgr.unmount_file_system_in_error_case(staged_source, repository, source_config) raise - logger.debug("Completed Post-snapshot") + def start_staging(staged_source, repository, source_config): @@ -81,6 +95,9 @@ def start_staging(staged_source, repository, source_config): link_cbbkpmgr.start_staging_cbbkpmgr(staged_source, repository, source_config) elif staged_source.parameters.d_source_type == constants.XDCR: link_xdcr.start_staging_xdcr(staged_source, repository, source_config) + logger.debug("D_SOURCE:{} enabled".format(source_config.pretty_name)) + except UserError: + raise except Exception as err: logger.debug("Enable operation is failed!" + err.message) raise @@ -93,17 +110,18 @@ def stop_staging(staged_source, repository, source_config): link_cbbkpmgr.stop_staging_cbbkpmgr(staged_source, repository, source_config) elif staged_source.parameters.d_source_type == constants.XDCR: link_xdcr.stop_staging_xdcr(staged_source, repository, source_config) + logger.debug("D_SOURCE:{} disabled".format(source_config.pretty_name)) + except UserError: + raise except Exception as err: logger.debug("Disable operation is failed!" + err.message) raise - logger.debug("D_SOURCE:{} disabled".format(source_config.pretty_name)) + def d_source_status(staged_source, repository, source_config): - if staged_source.parameters.d_source_type == constants.CBBKPMGR: - return link_cbbkpmgr.d_source_status_cbbkpmgr(staged_source, repository, source_config) - elif staged_source.parameters.d_source_type == constants.XDCR: - return link_xdcr.d_source_status_xdcr(staged_source, repository, source_config) + return linking.d_source_status(staged_source, repository, source_config) + #This function verifies that LOCK_SNAPSYNC_OPERATION or LOCK_SYNC_OPERATION is present in hidden folder or not @@ -115,9 +133,9 @@ def check_mount_path(staged_source, repository): snapsync_filename = mount_path_check.create_config_dir() + "/" + db_commands.constants.LOCK_SNAPSYNC_OPERATION sync_filename = mount_path_check.create_config_dir() + "/" + db_commands.constants.LOCK_SYNC_OPERATION if helper_lib.check_file_present(staged_source.staged_connection, snapsync_filename) : - raise MountPathError("Another Snap-Sync process is in progress ").to_user_error(), None, sys.exc_info()[2] + raise MultipleSnapSyncError("Another Snap-Sync process is in progress ", snapsync_filename).to_user_error() if helper_lib.check_file_present(staged_source.staged_connection, sync_filename): - raise MountPathError("Another Sync process is in progress ").to_user_error(), None, sys.exc_info()[2] + raise MultipleSnapSyncError("Another Sync process is in progress ", sync_filename).to_user_error() return True diff --git a/src/operations/linking.py b/src/operations/linking.py new file mode 100644 index 0000000..2886e8a --- /dev/null +++ b/src/operations/linking.py @@ -0,0 +1,193 @@ +# +# Copyright (c) 2020-2021 by Delphix. All rights reserved. +# +####################################################################################################################### +# In this module, functions defined common ingestion modes - backup and xdrc +####################################################################################################################### + +import logging +import os +import json +import time + +from dlpx.virtualization.platform import Status + +import db_commands.constants +from controller import helper_lib +from controller.couchbase_operation import CouchbaseOperation +from controller.helper_lib import get_bucket_size_in_MB, get_sync_lock_file_name +from controller.resource_builder import Resource +from generated.definitions import SnapshotDefinition +from internal_exceptions.database_exceptions import DuplicateClusterError +from internal_exceptions.plugin_exceptions import MultipleSyncError, MultipleXDCRSyncError +from operations import config +from dlpx.virtualization.platform.exceptions import UserError + +logger = logging.getLogger(__name__) + +# potentially to remove - as checks are done on the mount points +def check_for_concurrent(couchbase_obj, dsource_type, dsource_name, couchbase_host): + config_dir = couchbase_obj.create_config_dir() + + config.SYNC_FILE_NAME = config_dir + "/" + helper_lib.get_sync_lock_file_name(dsource_type, dsource_name) + + + delphix_config_dir = couchbase_obj.get_config_directory() + logger.debug("Check if we have config dir in Delphix storage") + if not helper_lib.check_dir_present(couchbase_obj.connection, delphix_config_dir): + logger.debug("make a Delphix storage dir {}".format(delphix_config_dir)) + couchbase_obj.make_directory(delphix_config_dir) + + if not verify_sync_lock_file_for_this_job(couchbase_obj.connection, config.SYNC_FILE_NAME): + config.SYNC_FLAG_TO_USE_CLEANUP_ONLY_IF_CURRENT_JOB_CREATED = False + logger.debug("Sync file is already created by other dSource") + raise MultipleXDCRSyncError("Sync file is already created by other dSource") + else: + # creating sync file + msg = db_commands.constants.RESYNCE_OR_SNAPSYNC_FOR_OTHER_OBJECT_IN_PROGRESS.format(dsource_name, couchbase_host) + helper_lib.write_file(couchbase_obj.connection, msg, config.SYNC_FILE_NAME) + +def verify_sync_lock_file_for_this_job(rx_connection, sync_filename): + if helper_lib.check_file_present(rx_connection, sync_filename): + logger.debug("Sync File Present: {}".format(sync_filename)) + return True + config_dir = os.path.dirname(sync_filename) + + possible_sync_filename = "/*" + db_commands.constants.LOCK_SYNC_OPERATION + possible_sync_filename = config_dir + possible_sync_filename + logger.debug("Checking for {}".format(possible_sync_filename)) + if helper_lib.check_file_present(rx_connection, possible_sync_filename): + return False + return True + + +def configure_cluster(couchbase_obj): + # configure Couchbase cluster + + logger.debug("Checking cluster config") + if couchbase_obj.check_config(): + logger.debug("cluster config found - restoring") + couchbase_obj.stop_couchbase() + couchbase_obj.restore_config() + couchbase_obj.start_couchbase() + else: + logger.debug("cluster config not found - preparing node") + # no config in delphix directory + # initial cluster setup + couchbase_obj.stop_couchbase() + # we can't use normal monitor as server is not configured yet + couchbase_obj.start_couchbase(no_wait=True) + + end_time = time.time() + 3660 + + server_status = Status.INACTIVE + + #break the loop either end_time is exceeding from 1 hour or server is successfully started + while time.time() < end_time and server_status<>Status.ACTIVE: + helper_lib.sleepForSecond(1) # waiting for 1 second + server_status = couchbase_obj.staging_bootstrap_status() # fetching status + logger.debug("server status {}".format(server_status)) + + # check if cluster not configured and raise an issue + if couchbase_obj.check_cluster_notconfigured(): + logger.debug("Node not configured - creating a new cluster") + couchbase_obj.node_init() + couchbase_obj.cluster_init() + logger.debug("Cluster configured") + else: + logger.debug("Node configured but no configuration in Delphix - ???????") + if couchbase_obj.check_cluster_configured(): + logger.debug("Configured with staging user/password and alive so not a problem - continue") + else: + logger.debug("Cluster configured but not with user/password given in Delphix potentially another cluster") + raise UserError("Cluster configured but not with user/password given in Delphix potentially another cluster") + + +def buckets_precreation(couchbase_obj, bucket_details_source, bucket_details_staged): + # common steps for both XDCR & CB back up + # return a list of precreated buckets to process + logger.debug("buckets_precreation") + bucket_list = [] + config_setting = couchbase_obj.parameters.config_settings_prov + logger.debug("Bucket names passed for configuration: {}".format(config_setting)) + bucket_configured_staged = [] + if len(config_setting) > 0: + # process for list of buckets + logger.debug("Getting bucket information from config") + buckets_dict = { b["name"]:b for b in bucket_details_source } + for config_bucket in config_setting: + bucket_configured_staged.append(config_bucket["bucketName"]) + logger.debug("Filtering bucket name with size only from above output") + bucket = buckets_dict[config_bucket["bucketName"]] + logger.debug("Running bucket operations for {}".format(bucket)) + bkt_name = bucket['name'] + bkt_size = bucket['ram'] + bkt_type = bucket['bucketType'] + bkt_compression = bucket['compressionMode'] + + bkt_size_mb = helper_lib.get_bucket_size_in_MB(couchbase_obj.parameters.bucket_size, bkt_size) + + if config_bucket["bucketName"] not in bucket_details_staged: + couchbase_obj.bucket_create(config_bucket["bucketName"], bkt_size_mb, bkt_type, bkt_compression) + else: + logger.debug("Bucket {} already present in staged environment. Recreating bucket ".format( + config_bucket["bucketName"])) + couchbase_obj.bucket_remove(config_bucket["bucketName"]) + couchbase_obj.bucket_create(config_bucket["bucketName"], bkt_size_mb, bkt_type, bkt_compression) + + bucket_list.append(config_bucket["bucketName"]) + + + logger.debug("Finding buckets present at staged server") + bucket_details_staged = couchbase_obj.bucket_list() + filter_bucket_list = helper_lib.filter_bucket_name_from_output(bucket_details_staged) + extra_bucket = list(set(filter_bucket_list) - set(bucket_configured_staged)) + + logger.debug("Extra bucket found to delete:{} ".format(extra_bucket)) + for bucket in extra_bucket: + couchbase_obj.bucket_remove(bucket) + else: + # process for all buckets + filter_source_bucket = helper_lib.filter_bucket_name_from_json(bucket_details_source) + for items in bucket_details_source: + if items: + logger.debug("Running bucket operations for {}".format(items)) + bkt_name = items['name'] + bkt_size = items['ram'] + bkt_type = items['bucketType'] + bkt_compression = items['compressionMode'] + + bkt_size_mb = helper_lib.get_bucket_size_in_MB(couchbase_obj.parameters.bucket_size, bkt_size) + if bkt_name not in bucket_details_staged: + couchbase_obj.bucket_create(bkt_name, bkt_size_mb, bkt_type, bkt_compression) + else: + logger.debug( + "Bucket {} already present in staged environment. Recreating bucket ".format(bkt_name)) + couchbase_obj.bucket_remove(bkt_name) + couchbase_obj.bucket_create(bkt_name, bkt_size_mb, bkt_type, bkt_compression) + bucket_list.append(bkt_name) + + + return bucket_list + + +def build_indexes(couchbase_obj): + # create indexes based on the index definition + + logger.debug("index builder") + ind = couchbase_obj.get_indexes_definition() + logger.debug("indexes definition : {}".format(ind)) + for i in ind: + logger.debug(i) + couchbase_obj.build_index(i) + couchbase_obj.check_index_build() + + + +def d_source_status(staged_source, repository, source_config): + status_obj = CouchbaseOperation( + Resource.ObjectBuilder.set_staged_source(staged_source).set_repository(repository).set_source_config( + source_config).build()) + logger.debug("Checking status for D_SOURCE: {}".format(source_config.pretty_name)) + return status_obj.status() + diff --git a/src/operations/virtual.py b/src/operations/virtual.py index 32f6671..e01d4d4 100644 --- a/src/operations/virtual.py +++ b/src/operations/virtual.py @@ -9,6 +9,8 @@ ####################################################################################################################### import re +import json +import time # Auto generated libs import sys @@ -21,6 +23,11 @@ from controller.couchbase_operation import CouchbaseOperation import logging from controller.resource_builder import Resource +from dlpx.virtualization.common import RemoteEnvironment +from dlpx.virtualization.common import RemoteHost +from dlpx.virtualization.common import RemoteUser +from dlpx.virtualization.common import RemoteConnection +from dlpx.virtualization.platform import Status # Global logger for this File logger = logging.getLogger(__name__) @@ -32,41 +39,225 @@ def vdb_status(virtual_source, repository, source_config): source_config).build()) cb_status = provision_process.status() logger.debug("VDB Status is {}".format(cb_status)) + + if cb_status == Status.ACTIVE: + logger.debug("Checking mount point") + if helper_lib.check_stale_mountpoint(provision_process.connection, virtual_source.parameters.mount_path): + logger.debug("error with mount point - report inactive") + return Status.INACTIVE + else: + return Status.ACTIVE + return cb_status def vdb_unconfigure(virtual_source, repository, source_config): # delete all buckets + provision_process = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_source_config( + source_config).build()) + vdb_stop(virtual_source, repository, source_config) + provision_process.delete_config() + + + if provision_process.parameters.node_list is not None and len(provision_process.parameters.node_list) > 0: + for node in provision_process.parameters.node_list: + logger.debug("+++++++++++++++++++++++++++") + logger.debug(node) + logger.debug("+++++++++++++++++++++++++++") + addnode = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_source_config( + source_config).build(), + make_nonprimary_connection(provision_process.connection, node['environment'], node['environmentUser'])) + addnode.delete_config() + addnode.stop_couchbase() def vdb_reconfigure(virtual_source, repository, source_config, snapshot): # delete all buckets # calll configure - vdb_start(virtual_source, repository, source_config) + + logger.debug("In vdb_reconfigure...") + provision_process = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_source_config( + source_config).build()) + + + provision_process.stop_couchbase() + + if provision_process.parameters.node_list is not None and len(provision_process.parameters.node_list) > 0: + multinode = True + server_count = len(provision_process.parameters.node_list) + 1 + else: + multinode = False + + nodeno = 1 + provision_process.restore_config(what='current', nodeno=nodeno) + provision_process.start_couchbase(no_wait=multinode) + + + if provision_process.parameters.node_list is not None and len(provision_process.parameters.node_list) > 0: + for node in provision_process.parameters.node_list: + nodeno = nodeno + 1 + logger.debug("+++++++++++++++++++++++++++") + logger.debug(node) + logger.debug(nodeno) + logger.debug("+++++++++++++++++++++++++++") + addnode = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_source_config( + source_config).build(), + make_nonprimary_connection(provision_process.connection, node['environment'], node['environmentUser'])) + addnode.stop_couchbase() + addnode.restore_config(what='current', nodeno=nodeno) + addnode.start_couchbase(no_wait=multinode) + + + logger.debug("reconfigure for multinode: {}".format(multinode)) + + + if multinode == True: + + + active_servers = {} + logger.debug("wait for nodes") + logger.debug("server count: {} active servers: {}".format(server_count, sum(active_servers.values()))) + + end_time = time.time() + 3660 + + + + #break the loop either end_time is exceeding from 1 minute or server is successfully started + while time.time() < end_time and sum(active_servers.values()) <> server_count: + logger.debug("server count 2: {} active servers: {}".format(server_count, sum(active_servers.values()))) + nodeno = 1 + helper_lib.sleepForSecond(1) # waiting for 1 second + server_status = provision_process.status() # fetching status + logger.debug("server status {}".format(server_status)) + if server_status == Status.ACTIVE: + active_servers[nodeno] = 1 + + for node in provision_process.parameters.node_list: + nodeno = nodeno + 1 + logger.debug("+++++++++++++++++++++++++++") + logger.debug(node) + logger.debug(nodeno) + logger.debug("+++++++++++++++++++++++++++") + addnode = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_source_config( + source_config).build(), + make_nonprimary_connection(provision_process.connection, node['environment'], node['environmentUser'])) + server_status = addnode.status() # fetching status + logger.debug("server status {}".format(server_status)) + if server_status == Status.ACTIVE: + active_servers[nodeno] = 1 + + + + return _source_config(virtual_source, repository, source_config, snapshot) def vdb_configure(virtual_source, snapshot, repository): - try: - provision_process = CouchbaseOperation( - Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_snapshot( - snapshot).build()) - - provision_process.restart_couchbase() - provision_process.node_init() - provision_process.cluster_init() - _do_provision(provision_process, snapshot) - _cleanup(provision_process, snapshot) - src_cfg_obj = _source_config(virtual_source, repository, None, snapshot) - - return src_cfg_obj - except FailedToReadBucketDataFromSnapshot as err: - raise FailedToReadBucketDataFromSnapshot("Provision is failed. " + err.message).to_user_error(), None, \ - sys.exc_info()[2] - except Exception as err: - logger.debug("Provision is failed {}".format(err.message)) - raise + # try: + + logger.debug("VDB CONFIG START") + + provision_process = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_snapshot( + snapshot).build()) + + + + + + # TODO: + # fail if already has cluster ? + + # to make sure there is no config + provision_process.delete_config() + + + provision_process.restore_config(what='parent') + + # if bucket doesn't existing in target cluster + # couchbase will delete directory while starting + # so we have to rename it before start + + bucket_list_and_size = json.loads(snapshot.bucket_list) + + if not bucket_list_and_size: + raise FailedToReadBucketDataFromSnapshot("Snapshot Data is empty.") + else: + logger.debug("snapshot bucket data is: {}".format(bucket_list_and_size)) + + + + # for item in helper_lib.filter_bucket_name_from_output(bucket_list_and_size): + # logger.debug("Checking bucket: {}".format(item)) + # bucket_name = item.split(',')[0] + # # rename folder + # provision_process.move_bucket(bucket_name, 'save') + + provision_process.restart_couchbase(provision=True) + provision_process.rename_cluster() + #provision_process.node_init() + #provision_process.cluster_init() + + + #_do_provision(provision_process, snapshot) + #_cleanup(provision_process, snapshot) + + #_build_indexes(provision_process, snapshot) + + # if self.__node_local: + # logger.debug("it will start on main envioronment") + # connection = self.config.connection + # else: + # logger.debug("it will start on an additional environment {}".format(str(self.__node_environment))) + # connection=make_nonprimary_connection(self.config.connection, self.__node_environment, self.__node_envuser) + + + nodeno = 1 + + + logger.debug("MAIN CONNECTION HOST: {}".format(provision_process.connection.environment.host.name)) + + if provision_process.parameters.node_list is not None and len(provision_process.parameters.node_list) > 0: + for node in provision_process.parameters.node_list: + nodeno = nodeno + 1 + logger.debug("+++++++++++++++++++++++++++") + logger.debug(node) + logger.debug(nodeno) + logger.debug("+++++++++++++++++++++++++++") + addnode = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_snapshot( + snapshot).build(), + make_nonprimary_connection(provision_process.connection, node['environment'], node['environmentUser'])) + logger.debug("ADDITIONAL CONNECTION HOST: {}".format(provision_process.connection.environment.host.name)) + addnode.addnode(nodeno, node) + # TODO + # FINISH HERE + # addnode.delete_config() + # addnode.stop_couchbase() + + + src_cfg_obj = _source_config(virtual_source, repository, None, snapshot) + + return src_cfg_obj + # except FailedToReadBucketDataFromSnapshot as err: + # raise FailedToReadBucketDataFromSnapshot("Provision is failed. " + err.message).to_user_error(), None, \ + # sys.exc_info()[2] + # except Exception as err: + # logger.debug("Provision is failed {}".format(err.message)) + # raise + + +def make_nonprimary_connection(primary_connection, secondary_env_ref, secondary_user_ref): + dummy_host = primary_connection.environment.host + user = RemoteUser(name="unused", reference=secondary_user_ref) + environment = RemoteEnvironment(name="unused", reference=secondary_env_ref, host=dummy_host) + return RemoteConnection(environment=environment, user=user) def _do_provision(provision_process, snapshot): @@ -77,13 +268,45 @@ def _do_provision(provision_process, snapshot): else: logger.debug("snapshot bucket data is: {}".format(bucket_list_and_size)) - for item in bucket_list_and_size.split(':'): - logger.debug("Creating bucket is: {}".format(item)) + bucket_list_and_size = json.loads(bucket_list_and_size) + + try: + bucket_list = provision_process.bucket_list() + bucket_list = helper_lib.filter_bucket_name_from_output(bucket_list) + logger.debug(bucket_list) + except Exception as err: + logger.debug("Failed to get bucket list. Error is " + err.message) + + + renamed_folders = [] + + for item in bucket_list_and_size: + logger.debug("Checking bucket: {}".format(item)) # try: + bucket_name = item['name'] + bkt_size = item['ram'] + bkt_type = item['bucketType'] + bkt_compression = item['compressionMode'] + bkt_size_mb = helper_lib.get_bucket_size_in_MB(0, bkt_size) + if bucket_name not in bucket_list: + # a new bucket needs to be created + logger.debug("Creating bucket: {}".format(bucket_name)) + provision_process.bucket_create(bucket_name, bkt_size_mb, bkt_type, bkt_compression) + helper_lib.sleepForSecond(2) + else: + logger.debug("Bucket {} exist - no need to rename directory".format(bucket_name)) + + + provision_process.stop_couchbase() + + for item in helper_lib.filter_bucket_name_from_output(bucket_list_and_size): + logger.debug("Checking bucket: {}".format(item)) bucket_name = item.split(',')[0] - bkt_size_mb = int(item.split(',')[1].strip()) // 1024 // 1024 - provision_process.bucket_create(bucket_name, bkt_size_mb) - helper_lib.sleepForSecond(2) + logger.debug("restoring folders") + provision_process.move_bucket(bucket_name, 'restore') + + + provision_process.start_couchbase() # getting config directory path directory = provision_process.get_config_directory() @@ -93,10 +316,10 @@ def _do_provision(provision_process, snapshot): # This file path is being used to store the bucket information coming in snapshot config_file_path = provision_process.get_config_file_path() - content = "BUCKET_LIST=" + _find_bucket_name_from_snapshot(snapshot) + #content = "BUCKET_LIST=" + _find_bucket_name_from_snapshot(snapshot) # Adding bucket list in config file path .config file, inside .delphix folder - helper_lib.write_file(provision_process.connection, content, config_file_path) + #helper_lib.write_file(provision_process.connection, content, config_file_path) def _cleanup(provision_process, snapshot): @@ -174,6 +397,16 @@ def vdb_start(virtual_source, repository, source_config): logger.debug("Starting couchbase server") try: provision_process.start_couchbase() + if provision_process.parameters.node_list is not None and len(provision_process.parameters.node_list) > 0: + for node in provision_process.parameters.node_list: + logger.debug("+++++++++++++++++++++++++++") + logger.debug(node) + logger.debug("+++++++++++++++++++++++++++") + addnode = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_source_config( + source_config).build(), + make_nonprimary_connection(provision_process.connection, node['environment'], node['environmentUser'])) + addnode.start_couchbase() except Exception: raise CouchbaseServicesError(" Start").to_user_error(), None, sys.exc_info()[2] @@ -185,9 +418,40 @@ def vdb_stop(virtual_source, repository, source_config): logger.debug("Stopping couchbase server") provision_process.stop_couchbase() + if provision_process.parameters.node_list is not None and len(provision_process.parameters.node_list) > 0: + for node in provision_process.parameters.node_list: + logger.debug("+++++++++++++++++++++++++++") + logger.debug(node) + logger.debug("+++++++++++++++++++++++++++") + addnode = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_source_config( + source_config).build(), + make_nonprimary_connection(provision_process.connection, node['environment'], node['environmentUser'])) + addnode.stop_couchbase() def vdb_pre_snapshot(virtual_source, repository, source_config): logger.debug("In Pre snapshot...") + provision_process = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_source_config( + source_config).build()) + + + nodeno = 1 + + provision_process.save_config(what='current', nodeno=nodeno) + + if provision_process.parameters.node_list is not None and len(provision_process.parameters.node_list) > 0: + for node in provision_process.parameters.node_list: + nodeno = nodeno + 1 + logger.debug("+++++++++++++++++++++++++++") + logger.debug(node) + logger.debug(nodeno) + logger.debug("+++++++++++++++++++++++++++") + addnode = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_source_config( + source_config).build(), + make_nonprimary_connection(provision_process.connection, node['environment'], node['environmentUser'])) + addnode.save_config(what='current', nodeno=nodeno) def post_snapshot(virtual_source, repository, source_config): @@ -196,19 +460,32 @@ def post_snapshot(virtual_source, repository, source_config): provision_process = CouchbaseOperation( Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).set_source_config( source_config).build()) - config_file = provision_process.get_config_file_path() + # config_file = provision_process.get_config_file_path() + + # stdout, stderr, exit_code = helper_lib.read_file(virtual_source.connection, config_file) + # bucket_list = re.sub('BUCKET_LIST=', '', stdout) + + ind = [] + + #ind = provision_process.get_indexes_definition() + #logger.debug("indexes definition : {}".format(ind)) + + - stdout, stderr, exit_code = helper_lib.read_file(virtual_source.connection, config_file) - bucket_list = re.sub('BUCKET_LIST=', '', stdout) - logger.debug("BUCKET_LIST={}".format(bucket_list)) + + bucket_details = json.dumps(provision_process.bucket_list()) + logger.debug("BUCKET_LIST={}".format(bucket_details)) db_path = virtual_source.parameters.mount_path time_stamp = helper_lib.current_time() couchbase_port = virtual_source.parameters.couchbase_port couchbase_host = virtual_source.connection.environment.host.name snapshot_id = str(helper_lib.get_snapshot_id()) snapshot = SnapshotDefinition(db_path=db_path, couchbase_port=couchbase_port, couchbase_host=couchbase_host, - bucket_list=bucket_list, time_stamp=time_stamp, snapshot_id=snapshot_id) - logger.info("snapshot schema: {}".format(snapshot)) + bucket_list=bucket_details, time_stamp=time_stamp, snapshot_id=snapshot_id, indexes = ind) + + snapshot.couchbase_admin = provision_process.parameters.couchbase_admin + snapshot.couchbase_admin_password = provision_process.parameters.couchbase_admin_password + return snapshot except Exception as err: logger.debug("Snap shot is failed with error {}".format(err.message)) @@ -217,15 +494,16 @@ def post_snapshot(virtual_source, repository, source_config): # This function returns the bucket name from snapshot. def _find_bucket_name_from_snapshot(snapshot): - bucket_list_and_size = snapshot.bucket_list + bucket_list_and_size = json.loads(snapshot.bucket_list) logger.debug("SnapShot bucket data is: {}".format(bucket_list_and_size)) - # bucket_list_and_size contains the ramsize e.g. "Bucket1,122:Bucket2,3432" - # Filtering the size from above information. - bucket_list_and_size += ':' - # Parsing logic because there could be bucket name having some digit - # bucket details in snapshot : Bucket_name1,RamSize1:Bucket_name2,RamSize2: - bucket_name = re.sub(',[0-9]*:', ':', bucket_list_and_size) - bucket_name = bucket_name.strip(':') + # # bucket_list_and_size contains the ramsize e.g. "Bucket1,122:Bucket2,3432" + # # Filtering the size from above information. + # bucket_list_and_size += ':' + # # Parsing logic because there could be bucket name having some digit + # # bucket details in snapshot : Bucket_name1,RamSize1:Bucket_name2,RamSize2: + # bucket_name = re.sub(',[0-9]*:', ':', bucket_list_and_size) + # bucket_name = bucket_name.strip(':') + bucket_name = helper_lib.filter_bucket_name_from_output(bucket_list_and_size) return bucket_name @@ -240,3 +518,11 @@ def _find_bucket_size_byname(bucket_name, bucket_metadata): if data_found == 0: # raise exception. Ideally this condition should never occur raise Exception("Failed to find the bucket_name from bucket_metadata list") + + +def _build_indexes(provision_process, snapshot): + logger.debug("index builder") + + for i in snapshot.indexes: + logger.debug(i) + provision_process.build_index(i) \ No newline at end of file diff --git a/src/plugin_runner.py b/src/plugin_runner.py index 53abc4f..3f2f9a5 100644 --- a/src/plugin_runner.py +++ b/src/plugin_runner.py @@ -4,14 +4,28 @@ # from dlpx.virtualization.platform import Mount, MountSpecification, Plugin, Status +from dlpx.virtualization.platform import OwnershipSpecification from operations import discovery, linked, virtual from utils import setup_logger from db_commands.constants import EVICTION_POLICY +import logging +from dlpx.virtualization.common import RemoteEnvironment +from dlpx.virtualization.common import RemoteHost +from dlpx.virtualization.common import RemoteUser +from dlpx.virtualization.common import RemoteConnection +from controller.helper_lib import check_stale_mountpoint +from controller.helper_lib import clean_stale_mountpoint +from controller.couchbase_operation import CouchbaseOperation +from controller.resource_builder import Resource +from controller.helper_lib import check_server_is_used + plugin = Plugin() setup_logger._setup_logger() +logger = logging.getLogger(__name__) + # # Below is an example of the repository discovery operation. # @@ -46,17 +60,30 @@ def linked_post_snapshot(staged_source, repository, source_config, optional_snap @plugin.linked.mount_specification() def linked_mount_specification(staged_source, repository): mount_path = staged_source.parameters.mount_path + + if check_stale_mountpoint(staged_source.staged_connection, mount_path): + cleanup_process = CouchbaseOperation( + Resource.ObjectBuilder.set_staged_source(staged_source).set_repository(repository).build()) + cleanup_process.stop_couchbase() + clean_stale_mountpoint(staged_source.staged_connection, mount_path) + + check_server_is_used(staged_source.staged_connection, mount_path) + environment = staged_source.staged_connection.environment linked.check_mount_path(staged_source, repository) + logger.debug("Mounting path {}".format(mount_path)) mounts = [Mount(environment, mount_path)] - return MountSpecification(mounts) + logger.debug("Setting ownership to uid {} and gid {}".format(repository.uid, repository.gid)) + ownership_spec = OwnershipSpecification(repository.uid, repository.gid) + return MountSpecification(mounts, ownership_spec) @plugin.linked.pre_snapshot() def linked_pre_snapshot(staged_source, repository, source_config, optional_snapshot_parameters): if int(optional_snapshot_parameters.resync) == 1: linked.resync(staged_source, repository, source_config, staged_source.parameters) - linked.pre_snapshot(staged_source, repository, source_config, staged_source.parameters) + else: + linked.pre_snapshot(staged_source, repository, source_config, staged_source.parameters) @plugin.linked.status() @@ -73,6 +100,7 @@ def start_staging(staged_source, repository, source_config): linked.start_staging(staged_source, repository, source_config) + @plugin.virtual.configure() def configure(virtual_source, snapshot, repository): return virtual.vdb_configure(virtual_source, snapshot, repository) @@ -106,15 +134,83 @@ def stop(virtual_source, repository, source_config): @plugin.virtual.mount_specification() def virtual_mount_specification(virtual_source, repository): mount_path = virtual_source.parameters.mount_path + + if check_stale_mountpoint(virtual_source.connection, mount_path): + cleanup_process = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).build()) + cleanup_process.stop_couchbase() + clean_stale_mountpoint(virtual_source.connection, mount_path) + + check_server_is_used(virtual_source.connection, mount_path) + mounts = [Mount(virtual_source.connection.environment, mount_path)] - return MountSpecification(mounts) + logger.debug("Mounting path {}".format(mount_path)) + logger.debug("Setting ownership to uid {} and gid {}".format(repository.uid, repository.gid)) + ownership_spec = OwnershipSpecification(repository.uid, repository.gid) + + logger.debug("in mounting: {}".format(str(virtual_source.parameters.node_list))) + + + + if virtual_source.parameters.node_list is not None and len(virtual_source.parameters.node_list) > 0: + # more nodes + for m in virtual_source.parameters.node_list: + logger.debug("in loop: {}".format(str(m))) + node_host = RemoteHost(name='foo', + reference=m["environment"].replace('_ENVIRONMENT', ''), + binary_path="", + scratch_path="" + ) + e = RemoteEnvironment("foo", m["environment"], node_host ) + mount = Mount(e, mount_path) + mounts.append(mount) + + + user = RemoteUser(name="unused", reference=m['environmentUser']) + environment = RemoteEnvironment(name="unused", reference=m['environment'], host=node_host) + clean_node_conn = RemoteConnection(environment=environment, user=user) + + + + if check_stale_mountpoint(clean_node_conn, mount_path): + clean_node = CouchbaseOperation( + Resource.ObjectBuilder.set_virtual_source(virtual_source).set_repository(repository).build(), + clean_node_conn ) + clean_node.stop_couchbase() + clean_stale_mountpoint(clean_node_conn, mount_path) + + check_server_is_used(clean_node_conn, mount_path) + + + return MountSpecification(mounts, ownership_spec) @plugin.virtual.status() def virtual_status(virtual_source, repository, source_config): + logger.debug("in status") return virtual.vdb_status(virtual_source, repository, source_config) @plugin.virtual.unconfigure() def unconfigure(virtual_source, repository, source_config): - virtual.vdb_stop(virtual_source, repository, source_config) + logger.debug("UNCONFIGURE") + virtual.vdb_unconfigure(virtual_source, repository, source_config) + + +@plugin.upgrade.virtual_source("2021.07.19") +def add_node_to_virtual(old_virtual_source): + new_virt = dict(old_virtual_source) + new_virt["node_list"] = [] + return new_virt + + +@plugin.upgrade.virtual_source("2021.10.06") +def add_node_to_virtual(old_virtual_source): + logger.debug("Doing upgrade to node_addr") + new_virt = dict(old_virtual_source) + logger.debug(new_virt) + for i in new_virt["node_list"]: + i["node_addr"] = "" + logger.debug("After changes") + logger.debug(new_virt) + return new_virt diff --git a/src/utils/utilities.py b/src/utils/utilities.py index 621b685..1b1c254 100644 --- a/src/utils/utilities.py +++ b/src/utils/utilities.py @@ -34,7 +34,7 @@ def execute_bash(source_connection, command_name, callback_func=None, environmen # Verify the exit code of each executed command. 0 means command ran successfully and for other code it is failed. # For failed cases we need to find the scenario in which programs will die and otherwise execution will continue. - _handle_exit_code(exit_code, error, output, callback_func) + #_handle_exit_code(exit_code, error, output, callback_func) return [output, error, exit_code]