Merge branch 'feature/idp-cli-deploy-from-code' into 'develop'

feat: IDP CLI --from-code Flag for Local Development Deployment, and --no-rollback Flag for Stack Deployment Troubleshooting

See merge request genaiic-reusable-assets/engagement-artifacts/genaiic-idp-accelerator!434

Author: rstrahan

CHANGELOG.md

@@ -5,8 +5,17 @@ SPDX-License-Identifier: MIT-0
 
 ## [Unreleased]
 
+## [0.4.4]
+
 ### Added
 
+- **IDP CLI --from-code Flag for Local Development Deployment**
+  - Added `--from-code` flag to `idp-cli deploy` command enabling deployment directly from local source code
+  - Automatically builds project using `publish.py` script with streaming output for real-time build progress
+- **IDP CLI --no-rollback Flag for Stack Deployment Troubleshooting**
+  - Added `--no-rollback` flag to `idp-cli deploy` command to disable automatic rollback on CloudFormation stack creation failure
+  - When enabled, failed stacks remain in `CREATE_FAILED` state instead of rolling back, allowing inspection of failed resources for troubleshooting
+
 ### Fixed
 
 ## [0.4.3]

idp_cli/idp_cli/cli.py

@@ -8,10 +8,13 @@
 """
 
 import logging
+import os
+import subprocess
 import sys
 import time
 from typing import Optional
 
+import boto3
 import click
 from rich.console import Console
 from rich.live import Live
@@ -31,6 +34,105 @@
 
 console = Console()
 
+
+def _build_from_local_code(from_code_dir: str, region: str, stack_name: str) -> tuple:
+    """
+    Build project from local code using publish.py
+
+    Args:
+        from_code_dir: Path to project root directory
+        region: AWS region
+        stack_name: CloudFormation stack name (unused but kept for signature compatibility)
+
+    Returns:
+        Tuple of (template_path, None) on success
+
+    Raises:
+        SystemExit: On build failure
+    """
+    # Verify publish.py exists
+    publish_script = os.path.join(from_code_dir, "publish.py")
+    if not os.path.isfile(publish_script):
+        console.print(f"[red]✗ Error: publish.py not found in {from_code_dir}[/red]")
+        console.print(
+            "[yellow]Tip: --from-code should point to the project root directory[/yellow]"
+        )
+        sys.exit(1)
+
+    # Get AWS account ID
+    try:
+        sts = boto3.client("sts", region_name=region)
+        account_id = sts.get_caller_identity()["Account"]
+    except Exception as e:
+        console.print(f"[red]✗ Error: Failed to get AWS account ID: {e}[/red]")
+        sys.exit(1)
+
+    # Set parameters for publish.py
+    cfn_bucket_basename = f"idp-accelerator-artifacts-{account_id}"
+    cfn_prefix = "idp-cli"
+
+    console.print("[bold cyan]Building project from source...[/bold cyan]")
+    console.print(f"[dim]Bucket: {cfn_bucket_basename}[/dim]")
+    console.print(f"[dim]Prefix: {cfn_prefix}[/dim]")
+    console.print(f"[dim]Region: {region}[/dim]")
+    console.print()
+
+    # Build command
+    cmd = [
+        sys.executable,  # Use same Python interpreter
+        publish_script,
+        cfn_bucket_basename,
+        cfn_prefix,
+        region,
+    ]
+
+    console.print(f"[dim]Running: {' '.join(cmd)}[/dim]")
+    console.print()
+
+    # Run with streaming output
+    try:
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1,
+            cwd=from_code_dir,
+        )
+
+        # Stream output line by line
+        for line in process.stdout:
+            # Print each line immediately (preserve formatting from publish.py)
+            print(line, end="")
+
+        process.wait()
+
+        if process.returncode != 0:
+            console.print("[red]✗ Build failed. See output above for details.[/red]")
+            sys.exit(1)
+
+    except Exception as e:
+        console.print(f"[red]✗ Error running publish.py: {e}[/red]")
+        sys.exit(1)
+
+    # Verify template was created
+    template_path = os.path.join(from_code_dir, ".aws-sam", "idp-main.yaml")
+    if not os.path.isfile(template_path):
+        console.print(
+            f"[red]✗ Error: Built template not found at {template_path}[/red]"
+        )
+        console.print(
+            "[yellow]The build may have failed or the template was not generated.[/yellow]"
+        )
+        sys.exit(1)
+
+    console.print()
+    console.print(f"[green]✓ Build complete. Using template: {template_path}[/green]")
+    console.print()
+
+    return template_path, None
+
+
 # Region-specific template URLs
 TEMPLATE_URLS = {
     "us-west-2": "https://s3.us-west-2.amazonaws.com/aws-ml-blog-us-west-2/artifacts/genai-idp/idp-main.yaml",
@@ -40,7 +142,7 @@
 
 
 @click.group()
-@click.version_option(version="0.4.2")
+@click.version_option(version="0.4.4")
 def cli():
     """
     IDP CLI - Batch document processing for IDP Accelerator
@@ -64,6 +166,11 @@ def cli():
 @click.option(
     "--admin-email", help="Admin user email address (required for new stacks)"
 )
+@click.option(
+    "--from-code",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
+    help="Deploy from local code by building with publish.py (path to project root)",
+)
 @click.option(
     "--template-url",
     help="URL to CloudFormation template in S3 (default: auto-selected based on region)",
@@ -93,11 +200,15 @@ def cli():
 )
 @click.option("--parameters", help="Additional parameters as key=value,key2=value2")
 @click.option("--wait", is_flag=True, help="Wait for stack creation to complete")
+@click.option(
+    "--no-rollback", is_flag=True, help="Disable rollback on stack creation failure"
+)
 @click.option("--region", help="AWS region (optional)")
 def deploy(
     stack_name: str,
     pattern: str,
     admin_email: str,
+    from_code: Optional[str],
     template_url: str,
     max_concurrent: int,
     log_level: str,
@@ -106,6 +217,7 @@ def deploy(
     custom_config: Optional[str],
     parameters: Optional[str],
     wait: bool,
+    no_rollback: bool,
     region: Optional[str],
 ):
     """
@@ -119,9 +231,15 @@ def deploy(
       # Create new stack with Pattern 2
       idp-cli deploy --stack-name my-idp --pattern pattern-2 --admin-email user@example.com
       
-      # Update existing stack with local config file (NEW!)
+      # Deploy from local code (NEW!)
+      idp-cli deploy --stack-name my-idp --from-code . --pattern pattern-2 --admin-email user@example.com --wait
+      
+      # Update existing stack with local config file
       idp-cli deploy --stack-name my-idp --custom-config ./my-config.yaml
       
+      # Update existing stack from local code
+      idp-cli deploy --stack-name my-idp --from-code . --wait
+      
       # Update existing stack with custom settings
       idp-cli deploy --stack-name my-idp --max-concurrent 200 --wait
       
@@ -131,6 +249,13 @@ def deploy(
           --parameters "DataRetentionInDays=90,ErrorThreshold=5"
     """
     try:
+        # Validate mutually exclusive options
+        if from_code and template_url:
+            console.print(
+                "[red]✗ Error: Cannot specify both --from-code and --template-url[/red]"
+            )
+            sys.exit(1)
+
         # Auto-detect region if not provided
         if not region:
             import boto3
@@ -142,8 +267,15 @@ def deploy(
                     "Region could not be determined. Please specify --region or configure AWS_DEFAULT_REGION"
                 )
 
+        # Handle deployment from local code
+        template_path = None
+        if from_code:
+            template_path, template_url = _build_from_local_code(
+                from_code, region, stack_name
+            )
+
         # Determine template URL (user-provided takes precedence)
-        if not template_url:
+        elif not template_url:
             if region in TEMPLATE_URLS:
                 template_url = TEMPLATE_URLS[region]
                 console.print(f"[bold]Using template for region: {region}[/bold]")
@@ -224,12 +356,24 @@ def deploy(
 
         # Deploy stack
         with console.status("[bold green]Deploying stack..."):
-            result = deployer.deploy_stack(
-                stack_name=stack_name,
-                template_url=template_url,
-                parameters=cfn_parameters,
-                wait=wait,
-            )
+            if template_path:
+                # Deploy from local template (built from code)
+                result = deployer.deploy_stack(
+                    stack_name=stack_name,
+                    template_path=template_path,
+                    parameters=cfn_parameters,
+                    wait=wait,
+                    no_rollback=no_rollback,
+                )
+            else:
+                # Deploy from template URL
+                result = deployer.deploy_stack(
+                    stack_name=stack_name,
+                    template_url=template_url,
+                    parameters=cfn_parameters,
+                    wait=wait,
+                    no_rollback=no_rollback,
+                )
 
         # Show results
         # Success if operation completed successfully OR was successfully initiated

idp_cli/idp_cli/deployer.py

@@ -41,6 +41,7 @@ def deploy_stack(
         template_url: Optional[str] = None,
         parameters: Dict[str, str] = None,
         wait: bool = False,
+        no_rollback: bool = False,
     ) -> Dict:
         """
         Deploy CloudFormation stack
@@ -51,6 +52,7 @@ def deploy_stack(
             template_url: URL to CloudFormation template in S3 (optional)
             parameters: Stack parameters
             wait: Whether to wait for stack creation to complete
+            no_rollback: If True, disable rollback on failure (DO_NOTHING)
 
         Returns:
             Dictionary with deployment result
@@ -67,7 +69,18 @@ def deploy_stack(
         else:
             # Read template from local file
             template_body = self._read_template(template_path)
-            template_param = {"TemplateBody": template_body}
+
+            # Check template size - CloudFormation has a 51,200 byte limit for direct upload
+            template_size = len(template_body.encode("utf-8"))
+            if template_size > 51200:  # 50KB limit
+                logger.info(
+                    f"Template size ({template_size} bytes) exceeds limit, uploading to S3"
+                )
+                template_url = self._upload_template_to_s3(template_body, stack_name)
+                template_param = {"TemplateURL": template_url}
+                logger.info(f"Template uploaded to: {template_url}")
+            else:
+                template_param = {"TemplateBody": template_body}
 
         # Convert parameters dict to CloudFormation format
         cfn_parameters = [
@@ -94,6 +107,8 @@ def deploy_stack(
                 operation = "UPDATE"
             else:
                 logger.info(f"Creating new stack: {stack_name}")
+                # Set OnFailure based on no_rollback flag
+                on_failure = "DO_NOTHING" if no_rollback else "ROLLBACK"
                 response = self.cfn.create_stack(
                     StackName=stack_name,
                     **template_param,
@@ -103,7 +118,7 @@ def deploy_stack(
                         "CAPABILITY_NAMED_IAM",
                         "CAPABILITY_AUTO_EXPAND",
                     ],
-                    OnFailure="ROLLBACK",
+                    OnFailure=on_failure,
                 )
                 operation = "CREATE"
 
@@ -136,6 +151,31 @@ def _read_template(self, template_path: str) -> str:
 
         return template_file.read_text()
 
+    def _upload_template_to_s3(self, template_body: str, stack_name: str) -> str:
+        """Upload template to S3 and return URL"""
+        from datetime import datetime
+
+        import boto3
+
+        # Get or create bucket for templates
+        bucket_name = get_or_create_config_bucket(self.region)
+
+        # Generate unique key
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        s3_key = f"idp-cli/templates/{stack_name}_{timestamp}.yaml"
+
+        # Upload template
+        s3 = boto3.client("s3", region_name=self.region)
+        s3.put_object(
+            Bucket=bucket_name,
+            Key=s3_key,
+            Body=template_body.encode("utf-8"),
+            ServerSideEncryption="AES256",
+        )
+
+        # Return S3 URL
+        return f"https://s3.{self.region}.amazonaws.com/{bucket_name}/{s3_key}"
+
     def _stack_exists(self, stack_name: str) -> bool:
         """Check if stack exists"""
         try:

idp_cli/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "idp-cli"
-version = "0.4.2"
+version = "0.4.4"
 description = "Command-line interface for IDP Accelerator batch document processing"
 authors = [{name = "AWS"}]
 license = {text = "MIT-0"}