AwsDirectDeployer.java

package com.cloudforgeci.api.deploy.aws;

import com.cloudforge.core.config.DeploymentConfig;
import com.cloudforge.core.enums.RuntimeType;
import com.cloudforge.core.local.DeploymentTarget;
import com.cloudforge.core.manager.ManagerEndpointSupport;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.core.waiters.WaiterOverrideConfiguration;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.cloudformation.CloudFormationClient;
import software.amazon.awssdk.services.cloudformation.model.Capability;
import software.amazon.awssdk.services.cloudformation.model.ChangeSetStatus;
import software.amazon.awssdk.services.cloudformation.model.ChangeSetType;
import software.amazon.awssdk.services.cloudformation.model.CloudFormationException;
import software.amazon.awssdk.services.cloudformation.model.CreateChangeSetRequest;
import software.amazon.awssdk.services.cloudformation.model.DeleteChangeSetRequest;
import software.amazon.awssdk.services.cloudformation.model.DeleteStackRequest;
import software.amazon.awssdk.services.cloudformation.model.DescribeChangeSetRequest;
import software.amazon.awssdk.services.cloudformation.model.DescribeStackEventsRequest;
import software.amazon.awssdk.services.cloudformation.model.DescribeStacksRequest;
import software.amazon.awssdk.services.cloudformation.model.ExecuteChangeSetRequest;
import software.amazon.awssdk.services.cloudformation.model.GetTemplateRequest;
import software.amazon.awssdk.services.cloudformation.model.Output;
import software.amazon.awssdk.services.cloudformation.model.ResourceChange;
import software.amazon.awssdk.services.cloudformation.model.DescribeChangeSetResponse;
import software.amazon.awssdk.services.cloudformation.model.StackEvent;
import software.amazon.awssdk.services.cloudformation.model.Tag;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException;
import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.sts.StsClient;

import com.cloudforgeci.localstack.LocalStackCdkAssetPublisher;

import java.io.IOException;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.logging.Logger;

import com.cloudforgeci.api.deploy.aws.StackProgressListener.StackProgressEvent;

/**
 * Creates and incrementally updates CloudFormation stacks on AWS via change sets.
 *
 * <p>Backs {@code CloudForgeDeployment}'s {@code AWS} case: the direct-deploy counterpart to
 * {@code LocalStackDeployer}/{@code MiniStackDeployer}, without the emulator-specific steps
 * (Cognito/RDS secret reconciliation, ECS restarts after secret sync), which AWS handles through
 * CDK's Secrets Manager integration.</p>
 *
 * <p><b>Targets AWS by default, but redirects to a local emulator when Manager itself runs
 * inside one</b>, using the same {@code LOCALSTACK_ENDPOINT}/{@code AWS_ENDPOINT_URL} detection
 * as {@link com.cloudforgeci.localstack.LocalStackDeployer#resolveEndpoint()}. Without the
 * endpoint override, a Manager instance hosted on LocalStack would call the public
 * CloudFormation endpoint instead of the emulator.</p>
 *
 * <p>Every stack created or updated is tagged with the {@code cloudforge:managed}/
 * {@code cloudforge:application}/{@code cloudforge:runtime} convention that
 * {@code ApplicationFargateStack}/{@code ApplicationEc2Stack} apply via
 * {@code Tags.of(this).add(...)}. The tags are required by Manager's inventory
 * ({@code StackListingPolicy}) and by the {@code aws:RequestTag}/{@code aws:ResourceTag}
 * conditions in {@code ManagerOperatorIamSupport.deployStatements}. Those conditions evaluate the
 * stack-level {@code Tags} parameter on the API call, not resource-level tags in the template;
 * this class controls the former.</p>
 *
 * <p><b>Test coverage:</b> {@code AwsDirectDeployerTest} runs this class against LocalStack via
 * the injectable-client constructor, which exercises every code path except AWS's own network
 * and authentication. The change-set create/execute/wait sequence mirrors
 * {@code LocalStackDeployer}'s.</p>
 */
public final class AwsDirectDeployer implements AutoCloseable {

    private static final ObjectMapper MAPPER = new ObjectMapper();
    private static final Logger LOG = Logger.getLogger(AwsDirectDeployer.class.getName());
    private static final Duration OPERATION_TIMEOUT = Duration.ofMinutes(30);
    private static final Duration POLL_INTERVAL = Duration.ofSeconds(1);
    /**
     * Caps how long the stack create/update waiter ({@code
     * cloudFormation.waiter().waitUntilStackCreateComplete(...)}) waits for a LocalStack target.
     * AWS targets keep the SDK's default waiter configuration (up to about 60 minutes, since large
     * stacks can take that long). A LocalStack resource that hangs is an emulation gap, so a
     * deploy that has not finished in 5 minutes is treated as hung rather than left "RUNNING".
     */
    private static final Duration LOCAL_EMULATOR_WAITER_TIMEOUT = Duration.ofMinutes(5);
    /** CloudFormation inline template body limit (same on real AWS as LocalStack). */
    private static final int MAX_INLINE_TEMPLATE_BYTES = 51_200;
    /** Shared with {@code ManagerOperatorIamSupport}'s S3 grant for this bucket — keep the two in
     *  sync rather than duplicating the literal. */
    public static final String TEMPLATE_BUCKET_PREFIX = "cfc-cfn-templates-";

    public static final String TAG_MANAGED = "cloudforge:managed";
    public static final String TAG_APPLICATION = "cloudforge:application";
    public static final String TAG_RUNTIME = "cloudforge:runtime";

    private final CloudFormationClient cloudFormation;
    private final S3Client s3;
    private final String applicationId;
    private final String runtimeTag;
    private final String templateBucket;
    private final boolean localEmulatorTarget;
    private final Region region;
    private final AwsCredentialsProvider credentialsOverride;

    /**
     * Real AWS by default (default credential/region chain, region from {@code config.region});
     * redirects to a local emulator instead when {@code target} is {@link DeploymentTarget#LOCALSTACK}/
     * {@link DeploymentTarget#MINISTACK} and {@link ManagerEndpointSupport#resolveLocalEmulatorEndpoint}
     * finds one — see class javadoc. {@code target} must be the caller's own already-known,
     * validated target (never re-derived from env vars here — see that method's own javadoc for
     * why). Equivalent to {@code AwsDirectDeployer(config, target, null)}.
     */
    public AwsDirectDeployer(DeploymentConfig config, DeploymentTarget target) {
        this(config, target, null);
    }

    /**
     * Same as {@link #AwsDirectDeployer(DeploymentConfig, DeploymentTarget)}, but with an
     * explicit credentials provider for AWS calls instead of the default chain, e.g. for a caller
     * that assumed a cross-account IAM role. {@code null} means the default credential chain.
     *
     * <p>Ignored when {@code target} resolves to a local emulator, which always uses the fixed
     * {@code test}/{@code test} static credentials.</p>
     */
    public AwsDirectDeployer(
            DeploymentConfig config, DeploymentTarget target, AwsCredentialsProvider credentialsOverride) {
        this(
            cloudFormationClient(config, target, credentialsOverride),
            s3Client(config, target, credentialsOverride),
            config.applicationId,
            runtimeTag(config.runtime),
            // null: resolvedTemplateBucket() computes the account-scoped name lazily so the
            // constructor makes no network call (see resolveAccountId()).
            null,
            ManagerEndpointSupport.resolveLocalEmulatorEndpoint(target) != null,
            Region.of(config.region == null ? "us-east-1" : config.region),
            credentialsOverride);
    }

    private static CloudFormationClient cloudFormationClient(
            DeploymentConfig config, DeploymentTarget target, AwsCredentialsProvider credentialsOverride) {
        Region region = Region.of(config.region == null ? "us-east-1" : config.region);
        String localEndpoint = ManagerEndpointSupport.resolveLocalEmulatorEndpoint(target);
        if (localEndpoint == null) {
            return CloudFormationClient.builder()
                .region(region)
                .credentialsProvider(credentialsOverride != null
                    ? credentialsOverride : DefaultCredentialsProvider.create())
                .build();
        }
        // Local emulator always wins, regardless of any credentialsOverride — see
        // AwsDirectDeployer(DeploymentConfig, DeploymentTarget, AwsCredentialsProvider)'s javadoc.
        return CloudFormationClient.builder()
            .region(region)
            .endpointOverride(URI.create(localEndpoint))
            .credentialsProvider(StaticCredentialsProvider.create(
                AwsBasicCredentials.create("test", "test")))
            .build();
    }

    private static S3Client s3Client(
            DeploymentConfig config, DeploymentTarget target, AwsCredentialsProvider credentialsOverride) {
        Region region = Region.of(config.region == null ? "us-east-1" : config.region);
        String localEndpoint = ManagerEndpointSupport.resolveLocalEmulatorEndpoint(target);
        if (localEndpoint == null) {
            return S3Client.builder()
                .region(region)
                .credentialsProvider(credentialsOverride != null
                    ? credentialsOverride : DefaultCredentialsProvider.create())
                .build();
        }
        return S3Client.builder()
            .region(region)
            .endpointOverride(URI.create(localEndpoint))
            .forcePathStyle(true)
            .credentialsProvider(StaticCredentialsProvider.create(
                AwsBasicCredentials.create("test", "test")))
            .build();
    }

    /**
     * Returns the target account ID: LocalStack's fixed test account without a network call, or
     * the caller's account via {@code sts:GetCallerIdentity} on AWS. Used to substitute CDK's
     * {@code ${AWS::AccountId}} token in asset-manifest bucket names (see
     * {@code LocalStackCdkAssetPublisher}). Called lazily from {@link #deploy} so that
     * construction never reaches the network (relied on by
     * {@code realAwsConstructorDefaultsRegionWhenConfigOmitsIt}).
     */
    private String resolveAccountId() {
        if (localEmulatorTarget) {
            return "000000000000";
        }
        try (StsClient sts = StsClient.builder()
                .region(region)
                .credentialsProvider(credentialsOverride != null
                    ? credentialsOverride : DefaultCredentialsProvider.create())
                .build()) {
            return sts.getCallerIdentity().account();
        }
    }

    /**
     * Visible for tests — inject pre-built clients (e.g. pointed at LocalStack, or mocks).
     * Defaults {@code localEmulatorTarget} to {@code false} (no CDK-bootstrap-parameter
     * adaptation) since no existing test deploys a template containing one; use the 7-arg
     * overload to test that behavior, or {@link #resolveAccountId}, specifically.
     */
    AwsDirectDeployer(
            CloudFormationClient cloudFormation,
            S3Client s3,
            String applicationId,
            String runtimeTag,
            String templateBucket) {
        this(cloudFormation, s3, applicationId, runtimeTag, templateBucket, false, Region.US_EAST_1);
    }

    /** Visible for tests — same as the 5-arg overload, with explicit control over adaptation. */
    AwsDirectDeployer(
            CloudFormationClient cloudFormation,
            S3Client s3,
            String applicationId,
            String runtimeTag,
            String templateBucket,
            boolean localEmulatorTarget,
            Region region) {
        this(cloudFormation, s3, applicationId, runtimeTag, templateBucket, localEmulatorTarget, region, null);
    }

    /** Visible for tests — same as the 7-arg overload, with an explicit credentials override
     *  (e.g. asserting a cross-account provider actually reaches {@link #resolveAccountId}). */
    AwsDirectDeployer(
            CloudFormationClient cloudFormation,
            S3Client s3,
            String applicationId,
            String runtimeTag,
            String templateBucket,
            boolean localEmulatorTarget,
            Region region,
            AwsCredentialsProvider credentialsOverride) {
        this.cloudFormation = cloudFormation;
        this.s3 = s3;
        this.applicationId = applicationId;
        this.runtimeTag = runtimeTag;
        this.templateBucket = templateBucket;
        this.localEmulatorTarget = localEmulatorTarget;
        this.region = region;
        this.credentialsOverride = credentialsOverride;
    }

    static String runtimeTag(RuntimeType runtime) {
        return runtime == null ? "unknown" : runtime.name().toLowerCase(Locale.ROOT);
    }

    /**
     * Template bucket name, including the account ID because S3 bucket names are global: a name
     * owned by another account returns 403 Access Denied on every call rather than an
     * "already exists" error. Mirrors CDK's bootstrap bucket convention ({@code
     * cdk-hnb659fds-assets-<account>-<region>}).
     */
    static String templateBucketName(String accountId, String region) {
        return TEMPLATE_BUCKET_PREFIX + accountId + "-"
            + (region == null || region.isBlank() ? "us-east-1" : region);
    }

    /**
     * The CloudFormation stack name used for AWS/S3 API calls: {@code stackName} plus a
     * {@code -localstack} suffix when targeting a local emulator, matching the
     * {@code LocalStackDeployer} convention. Manager's {@code StackListingPolicy.acceptsName}
     * requires the suffix to list the stack under its LocalStack target. Public methods accept
     * and return only the logical, unsuffixed name (including
     * {@link AwsStackDeployResult#stackName()}).
     */
    String physicalStackName(String stackName) {
        return localEmulatorTarget ? stackName + "-localstack" : stackName;
    }

    /** The three CloudForge stack tags applied to every {@code CreateStack}/{@code UpdateStack} call. */
    List<Tag> managedTags() {
        return List.of(
            Tag.builder().key(TAG_MANAGED).value("true").build(),
            Tag.builder().key(TAG_APPLICATION).value(applicationId == null ? "unknown" : applicationId).build(),
            Tag.builder().key(TAG_RUNTIME).value(runtimeTag).build());
    }

    /**
     * Creates or updates {@code stackName} from {@code template} via a change set. No-op
     * (returns {@code noOp = true}) when the stack already matches the candidate template.
     */
    public AwsStackDeployResult deploy(String stackName, Path template) throws IOException {
        return deploy(stackName, template, null);
    }

    /**
     * Same as {@link #deploy(String, Path)}, additionally reporting each CloudFormation stack
     * event to {@code listener} as it happens (may be {@code null} to skip live reporting).
     */
    public AwsStackDeployResult deploy(String stackName, Path template, StackProgressListener listener)
            throws IOException {
        String physical = physicalStackName(stackName);
        // The logical stackName, not the physical one: the asset manifest is named after the
        // construct id CloudForgeSynthesizer used.
        //
        // Assets are always published before the CloudFormation call, as `cdk deploy` does;
        // otherwise asset-backed resources (including CDK's LogRetention custom resource) fail.
        //
        // createBucketIfMissing = localEmulatorTarget: emulators have no separate bootstrap
        // step, so creating the asset bucket on first use serves as bootstrap. On AWS the bucket
        // belongs to `cdk bootstrap`; creating a bare bucket with that name would block bootstrap
        // from creating a correctly configured one, so an unbootstrapped account instead gets an
        // IOException telling the user to run `cdk bootstrap`.
        LocalStackCdkAssetPublisher.publish(
            template.getParent(), stackName, s3, resolveAccountId(), localEmulatorTarget);

        boolean exists = stackExists(physical) && stackIsDeployable(physical);
        // Every target, not just local emulators: a self-contained template (this class's whole
        // purpose -- see AssetFreeCustomResource) must never depend on the deploying account
        // being CDK-bootstrapped. An AWS Marketplace buyer launching this template directly never
        // runs `cdk bootstrap` first, so an unrewritten BootstrapVersion SSM-parameter reference
        // fails CloudFormation's own parameter resolution before a single resource is created --
        // the exact same failure mode this rewrite already fixed for LocalStack.
        String templateBody = resolveCdkBootstrapParameters(Files.readString(template));

        if (exists && templateMatches(physical, templateBody)) {
            return new AwsStackDeployResult(stackName, false, true, List.of(), outputs(physical));
        }

        String changeSetName = "cfc-" + UUID.randomUUID().toString().substring(0, 8);
        var changeSet = createAndWaitForChangeSet(physical, templateBody, exists, changeSetName);
        if (changeSet.status() == ChangeSetStatus.FAILED) {
            String reason = changeSet.statusReason() == null ? "" : changeSet.statusReason();
            deleteChangeSet(physical, changeSetName);
            if (isNoOp(reason)) {
                return new AwsStackDeployResult(stackName, false, true, List.of(), outputs(physical));
            }
            throw new IOException("AWS change set failed for " + physical + ": " + reason);
        }

        List<String> changeSummaries = changeSet.changes().stream()
            .map(change -> summarize(change.resourceChange()))
            .toList();

        // Captured before the operation starts, not before this deploy() call began, so a stack
        // event from an earlier deploy/update on this same stack (returned on the very first
        // describeStackEvents page, since CloudFormation keeps full history) is never mistaken
        // for this operation's own progress.
        Instant operationStart = Instant.now();
        cloudFormation.executeChangeSet(ExecuteChangeSetRequest.builder()
            .stackName(physical)
            .changeSetName(changeSetName)
            .build());

        var request = DescribeStacksRequest.builder().stackName(physical).build();
        waitWithProgress(physical, listener, operationStart, () -> {
            if (localEmulatorTarget) {
                WaiterOverrideConfiguration override = WaiterOverrideConfiguration.builder()
                    .waitTimeout(LOCAL_EMULATOR_WAITER_TIMEOUT)
                    .build();
                if (exists) {
                    cloudFormation.waiter().waitUntilStackUpdateComplete(request, override);
                } else {
                    cloudFormation.waiter().waitUntilStackCreateComplete(request, override);
                }
            } else if (exists) {
                cloudFormation.waiter().waitUntilStackUpdateComplete(request);
            } else {
                cloudFormation.waiter().waitUntilStackCreateComplete(request);
            }
        }, "AWS deployment failed for " + physical);

        return new AwsStackDeployResult(
            stackName, !exists, false, changeSummaries, outputs(physical));
    }

    /**
     * Runs {@code blockingWait} (one of the SDK's own {@code waitUntilStack*} calls) on a
     * background thread, while this thread polls {@code describe-stack-events} and reports each
     * event this stack hasn't already been seen having, in the order CloudFormation reported it,
     * to {@code listener} (skipped entirely when {@code listener} is {@code null}). Reuses the
     * SDK waiter's own terminal-state detection rather than reimplementing it; this only adds
     * visibility into what it's waiting on.
     *
     * <p>On failure, the exception message includes every event captured during the wait, with
     * events that carry a real failure reason (not a generic cancellation, from the cascade every
     * other in-flight resource gets once CloudFormation starts rolling back) surfaced first —
     * unlike the fixed-size event window a one-shot {@code describe-stack-events} call after the
     * fact would return, which can push the actual cause out of view entirely on a stack with
     * many resources.</p>
     */
    private void waitWithProgress(String stackName, StackProgressListener listener,
            Instant operationStart, Runnable blockingWait, String failureMessagePrefix) throws IOException {
        Set<String> seenEventIds = new HashSet<>();
        List<StackEvent> allEvents = new ArrayList<>();
        ExecutorService executor = Executors.newSingleThreadExecutor();
        Future<?> waiterFuture = executor.submit(blockingWait);
        try {
            while (!waiterFuture.isDone()) {
                pollNewEvents(stackName, operationStart, seenEventIds, allEvents, listener);
                Thread.sleep(POLL_INTERVAL.toMillis());
            }
            waiterFuture.get();
            pollNewEvents(stackName, operationStart, seenEventIds, allEvents, listener);
        } catch (ExecutionException e) {
            pollNewEvents(stackName, operationStart, seenEventIds, allEvents, listener);
            throw new IOException(
                failureMessagePrefix + ":\n" + formatFailureSummary(allEvents), e.getCause());
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new IOException("Interrupted while waiting for " + stackName, e);
        } finally {
            executor.shutdownNow();
        }
    }

    /**
     * Fetches this stack's events newer than {@code operationStart}, paginating as needed, and
     * reports any not already in {@code seenEventIds}. Events older than {@code operationStart}
     * are excluded so a stack event from an earlier deploy/update on the same stack (returned on
     * the first page, since {@code describeStackEvents} returns full history) is never reported
     * as this operation's own progress or folded into its failure summary.
     *
     * <p>Progress reporting only adds visibility -- it must never decide {@link #deploy}/
     * {@link #delete}'s outcome, which is the background waiter's job alone. A polling failure
     * (other than the stack having just finished deleting, an expected race with this poll) is
     * logged, not thrown; a listener that throws is isolated per-event so one bad callback can't
     * stop the rest of the batch from being reported.</p>
     */
    private void pollNewEvents(String stackName, Instant operationStart, Set<String> seenEventIds,
            List<StackEvent> allEvents, StackProgressListener listener) {
        // Newest-first within a page and across pages; walking oldest-to-newest within the
        // collected list happens below, once collection is done.
        List<StackEvent> newEvents = new ArrayList<>();
        String nextToken = null;
        try {
            do {
                var response = cloudFormation.describeStackEvents(DescribeStackEventsRequest.builder()
                    .stackName(stackName)
                    .nextToken(nextToken)
                    .build());
                PageScanResult scan = scanPageForNewEvents(response.stackEvents(), operationStart, seenEventIds);
                newEvents.addAll(scan.newEvents());
                nextToken = scan.stopPaginating() ? null : response.nextToken();
            } while (nextToken != null);
        } catch (CloudFormationException e) {
            if (e.statusCode() != 400 && e.statusCode() != 404) {
                LOG.warning("describeStackEvents failed for " + stackName + ": " + e.getMessage());
            }
            return;
        } catch (SdkException e) {
            LOG.warning("describeStackEvents failed for " + stackName + ": " + e.getMessage());
            return;
        }
        for (int i = newEvents.size() - 1; i >= 0; i--) {
            StackEvent event = newEvents.get(i);
            allEvents.add(event);
            if (listener != null) {
                try {
                    listener.onEvent(new StackProgressEvent(
                        String.valueOf(event.timestamp()),
                        event.resourceType(),
                        event.logicalResourceId(),
                        event.physicalResourceId(),
                        event.resourceStatusAsString(),
                        event.resourceStatusReason()));
                } catch (RuntimeException e) {
                    LOG.warning("StackProgressListener threw for " + stackName + ": " + e.getMessage());
                }
            }
        }
    }

    /**
     * One {@code describeStackEvents} page's worth of new events (newest-first, matching the
     * page's own order) and whether pagination should stop -- {@code true} once a pre-cutoff or
     * already-seen event is reached, since {@code describeStackEvents} guarantees strict
     * newest-first ordering: everything after that point, on this page or any later one, is at
     * least as old and was necessarily already covered by an earlier poll.
     */
    record PageScanResult(List<StackEvent> newEvents, boolean stopPaginating) {}

    /** Package-visible for {@code AwsDirectDeployerTest} -- see {@link #pollNewEvents}. */
    static PageScanResult scanPageForNewEvents(
            List<StackEvent> pageEvents, Instant operationStart, Set<String> seenEventIds) {
        List<StackEvent> newEvents = new ArrayList<>();
        for (StackEvent event : pageEvents) {
            if (event.timestamp() != null && event.timestamp().isBefore(operationStart)) {
                return new PageScanResult(newEvents, true);
            }
            if (!seenEventIds.add(event.eventId())) {
                return new PageScanResult(newEvents, true);
            }
            newEvents.add(event);
        }
        return new PageScanResult(newEvents, false);
    }

    /**
     * Events with a real failure reason (not a generic cancellation) are the actual cause and are
     * listed first; the full event list follows for complete context. Unlike {@link
     * #formatEvent}'s previous caller, this has no arbitrary length cap -- every event captured
     * during the wait is included.
     */
    static String formatFailureSummary(List<StackEvent> events) {
        StringBuilder summary = new StringBuilder();
        List<StackEvent> rootCauses = events.stream()
            .filter(e -> e.resourceStatusAsString() != null
                && e.resourceStatusAsString().contains("FAILED"))
            .filter(e -> e.resourceStatusReason() != null
                && !e.resourceStatusReason().isBlank()
                && !e.resourceStatusReason().contains("cancelled"))
            .toList();
        if (!rootCauses.isEmpty()) {
            summary.append("Root cause(s):\n");
            rootCauses.forEach(e -> summary.append(formatEvent(e)).append('\n'));
            summary.append('\n');
        }
        summary.append("All events:\n");
        events.forEach(e -> summary.append(formatEvent(e)).append('\n'));
        return summary.toString();
    }

    /**
     * Creates a change set for {@code stackName} from {@code template} and returns its
     * resource-change summary without executing it — the create-time equivalent of
     * {@code cdk deploy --no-execute}. The change set is deleted immediately after being
     * described, so nothing in the account outlives this call; unlike the CDK CLI's own
     * {@code --no-execute}, there's no changeset left behind to inspect or execute later, since
     * this is a one-shot preview, not a staged deploy. No-op (returns {@code noOp = true}) when
     * the stack already matches the candidate template, exactly like {@link #deploy}.
     */
    public AwsStackDeployResult previewChangeSet(String stackName, Path template) throws IOException {
        String physical = physicalStackName(stackName);
        LocalStackCdkAssetPublisher.publish(
            template.getParent(), stackName, s3, resolveAccountId(), localEmulatorTarget);

        boolean exists = stackExists(physical) && stackIsDeployable(physical);
        // See the identical comment in deploy() -- applies to every target now, not just local
        // emulators.
        String templateBody = resolveCdkBootstrapParameters(Files.readString(template));

        if (exists && templateMatches(physical, templateBody)) {
            return new AwsStackDeployResult(stackName, false, true, List.of(), Map.of());
        }

        String changeSetName = "cfc-preview-" + UUID.randomUUID().toString().substring(0, 8);
        var changeSet = createAndWaitForChangeSet(physical, templateBody, exists, changeSetName);
        if (changeSet.status() == ChangeSetStatus.FAILED) {
            String reason = changeSet.statusReason() == null ? "" : changeSet.statusReason();
            deleteChangeSet(physical, changeSetName);
            if (isNoOp(reason)) {
                return new AwsStackDeployResult(stackName, false, true, List.of(), Map.of());
            }
            throw new IOException("AWS change set failed for " + physical + ": " + reason);
        }

        List<String> changeSummaries = changeSet.changes().stream()
            .map(change -> summarize(change.resourceChange()))
            .toList();
        deleteChangeSet(physical, changeSetName);

        return new AwsStackDeployResult(stackName, !exists, false, changeSummaries, Map.of());
    }

    public void delete(String stackName) throws IOException {
        delete(stackName, null);
    }

    /**
     * Same as {@link #delete(String)}, additionally reporting each CloudFormation stack event to
     * {@code listener} as it happens (may be {@code null} to skip live reporting).
     */
    public void delete(String stackName, StackProgressListener listener) throws IOException {
        String physical = physicalStackName(stackName);
        if (!stackExists(physical)) {
            return;
        }
        var request = DescribeStacksRequest.builder().stackName(physical).build();
        try {
            Instant operationStart = Instant.now();
            cloudFormation.deleteStack(DeleteStackRequest.builder().stackName(physical).build());
            waitWithProgress(physical, listener, operationStart,
                () -> cloudFormation.waiter().waitUntilStackDeleteComplete(request),
                "Failed to delete AWS stack " + physical);
        } catch (IOException e) {
            if (stackExists(physical)) {
                throw e;
            }
        }
    }

    public boolean stackExists(String stackName) {
        try {
            var stacks = cloudFormation.describeStacks(
                    DescribeStacksRequest.builder().stackName(stackName).build())
                .stacks();
            if (stacks == null || stacks.isEmpty()) {
                return false;
            }
            String status = stacks.getFirst().stackStatusAsString();
            // DELETE_COMPLETE (and in-progress deletes) must not short-circuit redeploy.
            return status == null || !status.toUpperCase(Locale.ROOT).contains("DELETE");
        } catch (CloudFormationException e) {
            if (e.statusCode() == 400 || e.statusCode() == 404) {
                return false;
            }
            throw e;
        }
    }

    public Map<String, String> outputs(String stackName) {
        List<Output> stackOutputs = cloudFormation.describeStacks(
                DescribeStacksRequest.builder().stackName(stackName).build())
            .stacks().getFirst().outputs();
        Map<String, String> result = new LinkedHashMap<>();
        for (Output output : stackOutputs) {
            result.put(output.outputKey(), output.outputValue());
        }
        return Map.copyOf(result);
    }

    /**
     * Confirm the stack exists and return its outputs.
     *
     * @throws IOException when the stack is missing or outputs cannot be read
     */
    public Map<String, String> verifyDeployment(String stackName) throws IOException {
        String physical = physicalStackName(stackName);
        if (!stackExists(physical)) {
            throw new IOException("Stack does not exist: " + stackName);
        }
        return outputs(physical);
    }

    private boolean stackIsDeployable(String stackName) {
        try {
            String status = cloudFormation.describeStacks(
                    DescribeStacksRequest.builder().stackName(stackName).build())
                .stacks().getFirst().stackStatusAsString();
            if (status == null) {
                return false;
            }
            String normalized = status.toUpperCase(Locale.ROOT);
            return normalized.contains("COMPLETE") && !normalized.contains("ROLLBACK");
        } catch (CloudFormationException e) {
            return false;
        }
    }

    /**
     * CDK injects {@code BootstrapVersion} as {@code AWS::SSM::Parameter::Value<String>}, whose
     * default resolves from {@code /cdk-bootstrap/hnb659fds/version} in SSM Parameter Store. That
     * parameter exists in a bootstrapped AWS account but never in a local emulator or an AWS
     * Marketplace buyer's own (never-bootstrapped) account, where CloudFormation rejects the
     * reference ({@code "Parameter BootstrapVersion should either have input value or default
     * value"}) before creating a single resource. Applies the same rewrite as {@code
     * LocalStackTemplateAdapter.resolveCdkBootstrapParameters}, duplicated because {@code
     * cloudforge-api} cannot depend on {@code cloudforge-localstack}. Called for every deploy
     * target, not just a local emulator's -- a self-contained template must never depend on the
     * deploying account's bootstrap state. Public so {@code CloudForgeSynthesizer
     * #synthesizeForMarketplace} can also apply it directly to the template file it writes to
     * disk, not just to the in-memory body this class sends to CloudFormation -- a file meant to
     * be uploaded to AWS Marketplace has no later deploy-time rewrite step to rely on.
     */
    public static String resolveCdkBootstrapParameters(String templateBody) throws IOException {
        JsonNode root = MAPPER.readTree(templateBody);
        if (!(root instanceof ObjectNode template)) {
            return templateBody;
        }
        JsonNode parametersNode = template.get("Parameters");
        if (!(parametersNode instanceof ObjectNode parameters)) {
            return templateBody;
        }
        // Scoped to the literal "BootstrapVersion" name CDK always gives this specific parameter
        // -- matching by Type alone would also catch any other AWS::SSM::Parameter::Value the
        // template happens to declare (an AMI ID lookup, for instance) and overwrite its real
        // default with "21", corrupting an unrelated resource.
        JsonNode bootstrapVersionNode = parameters.get("BootstrapVersion");
        if (!(bootstrapVersionNode instanceof ObjectNode parameter)) {
            return templateBody;
        }
        String type = parameter.path("Type").asText();
        if (type == null || !type.startsWith("AWS::SSM::Parameter::Value")) {
            return templateBody;
        }
        String defaultValue = parameter.path("Default").asText(null);
        String resolved = (defaultValue == null || defaultValue.isBlank() || defaultValue.startsWith("/"))
            ? "21"
            : defaultValue;
        parameter.put("Type", "String");
        parameter.put("Default", resolved);
        return MAPPER.writeValueAsString(template);
    }

    private boolean templateMatches(String stackName, String candidate) throws IOException {
        try {
            String deployed = cloudFormation.getTemplate(
                GetTemplateRequest.builder().stackName(stackName).build()).templateBody();
            if (deployed == null || deployed.isBlank()) {
                return false;
            }
            return MAPPER.readTree(deployed).equals(MAPPER.readTree(candidate));
        } catch (CloudFormationException e) {
            // Incomplete / missing template — force a real create/update path
            return false;
        } catch (Exception e) {
            throw new IOException("Unable to compare deployed AWS template", e);
        }
    }

    /**
     * Creates a change set for {@code physical} from {@code templateBody} and waits for it to
     * settle. Shared by {@link #deploy} (which executes the result) and {@link #previewChangeSet}
     * (which deletes it unexecuted) — both need the identical creation and template-size-based
     * inline-vs-S3 branching.
     */
    private DescribeChangeSetResponse
            createAndWaitForChangeSet(String physical, String templateBody, boolean exists, String changeSetName)
            throws IOException {
        CreateChangeSetRequest.Builder changeSetBuilder = CreateChangeSetRequest.builder()
            .stackName(physical)
            .changeSetName(changeSetName)
            .changeSetType(exists ? ChangeSetType.UPDATE : ChangeSetType.CREATE)
            .capabilities(Capability.CAPABILITY_IAM, Capability.CAPABILITY_NAMED_IAM)
            .tags(managedTags());

        if (templateBody.getBytes(StandardCharsets.UTF_8).length > MAX_INLINE_TEMPLATE_BYTES) {
            changeSetBuilder.templateURL(uploadTemplateToS3(physical, templateBody));
        } else {
            changeSetBuilder.templateBody(templateBody);
        }

        cloudFormation.createChangeSet(changeSetBuilder.build());
        return waitForChangeSet(physical, changeSetName);
    }

    private DescribeChangeSetResponse
            waitForChangeSet(String stackName, String changeSetName) throws IOException {
        Instant deadline = Instant.now().plus(OPERATION_TIMEOUT);
        DescribeChangeSetRequest request = DescribeChangeSetRequest.builder()
            .stackName(stackName)
            .changeSetName(changeSetName)
            .build();
        while (Instant.now().isBefore(deadline)) {
            var response = cloudFormation.describeChangeSet(request);
            if (response.status() == ChangeSetStatus.CREATE_COMPLETE
                    || response.status() == ChangeSetStatus.FAILED) {
                return response;
            }
            try {
                Thread.sleep(POLL_INTERVAL);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new IOException("Interrupted while waiting for AWS change set", e);
            }
        }
        throw new IOException("Timed out waiting for AWS change set " + changeSetName);
    }

    private void deleteChangeSet(String stackName, String changeSetName) {
        cloudFormation.deleteChangeSet(DeleteChangeSetRequest.builder()
            .stackName(stackName)
            .changeSetName(changeSetName)
            .build());
    }

    private static String formatEvent(StackEvent event) {
        return "%s %s %s %s".formatted(
            event.resourceStatusAsString(),
            event.resourceType(),
            event.logicalResourceId(),
            event.resourceStatusReason() == null ? "" : event.resourceStatusReason());
    }

    private static String summarize(ResourceChange change) {
        return "%s %s (%s)%s".formatted(
            change.actionAsString(),
            change.logicalResourceId(),
            change.resourceType(),
            change.replacementAsString() == null ? "" : " replacement=" + change.replacementAsString());
    }

    private String uploadTemplateToS3(String stackName, String templateBody) throws IOException {
        String bucket = resolvedTemplateBucket();
        ensureTemplateBucket(bucket);
        String key = stackName + ".template.json";
        s3.putObject(
            PutObjectRequest.builder()
                .bucket(bucket)
                .key(key)
                .contentType("application/json")
                .build(),
            RequestBody.fromString(templateBody, StandardCharsets.UTF_8));
        return s3.utilities().getUrl(builder -> builder.bucket(bucket).key(key)).toString();
    }

    /**
     * Resolves the template bucket name lazily, because it includes the account ID and the
     * constructor must not make network calls (see {@link #resolveAccountId}). An explicit bucket
     * name injected through a test constructor takes precedence.
     */
    private String resolvedTemplateBucket() {
        return templateBucket != null ? templateBucket : templateBucketName(resolveAccountId(), region.id());
    }

    private void ensureTemplateBucket(String bucket) {
        try {
            s3.headBucket(HeadBucketRequest.builder().bucket(bucket).build());
        } catch (NoSuchBucketException e) {
            try {
                s3.createBucket(CreateBucketRequest.builder().bucket(bucket).build());
            } catch (BucketAlreadyExistsException | BucketAlreadyOwnedByYouException ignored) {
                // concurrent create
            }
        }
    }

    private static boolean isNoOp(String reason) {
        String normalized = reason.toLowerCase(Locale.ROOT);
        return normalized.contains("didn't contain changes")
            || normalized.contains("no updates")
            || normalized.contains("no changes");
    }

    @Override
    public void close() {
        cloudFormation.close();
        s3.close();
    }
}