Frontend deploy workflow - Mix of old and new instances in production CMS load balancer during the deploy Push stage

[barrydeveire]

Describe the bug
Frontend deploy workflow - Mix of old and new instances in production CMS load balancer during the deploy Push stage

Issue is observed at the push stage of the deploy workflow where the prod CMS load balancer has a mix of old instances and new instances with the new build.

Issue seems to be isolated to gyro projects where there is a separate ASG for the CMS load balancer.

To Reproduce
Production CMS ASG configs
`# -- CMS Launch Configuration/Autoscaling Group

@for container -in $containers
beam::container-config "$(environment)-cms-$(container)"
name: $container
file-backend: 'builds'
@beam::import-yaml: $container $environment cms
end
@EnD

aws::launch-configuration cms
name: "$project cms prod v$serial $brightspot-build"
ami: $ami-id
instance-type: "m5.xlarge"
key: $(aws::key-pair "base/key-pair")
instance-profile: $(aws::iam-instance-profile frontend)
security-groups: [$(aws::security-group frontend)]
enable-monitoring: true

@beam::launch-containers
    layer: cms
    brightspot-build: $(brightspot::build production-build)
    containers: $(beam::container-config "$(environment)-cms-*")
@end

end

aws::autoscaling-group cms
name: $(aws::launch-configuration cms).name
launch-configuration: $(aws::launch-configuration cms)
subnets: [
$(aws::subnet "base/private-$(base-region)a"),
$(aws::subnet "base/private-$(base-region)b")
]
health-check-type: "EC2"
classic-load-balancers: [
$(aws::load-balancer cms)
]

min-size: 4
max-size: 24

scaling-policy
    policy-name: "Cms Scale Up"
    adjustment-type: "ChangeInCapacity"
    policy-type: "SimpleScaling"
    cooldown: 120
    scaling-adjustment: 4
end

scaling-policy
    policy-name: "Cms Scale Down"
    adjustment-type: "ChangeInCapacity"
    policy-type: "SimpleScaling"
    cooldown: 300
    scaling-adjustment: -2
end

tags: {
    'Name': $(aws::launch-configuration cms).name,
    'Project': $project,
    'beam.project': $project,
    'beam.env': $environment,
    'beam.layer': cms,
    'beam.serial': $serial
}

propagate-at-launch-tags: ['Name', 'Project', 'beam.project', 'beam.serial', 'beam.layer', 'beam.serial', 'beam.env']

end`

Deployment workflow
`# -- Deployment Workflow

frontend-verify-hostname: "verify.($(aws::route53-hosted-zone base/public-zone).name)"
wildcard-frontend-verify-hostname: "*.verify.($(aws::route53-hosted-zone base/public-zone).name)"
cms-verify-hostname: "cms.verify.($(aws::route53-hosted-zone base/public-zone).name)"

@workflow::define aws::autoscaling-group frontend-deploy

stage verify
    confirm-diff: true

    @workflow::create aws::load-balancer frontend-verify
        @extends: $(aws::load-balancer frontend)

        name: "\($(aws::load-balancer frontend).name)-v"
        instances: []
    @end

    @workflow::create aws::load-balancer cms-verify
        @extends: $(aws::load-balancer cms)

        name: "\($(aws::load-balancer cms).name)-v"
        instances: []
    @end

    @workflow::create aws::route53-record-set frontend-elb-verify
        hosted-zone: $(aws::route53-hosted-zone base/public-zone)
        name: $frontend-verify-hostname
        type: "A"
        alias
            hosted-zone-id: $(aws::load-balancer frontend-verify).hosted-zone-id
            evaluate-target-health: false
            dns-name: $(aws::load-balancer frontend-verify).*.dns-name
        end
    @end

    @workflow::create aws::route53-record-set wildcard-frontend-elb-verify
        hosted-zone: $(aws::route53-hosted-zone base/public-zone)
        name: $wildcard-frontend-verify-hostname
        type: "A"
        alias
            hosted-zone-id: $(aws::load-balancer frontend-verify).hosted-zone-id
            evaluate-target-health: false
            dns-name: $(aws::load-balancer frontend-verify).*.dns-name
        end
    @end

    @workflow::create aws::route53-record-set cms-elb-verify
        hosted-zone: $(aws::route53-hosted-zone base/public-zone)
        name: $cms-verify-hostname
        type: "A"
        alias
            hosted-zone-id: $(aws::load-balancer cms-verify).hosted-zone-id
            evaluate-target-health: false
            dns-name: $(aws::load-balancer cms-verify).*.dns-name
        end
    @end

    @workflow::create aws::launch-configuration cms-verify
        @extends: $(aws::launch-configuration cms)

        @beam::launch-containers
            layer: cms
            brightspot-build: $(brightspot::build production-build)
            containers: $(beam::container-config "$(environment)-$(layer)-*")
        @end
    @end

    @workflow::create aws::autoscaling-group cms-verify
        @extends: $(aws::autoscaling-group cms)

        min-size: $(external-query aws::autoscaling-group {name: $(CURRENT).name}).0.desired-capacity
        desired-capacity: $(external-query aws::autoscaling-group {name: $(CURRENT).name}).0.desired-capacity

        name: $(aws::launch-configuration cms-verify).name
        launch-configuration: $(aws::launch-configuration cms-verify)
        classic-load-balancers: [
            $(aws::load-balancer cms-verify)
        ]
        tags: {
            'Name': $(aws::launch-configuration cms-verify).name,
            'Project': $project,
            'beam.project': $project,
            'beam.env': $environment,
            'beam.layer': cms,
            'beam.serial': $serial
        }
    @end

    @workflow::create aws::launch-configuration frontend-verify
        @extends: $(aws::launch-configuration frontend)

        @beam::launch-containers
            brightspot-build: $(brightspot::build production-build)
            containers: $(beam::container-config "$(environment)-$(layer)-*")
        @end
    @end

    @workflow::create aws::autoscaling-group frontend-verify
        @extends: $(aws::autoscaling-group frontend)

        min-size: $(external-query aws::autoscaling-group {name: $(CURRENT).name}).0.desired-capacity
        desired-capacity: $(external-query aws::autoscaling-group {name: $(CURRENT).name}).0.desired-capacity

        name: $(aws::launch-configuration frontend-verify).name
        launch-configuration: $(aws::launch-configuration frontend-verify)
        classic-load-balancers: [
            $(aws::load-balancer frontend-verify)
        ]
        tags: {
            'Name': $(aws::launch-configuration frontend-verify).name,
            'Project': $project,
            'beam.project': $project,
            'beam.env': $environment,
            'beam.layer': frontend,
            'beam.serial': $serial
        }

        @wait: $(aws::load-balancer frontend-verify).instance-health.InService = $desired-capacity -check-every 10 -at-most 120
        @wait: $(aws::load-balancer cms-verify).instance-health.InService = $desired-capacity -check-every 10 -at-most 120
    @end

    @workflow::update $(aws::autoscaling-group frontend)
        name: $(CURRENT).name
        launch-configuration: $(CURRENT).launch-configuration
        min-size: $(external-query aws::autoscaling-group {name: $(CURRENT).name}).0.desired-capacity
        tags: $(CURRENT).tags
    @end

    transition verify-finish
        to: verify-finish
    end
end

stage verify-finish
    confirm-diff: true
    transition-prompt: "\nL'Opinion Frontend Verification Load Balancer: $(frontend-verify-hostname)\nL'Agefi Frontend Verification Load Balancer: lagefi.verify.l-opinion.psdops.com\nCMS Verification Load Balancer: $(cms-verify-hostname)\nNext Stage?"

    @workflow::update $(aws::cloudwatch-metric-alarm scale-up)
        alarm-actions: [
            "\($($(aws::autoscaling-group frontend).scaling-policy | policy-name = 'Scale Up').'0'.policy-arn)",
            "\($($(aws::autoscaling-group frontend-verify).scaling-policy | policy-name = 'Scale Up').'0'.policy-arn)"
        ]
    @end

    @workflow::update $(aws::cloudwatch-metric-alarm scale-down)
        alarm-actions: [
            "\($($(aws::autoscaling-group frontend).scaling-policy | policy-name = 'Scale Down').'0'.policy-arn)",
            "\($($(aws::autoscaling-group frontend-verify).scaling-policy | policy-name = 'Scale Down').'0'.policy-arn)"
        ]
    @end

    @workflow::update $(aws::cloudwatch-metric-alarm cms-scale-up)
        alarm-actions: [
            "\($($(aws::autoscaling-group cms).scaling-policy | policy-name = 'Cms Scale Up').'0'.policy-arn)",
            "\($($(aws::autoscaling-group cms-verify).scaling-policy | policy-name = 'Cms Scale Up').'0'.policy-arn)"
        ]
    @end

    @workflow::update $(aws::cloudwatch-metric-alarm cms-scale-down)
        alarm-actions: [
            "\($($(aws::autoscaling-group cms).scaling-policy | policy-name = 'Cms Scale Down').'0'.policy-arn)",
            "\($($(aws::autoscaling-group cms-verify).scaling-policy | policy-name = 'Cms Scale Down').'0'.policy-arn)"
        ]
    @end

    transition push
        to: push
        description: 'Push the verification instances into the production load balancer and deregister old instances.'
    end

    transition reset
        to: reset
        description: 'Delete verification instances and load balancer, then exit Gyro.'
    end
end

stage push
    @workflow::update $(aws::autoscaling-group frontend-verify)
        classic-load-balancers: [
            $(aws::load-balancer frontend)
        ]

    @end

    @workflow::update $(aws::autoscaling-group cms-verify)
        classic-load-balancers: [
            $(aws::load-balancer cms)
        ]

        @wait: $(aws::load-balancer frontend).instance-health.InService = $(aws::load-balancer frontend).instance-health.Total -check-every 10 -at-most 120
        @wait: $(aws::load-balancer cms).instance-health.InService = $(aws::load-balancer cms).instance-health.Total -check-every 10 -at-most 120

    @end

    transition push-finish
        to: push-finish
    end
end

stage push-finish
    @workflow::update $(aws::autoscaling-group frontend)
        classic-load-balancers: [ ]
    @end

    transition revert
        to: revert
        description: 'Revert the verification instances. Move the verification instances to the verify load balancer and put the production instances back'
    end

    transition commit
        to: finished
        description: 'Commit verification instances to production load balancer.'
    end
end

stage revert
    transition-prompt: "L'Opinion Frontend Verification Load Balancer: $(frontend-verify-hostname)\nL'Agefi Frontend Verification Load Balancer: lagefi.verify.l-opinion.psdops.com\nCMS Verification Load Balancer: $(cms-verify-hostname)\nNext Stage?"
    @workflow::update $(aws::autoscaling-group frontend)
        classic-load-balancers: [
            $(aws::load-balancer frontend)
        ]

        @wait: $(aws::load-balancer frontend).instance-health.InService = $(aws::load-balancer frontend).instance-health.Total -check-every 10 -at-most 120
        @wait: $(aws::load-balancer cms).instance-health.InService = $(aws::load-balancer cms).instance-health.Total -check-every 10 -at-most 120
    @end

    transition revert-finish
        to: revert-finish
    end
end

stage revert-finish
    @workflow::update $(aws::autoscaling-group cms-verify)
        classic-load-balancers: [
            $(aws::load-balancer cms-verify)
        ]
    @end

    @workflow::update $(aws::autoscaling-group frontend-verify)
        classic-load-balancers: [
            $(aws::load-balancer frontend-verify)
        ]

        @wait: $(aws::load-balancer frontend-verify).instance-health.InService = $(aws::load-balancer frontend-verify).instance-health.Total -check-every 10 -at-most 120
        @wait: $(aws::load-balancer cms-verify).instance-health.InService = $(aws::load-balancer cms-verify).instance-health.Total -check-every 10 -at-most 120
    @end

    transition push
        to: push
        description: 'Push the verification instances into the production load balancer and deregister old instances.'
    end

    transition reset
        to: reset
        description: 'Delete verification instances and load balancer, then exit Gyro.'
    end
end

stage finished
    @workflow::update $(aws::cloudwatch-metric-alarm scale-up)
        alarm-actions: [
            "\($($(aws::autoscaling-group frontend-verify).scaling-policy | policy-name = 'Scale Up').'0'.policy-arn)"
        ]
    @end

    @workflow::update $(aws::cloudwatch-metric-alarm scale-down)
        alarm-actions: [
            "\($($(aws::autoscaling-group frontend-verify).scaling-policy | policy-name = 'Scale Down').'0'.policy-arn)"
        ]
    @end

    @workflow::update $(aws::cloudwatch-metric-alarm cms-scale-up)
        alarm-actions: [
            "\($($(aws::autoscaling-group cms-verify).scaling-policy | policy-name = 'Cms Scale Up').'0'.policy-arn)"
        ]
    @end

    @workflow::update $(aws::cloudwatch-metric-alarm cms-scale-down)
        alarm-actions: [
            "\($($(aws::autoscaling-group cms-verify).scaling-policy | policy-name = 'Cms Scale Down').'0'.policy-arn)"
        ]
    @end

    @workflow::replace: $(aws::launch-configuration frontend) $(aws::launch-configuration frontend-verify)
    @workflow::replace: $(aws::autoscaling-group frontend) $(aws::autoscaling-group frontend-verify)
    @workflow::replace: $(aws::launch-configuration cms) $(aws::launch-configuration cms-verify)
    @workflow::replace: $(aws::autoscaling-group cms) $(aws::autoscaling-group cms-verify)
    @workflow::delete: $(aws::route53-record-set frontend-elb-verify)
    @workflow::delete: $(aws::route53-record-set wildcard-frontend-elb-verify)
    @workflow::delete: $(aws::route53-record-set cms-elb-verify)
    @workflow::delete: $(aws::load-balancer frontend-verify)
    @workflow::delete: $(aws::load-balancer cms-verify)

    @audit::metadata
        metaData : ['Install']
    @end
end

stage reset
    @workflow::update $(aws::cloudwatch-metric-alarm scale-up)
        alarm-actions: [
            "\($($(aws::autoscaling-group frontend).scaling-policy | policy-name = 'Scale Up').'0'.policy-arn)"
        ]
    @end

    @workflow::update $(aws::cloudwatch-metric-alarm scale-down)
        alarm-actions: [
            "\($($(aws::autoscaling-group frontend).scaling-policy | policy-name = 'Scale Down').'0'.policy-arn)"
        ]
    @end

    @workflow::update $(aws::cloudwatch-metric-alarm cms-scale-up)
        alarm-actions: [
            "\($($(aws::autoscaling-group cms).scaling-policy | policy-name = 'Cms Scale Up').'0'.policy-arn)"
        ]
    @end

    @workflow::update $(aws::cloudwatch-metric-alarm cms-scale-down)
        alarm-actions: [
            "\($($(aws::autoscaling-group cms).scaling-policy | policy-name = 'Cms Scale Down').'0'.policy-arn)"
        ]
    @end

    @workflow::delete: $(aws::route53-record-set frontend-elb-verify)
    @workflow::delete: $(aws::route53-record-set wildcard-frontend-elb-verify)
    @workflow::delete: $(aws::route53-record-set cms-elb-verify)
    @workflow::delete: $(aws::autoscaling-group frontend-verify)
    @workflow::delete: $(aws::launch-configuration frontend-verify)
    @workflow::delete: $(aws::autoscaling-group cms-verify)
    @workflow::delete: $(aws::launch-configuration cms-verify)
    @workflow::delete: $(aws::load-balancer frontend-verify)
    @workflow::delete: $(aws::load-balancer cms-verify)
end

@end`

Expected behavior
At the push stage only instances with the new build should be present in the production CMS load balancer

Stacktrace
If applicable, cut and paste any stacktrace or error message to help explain your problem.

Additional context
Opsdesk event https://beam-enterprise.opsdesk.cloud/p/l-opinion/events/00000193-01cd-d32a-a5ff-e5ef1feb0000