aws_instance_cloudwatch_utilization.pt

name "AWS Inefficient Instance Utilization using CloudWatch"
rs_pt_ver 20180301
type "policy"
short_description "**Deprecated: This policy is no longer being updated. Please see [README](https://github.com/flexera-public/policy_templates/tree/master/cost/aws/instance_cloudwatch_utilization/) for more details.** Checks inefficient instance utilization using provided CPU and Memory thresholds. Instances matching the criteria can be resized after user approval. See the [README](https://github.com/flexera-public/policy_templates/tree/master/cost/aws/instance_cloudwatch_utilization/) and [docs.flexera.com/flexera/EN/Automation](https://docs.flexera.com/flexera/EN/Automation/AutomationGS.htm) to learn more."
long_description ""
severity "low"
category "Cost"
default_frequency "daily"
info(
  version: "3.3.3",
  provider: "AWS",
  service: "Compute",
  policy_set: "Inefficient Instance Usage",
  deprecated: "true",
  hide_skip_approvals: "true"
)

###############################################################################
# Parameters
###############################################################################

parameter "param_allowed_regions_allow_or_deny" do
  type "string"
  label "Allow/Deny Regions"
  description "Allow or Deny entered regions. See the README for more details"
  allowed_values "Allow", "Deny"
  default "Allow"
end

parameter "param_allowed_regions" do
  type "list"
  label "Regions"
  allowed_pattern /^([a-zA-Z-_]+-[a-zA-Z0-9-_]+-[0-9-_]+,*|)+$/
  description "A list of allowed or denied regions. See the README for more details"
end

parameter "param_email" do
  type "list"
  label "Email addresses to notify"
  description "Email addresses of the recipients you wish to notify when new incidents are created"
end

parameter "param_aws_account_number" do
  type "string"
  category "Policy Settings"
  label "Account Number"
  description "Leave blank; this is for automated use with Meta Policies. See README for more details."
  default ""
end

parameter "param_avg_used_memory" do
  type "number"
  label "Average used memory percentage"
  description "Set to -1 to ignore memory utilization"
  default 60
  min_value -1
  max_value 100
end

parameter "param_avg_cpu" do
  type "number"
  label "Average used CPU percentage"
  description "Set to -1 to ignore CPU utilization"
  default 60
  min_value -1
  max_value 100
end

parameter "param_exclusion_tag_key" do
  category "User Inputs"
  label "Exclusion Tag Key:Value"
  description "Cloud native tag key to ignore instances. Format: Key:Value"
  type "string"
  allowed_pattern /(^$)|([\w]?)+\:([\w]?)+/
end

parameter "param_automatic_action" do
  type "list"
  label "Automatic Actions"
  description "When this value is set, this policy will automatically take the selected action(s)"
  allowed_values ["Downsize Instances"]
  default []
end

parameter "param_log_to_cm_audit_entries" do
  type "string"
  label "Log to CM Audit Entries"
  description "Boolean for whether or not to log any debugging information from actions to CM Audit Entries, this should be left set to No on Flexera EU"
  default "No"
  allowed_values "Yes", "No"
end

###############################################################################
# Authentication
###############################################################################

#authenticate with AWS
credentials "auth_aws" do
  schemes "aws","aws_sts"
  label "AWS"
  description "Select the AWS Credential from the list"
  tags "provider=aws"
  aws_account_number $param_aws_account_number
end

###############################################################################
# Pagination
###############################################################################

pagination "aws_pagination_xml" do
  get_page_marker do
    body_path "//DescribeInstancesResponse/nextToken"
  end
  set_page_marker do
    query "NextToken"
  end
end

###############################################################################
# Datasources
###############################################################################

datasource "ds_get_caller_identity" do
  request do
    auth $auth_aws
    host "sts.amazonaws.com"
    path "/"
    header "User-Agent", "RS Policies"
    query "Action", "GetCallerIdentity"
    query "Version", "2011-06-15"
  end
  result do
    encoding "xml"
    collect xpath(response, "//GetCallerIdentityResponse/GetCallerIdentityResult") do
      field "account",xpath(col_item, "Account")
    end
  end
end

# ds_region_list is a list of regions that are opted-in or opt-in-not-required
datasource "ds_regions_list" do
  # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeRegions.html
  request do
    auth $auth_aws
    host "ec2.amazonaws.com"
    path "/"
    query "Action", "DescribeRegions"
    query "Version", "2016-11-15"
    query "Filter.1.Name", "opt-in-status"
    query "Filter.1.Value.1", "opt-in-not-required"
    query "Filter.1.Value.2", "opted-in"
  end
  result do
    encoding "xml"
    collect xpath(response, "//DescribeRegionsResponse/regionInfo/item", "array") do
      field "region", xpath(col_item, "regionName")
    end
  end
end

# Get only SCP enabled regions
datasource "ds_regions" do
  run_script $js_regions, $param_allowed_regions, $ds_regions_list, $param_allowed_regions_allow_or_deny
end

datasource "ds_instances_set" do
  iterate $ds_regions
  request do
    auth $auth_aws
    pagination $aws_pagination_xml
    host join(["ec2.", val(iter_item, "region"), ".amazonaws.com"])
    path "/"
    query "Action", "DescribeInstances"
    query "Version", "2016-11-15"
    query "Filter.1.Name", "instance-state-name"
    query "Filter.1.Value.1", "running"
    header "User-Agent", "RS Policies"
    header "Content-Type", "text/xml"
  end
  result do
    encoding "xml"
    collect xpath(response, "//DescribeInstancesResponse/reservationSet/item", "array") do
      field "instances_set" do
        collect xpath(col_item,"instancesSet/item","array") do
          field "region",val(iter_item, "region")
          field "instanceId", xpath(col_item,"instanceId")
          field "imageId", xpath(col_item,"imageId")
          field "instanceType", xpath(col_item, "instanceType")
          field "platform", xpath(col_item, "platform")
          field "privateDnsName", xpath(col_item, "privateDnsName")
          field "tag_set" do
            collect xpath(col_item,"tagSet/item","array") do
              field "key", xpath(col_item, "key")
              field "value", xpath(col_item, "value")
            end
          end
        end
      end
    end
  end
end

datasource "ds_instances" do
  run_script $js_instances, $ds_instances_set, $param_exclusion_tag_key
end

datasource "ds_cloudwatch_cpu_usage" do
  iterate $ds_instances
  request do
    run_script $js_cloudwatch_cpu_usage, val(iter_item, "region"), val(iter_item, "instanceId")
  end
  result do
    encoding "json"
    collect jmes_path(response, "GetMetricStatisticsResponse.GetMetricStatisticsResult.Datapoints[*]") do
      field "region", val(iter_item, "region")
      field "instanceId", val(iter_item, "instanceId")
      field "instanceType", val(iter_item, "instanceType")
      field "platform", val(iter_item, "platform")
      field "privateDnsName", val(iter_item, "privateDnsName")
      field "tag_set", val(iter_item, "tag_set")
      field "cpu_unit", jmes_path(col_item,"Unit")
      field "cpu_maximum", jmes_path(col_item, "Maximum")
      field "cpu_minimum", jmes_path(col_item, "Minimum")
      field "cpu_average", jmes_path(col_item, "Average")
    end
  end
end

datasource "ds_cloudwatch_cpu_highest_usage_value" do
  run_script $js_cloudwatch_cpu_highest_usage_value, $ds_cloudwatch_cpu_usage
end

datasource "ds_cloudwatch_mem_usage" do
  iterate $ds_instances
  request do
    run_script $js_cloudwatch_mem_usage, val(iter_item, "region"), val(iter_item, "instanceId"), val(iter_item,"privateDnsName")
  end
  result do
    encoding "json"
    collect jmes_path(response,"GetMetricStatisticsResponse.GetMetricStatisticsResult.Datapoints[*]") do
      field "region", val(iter_item, "region")
      field "instanceId", val(iter_item, "instanceId")
      field "privateDnsName", val(iter_item, "privateDnsName")
      field "tag_set", val(iter_item, "tag_set")
      field "mem_unit", jmes_path(col_item,"Unit")
      field "mem_maximum", jmes_path(col_item, "Maximum")
      field "mem_minimum", jmes_path(col_item, "Minimum")
      field "mem_average", jmes_path(col_item, "Average")
    end
  end
end

datasource "ds_cloudwatch_mem_highest_usage_value" do
  run_script $js_cloudwatch_memory_highest_usage_value, $ds_cloudwatch_mem_usage
end

datasource "ds_cloudwatch_windows_mem_usage" do
  iterate $ds_instances
  request do
    run_script $js_cloudwatch_windows_memory_usage, val(iter_item, "region"), val(iter_item, "instanceId"), val(iter_item, "imageId"), val(iter_item, "instanceType")
  end
  result do
    encoding "json"
    collect jmes_path(response,"GetMetricStatisticsResponse.GetMetricStatisticsResult.Datapoints[*]") do
      field "region", val(iter_item, "region")
      field "instanceId", val(iter_item, "instanceId")
      field "privateDnsName", val(iter_item, "privateDnsName")
      field "tag_set", val(iter_item, "tag_set")
      field "mem_unit", jmes_path(col_item,"Unit")
      field "mem_maximum", jmes_path(col_item, "Maximum")
      field "mem_minimum", jmes_path(col_item, "Minimum")
      field "mem_average", jmes_path(col_item, "Average")
    end
  end
end

datasource "ds_cloudwatch_windows_mem_highest_usage_value" do
  run_script $js_cloudwatch_windows_memory_highest_usage_value, $ds_cloudwatch_windows_mem_usage
end

# get AWS instance size map
datasource "ds_aws_instance_size_map" do
  request do
    host "raw.githubusercontent.com"
    path "/rightscale/policy_templates/master/data/aws/instance_types.json"
    header "User-Agent", "RS Policies"
  end
end

datasource "ds_merged_metrics" do
  run_script $js_merged_metrics, $ds_cloudwatch_cpu_highest_usage_value, $ds_cloudwatch_mem_highest_usage_value, $ds_cloudwatch_windows_mem_highest_usage_value
end

datasource "ds_inefficient_utilization_metrics" do
  run_script $js_inefficient_utilization_metrics, $ds_aws_instance_size_map, $ds_merged_metrics, $param_avg_used_memory, $param_avg_cpu, $ds_get_caller_identity
end

###############################################################################
# Scripts
###############################################################################

script "js_regions", type:"javascript" do
  parameters "user_entered_regions", "all_regions", "regions_allow_or_deny"
  result "regions"
  code <<-EOS
    if(_.isEmpty(user_entered_regions)){
      regions = all_regions;
    }else{
      //Filter unique regions
      var uniqueRegions = _.uniq(user_entered_regions);
      var all_regions_list = [];
      //Filter and remove denied regions from all_regions
      if (regions_allow_or_deny == "Deny"){
        var all_regions = all_regions.filter(function(obj){
          return user_entered_regions.indexOf(obj.region) === -1;
        });
      }
      all_regions.forEach(function(all_region){
        all_regions_list.push(all_region.region)
      })

      //Filter valid regions
      var valid_regions = [];
      _.map(uniqueRegions, function(uniqueRegion){
        if(all_regions_list.indexOf(uniqueRegion) > -1){
          valid_regions.push({"region": uniqueRegion})
        }
      })

      //Throw an error if no valid regions found
      if (_.isEmpty(valid_regions)) {
        regions = all_regions;
      }else{
        regions = valid_regions
      }
    }
  EOS
end

script "js_instances", type: "javascript" do
  result "results"
  parameters "ds_instance_set", "param_exclusion_tag_key"
  code <<-EOS
  var tag_key = param_exclusion_tag_key.split(':')[0]
  var tag_value = param_exclusion_tag_key.split(':')[1]
  var results = []
  for ( n=0; n < ds_instance_set.length; n++) {
    var instance_set = ds_instance_set[n].instances_set
    for ( i=0; i < instance_set.length; i++) {
      var instance = instance_set[i]
      var tags = instance.tag_set
      if (_.contains(_.pluck(tags,'key'), tag_key) && _.contains(_.pluck(tags,'value'), tag_value)){
        continue
      } else {
        results.push({
          "region": instance.region,
          "instanceId": instance.instanceId,
          "imageId": instance.imageId,
          "instanceType": instance.instanceType,
          "platform": instance.platform,
          "privateDnsName": instance.privateDnsName,
          "tag_set": instance.tag_set
        })
      }
    }
  }

  results = _.sortBy(results, 'region');

EOS
end

script "js_cloudwatch_cpu_usage", type: "javascript" do
  result "results"
  parameters "region","instance_id"
  code <<-EOS
  var end_date = new Date().toISOString()
  var start_date = new Date(new Date().setDate(new Date().getDate() - 30)).toISOString();
  results = {
    "auth": "auth_aws",
    "host": 'monitoring.'+region+'.amazonaws.com',
    "verb": "GET",
    "path": "/",
    "headers": {
      "User-Agent": "RS Policies",
      "Content-Type": "application/json",
      "x-amz-target": "GraniteServiceVersion20100801.GetMetricStatistics",
      "Accept": "application/json",
      "Content-Encoding": "amz-1.0"
    }
    "query_params": {
      'Action': 'GetMetricStatistics',
      'Version': '2010-08-01',
      'Namespace': 'AWS/EC2',
      'MetricName': 'CPUUtilization',
      'Dimensions.member.1.Name': 'InstanceId',
      'Dimensions.member.1.Value': instance_id,
      'StartTime': start_date,
      'EndTime': end_date,
      'Period': "2592000",
      'Statistics.member.1': 'Maximum',
      'Statistics.member.2': 'Average',
      'Statistics.member.3': 'Minimum'
    }
  }
EOS
end

script "js_cloudwatch_cpu_highest_usage_value", type: "javascript" do
  result "instances"
  parameters "datapoints"
  code <<-EOS
    var instances = [];
    var instanceIds = [];
    var distinctInstances = [];
    datapoints.forEach(function(datapoint){
      if(instanceIds.indexOf(datapoint.instanceId) === -1){
        instanceIds.push(datapoint.instanceId)
        distinctInstances.push(datapoint)
      }
    })
    _.each(distinctInstances, function(currentInstance){
      var max_cpu_value = currentInstance.cpu_maximum; var min_cpu_value = currentInstance.cpu_minimum; var avg_cpu_value = currentInstance.cpu_average;
      _.each(datapoints, function(instance){
        if(currentInstance.instanceId == instance.instanceId){
          if(instance.cpu_average > avg_cpu_value){
            max_cpu_value = instance.cpu_maximum;
            min_cpu_value = instance.cpu_minimum;
            avg_cpu_value = instance.cpu_average;
          }
        }
      })
      currentInstance.cpu_maximum = max_cpu_value; currentInstance.cpu_minimum = min_cpu_value; currentInstance.cpu_average = avg_cpu_value;
      instances.push(currentInstance)
    })
  EOS
end

script "js_cloudwatch_mem_usage", type: "javascript" do
  result "results"
  parameters "region","instance_id","privateDnsName"
  code <<-EOS
  if (privateDnsName != null && privateDnsName !== undefined && privateDnsName.length != 0) {
    var hostname = privateDnsName.split('.')[0]
  } else {
    var hostname = "ip-127-0-0-1"
  }
  var end_date = new Date().toISOString()
  var start_date = new Date(new Date().setDate(new Date().getDate() - 30)).toISOString();
  results = {
    "auth": "auth_aws",
    "host": 'monitoring.'+region+'.amazonaws.com',
    "verb": "GET",
    "path": "/",
    "headers": {
      "User-Agent": "RS Policies",
      "Content-Type": "application/json",
      "x-amz-target": "GraniteServiceVersion20100801.GetMetricStatistics",
      "Accept": "application/json",
      "Content-Encoding": "amz-1.0"
    }
    "query_params": {
      'Action': 'GetMetricStatistics',
      'Version': '2010-08-01',
      'Namespace': 'CWAgent',
      'MetricName': 'mem_used_percent',
      'Dimensions.member.1.Name': 'host',
      'Dimensions.member.1.Value': hostname,
      'StartTime': start_date,
      'EndTime': end_date,
      'Period': "2592000",
      'Statistics.member.1': 'Maximum',
      'Statistics.member.2': 'Average',
      'Statistics.member.3': 'Minimum'
    }
  }
EOS
end

script "js_cloudwatch_memory_highest_usage_value", type: "javascript" do
  result "instances"
  parameters "datapoints"
  code <<-EOS
    var instances = [];
    var instanceIds = [];
    var distinctInstances = [];
    datapoints.forEach(function(datapoint){
      if(instanceIds.indexOf(datapoint.instanceId) === -1){
        instanceIds.push(datapoint.instanceId)
        distinctInstances.push(datapoint)
      }
    })
    _.each(distinctInstances, function(currentInstance){
      var max_mem_value = currentInstance.mem_maximum; var min_mem_value = currentInstance.mem_minimum; var avg_mem_value = currentInstance.mem_average;
      _.each(datapoints, function(instance){
        if(currentInstance.instanceId == instance.instanceId){
          if(instance.mem_average > avg_mem_value){
            max_mem_value = instance.mem_maximum;
            min_mem_value = instance.mem_minimum;
            avg_mem_value = instance.mem_average;
          }
        }
      })
      currentInstance.mem_maximum = max_mem_value; currentInstance.mem_minimum = min_mem_value; currentInstance.mem_average = avg_mem_value;
      instances.push(currentInstance)
    })
  EOS
end

script "js_cloudwatch_windows_memory_usage", type: "javascript" do
  result "results"
  parameters "region","instance_id", "image_id", "instance_type"
  code <<-EOS
  var end_date = new Date().toISOString()
  var start_date = new Date(new Date().setDate(new Date().getDate() - 30)).toISOString();
  results = {
    "auth": "auth_aws",
    "host": 'monitoring.'+region+'.amazonaws.com',
    "verb": "GET",
    "path": "/",
    "headers": {
      "User-Agent": "RS Policies",
      "Content-Type": "application/json",
      "x-amz-target": "GraniteServiceVersion20100801.GetMetricStatistics",
      "Accept": "application/json",
      "Content-Encoding": "amz-1.0"
    }
    "query_params": {
      'Action': 'GetMetricStatistics',
      'Version': '2010-08-01',
      'Namespace': 'CWAgent',
      'MetricName': 'Memory % Committed Bytes In Use',
      'Dimensions.member.1.Name': 'ImageId',
      'Dimensions.member.1.Value': image_id,
      'Dimensions.member.2.Name': 'InstanceId',
      'Dimensions.member.2.Value': instance_id,
      'Dimensions.member.3.Name': 'InstanceType',
      'Dimensions.member.3.Value': instance_type,
      'Dimensions.member.4.Name': 'objectname',
      'Dimensions.member.4.Value': 'Memory',
      'StartTime': start_date,
      'EndTime': end_date,
      'Period': "2592000",
      'Statistics.member.1': 'Maximum',
      'Statistics.member.2': 'Average',
      'Statistics.member.3': 'Minimum'
    }
  }
EOS
end

script "js_cloudwatch_windows_memory_highest_usage_value", type: "javascript" do
  result "instances"
  parameters "datapoints"
  code <<-EOS
    var instances = [];
    var instanceIds = [];
    var distinctInstances = [];
    datapoints.forEach(function(datapoint){
      if(instanceIds.indexOf(datapoint.instanceId) === -1){
        instanceIds.push(datapoint.instanceId)
        distinctInstances.push(datapoint)
      }
    })
    _.each(distinctInstances, function(currentInstance){
      var max_mem_value = currentInstance.mem_maximum; var min_mem_value = currentInstance.mem_minimum; var avg_mem_value = currentInstance.mem_average;
      _.each(datapoints, function(instance){
        if(currentInstance.instanceId == instance.instanceId){
          if(instance.mem_average > avg_mem_value){
            max_mem_value = instance.mem_maximum;
            min_mem_value = instance.mem_minimum;
            avg_mem_value = instance.mem_average;
          }
        }
      })
      currentInstance.mem_maximum = max_mem_value; currentInstance.mem_minimum = min_mem_value; currentInstance.mem_average = avg_mem_value;
      instances.push(currentInstance)
    })
  EOS
end

script "js_merged_metrics", type: "javascript" do
  parameters "ds_cloudwatch_cpu_usage", "ds_cloudwatch_mem_usage", "ds_cloudwatch_windows_mem_usage"
  result "merged_results"
  code <<-EOS
  var merged_results = []
  for ( i = 0; i < ds_cloudwatch_cpu_usage.length; i++){
    var instance = ds_cloudwatch_cpu_usage[i]
    var instance_id = instance.instanceId
    if (instance.platform == 'windows'){
      var mem_usage_record = _.find(ds_cloudwatch_windows_mem_usage, function(record) { return record.instanceId == instance_id; })

      if (mem_usage_record != null && mem_usage_record !== undefined) {
        var mem_maximum = parseFloat(mem_usage_record.mem_maximum).toFixed(2)
        var mem_average = parseFloat(mem_usage_record.mem_average).toFixed(2)
        var mem_minimum = parseFloat(mem_usage_record.mem_minimum).toFixed(2)
      } else {
        var mem_maximum = "101"
        var mem_average = "101"
        var mem_minimum = "101"
      }
    } else {
      instance.platform = 'linux'
      var mem_usage_record = _.find(ds_cloudwatch_mem_usage, function(record) { return record.instanceId == instance_id; })

      if (mem_usage_record != null && mem_usage_record !== undefined) {
        var mem_maximum = parseFloat(mem_usage_record.mem_maximum).toFixed(2)
        var mem_average = parseFloat(mem_usage_record.mem_average).toFixed(2)
        var mem_minimum = parseFloat(mem_usage_record.mem_minimum).toFixed(2)
      } else {
        var mem_maximum = "101"
        var mem_average = "101"
        var mem_minimum = "101"
      }
    }
    merged_results.push(
      {
        "region": instance.region,
        "instanceId": instance.instanceId,
        "platform": instance.platform,
        "privateDnsName": instance.privateDnsName,
        "hostname": instance.privateDnsName.split('.')[0],
        "tag_set": instance.tag_set,
        "cpu_minimum": parseFloat(instance.cpu_minimum).toFixed(2),
        "cpu_maximum": parseFloat(instance.cpu_maximum).toFixed(2),
        "cpu_average": parseFloat(instance.cpu_average).toFixed(2),
        "mem_maximum": mem_maximum,
        "mem_minimum": mem_minimum,
        "mem_average": mem_average,
        "instanceType": instance.instanceType
      }
    )
  }
EOS
end

script "js_inefficient_utilization_metrics", type: "javascript" do
  result "inefficient_utilization_results"
  parameters "ds_aws_instance_size_map", "ds_merged_metrics", "param_avg_used_memory", "param_avg_cpu", "ds_get_caller_identity"
  code <<-EOS
  var inefficient_utilization_results = []
  for ( i = 0; i < ds_merged_metrics.length; i++){
    var instance = ds_merged_metrics[i]
    var cpu_average = instance.cpu_average
    var mem_average = instance.mem_average
    var next_instance_size = "Unknown";
    if(ds_aws_instance_size_map[instance.instanceType] !== undefined && ds_aws_instance_size_map[instance.instanceType] != null){
      next_instance_size = ds_aws_instance_size_map[instance.instanceType]["down"]
    }
    if(next_instance_size == null && typeof next_instance_size == "undefined" ){
      next_instance_size="N/A";
    }
    var maximumMemoryField = instance.mem_maximum
    if(maximumMemoryField == "101" || maximumMemoryField === "101"){
      maximumMemoryField = "N/A"
    }
    var averageMemoryField = instance.mem_average
    if(averageMemoryField == "101" || averageMemoryField === "101"){
      averageMemoryField = "N/A"
    }
    if ((cpu_average < param_avg_cpu || mem_average < param_avg_used_memory)){
      inefficient_utilization_results.push({
        "accountId": ds_get_caller_identity[0]['account'],
        "region": instance.region,
        "id": instance.instanceId,
        "platform": instance.platform,
        "privateDnsName": instance.privateDnsName,
        "hostname": instance.privateDnsName.split('.')[0],
        "tag_set": instance.tag_set,
        "cpu_minimum": instance.cpu_minimum,
        "cpu_maximum": instance.cpu_maximum,
        "cpu_average": cpu_average,
        "mem_maximum": instance.mem_maximum,
        "mem_minimum": instance.mem_minimum,
        "mem_average": mem_average,
        "instanceType": instance.instanceType,
        "next_instance_size": next_instance_size,
        "maximumMemoryField": maximumMemoryField,
        "averageMemoryField":averageMemoryField
      })
    }
  }
  inefficient_utilization_results = _.sortBy(inefficient_utilization_results, 'region');
EOS
end

###############################################################################
# Policy
###############################################################################

policy "pol_utilization" do
  validate_each $ds_inefficient_utilization_metrics do
    summary_template "AWS Account ID: {{with index data 0}}{{ .accountId }}{{end}} - {{ len data }} AWS instances with inefficient utilization"
    escalate $email
    check logic_or(eq($param_avg_used_memory, -1), gt(to_n(val(item,"mem_average")), $param_avg_used_memory))
    check logic_or(eq($param_avg_cpu, -1), gt(to_n(val(item,"cpu_average")),$param_avg_cpu))
    escalate $esc_downsize_instances
    export do
      resource_level true
      field "accountId" do
        label "Account Id"
      end
      field "region" do
        label "Region"
      end
      field "id" do
        label "Instance Id"
      end
      field "instanceType" do
        label "Current Instance Type"
      end
      field "next_instance_size" do
        label "Next Instance Type"
      end
      field "platform" do
        label "platform"
      end
      field "hostname" do
        label "Hostname"
      end
      field "privateDnsName" do
        label "Private DNS Name"
      end
      field "cpu_maximum" do
        label "CPU Maximum %"
      end
      field "cpu_average" do
        label "CPU Average %"
      end
      field "maximumMemoryField" do
        label "Memory Maximum %"
      end
      field "averageMemoryField" do
        label "Memory Average %"
      end
    end
  end
end


###############################################################################
# Escalations
###############################################################################

escalation "email" do
  automatic true
  label "Send Email"
  description "Send incident email"
  email $param_email
end

escalation "esc_downsize_instances" do
  automatic contains($param_automatic_action, "Downsize Instances")
  label "Downsize Instances"
  description "Approval to downsize all selected instances"
  run "downsize_instances", data, $param_log_to_cm_audit_entries, rs_optima_host
end

###############################################################################
# Cloud Workflow
###############################################################################

define downsize_instances($data,$param_log_to_cm_audit_entries, $$rs_optima_host) return $all_responses do
  $$debug = $param_log_to_cm_audit_entries == "Yes"
  $status_code=''
  $go_next = true
  foreach $item in $data do
    if $item["next_instance_size"]!= "N/A"
      sub on_error: rollback($item) do
        call stop_instances($item) retrieve $status_code
        if(to_s($status_code)=="200")
          call set_instance_type($item) retrieve $status_code
        else
          call rollback($item)
          $go_next = false
        end
        if(to_s($status_code)=="200")
          call start_instances($item) retrieve $status_code
        elsif ($go_next)
          call rollback($item)
          $go_next = false
        end
        if($go_next && to_s($status_code)!="200")
          call rollback($item)
        end
      end
    end
  end
end


define stop_instances($item) return $status_code do
  $status_code={}
  $response={}
  sub on_error: retry, timeout: 20m, on_timeout: skip do
    $response = http_request(
      verb: "post",
      host: "ec2.amazonaws.com",
      auth: $$auth_aws,
      href: join(["/", "?Action=StopInstances", "&InstanceId.1=", $item["id"], "&Version=2016-11-15"]),
      https: true,
      headers:{
        "cache-control": "no-cache",
        "content-type": "application/json"
      }
    )
    call sys_log("StopInstanceResponseCode", to_s($response["code"]))
  end
  $status_code=$response["code"]
  if($response["code"]==200)
    $wake_condition = "stopped"
    $state = ''
    while $state !~ $wake_condition do
      sub on_error: skip do
        call get_instance($item) retrieve $status_response
        $status_responses=$status_response["body"]["DescribeInstancesResponse"]["reservationSet"]["item"]["instancesSet"]["item"]
        $state = to_s($status_responses["instanceState"]["name"])
        $status_code = to_s($status_response["code"])
      end
    end
  end
end

define set_instance_type($item) return $status_code do
  $response = {}
  sub on_error: retry, timeout: 20m, on_timeout: skip do
    $response = http_request(
      verb: "post",
      host: "ec2.amazonaws.com",
      auth: $$auth_aws,
      href: join(["/", "?Action=ModifyInstanceAttribute", "&InstanceId.1=", $item["id"], "&InstanceType.Value=", $item["next_instance_size"], "&Version=2016-11-15"]),
      https: true,
      headers: {
        "cache-control": "no-cache",
        "content-type": "application/json"
      }
    )
    call sys_log("ModifyInstanceResponseCode", to_s($response["code"]))
  end
  $status_code=$response["code"]
end

define start_instances($item) return $status_code do
  $response = {}
  sub on_error: retry, timeout: 20m, on_timeout: skip do
    $response = http_request(
      verb: "post",
      host: "ec2.amazonaws.com",
      auth: $$auth_aws,
      href: join(["/", "?Action=StartInstances", "&InstanceId.1=", $item["id"], "&Version=2016-11-15"]),
      https: true,
      headers: {
        "cache-control": "no-cache",
        "content-type": "application/json"
      }
    )
    call sys_log("StartInstanceResponseCode", to_s($response["code"]))
  end
  $status_code=$response["code"]
  if($response["code"]==200)
    $wake_condition = "running"
    $state = ''
    while $state !~ $wake_condition do
      sub on_error: skip do
        call get_instance($item) retrieve $status_response
        $status_responses=$status_response["body"]["DescribeInstancesResponse"]["reservationSet"]["item"]["instancesSet"]["item"]
        $state = to_s($status_responses["instanceState"]["name"])
        $status_code = $status_response["code"]
      end
    end
  end
end


define get_instance($item) return $status_response do
  $status_response = {}
  sub on_error: retry, timeout: 20m, on_timeout: skip do
    $response = http_request(
      verb: "get",
      host: "ec2.amazonaws.com",
      auth: $$auth_aws,
      href: join(["/", "?Action=DescribeInstances", "&InstanceId.1=", $item["id"], "&Version=2016-11-15"]),
      https: true,
      headers: {
        "cache-control": "no-cache",
        "Accept": "application/json"
      }
    )
    $status_response=$response
  end
end

define rollback($item) do
  call get_instance($item) retrieve $status_response
  $status_responses=$status_response["body"]["DescribeInstancesResponse"]["reservationSet"]["item"]["instancesSet"]["item"]
  call sys_log("Rollback_statusResponse", to_s($status_responses))
  $state = to_s($status_responses["instanceState"]["name"])
  $started_condition = "running"
  if include?(to_s($status_responses["instanceType"]), to_s($item["instanceType"]))
    # Instance is still the original size, just make sure its running
    if $state =~ $started_condition
      # instance is running nothing to do
    else
      # instance is not running, start it
      call start_instances($item) retrieve $start_response
    end
  else
    # Instance is the new size. Roll back
    if $state == $started_condition
      call stop_instances($item)
    end
    call rollback_instance_type($item)
    call start_instances($item)
  end
end

define rollback_instance_type($item) do
  sub on_error: skip do
      $response = http_request({
        verb: "post",
        host: "ec2.amazonaws.com",
        auth: $$auth_aws,
        href: join(["/", "?Action=ModifyInstanceAttribute", "&InstanceId.1=", $item["id"], "&InstanceType.Value=", $item["instanceType"], "&Version=2016-11-15"]),
        https: true,
        headers: {
          "cache-control": "no-cache",
          "content-type": "application/json"
        }
      })
  end
end

define sys_log($subject, $detail) do
  # Create empty errors array if doesn't already exist
  if !$$errors
    $$errors = []
  end
  # Check if debug is enabled
  if $$debug
    # Append to global $$errors
    # This is the suggested way to capture errors
    $$errors << "Unexpected error for " + $subject + "\n  " + to_s($detail)
    # If Flexera NAM Zone, create audit_entries [to be deprecated]
    # This is the legacy method for capturing errors and only supported on Flexera NAM
    if $$rs_optima_host == "api.optima.flexeraeng.com"
      # skip_error_and_append is used to catch error if rs_cm.audit_entries.create fails unexpectedly
      $task_label = "Creating audit entry for " + $subject
      sub task_label: $task, on_error: skip_error_and_append($task) do
        rs_cm.audit_entries.create(
          notify: "None",
          audit_entry: {
            auditee_href: @@account,
            summary: $subject,
            detail: $detail
          }
        )
      end # End sub on_error
    end # End if rs_optima_host
  end # End if debug is enabled
end

define skip_error_and_append($subject) do
  $$errors << "Unexpected error for " + $subject + "\n  " + to_s($_error)
  $_error_behavior = "skip"
end