Source code for awslimitchecker.services.ec2

"""
awslimitchecker/services/ec2.py

The latest version of this package is available at:
<https://github.com/jantman/awslimitchecker>

################################################################################
Copyright 2015-2017 Jason Antman <jason@jasonantman.com>

    This file is part of awslimitchecker, also known as awslimitchecker.

    awslimitchecker is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    awslimitchecker is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with awslimitchecker.  If not, see <http://www.gnu.org/licenses/>.

The Copyright and Authors attributions contained herein may not be removed or
otherwise altered, except to add the Author attribution of a contributor to
this work. (Additional Terms pursuant to Section 7b of the AGPL v3)
################################################################################
While not legally required, I sincerely request that anyone who finds
bugs please submit them at <https://github.com/jantman/awslimitchecker> or
to me via email, and that you send any contributions or improvements
either as a pull request on GitHub, or to me via email.
################################################################################

AUTHORS:
Jason Antman <jason@jasonantman.com> <http://www.jasonantman.com>
################################################################################
"""

import abc  # noqa
import logging
from collections import defaultdict
from copy import deepcopy

import botocore

from .base import _AwsService
from ..limit import AwsLimit

logger = logging.getLogger(__name__)

RI_NO_AZ = 'xxREGIONAL_BENEFIT-NO_AZxx'


[docs]class _Ec2Service(_AwsService): service_name = 'EC2' api_name = 'ec2'
[docs] def find_usage(self): """ Determine the current usage for each limit of this service, and update corresponding Limit via :py:meth:`~.AwsLimit._add_current_usage`. """ logger.debug("Checking usage for service %s", self.service_name) self.connect() self.connect_resource() for lim in self.limits.values(): lim._reset_usage() self._find_usage_instances() self._find_usage_networking_sgs() self._find_usage_networking_eips() self._find_usage_networking_eni_sg() self._find_usage_spot_instances() self._find_usage_spot_fleets() self._have_usage = True logger.debug("Done checking usage.")
[docs] def _find_usage_instances(self): """calculate On-Demand instance usage for all types and update Limits""" # update our limits with usage inst_usage = self._instance_usage() res_usage = self._get_reserved_instance_count() logger.debug('Reserved instance count: %s', res_usage) # subtract reservations from instance usage ondemand_usage = defaultdict(int) for az in inst_usage: if az not in res_usage: for i_type, count in inst_usage[az].items(): ondemand_usage[i_type] += count continue # else we have reservations for this AZ for i_type, count in inst_usage[az].items(): if i_type not in res_usage[az]: # no reservations for this type ondemand_usage[i_type] += count continue od = count - res_usage[az][i_type] if od < 1: # we have unused reservations continue ondemand_usage[i_type] += od # subtract any "Regional Benefit" AZ-less reservations for i_type in ondemand_usage.keys(): if RI_NO_AZ in res_usage and i_type in res_usage[RI_NO_AZ]: logger.debug('Subtracting %d AZ-less "Regional Benefit" ' 'Reserved Instances from %d running %s instances', res_usage[RI_NO_AZ][i_type], ondemand_usage[i_type], i_type) # we have Regional Benefit reservations for this type; # we don't want to show negative usage, even if we're # over-reserved ondemand_usage[i_type] = max( ondemand_usage[i_type] - res_usage[RI_NO_AZ][i_type], 0 ) total_instances = 0 for i_type, usage in ondemand_usage.items(): key = 'Running On-Demand {t} instances'.format( t=i_type) self.limits[key]._add_current_usage( usage, aws_type='AWS::EC2::Instance', ) total_instances += usage # limit for ALL On-Demand EC2 instances key = 'Running On-Demand EC2 instances' self.limits[key]._add_current_usage( total_instances, aws_type='AWS::EC2::Instance' )
[docs] def _find_usage_spot_instances(self): """calculate spot instance request usage and update Limits""" logger.debug('Getting spot instance request usage') try: res = self.conn.describe_spot_instance_requests() except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'UnsupportedOperation': return raise count = 0 for req in res['SpotInstanceRequests']: if req['State'] in ['open', 'active']: count += 1 logger.debug('Counting spot instance request %s state=%s', req['SpotInstanceRequestId'], req['State']) else: logger.debug('NOT counting spot instance request %s state=%s', req['SpotInstanceRequestId'], req['State']) self.limits['Max spot instance requests per region']._add_current_usage( count, aws_type='AWS::EC2::SpotInstanceRequest' )
[docs] def _find_usage_spot_fleets(self): """calculate spot fleet request usage and update Limits""" logger.debug('Getting spot fleet request usage') try: res = self.conn.describe_spot_fleet_requests() except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'UnsupportedOperation': return raise if 'NextToken' in res: logger.error('Error: describe_spot_fleet_requests() response ' 'includes pagination token, but pagination not ' 'configured in awslimitchecker.') active_fleets = 0 total_target_cap = 0 lim_cap_per_fleet = self.limits['Max target capacity per spot fleet'] lim_launch_specs = self.limits[ 'Max launch specifications per spot fleet'] for fleet in res['SpotFleetRequestConfigs']: _id = fleet['SpotFleetRequestId'] if fleet['SpotFleetRequestState'] != 'active': logger.debug('Skipping spot fleet request %s in state %s', _id, fleet['SpotFleetRequestState']) continue active_fleets += 1 cap = fleet['SpotFleetRequestConfig']['TargetCapacity'] launch_specs = len( fleet['SpotFleetRequestConfig']['LaunchSpecifications']) total_target_cap += cap lim_cap_per_fleet._add_current_usage( cap, resource_id=_id, aws_type='AWS::EC2::SpotFleetRequest') lim_launch_specs._add_current_usage( launch_specs, resource_id=_id, aws_type='AWS::EC2::SpotFleetRequest') self.limits['Max active spot fleets per region']._add_current_usage( active_fleets, aws_type='AWS::EC2::SpotFleetRequest' ) self.limits['Max target capacity for all spot ' 'fleets in region']._add_current_usage( total_target_cap, aws_type='AWS::EC2::SpotFleetRequest' )
[docs] def _get_reserved_instance_count(self): """ For each availability zone, get the count of current instance reservations of each instance type. Return as a nested dict of AZ name to dict of instance type to reservation count. :rtype: dict """ reservations = defaultdict(int) az_to_res = {} logger.debug("Getting reserved instance information") res = self.conn.describe_reserved_instances() for x in res['ReservedInstances']: if x['State'] != 'active': logger.debug("Skipping ReservedInstance %s with state %s", x['ReservedInstancesId'], x['State']) continue if 'AvailabilityZone' not in x: # "Regional Benefit" AZ-less reservation x['AvailabilityZone'] = RI_NO_AZ if x['AvailabilityZone'] not in az_to_res: az_to_res[x['AvailabilityZone']] = deepcopy(reservations) az_to_res[x['AvailabilityZone']][ x['InstanceType']] += x['InstanceCount'] # flatten and return for x in az_to_res: az_to_res[x] = dict(az_to_res[x]) return az_to_res
[docs] def _instance_usage(self): """ Find counts of currently-running EC2 Instances (On-Demand or Reserved) by placement (Availability Zone) and instance type (size). Return as a nested dict of AZ name to dict of instance type to count. :rtype: dict """ # On-Demand instances by type ondemand = {} for t in self._instance_types(): ondemand[t] = 0 az_to_inst = {} logger.debug("Getting usage for on-demand instances") for inst in self.resource_conn.instances.all(): if inst.spot_instance_request_id: logger.info("Spot instance found (%s); skipping from " "Running On-Demand Instances count", inst.id) continue if inst.state['Name'] in ['stopped', 'terminated']: logger.debug("Ignoring instance %s in state %s", inst.id, inst.state['Name']) continue if inst.placement['AvailabilityZone'] not in az_to_inst: az_to_inst[ inst.placement['AvailabilityZone']] = deepcopy(ondemand) try: az_to_inst[ inst.placement['AvailabilityZone']][inst.instance_type] += 1 except KeyError: logger.error("ERROR - unknown instance type '%s'; not " "counting", inst.instance_type) return az_to_inst
[docs] def get_limits(self): """ Return all known limits for this service, as a dict of their names to :py:class:`~.AwsLimit` objects. :returns: dict of limit names to :py:class:`~.AwsLimit` objects :rtype: dict """ if self.limits != {}: return self.limits limits = {} limits.update(self._get_limits_instances()) limits.update(self._get_limits_networking()) limits.update(self._get_limits_spot()) self.limits = limits return self.limits
[docs] def _update_limits_from_api(self): """ Query EC2's DescribeAccountAttributes API action, and update limits with the quotas returned. Updates ``self.limits``. """ self.connect() self.connect_resource() logger.info("Querying EC2 DescribeAccountAttributes for limits") # no need to paginate attribs = self.conn.describe_account_attributes() for attrib in attribs['AccountAttributes']: aname = attrib['AttributeName'] val = attrib['AttributeValues'][0]['AttributeValue'] lname = None if aname == 'max-elastic-ips': lname = 'Elastic IP addresses (EIPs)' elif aname == 'max-instances': lname = 'Running On-Demand EC2 instances' elif aname == 'vpc-max-elastic-ips': lname = 'VPC Elastic IP addresses (EIPs)' elif aname == 'vpc-max-security-groups-per-interface': lname = 'VPC security groups per elastic network interface' if lname is not None: if int(val) == 0: continue self.limits[lname]._set_api_limit(int(val)) logger.debug("Done setting limits from API")
[docs] def _get_limits_instances(self): """ Return a dict of limits for EC2 instances only. This method should only be used internally by :py:meth:~.get_limits`. :rtype: dict """ # from: http://aws.amazon.com/ec2/faqs/ # (On-Demand, Reserved, Spot) default_limits = (20, 20, 5) special_limits = { 'c4.4xlarge': (10, 20, 5), 'c4.8xlarge': (5, 20, 5), 'cg1.4xlarge': (2, 20, 5), 'cr1.8xlarge': (2, 20, 5), 'd2.4xlarge': (10, 20, 5), 'd2.8xlarge': (5, 20, 5), 'g2.2xlarge': (5, 20, 5), 'g2.8xlarge': (2, 20, 5), 'hi1.4xlarge': (2, 20, 5), 'hs1.8xlarge': (2, 20, 0), 'i2.xlarge': (8, 20, 0), 'i2.2xlarge': (8, 20, 0), 'i2.4xlarge': (4, 20, 0), 'i2.8xlarge': (2, 20, 0), 'i3.large': (2, 20, 0), 'i3.xlarge': (2, 20, 0), 'i3.2xlarge': (2, 20, 0), 'i3.4xlarge': (2, 20, 0), 'i3.8xlarge': (2, 20, 0), 'i3.16xlarge': (2, 20, 0), 'm4.4xlarge': (10, 20, 5), 'm4.10xlarge': (5, 20, 5), 'm4.16xlarge': (5, 20, 5), 'p2.xlarge': (1, 20, 5), 'p2.8xlarge': (1, 20, 5), 'p2.16xlarge': (1, 20, 5), 'r3.4xlarge': (10, 20, 5), 'r3.8xlarge': (5, 20, 5), } limits = {} for i_type in self._instance_types(): key = 'Running On-Demand {t} instances'.format( t=i_type) lim = default_limits[0] if i_type in special_limits: lim = special_limits[i_type][0] limits[key] = AwsLimit( key, self, lim, self.warning_threshold, self.critical_threshold, limit_type='On-Demand instances', limit_subtype=i_type, ta_limit_name='On-Demand instances - %s' % i_type ) # limit for ALL running On-Demand instances key = 'Running On-Demand EC2 instances' limits[key] = AwsLimit( key, self, default_limits[0], self.warning_threshold, self.critical_threshold, limit_type='On-Demand instances', ) return limits
[docs] def _get_limits_spot(self): """ Return a dict of limits for spot requests only. This method should only be used internally by :py:meth:~.get_limits`. :rtype: dict """ limits = {} limits['Max spot instance requests per region'] = AwsLimit( 'Max spot instance requests per region', self, 20, self.warning_threshold, self.critical_threshold, limit_type='Spot instance requests' ) limits['Max active spot fleets per region'] = AwsLimit( 'Max active spot fleets per region', self, 1000, self.warning_threshold, self.critical_threshold, ) limits['Max launch specifications per spot fleet'] = AwsLimit( 'Max launch specifications per spot fleet', self, 50, self.warning_threshold, self.critical_threshold, ) limits['Max target capacity per spot fleet'] = AwsLimit( 'Max target capacity per spot fleet', self, 3000, self.warning_threshold, self.critical_threshold ) limits['Max target capacity for all spot fleets in region'] = AwsLimit( 'Max target capacity for all spot fleets in region', self, 5000, self.warning_threshold, self.critical_threshold ) return limits
[docs] def _find_usage_networking_sgs(self): """calculate usage for VPC-related things""" logger.debug("Getting usage for EC2 VPC resources") sgs_per_vpc = defaultdict(int) rules_per_sg = defaultdict(int) for sg in self.resource_conn.security_groups.all(): if sg.vpc_id is not None: sgs_per_vpc[sg.vpc_id] += 1 rules_per_sg[sg.id] = len(sg.ip_permissions) # set usage for vpc_id, count in sgs_per_vpc.items(): self.limits['Security groups per VPC']._add_current_usage( count, aws_type='AWS::EC2::VPC', resource_id=vpc_id, ) for sg_id, count in rules_per_sg.items(): self.limits['Rules per VPC security group']._add_current_usage( count, aws_type='AWS::EC2::SecurityGroupRule', resource_id=sg_id, )
[docs] def _find_usage_networking_eips(self): logger.debug("Getting usage for EC2 EIPs") vpc_addrs = self.resource_conn.vpc_addresses.all() self.limits['VPC Elastic IP addresses (EIPs)']._add_current_usage( sum(1 for a in vpc_addrs if a.domain == 'vpc'), aws_type='AWS::EC2::EIP', ) # the EC2 limits screen calls this 'EC2-Classic Elastic IPs' # but Trusted Advisor just calls it 'Elastic IP addresses (EIPs)' classic_addrs = self.resource_conn.classic_addresses.all() self.limits['Elastic IP addresses (EIPs)']._add_current_usage( sum(1 for a in classic_addrs if a.domain == 'standard'), aws_type='AWS::EC2::EIP', )
[docs] def _find_usage_networking_eni_sg(self): logger.debug("Getting usage for EC2 Network Interfaces") ints = self.resource_conn.network_interfaces.all() for iface in ints: if iface.vpc is None: continue self.limits['VPC security groups per elastic network ' 'interface']._add_current_usage( len(iface.groups), aws_type='AWS::EC2::NetworkInterface', resource_id=iface.id, )
[docs] def _get_limits_networking(self): """ Return a dict of VPC-related limits only. This method should only be used internally by :py:meth:~.get_limits`. :rtype: dict """ limits = {} limits['Security groups per VPC'] = AwsLimit( 'Security groups per VPC', self, 500, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::SecurityGroup', limit_subtype='AWS::EC2::VPC', ) limits['Rules per VPC security group'] = AwsLimit( 'Rules per VPC security group', self, 50, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::SecurityGroup', limit_subtype='AWS::EC2::VPC', ) limits['VPC Elastic IP addresses (EIPs)'] = AwsLimit( 'VPC Elastic IP addresses (EIPs)', self, 5, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::EIP', limit_subtype='AWS::EC2::VPC', ta_service_name='VPC' # TA shows this as VPC not EC2 ) # the EC2 limits screen calls this 'EC2-Classic Elastic IPs' # but Trusted Advisor just calls it 'Elastic IP addresses (EIPs)' limits['Elastic IP addresses (EIPs)'] = AwsLimit( 'Elastic IP addresses (EIPs)', self, 5, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::EIP', ) limits['VPC security groups per elastic network interface'] = AwsLimit( 'VPC security groups per elastic network interface', self, 5, self.warning_threshold, self.critical_threshold, limit_type='AWS::EC2::SecurityGroup', limit_subtype='AWS::EC2::NetworkInterface', ) return limits
[docs] def required_iam_permissions(self): """ Return a list of IAM Actions required for this Service to function properly. All Actions will be shown with an Effect of "Allow" and a Resource of "*". :returns: list of IAM Action strings :rtype: list """ return [ "ec2:DescribeAccountAttributes", "ec2:DescribeAddresses", "ec2:DescribeInstances", "ec2:DescribeInternetGateways", "ec2:DescribeNetworkAcls", "ec2:DescribeNetworkInterfaces", "ec2:DescribeReservedInstances", "ec2:DescribeRouteTables", "ec2:DescribeSecurityGroups", "ec2:DescribeSnapshots", "ec2:DescribeSpotDatafeedSubscription", "ec2:DescribeSpotFleetInstances", "ec2:DescribeSpotFleetRequestHistory", "ec2:DescribeSpotFleetRequests", "ec2:DescribeSpotInstanceRequests", "ec2:DescribeSpotPriceHistory", "ec2:DescribeSubnets", "ec2:DescribeVolumes", "ec2:DescribeVpcs", ]
[docs] def _instance_types(self): """ Return a list of all known EC2 instance types :returns: list of all valid known EC2 instance types :rtype: list """ GENERAL_TYPES = [ 't2.nano', 't2.micro', 't2.small', 't2.medium', 't2.large', 't2.xlarge', 't2.2xlarge', 'm3.medium', 'm3.large', 'm3.xlarge', 'm3.2xlarge', 'm4.large', 'm4.xlarge', 'm4.2xlarge', 'm4.4xlarge', 'm4.10xlarge', 'm4.16xlarge' ] PREV_GENERAL_TYPES = [ 't1.micro', 'm1.small', 'm1.medium', 'm1.large', 'm1.xlarge', ] MEMORY_TYPES = [ 'r3.large', 'r3.xlarge', 'r3.2xlarge', 'r3.4xlarge', 'r3.8xlarge', 'r4.large', 'r4.xlarge', 'r4.2xlarge', 'r4.4xlarge', 'r4.8xlarge', 'r4.16xlarge', 'x1.16xlarge', 'x1.32xlarge' ] PREV_MEMORY_TYPES = [ 'm2.xlarge', 'm2.2xlarge', 'm2.4xlarge', 'cr1.8xlarge', ] COMPUTE_TYPES = [ 'c3.large', 'c3.xlarge', 'c3.2xlarge', 'c3.4xlarge', 'c3.8xlarge', 'c4.large', 'c4.xlarge', 'c4.2xlarge', 'c4.4xlarge', 'c4.8xlarge', ] PREV_COMPUTE_TYPES = [ 'c1.medium', 'c1.xlarge', 'cc2.8xlarge', ] ACCELERATED_COMPUTE_TYPES = [ 'p2.xlarge', 'p2.8xlarge', 'p2.16xlarge' ] STORAGE_TYPES = [ 'i2.xlarge', 'i2.2xlarge', 'i2.4xlarge', 'i2.8xlarge', 'i3.large', 'i3.xlarge', 'i3.2xlarge', 'i3.4xlarge', 'i3.8xlarge', 'i3.16xlarge', ] PREV_STORAGE_TYPES = [ 'hi1.4xlarge', 'hs1.8xlarge', ] DENSE_STORAGE_TYPES = [ 'd2.xlarge', 'd2.2xlarge', 'd2.4xlarge', 'd2.8xlarge', ] GPU_TYPES = [ 'g2.2xlarge', 'g2.8xlarge', ] PREV_GPU_TYPES = [ 'cg1.4xlarge', ] FPGA_TYPES = [ # note, as of 2016-12-17, these are still in Developer Preview; # there isn't a published instance limit yet, so we'll assume # it's the default... 'f1.2xlarge', 'f1.16xlarge' ] return ( GENERAL_TYPES + PREV_GENERAL_TYPES + MEMORY_TYPES + PREV_MEMORY_TYPES + COMPUTE_TYPES + PREV_COMPUTE_TYPES + ACCELERATED_COMPUTE_TYPES + STORAGE_TYPES + PREV_STORAGE_TYPES + DENSE_STORAGE_TYPES + GPU_TYPES + PREV_GPU_TYPES + FPGA_TYPES )