"""
awslimitchecker/services/ec2.py
The latest version of this package is available at:
<https://github.com/jantman/awslimitchecker>
################################################################################
Copyright 2015 Jason Antman <jason@jasonantman.com> <http://www.jasonantman.com>
This file is part of awslimitchecker, also known as awslimitchecker.
awslimitchecker is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
awslimitchecker is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with awslimitchecker. If not, see <http://www.gnu.org/licenses/>.
The Copyright and Authors attributions contained herein may not be removed or
otherwise altered, except to add the Author attribution of a contributor to
this work. (Additional Terms pursuant to Section 7b of the AGPL v3)
################################################################################
While not legally required, I sincerely request that anyone who finds
bugs please submit them at <https://github.com/jantman/awslimitchecker> or
to me via email, and that you send any contributions or improvements
either as a pull request on GitHub, or to me via email.
################################################################################
AUTHORS:
Jason Antman <jason@jasonantman.com> <http://www.jasonantman.com>
################################################################################
"""
import abc # noqa
import boto
import logging
from collections import defaultdict
from copy import deepcopy
from .base import _AwsService
from ..limit import AwsLimit
from ..utils import boto_query_wrapper
logger = logging.getLogger(__name__)
[docs]class _Ec2Service(_AwsService):
service_name = 'EC2'
[docs] def connect(self):
"""Connect to API if not already connected; set self.conn."""
if self.conn is not None:
return
elif self.region:
self.conn = self.connect_via(boto.ec2.connect_to_region)
else:
self.conn = boto.connect_ec2()
[docs] def find_usage(self):
"""
Determine the current usage for each limit of this service,
and update corresponding Limit via
:py:meth:`~.AwsLimit._add_current_usage`.
"""
logger.debug("Checking usage for service %s", self.service_name)
self.connect()
for lim in self.limits.values():
lim._reset_usage()
self._find_usage_instances()
self._find_usage_networking_sgs()
self._find_usage_networking_eips()
self._find_usage_networking_eni_sg()
self._have_usage = True
logger.debug("Done checking usage.")
[docs] def _find_usage_instances(self):
"""calculate On-Demand instance usage for all types and update Limits"""
# update our limits with usage
inst_usage = self._instance_usage()
res_usage = self._get_reserved_instance_count()
# subtract reservations from instance usage
ondemand_usage = defaultdict(int)
for az in inst_usage:
if az not in res_usage:
for i_type, count in inst_usage[az].items():
ondemand_usage[i_type] += count
continue
# else we have reservations for this AZ
for i_type, count in inst_usage[az].items():
if i_type not in res_usage[az]:
# no reservations for this type
ondemand_usage[i_type] += count
continue
od = count - res_usage[az][i_type]
if od < 1:
# we have unused reservations
continue
ondemand_usage[i_type] += od
total_instances = 0
for i_type, usage in ondemand_usage.items():
key = 'Running On-Demand {t} instances'.format(
t=i_type)
self.limits[key]._add_current_usage(
usage,
aws_type='AWS::EC2::Instance',
)
total_instances += usage
# limit for ALL On-Demand EC2 instances
key = 'Running On-Demand EC2 instances'
self.limits[key]._add_current_usage(
total_instances,
aws_type='AWS::EC2::Instance'
)
[docs] def _get_reserved_instance_count(self):
"""
For each availability zone, get the count of current instance
reservations of each instance type. Return as a nested
dict of AZ name to dict of instance type to reservation count.
:rtype: dict
"""
reservations = defaultdict(int)
az_to_res = {}
logger.debug("Getting reserved instance information")
res = boto_query_wrapper(self.conn.get_all_reserved_instances)
for x in res:
if x.state != 'active':
logger.debug("Skipping ReservedInstance %s with state %s",
x.id, x.state)
continue
if x.availability_zone not in az_to_res:
az_to_res[x.availability_zone] = deepcopy(reservations)
az_to_res[x.availability_zone][x.instance_type] += x.instance_count
# flatten and return
for x in az_to_res:
az_to_res[x] = dict(az_to_res[x])
return az_to_res
[docs] def _instance_usage(self):
"""
Find counts of currently-running EC2 Instances
(On-Demand or Reserved) by placement (Availability
Zone) and instance type (size). Return as a nested dict
of AZ name to dict of instance type to count.
:rtype: dict
"""
# On-Demand instances by type
ondemand = {}
for t in self._instance_types():
ondemand[t] = 0
az_to_inst = {}
logger.debug("Getting usage for on-demand instances")
for res in boto_query_wrapper(self.conn.get_all_reservations):
for inst in res.instances:
if inst.spot_instance_request_id:
logger.warning("Spot instance found (%s); awslimitchecker "
"does not yet support spot "
"instances.", inst.id)
continue
if inst.placement not in az_to_inst:
az_to_inst[inst.placement] = deepcopy(ondemand)
try:
az_to_inst[inst.placement][inst.instance_type] += 1
except KeyError:
logger.error("ERROR - unknown instance type '%s'; not "
"counting", inst.instance_type)
return az_to_inst
[docs] def get_limits(self):
"""
Return all known limits for this service, as a dict of their names
to :py:class:`~.AwsLimit` objects.
:returns: dict of limit names to :py:class:`~.AwsLimit` objects
:rtype: dict
"""
if self.limits != {}:
return self.limits
limits = {}
limits.update(self._get_limits_instances())
limits.update(self._get_limits_networking())
self.limits = limits
return self.limits
[docs] def _update_limits_from_api(self):
"""
Query EC2's DescribeAccountAttributes API action, and update limits
with the quotas returned. Updates ``self.limits``.
"""
self.connect()
logger.info("Querying EC2 DescribeAccountAttributes for limits")
attribs = boto_query_wrapper(self.conn.describe_account_attributes)
for attrib in attribs:
aname = attrib.attribute_name
val = attrib.attribute_values[0]
lname = None
if aname == 'max-elastic-ips':
lname = 'Elastic IP addresses (EIPs)'
elif aname == 'max-instances':
lname = 'Running On-Demand EC2 instances'
elif aname == 'vpc-max-elastic-ips':
lname = 'VPC Elastic IP addresses (EIPs)'
elif aname == 'vpc-max-security-groups-per-interface':
lname = 'VPC security groups per elastic network interface'
if lname is not None:
self.limits[lname]._set_api_limit(int(val))
logger.debug("Done setting limits from API")
[docs] def _get_limits_instances(self):
"""
Return a dict of limits for EC2 instances only.
This method should only be used internally by
:py:meth:~.get_limits`.
:rtype: dict
"""
# from: http://aws.amazon.com/ec2/faqs/
# (On-Demand, Reserved, Spot)
default_limits = (20, 20, 5)
special_limits = {
'c4.4xlarge': (10, 20, 5),
'c4.8xlarge': (5, 20, 5),
'cg1.4xlarge': (2, 20, 5),
'hi1.4xlarge': (2, 20, 5),
'hs1.8xlarge': (2, 20, 0),
'cr1.8xlarge': (2, 20, 5),
'g2.2xlarge': (5, 20, 5),
'g2.8xlarge': (2, 20, 5),
'r3.4xlarge': (10, 20, 5),
'r3.8xlarge': (5, 20, 5),
'i2.xlarge': (8, 20, 0),
'i2.2xlarge': (8, 20, 0),
'i2.4xlarge': (4, 20, 0),
'i2.8xlarge': (2, 20, 0),
'd2.4xlarge': (10, 20, 5),
'd2.8xlarge': (5, 20, 5),
}
limits = {}
for i_type in self._instance_types():
key = 'Running On-Demand {t} instances'.format(
t=i_type)
lim = default_limits[0]
if i_type in special_limits:
lim = special_limits[i_type][0]
limits[key] = AwsLimit(
key,
self,
lim,
self.warning_threshold,
self.critical_threshold,
limit_type='On-Demand instances',
limit_subtype=i_type
)
# limit for ALL running On-Demand instances
key = 'Running On-Demand EC2 instances'
limits[key] = AwsLimit(
key,
self,
default_limits[0],
self.warning_threshold,
self.critical_threshold,
limit_type='On-Demand instances',
)
return limits
[docs] def _find_usage_networking_sgs(self):
"""calculate usage for VPC-related things"""
logger.debug("Getting usage for EC2 VPC resources")
sgs_per_vpc = defaultdict(int)
rules_per_sg = defaultdict(int)
for sg in boto_query_wrapper(self.conn.get_all_security_groups):
if sg.vpc_id is not None:
sgs_per_vpc[sg.vpc_id] += 1
rules_per_sg[sg.id] = len(sg.rules)
# set usage
for vpc_id, count in sgs_per_vpc.items():
self.limits['Security groups per VPC']._add_current_usage(
count,
aws_type='AWS::EC2::VPC',
resource_id=vpc_id,
)
for sg_id, count in rules_per_sg.items():
self.limits['Rules per VPC security group']._add_current_usage(
count,
aws_type='AWS::EC2::SecurityGroupRule',
resource_id=sg_id,
)
[docs] def _find_usage_networking_eips(self):
logger.debug("Getting usage for EC2 EIPs")
addrs = boto_query_wrapper(self.conn.get_all_addresses)
self.limits['VPC Elastic IP addresses (EIPs)']._add_current_usage(
sum(1 for a in addrs if a.domain == 'vpc'),
aws_type='AWS::EC2::EIP',
)
# the EC2 limits screen calls this 'EC2-Classic Elastic IPs'
# but Trusted Advisor just calls it 'Elastic IP addresses (EIPs)'
self.limits['Elastic IP addresses (EIPs)']._add_current_usage(
sum(1 for a in addrs if a.domain == 'standard'),
aws_type='AWS::EC2::EIP',
)
[docs] def _find_usage_networking_eni_sg(self):
logger.debug("Getting usage for EC2 Network Interfaces")
ints = boto_query_wrapper(self.conn.get_all_network_interfaces)
for iface in ints:
self.limits['VPC security groups per elastic network '
'interface']._add_current_usage(
len(iface.groups),
aws_type='AWS::EC2::NetworkInterface',
resource_id=iface.id,
)
[docs] def _get_limits_networking(self):
"""
Return a dict of VPC-related limits only.
This method should only be used internally by
:py:meth:~.get_limits`.
:rtype: dict
"""
limits = {}
limits['Security groups per VPC'] = AwsLimit(
'Security groups per VPC',
self,
100,
self.warning_threshold,
self.critical_threshold,
limit_type='AWS::EC2::SecurityGroup',
limit_subtype='AWS::EC2::VPC',
)
limits['Rules per VPC security group'] = AwsLimit(
'Rules per VPC security group',
self,
50,
self.warning_threshold,
self.critical_threshold,
limit_type='AWS::EC2::SecurityGroup',
limit_subtype='AWS::EC2::VPC',
)
limits['VPC Elastic IP addresses (EIPs)'] = AwsLimit(
'VPC Elastic IP addresses (EIPs)',
self,
5,
self.warning_threshold,
self.critical_threshold,
limit_type='AWS::EC2::EIP',
limit_subtype='AWS::EC2::VPC',
)
# the EC2 limits screen calls this 'EC2-Classic Elastic IPs'
# but Trusted Advisor just calls it 'Elastic IP addresses (EIPs)'
limits['Elastic IP addresses (EIPs)'] = AwsLimit(
'Elastic IP addresses (EIPs)',
self,
5,
self.warning_threshold,
self.critical_threshold,
limit_type='AWS::EC2::EIP',
)
limits['VPC security groups per elastic network interface'] = AwsLimit(
'VPC security groups per elastic network interface',
self,
5,
self.warning_threshold,
self.critical_threshold,
limit_type='AWS::EC2::SecurityGroup',
limit_subtype='AWS::EC2::NetworkInterface',
)
return limits
[docs] def required_iam_permissions(self):
"""
Return a list of IAM Actions required for this Service to function
properly. All Actions will be shown with an Effect of "Allow"
and a Resource of "*".
:returns: list of IAM Action strings
:rtype: list
"""
return [
"ec2:DescribeAccountAttributes",
"ec2:DescribeAddresses",
"ec2:DescribeInstances",
"ec2:DescribeInternetGateways",
"ec2:DescribeNetworkAcls",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribeReservedInstances",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSnapshots",
"ec2:DescribeSubnets",
"ec2:DescribeVolumes",
"ec2:DescribeVpcs",
]
[docs] def _instance_types(self):
"""
Return a list of all known EC2 instance types
:returns: list of all valid known EC2 instance types
:rtype: list
"""
GENERAL_TYPES = [
't2.micro',
't2.small',
't2.medium',
't2.large',
'm3.medium',
'm3.large',
'm3.xlarge',
'm3.2xlarge',
'm4.large',
'm4.xlarge',
'm4.2xlarge',
'm4.4xlarge',
'm4.8xlarge',
]
PREV_GENERAL_TYPES = [
't1.micro',
'm1.small',
'm1.medium',
'm1.large',
'm1.xlarge',
]
MEMORY_TYPES = [
'r3.large',
'r3.xlarge',
'r3.2xlarge',
'r3.4xlarge',
'r3.8xlarge',
]
PREV_MEMORY_TYPES = [
'm2.xlarge',
'm2.2xlarge',
'm2.4xlarge',
'cr1.8xlarge',
]
COMPUTE_TYPES = [
'c3.large',
'c3.xlarge',
'c3.2xlarge',
'c3.4xlarge',
'c3.8xlarge',
'c4.large',
'c4.xlarge',
'c4.2xlarge',
'c4.4xlarge',
'c4.8xlarge',
]
PREV_COMPUTE_TYPES = [
'c1.medium',
'c1.xlarge',
'cc2.8xlarge',
]
STORAGE_TYPES = [
'i2.xlarge',
'i2.2xlarge',
'i2.4xlarge',
'i2.8xlarge',
]
PREV_STORAGE_TYPES = [
'hi1.4xlarge',
'hs1.8xlarge',
]
DENSE_STORAGE_TYPES = [
'd2.xlarge',
'd2.2xlarge',
'd2.4xlarge',
'd2.8xlarge',
]
GPU_TYPES = [
'g2.2xlarge',
'g2.8xlarge',
]
PREV_GPU_TYPES = [
'cg1.4xlarge',
]
return (
GENERAL_TYPES +
PREV_GENERAL_TYPES +
MEMORY_TYPES +
PREV_MEMORY_TYPES +
COMPUTE_TYPES +
PREV_COMPUTE_TYPES +
STORAGE_TYPES +
PREV_STORAGE_TYPES +
DENSE_STORAGE_TYPES +
GPU_TYPES +
PREV_GPU_TYPES
)