Commit 63ab49b9 authored by Nagarjuna D N's avatar Nagarjuna D N
Browse files

fixes

parent d336a2d7
This AMI Backup code has refered with a small enhancement:https://gist.github.com/bkozora/724e01903a9ad481d21e
This AMI Cleanup code has refered with a small enhancement:https://gist.github.com/bkozora/d4f1cf0e5cf26acdd377
# Automated AMI Backups
• This script will first search for all ec2 instances having a tag with "Backup” on it and the instances in "Running" state.
• As soon as it has the instances list, it loops through each instance and then creates an AMI of it.
• Also, it will look for a "Retention" tag key which will be used as a retention policy number in days. If there is no tag with that name, it will use a 7 days default value for each AMI.
• After creating the AMI it creates a "DeleteOn" tag on the AMI indicating when it will be deleted using the Retention value and another Lambda function.
# Automated AMI and Snapshot Deletion
• This script first searches for all ec2 instances having a tag with "Backup” on it and the instances in "Running" state.
• As soon as it has the instances list, it loops through each instance and reference the AMIs of that instance.
• It checks that the latest daily backup succeeded then it stores every image that's reached its DeleteOn tag's date for deletion.
• It then loops through the AMIs, de-registers them and removes all the snapshots associated with that AMI.
# Automated AMI Backups
# This code refered with slight changes: https://gist.github.com/bkozora/724e01903a9ad481d21e
#
# This script will search for all instances having a tag with "Backup"
# on it and are in runnig state. As soon as it has the instances list, it loop through each instance
# and create an AMI of it. Also, it will look for a "Retention" tag key which
# will be used as a retention policy number in days. If there is no tag with
# that name, it will use a 4 days default value for each AMI.If there is no tag with that name, it will use a 7 days default value for each AMI.
#
# After creating the AMI it creates a "DeleteOn" tag on the AMI indicating when
# it will be deleted using the Retention value and another Lambda function
import boto3
import collections
import datetime
#By the time I used this script, the Lamda is not available in Mumbai region. So, I chosen Singapore region.
#Specify the region in which EC2 Instances located and to create AMI's. Ex: Mumbai region (ap-south-1)
ec = boto3.client('ec2', 'ap-south-1')
#ec = boto3.client('ec2')
def lambda_handler(event, context):
reservations = ec.describe_instances(
Filters=[
{'Name': 'tag-key', 'Values': ['Backup']},
{ 'Name': 'instance-state-name','Values': ['running'] }
]
).get(
'Reservations', []
)
instances = sum(
[
[i for i in r['Instances']]
for r in reservations
], [])
print "Found %d instances that need backing up" % len(instances)
to_tag = collections.defaultdict(list)
for instance in instances:
print "Instance name:" + [res['Value'] for res in instance['Tags'] if res['Key'] == 'Name'][0]
#Default retention for 7 days if the tag is not specified
try:
retention_days = [
int(t.get('Value')) for t in instance['Tags']
if t['Key'] == 'Retention'][0]
except IndexError:
retention_days = 7
finally:
create_time = datetime.datetime.now()
#create_fmt = create_time.strftime('%d-%m-%Y.%H.%M.%S')
create_fmt = create_time.strftime('%d-%m-%Y')
try:
#Check for instance in running state
# if(ec.describe_instance_status(InstanceIds=[instance['InstanceId']],Filters=[{ 'Name': 'instance-state-name','Values': ['running'] }])['InstanceStatuses'][0]['InstanceState']['Name'] == 'running'):
#To make sure instance NoReboot enabled and to name the AMI
AMIid = ec.create_image(InstanceId=instance['InstanceId'], Name="Lambda - " + [result['Value'] for result in instance['Tags'] if result['Key'] == 'Name'][0] + " - " + " From " + create_fmt, Description="Lambda created AMI of instance " + instance['InstanceId'], NoReboot=True, DryRun=False)
to_tag[retention_days].append(AMIid['ImageId'])
print "Retaining AMI %s of instance %s for %d days" % (
AMIid['ImageId'],
instance['InstanceId'],
retention_days,
)
for retention_days in to_tag.keys():
delete_date = datetime.date.today() + datetime.timedelta(days=retention_days)
delete_fmt = delete_date.strftime('%d-%m-%Y')
print "Will delete %d AMIs on %s" % (len(to_tag[retention_days]), delete_fmt)
#To create a tag to an AMI when it can be deleted after retention period expires
ec.create_tags(
Resources=to_tag[retention_days],
Tags=[
{'Key': 'DeleteOn', 'Value': delete_fmt},
]
)
#If the instance is not in running state
except IndexError as e:
print "Unexpected error, instance "+[res['Value'] for res in instance['Tags'] if res['Key'] == 'Name'][0]+"-"+"\""+instance['InstanceId']+"\""+" might be in the state other then 'running'. So, AMI creation skipped."
\ No newline at end of file
# Automated AMI and Snapshot Deletion
# This code refered with slight changes: https://gist.github.com/bkozora/d4f1cf0e5cf26acdd377
#
# This script will search for all instances having a tag with "Backup"
# on it. As soon as it has the instances list, it loop through each instance
# and reference the AMIs of that instance. It check that the latest daily backup
# succeeded then it store every image that's reached its DeleteOn tag's date for
# deletion. It then loop through the AMIs, deregister them and remove all the
# snapshots associated with that AMI.
import boto3
import collections
import datetime
import time
import sys
#specify the region in which EC2 Instances located and to cleanup AMI's. Ex: Mumbai region (ap-south-1)
ec = boto3.client('ec2', 'ap-south-1')
ec2 = boto3.resource('ec2', 'ap-south-1')
images = ec2.images.filter(Owners=["XXXXXXXX"])# Specify your AWS account owner id in place of "XXXXX" at all the places in this script
def lambda_handler(event, context):
reservations = ec.describe_instances(
Filters=[
{'Name': 'tag-key', 'Values': ['Backup']},
{ 'Name': 'instance-state-name','Values': ['running'] }
]
).get(
'Reservations', []
)
instances = sum(
[
[i for i in r['Instances']]
for r in reservations
], [])
print "Found %d instances that need evaluated" % len(instances)
to_tag = collections.defaultdict(list)
date = datetime.datetime.now()
date_fmt = date.strftime('%d-%m-%Y')
print "Present date and time:" + date.strftime('%d-%m-%Y:%H.%m.%s')
imagesList = []
# Set to true once we confirm we have a backup taken today
backupSuccess = False
# Loop through all of our instances with a tag named "Backup"
for instance in instances:
imagecount = 0
# Loop through each image of our current instance
for image in images:
# Our other Lambda Function names its AMIs Lambda - Instance Name.
# We now know these images are auto created
if image.name.startswith('Lambda - ' + [result['Value'] for result in instance['Tags'] if result['Key'] == 'Name'][0]):
#if image.name.startswith('Lambda ec2 tag - ' + [result['Value'] for result in instance['Tags'] if result['Key'] == 'Name'][0]):
#print "FOUND IMAGE " + image.id + " FOR INSTANCE " + instance['InstanceId']
# Count this image's occcurance
imagecount = imagecount + 1
try:
if image.tags is not None:
deletion_date = [
t.get('Value') for t in image.tags
if t['Key'] == 'DeleteOn'][0]
delete_date = time.strptime(deletion_date, "%d-%m-%Y")
except IndexError:
deletion_date = False
delete_date = False
today_time = datetime.datetime.now().strftime('%d-%m-%Y')
# today_fmt = today_time.strftime('%m-%d-%Y')
today_date = time.strptime(today_time, '%d-%m-%Y')
# If image's DeleteOn date is less than or equal to today,
# add this image to our list of images to process later
if delete_date <= today_date:
imagesList.append(image.id)
# Make sure we have an AMI from today and mark backupSuccess as true
if image.name.endswith(date_fmt):
# Our latest backup from our other Lambda Function succeeded
backupSuccess = True
print "Latest backup from " + date_fmt + " was a success"
print "instance " + instance['InstanceId'] + " has " + str(imagecount) + " AMIs"
print "============="
print "About to process the following AMIs:"
print imagesList
if backupSuccess == True:
snapshotList = []
for image in imagesList:
#print image
desc_image_snapshots = ec.describe_images(ImageIds=[image],Owners=['XXXXXXXX',])['Images'][0]['BlockDeviceMappings']
# print (desc_image_snapshots)
try:
for desc_image_snapshot in desc_image_snapshots:
snapshot = ec.describe_snapshots(SnapshotIds=[desc_image_snapshot['Ebs']['SnapshotId'],], OwnerIds=['XXXXXXX'])['Snapshots'][0]
#if snapshot['Description'].find(image) > 0:
snapshotList.append(snapshot['SnapshotId'])
#else:
# continue
# print "Snapshot is not associated with an AMI"
except Exception as e:
print "Ignore Index Error:%s" % e.message
print "Deregistering image %s" % image
amiResponse = ec.deregister_image(
DryRun=False,
ImageId=image,
)
print "============="
print "About to process the following Snapshots associated with above Images:"
print (snapshotList)
print "The timer is started for 5 seconds to wait for images to deregister before deleting the snapshots associated to it"
time.sleep(5)# This should be set to higher value if the image in the imagesList takes more time to deregister
for snapshot in snapshotList:
try:
snap = ec.delete_snapshot(SnapshotId=snapshot)
print "Deleted snapshot " + snapshot
except Exception as e:
print "%s" % e.message
print "-------------"
else:
print "No current backup found. Termination suspended."
\ No newline at end of file
......@@ -28,9 +28,10 @@ def copy_latest_image():
#Copy todays images
if image_date.date() == (datetime.datetime.today()).date():
#Copy previous day images
#To Copy previous day images
#if image_date.date() == (datetime.datetime.today()-datetime.timedelta(1)).date():
if not dest_image_client.describe_images(Owners=['XXXXX',],Filters=[{'Name':'name','Values':[image.name]}])['Images']:
#if not dest_image_client.describe_images(Owners=['XXXXX',])['Images']:
......
......@@ -71,10 +71,15 @@ def lambda_handler(event, context):
print "Ignore Index Error:%s" % e.message
print "Deregistering image %s" % image
amiResponse = ec.deregister_image(
DryRun=False,
try:
amiResponse = ec.deregister_image(
DryRun=False,
ImageId=image,
)
)
#print "For testing, commented ami de-register"
except Exception as e:
print "%s" % e.message
print "============="
......
import boto3
import collections
import datetime
import time
import re
#Global objects
ec = boto3.resource('ec2', 'ap-southeast-1')
images = ec.images.filter(Owners=["XXXXX"])# Specify your AWS account owner id in place of "XXXXX" at all the places in this script
autoscale = boto3.client('autoscaling', 'ap-southeast-1')
##To sort out the ami's date
today = datetime.datetime.now()
#To sort out the ami's that matches the date to be updated in the new launch configuration
days_old = today - datetime.timedelta(days=1) # Zero means todays date and 1 means yesterday
days_old_date = days_old.strftime('%d-%m-%Y')
#Keep adding more number of AMI prefix names created from lambda here
apache_app = 'Lambda - '+'Apache-app' # 'Apache-app' can be the prefix name for ASG Ex: Apache-app-DR-ASG
tomcat_app = 'Lambda - '+'Tomcat-app'
#The variable should be comma separated
image_names_array = [apache_app,tomcat_app]
def lambda_handler(event, context):
imagesList = []
for image_name in image_names_array:
for image in images:
if ((image.name.startswith(image_name)) and (image.name.endswith(days_old_date))):
imagesList.append(image.name)
print ("\nThe new AMI to be used with new LC is: \"%s\"" %image.name)
#Existing Autoscaling group details, ASG be must be manually created.
#lambda_prefix_asg_name = trunc_at(image.name, "-",2) # Ex: Lambda - Apache
lambda_prefix_asg_name = trunc_at(image.name, "-",3) # Ex: Lambda - Apache-app
ASG_NAME = lambda_prefix_asg_name.replace('Lambda - ', '')+'-DR-ASG'
try:
update_autoscale_group = autoscale.update_auto_scaling_group(
AutoScalingGroupName=ASG_NAME,
MinSize=1, # remove 1 instance
)
except Exception as e:
print "%s" % e.message
def trunc_at(s, d, n=3):
"Returns s truncated at the n'th (3rd by default) occurrence of the delimiter, d."
return d.join(s.split(d)[:n])
\ No newline at end of file
import boto3
import collections
import datetime
import time
import re
import base64
#Global objects
ec = boto3.resource('ec2', 'ap-southeast-1')
images = ec.images.filter(Owners=["XXXXXX"])# Specify your AWS account owner id in place of "XXXXX" at all the places in this script
autoscale = boto3.client('autoscaling', 'ap-southeast-1')
##To sort out the ami's date
today = datetime.datetime.now()
#To sort out the ami's that matches the date to be updated in the new launch configuration
days_old = today - datetime.timedelta(days=1) # Zero means todays date and 1 means yesterday
days_old_date = days_old.strftime('%d-%m-%Y')
#Keep adding more number of AMI prefix names created from lambda here
apache_app = 'Lambda - '+'Apache-app' # 'Apache-app' can be the prefix name for ASG Ex: Apache-app-DR-ASG
tomcat_app = 'Lambda - '+'Tomcat-app'
#The variable should be comma separated
image_names_array = [apache_app,tomcat_app]
def lambda_handler(event, context):
imagesList = []
for image_name in image_names_array:
for image in images:
try:
if ((image.name.startswith(image_name)) and (image.name.endswith(days_old_date))):
imagesList.append(image.name)
#print ("array image_name %s" %image_name)
print ("\nThe new AMI to be used with new LC is: \"%s\"" %image.name)
#Existing Autoscaling group details, ASG be must be manually created.
#lambda_prefix_asg_name = trunc_at(image.name, "-",2) # Ex: Lambda - Admin
lambda_prefix_asg_name = trunc_at(image.name, "-",3) # Ex: Lambda - Admin-app
ASG_NAME = lambda_prefix_asg_name.replace('Lambda - ', '')+'-DR-ASG'
try:
autoscale_group = autoscale.describe_auto_scaling_groups(
AutoScalingGroupNames= [
ASG_NAME
]
)
autoscale_lc_name = [aslcn['LaunchConfigurationName'] for aslcn in autoscale_group['AutoScalingGroups'] if 'LaunchConfigurationName' in aslcn][0]
print ("The Autoscaling group is: %s" % ASG_NAME) # Ex: Apache-app-DR-ASG
print ("The current LC being used in autoscaling is: %s" %autoscale_lc_name)
lc_config = autoscale.describe_launch_configurations(
LaunchConfigurationNames=[
autoscale_lc_name
],
)
#Save the current lc details to variables
print ("\nThe configurations to create new LC are below......")
key_pair = [lck['KeyName'] for lck in lc_config['LaunchConfigurations'] if 'KeyName' in lck][0]
print ("Current LC key-pair: %s" % key_pair)
instance_type = [lcit['InstanceType'] for lcit in lc_config['LaunchConfigurations'] if 'InstanceType' in lcit][0]
print ("Current LC instance type: %s" % instance_type)
space_old_launch_config_name = [lcn['LaunchConfigurationName'] for lcn in lc_config['LaunchConfigurations'] if 'LaunchConfigurationName' in lcn][0]
old_launch_config_name = space_old_launch_config_name.replace(" ", "") # Without white spaces
print ("Current launch config name: %s" % old_launch_config_name)
security_group_id = [lcsgi['SecurityGroups'][0] for lcsgi in lc_config['LaunchConfigurations'] if 'SecurityGroups' in lcsgi][0]
print ("Current LC instance type: %s" %security_group_id)
new_ami_id = image.id
print ("New AMI Id to be updated: %s" %new_ami_id)
user_data = [lcud['UserData'] for lcud in lc_config['LaunchConfigurations'] if 'UserData' in lcud][0]
#to_string = "\""+user_data+"\""
decoded_user_data = base64.b64decode(user_data)
print ("-------------------------------------------------------------------------------------------------")
print ("Current LC encoded User data:\n %s" % user_data)
print ("\nCurrent LC decoded data is:\n %s" % decoded_user_data)
print ("-------------------------------------------------------------------------------------------------")
#To extract todays date to append with the new launch configuration name
#print (days_old_date)
todays = today - datetime.timedelta(days=0)
today_date = todays.strftime('%d-%m-%Y')
#print (today_date)
#print (str(old_launch_config_name))
#space_new_launch_config_name = old_launch_config_name.replace(str(days_old_date), str(today_date))
new_launch_config_name = image_name.replace(" ", "")+'-'+today_date+'-LC' # Without white spaces
print ("\nThe new launch configuration name is: %s" %new_launch_config_name)
try:
create_launch_config = autoscale.create_launch_configuration(
LaunchConfigurationName=new_launch_config_name,
ImageId=new_ami_id,
KeyName=key_pair,
SecurityGroups=[
security_group_id,
],
InstanceType=instance_type,
AssociatePublicIpAddress=True,
# UserData = user_data,
UserData =decoded_user_data
)
print ("The new LC being successfully created is %s" % new_launch_config_name)
except Exception as e:
print "Failed to create new LC. %s" % e.message
try:
update_autoscale_group = autoscale.update_auto_scaling_group(
AutoScalingGroupName=ASG_NAME,
MinSize=2, # Launch additional instance with the new launch configurations
LaunchConfigurationName=new_launch_config_name,
)
print ("The LC being successfully updated with %s" % ASG_NAME)
except Exception as e:
print "Failed to update ASG with new LC. %s" % e.message
try:
delete_launch_config = autoscale.delete_launch_configuration(
LaunchConfigurationName=old_launch_config_name
)
print "uncomment above and delete this line in prod"
except Exception as e:
print "%s" % e.message
print ("%d AMI(s) are successfully being updated with the new launch configurations which are: %s" %len(imagesList) %(imagesList))
except Exception as e:
print "ignore index error. %s" %e.message
else:
continue
except Exception as e:
print "image not found, ignore. %s" %e.message
def trunc_at(s, d, n=3):
"Returns s truncated at the n'th (3rd by default) occurrence of the delimiter, d."
return d.join(s.split(d)[:n])
import boto3
import collections
import datetime
import time
import base64
#Global objects
ec2 = boto3.client('ec2', 'ap-south-1')
images = ec.images.filter(Owners=["XXXXX"])# Specify your AWS account owner id in place of "XXXXX" at all the places in this script
autoscale = boto3.client('autoscaling', 'ap-south-1')
today = datetime.datetime.now()
#Auto-scaling group names list
asg_names_array = ['Apache-app-ASG','Tomcat-app-ASG']
def lambda_handler(event, context):
for ASG_NAME in asg_names_array:
try:
autoscale_group = autoscale.describe_auto_scaling_groups(
AutoScalingGroupNames= [
ASG_NAME
]
)
new_ami_id = []
try:
#Get the instance id and create an ami
for autoscale_instance in autoscale_group['AutoScalingGroups'][0]['Instances']:
if (autoscale_instance['HealthStatus'] == 'Healthy' and autoscale_instance['LifecycleState'] == 'InService'):
print ("Healthy Instance is: %s" %autoscale_instance['InstanceId'])
print ("Creating AMI for the instance: %s" %autoscale_instance['InstanceId'])
AMIid = ec2.create_image(InstanceId=autoscale_instance['InstanceId'], Name=ASG_NAME, Description="Lambda created AMI for ASG " + ASG_NAME, NoReboot=True, DryRun=False)
new_ami_id.append(AMIid['ImageId'])
# new_ami_id.append('123')
break
#print (new_ami_id[0])
new_ami_id = new_ami_id[0]
autoscale_lc_name = [aslcn['LaunchConfigurationName'] for aslcn in autoscale_group['AutoScalingGroups'] if 'LaunchConfigurationName' in aslcn][0]
print ("The Autoscaling group is: %s" % ASG_NAME)
print ("The current LC being used in autoscaling is: %s" %autoscale_lc_name)
lc_config = autoscale.describe_launch_configurations(
LaunchConfigurationNames=[
autoscale_lc_name
],
)
#Save the current lc details to variables
print ("\nThe configurations to create new LC are below......")
key_pair = [lck['KeyName'] for lck in lc_config['LaunchConfigurations'] if 'KeyName' in lck][0]
print ("Current LC key-pair: %s" % key_pair)
instance_type = [lcit['InstanceType'] for lcit in lc_config['LaunchConfigurations'] if 'InstanceType' in lcit][0]
print ("Current LC instance type: %s" % instance_type)
space_old_launch_config_name = [lcn['LaunchConfigurationName'] for lcn in lc_config['LaunchConfigurations'] if 'LaunchConfigurationName' in lcn][0]
old_launch_config_name = space_old_launch_config_name.replace(" ", "") # Without white spaces
print ("Current launch config name: %s" % old_launch_config_name)
security_group_id = [lcsgi['SecurityGroups'][0] for lcsgi in lc_config['LaunchConfigurations'] if 'SecurityGroups' in lcsgi][0]
print ("Current LC Security Group Id: %s" %security_group_id)
old_ami_id = [lcami['ImageId'] for lcami in lc_config['LaunchConfigurations'] if 'ImageId' in lcami][0]
print ("Current LC ami id: %s" %old_ami_id)
print ("New AMI Id to be updated: %s" %new_ami_id)
user_data = [lcud['UserData'] for lcud in lc_config['LaunchConfigurations'] if 'UserData' in lcud][0]
decoded_user_data = base64.b64decode(user_data)
print ("-------------------------------------------------------------------------------------------------")
print ("Current LC encoded User data:\n %s" % user_data)
print ("\nCurrent LC decoded data is:\n %s" % decoded_user_data)
print ("-------------------------------------------------------------------------------------------------")
#To extract todays date to append with the new launch configuration name
todays = today - datetime.timedelta(days=0)
today_date = todays.strftime('%d-%m-%Y')
#space_new_launch_config_name = old_launch_config_name.replace(str(days_old_date), str(today_date))
new_launch_config_name = ASG_NAME.replace(" ", "")+'-'+today_date+'-LC' # Without white spaces