Showing posts with label EBS. Show all posts
Showing posts with label EBS. Show all posts

Thursday, April 16, 2009

Programmatically monitoring your EC2 instances

The EC2 C# library provides several classes that allow you to monitor and pass commands to your running instances and volumes. Each request class is coupled with a response class that is returned as an instance when you make a functions call using an instance of the EC2Service class.

The EC2Service class is the controller trough which requests and responses are passed. Classes used to monitor running instances use the Describe prefix, such as the DescribeVolumesRequest or DescribeInstancesRequest classes. Note that both the DescribeInstancesRequest and the DescribeVolumesRequest class expect lists of IDs. This means that you can request status reports from multiple as well as single instances and volumes.

I have created a wrapper class that can be used for the following:

1. Initialize an EC2 instances using an existing AMI.
2. Terminate an EC2 instance using an existing AMI.
3. Attach an EBS volume.
4. Detach an EBS volume.
5. Monitor the EBS volume attachment process.
6. Monitor the EBS volume detachment process.
7. Monitor the EC2 instance startup process.
8. Monitor the EC2 instance closing process.

EC2 wrapper:



using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Amazon.EC2;
using Amazon.EC2.Model;

namespace consoleEBS_And_EC2
{
class EC2Wrapper
{

private AmazonEC2 service;
private string _instance;
public string instance { get { return _instance; } set { _instance = value; } }
private bool _isRunning = false;
public bool isRunning { get { return _isRunning; } set { _isRunning = value; } }
private bool _volAttached;
public bool volAttached { get { return _volAttached; } set { _volAttached = value;} }
private string _volID;
public string volID { get { return _volID ;} set { _volID = value;} }
private string _runReservationId;
public string runReservationId { get { return this._runReservationId; } set { this._runReservationId = value;} }
private bool _detaching = false;
public bool detaching { get { return _detaching;} set {_detaching = value;} }
private bool _attaching = false;
public bool attaching { get { return _attaching; } set { _attaching = value ;} }





public EC2Wrapper(string accessKeyId, string secretAccessKey,string amiId)
{
try
{
service = new AmazonEC2Client(accessKeyId, secretAccessKey);


runYourInstances(amiId);



}
catch (Exception ex)
{
//Call logger here, then throw user friendly message.
//throw new Exception("Your instance did not start. Check your eventlog for details.");
throw ex;
}
}

private void runYourInstances(string amiId)
{
RunInstancesRequest request = new RunInstancesRequest();
////Set allowed number of instances:
request.MinCount = 1;
request.MaxCount = 1;
request.InstanceType = "m1.small";
//request.
Placement p = new Placement();
p.AvailabilityZone = "us-east-1a";
request.Placement = p;
request.ImageId = amiId;
RunInstancesResponse risp = this.service.RunInstances(request);
this.runReservationId = risp.RunInstancesResult.Reservation.ReservationId;
this.instance = (from r in risp.RunInstancesResult.Reservation.RunningInstance where risp.RunInstancesResult.Reservation.ReservationId == this.runReservationId select r.InstanceId).FirstOrDefault();

}
/// <summary>
/// Monitors running request until it either the timeout has elapsed (12 minutes) or the status is running.
/// </summary>
/// <param name="id"></param>
/// <returns></returns>
public bool monitorStartingProcess()
{

int timeOutIterations = 0;
try
{

while (true)
{
RunningInstance ri = getRunningInstance();
if (ri != null)
{
this.isRunning = true;
break;
}
System.Threading.Thread.Sleep(1000);
timeOutIterations += 1;
if (timeOutIterations == 720)
break;

}
}
catch (Exception ex)
{
throw ex;
}
return this.isRunning;
}

private RunningInstance getRunningInstance()
{
DescribeInstancesRequest rs = new DescribeInstancesRequest();
rs.InstanceId = new List<string>() { this.instance };
DescribeInstancesResponse rsp = service.DescribeInstances(rs);
Reservation res = (from r in rsp.DescribeInstancesResult.Reservation where r.ReservationId == this.runReservationId select r).FirstOrDefault();
RunningInstance ri = (from r in res.RunningInstance where r.InstanceId == this.instance && r.InstanceState.Name == "running" select r).FirstOrDefault();

return ri;
}
/// <summary>
/// Terminates running instance.
/// </summary>
public void terminateRunning()
{
try
{
TerminateInstancesRequest rq = new TerminateInstancesRequest();
rq.InstanceId.Add(this.instance);
TerminateInstancesResponse rsp = this.service.TerminateInstances(rq);
this.isRunning = false;
}
catch (Exception ex)
{
throw ex;
}
}

public bool Attach_EBS_Volume(string id)
{
bool result = false;
AttachVolumeRequest vrq = new AttachVolumeRequest();
this.volID = id;
vrq.VolumeId = this.volID;
vrq.InstanceId = this.instance;
vrq.Device = "/dev/sda";
AttachVolumeResponse vrp = this.service.AttachVolume(vrq);
if (vrp.AttachVolumeResult.Attachment.Status == "attaching")
{
result = true;
this.attaching = true;
}
else
result = false;
return result;
}



public bool monitorVolumeAttachment()
{

int timeOutIterations = 0;
try
{

while (true)
{
DescribeVolumesRequest rs = new DescribeVolumesRequest();
rs.VolumeId = new List<string>() { this.volID };
DescribeVolumesResponse rsp = service.DescribeVolumes(rs);
Volume vol = (from v in rsp.DescribeVolumesResult.Volume where v.VolumeId == this.volID && v.Status == "in-use" select v).FirstOrDefault();
if (vol != null)
{
this.volAttached = true;
this.attaching = false;
break;
}
System.Threading.Thread.Sleep(1000);
timeOutIterations += 1;
if (timeOutIterations == 720)
break;

}
}
catch (Exception ex)
{
throw ex;
}
return this.isRunning;
}

public bool Detach_EBS_Volume()
{
bool result = false;
DetachVolumeRequest dvr = new DetachVolumeRequest();
dvr.InstanceId = this.instance;
dvr.Device = "/dev/sda";
dvr.Force = false;
dvr.VolumeId = this.volID;
DetachVolumeResponse dvs = this.service.DetachVolume(dvr);
if (dvs.DetachVolumeResult.Attachment.Status == "detaching")
{
result = true;
this.detaching = true;
}
else
result = false;
return result;
}


public bool monitorDetachVolumeRequest(string id)
{
int timeOutIterations = 0;
try
{

while (true)
{
DescribeVolumesRequest rs = new DescribeVolumesRequest();
rs.VolumeId = new List<string>() { id };
DescribeVolumesResponse rsp = service.DescribeVolumes(rs);
Volume vol = (from v in rsp.DescribeVolumesResult.Volume where v.Status == "available" && v.VolumeId == id select v).FirstOrDefault();
if (vol != null)
{
this.volAttached = false;
this.detaching = false;
break;
}
System.Threading.Thread.Sleep(1000);
timeOutIterations += 1;
if (timeOutIterations == 720)
break;

}
}
catch (Exception ex)
{
throw ex;
}
return this.volAttached;
}

public static bool isInstanceRunning(string id,string accesskey,string secretkey)
{
bool result = false;
AmazonEC2 ae = new AmazonEC2Client(accesskey,secretkey);
DescribeInstancesRequest rs = new DescribeInstancesRequest();
rs.InstanceId = new List<string>() { id };

DescribeInstancesResponse rsp = ae.DescribeInstances(rs);


var res = (from r in rsp.DescribeInstancesResult.Reservation select new { state =
(from ri in r.RunningInstance where ri.InstanceId == id && ri.InstanceState.Name == "running" select ri.InstanceState.Name.ToString()).FirstOrDefault()});
foreach (var inst in res)
if ((string) inst.state == "running")
result = true;
return result;

}
}
}


Invocation of wrapper:



using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace consoleEBS_And_EC2
{
class Program
{

static void Main(string[] args)
{

//Ubuntu 8.10 image, can use the small instance type which will save us money.
String ami = "ami-7dfd1a14";
string EBS_id = null;
try
{

Console.WriteLine("Type accessKeyId: ");
string accessKeyId = Console.ReadLine();
Console.WriteLine("Type secretAccessKey: ");
string secretAccessKey = Console.ReadLine();
EC2Wrapper wrp = new EC2Wrapper(accessKeyId, secretAccessKey, ami);
Console.WriteLine("Instance start request sent");
loopDescribeInstance(wrp, ami);
if (wrp.isRunning)
{
Console.WriteLine("Type the Id of the EBS volume you want to attach: ");
EBS_id = Console.ReadLine();
if (wrp.Attach_EBS_Volume(EBS_id))
{
Console.WriteLine("Volume attach request sent");
if (wrp.monitorVolumeAttachment())
{
Console.WriteLine("Volume attached successfully");
System.Threading.Thread.Sleep(20000);
if (wrp.Detach_EBS_Volume())
{
Console.WriteLine("Volume detached request sent");
if (!wrp.monitorDetachVolumeRequest(EBS_id))
{
Console.WriteLine("Volume detached successfully");
}
else
{
Console.WriteLine("Detach operation failed");
}
}
else
{
Console.WriteLine("Detach request failed");
}
}
else
{
Console.WriteLine("Attach operation timed out");
}
}
else
{
Console.WriteLine("Volume attach request failed");
}
}

Stop(wrp);
Console.ReadLine();
}
catch (Amazon.EC2.AmazonEC2Exception aex)
{
Console.WriteLine(aex.ToString());
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}

}

static void loopDescribeInstance(EC2Wrapper w, string id)
{

try
{

if (w.monitorStartingProcess())
{
Console.WriteLine("Instance started");

}
else
{
Console.WriteLine("Instance not started");

}

}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}

}

static void Stop(EC2Wrapper srvc)
{
srvc.terminateRunning();
Console.WriteLine("Running instance stopped");
}

}
}


Note that the EC2Wrapper class only allows us to start one instance, and that it has two properties called attaching or detaching which are used to indicate whether an instance is in the process of attaching or detaching an EBS volume. Once the volume is attached or detached, we leave the transitional state and the attaching and detaching properties are set to 'false'.

A drawback to the EC2 C# library is that it only contains one exception class, Amazon.EC2.AmazonEC2Exception, but the error codes of this class are fairly informative so it would be totally feasible to trap AmazonEC2Exceptions, read the error code and then throw our own custom exceptions based on the error code.

Monday, March 23, 2009

Creating and using Amazon Machine Images and Amazon Elastic Block Storage

Let's say you have managed to get an instance running SQL Server 2005 on a Windows 2003 operating system up and running. You are running concurrent CRUD operations against a database, but you decide you need to install a service package and restart the instance. That should be no problem right?

Wrong!

When you terminate a running instance, any data changes made during the session are lost unless they were saved on Amazon Elastic Block Store (henceforth referred to as EBS). The only way to save any data on your instance that is not stored on EBS is by bundling the image. Bundling an image means saving the instance and it's current configuration to Amazons's Simple Storage Service (henceforth referred to as S3). Typically what you would do is to create your image, test it and save it again as part of an iterative process moving towards perfection. Once all the necessary applications and configuration settings have been made and you have made sure everything works, you should only need to save production data, and that should be stored on an EBS volume. One of the biggest advantages with EBS volumes is that if your image becomes corrupt, you can simply attach the EBS volume to a new image and retrieve the data. You can also create snapshots of EBS volumes.

The easiest way to create and save an image is to launch one of the images in Amazon's public image directory, install any additional software and service packages, then bundle it using the management console or Elasticfox, unless you are John Wayne or somebody who insists on doing it the hard way by using Amazon's command line tools. The image you bundle can be registered as either private or public. A public image can be accessed and launched by anyone with an AWS account. You can also create images on your own machine and upload to Amazon. In this posting however, we will focus on creating new images from running instances initalized using existing images already registered with AWS.


To create an image from a running instance using Elasticfox:

1. Select the 'Instances' tab

2. Right click the instance and select the 'Bundle into an AMI' option.

3. Type a bucket name and an image name for your S3 folder (a new bucket will be created if a bucket with the specified name does not exist, then click ok.

4.Click the 'Bundle Task' tab and you will see the progress of the bundle task you just created. It can take a while so be patient. Bundling actually consists of two steps, bundling or packaging the image, then saving it to S3. The progress will be displayed in the Bundle State column. Once the process is finished the Bundle State column will display as 'complete'.

5. Once the 'complete' status is reached, you need to right click the image file in the Bundle Tasks view and select 'Register AMI' to register it before you can use it. Adding a descriptive tag is optional but useful if you need to keep track of a lot of images.

6. The image you created is now registered and will appear among your images with the Visibility value set to 'Private'.

7. To launch the image you created, right click and select the 'Launch instance(s) of this AMI' option.



As mentioned above you need to have an EBS volume to persist data between sessions. So how do you create an EBS volume?

To create an EBS volume using Elasticfox:

1. Click on the Volumes and Snapshots tab, then click the add(+) button to create your EBS volume.

2.Type desired size (anything between 1 GB and 1 TB), add a tag, click create, and Bob's yer uncle.

Attaching the EBS volume is even simpler:

1.In Elasticfox, right click your running instance and select 'Attach an EBS volume'.

2.In the popup, select the id of the volume you want to attach and click 'Attach'.

Assuming that you attached the EBS volume to a Windows instance, all you need to do now is to log on to your running image, then go to Administrative Tools, select Computer Management and format the volume. Note that your instance and your image must be running in the same availability zone, otherwise you will get an error.

The C# library also offers functions for handling EBS volumes. The following code example shows how to create an EBS volume and attach it to a running instance (for an example on how to launch an instance, see my previous post Initializing and Controlling your instance on EC2):

First of all, make sure you are running your instance in the same availability zone
as the EBS volume. If the volume is located in the us-east-1a zone, your instance must be launched there as well by setting the Placement property for an instance of the RunInstancesRequest class:

Placement p = new Placement();
p.AvailabilityZone = "us-east-1a";
request.Placement = p;

Assuming that we have launched the instance successfully, we can now attach our EBS volume (for initialization of the service class see Initializing and Controlling your instance on EC2):

AttachVolumeRequest request = new AttachVolumeRequest();
request.VolumeId = "vol-e38d6c8a";
request.InstanceId = "i-a05e3bc9";
request.Device = "xvdg";
AttachVolumeResponse response = service.AttachVolume(request);

If the request was successful, the response.Attachment.Status property will be set to 'attaching'. You can also check the result by opening Elasticfox and clicking the Volumes and Snaphots tab and checking the Attachment Status field of the EBS volume specified in the function call. To monitor your instances and volumes use the DescribeInstancesRequest and DescribeVolumesRequest classes, but more about that some other time. Sampai nanti as they say in Indonesian.