#
# Django Code Deploy
#
# Copyright 2015 - 2016 devops.center
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import logging
logger = logging.getLogger(__name__)
handler = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s %(levelname)-8s %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.INFO)
from fabric.api import *
from time import gmtime, strftime
import os
import sys
from git import Repo
class FabricException(Exception):
pass
# Set some global defaults for all operations
env.user = "ubuntu"
env.key_filename = []
ACCESS_KEY_PATH = "~/.ssh/"
env.connection_attempts = 3
TRUTH_VALUES = ['True', 'TRUE', '1', 'true', 't', 'Yes',
'YES', 'yes', 'y'] # arguments in fab are always strings
import boto
import urllib2
from boto.ec2 import connect_to_region
import distutils.sysconfig
# objects
import collections
AWSAddress = collections.namedtuple(
'AWSAddress', 'name publicdns privateip shard')
# set_hosts selects all instances that match the filter criteria.
# type is web, worker, db
# environment is dev, staging, prod
# appname is application name, such as "fresco", "topopps", "mojo", etc.
# action is the deferred action needed, such as "deploy", "security-updates", etc.
# region is aws region
@task
def set_hosts(type, primary=None, appname=None, action=None, region=None,
shard='all', aRole=None):
if appname is None:
local('echo "ERROR: appname option is not set"')
if region is None:
local('echo "ERROR: region option is not set"')
environment = os.environ["AWS_ENVIRONMENT"]
awsaddresses = _get_awsaddress(type, primary, environment, appname,
action, region, shard, aRole)
env.hosts = list(item.publicdns for item in awsaddresses)
env.host_names = list(item.name for item in awsaddresses)
_log_hosts(awsaddresses)
# set_one_host picks a single instance out of the set.
# filters are the same as with set_hosts.
@task
def set_one_host(type, primary=None, appname=None, action=None, region=None,
shard='all', aRole=None):
if appname is None:
local('echo "ERROR: appname option is not set"')
if region is None:
local('echo "ERROR: region option is not set"')
environment = os.environ["AWS_ENVIRONMENT"]
awsaddresses = _get_awsaddress(type, primary, environment, appname,
action, region, shard, aRole)
awsaddresses = [awsaddresses[0]]
env.hosts = [awsaddresses[0].publicdns]
env.host_names = [awsaddresses[0].name]
_log_hosts(awsaddresses)
@task
def set_one_host_per_shard(type, primary=None, appname=None, action=None,
region=None, shard='all', aRole=None):
if appname is None:
local('echo "ERROR: appname option is not set"')
if region is None:
local('echo "ERROR: region option is not set"')
environment = os.environ["AWS_ENVIRONMENT"]
awsaddresses = _get_awsaddress(type, primary, environment, appname,
action, region, shard, aRole)
pruned_list = []
for ahost in awsaddresses:
if not next((True for bhost in pruned_list if ahost.shard == bhost.shard), False):
pruned_list.append(ahost)
env.hosts = list(item.publicdns for item in pruned_list)
env.host_names = list(item.name for item in pruned_list)
_log_hosts(pruned_list)
def _log_hosts(awsaddresses):
logger.info("")
logger.info(
"Instances to operate upon - name, public dns, private ip, shard")
logger.info(
"---------------------------------------------------------------")
for instance in awsaddresses:
logger.info("%s %s %s %s", instance.name,
instance.publicdns, instance.privateip, instance.shard)
logger.info("")
logger.info("")
logger.info("keys: %s", env.key_filename)
logger.info("")
@task
def dev():
os.environ["AWS_ENVIRONMENT"] = "dev"
@task
def staging():
os.environ["AWS_ENVIRONMENT"] = "staging"
@task
def prod():
os.environ["AWS_ENVIRONMENT"] = "prod"
@task
def set_environment(environment):
os.environ["AWS_ENVIRONMENT"] = environment
@task
def set_access_key(accessKeyPath):
env.key_filename = [accessKeyPath]
@task
def set_access_key_path(anAccessKeyPath):
global ACCESS_KEY_PATH
if(anAccessKeyPath.endswith('/')):
ACCESS_KEY_PATH = anAccessKeyPath
else:
ACCESS_KEY_PATH = anAccessKeyPath + "/"
@task
def set_user(loginName):
env.user = loginName
@task
def show_environment():
run('env')
# Private method to get public DNS name for instance with given tag key
# and value pair
def _get_awsaddress(type, primary, environment, appname, action, region, shard,
aRole):
awsaddresses = []
connection = _create_connection(region)
aws_tags = {"tag:Type": type, "tag:Env": environment,
"tag:App": appname, "instance-state-name": "running"}
if action:
aws_tags["tag:ActionNeeded"] = action
if primary:
aws_tags["tag:Primary"] = primary
if aRole:
aws_tags["tag:role"] = aRole
logger.info("Filtering via tags=%s", aws_tags)
instances = connection.get_only_instances(filters=aws_tags)
shards = [e for e in shard.split(' ')]
for instance in instances:
if instance.public_dns_name: # make sure there's really an instance here
shardt = (
"None" if not 'Shard' in instance.tags else instance.tags['Shard'])
awsaddress = AWSAddress(name=instance.tags['Name'], publicdns=instance.public_dns_name,
privateip=instance.private_ip_address, shard=shardt)
if (shard == 'all') or (shardt in shards):
awsaddresses.append(awsaddress)
if instance.key_name not in env.key_filename:
env.key_filename.append(instance.key_name)
# convert any AWS key-pair names to a file path for the actual key pair
# locally
env.key_filename = [key if os.path.isfile(
key) else ACCESS_KEY_PATH + key + ".pem" for key in env.key_filename]
return awsaddresses
# Private method for getting AWS connection
def _create_connection(region):
logger.info("")
logger.info("Connecting to AWS region %s", region)
connection = connect_to_region(
region_name=region
)
logger.info("Connection with AWS established")
return connection
timest = strftime("%Y-%m-%d_%H-%M-%S", gmtime())
# deploy directories have timestamps for names.
UPLOAD_CODE_PATH = os.path.join("/data/deploy", timest)
TAR_NAME = "devops"
@task
def tar_from_git(branch):
local('rm -rf %s.tar.gz' % TAR_NAME)
local('git archive %s --format=tar.gz --output=%s.tar.gz' %
(branch, TAR_NAME))
@task
def unpack_code():
cmd = "mkdir -p " + UPLOAD_CODE_PATH
sudo(cmd)
put('%s.tar.gz' % TAR_NAME, '%s' % UPLOAD_CODE_PATH, use_sudo=True)
with cd('%s' % UPLOAD_CODE_PATH):
sudo('tar zxf %s.tar.gz' % TAR_NAME)
@task
def link_new_code():
try:
sudo('unlink /data/deploy/pending')
except:
pass
sudo('ln -s %s /data/deploy/pending' % UPLOAD_CODE_PATH)
with cd('/data/deploy'):
# keep onlly 5 most recent deploys, excluding any symlinks or other purely alpha directories. The steps are
