#! /usr/bin/env python
'''
*
*   Copyright [2011] [Red Hat, Inc.]
*
*   Licensed under the Apache License, Version 2.0 (the "License");
*   you may not use this file except in compliance with the License.
*   You may obtain a copy of the License at
*
*   http://www.apache.org/licenses/LICENSE-2.0
*
*   Unless required by applicable law or agreed to in writing, software
*   distributed under the License is distributed on an "AS IS" BASIS,
*   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*   See the License for the specific language governing permissions and
*  limitations under the License.
*
'''

'''
Audrey Startup (AS)

Note: The source file is named audrey_start.in.py The make process generates
      audrey_start.py. audrey_start.py should not be manually modified.

Invoked at instance launch to interface with the Config Server (CS)

For prototype end to end testing this file needs to be installed
at: /usr/bin/audrey

Algorithim:
    Get CF info
    Loop:
        Get and validate required configuration from CS
        Configure system using required configuration
        Get and validate provides parameters from CS
        Gather provided parameter data from system
        Put provided parameter data to CS
    if not done then goto Loop

'''

import argparse
import base64
import httplib2
import logging
import os
import shutil
import sys
import tarfile
import tarfile as tf # To simplify exception names.
import tempfile
import urllib
import oauth2 as oauth

from time import sleep
from collections import deque
from subprocess import Popen, PIPE

EC2_USER_DATA_URL = 'http://169.254.169.254/latest/user-data'
CLOUD_INFO_FILE = '/etc/sysconfig/cloud-info'

# Location of the config tooling.
TOOLING_DIR = '/var/audrey/tooling/'

# Log file
LOG = '/var/log/audrey.log'
LOGGER = None
CS_API_VER = 1
# The VERSION string is filled in during the make process.
AUDREY_VER = '0.4.2'

# When running on condor-cloud, the Config Server (CS) contact
# information will be stored in the smbios.
# These are the dmi files where the smbios information is stored.
CONDORCLOUD_CS_ADDR = '/sys/devices/virtual/dmi/id/sys_vendor'
CONDORCLOUD_CS_UUID = '/sys/devices/virtual/dmi/id/product_name'

#
# Error Handling methods:
#
class ASError(Exception):
    '''
    Some sort of error occurred. The exact cause of the error should
    have been logged. So, this just indicates that something is wrong.
    '''
    pass

def _raise_ASError(err_msg):
    '''
    Log an error message and raise ASError
    '''
    LOGGER.error(err_msg)
    raise ASError(err_msg)

class _run_cmd_return_subproc(object):
    '''
    Used to pass return code to caller if no subprocess object
    is generated by Popen() due to an error.
    '''
    returncode = 127

#
# Misc. Supporting Methods
#
def _run_cmd(cmd, my_cwd=None):
    '''
    Description:
        Run a command given by a dictionary, check for stderr output,
        return code.

        To check the return code catch SystemExit then examine:
        ret['subproc'].returncode.

    Input:

        cmd - a list containing the command to execute.
            e.g.: cmd = ['ls', '/tmp']

    Returns:

        ret - a dictionary upon return contains the keys:
            'subproc', 'err', 'out'

        ret['subproc'].returncode = subprocess return code
        ret['err'] = command errors string.
        ret['out'] = command out list.

    Example:

        cmd = ['ls', '/tmp']
        ret = _run_cmd(cmd)

        ret.keys()
        ['subproc', 'err', 'out']
        ret['subproc'].returncode
        0
        ret['err']
        ''
        ret['out']

    '''

    pfail = _run_cmd_return_subproc()

    # Return dictionary to contain keys: 'cmd', 'subproc', 'err', 'out'
    ret = {'subproc' : None, 'err' : '' , 'out' : ''}

    try:
        ret['subproc'] = Popen(cmd, cwd=my_cwd, stdout=PIPE, stderr=PIPE)

    # unable to find command will result in an OSError
    except OSError, err:
        if not ret['subproc']:
            ret['subproc'] = pfail

        ret['subproc'].returncode = 127 # command not found
        ret['err'] = str(err)
        return ret

    # fill ret['out'] with stdout and ret['err'] with stderr
    ret.update(zip(['out', 'err'], ret['subproc'].communicate()))

    return ret

def _run_pipe_cmd(cmd1, cmd2):
    '''
    Description:
        Run one command piped into another. Commands are given as
        dictionaries, check for stderr output, return code.

        To check the return code catch SystemExit then examine:
        ret['subproc'].returncode.

        That is this routine can be used to execute a command
        of the form:

    Input:

        cmd1 - a list containing the command to execute.
            e.g.: cmd = ['ls', '/tmp']

        cmd2 - a list containing the command to pipe the output
            of cmd1 to.
            e.g.: cmd = ['grep', 'a_file']

    Returns:

        ret - a dictionary upon return contains the keys:
            'subproc', 'err', 'out'

        ret['subproc'].returncode = subprocess return code
        ret['err'] = command errors string.
        ret['out'] = command out list.

    Example:

        cmd1 = ['ls', '/tmp']
        cmd2 = ['grep', 'a_file']
        ret = _run_pipe_cmd(cmd1, cmd2)

        ret.keys()
        ['subproc', 'err', 'out']
        ret['subproc'].returncode
        0
        ret['err']
        ''
        ret['out']

    '''

    # Return dictionary to contain keys: 'cmd', 'subproc', 'err', 'out'
    ret = {'subproc' : None, 'err' : '' , 'out' : ''}

    p1 = None
    p2 = None
    pfail = _run_cmd_return_subproc()

    # Execute the first command:
    try:
        p1 = Popen(cmd1, stdout=PIPE)
        p2 = Popen(cmd2, stdin=p1.stdout, stdout=PIPE )
        p1.stdout.close()

        # fill ret['out'] with stdout and ret['err'] with stderr
        # ret.update(zip(['out', 'err'], ret['subproc'].communicate()[0]))
        ret.update(zip(['out', 'err'], p2.communicate()))
        ret['subproc'] = p2

    # unable to find command will result in an OSError
    except OSError, err:
        if p2:
            ret['subproc'] = p2
        elif p1:
            ret['subproc'] = p1
        else:
            ret['subproc'] = pfail

        ret['subproc'].returncode = 127 # command not found
        ret['err'] = str(err)
        return ret

    return ret

class ServiceParams(object):
    '''
    Description:
        Used for storing a service and all of it's associated parameters
        as provided by the Config Server in the "required" parameters
        API message.

        services = [
                ServiceParams('serviceA', ['n&v', 'n&v', 'n&v',...]),
                ServiceParams('serviceB', ['n&v', 'n&v', 'n&v',...]),
                ServiceParams('serviceB', ['n&v', 'n&v', 'n&v',...]),
        ]

        This structure aids in tracking the parsed required config
        parameters which is useful when doing UNITTESTing.

    '''
    def __init__(self, name=None):
        if name == None:
            name = ''
        self.name = name # string
        self.params = [] # start with an empty list
    def add_param(self, param):
        '''
        Description:
            Add a parameter provided by   the Config Server to the list.
        '''
        self.params.append(param)
    def __repr__(self):
        return repr((self.name, self.params))

#
# Methods used to parse the CS<->AS text based API
#
def _common_validate_message(src):
    '''
    Perform validation of the text message sent from the Config Server.
    '''

    if not src.startswith('|') or not src.endswith('|'):
        _raise_ASError(('Invalid start and end characters: %s') % (src))

def gen_env(serv_name, param_val):
    '''
    Description:
        Generate the os environment variables from the required config string.

    Input:
        serv_name - A service name
            e.g.:
            jon_agent_config

        param_val - A parameter name&val pair. The value is base64 encoded.
            e.g.:
            jon_server_ip&MTkyLjE2OC4wLjE=

    Output:
        Set environment variables of the form:
        <name>=<value>
            e.g.:
            jon_server_ip=base64.b64decode('MTkyLjE2OC4wLjE=')
            jon_server_ip='192.168.0.1

    Raises ASError when encountering an error.

    '''
    LOGGER.debug('Invoked gen_env()')

    # If the param_val is missing treat as an exception.
    if param_val == '':
        _raise_ASError(('Missing parameter name. %s') % \
            (str(param_val)))

    # If serv_name is not blank an extra "_" must be added to
    # the environment variable name.
    if serv_name != '':
        serv_name = serv_name + '_'

    name_val = param_val.split('&')
    var_name = 'AUDREY_VAR_' + serv_name + name_val[0]
    os.environ[var_name] = \
        base64.b64decode(name_val[1])

    # Get what was set and log it.
    cmd = ['/usr/bin/printenv', var_name]
    ret = _run_cmd(cmd)
    LOGGER.debug(var_name + '=' + str(ret['out'].strip()))

def parse_require_config(src):
    '''
    Description:
        Parse the required config text message sent from the Config Server.

    Input:
        The required config string obtained from the Config Server,
        delimited by an | and an &

        Two tags will mark the sections of the data,
        '|service|' and  '|parameters|'

        To ensure all the data was received the entire string will be
        terminated with an "|".

        The string "|service|" will precede a service names.

        The string "|parameters|" will precede the parameters for
        the preceeding service, in the form: names&<b64 encoded values>.

    This will be a continuous text string (no CR or New Line).

        Format (repeating for each service):

        |service|<s1>|parameters|name1&<b64val>|name2&<b64val>...|nameN&<b64v>|


        e.g.:
        |service|ssh::server|parameters|ssh_port&<b64('22')>
        |service|apache2::common|apache_port&<b64('8081')>|

    Returns:
        - A list of ServiceParams objects.
    '''

    services = []
    new = None

    _common_validate_message(src)

    # Message specific validation
    if src == '||':
        # special case indicating no required config needed.
        return []

    if src.find('|service|') != 0:
        _raise_ASError(('|service| is not the first tag found. %s') % (src))


    src_q = deque(src.split('|'))

    # remove leading and trailing elements from the src_q since they are
    # empty strings generated by the split('|') because of the leading
    # and trailing '|'
    token = src_q.popleft()
    token = src_q.pop()

    while True:
        try:
            token = src_q.popleft()
            if token == 'service':
                token = src_q.popleft() # next token is service name

                # Raise an error if the service name is invalid.
                if token.find('&') != -1 or \
                    token == 'service' or \
                    token == 'parameters':
                    _raise_ASError(('ERROR invalid service name: %s') % \
                       (str(token)))

                new = ServiceParams(token)
                services.append(new)
            elif token == 'parameters' or token == '':
                pass
            else: # token is a name&value pair.
                if token.find('&') == -1:
                    _raise_ASError(('ERROR name&val: %s missing delimiter') % \
                       (str(token)))
                if new:
                    new.add_param(token)
                    gen_env(new.name, token)
                else:
                    _raise_ASError(('ERROR missing service tag %s') % \
                         (str(src)))
        except IndexError:
            break

    return services

def _get_system_info():
    '''
    Description:
        Get the system info to be used for generating this instances
        provides back to the Config Server.

        Currently utilizes Puppet's facter via a Python subprocess call.

    Input:
        None

    Returns:
        A dictionary of system info name/value pairs.

    '''

    cmd = ['/usr/bin/facter']
    ret = _run_cmd(cmd)
    if ret['subproc'].returncode != 0:
        _raise_ASError(('Failed command: \n%s \nError: \n%s') % \
            (' '.join(cmd), str(ret['err'])))

    facts = {}
    for fact in ret['out'].split('\n'):
        if fact: # Handle the new line at the end of the facter output
            name, val = fact.split(' => ')
            facts[ name ] = val.rstrip()

    return facts

def parse_provides_params(src):
    '''
    Description:
        Parse the provides parameters text message sent from the
        Config Server.

    Input:
        The provides parameters string obtained from the Config Server.

        The delimiters will be an | and an &

        To ensure all the data was received the entire string will be
        terminated with an "|".

        This will be a continuous text string (no CR or New Line).

        Format:
        |name1&name2...&nameN|

        e.g.:
        |ipaddress&virtual|

    Returns:
        - a list of parameter names.
    '''

    _common_validate_message(src)

    # Message specific validation
    if src == '||':
        # special case indicating no provides parameters requested.
        return ['']

    params_str = src[src.find('|')+1:len(src)-1]

    return params_str.split('&')

def generate_provides(src):
    '''
    Description:
        Generate the provides parameters list.
        Uses parse_provides_params()

    Input:
        The provides parameters string obtained from the Config Server.

    Returns:
        A string to send back to the Config Server  with prifix
        'audrey_data='<url encoded return data>'

        The return portion will be delimited with an | and an &

        To ensure all the data is transmitted the entire string will be
        terminated with an "|".

        This will be a continuous text string (no CR or New Line).

        Data portion Format:
        |name1&val1|name2&val...|nameN$valN|

        e.g.:
        |ipaddress&<b64/10.118.46.205>|virtual&<b64/xenu>|

        The return string format:
        "audrey_data=<url encoded data portion>"


    '''
    LOGGER.info('Invoked generate_provides()')

    provides_dict = {}
    params_list = parse_provides_params(src)

    system_info_dict = _get_system_info()

    for param in params_list:
        try:
            provides_dict.update( \
                {param:base64.b64encode(system_info_dict[param])})
        except KeyError:
            # A specified parameter is not found. Provide value ''
            provides_dict.update({param:''})


    # Create string to send to Config Server
    provides_list = ['']
    for key in provides_dict.keys():
        provides_list.append(str(key) + '&' + str(provides_dict[key]))
    provides_list.append('')

    return urllib.urlencode({'audrey_data':'|'.join(provides_list)})

class ConfigTooling(object):
    '''
    TBD - Consider making this class derived from dictionary or a mutable
    mapping.

    Description:
        Interface to configuration tooling:
        - Getting optional user supplied tooling from CS
        - Verify and Unpack optional user supplied tooling retrieved
          from CS
        - Is tooling for a given service user supplied
        - Is tooling for a given service Red Hat supplied
        - Find tooling for a given service Red Hat supplied
        - List tooling for services and indicate if it is user or Red
          Hat supplied.
    '''

    def __init__(self, tool_dir=TOOLING_DIR):
        '''
        Description:
            Set initial state so it can be tracked. Valuable for
            testing and debugging.
        '''
        self.tool_dir = tool_dir
        self.user_dir = tool_dir + 'user/'
        self.log = tool_dir + 'log'
        self.tarball = ''

        # Create the extraction destination
        try:
            os.makedirs(self.user_dir)
        except OSError, (errno, strerror):
            if errno is 17: # File exists
                pass
            else:
                _raise_ASError(('Failed to create directory %s. ' + \
                    'Error: %s') % (self.user_dir, strerror))

        self.ct_logger = logging.getLogger('ConfigTooling')
        self.ct_logger.addHandler(logging.FileHandler(self.log))

    def __str__(self):
        '''
        Description:
            Called by the str() function and by the print statement to
            produce the informal string representation of an object.
        '''
        return('\n<Instance of: %s\n' \
               '\tTooling Dir: %s\n' \
               '\tUnpack User Tooling Tarball Dir: %s\n' \
               '\tLog File: %s\n' \
               '\ttarball Name: %s\n' \
               'eot>' %
            (self.__class__.__name__,
            str(self.tool_dir),
            str(self.user_dir),
            str(self.log),
            str(self.tarball),
            ))

    def log_info(self, log_str):
        '''
        Description:
            Used for logging the commands that have been executed
            along with their output and return codes.

            Simply logs the provided input string.
        '''
        self.ct_logger.info(log_str)

    def log_error(self, log_str):
        '''
        Description:
            Used for logging errors encountered when attempting to
            execute the service command.

            Simply logs the provided input string.
        '''
        self.ct_logger.error(log_str)

    def invoke_tooling(self, services):
        '''
        Description:
            Invoke the configuration tooling for the specified services.

        Input:
            services - A list of ServiceParams objects.

        '''

        # For now invoke them all. Later versions will invoke the service
        # based on the required params from the Config Server.
        LOGGER.debug('Invoked ConfigTooling.invoke_tooling()')
        LOGGER.debug(str(services))
        for service in services:

            try:
                top_level, tooling_path = self.find_tooling(service.name)
            except ASError:
                # No tooling found. Try the next service.
                continue

            cmd = [tooling_path]
            cmd_dir = os.path.dirname(tooling_path)
            ret = _run_cmd(cmd, cmd_dir)
            self.log_info('Execute Tooling command: ' + ' '.join(cmd))

            retcode = ret['subproc'].returncode
            if retcode == 0:
                # Command successed, log the output.
                self.log_info('return code: ' + str(retcode))
                self.log_info('\n\tStart Output of: ' + ' '.join(cmd) + \
                    ' >>>\n' +  \
                    str(ret['out']) + \
                    '\n\t<<< End Output')
            else:
                # Command failed, log the errors.
                self.log_info('\n\tStart Output of: ' + ' '.join(cmd) + \
                    ' >>>\n' +  \
                    str(ret['out']) + \
                    '\n\t<<< End Output')
                self.log_error('error code: ' + str(retcode))
                self.log_error('error msg:  ' + str(ret['err']))

            # If tooling was provided at the top level only run it once
            # for all services listed in the required config params.
            if top_level:
                break

    def unpack_tooling(self, tarball):
        '''
        Description:
            Methods used to untar the user provided tarball

            Perform validation of the text message sent from the
            Config Server. Validate, open and write out the contents
            of the user provided tarball.
        '''
        LOGGER.info('Invoked unpack_tooling()')
        LOGGER.debug('tarball: ' + str(tarball) + \
            'Target Direcory: ' + str(self.user_dir))

        self.tarball = tarball

        # Validate the specified tarfile.
        try:
            if not tarfile.is_tarfile(self.tarball):
                # If file exists but is not a tar file force IOError.
                raise IOError
        except IOError, (errno, strerror):
            _raise_ASError(('File was not found or is not a tar file: %s ' + \
                    'Error: %s %s') % (self.tarball, errno, strerror))

        # Attempt to extract the contents from the specified tarfile.
        #
        # If tarfile access or content is bad report to the user to aid
        # problem resolution.
        try:
            tarf = tarfile.open(self.tarball)
            tarf.extractall(path=self.user_dir)
            tarf.close()
        except IOError, (errno, strerror):
            _raise_ASError(('Failed to access tar file %s. Error: %s') %  \
                (self.tarball, strerror))
        # Capture and report errors with the tarfile
        except (tf.TarError, tf.ReadError, tf.CompressionError, \
            tf.StreamError, tf.ExtractError), (strerror):

            _raise_ASError(('Failed to access tar file %s. Error: %s') %  \
                (self.tarball, strerror))

    def is_user_supplied(self):
        '''
        Description:
            Is the the configuration tooling for the specified service
            supplied by the user?

            TBD: Take in a service_name and evaluate.
            def is_user_supplied(self, service_name):
        '''
        return True

    def is_rh_supplied(self):
        '''
        Description:
            Is the the configuration tooling for the specified service
            supplied by Red Hat?

            TBD: Take in a service_name and evaluate.
            def is_rh_supplied(self, service_name):
        '''
        return False

    def find_tooling(self, service_name):
        '''
        Description:
            Given a service name return the path to the configuration
            tooling.

            Search for the service start executable in the user
            tooling directory.
                self.tool_dir + '/user/<service name>/start'

            If not found there search for the it in the documented directory
            here built in tooling should be placed.
                self.tool_dir + '/AUDREY_TOOLING/<service name>/start'

            If not found there search for the it in the Red Hat tooling
            directory.
                self.tool_dir + '/REDHAT/<service name>/start'

           If not found there raise an error.

        Returns:
            return 1 - True if top level tooling found, False otherwise.
            return 2 - path to tooling
        '''

        top_path = self.tool_dir + 'user/start'
        if os.access(top_path, os.X_OK):
            return True, top_path

        service_user_path = self.tool_dir + 'user/' + \
            service_name + '/start'
        if os.access(service_user_path, os.X_OK):
            return False, service_user_path

        service_redhat_path = self.tool_dir + 'AUDREY_TOOLING/' + \
            service_name + '/start'
        if os.access(service_redhat_path, os.X_OK):
            return False, service_redhat_path

        service_redhat_path = self.tool_dir + 'REDHAT/' + \
            service_name + '/start'
        if os.access(service_redhat_path, os.X_OK):
            return False, service_redhat_path

        # No tooling found. Raise an error.
        _raise_ASError(('No configuration tooling found for service: %s') % \
            (service_name))

class CSClient(object):
    '''
    Description:
        Client interface to Config Server (CS)
    '''

    def __init__(self, endpoint, oauth_key, oauth_secret, **kwargs):
        '''
        Description:
            Set initial state so it can be tracked. Valuable for
            testing and debugging.
        '''

        self.version = CS_API_VER
        self.cs_endpoint = endpoint
        self.cs_oauth_key = oauth_key
        self.cs_oauth_secret = oauth_secret
        self.ec2_user_data_url = EC2_USER_DATA_URL
        self.cs_params = ''
        self.cs_configs = ''
        self.tmpdir = ''
        self.tarball = ''

        # create an oauth client for communication with the cs
        consumer = oauth.Consumer(self.cs_oauth_key, self.cs_oauth_secret)
        # 2 legged auth, token unnessesary
        token = None #oauth.Token('access-key-here','access-key-secret-here')
        client = oauth.Client(consumer, token)
        self.http = client

    def __del__(self):
        '''
        Description:
            Class destructor
        '''
        try:
            shutil.rmtree(self.tmpdir)
        except OSError:
            pass # ignore any errors when attempting to remove the temp dir.

    def __str__(self):
        '''
        Description:
            Called by the str() function and by the print statement to
            produce the informal string representation of an object.
        '''
        return('\n<Instance of: %s\n' \
               '\tVersion: %s\n' \
               '\tConfig Server Endpoint: %s\n' \
               '\tConfig Server oAuth Key: %s\n' \
               '\tConfig Server oAuth Secret: %s\n' \
               '\tConfig Server Params: %s\n' \
               '\tConfig Server Configs: %s\n' \
               '\tTemporary Directory: %s\n' \
               '\tTarball Name: %s\n' \
               'eot>' %
            (self.__class__.__name__,
            str(self.version),
            str(self.cs_endpoint),
            str(self.cs_oauth_key),
            str(self.cs_oauth_secret),
            str(self.cs_params),
            str(self.cs_configs),
            str(self.tmpdir),
            str(self.tarball),
            ))

    def _cs_url(self, url_type):
        '''
        Description:
            Generate the Config Server (CS) URL.
        '''
        return '%s/%s/%s/%s' % \
            (self.cs_endpoint, url_type, self.version, self.cs_oauth_key)

    def _get(self, url, headers=None):
        '''
        Description:
            Issue the http get to the the Config Server.
        '''
        try:
            return self.http.request(url, method='GET', headers=headers)
        except Exception, e:
            return (e, None)

    def _put(self, url, body=None, headers=None):
        '''
        Description:
            Issue the http put to the the Config Server.
        '''
        try:
            return self.http.request(url, method='PUT',
                                body=body, headers=headers)
        except Exception, e:
            return (e, None)

    def _validate_http_status(self, status):
        '''
        Description:
            Confirm the http status is one of:
            200 HTTP OK - Success and no more data of this type
            202 HTTP Accepted - Success and more data of this type
            404 HTTP Not Found - This may be temporary so try again
        '''
        if (status != 200) and (status != 202) and (status != 404):
            _raise_ASError(('Invalid HTTP status code: %s') % \
                (str(status)))

    # Public interfaces
    def get_cs_configs(self):
        '''
        Description:
            get the required configuration from the Config Server.
        '''
        LOGGER.info('Invoked CSClient.get_cs_configs()')
        url = self._cs_url('configs')
        headers = {'Accept': 'text/plain'}

        response, body = self._get(url, headers=headers)
        self.cs_configs = body
        self._validate_http_status(response.status)

        return response.status, body

    def get_cs_params(self):
        '''
        Description:
            get the provides parameters from the Config Server.
        '''
        LOGGER.info('Invoked CSClient.get_cs_params()')
        url = self._cs_url('params')
        headers = {'Accept': 'text/plain'}

        response, body = self._get(url, headers=headers)
        self.cs_params = body
        self._validate_http_status(response.status)

        return response.status, body

    def put_cs_params_values(self, params_values):
        '''
        Description:
            put the provides parameters to the Config Server.
        '''
        LOGGER.info('Invoked CSClient.put_cs_params_values()')
        url = self._cs_url('params')
        headers = {'Content-Type': 'application/x-www-form-urlencoded'}

        response, body = self._put(url, body=params_values, headers=headers)
        return response.status, body

    def get_cs_tooling(self):
        '''
        Description:
            get any optional user supplied tooling which is
            provided as a tarball
        '''
        LOGGER.info('Invoked CSClient.get_cs_tooling()')
        url = self._cs_url('files')
        headers = {'Accept': 'content-disposition'}

        tarball = ''
        response, body = self._get(url, headers=headers)
        self._validate_http_status(response.status)

        # Parse the file name burried in the response header
        # at: response['content-disposition']
        # as: 'attachment; tarball="tarball.tgz"'
        if (response.status == 200) or (response.status == 202):
            tarball = response['content-disposition']. \
                lstrip('attachment; filename=').replace('"','')

            # Create the temporary tarfile
            try:
                self.tmpdir = tempfile.mkdtemp()
                self.tarball = self.tmpdir + '/' + tarball
                f  = open(self.tarball, 'w')
                f.write(body)
                f.close()
            except IOError, (errno, strerror):
                _raise_ASError(('File not found or not a tar file: %s ' + \
                        'Error: %s %s') % (self.tarball, errno, strerror))

        return response.status, self.tarball

def discover_config_server(cloud_info_file=CLOUD_INFO_FILE,
                           condor_addr_file=CONDORCLOUD_CS_ADDR,
                           condor_uuid_file=CONDORCLOUD_CS_UUID,
                           ec2_user_data=EC2_USER_DATA_URL,
                           http=httplib2.Http()):
    '''
    Description:
        Discover the Config Server access info.
        If not discover it using the cloud provider specific method.
    '''
    #
    # What Cloud Backend?
    #
    # Read the file populated with Cloud back end type.
    # e.g.: CLOUD_TYPE="EC2"
    #

    def _parse_user_data(data, condor=None):
        '''
        Take a string in form version|cs_endpoint|oauth_key|oauth_secret
        and populate the respective self vars.
        Conductor puts the UUID into the oauth_key field.
        At minimum this function expects to find a | in the string
        this is in effort not to log oauth secrets.
        '''
        LOGGER.debug('Parsing User Data')
        user_data = data.split('|')
        if len(user_data) > 1:
            if user_data[0] == '1':
                if condor:
                    ud_version, endpoint, \
                        oauth_secret = user_data
                    oauth_key = condor
                else:
                    ud_version, endpoint, \
                        oauth_key, oauth_secret = user_data
                return {'endpoint': endpoint,
                        'oauth_key': oauth_key,
                        'oauth_secret': oauth_secret,}
            #elif ud[0] == nextversion
            #    parse code for version
            else:
                _raise_ASError('Invalid User Data Version: %s' % user_data[0])
        else:
            _raise_ASError('Could not get user data version, parse failed')

    # This could be done using "with open()" but that's not available
    # in Python 2.4 as used on RHEL5
    try:
        fp = open(cloud_info_file, 'r')
        try:
            read_data = fp.read()
        finally:
            fp.close()
    except IOError:
        _raise_ASError(('Failed accessing file %s') % \
            (cloud_info_file))

    #
    # Discover the Config Server access info.
    #
    cloud_type = read_data.upper()
    if 'EC2' in cloud_type:
        #
        # If on EC2 the user data will contain the Config Server
        # access info.
        #

        try:
            max_attempts = 5
            headers = {'Accept': 'text/plain'}
            for attempt in range(1, max_attempts):
                response, body = http.request(ec2_user_data,
                                              headers=headers)
                if response.status == 200:
                    break
            if response.status != 200:
                _raise_ASError('Max attempts to get EC2 user data \
                        exceeded.')

            if '|' not in body:
                body = base64.b64decode(body)
            return _parse_user_data(body)

        except Exception, e:
            _raise_ASError('Failed accessing EC2 user data: %s' % e)

    elif 'CONDORCLOUD' in cloud_type:
        #
        # If on Condor Cloud, the user data will be in smbios
        # Uses the dmi files to access the stored smbios information.
        #
        try:
            return _parse_user_data(open(condor_addr_file, 'r').read().strip(),
                                    open(condor_uuid_file, 'r').read().strip())
        except Exception, e:
            _raise_ASError('Failed accessing Config Server data: %s' % e)

    elif 'RHEV' in cloud_type:
        #
        # If on RHEV-M the user data will be contained on the
        # floppy device in file deltacloud-user-data.txt.
        # To access it:
        #    modprobe floppy
        #    mount /dev/fd0 /media
        #    read /media/deltacloud-user-data.txt
        #
        # Note:
        # On RHEVm the deltacloud drive had been delivering the user
        # data base64 decoded at one point that changed such that the
        # deltacloud drive leaves the date base64 encoded. This
        # Code segment will handle both base64 encoded and decoded
        # user data.
        #
        # Since ':' is used as a field delimiter in the user data
        # and is not a valid base64 char, if ':' is found assume
        # the data is already base64 decoded.
        #
        #    modprobe floppy
        cmd = ['/sbin/modprobe', 'floppy']
        ret = _run_cmd(cmd)
        if ret['subproc'].returncode != 0:
            _raise_ASError(('Failed command: \n%s \nError: \n%s') % \
                (' '.join(cmd), str(ret['err'])))

        cmd = ['/bin/mkdir', '/media']
        ret = _run_cmd(cmd)
        # If /media is already there (1) or any other error (0)
        if (ret['subproc'].returncode != 1) and  \
           (ret['subproc'].returncode != 0):
            _raise_ASError(('Failed command: \n%s \nError: \n%s') % \
                (' '.join(cmd), str(ret['err'])))

        cmd = ['/bin/mount', '/dev/fd0', '/media']
        ret = _run_cmd(cmd)
        # If /media is already mounted (32) or any other error (0)
        if (ret['subproc'].returncode != 32) and  \
           (ret['subproc'].returncode != 0):
            _raise_ASError(('Failed command: \n%s \nError: \n%s') % \
                (' '.join(cmd), str(ret['err'])))

        # Condfig Server (CS) address:port.
        # This could be done using "with open()" but that's not available
        # in Python 2.4 as used on RHEL5
        try:
            fp = open('/media/deltacloud-user-data.txt', 'r')
            try:
                line = fp.read().strip()
                if '|' not in line:
                    line = base64.b64decode(line)
                return _parse_user_data(line)
            finally:
                fp.close()
        except:
            _raise_ASError('Failed accessing RHEVm user data.')

    elif 'VSPHERE' in cloud_type:
        #
        # If on vSphere the user data will be contained on the
        # floppy device in file deltacloud-user-data.txt.
        # To access it:
        #    mount /dev/fd0 /media
        #    read /media/deltacloud-user-data.txt
        #
        # Note:
        # On vSphere the deltacloud drive had been delivering the user
        # data base64 decoded at one point that changed such that the
        # deltacloud drive leaves the date base64 encoded. This
        # Code segment will handle both base64 encoded and decoded
        # user data.
        #
        # Since ':' is used as a field delimiter in the user data
        # and is not a valid base64 char, if ':' is found assume
        # the data is already base64 decoded.
        #
        cmd = ['/bin/mkdir', '/media']
        ret = _run_cmd(cmd)
        # If /media is already there (1) or any other error (0)
        if (ret['subproc'].returncode != 1) and  \
           (ret['subproc'].returncode != 0):
            _raise_ASError(('Failed command: \n%s \nError: \n%s') % \
                (' '.join(cmd), str(ret['err'])))

        cmd = ['/bin/mount', '/dev/cdrom', '/media']
        ret = _run_cmd(cmd)
        # If /media is already mounted (32) or any other error (0)
        if (ret['subproc'].returncode != 32) and  \
           (ret['subproc'].returncode != 0):
            _raise_ASError(('Failed command: \n%s \nError: \n%s') % \
                (' '.join(cmd), str(ret['err'])))

        # Condfig Server (CS) address:port.
        # This could be done using "with open()" but that's not available
        # in Python 2.4 as used on RHEL5
        try:
            fp = open('/media/deltacloud-user-data.txt', 'r')
            try:
                line = fp.read().strip()
                if '|' not in line:
                    line = base64.b64decode(line)
                return _parse_user_data(line)
            finally:
                fp.close()
        except:
            _raise_ASError('Failed accessing vSphere user data.')

def setup_logging(level=logging.INFO, logfile_name=LOG):
    '''
    Description:
        Establish the output logging.
    '''

    global LOGGER

    # If not run as root create the log file in the current directory.
    # This allows minimal functionality, e.g.: --help
    if not os.geteuid() == 0:
        logfile_name = './audrey.log'

    # set up logging
    LOG_FORMAT = ('%(asctime)s - %(levelname)-8s: '
        '%(filename)s:%(lineno)d %(message)s')
    LOG_LEVEL_INPUT = 5
    LOG_NAME_INPUT = 'INPUT'

    logging.basicConfig(filename=logfile_name,
        level=level, filemode='w', format=LOG_FORMAT)

    logging.addLevelName(LOG_LEVEL_INPUT, LOG_NAME_INPUT)

    LOGGER = logging.getLogger('Audrey')

def parse_args():
    '''
    Description:
        Gather any Config Server access info optionally passed
        on the command line. If being provided on the command
        line all of it must be provided.

        oAuth Secret is prompted for and not allowed as an argument.
        This is to avoid a ps on the system from displaying the
        oAuth Secret argument.

    Return:
        dict - of parser keys and values
    '''
    desc_txt = 'The Aeolus Audrey Startup Agent, a script which ' + \
               'runs on a booting cloud instance to retrieve ' + \
               'configuration data from the Aeolus Config Server.'

    log_level_dict={'DEBUG' : logging.DEBUG,
        'INFO' : logging.INFO,
        'WARNING' : logging.WARNING,
        'ERROR' : logging.ERROR,
        'CRITICAL' : logging.CRITICAL}

    parser = argparse.ArgumentParser(description=desc_txt)
    parser.add_argument('-e', '--endpoint', dest='endpoint',
        required=False, help='Config Server endpoint url')
    parser.add_argument('-k', '--key', dest='oauth_key', required=False,
        help='oAuth Key. If specified prompt for the oAuth Secret.')
    parser.add_argument('-p', '--pwd', action='store_true', default=False,
        required=False, help='Log and look for configs in pwd',)
    parser.add_argument('-L', '--log-level', dest='log_level',
        required=False, default='INFO', help='Audrey Agent Logging Level',
        choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']),
    parser.add_argument('-V', '-v', '--version', dest='version',
        action='store_true', default=False, required=False,
        help='Displays the program\'s version number and exit.')

    args = parser.parse_args()
    args.log_level = log_level_dict[args.log_level]

    if args.version:
        print AUDREY_VER
        sys.exit()

    if args.oauth_key:
        # Prompt for oAuth secret so ps won't display it.
        args.oauth_secret = raw_input('oAuth Secret: ')

    return args

def audrey_script_main(client_http=None):
    '''
    Description:
        This script will be used on EC2 for configuring the running
        instance based on Cloud Engine configuration supplied at
        launch time in the user data.

        Config Server Status:
        200 HTTP OK - Success and no more data of this type
        202 HTTP Accepted - Success and more data of this type
        404 HTTP Not Found - This may be temporary so try again
    '''
    # parse the args and setup logging
    conf = parse_args()
    if 'pwd' in conf and conf.pwd:
        log_file = 'audrey.log'
        tool_dir = 'tooling'
        cloud_info = 'cloud_info'
    else:
        log_file = LOG
        tool_dir = TOOLING_DIR
        cloud_info = CLOUD_INFO_FILE

    setup_logging(level=conf.log_level,
            logfile_name=log_file)

    if not conf.endpoint:
        if client_http:
            conf = discover_config_server(cloud_info_file=cloud_info,
                                          http=client_http)
        else:
            # discover the cloud I'm on
            conf = discover_config_server(cloud_info_file=cloud_info)

    # ensure the conf it a dictionary, not a namespace
    if hasattr(conf, '__dict__'):
        conf = vars(conf)

    LOGGER.info('Invoked audrey_script_main')

    # 0 means don't run again
    # -1 is non zero so initial runs will happen
    config_status = -1
    param_status = -1
    tooling_status = -1

    max_retry = 5
    services = []

    # Create the Client Object
    cs_client = CSClient(**conf)
    if client_http:
        cs_client.http = client_http
    # test connectivity, try and wait for it if it's not there
    url = cs_client._cs_url('version')
    while isinstance(cs_client._get(url)[0], Exception):
        if max_retry:
            max_retry-=1
            LOGGER.info('Failed attempt to contact config server')
            sleep(10)
        else:
            LOGGER.error('Failed to connect to the Configserver')
            exit(1)

    max_retry = 5

    LOGGER.info(str(cs_client))

    LOGGER.debug('Get optional tooling from the Config Server')
    # Get any optional tooling from the Config Server
    tooling = ConfigTooling(tool_dir=tool_dir)
    tooling_status, tarball = cs_client.get_cs_tooling()
    if (tooling_status == 200) or (tooling_status == 202):
        tooling.unpack_tooling(tarball)
    else:
        LOGGER.info('No optional config tooling provided. status: ' + \
                str(tooling_status))
    LOGGER.debug(str(tooling))

    LOGGER.debug('Process the Requires and Provides parameters')

    # Process the Requires and Provides parameters until the HTTP status
    # from the get_cs_configs and the get_cs_params both return 200
    while config_status or param_status:

        LOGGER.debug('Config Parameter status: ' + str(config_status))
        LOGGER.debug('Return Parameter status: ' + str(param_status))

        # Get the Required Configs from the Config Server
        if config_status:
            config_status, configs = cs_client.get_cs_configs()

            # Configure the system with the provided Required Configs
            if config_status == 200:
                services = parse_require_config(configs)
                tooling.invoke_tooling(services)
                # don't do any more config status work
                # now that the tooling has run
                config_status = 0
            else:
                LOGGER.info('No configuration parameters provided. status: ' + \
                    str(config_status))

        # Get the requested provides from the Config Server
        if param_status:
            get_status, params = cs_client.get_cs_params()

            # Gather the values from the system for the requested provides
            if get_status == 200:
                params_values = generate_provides(params)
            else:
                params_values = '||'

            # Put the requested provides with values to the Config Server
            param_status, body = cs_client.put_cs_params_values(params_values)
            if param_status == 200:
                # don't operate on params anymore, all have been provided.
                param_status = 0

        # Retry a number of times if 404 HTTP Not Found is returned.
        if config_status == 404 or param_status == 404:
            LOGGER.error('Requiest to Config Server failed or more to come.')
            LOGGER.error('Required Config Parameter status: ' + \
                str(config_status))
            LOGGER.info('Return Parameter status: ' + str(param_status))

            max_retry -= 1
            if max_retry < 0:
                _raise_ASError('Too many erroneous Config Server responses.')

        sleep(10)

if __name__ == '__main__':

    audrey_script_main()
