#!/usr/bin/python

import re
import sys
import os
import time
import subprocess
import logging
import logging.handlers
import traceback

from pwd import getpwnam

import pyudev

import json
import xml.etree.ElementTree as ET

from optparse import OptionParser

# Create logger
progname = os.path.basename(os.path.abspath(sys.argv[0]))

logger = logging.getLogger(progname)
logger.setLevel(logging.INFO)

logout = logging.getLogger(progname + '-std')
logout.setLevel(logging.INFO)
outhandler = logging.StreamHandler(sys.stdout)
outhandler.setFormatter(logging.Formatter(progname + ': ' + '%(message)s'))
logout.addHandler(outhandler)

if os.isatty(0):
    handler = logging.StreamHandler(sys.stderr)
    handler.setFormatter(logging.Formatter(progname + ': %(levelname)s: ' + '%(message)s'))
else:
    handler = logging.handlers.SysLogHandler(address='/dev/log')
    handler.setFormatter(logging.Formatter(progname + '[%(process)d]: ' + '%(message)s'))

logger.addHandler(handler)

reg_detl_attr = re.compile("[ ]*([A-Za-z0-9 ]+) : (([^ \n]|.[^ \n])+)")
reg_detl_memb = re.compile("[ ]*([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+(([A-Za-z]|[ ][A-Za-z])+)[ ]*([/a-z0-9]*)")

reg_scan_array = re.compile("ARRAY ([a-zA-z/]+)([0-9]+) .+")
reg_scan_membs = re.compile("[ ]+devices=([^ \n]+)")

reg_stat_ident = re.compile("(md[0-9]+) : [a-z]+ ([(][a-zA-Z0-9-]+[)] )?raid([0-9]+) ([^\n]+)")
reg_stat_membs = re.compile("([a-z0-9]+)\\[([0-9]+)\\](\\(([A-Za-z]+)\\))?")
reg_stat_super = re.compile("[ ]+([0-9]+) blocks super ([^ ]+) \\[([0-9]+)/([0-9]+)\\] \\[([^\n]+)\\]")
reg_stat_recvr = re.compile("[ ]+\\[(.+)\\][ ]+(recovery|resync) =[ ]+(([0-9.%]+) [(]([0-9]+)/([0-9]+)[)] finish=([^ ]+) speed=([^\n ]+))|([A-Z]+)")

reg_info_parm  = re.compile("[ ]+Model=(([^, ]+) )?([^,]+), FwRev=([^,]+), SerialNo=(.+)")
reg_info_sctl = re.compile("([^:]+):[ ]+([^ ].*)")
####

def system_echo(cmdline):
    logger.debug('+ executing "' + cmdline + '"...')
    return os.system(cmdline)

def get_block_size(devname):
    try:
        # sector count -> mb
        num = int(file("/sys/block/" + devname + "/size").readline()) / 2048

        if num > 1000:
            return '%0.2f' % (float(num) / 1000) + ' GB'
        else:
            return str(num) + ' MB'

    except Exception, e:
        logger.debug('unable to read block device size: ' + str(e))
        return 0

def normalize_name(part, withdev=True, withpart=False):
    match = re.match(r"(/dev/)?(.+[a-zA-Z])([0-9]*)", part)

    if match:
        ret  = [ '', '/dev/' ][int(withdev)]
        ret += match.group(2)
        ret += [ '', match.group(3) ][int(withpart)]

        return ret

    return None


####

## unused for now, kept for possible future uses
def mdadm_scan(verbose):
    rets = {}
    args = [ "mdadm", "--examine", "--scan" ]

    if verbose:
        args.append("--verbose")

    out, err = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()
    #out = file('scan.txt', 'r')

    array = None

    for line in out.splitlines():
    #for line in out.readlines():
        if not array:
            match1 = reg_scan_array.match(line)
            if match1:
                array = 'md' + match1.group(2)
                if not verbose:
                    rets[array]=[]
                    array = None
        else:
            match2 = reg_scan_membs.match(line)
            if match2:
                rets[array] = match2.group(1).split(',')
                array = None

    return rets

## unused for now, kept for possible future uses
def mdadm_detail(array):
    attrs = {}
    membs = []

    args = [ "mdadm", "--detail", array ]

    out, err = subprocess.Popen(args, stdout=subprocess.PIPE).communicate()

    for line in out.splitlines():
        match1 = reg_detl_attr.match(line)
        if match1:
            attrs[match1.group(1)] = match1.group(2)
        else:
            match2 = reg_detl_memb.match(line)
            if match2:
                membs.append({'device' : match2.group(7), 'state' : match2.group(5)})

    return {'attributes' : attrs, 'members' : membs}

####

def process_mdstat(mdstat):
    data = {}
    numb = None

    if mdstat is None:
        mdstat = '/proc/mdstat'

    for line in file(mdstat).readlines():
        match1 = reg_stat_ident.match(line)
        if match1:
            membs = []
            membl = []
            for memb in match1.group(4).split(' '):
                match1a = reg_stat_membs.match(memb)
                if match1a:
                    state = 'online'
                    flags = match1a.group(4)

                    if flags:
                        if   flags == 'F':
                            state = 'failed'
                        elif flags == 'S':
                            state = 'stopped'

                    membs.append({'device' : match1a.group(1), 'seqnum' : match1a.group(2), 'state': state})
                    membl.append(match1a.group(1))

            numb = match1.group(1)

            data[numb] = {}
            data[numb]['attributes'] = { 'level' : match1.group(3), 'members': membl }
            data[numb]['status']     = { 'members' : membs }

        elif numb is not None:
            match2 = reg_stat_super.match(line)
            if match2:
                data[numb]['attributes']['super-version'] = match2.group(2)
                data[numb]['attributes']['device-total'] = match2.group(3)
                data[numb]['attributes']['block-count'] = match2.group(1)

                data[numb]['status']['device-total'] = match2.group(3)
                data[numb]['status']['device-count'] = match2.group(4)
                data[numb]['status']['device-state'] = match2.group(5)
            else:
                match3 = reg_stat_recvr.match(line)
                if match3:
                    if match3.group(3) is not None:
                        data[numb]['status'][match3.group(2)] = {
                            'state': 'ongoing',
                            'progress-bar': match3.group(1),
                            'progress': match3.group(4),
                            'block-current': match3.group(5),
                            'block-total': match3.group(6),
                            'eta': match3.group(7),
                            'rate': match3.group(8) }
                    else:
                        data[numb]['status'][match3.group(2)] = {
                            'state': match3.group(9).lower() }

    return data

####

def info_disk_smartctl(part):
    try:
        disk = normalize_name(part)
        name = normalize_name(part, withdev=False)
        args = [ "/usr/local/sng/bin/sng-disk-info", "smartctl", disk ]

        out, err = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()

        data = {}
        for line in out.splitlines():
            match2 = reg_info_sctl.match(line)
            if match2:
                data[match2.group(1)] = match2.group(2)

        vendor = data.get('Vendor', data.get('Model Family'))
        model  = data.get('Product', data.get('Device Model'))

        if vendor is None:
            ret = model.split(' ', 2)

            if len(ret) == 2:
                vendor = ret[0]
                model = ret[1]
            else:
                vendor = 'Unknown'

        serial = data.get('Serial Number')

        return (name, vendor, model, serial, get_block_size(name))

    except Exception, e:
        logger.debug('unable to get smartctl info: ' + str(e) + ' (' + part + ')')

    return None

def info_disk_hdparm(part):
    try:
        disk = normalize_name(part)
        name = normalize_name(part, withdev=False)
        args = [ "/usr/local/sng/bin/sng-disk-info", "hdparm", disk ]

        out, err = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()

        for line in out.splitlines():
            match2 = reg_info_parm.match(line)
            if match2:
                return (name, match2.group(2), match2.group(3), match2.group(5), get_block_size(name))

    except Exception, e:
        logger.debug('unable to get hdparm info: ' + str(e) + ' (' + part + ')')

    return None

def info_disk_udev(part):
    ret = None

    try:
        disk = normalize_name(part, withdev=False)
        basedir = "/sys/block/" + disk + "/device"
        if os.access(basedir, os.R_OK):
            model  = open(basedir + "/model").read().strip()
            vendor = open(basedir + "/vendor").read().strip()
            modeltoks = model.split(' ', 2)

            if len(modeltoks) == 1 or (vendor <> "ATA"):
                ret = (disk, vendor, model, None, get_block_size(disk))
            else:
                ret = (disk, modeltoks[0], modeltoks[1], None, get_block_size(disk))

    except Exception, e:
        logger.debug('unable to get udev info: ' + str(e) + ' (' + part + ')')

    return ret

def info_disk(part):
    ret = info_disk_hdparm(part)

    if ret is None:
        ret = info_disk_udev(part)

    if ret is not None:
        return { 'name' : ret[0], 'vendor' : ret[1], 'model' : ret[2], 'serial' : ret[3], 'size': ret[4] }

    return { 'name' : normalize_name(part, False) }


####

def get_all_disks():
    devices = []
    context = pyudev.Context()

    for device in context.list_devices(subsystem='block', DEVTYPE='disk'):

        idtype = device.get('ID_TYPE', 'none')
        idbus  = device.get('ID_BUS', 'none')

        if idtype == 'disk' and idbus <> 'usb' and idbus <> 'none':
            devices.append(device.sys_name)

    return devices

####

def get_inventory_status(mdstat, alldisks, has_inventory=True, has_status=True):
    arrays = []
    mdisks = []
    udisks = []

    status = []

    memlst = []

    for array in mdstat.keys():
        if has_inventory:
            attrs = { 'name': array }

            for key in mdstat[array]['attributes'].keys():
                attrs[key] = mdstat[array]['attributes'][key]

            arrays.append(attrs)

        if has_status:
            stats = { 'name': array }

            for key in mdstat[array]['status'].keys():
                stats[key] = mdstat[array]['status'][key]

            if   mdstat[array]['status'].get('recovery') is not None:
                    stats['state']='recovery'
            elif mdstat[array]['status'].get('resync') is not None:
                    stats['state']='resync'
            else:
                if mdstat[array]['status']['device-total'] == mdstat[array]['status']['device-count']:
                    stats['state']='normal'
                else:
                    stats['state']='degraded'

            status.append(stats)

        for memb in mdstat[array]['status']['members']:
            disk = normalize_name(memb['device'])
            if disk not in memlst:
                memlst.append(disk)

    ret_inventory = None
    ret_status = None

    if has_inventory:
        for disk in memlst:
            mdisks.append(info_disk(disk))

        for disk in alldisks:
            info = info_disk(disk)
            if info not in mdisks:
                udisks.append(info)

        ret_inventory = { 'arrays': arrays, 'devices': mdisks, 'spare-disks': udisks }

    if has_status:
        ret_status = { 'arrays': status }

    return ret_inventory, ret_status

def string_of_dict(dic):
    ret = []
    for key in dic.keys():
        if dic[key] is not None:
            ret.append(key + ':' + dic[key])
        else:
            ret.append(key)

    return ';'.join(ret)

def dict_of_string(ss):
    ret = {}
    for item in ss.split(';'):
        ls = item.split(':')

        if   len(ls) == 2:
            ret[ls[0]] = ls[1]
        elif len(ls) == 1:
            ret[ls[0]] = None

    return ret

def descr_of_device(dev):
    vendor = dev.get('vendor')
    model = dev.get('model', '')
    name = dev.get('name', '')

    if vendor is not None:
        ret = vendor + ' ' + model + ' [' + dev.get('name') + ']'
    else:
        ret = model + ' [' + dev.get('name') + ']'

    extra = []

    if dev.get('serial'):
        extra.append('serial ' + dev.get('serial'))

    if dev.get('size'):
        extra.append('size ' + dev.get('size'))

    if len(extra) <> 0:
        ret = ret + ' (' + ', '.join(extra) + ')'

    return ret

def get_changes(curscan, newscan, mdstat, alldisks, allchanges=False):
    allscan = set()

    for key in curscan.keys():
        allscan.add(key)
    for key in newscan.keys():
        allscan.add(key)

    alldevs = []

    newarray = []
    delarray = []

    newdevice = {}
    deldevice = {}
    errdevice = {}
    rcvdevice = {}

    cmmdevice = {}

    for key in allscan:
        curdata = curscan.get(key)
        newdata = newscan.get(key)

        if curdata is None:
            newarray.append(key)
        elif newdata is None:
            delarray.append(key)
        else:
            curmembs = []
            newmembs = []

            for memb in curdata:
                curmembs.append(memb)

            for memb in newdata:
                newmembs.append(memb)

            for memb in curdata:
                if memb in newmembs:
                    newmembs.remove(memb)
                    lst = cmmdevice.get(string_of_dict(memb), [])
                    lst.append(key)
                    cmmdevice[string_of_dict(memb)]=lst

            for memb in newdata:
                if memb in curmembs:
                    curmembs.remove(memb)
                    lst = cmmdevice.get(string_of_dict(memb), [])
                    lst.append(key)
                    cmmdevice[string_of_dict(memb)]=lst

            for dev in curmembs:
                lst = deldevice.get(string_of_dict(dev), [])
                lst.append(key)
                deldevice[string_of_dict(dev)]=lst

                if dev not in alldevs:
                    alldevs.append(dev)

            for dev in newmembs:
                lst = newdevice.get(string_of_dict(dev), [])
                lst.append(key)
                newdevice[string_of_dict(dev)]=lst

                if dev not in alldevs:
                    alldevs.append(dev)

    has_degraded = False
    has_recovery = False

    block_cur = 0
    block_sum = 0

    for key in newscan:
        newdata = newscan[key]

        doing_resync = False

        if mdstat[key]['status']['device-total'] <> mdstat[key]['status']['device-count']:
            recovs = mdstat[key]['status'].get('recovery')
            resync = mdstat[key]['status'].get('resync')

            if recovs is None and resync is None:
                num = int(mdstat[key]['attributes']['block-count'])
                #block_sum = block_sum + num

                logger.debug('array %s degraded, %d blocks total', key, num)

                has_degraded = True

            elif recovs is not None:
                num = int(recovs.get('block-total', mdstat[key]['attributes']['block-count']))
                cur = int(recovs.get('block-current', '0'))
                block_cur = block_cur + cur
                block_sum = block_sum + num

                logger.debug('array %s in recovery, %d of %d blocks recovered', key, cur, num)

                has_recovery = True
                doing_resync = True

            elif resync is not None:
                num = int(resync.get('block-total', mdstat[key]['attributes']['block-count']))
                cur = int(resync.get('block-current', '0'))
                block_cur = block_cur + cur
                block_sum = block_sum + num

                logger.debug('array %s in resync, %d of %d blocks recovered', key, cur, num)

                doing_resync = True
        else:
            num = int(mdstat[key]['attributes']['block-count'])
            block_cur = block_cur + num
            block_sum = block_sum + num

            logger.debug('array %s OK, %d blocks total', key, num)

        lastmem = { 'seqnum': '0' }

        for memb in mdstat[key]['status']['members']:

            logger.debug('checking member %s...', memb['device'])

            if int(lastmem['seqnum']) <= int(memb['seqnum']):
                logger.debug('member %s is now the last one', memb['device'])
                lastmem = memb

            if memb['state'] == 'failed':
                dev = info_disk('/dev/' + memb['device'])

                lst = errdevice.get(string_of_dict(dev), [])
                lst.append(key)
                errdevice[string_of_dict(dev)]=lst

                if dev not in alldevs:
                    alldevs.append(dev)

        if doing_resync and lastmem is not None:
            logger.debug('array %s recoving device %s', key, lastmem)

            dev = info_disk(normalize_name(lastmem['device']))

            lst = rcvdevice.get(string_of_dict(dev), [])
            lst.append(key)
            rcvdevice[string_of_dict(dev)]=lst

            if dev not in alldevs:
                alldevs.append(dev)

    changes = []

    if allchanges:
        for array in allscan:
            if   array in newarray and array not in delarray:
                changes.append({ "type": "array-create",  "array": { 'name': array } })
            elif array not in newarray and array in delarray:
                changes.append({ "type": "array-destroy", "array": { 'name': array } })

    infodisks = []

    for disk in alldisks:
        info = info_disk(disk)
        if info not in infodisks:
            infodisks.append(info)

    if has_degraded:
        for dev in infodisks:
            if len(errdevice.get(string_of_dict(dev), [])) <> 0:
                logger.debug('device %s in error list, skipping...', dev)
                continue
            if len(newdevice.get(string_of_dict(dev), [])) <> 0:
                logger.debug('device %s in new list, skipping...', dev)
                continue
            if len(cmmdevice.get(string_of_dict(dev), [])) <> 0:
                logger.debug('device %s in common list, skipping...', dev)
                continue

            repkey = None
            repdev = None

            # for each spare, remove one error or remove
            if not allchanges:
                logger.debug('spare change device="%s", fail=%d, remove=%d' % (string_of_dict(dev), len(errdevice), len(deldevice)))
                if   len(errdevice) <> 0:
                    key = sorted(errdevice.keys())[0]
                    repkey = 'device-fail'
                    for dev2 in alldevs:
                        if string_of_dict(dev2) == key:
                            repdev = dev2
                            break
                    del errdevice[key]
                elif len(deldevice) <> 0:
                    key = sorted(deldevice.keys())[0]
                    repkey = 'device-remove'
                    for dev2 in alldevs:
                        if string_of_dict(dev2) == key:
                            repdev = dev2
                            break
                    del deldevice[key]

            change = { "type": "device-spare", "device": dev }

            if repkey is not None and repdev is not None:
                change['replaces'] = { 'cause': repkey, 'device': repdev }

            changes.append(change)

    newchanges = []
    delchanges = []
    errchanges = []

    for dev in alldevs:
        errlist = errdevice.get(string_of_dict(dev), [])
        newlist = newdevice.get(string_of_dict(dev), [])
        dellist = deldevice.get(string_of_dict(dev), [])
        rcvlist = rcvdevice.get(string_of_dict(dev), [])

        alllist = set()
        for e in errlist:
            alllist.add(e)
        for e in newlist:
            alllist.add(e)
        for e in dellist:
            alllist.add(e)
        for e in rcvlist:
            alllist.add(e)

        newarrays = []
        delarrays = []
        errarrays = []
        rcvarrays = []

        for array in alllist:
            if   array in newlist and array not in dellist and array not in errlist and array not in rcvlist:
                newarrays.append({ 'name': array })
            elif array not in newlist and array in dellist and array not in errlist and array not in rcvlist:
                delarrays.append({ 'name': array })
            elif array not in newlist and array not in dellist and array in errlist and array not in rcvlist:
                errarrays.append({ 'name': array })
            elif array not in newlist and array not in dellist and array not in errlist and array in rcvlist:
                rcvarrays.append({ 'name': array })

        if len(newarrays) <> 0:
            newchanges.append({ "device": dev, "arrays": newarrays })

        if len(delarrays) <> 0:
            delchanges.append({ "device": dev, "arrays": delarrays })

        if len(errarrays) <> 0:
            errchanges.append({ "device": dev, "arrays": errarrays })

        if has_recovery:
            if len(rcvarrays) <> 0:
                changes.append({ "type": "device-recovery", "device": dev, "arrays": rcvarrays, "progress": str((block_cur * 100) / block_sum) + '%' })

    replaces = {}

    for errchange in errchanges:
        errchange['type'] = 'device-fail'
        changes.append(errchange)

    for newchange in list(newchanges):
        for delchange in list(delchanges):
            newset = set()
            delset = set()

            for newelm in newchange['arrays']:
                newset.add(newelm['name'])

            for delelm in delchange['arrays']:
                delset.add(delelm['name'])

            if newset == delset and newchange['device']['name'] == delchange['device']['name']:
                num = 0
                if newchange in newchanges:
                    newchanges.remove(newchange)
                    num = num + 1
                if delchange in delchanges:
                    delchanges.remove(delchange)
                    num = num + 1

                logger.debug('new change: ' + str(newchange['device']))
                logger.debug('del change: ' + str(delchange['device']))

                if num == 2:
                    logger.info('device ' + descr_of_device(newchange['device']) + ' replaced ' + descr_of_device(delchange['device']) + '.')

                replaces[string_of_dict(delchange['device'])]=string_of_dict(newchange['device'])

    if allchanges:
        for newchange in newchanges:
            newchange['type'] = 'array-attach'
            changes.append(newchange)

    for delchange in delchanges:
        if dev in infodisks:
            if allchanges:
                delchange['type'] = 'array-detach'
            else:
                delchange['type'] = 'device-fail'
        else:
            delchange['type'] = 'device-remove'

        changes.append(delchange)

    return changes, replaces

def get_detail(arrays):
    results = []
    for array in arrays.keys():
        data = mdadm_detail('/dev/' + array)

        orig = data['attributes']
        dest = {}

        for item in orig.keys():
            dest[item.replace(' ', '-')] = orig[item]

        result = { 'name': array, 'attributes': dest, 'members': data['members'] }
        results.append(result)
    return results

####

def read_part_tables(disk):
    logout.debug('re-reading partition tables from ' + disk + '...')
    for i in xrange(0, 15):
        if system_echo("sfdisk -R " + disk) == 0:
            time.sleep(1.0)
            return True

        time.sleep(2.0)

    return False

def copy_boot_record(orig, dest):
    logout.debug('copying mbr from ' + orig + ' to ' + dest + '...')
    if system_echo("dd if=" + orig + " of=" + dest + " bs=446 count=1 >/dev/null") <> 0:
        return False

    return True

def copy_part_tables(orig, dest):
    logout.debug('copying partitions from ' + orig + ' to ' + dest + '...')
    if system_echo("sfdisk -d " + orig + " | sfdisk -f " + dest) <> 0:
        return False

    time.sleep(1.0)

    return read_part_tables(dest)

def first_spare_disk(devices, disks):
    memblist = []
    for key in devices.keys():
        for memb in devices[key]:
            if memb in memblist:
                continue

            memblist.append(memb)

    diskinfo = []
    for disk in disks:
        info = info_disk(disk)
        if info in memblist:
            continue
        diskinfo.append(info)

    if len(diskinfo) <> 0:
        return diskinfo[0]
    else:
        return None

def recover_arrays(arrays, newdisk, mdstat):
    srcdisk = None
    fwarray = {}

    if len(arrays) == 1: # container array is ignored
        for array in arrays.keys():
            super_version = mdstat[array]['attributes']['super-version']
            if super_version.startswith('external:'):
                logout.info('recovering RAID managed by SATA controller.')
                fwarray[array] = super_version.split('/')[1]

    if not os.access(newdisk, os.F_OK):
        logout.error('invalid destination disk, aborting recovery')
        return False

    logout.debug('searching for source disk...')

    for array in arrays.keys():
        skiparray = False
        for disk in arrays[array]:
            logout.debug('checking ' + disk + ' in ' + array + '...')
            mdisk = normalize_name(disk)
            if mdisk is not None:
                if mdisk == newdisk:
                    continue

                srcdisk = mdisk
                break

        if srcdisk is not None:
            break

    if srcdisk is None:
        logout.error('unable to find source disk or disk already in use. aborting.')
        return False

    logout.debug('using ' + srcdisk + ' as source disk')

    # remove any association first
    for array in arrays.keys():
        for memb in arrays[array]:
            mdisk = normalize_name(memb)
            if newdisk == mdisk:
                logout.debug('removing ' + membname + ' from array ' + array + '...')
                membname = normalize_name(memb, withpart=True)
                system_echo('mdadm --fail /dev/' + array + ' ' + membname)
                time.sleep(0.5)
                realarray = fwarray.get(array, array) # get container if there is any
                system_echo('mdadm --remove /dev/' + realarray + ' ' + membname)
                time.sleep(0.5)

    # clone partition data
    if not copy_boot_record(mdisk, newdisk):
        return False

    if len(fwarray) == 0:
        # clone partition data
        if not copy_part_tables(mdisk, newdisk):
            return False
    else:
        logout.debug('restarting mdmon...')
        system_echo('pgrep mdmon | xargs -r kill -TERM')
        time.sleep(1.0)

        for array in fwarray.keys():
            system_echo('mdmon ' + fwarray[array])

    # clone members
    for array in arrays.keys():
        logout.info('recovering array ' + array + '...')
        donedisk = False
        for disk in arrays[array]:
            mdisk = normalize_name(disk, withdev=True, withpart=True)
            membname = mdisk.replace(srcdisk, newdisk)
            if membname.find(newdisk) > -1:
                logout.info('initializing ' + membname + '...')
                if not os.access(membname, os.F_OK):
                    logout.debug('waiting for device "' + membname + '"...')

                    while True:
                        if os.access(membname, os.F_OK):
                            break

                        time.sleep(0.75)

                logout.debug('device "' + membname + '" found, adding to array...')

                for num in xrange(0,15):
                    realarray = fwarray.get(array, array)
                    if system_echo('mdadm --manage /dev/' + realarray + ' --add ' + membname) == 0:
                        donedisk = True
                        break

                    time.sleep(2.0)

                if not donedisk:
                    logout.error('unable to add ' + membname + ' to array ' + array + ', aborting.')
                    return False

            if donedisk:
                break


    return True

####

def arrays_of_mdstat(mdstat):
    res = {}
    for array in mdstat.keys():
        members = []

        for memb in mdstat[array]['status']['members']:
            members.append(memb['device'])
        res[array]=members

    return res

def devices_of_arrays(arrays):
    devices = {}
    for key in arrays.keys():
        memblist = []
        for memb in arrays[key]:
            if memb <> "":
                memblist.append(info_disk(memb))
        devices[key]=memblist
    return devices

def devices_filter(curdevices, newdevices, replace):
    allarrays = set()

    for key in curdevices.keys():
        allarrays.add(key)
    for key in newdevices.keys():
        allarrays.add(key)

    result = {}

    for array in allarrays:
        curlst = curdevices.get(array, [])
        newlst = newdevices.get(array, [])

        if len(newlst) > 1 and len(newlst) >= len(curlst):
            result[array] = newlst
        elif len(curlst) > 1:
            if len(replace) <> 0:
                replst = []

                for memb in curlst:
                    repstr = replace.get(string_of_dict(memb))

                    if repstr is None:
                        replst.append(memb)
                    else:
                        replst.append(dict_of_string(repstr))

                result[array] = replst
            else:
                result[array] = curlst

    return result

def xml_of_data(name, data):
    def singular(s):
        if len(s) > 0:
            if s[-1] == 's':
                return s[:-1]

        return s + '-element'

    def recurse(elm, name, data):
        # TODO: inspect objects

        if   type(data) == type({}):
            for key in data.keys():
                if key == 'type' or key == 'name':
                    elm.set(key, str(data[key]))
                else:
                    recurse(ET.SubElement(elm, key.lower()), key, data[key])

        elif type(data) == type([]):
            name = singular(name).lower()
            for e in data:
                recurse(ET.SubElement(elm, name), name, e)

        elif data is not None:
            elm.text = str(data)

        return elm

    return ET.tostring(recurse(ET.Element(name), name, data))

def statefile_path():
    return '/var/sng/raid'

def statefile_name(suffix=None):
    name = statefile_path() + '/status'

    if suffix:
        name = name + '.' + suffix

    return name

def statefile_process(devices=None, mdstat=None):
    last = None
    mode = 'r'
    path = statefile_path()

    if devices is not None:
        last = 'new'
        mode = 'w'

        if not os.path.exists(path):
            os.makedirs(path)

            try:
                emsg = 'get login name'
                if os.getuid() == 0:

                    emsg = 'get uid for webconfig user'
                    udata = getpwnam('webconfig')

                    emsg = 'set owner for state file'
                    os.chown(path, udata.pw_uid, 0) # webconfig:root

            except BaseException, e:
                logger.warning('failed ' + emsg + ': ' + str(e))

    emsg = ''
    name = statefile_name(last)

    try:
        emsg = 'open "' + name + '"'
        fdes = file(name, mode)

        if devices is not None:
            emsg = 'dump "' + name + '"'
            json.dump(devices, fdes)
            fdes.close()

            try:
                emsg = 'get login name'
                if os.getuid() == 0:

                    emsg = 'get uid for webconfig user'
                    udata = getpwnam('webconfig')

                    emsg = 'set owner for state file'
                    os.chown(name, udata.pw_uid, 0) # webconfig:root

            except BaseException, e:
                logger.warning('failed ' + emsg + ': ' + str(e))

            return True

        else:
            emsg = 'load "' + name + '"'
            data = json.load(fdes)
            fdes.close()
            return data

    except Exception, e:
        if os.path.exists(path):
            logger.error(emsg + ' failed: ' + str(e))

        if devices is not None:
            return False
        else:
            return {}

def has_any_degraded(mdstat):
    for key in mdstat.keys():
        if mdstat[key]['status']['device-total'] <> mdstat[key]['status']['device-count']:
            recovs = mdstat[key]['status'].get('recovery')
            resync = mdstat[key]['status'].get('resync')

            logger.debug(key + ': checking state..')

            if recovs is None and resync is None:
                return True

        for memb in mdstat[key]['status']['members']:
            logger.debug(key + ': checking ' + memb['device'] + ' state (' + memb['state'] + ')')
            if memb['state'] <> 'online':
                return True

    logger.debug('nothing degraded')
    return False

###############################################

parser = OptionParser()
query = False

def set_option(option, optstr, value, parser):
    global query
    query = True

    setattr(parser.values, option.dest, True)

parser.add_option("-d", "--debug",     dest="debug",  action="store_true", help='enable debug messages')
parser.add_option("-p", "--pretty",    dest="pretty", action="store_true", help='report everything in one line')

parser.add_option('-m', "--mdstat",    dest="mdstat", help='test input from file MDSTAT instead of /proc/mdstat')
parser.add_option('-r', "--recover",   dest="disk",   help='recover degraded arrays by adding a new disk')

parser.add_option('-i', "--inventory", dest="inventory", action="callback", help='report inventory', callback=set_option)
parser.add_option('-s', "--status",    dest="status",    action="callback", help='report status', callback=set_option)
parser.add_option('-c', "--changes",   dest="changes",   action="callback", help='report changes', callback=set_option)
parser.add_option('-o', "--commit",    dest="commit",    action="callback", help='save current state as default', callback=set_option)

parser.add_option('-e', "--detail",    dest="detail",    action="store_true", help='show data from mdadm --detail')

(options, args) = parser.parse_args()

if options.debug is not None:
    logger.setLevel(logging.DEBUG)
    logout.setLevel(logging.DEBUG)

if not query and options.disk is None and options.detail is None:
    logger.error('nothing to do, please specify some action.')
    sys.exit(1)

retcode = 0
xmltext = ''
replace = {}

try:
    mdstat = process_mdstat(options.mdstat)
    arrays = arrays_of_mdstat(mdstat)

    if options.detail is not None:
        detail = get_detail(arrays)
        xmltext += xml_of_data("arrays", detail)

    alldisks = []

    curdevices = {}
    newdevices = {}

    if query or options.disk is not None:
        alldisks = get_all_disks()

        curdevices = statefile_process(mdstat=mdstat)
        newdevices = devices_of_arrays(arrays)

        if options.inventory is not None or options.status is not None:
            inventory, status = get_inventory_status(mdstat, alldisks, has_inventory=options.inventory, has_status=options.status)

            if inventory is not None:
                xmltext += xml_of_data("inventory", inventory)

            if status is not None:
                xmltext += xml_of_data("status", status)

        if options.changes is not None:
            changes, replace = get_changes(curdevices, newdevices, mdstat, alldisks)

            xmltext += xml_of_data("changes", changes)

    if xmltext <> '':
        if options.pretty is None:
            print xmltext
        else:
            import xml.dom.minidom as xml
            print xml.parseString(xmltext).toprettyxml()

    if options.disk is not None:
        mdisk = None

        if options.disk == 'auto':
            devdisk = first_spare_disk(newdevices, alldisks)

            if devdisk is None:
                if has_any_degraded(mdstat):
                    logout.error('unable to find spare disk, aborting recovery')
                    retcode = 1
                else:
                    logout.info('nothing to do')
            else:
                mdisk = normalize_name(devdisk['name'])
                logout.info('found spare disk "' + mdisk + '", starting recovery...')

        else:
            mdisk = normalize_name(options.disk)

            if mdisk is None:
                logout.error('disk name "' + options.disk + '" invalid, please provide a valid device name.')
                retcode = 1

        if mdisk is not None:
            if not recover_arrays(arrays, mdisk, mdstat):
                retcode = 1

    if not statefile_process(devices=devices_filter(curdevices, newdevices, replace)):
        retcode = 1

except BaseException, e:
    logger.error(str(e))

    for line in traceback.format_tb(sys.exc_info()[2]):
        logger.error('  ' + line)

    retcode = 123

if retcode == 0 and (len(curdevices) == 0 or len(replace) <> 0 or options.commit is not None):
    try:
        os.rename(statefile_name('new'), statefile_name())
    except Exception, e:
        logger.error('unable to save new state: ' + str(e))
else:
    fname = statefile_name('new')

    try:
        if os.access(fname, os.F_OK):
            os.unlink(fname)
    except Exception, e:
        logger.error('uname to remove temporary state: ' + str(e))

sys.exit(retcode)
