Skip to content
Snippets Groups Projects
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
ICCFitTools.py 49.52 KiB
from __future__ import (absolute_import, division, print_function)
import matplotlib.pyplot as plt
import numpy as np
import sys
from scipy.misc import factorial
from scipy.optimize import curve_fit
from mantid.simpleapi import *
from mantid.kernel import V3D
import ICConvoluted as ICC
import itertools
from functools import reduce
from scipy.ndimage.filters import convolve
plt.ion()


def parseConstraints(peaks_ws):
    """
    returns a dictionary containing parameters for ICC fitting. Parameters
    are derived from instrument parameters files (see MANDI_Parameters.xml
    for an example).
    """
    possibleKeys = ['iccA', 'iccB', 'iccR', 'iccT0', 'iccScale0', 'iccHatWidth', 'iccKConv']
    d = {}
    for paramName in possibleKeys:
        if peaks_ws.getInstrument().hasParameter(paramName):
            vals = np.array(peaks_ws.getInstrument().getStringParameter(paramName)[0].split(),dtype=float)
            d[paramName] = vals
    return d


def scatFun(x, A, bg):
    """
    scatFun: returns A/x+bg.  Used for background estimation.
    """
    return A / x + bg


def oldScatFun(x, A, k, bg):
    """
    oldScatFun: returns 1.0*A*np.exp(-k*x) + bg.  Used for background estimation.
    """
    return 1.0 * A * np.exp(-1. * k * x) + bg


def calcSomeTOF(box, peak, refitIDX=None, q_frame='sample'):
    xaxis = box.getXDimension()
    qx = np.linspace(xaxis.getMinimum(), xaxis.getMaximum(), xaxis.getNBins())
    yaxis = box.getYDimension()
    qy = np.linspace(yaxis.getMinimum(), yaxis.getMaximum(), yaxis.getNBins())
    zaxis = box.getZDimension()
    qz = np.linspace(zaxis.getMinimum(), zaxis.getMaximum(), zaxis.getNBins())
    QX, QY, QZ = getQXQYQZ(box)

    if refitIDX is None:
        refitIDX = np.ones_like(QX).astype(np.bool)

    if q_frame == 'lab':
        qS0 = peak.getQLabFrame()
    elif q_frame == 'sample':
        qS0 = peak.getQSampleFrame()
    else:
        raise ValueError(
            'ICCFT:calcSomeTOF - q_frame must be either \'lab\' or \'sample\'; %s was provided' % q_frame)
    PIXELFACTOR = np.ones_like(QX) * (peak.getL1() + peak.getL2()) * np.sin(0.5 * peak.getScattering())
    for i, x in enumerate(qx):
        for j, y in enumerate(qy):
            for k, z in enumerate(qz):
                if refitIDX[i, j, k]:
                    qNew = V3D(x, y, z)
                    peak.setQSampleFrame(qNew)
                    L = peak.getL1() + peak.getL2()
                    HALFSCAT = 0.5 * peak.getScattering()
                    PIXELFACTOR[i, j, k] = L*np.sin(HALFSCAT)
    peak.setQSampleFrame(qS0)

    tofBox = 3176.507 * PIXELFACTOR * 1.0/np.sqrt(QX**2 + QY**2 + QZ**2)
    return tofBox


def cart2sph(x, y, z):
    """
    cart2sph takes in spherical coordinates (x,y,z) and returns
    spherical coordinates (r,phi,theta)
    """
    hxy = np.hypot(x, y)
    r = np.hypot(hxy, z)
    el = np.arctan2(z, hxy)
    az = np.arctan2(y, x)
    return r, az, el


def getQXQYQZ(box):
    """
    getQXQYQZ - returns meshed coordinates from an MDHistoWorkspace
    input: box - a 3D MDHistoWorkspace of a*b*c bins
    output: QX,QY,QZ - a*b*c numpy arrays containing evenly spaced coordinates
                        over the range of each axis.
    """
    xaxis = box.getXDimension()
    qx = np.linspace(xaxis.getMinimum(), xaxis.getMaximum(), xaxis.getNBins())
    yaxis = box.getYDimension()
    qy = np.linspace(yaxis.getMinimum(), yaxis.getMaximum(), yaxis.getNBins())
    zaxis = box.getZDimension()
    qz = np.linspace(zaxis.getMinimum(), zaxis.getMaximum(), zaxis.getNBins())
    QX, QY, QZ = np.meshgrid(qx, qy, qz, indexing='ij', copy=False)
    return QX, QY, QZ


def getQuickTOFWS(box, peak, padeCoefficients, goodIDX=None, dtSpread=0.03, qMask=None,
                  pp_lambda=None, minppl_frac=0.8, maxppl_frac=1.5, mindtBinWidth=1, maxdtBinWidth=50,
                  constraintScheme=1, peakMaskSize=5, iccFitDict=None):
    """
    getQuickTOFWS - generates a quick-and-dirty TOFWS.  Useful for determining the background.
    Input:
        box: the MDHistoWorkspace for the peak we are fitting
        peak: the peak object we are trying to fit
        padeCoefficients: the dictionary containing coefficients for the Pade coefficients describing thd moderator.
        goodIDX - which indices to consider.  Should be a numpy array with the same shape as box.  Will use all
                    voxels if this is None.
        dtSpread - how far on each side of the peak TOF to consider.
        qMask - which voxels to consider as a result of only keeping (h-eta, k-eta,l-eta) to (h+eta, k+eta, l+eta)
        pp_lambda - nominal background level.  Will be calculated if set to None.
        minppl_frac, maxppl_frac: fraction of the predicted pp_lambda to try if calculating pp_lambda
        mindtBinWidth - the minimum binwidth (in us) that we will allow when histogramming.
        maxdtBinWidth - the maximum binwidth (in us) that we will allow when histogramming.
        constraintScheme - which constraint scheme we use.  Typically set to 1
        iccFitDict - a dictionary containing ICC fit constraints and possibly initial guesses
    Output:
        chiSq - reduced chiSquared from fitting the TOF profile
        h - list of [Y, X], with Y and X being numpy arrays of the Y and X of the tof profile
        intensity - fitted intensity
        sigma - fitted sig(int)
    """
    tof = peak.getTOF()  # in us
    wavelength = peak.getWavelength()  # in Angstrom
    flightPath = peak.getL1() + peak.getL2()  # in m
    scatteringHalfAngle = 0.5*peak.getScattering()
    energy = 81.804 / wavelength**2 / 1000.0  # in eV
    if qMask is None:
        qMask = np.ones_like(box.getNumEventsArray()).astype(np.bool)

    calc_pp_lambda = False
    if pp_lambda is None:
        calc_pp_lambda = True

    tofWS, ppl = getTOFWS(box, flightPath, scatteringHalfAngle, tof, peak, qMask, dtSpread=dtSpread,
                          minFracPixels=0.01, neigh_length_m=3, zBG=1.96, pp_lambda=pp_lambda,
                          calc_pp_lambda=calc_pp_lambda, pplmin_frac=minppl_frac, pplmax_frac=minppl_frac,
                          mindtBinWidth=mindtBinWidth, maxdtBinWidth=maxdtBinWidth,
                          peakMaskSize=peakMaskSize, iccFitDict=iccFitDict)
    fitResults, fICC = doICCFit(
        tofWS, energy, flightPath, padeCoefficients, fitOrder=1, constraintScheme=constraintScheme,
        iccFitDict=iccFitDict)
    h = [tofWS.readY(0), tofWS.readX(0)]
    chiSq = fitResults.OutputChi2overDoF

    r = mtd['fit_Workspace']
    param = mtd['fit_Parameters']
    n_events = box.getNumEventsArray()

    iii = fICC.numParams() - 1
    fitBG = [param.row(int(iii+i+1))['Value'] for i in range(1+1)]

    # Set the intensity before moving on to the next peak
    icProfile = r.readY(1)
    bgCoefficients = fitBG[::-1]

    intensity, sigma, xStart, xStop = integratePeak(
        r.readX(0), icProfile, r.readY(0), np.polyval(bgCoefficients, r.readX(1)), pp_lambda=pp_lambda,
        fracStop=0.01, totEvents=np.sum(n_events[goodIDX*qMask]), bgEvents=np.sum(goodIDX*qMask)*pp_lambda,
        varFit=chiSq)

    return chiSq, h, intensity, sigma


def getPoissionGoodIDX(n_events, zBG=1.96, neigh_length_m=3):
    """
    getPoissionGoodIDX - returns a numpy arrays which is true if the voxel contains events at
            the zBG z level (1.96=95%CI).  This is based only on Poission statistics.
    Input:
        n_events: 3D numpy array containing counts
        zBG: the z value at which we will set the cutoff
        neigh_lengh_m: the number of voxels we will smooth over (via np.convolve)
    Output:
        goodIDX: a numpy arrays the same size as n_events that is True of False for if it contains
                counts at a statistically significant level.

        pp_lambda: the most likely number of background events
    """
    hasEventsIDX = n_events > 0
    # Set up some things to only consider good pixels
    # Set to zero for "this pixel only" mode - performance is optimized for neigh_length_m=0
    neigh_length_m = neigh_length_m

    # Get the most probably number of events
    pp_lambda = get_pp_lambda(n_events, hasEventsIDX)
    found_pp_lambda = False
    convBox = 1.0*np.ones([neigh_length_m, neigh_length_m,
                           neigh_length_m]) / neigh_length_m**3
    conv_n_events = convolve(n_events, convBox)
    allEvents = np.sum(n_events[hasEventsIDX])
    if allEvents > 0:
        while not found_pp_lambda:
            goodIDX = np.logical_and(
                hasEventsIDX, conv_n_events > pp_lambda+zBG*np.sqrt(pp_lambda/(2*neigh_length_m+1)**3))
            boxMean = n_events[goodIDX]
            if allEvents > np.sum(boxMean):
                found_pp_lambda = True
            else:
                pp_lambda *= 1.05
    return goodIDX, pp_lambda


def getOptimizedGoodIDX(n_events, padeCoefficients, zBG=1.96, neigh_length_m=3, qMask=None,
                        peak=None, box=None, pp_lambda=None, peakNumber=-1, minppl_frac=0.8,
                        maxppl_frac=1.5, mindtBinWidth=1, maxdtBinWidth=50,
                        constraintScheme=1, peakMaskSize=5, iccFitDict=None):
    """
    getOptimizedGoodIDX - returns a numpy arrays which is true if the voxel contains events at
            the zBG z level (1.96=95%CI).  Rather than using Poission statistics, this function
            will determine the optimal background level by trying to fit the TOF profile
            at each possible background level and returning the one which is best fit by the
            predicted moderator output (as described by the padeCoefficients.)
    Input:
        n_events - 3D numpy array containing counts
        padeCoefficients - A dictionary of Pade coefficients describing moderator emission.
        zBG - the z value at which we will set the cutoff
        neigh_lengh_m: the number of voxels we will smooth over (via np.convolve)
        qMask - the voxels of n_events to consider based on keeping only a fraction around (h,k,l)
        peak - the IPeak object for the peak we're trying to find the local background of.
        box - a 3D MDHisto workspace containing the intensity of each voxel for the peak we're fitting.
        pp_lambda - Currently unused.  Leave as None. TODO: remove this.
        peakNumber - currently unused.  TODO: Remove this.
        minppl_frac, maxppl_frac; range around predicted pp_lambda to check.
        mindtBinWidth - the smallest dt (in us) allowed for constructing the TOF profile.
        mindtBinWidth - the largest dt (in us) allowed for constructing the TOF profile.
        constraintScheme - sets the constraints for TOF profile fitting.  Leave as 1 if you're
                not sure how to modify this.
        iccFitDict - a dictionary containing ICC fit constraints and possibly initial guesses

    Output:
        goodIDX: a numpy arrays the same size as n_events that is True of False for if it contains
                counts to be included in the TOF profile.
        pp_lambda: the most likely number of background events
    """
    # Set up some things to only consider good pixels
    hasEventsIDX = n_events > 0
    # Set to zero for "this pixel only" mode - performance is optimized for neigh_length_m=0
    neigh_length_m = neigh_length_m
    convBox = 1.0*np.ones([neigh_length_m, neigh_length_m,
                           neigh_length_m]) / neigh_length_m**3
    conv_n_events = convolve(n_events, convBox)
    # Get the most probable number of events
    pp_lambda = get_pp_lambda(n_events, hasEventsIDX)

    pp_lambda_toCheck = np.unique(conv_n_events)
    pp_lambda_toCheck = pp_lambda_toCheck[1:][np.diff(
        pp_lambda_toCheck) > 0.001]

    # Get the average background level
    nX, nY, nZ = n_events.shape
    cX = nX//2
    cY = nY//2
    cZ = nZ//2
    dP = peakMaskSize

    peakMask = qMask.copy()
    peakMask[cX-dP:cX+dP, cY-dP:cY+dP, cZ-dP:cZ+dP] = 0
    neigh_length_m=3
    convBox = 1.0 * np.ones([neigh_length_m, neigh_length_m,
                             neigh_length_m]) / neigh_length_m**3
    conv_n_events = convolve(n_events, convBox)
    bgMask = np.logical_and(conv_n_events>0, peakMask>0)
    meanBG = np.mean(n_events[bgMask])

    # We set it at slightly lower than 1:1 because we don't
    # want to fit a peak where most of the counts have been removed.
    # The factor of 1.96 comes from a historical definition of pp_lambda
    # where background was considered at the 95% confidence interval.
    pred_ppl = np.polyval([0.98,0],meanBG)*1.96
    minppl = minppl_frac*pred_ppl
    maxppl = maxppl_frac*pred_ppl
    pp_lambda_toCheck = pp_lambda_toCheck[pp_lambda_toCheck > minppl]
    pp_lambda_toCheck = pp_lambda_toCheck[pp_lambda_toCheck < maxppl]

    chiSqList = 1.0e30*np.ones_like(pp_lambda_toCheck)
    ISIGList = 1.0e-30*np.ones_like(pp_lambda_toCheck)
    IList = 1.0e-30*np.ones_like(pp_lambda_toCheck)
    oldGoodIDXSum = -1.0
    for i, pp_lambda in enumerate(pp_lambda_toCheck):
        try:
            goodIDX = np.logical_and(
                hasEventsIDX, conv_n_events > pp_lambda+zBG*np.sqrt(pp_lambda/(2*neigh_length_m+1)**3))
            if np.sum(goodIDX) == oldGoodIDXSum:  # No new points removed, we skip this
                continue
            else:
                oldGoodIDXSum = np.sum(goodIDX)
            try:
                chiSq, h, intens, sigma = getQuickTOFWS(box, peak, padeCoefficients, goodIDX=goodIDX, qMask=qMask, pp_lambda=pp_lambda,
                                                        minppl_frac=minppl_frac, maxppl_frac=maxppl_frac, mindtBinWidth=mindtBinWidth,
                                                        maxdtBinWidth=maxdtBinWidth, constraintScheme=constraintScheme,
                                                        peakMaskSize=peakMaskSize, iccFitDict=iccFitDict)
            except:
                #raise
                break
            chiSqList[i] = chiSq
            ISIGList[i] = intens/sigma
            IList[i] = intens
            #hList.append((pp_lambda, chiSq, h))
            # or np.sum(h[0])<10: #or (chiSq > 100 and np.min(chiSqList)<5):
            if len(h[0]) < 10:
                break
        except RuntimeError:
            # This is caused by there being fewer datapoints remaining than parameters.  For now, we just hope
            # we found a satisfactory answer.
            raise
            break
        except KeyboardInterrupt:
            sys.exit()
    use_ppl = np.argmin(np.abs(chiSqList[:i+1]-1.0))
    pp_lambda = pp_lambda_toCheck[use_ppl]
    goodIDX, _ = getBGRemovedIndices(n_events, pp_lambda=pp_lambda)
    chiSq, h, intens, sigma = getQuickTOFWS(box, peak, padeCoefficients, goodIDX=goodIDX, qMask=qMask,
                                            pp_lambda=pp_lambda, minppl_frac=minppl_frac, maxppl_frac=maxppl_frac,
                                            mindtBinWidth=mindtBinWidth, maxdtBinWidth=maxdtBinWidth,
                                            peakMaskSize=peakMaskSize,
                                            iccFitDict=iccFitDict)
    if qMask is not None:
        return goodIDX*qMask, pp_lambda
    return goodIDX, pp_lambda


def getBGRemovedIndices(n_events, zBG=1.96, calc_pp_lambda=False, neigh_length_m=3, qMask=None,
                        peak=None, box=None, pp_lambda=None, peakNumber=-1, padeCoefficients=None,
                        pplmin_frac=0.8, pplmax_frac=1.5, mindtBinWidth=1, maxdtBinWidth=50,
                        constraintScheme=1, peakMaskSize=5, iccFitDict=None):
    """
    getBGRemovedIndices - A wrapper for getOptimizedGoodIDX
    Input:
        n_events - 3D numpy array containing counts
        padeCoefficients - A dictionary of Pade coefficients describing moderator emission.
        zBG - the z value at which we will set the cutoff
        calc_pp_lambda - a boolean for if pp_lambda should be calculated
        neigh_lengh_m: the number of voxels we will smooth over (via np.convolve)
        qMask - the voxels of n_events to consider based on keeping only a fraction around (h,k,l)
        peak - the IPeak object for the peak we're trying to find the local background of.
        box - a 3D MDHisto workspace containing the intensity of each voxel for the peak we're fitting.
        pp_lambda - Currently unused.  Leave as None. TODO: remove this.
        peakNumber - currently unused.  TODO: Remove this.
        minppl_frac, maxppl_frac; range around predicted pp_lambda to check.
        mindtBinWidth - the smallest dt (in us) allowed for constructing the TOF profile.
        maxdtBinWidth - the largest dt (in us) allowed for constructing the TOF profile.
        constraintScheme - sets the constraints for TOF profile fitting.  Leave as 1 if you're
                not sure how to modify this.
        iccFitDict - a dictionary containing ICC fit constraints and possibly initial guesses

    Output:
        goodIDX: a numpy arrays the same size as n_events that is True of False for if it contains
                counts to be included in the TOF profile.
        pp_lambda: the most likely number of background events
    """

    if calc_pp_lambda is True and pp_lambda is not None:
        sys.exit(
            'Error in ICCFT:getBGRemovedIndices: You should not calculate and specify pp_lambda.')

    if calc_pp_lambda is True and padeCoefficients is None:
        sys.exit(
            'Error in ICCFT:getBGRemovedIndices: calc_pp_lambda is True, but no moderator coefficients are provided.')

    if pp_lambda is not None:
        # Set up some things to only consider good pixels
        hasEventsIDX = n_events > 0
        # Set to zero for "this pixel only" mode - performance is optimized for neigh_length_m=0
        neigh_length_m = neigh_length_m
        convBox = 1.0 * \
            np.ones([neigh_length_m, neigh_length_m,
                     neigh_length_m]) / neigh_length_m**3
        conv_n_events = convolve(n_events, convBox)
        goodIDX = np.logical_and(hasEventsIDX, conv_n_events >
                                 pp_lambda+zBG*np.sqrt(pp_lambda/(2*neigh_length_m+1)**3))
        return goodIDX, pp_lambda

    if calc_pp_lambda is False:
        return getPoissionGoodIDX(n_events, zBG=zBG, neigh_length_m=neigh_length_m)

    if peak is not None and box is not None and padeCoefficients is not None:
        while pplmin_frac >= 0.0:
            try:
                return getOptimizedGoodIDX(n_events, padeCoefficients, zBG=1.96, neigh_length_m=neigh_length_m,
                                           minppl_frac=pplmin_frac, maxppl_frac=pplmax_frac, qMask=qMask, peak=peak,
                                           box=box, pp_lambda=pp_lambda, peakNumber=peakNumber,
                                           mindtBinWidth=mindtBinWidth, maxdtBinWidth=maxdtBinWidth,
                                           constraintScheme=constraintScheme,
                                           peakMaskSize=peakMaskSize, iccFitDict=iccFitDict)
            except KeyboardInterrupt:
                sys.exit()
            except:
                #raise
                pplmin_frac -= 0.4
    logger.warning('ERROR WITH ICCFT:getBGRemovedIndices!')


def getDQFracHKL(UB, frac=0.5):
    """
    UBmatrix as loaded by LoadIsawUB().  Only works in the
    return dQ -  a 3x2 numpy array saying how far in each direction you have
        to go to get all of (h-frak,k-frac,l-frac; h+frac, k+frac, l+frac_)
    """

    dQ = np.zeros((3, 2))

    q = [2*np.pi*frac*UB.dot(v)
         for v in [seq for seq in itertools.product([-1.0, 1.0], repeat=3)]]
    dQ[:, 0] = np.max(q, axis=0)  # TODO THIS CAN BE 1D since it's symmetric
    dQ[:, 1] = np.min(q, axis=0)
    return dQ


def getHKLMask(UB, frac=0.5, dQPixel=0.005, dQ=None):
    """
    getHKLMask returns the qMask used throughout most of the fitting.
    Intput:
        UB: the UB matrix (3x3 numpy array)
        frac: the fraction around (hkl) you want to consider.
        dQPixel: the side length of each voxel
        dQ: how far to go.  Will calculate if set to None
    returns
        mask: the qMask used by many fitting functions.
    """
    if dQ is None:
        dQ = np.abs(getDQFracHKL(UB, frac=frac))
        dQ[dQ > 0.5] = 0.5
    nPtsQ = np.round(np.sum(dQ/dQPixel, axis=1)).astype(int)
    h0 = 1.0
    k0 = 27.0
    l0 = 7.0
    qDummy = 2*np.pi*UB.dot(np.asarray([h0, k0, l0]))
    qx = np.linspace(qDummy[0]-dQ[0, 0], qDummy[0]+dQ[0, 1], nPtsQ[0])
    qy = np.linspace(qDummy[1]-dQ[1, 0], qDummy[1]+dQ[1, 1], nPtsQ[1])
    qz = np.linspace(qDummy[2]-dQ[2, 0], qDummy[2]+dQ[2, 1], nPtsQ[2])
    QX, QY, QZ = np.meshgrid(qx, qy, qz, indexing='ij', copy=False)
    UBinv = np.linalg.inv(UB)
    tHKL = UBinv.dot([QX.ravel(), QY.ravel(), QZ.ravel()])/2/np.pi
    H = np.reshape(tHKL[0, :], QX.shape)
    K = np.reshape(tHKL[1, :], QX.shape)
    L = np.reshape(tHKL[2, :], QX.shape)
    mask = reduce(np.logical_and,  [
                  H > h0-frac, H < h0+frac, K > k0-frac, K < k0+frac, L > l0-frac, L < l0+frac])
    return mask


def padeWrapper(x, a, b, c, d, f, g, h, i, j, k):
    """
    padeWrapper is a wrapper for the pade(c,x) function for compatability with curve_fit.
    Inputs are x (numpy array) and the 8 coefficients for the approximant.
    Output are the values of the pade approximant at each value of x.
    """
    pArr = np.zeros(10)
    pArr[0] = a
    pArr[1] = b
    pArr[2] = c
    pArr[3] = d
    pArr[4] = f
    pArr[5] = g
    pArr[6] = h
    pArr[7] = i
    pArr[8] = j
    pArr[9] = k
    return pade(pArr, x)


def pade(c, x):
    """
    Stadnard 4th order pade approximant.
    For use with moderator characterization, c are the coefficients
    and x is the energy (in eV)
    """
    return c[0]*x**c[1]*(1+c[2]*x+c[3]*x**2+(x/c[4])**c[5])/(1+c[6]*x+c[7]*x**2+(x/c[8])**c[9])


def integratePeak(x, yFit, yData, bg, pp_lambda=0, fracStop=0.01, totEvents=1, bgEvents=1, varFit=0.):
    """
    integratePeak does 1D integration of the peak.  This is currently done by rectangular integration (i.e.
    we just sum the values at each point on the curve for values considered to be the peak.)  The peak is
    defined as all x values with yFit/max(yFit) > fracStop.
    Note that this function will only integrate the 1D TOF peak - for 3D profile fitting, integration is done in
    doBVGFit or (for mantid) in the algorithm itself.)

    Input:
        x - the TOf values for the TOF histogram
        yFit - the fit for the TOF histogram
        yData - the counts for the TOF histogram
        pp_lambda - the pp_lambda background level.  Not used in integration. TODO: Remove this
        fracStop - the threshold at which we consider the peak to be a peak
        totEvents - not used
        bgEvents - total number of background events including events removed by pp_lambda filtering.
        varFit - the variance of the fit (float)
    Output:
        intensity - total number of counts in the peak
        sigma - sigma(I) for the peak
        xStart - the time (in us) the peak started at
        xStop - the time (in us) the peak stopped at

    """
    # Find out start/stop point
    yScaled = (yFit-bg) / np.max(yFit-bg)
    goodIDX = yScaled > fracStop
    if np.sum(goodIDX) > 0:
        iStart = np.min(np.where(goodIDX))
        iStop = np.max(np.where(goodIDX))
        xStart = x[iStart]
        xStop = x[iStop]
    else:
        logger.warning('ICCFITTOOLS:integratePeak - NO GOOD START/STOP POINT!!')
        return 0.0, 1.0, x[0], x[-1]

    # Do the integration
    intensity = np.sum(yFit[iStart:iStop] - bg[iStart:iStop]) - bgEvents

    # Calculate the background sigma = sqrt(var(Fit) + sum(BG))
    # sigma = np.sqrt(totEvents + bgEvents)
    sigma = np.sqrt(intensity + 2.0*bgEvents + varFit)
    return intensity, sigma, xStart, xStop


def poisson(k, lam):
    return (lam**k)*(np.exp(-lam)/factorial(k))


def normPoiss(k, lam, eventHist):
    pp_dist = poisson(k, lam)
    pp_n = np.dot(pp_dist, eventHist)/np.dot(pp_dist, pp_dist)
    return pp_dist*pp_n


def get_pp_lambda(n_events, hasEventsIDX):
    """
    #Estimate the most likely number of events based on a Poission
    #distribution of the box.  n_events an ND array (N=3 for Qx,Qy,Qz)
    #containing the number of events in each pixel.  Returns pp_lambda,
    #the most likely number of events.
    """
    eventValues = n_events[hasEventsIDX]
    eventHist = np.bincount(eventValues.astype('int'))[1:]
    eventX = np.array(range(len(eventHist))) + 1

    def normPoissL(k, lam): return normPoiss(k, lam, eventHist)
    pp_lambda, cov = curve_fit(normPoissL, eventX, eventHist, p0=0.1)
    return pp_lambda


def getTOFWS(box, flightPath, scatteringHalfAngle, tofPeak, peak, qMask, zBG=-1.0, dtSpread=0.02,
             minFracPixels=0.005, workspaceNumber=None, neigh_length_m=0, pp_lambda=None, calc_pp_lambda=False,
             padeCoefficients=None, pplmin_frac=0.8, pplmax_frac=1.5, peakMaskSize=5,
             mindtBinWidth=1, maxdtBinWidth=50, constraintScheme=1, iccFitDict=None):
    """
    Builds a TOF profile from the data in box which is nominally centered around a peak.
    Input:
        box - in a binned MD box.
        flightPath - L1+L2 (units: m)
        scatteringHalfAngle - the scattering half angle (units: rad)
        tofPeak - the nominal TOF of the peak (units: us)
        peak - the IPeak object for the peak we're fitting
        qMask - a numpy array telling which voxels to use within a fraction of some (hkl).  True for use.
        zBG - the z score at which we will accept pixels (i.e. 1.96 for 95% CI).  If zBG<0
             then we will not remove background and will instead just consider each pixel
             individually
        dtSpread - the fraction of t around tofPeak we will consider
        minFracPixels -
        workspaceNumber - None of not doing multiple fits.  Otherwise it will append an integer in the mtd[] object so
            not to overwrite.  Probably not needed in most cases.
        neigh_length_m - integer; how large of a convolution box to use.
        pp_lambda - the most likely backgorund level; set to None if want to calculate
        calc_pp_lambda - boolean; True if you want to calculate pp_lambda using TOF profile fitting.  If you do not
            want to, you can feed the value in as pp_lambda (calculated elsewhere).
        minppl_frac, maxppl_frac; range around predicted pp_lambda to check.
        mindtBinWidth - the small dt (in us) allowed for constructing the TOF profile.
        maxdtBinWidth - the largest dt (in us) allowed for constructing the TOF profile.
        constraintScheme - sets the constraints for TOF profile fitting.  Leave as 1 if you're
                not sure how to modify this.
        iccFitDict - a dictionary containing ICC fit constraints and possibly initial guesses

    Output:
        tofWS - a mantid containing the TOF profile.  X-axis is TOF (units: us) and
               Y-axis is the number of events.
        pp_lambda - the most likey number of bg events.
    """
    # Find the qVoxels to use
    n_events = box.getNumEventsArray()
    hasEventsIDX = np.logical_and(n_events > 0, qMask)
    # Set up some things to only consider good pixels
    if zBG >= 0:
        if pp_lambda is None:
            calc_pp_lambda = True
        goodIDX, pp_lambda = getBGRemovedIndices(n_events, box=box, qMask=qMask, peak=peak, pp_lambda=pp_lambda,
                                                 calc_pp_lambda=calc_pp_lambda, padeCoefficients=padeCoefficients,
                                                 pplmin_frac=pplmin_frac, pplmax_frac=pplmax_frac,
                                                 mindtBinWidth=mindtBinWidth, maxdtBinWidth=maxdtBinWidth,
                                                 constraintScheme=constraintScheme,
                                                 peakMaskSize=peakMaskSize, iccFitDict=iccFitDict)
        hasEventsIDX = np.logical_and(goodIDX, qMask)
        boxMeanIDX = np.where(hasEventsIDX)
    else:  # don't do background removal - just consider one pixel at a time
        pp_lambda = 0
        boxMeanIDX = np.where(hasEventsIDX)
    boxMeanIDX = np.asarray(boxMeanIDX)

    # Setup our axes -- ask if there is a way to just get this
    xaxis = box.getXDimension()
    qx = np.linspace(xaxis.getMinimum(), xaxis.getMaximum(), xaxis.getNBins())
    yaxis = box.getYDimension()
    qy = np.linspace(yaxis.getMinimum(), yaxis.getMaximum(), yaxis.getNBins())
    zaxis = box.getZDimension()
    qz = np.linspace(zaxis.getMinimum(), zaxis.getMaximum(), zaxis.getNBins())
    QX, QY, QZ = np.meshgrid(qx, qy, qz, indexing='ij', copy=False)

    # Create our TOF distribution from bg corrected data
    tList = 1.0/np.sqrt(QX[hasEventsIDX]**2 +
                        QY[hasEventsIDX]**2 + QZ[hasEventsIDX]**2)
    tList = 3176.507 * flightPath * \
        np.sin(scatteringHalfAngle) * tList  # convert to microseconds

    # Set up our bins for histogramming
    tMin = np.min(tList)
    tMax = np.max(tList)
    dt = tofPeak*dtSpread  # time in us on either side of the peak position to consider
    tMin = min(tMin, tofPeak - dt)
    tMax = max(tMax, tofPeak + dt)

    qCorners = np.array([[qx[v[0]], qy[v[1]], qz[v[2]]]
                         for v in itertools.product((1, -1), repeat=3)])
    qMagCorn = np.array(
        [np.sqrt(q[0]*q[0] + q[1]*q[1] + q[2]*q[2]) for q in qCorners])
    tofCorners = 3176.507 * flightPath * np.sin(scatteringHalfAngle) / qMagCorn
    tMin = np.min(tofCorners)
    tMax = np.max(tofCorners)
    # this will set dtBinWidth as the resolution at the center of the box
    tC = 3176.507 * flightPath * np.sin(scatteringHalfAngle)/np.linalg.norm(
        [qx[qx.shape[0]//2], qy[qy.shape[0]//2], qz[qz.shape[0]//2]])
    tD = 3176.507 * flightPath * np.sin(scatteringHalfAngle)/np.linalg.norm(
        [qx[qx.shape[0]//2 + 1], qy[qy.shape[0]//2+1], qz[qz.shape[0]//2+1]])
    dtBinWidth = np.abs(tD-tC)
    dtBinWidth = max(mindtBinWidth, dtBinWidth)
    dtBinWidth = min(maxdtBinWidth, dtBinWidth)
    tBins = np.arange(tMin, tMax, dtBinWidth)
    weightList = n_events[hasEventsIDX]  # - pp_lambda
    h = np.histogram(tList, tBins, weights=weightList)
    # For and plot the TOF distribution
    tPoints = 0.5*(h[1][1:] + h[1][:-1])
    yPoints = h[0]

    if workspaceNumber is None:
        tofWS = CreateWorkspace(
            OutputWorkspace='tofWS', DataX=tPoints, DataY=yPoints, DataE=np.sqrt(yPoints))
    else:
        tofWS = CreateWorkspace(OutputWorkspace='tofWS%i' % workspaceNumber,
                                DataX=tPoints, DataY=yPoints, DataE=np.sqrt(yPoints))
    return tofWS, float(pp_lambda)


def getT0Shift(E, L):
    """
    getT0Shift(E,L) returns the time it takes a neutron of energy E
    (in eV) to go a distance L (in m).  Time is returned in us.
    """
    # E is energy in eV, L is flight path in m
    mn = 1.674929e-27  # neutron mass, kg
    E = E * 1.60218e-19  # convert eV to J
    t0Shift = L*np.sqrt(mn/2/E)  # units = s
    t0Shift = t0Shift * 1.0e6  # units =us
    return t0Shift


def getModeratorCoefficients(fileName):
    """
    getModeratorCoefficients takes coefficients saved via numpy and loads
    them as a dictionary for use in other functions.
    """
    r = np.loadtxt(fileName)
    d = dict()
    d['A'] = r[0]
    d['B'] = r[1]
    d['R'] = r[2]
    d['T0'] = r[3]
    return d


def oneOverXSquared(x, A, bg):
    return A/np.sqrt(x) + bg


def getInitialGuess(tofWS, paramNames, energy, flightPath, padeCoefficients):
    """
    Returns intial parameters for fitting based on a few quickly derived TOF
     profile parameters.  tofWS is a worskapce containng the TOF profile,
     paramNames is the list of parameter names
     energy is the energy of the peak (units: eV)
     flightPath is L = L1 + L2 (units: m)
    """
    x0 = np.zeros(len(paramNames))
    x = tofWS.readX(0)
    y = tofWS.readY(0)
    x0[0] = pade(padeCoefficients['A'], energy)
    x0[1] = pade(padeCoefficients['B'], energy)
    x0[2] = pade(padeCoefficients['R'], energy)
    x0[3] = pade(padeCoefficients['T0'], energy) + getT0Shift(energy,
                                                              flightPath)  # extra is for ~18m downstream we are

    # These are still phenomenological
    #x0[0] /= 1.2
    #x0[2] += 0.05
    # x0[3] -= 10 #This is lazy - we can do it detector-by-detector
    x0[3] = np.mean(x[np.argsort(y)[::-1][:min(5, np.sum(y > 0))]])
    x0[4] = (np.max(y))/x0[0]*2*2.5  # Amplitude - gets rescaled later anyway

    x0[5] = 0.5  # hat width in IDX units
    x0[6] = 120.0  # Exponential decay rate for convolution
    return x0


def getSample(run, DetCalFile,  workDir, fileName, qLow=-25, qHigh=25, q_frame='sample'):
    """
    getSample loads the event workspace, converts it to reciprocal space as an MDWorkspace.
        The event workspace will be removed from memory.
    Input:
        run - int; the run number
        DetCalFile - string; path to the DetCalFile.  If None, default calibration is used.
        workDir - not used.  TODO - remove this.
        fileName - str; the events file to load.  Should probably be an absolute path.
        qLow, qHigh - the returned MDWorkspace will range from qLow to qHigh in all 3 directions.
        q_frame - either 'sample' or 'lab'.  Wether to return in the lab or sample coordinate system.
    Returns:
        MDdata - a handle for the mtd['MDdata'] object, which contains the loaded run in reciprocal space.
    """

    # data
    logger.information('Loading file' + fileName)
    data = Load(Filename=fileName)
    if DetCalFile is not None:
        LoadIsawDetCal(InputWorkspace=data, Filename=DetCalFile)

    if q_frame == 'lab':
        Q3DFrame = 'Q_lab'
    elif q_frame == 'sample':
        Q3DFrame = 'Q_sample'
    else:
        raise ValueError(
            'ICCFT:calcSomeTOF - q_frame must be either \'lab\' or \'sample\'; %s was provided' % q_frame)

    MDdata = ConvertToMD(InputWorkspace=data, QDimensions='Q3D', dEAnalysisMode='Elastic',
                         Q3DFrames=Q3DFrame, QConversionScales='Q in A^-1',
                         MinValues='%f, %f, %f' % (qLow, qLow, qLow), Maxvalues='%f, %f, %f' % (qHigh, qHigh, qHigh), MaxRecursionDepth=10,
                         LorentzCorrection=False)
    mtd.remove('data')
    return MDdata


def plotFit(filenameFormat, r, tofWS, fICC, runNumber, peakNumber, energy, chiSq, bgFinal, xStart, xStop, bgx0=None):
    """
    Function to make and save plots of the fits. bgx0=polynomial coefficients (polyfit order)
    for the initial guess
    """
    plt.figure(1)
    plt.clf()
    plt.plot(r.readX(0), r.readY(0), 'o', label='Data')
    if bgx0 is not None:
        plt.plot(tofWS.readX(0), fICC.function1D(tofWS.readX(0)) +
                 np.polyval(bgx0, tofWS.readX(0)), 'b', label='Initial Guess')
    else:
        plt.plot(tofWS.readX(0), fICC.function1D(
            tofWS.readX(0)), 'b', label='Initial Guess')

    plt.plot(r.readX(1), r.readY(1), '.-', label='Fit')
    plt.plot(r.readX(1), np.polyval(
        bgFinal, r.readX(1)), 'r', label='Background')
    yLims = plt.ylim()
    plt.plot([xStart, xStart], yLims, 'k')
    plt.plot([xStop, xStop], yLims, 'k')
    plt.title('E0=%4.4f meV, redChiSq=%4.4e' % (energy*1000, chiSq))
    plt.legend(loc='best')
    plt.savefig(filenameFormat % (runNumber, peakNumber))


def getBoxFracHKL(peak, peaks_ws, MDdata, UBMatrix, peakNumber, dQ, dQPixel=0.005, fracHKL=0.5, fracHKLRefine=0.2, q_frame='sample'):
    """
    getBoxFracHKL returns the binned MDbox going from (x,y,z) - (dq_x, dq_y, dq_z) to (x,y,z) + (dq_x, dq_y, dq_z)
     Inputs:
        peak: peak object to be analyzed.  HKL and peak centers must be defined
        peaks_ws - not used; TODO: remove this.
        MDdata: the MD events workspace to be binned
                only the minmium number of constants is necessary for simpler systems (e.g.
                one value for cubic crystals).
        UBMatrix - not used; TODO: remove this.
        peakNumber - not used.  TODO: remove this
        dQ - a (3x2) numpy array stating how far to bin MDdata around the peak
        fracHKL - not used; TODO: remove this
        fracHKLRefine - not used;  TODO: remove this
        q_frame - str; either 'sample' or 'lab'
      Returns:
          Box, an MDWorkspace with histogrammed events around the peak
    """

    if q_frame == 'lab':
        q0 = peak.getQLabFrame()
    elif q_frame == 'sample':
        q0 = peak.getQSampleFrame()
    else:
        raise ValueError(
            'ICCFT:calcSomeTOF - q_frame must be either \'lab\' or \'sample\'; %s was provided' % q_frame)
    Qx = q0[0]
    Qy = q0[1]
    Qz = q0[2]
    dQ = np.abs(dQ)
    dQ[dQ > 0.5] = 0.5
    nPtsQ = np.round(np.sum(dQ/dQPixel, axis=1)).astype(int)

    Box = BinMD(InputWorkspace='MDdata',
                AlignedDim0='Q_%s_x,' % q_frame +
                str(Qx-dQ[0, 0])+','+str(Qx+dQ[0, 1])+','+str(nPtsQ[0]),
                AlignedDim1='Q_%s_y,' % q_frame +
                str(Qy-dQ[1, 0])+','+str(Qy+dQ[1, 1])+','+str(nPtsQ[1]),
                AlignedDim2='Q_%s_z,' % q_frame +
                str(Qz-dQ[2, 0])+','+str(Qz+dQ[2, 1])+','+str(nPtsQ[2]),
                OutputWorkspace='MDbox')
    return Box


def doICCFit(tofWS, energy, flightPath, padeCoefficients, constraintScheme=None, outputWSName='fit', fitOrder=1,
             iccFitDict=None):
    """
    doICCFit - Carries out the actual least squares fit for the TOF workspace.
    Intput:
        tofWS - a workspace with x being timepoints and y being the TOF profile
        energy - peak energy
        flightPath - L1 + L2 (in m)
        padeCoefficients - the dictinoary containing coefficients for the pade
            approximant describing moderator output.
        constraintScheme - defines the sets of constraints to use for fitting.
            None will force no constraints
            1 will set constraints to 50% of the initial guess either high or low.
            2 sets a very large range of physically possible (though not necessarily
            correct) values
            Other schemes can be implemented in this code by following the template
            below.
        outputWSName - the base name for output workspaces.  Leave as 'fit' unless you are
            doing multiple fits.
        fitOrder - the background polynomial order
        iccFitDict - a dictionary containing ICC fit constraints and possibly initial guesses
    Returns:
        fitResults - the output from Mantid's Fit() routine
        fICC - an IkedaCarpenterConvoluted function with parameters set to the fit values.
    """
    # Set up our inital guess
    fICC = ICC.IkedaCarpenterConvoluted()
    fICC.init()
    paramNames = [fICC.getParamName(x) for x in range(fICC.numParams())]
    x0 = getInitialGuess(tofWS, paramNames, energy,
                         flightPath, padeCoefficients)
    [fICC.setParameter(iii, v) for iii, v in enumerate(x0[:fICC.numParams()])]
    x = tofWS.readX(0)
    y = tofWS.readY(0)
    bgx0 = np.polyfit(x[np.r_[0:15, -15:0]], y[np.r_[0:15, -15:0]], fitOrder)

    nPts = x.size
    scaleFactor = np.max((y-np.polyval(bgx0, x))
                         [nPts//3:2*nPts//3])/np.max(fICC.function1D(x)[nPts//3:2*nPts//3])
    x0[4] = x0[4]*scaleFactor
    fICC.setParameter(4, x0[4])
    #fICC.setPenalizedConstraints(A0=[0.01, 1.0], B0=[0.005, 1.5], R0=[0.01, 1.0], T00=[0,1.0e10], KConv0=[10,500],penalty=1.0e20)
    if constraintScheme == 1:
        # Set these bounds as defaults - they can be changed for each instrument
        # They can be changed by setting parameters in the INSTRUMENT_Parameters.xml file.
        A0 = [0.5*x0[0], 1.5*x0[0]]
        B0 = [0.5*x0[1], 1.5*x0[1]]
        R0 = [0.5*x0[2], 1.5*x0[2]]
        T00 = [0.,1.e10]
        HatWidth0 = [0., 5.]
        Scale0 = [0., np.inf]
        KConv0 = [100, 140]

        # Now we see what instrument specific parameters we have
        if iccFitDict is not None:
            possibleKeys = ['iccA', 'iccB', 'iccR', 'iccT0', 'iccScale0', 'iccHatWidth', 'iccKConv']
            for keyIDX, (key, bounds) in enumerate(zip(possibleKeys, [A0, B0, R0, T00, Scale0, HatWidth0, KConv0])):
                if key in iccFitDict:
                    bounds[0] = iccFitDict[key][0]
                    bounds[1] = iccFitDict[key][1]
                    if len(iccFitDict[key] == 3):
                        x0[keyIDX] = iccFitDict[key][2]
                        fICC.setParameter(keyIDX, x0[keyIDX])
        try:
            fICC.setPenalizedConstraints(A0=A0, B0=B0, R0=R0, T00=T00, KConv0=KConv0, penalty=1.0e10)
        except:
            fICC.setPenalizedConstraints(A0=A0, B0=B0, R0=R0, T00=T00, KConv0=KConv0, penalty=None)
    if constraintScheme == 2:
        try:
            fICC.setPenalizedConstraints(A0=[0.0001, 1.0], B0=[0.005, 1.5], R0=[0.00, 1.], Scale0=[
                                         0.0, 1.0e10], T00=[0, 1.0e10], KConv0=[100., 140.], penalty=1.0e20)
        except:
            fICC.setPenalizedConstraints(A0=[0.0001, 1.0], B0=[0.005, 1.5], R0=[0.00, 1.], Scale0=[
                                         0.0, 1.0e10], T00=[0, 1.0e10], KConv0=[100, 140.], penalty=None)
    f = FunctionWrapper(fICC)
    bg = Polynomial(n=fitOrder)

    for i in range(fitOrder+1):
        bg['A'+str(fitOrder-i)] = bgx0[i]
    bg.constrain('-1.0 < A%i < 1.0' % fitOrder)
    fitFun = f + bg
    fitResults = Fit(Function=fitFun, InputWorkspace='tofWS',
                     Output=outputWSName)
    return fitResults, fICC


def integrateSample(run, MDdata, peaks_ws, paramList, UBMatrix, dQ, qMask, padeCoefficients,
                    figsFormat=None, dtSpread=0.02, fracHKL=0.5, minFracPixels=0.0000, fracStop=0.01,
                    dQPixel=0.005, p=None, neigh_length_m=0, zBG=-1.0, bgPolyOrder=1,
                    doIterativeBackgroundFitting=False, q_frame='sample',
                    progressFile=None, minpplfrac=0.8, maxpplfrac=1.5, mindtBinWidth=1, maxdtBinWidth=50,
                    keepFitDict=False, constraintScheme=1, peakMaskSize=5, iccFitDict=None):
    """
    integrateSample contains the loop that integrates over all of the peaks in a run and saves the results.  Importantly, it also handles
    errors (mostly by passing and recording special values for failed fits.)
    Input:
        run - int; the run number to process
        MDdata - MDWorkspace; the MDWorkspace from this run in q_frame coordinates
        peaks_ws - a Mantid peaks workspace containing at least one peak to be fit from run number run
        paramList - used to save TOF fit parameters.  Can pass an empty list [] or a list you want to append to.
        UBMatrix - the UBMatrix for the sample
        dQ - how far to extend the box
        qMask - mask stating which voxels to use to stay within a fraction of (h,k,l)
        padeCoefficients - dictionary containing the coefficients for pade approximants for the moderator output
        figsFormat - fileName format to save figure for each fit.  Will not save if set to None.
        dtSpread - how far on each side of the nominal peak TOF to consider.
        fracHKL - the fraction for generating qMask
        minFracPixels -
        fracStop - the minimum intensity (as a ratio of the maximum intensity) of bins we include in the peak
        dQPixel - the side length of each voxel for fitting
        p - array of peak numbers to fit.  Useful for troubleshooting.
        neigh_length_m - the number of voxels we will smooth over (via np.convolve)
        zBG - the z score at which we consider events to be signal.  Set to negative to not use.
`       bgPolyOrder - the polynomial order the background is fit to for the TOF profile.  Typically
            the background is removed by only keeping signal, so linear is sufficient to take care of
            any small residual bakcground.
        doIterativeBackgroundFitting - do not use; leave as False.  TODO: Remove this
        q_frame - str; either 'sample' or 'lab'
        progressFile - the name of a file which will write the current peak number every 100 peaks.  Useful
            for monitoring batch jobs.  Set to None to not write file.
        minpplfrac, maxpplfrac - the range of pp_lambdas to check around the predicted pp_lambda as a fraction
            of pp_lambda
        mindtBinWidth - the smallest dt bin width (in us) allowed for TOF profile construction
        mindtBinWidth - the largest dt bin width (in us) allowed for TOF profile construction
        keepFitDict= bool; if True then each fit will be saved in a dictionary and returned.  For large peak sets,
            this can take a lot of memory.
        constraintScheme - which constraint scheme we will use - leave as 1 if you're not sure what this does.
        iccFitDict - a dictionary containing ICC fit constraints and possibly initial guesses
    Returns:
        peaks_ws - the peaks_ws with updated I, sig(I)
        paramList - a list of fit parameters for each peak.  Parameters are in the order:
            [peakNumber, energy (eV), sum(fitIntensities), 0.0, redChiSq, alpha, beta, R, T0, amplitude/scale, hat_width,
            k_conv, Ai (the background coefficients, A0 and A1 for linear), reducedChiSquared, pp_lambda]
        fitDict - if keepFitDict is False, an empty dictionary.  If keepFitDict is true, a dictionary (integer peak number as key)
            containing the x, yData, yFit for each peak.
    """
    if p is None:
        p = range(peaks_ws.getNumberPeaks())
    fitDict = {}
    for i in p:
        if progressFile is not None and i % 100 == 0:
            with open(progressFile, 'w') as f:
                f.write('%i\n' % (i))
        peak = peaks_ws.getPeak(i)
        if peak.getRunNumber() == run:
            try:  # for ppppp in [3]:#try:
                Box = getBoxFracHKL(peak, peaks_ws, MDdata, UBMatrix, i,
                                    dQ, fracHKL=fracHKL, dQPixel=dQPixel, q_frame=q_frame)
                wavelength = peak.getWavelength()  # in Angstrom
                energy = 81.804 / wavelength**2 / 1000.0  # in eV
                flightPath = peak.getL1() + peak.getL2()  # in m
                logger.information( '---fitting peak {:d}'.format(i))
                if Box.getNEvents() < 1 or np.all(np.abs(peak.getHKL()) == 0):
                    logger.information("Peak {:d} has 0 events or is HKL=000. Skipping!".format(p))
                    peak.setIntensity(0)
                    peak.setSigmaIntensity(1)
                    paramLisg.append([i, energy, 0.0, 1.0e10, 1.0e10] +
                                     [0 for i in range(mtd['fit_parameters'].rowCount())]+[0])

                    mtd.remove('MDbox_'+str(run)+'_'+str(i))
                    continue
                n_events = Box.getNumEventsArray()
                goodIDX, pp_lambda = getBGRemovedIndices(n_events, peak=peak, box=Box, qMask=qMask,
                                                         calc_pp_lambda=True, padeCoefficients=padeCoefficients,
                                                         mindtBinWidth=mindtBinWidth,
                                                         maxdtBinWidth=maxdtBinWidth,
                                                         pplmin_frac=minpplfrac, pplmax_frac=maxpplfrac,
                                                         constraintScheme=constraintScheme,
                                                         peakMaskSize=peakMaskSize, iccFitDict=iccFitDict)
                # --IN PRINCIPLE!!! WE CALCULATE THIS BEFORE GETTING HERE
                tofWS = mtd['tofWS']

                fitResults, fICC = doICCFit(
                    tofWS, energy, flightPath, padeCoefficients, fitOrder=bgPolyOrder, constraintScheme=constraintScheme,
                    iccFitDict=iccFitDict)
                chiSq = fitResults.OutputChi2overDoF

                r = mtd['fit_Workspace']
                param = mtd['fit_Parameters']
                tofWS = mtd['tofWS']

                iii = fICC.numParams() - 1
                fitBG = [param.row(int(iii+bgIDX+1))['Value']
                         for bgIDX in range(bgPolyOrder+1)]

                # Set the intensity before moving on to the next peak
                icProfile = r.readY(1)
                bgCoefficients = fitBG[::-1]
                # peak.setSigmaIntensity(np.sqrt(np.sum(icProfile)))i

                convBox = 1.0 * \
                    np.ones([neigh_length_m, neigh_length_m,
                             neigh_length_m]) / neigh_length_m**3
                conv_n_events = convolve(n_events, convBox)

                totEvents = np.sum(n_events[goodIDX*qMask])
                bgIDX = reduce(np.logical_and, [
                               ~goodIDX, qMask, conv_n_events > 0])
                bgEvents = np.mean(n_events[bgIDX])*np.sum(goodIDX*qMask)
                intensity, sigma, xStart, xStop = integratePeak(r.readX(0), icProfile, r.readY(0), np.polyval(bgCoefficients, r.readX(
                    1)), pp_lambda=pp_lambda, fracStop=fracStop, totEvents=totEvents, bgEvents=bgEvents, varFit=chiSq)
                # subtract background
                icProfile = icProfile - np.polyval(bgCoefficients, r.readX(1))
                peak.setIntensity(intensity)
                peak.setSigmaIntensity(sigma)
                if figsFormat is not None:
                    plotFit(figsFormat, r, tofWS, fICC, peak.getRunNumber(
                    ), i, energy, chiSq, fitBG, xStart, xStop, bgx0=None)
                if keepFitDict:
                    fitDict[i] = np.array(
                        [r.readX(0), r.readY(0), r.readY(1), r.readY(2)])
                paramList.append([i, energy, np.sum(icProfile), 0.0, chiSq] +
                                 [param.row(i)['Value'] for i in range(param.rowCount())]+[pp_lambda])
                mtd.remove('MDbox_'+str(run)+'_'+str(i))

            except KeyboardInterrupt:
                logger.warning('KeyboardInterrupt: Exiting Program!!!!!!!')
                sys.exit()
            except:  # Error with fitting
                # raise
                peak.setIntensity(0)
                peak.setSigmaIntensity(1)
                logger.warning('Error with peak ' + str(i))
                paramList.append(
                    [i, energy, 0.0, 1.0e10, 1.0e10] + [0 for i in range(10)]+[0])
                #paramList.append([i, energy, 0.0, 1.0e10,1.0e10] + [0 for i in range(mtd['fit_parameters'].rowCount())]+[0])
                continue
        mtd.remove('MDbox_'+str(run)+'_'+str(i))
    return peaks_ws, paramList, fitDict