ref: 0fe9aab4da39efa4585fccb5e3d931389f7ee592
dir: /python/test/bench/onset/benchonset.py/
#! /usr/bin/python
from aubio.bench.node import *
from os.path import dirname,basename
def mmean(l):
return sum(l)/max(float(len(l)),1)
def stdev(l):
smean = 0
if not len(l): return smean
lmean = mmean(l)
for i in l:
smean += (i-lmean)**2
smean *= 1. / len(l)
return smean**.5
class benchonset(bench):
""" list of values to store per file """
valuenames = ['orig','missed','Tm','expc','bad','Td']
""" list of lists to store per file """
valuelists = ['l','labs']
""" list of values to print per dir """
printnames = [ 'mode', 'thres', 'dist', 'prec', 'recl',
'GD', 'FP',
'Torig', 'Ttrue', 'Tfp', 'Tfn', 'TTm', 'TTd',
'aTtrue', 'aTfp', 'aTfn', 'aTm', 'aTd',
'mean', 'smean', 'amean', 'samean']
""" per dir """
formats = {'mode': "%12s" , 'thres': "%5.4s",
'dist': "%5.4s", 'prec': "%5.4s", 'recl': "%5.4s",
'Torig': "%5.4s", 'Ttrue': "%5.4s", 'Tfp': "%5.4s", 'Tfn': "%5.4s",
'TTm': "%5.4s", 'TTd': "%5.4s",
'aTtrue':"%5.4s", 'aTfp': "%5.4s", 'aTfn': "%5.4s",
'aTm': "%5.4s", 'aTd': "%5.4s",
'mean': "%5.40s", 'smean': "%5.40s",
'amean': "%5.40s", 'samean': "%5.40s",
"GD": "%5.4s", "FP": "%5.4s",
"GDm": "%5.4s", "FPd": "%5.4s"}
def dir_eval(self):
""" evaluate statistical data over the directory """
v = self.v
v['mode'] = self.params.onsetmode
v['thres'] = self.params.threshold
v['bufsize'] = self.params.bufsize
v['hopsize'] = self.params.hopsize
v['silence'] = self.params.silence
v['mintol'] = self.params.mintol
v['Torig'] = sum(v['orig'])
v['TTm'] = sum(v['Tm'])
v['TTd'] = sum(v['Td'])
v['Texpc'] = sum(v['expc'])
v['Tbad'] = sum(v['bad'])
v['Tmissed'] = sum(v['missed'])
v['aTm'] = mmean(v['Tm'])
v['aTd'] = mmean(v['Td'])
v['mean'] = mmean(v['l'])
v['smean'] = stdev(v['l'])
v['amean'] = mmean(v['labs'])
v['samean'] = stdev(v['labs'])
# old type calculations
# good detection rate
v['GD'] = 100.*(v['Torig']-v['Tmissed']-v['TTm'])/v['Torig']
# false positive rate
v['FP'] = 100.*(v['Tbad']+v['TTd'])/v['Torig']
# good detection counting merged detections as good
v['GDm'] = 100.*(v['Torig']-v['Tmissed'])/v['Torig']
# false positives counting doubled as good
v['FPd'] = 100.*v['Tbad']/v['Torig']
# mirex type annotations
totaltrue = v['Texpc']-v['Tbad']-v['TTd']
totalfp = v['Tbad']+v['TTd']
totalfn = v['Tmissed']+v['TTm']
self.v['Ttrue'] = totaltrue
self.v['Tfp'] = totalfp
self.v['Tfn'] = totalfn
# average over the number of annotation files
N = float(len(self.reslist))
self.v['aTtrue'] = totaltrue/N
self.v['aTfp'] = totalfp/N
self.v['aTfn'] = totalfn/N
# F-measure
self.P = 100.*float(totaltrue)/max(totaltrue + totalfp,1)
self.R = 100.*float(totaltrue)/max(totaltrue + totalfn,1)
#if self.R < 0: self.R = 0
self.F = 2.* self.P*self.R / max(float(self.P+self.R),1)
self.v['dist'] = self.F
self.v['prec'] = self.P
self.v['recl'] = self.R
"""
Plot functions
"""
def plotroc(self,d,plottitle=""):
import Gnuplot, Gnuplot.funcutils
gd = []
fp = []
for i in self.vlist:
gd.append(i['GD'])
fp.append(i['FP'])
d.append(Gnuplot.Data(fp, gd, with='linespoints',
title="%s %s" % (plottitle,i['mode']) ))
def plotplotroc(self,d,outplot=0,extension='ps'):
import Gnuplot, Gnuplot.funcutils
from sys import exit
g = Gnuplot.Gnuplot(debug=0, persist=1)
if outplot:
if extension == 'ps': ext, extension = '.ps' , 'postscript'
elif extension == 'png': ext, extension = '.png', 'png'
elif extension == 'svg': ext, extension = '.svg', 'svg'
else: exit("ERR: unknown plot extension")
g('set terminal %s' % extension)
g('set output \'roc-%s%s\'' % (outplot,ext))
xmax = 30 #max(fp)
ymin = 50
g('set xrange [0:%f]' % xmax)
g('set yrange [%f:100]' % ymin)
# grid set
g('set grid')
g('set xtics 0,5,%f' % xmax)
g('set ytics %f,5,100' % ymin)
g('set key 27,65')
#g('set format \"%g\"')
g.title(basename(self.datadir))
g.xlabel('false positives (%)')
g.ylabel('correct detections (%)')
g.plot(*d)
def plotpr(self,d,plottitle=""):
import Gnuplot, Gnuplot.funcutils
x = []
y = []
for i in self.vlist:
x.append(i['prec'])
y.append(i['recl'])
d.append(Gnuplot.Data(x, y, with='linespoints',
title="%s %s" % (plottitle,i['mode']) ))
def plotplotpr(self,d,outplot=0,extension='ps'):
import Gnuplot, Gnuplot.funcutils
from sys import exit
g = Gnuplot.Gnuplot(debug=0, persist=1)
if outplot:
if extension == 'ps': ext, extension = '.ps' , 'postscript'
elif extension == 'png': ext, extension = '.png', 'png'
elif extension == 'svg': ext, extension = '.svg', 'svg'
else: exit("ERR: unknown plot extension")
g('set terminal %s' % extension)
g('set output \'pr-%s%s\'' % (outplot,ext))
g.title(basename(self.datadir))
g.xlabel('Recall (%)')
g.ylabel('Precision (%)')
g.plot(*d)
def plotfmeas(self,d,plottitle=""):
import Gnuplot, Gnuplot.funcutils
x,y = [],[]
for i in self.vlist:
x.append(i['thres'])
y.append(i['dist'])
d.append(Gnuplot.Data(x, y, with='linespoints',
title="%s %s" % (plottitle,i['mode']) ))
def plotplotfmeas(self,d,outplot="",extension='ps', title="F-measure"):
import Gnuplot, Gnuplot.funcutils
from sys import exit
g = Gnuplot.Gnuplot(debug=0, persist=1)
if outplot:
if extension == 'ps': terminal = 'postscript'
elif extension == 'png': terminal = 'png'
elif extension == 'svg': terminal = 'svg'
else: exit("ERR: unknown plot extension")
g('set terminal %s' % terminal)
g('set output \'fmeas-%s.%s\'' % (outplot,extension))
g.xlabel('threshold \\delta')
g.ylabel('F-measure (%)')
g('set xrange [0:1.2]')
g('set yrange [0:100]')
g.title(basename(self.datadir))
# grid set
#g('set grid')
#g('set xtics 0,5,%f' % xmax)
#g('set ytics %f,5,100' % ymin)
#g('set key 27,65')
#g('set format \"%g\"')
g.plot(*d)
def plotfmeasvar(self,d,var,plottitle=""):
import Gnuplot, Gnuplot.funcutils
x,y = [],[]
for i in self.vlist:
x.append(i[var])
y.append(i['dist'])
d.append(Gnuplot.Data(x, y, with='linespoints',
title="%s %s" % (plottitle,i['mode']) ))
def plotplotfmeasvar(self,d,var,outplot="",extension='ps', title="F-measure"):
import Gnuplot, Gnuplot.funcutils
from sys import exit
g = Gnuplot.Gnuplot(debug=0, persist=1)
if outplot:
if extension == 'ps': terminal = 'postscript'
elif extension == 'png': terminal = 'png'
elif extension == 'svg': terminal = 'svg'
else: exit("ERR: unknown plot extension")
g('set terminal %s' % terminal)
g('set output \'fmeas-%s.%s\'' % (outplot,extension))
g.xlabel(var)
g.ylabel('F-measure (%)')
#g('set xrange [0:1.2]')
g('set yrange [0:100]')
g.title(basename(self.datadir))
g.plot(*d)
def plotdiffs(self,d,plottitle=""):
import Gnuplot, Gnuplot.funcutils
v = self.v
l = v['l']
mean = v['mean']
smean = v['smean']
amean = v['amean']
samean = v['samean']
val = []
per = [0] * 100
for i in range(0,100):
val.append(i*.001-.05)
for j in l:
if abs(j-val[i]) <= 0.001:
per[i] += 1
total = v['Torig']
for i in range(len(per)): per[i] /= total/100.
d.append(Gnuplot.Data(val, per, with='fsteps',
title="%s %s" % (plottitle,v['mode']) ))
#d.append('mean=%f,sigma=%f,eps(x) title \"\"'% (mean,smean))
#d.append('mean=%f,sigma=%f,eps(x) title \"\"'% (amean,samean))
def plotplotdiffs(self,d,outplot=0,extension='ps'):
import Gnuplot, Gnuplot.funcutils
from sys import exit
g = Gnuplot.Gnuplot(debug=0, persist=1)
if outplot:
if extension == 'ps': ext, extension = '.ps' , 'postscript'
elif extension == 'png': ext, extension = '.png', 'png'
elif extension == 'svg': ext, extension = '.svg', 'svg'
else: exit("ERR: unknown plot extension")
g('set terminal %s' % extension)
g('set output \'diffhist-%s%s\'' % (outplot,ext))
g('eps(x) = 1./(sigma*(2.*3.14159)**.5) * exp ( - ( x - mean ) ** 2. / ( 2. * sigma ** 2. ))')
g.title(basename(self.datadir))
g.xlabel('delay to hand-labelled onset (s)')
g.ylabel('% number of correct detections / ms ')
g('set xrange [-0.05:0.05]')
g('set yrange [0:20]')
g.plot(*d)
def plothistcat(self,d,plottitle=""):
import Gnuplot, Gnuplot.funcutils
total = v['Torig']
for i in range(len(per)): per[i] /= total/100.
d.append(Gnuplot.Data(val, per, with='fsteps',
title="%s %s" % (plottitle,v['mode']) ))
#d.append('mean=%f,sigma=%f,eps(x) title \"\"'% (mean,smean))
#d.append('mean=%f,sigma=%f,eps(x) title \"\"'% (amean,samean))
def plotplothistcat(self,d,outplot=0,extension='ps'):
import Gnuplot, Gnuplot.funcutils
from sys import exit
g = Gnuplot.Gnuplot(debug=0, persist=1)
if outplot:
if extension == 'ps': ext, extension = '.ps' , 'postscript'
elif extension == 'png': ext, extension = '.png', 'png'
elif extension == 'svg': ext, extension = '.svg', 'svg'
else: exit("ERR: unknown plot extension")
g('set terminal %s' % extension)
g('set output \'diffhist-%s%s\'' % (outplot,ext))
g('eps(x) = 1./(sigma*(2.*3.14159)**.5) * exp ( - ( x - mean ) ** 2. / ( 2. * sigma ** 2. ))')
g.title(basename(self.datadir))
g.xlabel('delay to hand-labelled onset (s)')
g.ylabel('% number of correct detections / ms ')
g('set xrange [-0.05:0.05]')
g('set yrange [0:20]')
g.plot(*d)