id	sid	tid	token	lemma	pos
flr-250	1	1	frontline	frontline	NOUN
flr-250	1	2	learning	learn	VERB
flr-250	1	3	research	research	NOUN
flr-250	1	4	vol.5	vol.5	ADV
flr-250	1	5	no	no	NOUN
flr-250	1	6	.	.	NOUN
flr-250	1	7	3	3	NUM
flr-250	1	8	special	special	ADJ
flr-250	1	9	issue	issue	NOUN
flr-250	1	10	(	(	PUNCT
flr-250	1	11	2017	2017	NUM
flr-250	1	12	)	)	PUNCT
flr-250	1	13	31	31	NUM
flr-250	1	14	42	42	NUM
flr-250	1	15	issn	issn	VERB
flr-250	1	16	2295	2295	NUM
flr-250	1	17	-	-	SYM
flr-250	1	18	3159	3159	NUM
flr-250	1	19	elizabeth	elizabeth	PROPN
flr-250	1	20	a.	a.	PROPN
flr-250	1	21	krupinski	krupinski	PROPN
flr-250	1	22	,	,	PUNCT
flr-250	1	23	phd	phd	NOUN
flr-250	1	24	,	,	PUNCT
flr-250	1	25	department	department	NOUN
flr-250	1	26	of	of	ADP
flr-250	1	27	radiology	radiology	PROPN
flr-250	1	28	&	&	CCONJ
flr-250	1	29	imaging	imaging	PROPN
flr-250	1	30	sciences	sciences	PROPN
flr-250	1	31	,	,	PUNCT
flr-250	1	32	emory	emory	PROPN
flr-250	1	33	university	university	PROPN
flr-250	1	34	1364	1364	NUM
flr-250	1	35	clifton	clifton	PROPN
flr-250	1	36	rd	rd	PROPN
flr-250	1	37	ne	ne	PROPN
flr-250	1	38	d107	d107	PROPN
flr-250	1	39	atlanta	atlanta	PROPN
flr-250	1	40	,	,	PUNCT
flr-250	1	41	ga30322	ga30322	PROPN
flr-250	1	42	,	,	PUNCT
flr-250	1	43	united	united	PROPN
flr-250	1	44	states	states	PROPN
flr-250	1	45	.	.	PUNCT
flr-250	2	1	email	email	NOUN
flr-250	2	2	:	:	PUNCT
flr-250	2	3	ekrupin&emory.edu	ekrupin&emory.edu	X
flr-250	2	4	.	.	PUNCT
flr-250	3	1	doi	doi	NOUN
flr-250	3	2	:	:	PUNCT
flr-250	3	3	http://dx.doi.org/10.14786/flr.v5i3.250	http://dx.doi.org/10.14786/flr.v5i3.250	NOUN
flr-250	3	4	receiver	receiver	NOUN
flr-250	3	5	operating	operate	VERB
flr-250	3	6	characteristic	characteristic	NOUN
flr-250	3	7	(	(	PUNCT
flr-250	3	8	roc	roc	PROPN
flr-250	3	9	)	)	PUNCT
flr-250	3	10	analysis	analysis	NOUN
flr-250	3	11	elizabeth	elizabeth	PROPN
flr-250	3	12	a.	a.	PROPN
flr-250	3	13	krupinski	krupinski	PROPN
flr-250	3	14	emory	emory	PROPN
flr-250	3	15	university	university	PROPN
flr-250	3	16	,	,	PUNCT
flr-250	3	17	usa	usa	PROPN
flr-250	3	18	article	article	NOUN
flr-250	3	19	received	receive	VERB
flr-250	3	20	13	13	NUM
flr-250	3	21	april	april	PROPN
flr-250	3	22	/	/	PUNCT
flr-250	3	23	revised	revise	VERB
flr-250	3	24	23	23	NUM
flr-250	3	25	march	march	NOUN
flr-250	3	26	/	/	PUNCT
flr-250	3	27	accepted	accept	VERB
flr-250	3	28	23	23	NUM
flr-250	3	29	march	march	NOUN
flr-250	3	30	/	/	SYM
flr-250	3	31	available	available	ADJ
flr-250	3	32	online	online	ADJ
flr-250	3	33	14	14	NUM
flr-250	3	34	july	july	PROPN
flr-250	3	35	abstract	abstract	ADJ
flr-250	3	36	visual	visual	ADJ
flr-250	3	37	expertise	expertise	NOUN
flr-250	3	38	covers	cover	VERB
flr-250	3	39	a	a	DET
flr-250	3	40	broad	broad	ADJ
flr-250	3	41	range	range	NOUN
flr-250	3	42	of	of	ADP
flr-250	3	43	types	type	NOUN
flr-250	3	44	of	of	ADP
flr-250	3	45	studies	study	NOUN
flr-250	3	46	and	and	CCONJ
flr-250	3	47	methodologies	methodology	NOUN
flr-250	3	48	.	.	PUNCT
flr-250	4	1	many	many	ADJ
flr-250	4	2	studies	study	NOUN
flr-250	4	3	incorporate	incorporate	VERB
flr-250	4	4	some	some	DET
flr-250	4	5	measure(s	measure(s	NOUN
flr-250	4	6	)	)	PUNCT
flr-250	4	7	of	of	ADP
flr-250	4	8	observer	observer	NOUN
flr-250	4	9	performance	performance	NOUN
flr-250	4	10	or	or	CCONJ
flr-250	4	11	how	how	SCONJ
flr-250	4	12	well	well	ADV
flr-250	4	13	participants	participant	NOUN
flr-250	4	14	perform	perform	VERB
flr-250	4	15	on	on	ADP
flr-250	4	16	a	a	DET
flr-250	4	17	given	give	VERB
flr-250	4	18	task	task	NOUN
flr-250	4	19	.	.	PUNCT
flr-250	5	1	receiver	receiver	ADJ
flr-250	5	2	operating	operate	VERB
flr-250	5	3	characteristic	characteristic	NOUN
flr-250	5	4	(	(	PUNCT
flr-250	5	5	roc	roc	PROPN
flr-250	5	6	)	)	PUNCT
flr-250	5	7	analysis	analysis	NOUN
flr-250	5	8	is	be	AUX
flr-250	5	9	a	a	DET
flr-250	5	10	method	method	NOUN
flr-250	5	11	commonly	commonly	ADV
flr-250	5	12	used	use	VERB
flr-250	5	13	in	in	ADP
flr-250	5	14	signal	signal	ADJ
flr-250	5	15	detection	detection	NOUN
flr-250	5	16	tasks	task	NOUN
flr-250	5	17	(	(	PUNCT
flr-250	5	18	i.e.	i.e.	X
flr-250	5	19	,	,	PUNCT
flr-250	5	20	those	those	PRON
flr-250	5	21	in	in	ADP
flr-250	5	22	which	which	PRON
flr-250	5	23	the	the	DET
flr-250	5	24	observer	observer	NOUN
flr-250	5	25	must	must	AUX
flr-250	5	26	decide	decide	VERB
flr-250	5	27	whether	whether	SCONJ
flr-250	5	28	or	or	CCONJ
flr-250	5	29	not	not	PART
flr-250	5	30	a	a	DET
flr-250	5	31	target	target	NOUN
flr-250	5	32	is	be	AUX
flr-250	5	33	present	present	ADJ
flr-250	5	34	or	or	CCONJ
flr-250	5	35	absent	absent	ADJ
flr-250	5	36	;	;	PUNCT
flr-250	5	37	or	or	CCONJ
flr-250	5	38	must	must	AUX
flr-250	5	39	classify	classify	VERB
flr-250	5	40	a	a	DET
flr-250	5	41	given	give	VERB
flr-250	5	42	target	target	NOUN
flr-250	5	43	as	as	ADP
flr-250	5	44	belonging	belong	VERB
flr-250	5	45	to	to	ADP
flr-250	5	46	one	one	NUM
flr-250	5	47	category	category	NOUN
flr-250	5	48	or	or	CCONJ
flr-250	5	49	another	another	PRON
flr-250	5	50	)	)	PUNCT
flr-250	5	51	,	,	PUNCT
flr-250	5	52	especially	especially	ADV
flr-250	5	53	those	those	PRON
flr-250	5	54	in	in	ADP
flr-250	5	55	the	the	DET
flr-250	5	56	medical	medical	ADJ
flr-250	5	57	imaging	imaging	NOUN
flr-250	5	58	literature	literature	NOUN
flr-250	5	59	.	.	PUNCT
flr-250	6	1	this	this	DET
flr-250	6	2	frontline	frontline	NOUN
flr-250	6	3	paper	paper	NOUN
flr-250	6	4	will	will	AUX
flr-250	6	5	review	review	VERB
flr-250	6	6	some	some	PRON
flr-250	6	7	of	of	ADP
flr-250	6	8	the	the	DET
flr-250	6	9	core	core	NOUN
flr-250	6	10	theoretical	theoretical	ADJ
flr-250	6	11	underpinnings	underpinning	NOUN
flr-250	6	12	of	of	ADP
flr-250	6	13	roc	roc	PROPN
flr-250	6	14	analysis	analysis	NOUN
flr-250	6	15	,	,	PUNCT
flr-250	6	16	provide	provide	VERB
flr-250	6	17	an	an	DET
flr-250	6	18	overview	overview	NOUN
flr-250	6	19	of	of	ADP
flr-250	6	20	how	how	SCONJ
flr-250	6	21	to	to	PART
flr-250	6	22	conduct	conduct	VERB
flr-250	6	23	an	an	DET
flr-250	6	24	roc	roc	PROPN
flr-250	6	25	study	study	NOUN
flr-250	6	26	,	,	PUNCT
flr-250	6	27	and	and	CCONJ
flr-250	6	28	discuss	discuss	VERB
flr-250	6	29	some	some	PRON
flr-250	6	30	of	of	ADP
flr-250	6	31	the	the	DET
flr-250	6	32	key	key	ADJ
flr-250	6	33	variants	variant	NOUN
flr-250	6	34	of	of	ADP
flr-250	6	35	roc	roc	PROPN
flr-250	6	36	analysis	analysis	NOUN
flr-250	6	37	and	and	CCONJ
flr-250	6	38	their	their	PRON
flr-250	6	39	applications	application	NOUN
flr-250	6	40	.	.	PUNCT
flr-250	7	1	keywords	keyword	NOUN
flr-250	7	2	:	:	PUNCT
flr-250	7	3	receiver	receiver	ADJ
flr-250	7	4	operating	operate	VERB
flr-250	7	5	characteristic	characteristic	ADJ
flr-250	7	6	analysis	analysis	NOUN
flr-250	7	7	,	,	PUNCT
flr-250	7	8	roc	roc	PROPN
flr-250	7	9	,	,	PUNCT
flr-250	7	10	observer	observer	NOUN
flr-250	7	11	performance	performance	NOUN
flr-250	7	12	,	,	PUNCT
flr-250	7	13	visual	visual	ADJ
flr-250	7	14	expertise	expertise	NOUN
flr-250	7	15	http://dx.doi.org/10.14786/flr.v5i3.250	http://dx.doi.org/10.14786/flr.v5i3.250	PROPN
flr-250	7	16	krupinski	krupinski	PROPN
flr-250	8	1	|	|	ADV
flr-250	8	2	f	f	NOUN
flr-250	8	3	l	l	NOUN
flr-250	8	4	r	r	NOUN
flr-250	8	5	32	32	NUM
flr-250	8	6	1	1	NUM
flr-250	8	7	.	.	PUNCT
flr-250	9	1	introduction	introduction	NOUN
flr-250	9	2	visual	visual	ADJ
flr-250	9	3	expertise	expertise	NOUN
flr-250	9	4	can	can	AUX
flr-250	9	5	be	be	AUX
flr-250	9	6	measured	measure	VERB
flr-250	9	7	or	or	CCONJ
flr-250	9	8	assessed	assess	VERB
flr-250	9	9	in	in	ADP
flr-250	9	10	a	a	DET
flr-250	9	11	variety	variety	NOUN
flr-250	9	12	of	of	ADP
flr-250	9	13	ways	way	NOUN
flr-250	9	14	,	,	PUNCT
flr-250	9	15	but	but	CCONJ
flr-250	9	16	in	in	ADP
flr-250	9	17	many	many	ADJ
flr-250	9	18	cases	case	NOUN
flr-250	9	19	it	it	PRON
flr-250	9	20	is	be	AUX
flr-250	9	21	the	the	DET
flr-250	9	22	behavioral	behavioral	ADJ
flr-250	9	23	outcome	outcome	NOUN
flr-250	9	24	related	relate	VERB
flr-250	9	25	to	to	ADP
flr-250	9	26	visual	visual	ADJ
flr-250	9	27	expertise	expertise	NOUN
flr-250	9	28	that	that	PRON
flr-250	9	29	is	be	AUX
flr-250	9	30	of	of	ADP
flr-250	9	31	ultimate	ultimate	ADJ
flr-250	9	32	concern	concern	NOUN
flr-250	9	33	.	.	PUNCT
flr-250	10	1	for	for	ADP
flr-250	10	2	example	example	NOUN
flr-250	10	3	,	,	PUNCT
flr-250	10	4	in	in	ADP
flr-250	10	5	medical	medical	ADJ
flr-250	10	6	imaging	imaging	NOUN
flr-250	10	7	(	(	PUNCT
flr-250	10	8	e.g.	e.g.	ADV
flr-250	10	9	,	,	PUNCT
flr-250	10	10	radiology	radiology	NOUN
flr-250	10	11	,	,	PUNCT
flr-250	10	12	pathology	pathology	NOUN
flr-250	10	13	,	,	PUNCT
flr-250	10	14	telemedicine	telemedicine	NOUN
flr-250	10	15	)	)	PUNCT
flr-250	10	16	a	a	DET
flr-250	10	17	physician	physician	NOUN
flr-250	10	18	must	must	AUX
flr-250	10	19	view	view	VERB
flr-250	10	20	an	an	DET
flr-250	10	21	image	image	NOUN
flr-250	10	22	(	(	PUNCT
flr-250	10	23	e.g.	e.g.	ADV
flr-250	10	24	,	,	PUNCT
flr-250	10	25	x	x	ADJ
flr-250	10	26	-	-	NOUN
flr-250	10	27	ray	ray	NOUN
flr-250	10	28	exam	exam	NOUN
flr-250	10	29	,	,	PUNCT
flr-250	10	30	pathology	pathology	NOUN
flr-250	10	31	slide	slide	NOUN
flr-250	10	32	,	,	PUNCT
flr-250	10	33	photograph	photograph	NOUN
flr-250	10	34	of	of	ADP
flr-250	10	35	a	a	DET
flr-250	10	36	skin	skin	NOUN
flr-250	10	37	lesion	lesion	NOUN
flr-250	10	38	)	)	PUNCT
flr-250	10	39	and	and	CCONJ
flr-250	10	40	render	render	VERB
flr-250	10	41	a	a	DET
flr-250	10	42	diagnostic	diagnostic	ADJ
flr-250	10	43	decision	decision	NOUN
flr-250	10	44	(	(	PUNCT
flr-250	10	45	e.g.	e.g.	ADV
flr-250	10	46	,	,	PUNCT
flr-250	10	47	tumor	tumor	NOUN
flr-250	10	48	present	present	ADJ
flr-250	10	49	or	or	CCONJ
flr-250	10	50	absent	absent	ADJ
flr-250	10	51	)	)	PUNCT
flr-250	10	52	,	,	PUNCT
flr-250	10	53	prognosis	prognosis	NOUN
flr-250	10	54	(	(	PUNCT
flr-250	10	55	e.g.	e.g.	ADV
flr-250	10	56	,	,	PUNCT
flr-250	10	57	malignant	malignant	ADJ
flr-250	10	58	vs	vs	ADP
flr-250	10	59	benign	benign	NOUN
flr-250	10	60	)	)	PUNCT
flr-250	10	61	and/or	and/or	CCONJ
flr-250	10	62	a	a	DET
flr-250	10	63	recommendation	recommendation	NOUN
flr-250	10	64	for	for	ADP
flr-250	10	65	a	a	DET
flr-250	10	66	treatment	treatment	NOUN
flr-250	10	67	plan	plan	NOUN
flr-250	10	68	(	(	PUNCT
flr-250	10	69	e.g.	e.g.	ADV
flr-250	10	70	,	,	PUNCT
flr-250	10	71	additional	additional	ADJ
flr-250	10	72	exams	exam	NOUN
flr-250	10	73	or	or	CCONJ
flr-250	10	74	surgery	surgery	NOUN
flr-250	10	75	)	)	PUNCT
flr-250	10	76	.	.	PUNCT
flr-250	11	1	in	in	ADP
flr-250	11	2	the	the	DET
flr-250	11	3	military	military	NOUN
flr-250	11	4	,	,	PUNCT
flr-250	11	5	radar	radar	NOUN
flr-250	11	6	screens	screen	NOUN
flr-250	11	7	need	need	VERB
flr-250	11	8	to	to	PART
flr-250	11	9	be	be	AUX
flr-250	11	10	monitored	monitor	VERB
flr-250	11	11	for	for	ADP
flr-250	11	12	approaching	approach	VERB
flr-250	11	13	targets	target	NOUN
flr-250	11	14	(	(	PUNCT
flr-250	11	15	e.g.	e.g.	ADV
flr-250	11	16	,	,	PUNCT
flr-250	11	17	missiles	missile	NOUN
flr-250	11	18	)	)	PUNCT
flr-250	11	19	and	and	CCONJ
flr-250	11	20	decisions	decision	NOUN
flr-250	11	21	made	make	VERB
flr-250	11	22	as	as	ADP
flr-250	11	23	to	to	ADP
flr-250	11	24	whether	whether	SCONJ
flr-250	11	25	they	they	PRON
flr-250	11	26	are	be	AUX
flr-250	11	27	friend	friend	NOUN
flr-250	11	28	or	or	CCONJ
flr-250	11	29	foe	foe	VERB
flr-250	11	30	and	and	CCONJ
flr-250	11	31	whether	whether	SCONJ
flr-250	11	32	to	to	PART
flr-250	11	33	escalate	escalate	VERB
flr-250	11	34	the	the	DET
flr-250	11	35	finding	finding	NOUN
flr-250	11	36	to	to	ADP
flr-250	11	37	the	the	DET
flr-250	11	38	next	next	ADJ
flr-250	11	39	level	level	NOUN
flr-250	11	40	of	of	ADP
flr-250	11	41	action	action	NOUN
flr-250	11	42	.	.	PUNCT
flr-250	12	1	there	there	PRON
flr-250	12	2	are	be	VERB
flr-250	12	3	many	many	ADJ
flr-250	12	4	other	other	ADJ
flr-250	12	5	situations	situation	NOUN
flr-250	12	6	where	where	SCONJ
flr-250	12	7	these	these	DET
flr-250	12	8	types	type	NOUN
flr-250	12	9	of	of	ADP
flr-250	12	10	visual	visual	ADJ
flr-250	12	11	detection	detection	NOUN
flr-250	12	12	and/or	and/or	CCONJ
flr-250	12	13	classification	classification	NOUN
flr-250	12	14	tasks	task	NOUN
flr-250	12	15	take	take	VERB
flr-250	12	16	place	place	NOUN
flr-250	12	17	in	in	ADP
flr-250	12	18	real	real	ADJ
flr-250	12	19	life	life	NOUN
flr-250	12	20	,	,	PUNCT
flr-250	12	21	and	and	CCONJ
flr-250	12	22	where	where	SCONJ
flr-250	12	23	investigations	investigation	NOUN
flr-250	12	24	take	take	VERB
flr-250	12	25	place	place	NOUN
flr-250	12	26	to	to	PART
flr-250	12	27	assess	assess	VERB
flr-250	12	28	how	how	SCONJ
flr-250	12	29	expertise	expertise	NOUN
flr-250	12	30	impacts	impact	VERB
flr-250	12	31	these	these	DET
flr-250	12	32	decisions	decision	NOUN
flr-250	12	33	and	and	CCONJ
flr-250	12	34	what	what	PRON
flr-250	12	35	we	we	PRON
flr-250	12	36	can	can	AUX
flr-250	12	37	do	do	VERB
flr-250	12	38	to	to	PART
flr-250	12	39	improve	improve	VERB
flr-250	12	40	them	they	PRON
flr-250	12	41	.	.	PUNCT
flr-250	13	1	in	in	ADP
flr-250	13	2	medicine	medicine	NOUN
flr-250	13	3	this	this	PRON
flr-250	13	4	is	be	AUX
flr-250	13	5	particularly	particularly	ADV
flr-250	13	6	important	important	ADJ
flr-250	13	7	as	as	ADP
flr-250	13	8	when	when	SCONJ
flr-250	13	9	decision	decision	NOUN
flr-250	13	10	errors	error	NOUN
flr-250	13	11	are	be	AUX
flr-250	13	12	made	make	VERB
flr-250	13	13	they	they	PRON
flr-250	13	14	have	have	VERB
flr-250	13	15	direct	direct	ADJ
flr-250	13	16	and	and	CCONJ
flr-250	13	17	significant	significant	ADJ
flr-250	13	18	impacts	impact	NOUN
flr-250	13	19	on	on	ADP
flr-250	13	20	patient	patient	ADJ
flr-250	13	21	care	care	NOUN
flr-250	13	22	and	and	CCONJ
flr-250	13	23	well	well	ADV
flr-250	13	24	-	-	PUNCT
flr-250	13	25	being	be	AUX
flr-250	13	26	[	[	X
flr-250	13	27	1	1	NUM
flr-250	13	28	-	-	SYM
flr-250	13	29	2	2	NUM
flr-250	13	30	]	]	PUNCT
flr-250	13	31	.	.	PUNCT
flr-250	14	1	the	the	DET
flr-250	14	2	problem	problem	NOUN
flr-250	14	3	with	with	ADP
flr-250	14	4	“	"	PUNCT
flr-250	14	5	real	real	ADJ
flr-250	14	6	life	life	NOUN
flr-250	14	7	”	"	PUNCT
flr-250	14	8	is	be	AUX
flr-250	14	9	that	that	PRON
flr-250	14	10	is	be	AUX
flr-250	14	11	often	often	ADV
flr-250	14	12	very	very	ADV
flr-250	14	13	difficult	difficult	ADJ
flr-250	14	14	to	to	PART
flr-250	14	15	determine	determine	VERB
flr-250	14	16	how	how	SCONJ
flr-250	14	17	well	well	ADV
flr-250	14	18	the	the	DET
flr-250	14	19	interpreter	interpreter	NOUN
flr-250	14	20	is	be	AUX
flr-250	14	21	actually	actually	ADV
flr-250	14	22	performing	perform	VERB
flr-250	14	23	.	.	PUNCT
flr-250	15	1	feedback	feedback	NOUN
flr-250	15	2	is	be	AUX
flr-250	15	3	often	often	ADV
flr-250	15	4	not	not	PART
flr-250	15	5	provided	provide	VERB
flr-250	15	6	and	and	CCONJ
flr-250	15	7	when	when	SCONJ
flr-250	15	8	it	it	PRON
flr-250	15	9	is	be	AUX
flr-250	15	10	it	it	PRON
flr-250	15	11	is	be	AUX
flr-250	15	12	often	often	ADV
flr-250	15	13	separate	separate	ADJ
flr-250	15	14	in	in	ADP
flr-250	15	15	time	time	NOUN
flr-250	15	16	from	from	ADP
flr-250	15	17	the	the	DET
flr-250	15	18	actual	actual	ADJ
flr-250	15	19	decision	decision	NOUN
flr-250	15	20	making	make	VERB
flr-250	15	21	event	event	NOUN
flr-250	15	22	–	–	PUNCT
flr-250	15	23	often	often	ADV
flr-250	15	24	quite	quite	ADV
flr-250	15	25	disparate	disparate	ADJ
flr-250	15	26	in	in	ADP
flr-250	15	27	time	time	NOUN
flr-250	15	28	,	,	PUNCT
flr-250	15	29	making	make	VERB
flr-250	15	30	the	the	DET
flr-250	15	31	feedback	feedback	NOUN
flr-250	15	32	less	less	ADV
flr-250	15	33	impactful	impactful	ADJ
flr-250	15	34	.	.	PUNCT
flr-250	16	1	compounding	compound	VERB
flr-250	16	2	the	the	DET
flr-250	16	3	problem	problem	NOUN
flr-250	16	4	is	be	AUX
flr-250	16	5	that	that	SCONJ
flr-250	16	6	there	there	PRON
flr-250	16	7	is	be	VERB
flr-250	16	8	often	often	ADV
flr-250	16	9	not	not	PART
flr-250	16	10	a	a	DET
flr-250	16	11	single	single	ADJ
flr-250	16	12	correct	correct	ADJ
flr-250	16	13	answer	answer	NOUN
flr-250	16	14	.	.	PUNCT
flr-250	17	1	thus	thus	ADV
flr-250	17	2	,	,	PUNCT
flr-250	17	3	the	the	DET
flr-250	17	4	majority	majority	NOUN
flr-250	17	5	of	of	ADP
flr-250	17	6	performance	performance	NOUN
flr-250	17	7	assessments	assessment	NOUN
flr-250	17	8	are	be	AUX
flr-250	17	9	done	do	VERB
flr-250	17	10	in	in	ADP
flr-250	17	11	the	the	DET
flr-250	17	12	context	context	NOUN
flr-250	17	13	of	of	ADP
flr-250	17	14	research	research	NOUN
flr-250	17	15	and	and	CCONJ
flr-250	17	16	are	be	AUX
flr-250	17	17	often	often	ADV
flr-250	17	18	focused	focus	VERB
flr-250	17	19	on	on	ADP
flr-250	17	20	assessing	assess	VERB
flr-250	17	21	observer	observer	NOUN
flr-250	17	22	performance	performance	NOUN
flr-250	17	23	in	in	ADP
flr-250	17	24	the	the	DET
flr-250	17	25	context	context	NOUN
flr-250	17	26	of	of	ADP
flr-250	17	27	comparing	compare	VERB
flr-250	17	28	a	a	DET
flr-250	17	29	new	new	ADJ
flr-250	17	30	technique	technique	NOUN
flr-250	17	31	or	or	CCONJ
flr-250	17	32	technology	technology	NOUN
flr-250	17	33	to	to	ADP
flr-250	17	34	an	an	DET
flr-250	17	35	existing	exist	VERB
flr-250	17	36	one	one	NUM
flr-250	17	37	.	.	PUNCT
flr-250	18	1	to	to	PART
flr-250	18	2	deal	deal	VERB
flr-250	18	3	with	with	ADP
flr-250	18	4	the	the	DET
flr-250	18	5	complex	complex	ADJ
flr-250	18	6	nature	nature	NOUN
flr-250	18	7	of	of	ADP
flr-250	18	8	decision	decision	NOUN
flr-250	18	9	interpretation	interpretation	NOUN
flr-250	18	10	tasks	task	NOUN
flr-250	18	11	where	where	SCONJ
flr-250	18	12	it	it	PRON
flr-250	18	13	is	be	AUX
flr-250	18	14	important	important	ADJ
flr-250	18	15	to	to	PART
flr-250	18	16	understand	understand	VERB
flr-250	18	17	and	and	CCONJ
flr-250	18	18	balance	balance	VERB
flr-250	18	19	the	the	DET
flr-250	18	20	consequences	consequence	NOUN
flr-250	18	21	both	both	CCONJ
flr-250	18	22	correct	correct	ADJ
flr-250	18	23	and	and	CCONJ
flr-250	18	24	incorrect	incorrect	ADJ
flr-250	18	25	decisions	decision	NOUN
flr-250	18	26	,	,	PUNCT
flr-250	18	27	receiver	receiver	NOUN
flr-250	18	28	operating	operate	VERB
flr-250	18	29	characteristic	characteristic	NOUN
flr-250	18	30	(	(	PUNCT
flr-250	18	31	roc	roc	PROPN
flr-250	18	32	)	)	PUNCT
flr-250	18	33	analysis	analysis	NOUN
flr-250	18	34	is	be	AUX
flr-250	18	35	a	a	DET
flr-250	18	36	very	very	ADV
flr-250	18	37	valuable	valuable	ADJ
flr-250	18	38	tool	tool	NOUN
flr-250	18	39	.	.	PUNCT
flr-250	19	1	it	it	PRON
flr-250	19	2	is	be	AUX
flr-250	19	3	important	important	ADJ
flr-250	19	4	throughout	throughout	ADP
flr-250	19	5	this	this	DET
flr-250	19	6	review	review	NOUN
flr-250	19	7	to	to	PART
flr-250	19	8	keep	keep	VERB
flr-250	19	9	in	in	ADP
flr-250	19	10	mind	mind	NOUN
flr-250	19	11	that	that	SCONJ
flr-250	19	12	performance	performance	NOUN
flr-250	19	13	is	be	AUX
flr-250	19	14	not	not	PART
flr-250	19	15	a	a	DET
flr-250	19	16	constant	constant	ADJ
flr-250	19	17	.	.	PUNCT
flr-250	20	1	one	one	NUM
flr-250	20	2	’s	’s	PART
flr-250	20	3	performance	performance	NOUN
flr-250	20	4	on	on	ADP
flr-250	20	5	any	any	DET
flr-250	20	6	given	give	VERB
flr-250	20	7	task	task	NOUN
flr-250	20	8	will	will	AUX
flr-250	20	9	vary	vary	VERB
flr-250	20	10	as	as	ADP
flr-250	20	11	a	a	DET
flr-250	20	12	function	function	NOUN
flr-250	20	13	of	of	ADP
flr-250	20	14	a	a	DET
flr-250	20	15	number	number	NOUN
flr-250	20	16	of	of	ADP
flr-250	20	17	factors	factor	NOUN
flr-250	20	18	and	and	CCONJ
flr-250	20	19	thus	thus	ADV
flr-250	20	20	the	the	DET
flr-250	20	21	metrics	metric	NOUN
flr-250	20	22	and	and	CCONJ
flr-250	20	23	principles	principle	NOUN
flr-250	20	24	discussed	discuss	VERB
flr-250	20	25	will	will	AUX
flr-250	20	26	reflect	reflect	VERB
flr-250	20	27	those	those	DET
flr-250	20	28	changes	change	NOUN
flr-250	20	29	and	and	CCONJ
flr-250	20	30	differences	difference	NOUN
flr-250	20	31	.	.	PUNCT
flr-250	21	1	for	for	ADP
flr-250	21	2	example	example	NOUN
flr-250	21	3	,	,	PUNCT
flr-250	21	4	when	when	SCONJ
flr-250	21	5	someone	someone	PRON
flr-250	21	6	first	first	ADV
flr-250	21	7	learns	learn	VERB
flr-250	21	8	a	a	DET
flr-250	21	9	decision	decision	NOUN
flr-250	21	10	task	task	NOUN
flr-250	21	11	,	,	PUNCT
flr-250	21	12	their	their	PRON
flr-250	21	13	criteria	criterion	NOUN
flr-250	21	14	for	for	ADP
flr-250	21	15	rendering	render	VERB
flr-250	21	16	a	a	DET
flr-250	21	17	decision	decision	NOUN
flr-250	21	18	may	may	AUX
flr-250	21	19	be	be	AUX
flr-250	21	20	based	base	VERB
flr-250	21	21	more	more	ADV
flr-250	21	22	on	on	ADP
flr-250	21	23	didactic	didactic	ADJ
flr-250	21	24	learning	learning	NOUN
flr-250	21	25	that	that	SCONJ
flr-250	21	26	they	they	PRON
flr-250	21	27	have	have	AUX
flr-250	21	28	engaged	engage	VERB
flr-250	21	29	in	in	ADP
flr-250	21	30	and	and	CCONJ
flr-250	21	31	the	the	DET
flr-250	21	32	exact	exact	ADJ
flr-250	21	33	examples	example	NOUN
flr-250	21	34	of	of	ADP
flr-250	21	35	the	the	DET
flr-250	21	36	task	task	NOUN
flr-250	21	37	they	they	PRON
flr-250	21	38	have	have	AUX
flr-250	21	39	encountered	encounter	VERB
flr-250	21	40	to	to	ADP
flr-250	21	41	date	date	NOUN
flr-250	21	42	.	.	PUNCT
flr-250	22	1	as	as	SCONJ
flr-250	22	2	expertise	expertise	NOUN
flr-250	22	3	grows	grow	VERB
flr-250	22	4	and	and	CCONJ
flr-250	22	5	they	they	PRON
flr-250	22	6	encounter	encounter	VERB
flr-250	22	7	more	more	ADJ
flr-250	22	8	and	and	CCONJ
flr-250	22	9	varied	varied	ADJ
flr-250	22	10	examples	example	NOUN
flr-250	22	11	,	,	PUNCT
flr-250	22	12	the	the	DET
flr-250	22	13	criteria	criterion	NOUN
flr-250	22	14	they	they	PRON
flr-250	22	15	use	use	VERB
flr-250	22	16	in	in	ADP
flr-250	22	17	their	their	PRON
flr-250	22	18	decisions	decision	NOUN
flr-250	22	19	is	be	AUX
flr-250	22	20	likely	likely	ADJ
flr-250	22	21	to	to	PART
flr-250	22	22	change	change	VERB
flr-250	22	23	as	as	SCONJ
flr-250	22	24	their	their	PRON
flr-250	22	25	knowledge	knowledge	NOUN
flr-250	22	26	and	and	CCONJ
flr-250	22	27	skill	skill	NOUN
flr-250	22	28	develops	develop	VERB
flr-250	22	29	.	.	PUNCT
flr-250	23	1	even	even	ADV
flr-250	23	2	when	when	SCONJ
flr-250	23	3	one	one	PRON
flr-250	23	4	is	be	AUX
flr-250	23	5	an	an	DET
flr-250	23	6	expert	expert	NOUN
flr-250	23	7	,	,	PUNCT
flr-250	23	8	changes	change	NOUN
flr-250	23	9	in	in	ADP
flr-250	23	10	the	the	DET
flr-250	23	11	environment	environment	NOUN
flr-250	23	12	,	,	PUNCT
flr-250	23	13	the	the	DET
flr-250	23	14	stimuli	stimulus	NOUN
flr-250	23	15	and	and	CCONJ
flr-250	23	16	the	the	DET
flr-250	23	17	consequences	consequence	NOUN
flr-250	23	18	of	of	ADP
flr-250	23	19	the	the	DET
flr-250	23	20	decisions	decision	NOUN
flr-250	23	21	rendered	render	VERB
flr-250	23	22	may	may	AUX
flr-250	23	23	change	change	VERB
flr-250	23	24	,	,	PUNCT
flr-250	23	25	making	make	VERB
flr-250	23	26	it	it	PRON
flr-250	23	27	necessary	necessary	ADJ
flr-250	23	28	to	to	PART
flr-250	23	29	adjust	adjust	VERB
flr-250	23	30	one	one	NUM
flr-250	23	31	’s	’s	PART
flr-250	23	32	criteria	criterion	NOUN
flr-250	23	33	to	to	ADP
flr-250	23	34	the	the	DET
flr-250	23	35	new	new	ADJ
flr-250	23	36	situation	situation	NOUN
flr-250	23	37	.	.	PUNCT
flr-250	24	1	for	for	ADP
flr-250	24	2	example	example	NOUN
flr-250	24	3	,	,	PUNCT
flr-250	24	4	on	on	ADP
flr-250	24	5	a	a	DET
flr-250	24	6	chest	chest	NOUN
flr-250	24	7	x	x	NOUN
flr-250	24	8	-	-	NOUN
flr-250	24	9	ray	ray	NOUN
flr-250	24	10	image	image	NOUN
flr-250	24	11	cancer	cancer	NOUN
flr-250	24	12	looks	look	VERB
flr-250	24	13	like	like	ADP
flr-250	24	14	a	a	DET
flr-250	24	15	white	white	ADJ
flr-250	24	16	spot(s	spot(s	NOUN
flr-250	24	17	)	)	PUNCT
flr-250	24	18	on	on	ADP
flr-250	24	19	the	the	DET
flr-250	24	20	lung	lung	NOUN
flr-250	24	21	and	and	CCONJ
flr-250	24	22	even	even	ADV
flr-250	24	23	residents	resident	NOUN
flr-250	24	24	in	in	ADP
flr-250	24	25	training	training	NOUN
flr-250	24	26	quickly	quickly	ADV
flr-250	24	27	learn	learn	VERB
flr-250	24	28	to	to	PART
flr-250	24	29	detect	detect	VERB
flr-250	24	30	and	and	CCONJ
flr-250	24	31	diagnose	diagnose	NOUN
flr-250	24	32	lung	lung	NOUN
flr-250	24	33	cancer	cancer	NOUN
flr-250	24	34	.	.	PUNCT
flr-250	25	1	however	however	ADV
flr-250	25	2	,	,	PUNCT
flr-250	25	3	in	in	ADP
flr-250	25	4	the	the	DET
flr-250	25	5	southwest	southwest	PROPN
flr-250	25	6	united	united	PROPN
flr-250	25	7	states	states	PROPN
flr-250	25	8	and	and	CCONJ
flr-250	25	9	other	other	ADJ
flr-250	25	10	desert	desert	NOUN
flr-250	25	11	regions	region	NOUN
flr-250	25	12	there	there	PRON
flr-250	25	13	is	be	VERB
flr-250	25	14	a	a	DET
flr-250	25	15	condition	condition	NOUN
flr-250	25	16	known	know	VERB
flr-250	25	17	as	as	ADP
flr-250	25	18	valley	valley	NOUN
flr-250	25	19	fever	fever	NOUN
flr-250	25	20	(	(	PUNCT
flr-250	25	21	in	in	ADP
flr-250	25	22	infection	infection	NOUN
flr-250	25	23	caused	cause	VERB
flr-250	25	24	by	by	ADP
flr-250	25	25	the	the	DET
flr-250	25	26	fungus	fungus	NOUN
flr-250	25	27	coccidioidomycosis	coccidioidomycosis	NOUN
flr-250	25	28	)	)	PUNCT
flr-250	25	29	that	that	PRON
flr-250	25	30	appears	appear	VERB
flr-250	25	31	in	in	ADP
flr-250	25	32	the	the	DET
flr-250	25	33	lungs	lung	NOUN
flr-250	25	34	as	as	ADP
flr-250	25	35	a	a	DET
flr-250	25	36	white	white	ADJ
flr-250	25	37	spot(s	spot(s	NOUN
flr-250	25	38	)	)	PUNCT
flr-250	25	39	.	.	PUNCT
flr-250	26	1	radiologists	radiologist	NOUN
flr-250	26	2	who	who	PRON
flr-250	26	3	move	move	VERB
flr-250	26	4	to	to	ADP
flr-250	26	5	places	place	NOUN
flr-250	26	6	like	like	ADP
flr-250	26	7	arizona	arizona	PROPN
flr-250	26	8	where	where	SCONJ
flr-250	26	9	valley	valley	NOUN
flr-250	26	10	fever	fever	NOUN
flr-250	26	11	is	be	AUX
flr-250	26	12	quite	quite	ADV
flr-250	26	13	common	common	ADJ
flr-250	26	14	go	go	VERB
flr-250	26	15	through	through	ADP
flr-250	26	16	a	a	DET
flr-250	26	17	period	period	NOUN
flr-250	26	18	of	of	ADP
flr-250	26	19	criteria	criterion	NOUN
flr-250	26	20	adjustment	adjustment	NOUN
flr-250	26	21	–	–	PUNCT
flr-250	26	22	initially	initially	ADV
flr-250	26	23	calling	call	VERB
flr-250	26	24	nearly	nearly	ADV
flr-250	26	25	everything	everything	PRON
flr-250	26	26	cancer	cancer	NOUN
flr-250	26	27	(	(	PUNCT
flr-250	26	28	high	high	ADJ
flr-250	26	29	false	false	ADJ
flr-250	26	30	positive	positive	ADJ
flr-250	26	31	rate	rate	NOUN
flr-250	26	32	)	)	PUNCT
flr-250	26	33	but	but	CCONJ
flr-250	26	34	soon	soon	ADV
flr-250	26	35	learning	learn	VERB
flr-250	26	36	to	to	PART
flr-250	26	37	distinguish	distinguish	VERB
flr-250	26	38	valley	valley	NOUN
flr-250	26	39	fever	fever	NOUN
flr-250	26	40	from	from	ADP
flr-250	26	41	lung	lung	NOUN
flr-250	26	42	cancer	cancer	NOUN
flr-250	26	43	as	as	SCONJ
flr-250	26	44	they	they	PRON
flr-250	26	45	see	see	VERB
flr-250	26	46	more	more	ADJ
flr-250	26	47	cases	case	NOUN
flr-250	26	48	and	and	CCONJ
flr-250	26	49	learn	learn	VERB
flr-250	26	50	the	the	DET
flr-250	26	51	discriminating	discriminate	VERB
flr-250	26	52	features	feature	NOUN
flr-250	26	53	.	.	PUNCT
flr-250	27	1	2	2	X
flr-250	27	2	.	.	X
flr-250	27	3	basics	basic	NOUN
flr-250	27	4	of	of	ADP
flr-250	27	5	decision	decision	NOUN
flr-250	27	6	making	make	VERB
flr-250	27	7	for	for	ADP
flr-250	27	8	roc	roc	PROPN
flr-250	27	9	roc	roc	PROPN
flr-250	27	10	analysis	analysis	NOUN
flr-250	27	11	was	be	AUX
flr-250	27	12	developed	develop	VERB
flr-250	27	13	in	in	ADP
flr-250	27	14	the	the	DET
flr-250	27	15	early	early	ADJ
flr-250	27	16	1950s	1950	NOUN
flr-250	27	17	based	base	VERB
flr-250	27	18	on	on	ADP
flr-250	27	19	principles	principle	NOUN
flr-250	27	20	from	from	ADP
flr-250	27	21	signal	signal	NOUN
flr-250	27	22	-	-	PUNCT
flr-250	27	23	detection	detection	NOUN
flr-250	27	24	theory	theory	NOUN
flr-250	27	25	for	for	ADP
flr-250	27	26	evaluation	evaluation	NOUN
flr-250	27	27	of	of	ADP
flr-250	27	28	radar	radar	NOUN
flr-250	27	29	operators	operator	NOUN
flr-250	27	30	in	in	ADP
flr-250	27	31	the	the	DET
flr-250	27	32	detection	detection	NOUN
flr-250	27	33	of	of	ADP
flr-250	27	34	enemy	enemy	NOUN
flr-250	27	35	aircraft	aircraft	NOUN
flr-250	27	36	and	and	CCONJ
flr-250	27	37	missiles	missile	NOUN
flr-250	27	38	[	[	X
flr-250	27	39	3	3	NUM
flr-250	27	40	-	-	SYM
flr-250	27	41	4	4	NUM
flr-250	27	42	]	]	PUNCT
flr-250	27	43	,	,	PUNCT
flr-250	27	44	and	and	CCONJ
flr-250	27	45	additional	additional	ADJ
flr-250	27	46	contributions	contribution	NOUN
flr-250	27	47	were	be	AUX
flr-250	27	48	thereafter	thereafter	ADV
flr-250	27	49	made	make	VERB
flr-250	27	50	by	by	ADP
flr-250	27	51	researchers	researcher	NOUN
flr-250	27	52	in	in	ADP
flr-250	27	53	engineering	engineering	NOUN
flr-250	27	54	,	,	PUNCT
flr-250	27	55	psychology	psychology	NOUN
flr-250	27	56	,	,	PUNCT
flr-250	27	57	and	and	CCONJ
flr-250	27	58	mathematics	mathematic	NOUN
flr-250	28	1	[	[	X
flr-250	28	2	5	5	NUM
flr-250	28	3	-	-	SYM
flr-250	28	4	7	7	NUM
flr-250	28	5	]	]	PUNCT
flr-250	28	6	.	.	PUNCT
flr-250	29	1	roc	roc	PROPN
flr-250	29	2	was	be	AUX
flr-250	29	3	introduced	introduce	VERB
flr-250	29	4	into	into	ADP
flr-250	29	5	medicine	medicine	NOUN
flr-250	29	6	in	in	ADP
flr-250	29	7	the	the	DET
flr-250	29	8	1960s	1960s	NUM
flr-250	29	9	by	by	SCONJ
flr-250	29	10	lee	lee	PROPN
flr-250	29	11	lusted	lust	VERB
flr-250	29	12	[	[	PUNCT
flr-250	29	13	8	8	NUM
flr-250	29	14	-	-	SYM
flr-250	29	15	11	11	NUM
flr-250	29	16	]	]	PUNCT
flr-250	29	17	,	,	PUNCT
flr-250	29	18	with	with	ADP
flr-250	29	19	significant	significant	ADJ
flr-250	29	20	efforts	effort	NOUN
flr-250	29	21	devoted	devote	VERB
flr-250	29	22	to	to	ADP
flr-250	29	23	gaining	gain	VERB
flr-250	29	24	a	a	DET
flr-250	29	25	better	well	ADJ
flr-250	29	26	understanding	understanding	NOUN
flr-250	29	27	of	of	ADP
flr-250	29	28	decision	decision	NOUN
flr-250	29	29	-	-	PUNCT
flr-250	29	30	making	making	NOUN
flr-250	29	31	[	[	X
flr-250	29	32	12	12	NUM
flr-250	29	33	-	-	SYM
flr-250	29	34	15	15	NUM
flr-250	29	35	]	]	PUNCT
flr-250	29	36	.	.	PUNCT
flr-250	30	1	this	this	DET
flr-250	30	2	entrée	entrée	NOUN
flr-250	30	3	into	into	ADP
flr-250	30	4	medicine	medicine	NOUN
flr-250	30	5	was	be	AUX
flr-250	30	6	the	the	DET
flr-250	30	7	result	result	NOUN
flr-250	30	8	of	of	ADP
flr-250	30	9	a	a	DET
flr-250	30	10	series	series	NOUN
flr-250	30	11	of	of	ADP
flr-250	30	12	studies	study	NOUN
flr-250	30	13	in	in	ADP
flr-250	30	14	radiology	radiology	NOUN
flr-250	30	15	that	that	PRON
flr-250	30	16	began	begin	VERB
flr-250	30	17	soon	soon	ADV
flr-250	30	18	after	after	SCONJ
flr-250	30	19	world	world	PROPN
flr-250	30	20	war	war	PROPN
flr-250	30	21	ii	ii	PROPN
flr-250	30	22	to	to	PART
flr-250	30	23	determine	determine	VERB
flr-250	30	24	which	which	PRON
flr-250	30	25	of	of	ADP
flr-250	30	26	four	four	NUM
flr-250	30	27	radiographic	radiographic	ADJ
flr-250	30	28	and	and	CCONJ
flr-250	30	29	fluoroscopic	fluoroscopic	NOUN
flr-250	30	30	techniques	technique	NOUN
flr-250	30	31	(	(	PUNCT
flr-250	30	32	e.g.	e.g.	ADV
flr-250	30	33	,	,	PUNCT
flr-250	30	34	radiographic	radiographic	ADJ
flr-250	30	35	film	film	NOUN
flr-250	30	36	vs	vs	ADP
flr-250	30	37	slides	slide	NOUN
flr-250	30	38	)	)	PUNCT
flr-250	30	39	was	be	AUX
flr-250	30	40	better	well	ADJ
flr-250	30	41	for	for	ADP
flr-250	30	42	tuberculosis	tuberculosis	NOUN
flr-250	30	43	(	(	PUNCT
flr-250	30	44	tb	tb	NOUN
flr-250	30	45	)	)	PUNCT
flr-250	30	46	screening	screen	VERB
flr-250	31	1	[	[	X
flr-250	31	2	16	16	NUM
flr-250	31	3	-	-	SYM
flr-250	31	4	17	17	NUM
flr-250	31	5	]	]	PUNCT
flr-250	31	6	.	.	PUNCT
flr-250	32	1	the	the	DET
flr-250	32	2	goal	goal	NOUN
flr-250	32	3	was	be	AUX
flr-250	32	4	to	to	PART
flr-250	32	5	find	find	VERB
flr-250	32	6	a	a	DET
flr-250	32	7	single	single	ADJ
flr-250	32	8	imaging	imaging	NOUN
flr-250	32	9	technique	technique	NOUN
flr-250	32	10	that	that	PRON
flr-250	32	11	would	would	AUX
flr-250	32	12	outperform	outperform	VERB
flr-250	32	13	all	all	DET
flr-250	32	14	the	the	DET
flr-250	32	15	others	other	NOUN
flr-250	32	16	(	(	PUNCT
flr-250	32	17	in	in	ADP
flr-250	32	18	other	other	ADJ
flr-250	32	19	words	word	NOUN
flr-250	32	20	allow	allow	VERB
flr-250	32	21	krupinski	krupinski	NOUN
flr-250	33	1	|	|	ADV
flr-250	33	2	f	f	NOUN
flr-250	33	3	l	l	NOUN
flr-250	33	4	r	r	NOUN
flr-250	33	5	33	33	NUM
flr-250	33	6	the	the	DET
flr-250	33	7	radiologists	radiologist	NOUN
flr-250	33	8	to	to	PART
flr-250	33	9	reach	reach	VERB
flr-250	33	10	the	the	DET
flr-250	33	11	correct	correct	ADJ
flr-250	33	12	decision	decision	NOUN
flr-250	33	13	)	)	PUNCT
flr-250	33	14	.	.	PUNCT
flr-250	34	1	instead	instead	ADV
flr-250	34	2	what	what	PRON
flr-250	34	3	they	they	PRON
flr-250	34	4	found	find	VERB
flr-250	34	5	was	be	AUX
flr-250	34	6	that	that	SCONJ
flr-250	34	7	the	the	DET
flr-250	34	8	intra	intra	ADJ
flr-250	34	9	-	-	ADJ
flr-250	34	10	observer	observer	NOUN
flr-250	34	11	and	and	CCONJ
flr-250	34	12	interobserver	interobserver	ADJ
flr-250	34	13	variation	variation	NOUN
flr-250	34	14	was	be	AUX
flr-250	34	15	unexpectedly	unexpectedly	ADV
flr-250	34	16	so	so	ADV
flr-250	34	17	high	high	ADJ
flr-250	34	18	that	that	SCONJ
flr-250	34	19	it	it	PRON
flr-250	34	20	was	be	AUX
flr-250	34	21	impossible	impossible	ADJ
flr-250	34	22	to	to	PART
flr-250	34	23	determine	determine	VERB
flr-250	34	24	which	which	DET
flr-250	34	25	one	one	PRON
flr-250	34	26	was	be	AUX
flr-250	34	27	best	good	ADJ
flr-250	34	28	.	.	PUNCT
flr-250	35	1	this	this	PRON
flr-250	35	2	was	be	AUX
flr-250	35	3	unexpected	unexpected	ADJ
flr-250	35	4	as	as	SCONJ
flr-250	35	5	until	until	ADP
flr-250	35	6	then	then	ADV
flr-250	35	7	it	it	PRON
flr-250	35	8	was	be	AUX
flr-250	35	9	presumed	presume	VERB
flr-250	35	10	that	that	SCONJ
flr-250	35	11	given	give	VERB
flr-250	35	12	the	the	DET
flr-250	35	13	same	same	ADJ
flr-250	35	14	image	image	NOUN
flr-250	35	15	data	datum	NOUN
flr-250	35	16	all	all	DET
flr-250	35	17	radiologists	radiologist	NOUN
flr-250	35	18	looking	look	VERB
flr-250	35	19	at	at	ADP
flr-250	35	20	the	the	DET
flr-250	35	21	images	image	NOUN
flr-250	35	22	would	would	AUX
flr-250	35	23	be	be	AUX
flr-250	35	24	seeing	see	VERB
flr-250	35	25	the	the	DET
flr-250	35	26	same	same	ADJ
flr-250	35	27	diagnostic	diagnostic	ADJ
flr-250	35	28	features	feature	NOUN
flr-250	35	29	and	and	CCONJ
flr-250	35	30	information	information	NOUN
flr-250	35	31	,	,	PUNCT
flr-250	35	32	detecting	detect	VERB
flr-250	35	33	the	the	DET
flr-250	35	34	tb	tb	NOUN
flr-250	35	35	if	if	SCONJ
flr-250	35	36	it	it	PRON
flr-250	35	37	was	be	AUX
flr-250	35	38	present	present	ADJ
flr-250	35	39	,	,	PUNCT
flr-250	35	40	and	and	CCONJ
flr-250	35	41	rendering	render	VERB
flr-250	35	42	the	the	DET
flr-250	35	43	same	same	ADJ
flr-250	35	44	diagnostic	diagnostic	ADJ
flr-250	35	45	decision	decision	NOUN
flr-250	35	46	.	.	PUNCT
flr-250	36	1	the	the	DET
flr-250	36	2	idea	idea	NOUN
flr-250	36	3	that	that	SCONJ
flr-250	36	4	everyone	everyone	PRON
flr-250	36	5	may	may	AUX
flr-250	36	6	“	"	PUNCT
flr-250	36	7	see	see	VERB
flr-250	36	8	”	"	PUNCT
flr-250	36	9	something	something	PRON
flr-250	36	10	different	different	ADJ
flr-250	36	11	in	in	ADP
flr-250	36	12	the	the	DET
flr-250	36	13	images	image	NOUN
flr-250	36	14	,	,	PUNCT
flr-250	36	15	perhaps	perhaps	ADV
flr-250	36	16	as	as	ADP
flr-250	36	17	a	a	DET
flr-250	36	18	function	function	NOUN
flr-250	36	19	of	of	ADP
flr-250	36	20	experience	experience	NOUN
flr-250	36	21	or	or	CCONJ
flr-250	36	22	expertise	expertise	NOUN
flr-250	36	23	,	,	PUNCT
flr-250	36	24	had	have	AUX
flr-250	36	25	never	never	ADV
flr-250	36	26	been	be	AUX
flr-250	36	27	considered	consider	VERB
flr-250	36	28	.	.	PUNCT
flr-250	37	1	thus	thus	ADV
flr-250	37	2	,	,	PUNCT
flr-250	37	3	it	it	PRON
flr-250	37	4	was	be	AUX
flr-250	37	5	necessary	necessary	ADJ
flr-250	37	6	to	to	PART
flr-250	37	7	build	build	VERB
flr-250	37	8	systems	system	NOUN
flr-250	37	9	that	that	PRON
flr-250	37	10	could	could	AUX
flr-250	37	11	generate	generate	VERB
flr-250	37	12	better	well	ADJ
flr-250	37	13	images	image	NOUN
flr-250	37	14	so	so	SCONJ
flr-250	37	15	radiologists	radiologist	NOUN
flr-250	37	16	’	'	PUNCT
flr-250	37	17	performance	performance	NOUN
flr-250	37	18	could	could	AUX
flr-250	37	19	improve	improve	VERB
flr-250	37	20	(	(	PUNCT
flr-250	37	21	i.e.	i.e.	X
flr-250	37	22	,	,	PUNCT
flr-250	37	23	reduce	reduce	VERB
flr-250	37	24	observer	observer	NOUN
flr-250	37	25	variability	variability	NOUN
flr-250	37	26	)	)	PUNCT
flr-250	37	27	,	,	PUNCT
flr-250	37	28	and	and	CCONJ
flr-250	37	29	develop	develop	VERB
flr-250	37	30	methods	method	NOUN
flr-250	37	31	to	to	PART
flr-250	37	32	evaluate	evaluate	VERB
flr-250	37	33	these	these	DET
flr-250	37	34	new	new	ADJ
flr-250	37	35	systems	system	NOUN
flr-250	37	36	and	and	CCONJ
flr-250	37	37	assess	assess	VERB
flr-250	37	38	their	their	PRON
flr-250	37	39	impact	impact	NOUN
flr-250	37	40	on	on	ADP
flr-250	37	41	observer	observer	NOUN
flr-250	37	42	performance	performance	NOUN
flr-250	37	43	.	.	PUNCT
flr-250	38	1	although	although	SCONJ
flr-250	38	2	there	there	PRON
flr-250	38	3	are	be	VERB
flr-250	38	4	newer	new	ADJ
flr-250	38	5	methods	method	NOUN
flr-250	38	6	that	that	PRON
flr-250	38	7	allow	allow	VERB
flr-250	38	8	for	for	ADP
flr-250	38	9	more	more	ADJ
flr-250	38	10	than	than	ADP
flr-250	38	11	two	two	NUM
flr-250	38	12	decisions	decision	NOUN
flr-250	38	13	in	in	ADP
flr-250	38	14	the	the	DET
flr-250	38	15	roc	roc	PROPN
flr-250	38	16	task	task	PROPN
flr-250	38	17	environment	environment	NOUN
flr-250	38	18	[	[	X
flr-250	38	19	18	18	NUM
flr-250	38	20	-	-	SYM
flr-250	38	21	19	19	NUM
flr-250	38	22	]	]	PUNCT
flr-250	38	23	,	,	PUNCT
flr-250	38	24	roc	roc	PROPN
flr-250	38	25	is	be	AUX
flr-250	38	26	traditionally	traditionally	ADV
flr-250	38	27	a	a	DET
flr-250	38	28	binary	binary	ADJ
flr-250	38	29	decision	decision	NOUN
flr-250	38	30	task	task	NOUN
flr-250	38	31	–	–	PUNCT
flr-250	38	32	target	target	NOUN
flr-250	38	33	/	/	SYM
flr-250	38	34	signal	signal	NOUN
flr-250	38	35	(	(	PUNCT
flr-250	38	36	e.g.	e.g.	ADJ
flr-250	38	37	,	,	PUNCT
flr-250	38	38	lesion	lesion	NOUN
flr-250	38	39	,	,	PUNCT
flr-250	38	40	disease	disease	NOUN
flr-250	38	41	,	,	PUNCT
flr-250	38	42	missile	missile	NOUN
flr-250	38	43	)	)	PUNCT
flr-250	38	44	present	present	ADJ
flr-250	38	45	versus	versus	ADP
flr-250	38	46	target	target	NOUN
flr-250	38	47	/	/	SYM
flr-250	38	48	signal	signal	NOUN
flr-250	38	49	absent	absent	ADJ
flr-250	38	50	,	,	PUNCT
flr-250	38	51	or	or	CCONJ
flr-250	38	52	in	in	ADP
flr-250	38	53	the	the	DET
flr-250	38	54	case	case	NOUN
flr-250	38	55	of	of	ADP
flr-250	38	56	classification	classification	NOUN
flr-250	38	57	rather	rather	ADV
flr-250	38	58	than	than	ADP
flr-250	38	59	detection	detection	VERB
flr-250	38	60	the	the	DET
flr-250	38	61	target	target	NOUN
flr-250	38	62	/	/	SYM
flr-250	38	63	signal	signal	NOUN
flr-250	38	64	belongs	belong	VERB
flr-250	38	65	to	to	ADP
flr-250	38	66	class	class	NOUN
flr-250	38	67	1	1	NUM
flr-250	38	68	(	(	PUNCT
flr-250	38	69	e.g.	e.g.	ADV
flr-250	38	70	,	,	PUNCT
flr-250	38	71	cancer	cancer	NOUN
flr-250	38	72	,	,	PUNCT
flr-250	38	73	enemy	enemy	NOUN
flr-250	38	74	)	)	PUNCT
flr-250	38	75	or	or	CCONJ
flr-250	38	76	class	class	NOUN
flr-250	38	77	2	2	NUM
flr-250	38	78	(	(	PUNCT
flr-250	38	79	e.g.	e.g.	ADV
flr-250	38	80	,	,	PUNCT
flr-250	38	81	not	not	PART
flr-250	38	82	cancer	cancer	NOUN
flr-250	38	83	,	,	PUNCT
flr-250	38	84	friend	friend	NOUN
flr-250	38	85	)	)	PUNCT
flr-250	38	86	.	.	PUNCT
flr-250	39	1	for	for	ADP
flr-250	39	2	roc	roc	PROPN
flr-250	39	3	analysis	analysis	NOUN
flr-250	39	4	,	,	PUNCT
flr-250	39	5	these	these	DET
flr-250	39	6	two	two	NUM
flr-250	39	7	conditions	condition	NOUN
flr-250	39	8	must	must	AUX
flr-250	39	9	be	be	AUX
flr-250	39	10	mutually	mutually	ADV
flr-250	39	11	exclusive	exclusive	ADJ
flr-250	39	12	.	.	PUNCT
flr-250	40	1	there	there	PRON
flr-250	40	2	must	must	AUX
flr-250	40	3	also	also	ADV
flr-250	40	4	be	be	AUX
flr-250	40	5	some	some	DET
flr-250	40	6	sort	sort	NOUN
flr-250	40	7	of	of	ADP
flr-250	40	8	“	"	PUNCT
flr-250	40	9	truth	truth	NOUN
flr-250	40	10	”	"	PUNCT
flr-250	40	11	or	or	CCONJ
flr-250	40	12	gold	gold	NOUN
flr-250	40	13	standard	standard	NOUN
flr-250	40	14	for	for	ADP
flr-250	40	15	each	each	DET
flr-250	40	16	option	option	NOUN
flr-250	40	17	.	.	PUNCT
flr-250	41	1	in	in	ADP
flr-250	41	2	radiology	radiology	NOUN
flr-250	41	3	for	for	ADP
flr-250	41	4	example	example	NOUN
flr-250	41	5	,	,	PUNCT
flr-250	41	6	pathology	pathology	NOUN
flr-250	41	7	is	be	AUX
flr-250	41	8	often	often	ADV
flr-250	41	9	used	use	VERB
flr-250	41	10	as	as	ADP
flr-250	41	11	the	the	DET
flr-250	41	12	gold	gold	NOUN
flr-250	41	13	standard	standard	NOUN
flr-250	41	14	.	.	PUNCT
flr-250	42	1	in	in	ADP
flr-250	42	2	cases	case	NOUN
flr-250	42	3	where	where	SCONJ
flr-250	42	4	there	there	PRON
flr-250	42	5	is	be	VERB
flr-250	42	6	no	no	DET
flr-250	42	7	other	other	ADJ
flr-250	42	8	definitive	definitive	ADJ
flr-250	42	9	test	test	NOUN
flr-250	42	10	or	or	CCONJ
flr-250	42	11	method	method	NOUN
flr-250	42	12	for	for	ADP
flr-250	42	13	determining	determine	VERB
flr-250	42	14	the	the	DET
flr-250	42	15	truth	truth	NOUN
flr-250	42	16	,	,	PUNCT
flr-250	42	17	panels	panel	NOUN
flr-250	42	18	of	of	ADP
flr-250	42	19	experts	expert	NOUN
flr-250	42	20	are	be	AUX
flr-250	42	21	often	often	ADV
flr-250	42	22	used	use	VERB
flr-250	42	23	to	to	PART
flr-250	42	24	establish	establish	VERB
flr-250	42	25	the	the	DET
flr-250	42	26	gold	gold	ADJ
flr-250	42	27	standard	standard	NOUN
flr-250	43	1	[	[	X
flr-250	43	2	20	20	NUM
flr-250	43	3	-	-	SYM
flr-250	43	4	21	21	NUM
flr-250	43	5	]	]	PUNCT
flr-250	43	6	.	.	PUNCT
flr-250	44	1	given	give	VERB
flr-250	44	2	the	the	DET
flr-250	44	3	truth	truth	NOUN
flr-250	44	4	and	and	CCONJ
flr-250	44	5	the	the	DET
flr-250	44	6	decisions	decision	NOUN
flr-250	44	7	of	of	ADP
flr-250	44	8	the	the	DET
flr-250	44	9	observers	observer	NOUN
flr-250	44	10	in	in	ADP
flr-250	44	11	the	the	DET
flr-250	44	12	study	study	NOUN
flr-250	44	13	,	,	PUNCT
flr-250	44	14	a	a	DET
flr-250	44	15	2x2	2x2	NUM
flr-250	44	16	table	table	NOUN
flr-250	44	17	readily	readily	ADV
flr-250	44	18	summarizes	summarize	VERB
flr-250	44	19	all	all	DET
flr-250	44	20	four	four	NUM
flr-250	44	21	possible	possible	ADJ
flr-250	44	22	decisions	decision	NOUN
flr-250	44	23	:	:	PUNCT
flr-250	44	24	true	true	ADJ
flr-250	44	25	positive	positive	ADJ
flr-250	44	26	(	(	PUNCT
flr-250	44	27	tp	tp	NOUN
flr-250	44	28	)	)	PUNCT
flr-250	44	29	(	(	PUNCT
flr-250	44	30	target	target	NOUN
flr-250	44	31	present	present	ADJ
flr-250	44	32	,	,	PUNCT
flr-250	44	33	observer	observer	NOUN
flr-250	44	34	reports	report	VERB
flr-250	44	35	as	as	ADP
flr-250	44	36	present	present	ADJ
flr-250	44	37	)	)	PUNCT
flr-250	44	38	,	,	PUNCT
flr-250	44	39	false	false	ADJ
flr-250	44	40	negative	negative	ADJ
flr-250	44	41	(	(	PUNCT
flr-250	44	42	fn	fn	NOUN
flr-250	44	43	)	)	PUNCT
flr-250	44	44	(	(	PUNCT
flr-250	44	45	target	target	NOUN
flr-250	44	46	present	present	ADJ
flr-250	44	47	,	,	PUNCT
flr-250	44	48	observer	observer	NOUN
flr-250	44	49	reports	report	VERB
flr-250	44	50	as	as	ADP
flr-250	44	51	absent	absent	ADJ
flr-250	44	52	)	)	PUNCT
flr-250	44	53	,	,	PUNCT
flr-250	44	54	false	false	ADJ
flr-250	44	55	positive	positive	ADJ
flr-250	44	56	(	(	PUNCT
flr-250	44	57	fp	fp	NOUN
flr-250	44	58	)	)	PUNCT
flr-250	44	59	(	(	PUNCT
flr-250	44	60	target	target	NOUN
flr-250	44	61	absent	absent	ADJ
flr-250	44	62	,	,	PUNCT
flr-250	44	63	observer	observer	NOUN
flr-250	44	64	reports	report	NOUN
flr-250	44	65	as	as	ADP
flr-250	44	66	present	present	ADJ
flr-250	44	67	)	)	PUNCT
flr-250	44	68	,	,	PUNCT
flr-250	44	69	and	and	CCONJ
flr-250	44	70	true	true	ADJ
flr-250	44	71	negative	negative	ADJ
flr-250	44	72	(	(	PUNCT
flr-250	44	73	tn	tn	NOUN
flr-250	44	74	)	)	PUNCT
flr-250	44	75	(	(	PUNCT
flr-250	44	76	target	target	NOUN
flr-250	44	77	absent	absent	ADJ
flr-250	44	78	,	,	PUNCT
flr-250	44	79	observer	observer	NOUN
flr-250	44	80	reports	report	NOUN
flr-250	44	81	as	as	ADP
flr-250	44	82	absent	absent	ADJ
flr-250	44	83	)	)	PUNCT
flr-250	44	84	.	.	PUNCT
flr-250	45	1	the	the	DET
flr-250	45	2	tp	tp	PROPN
flr-250	45	3	and	and	CCONJ
flr-250	45	4	tn	tn	NOUN
flr-250	45	5	decisions	decision	NOUN
flr-250	45	6	are	be	AUX
flr-250	45	7	correct	correct	ADJ
flr-250	45	8	while	while	SCONJ
flr-250	45	9	the	the	DET
flr-250	45	10	fn	fn	NOUN
flr-250	45	11	and	and	CCONJ
flr-250	45	12	fp	fp	PROPN
flr-250	45	13	decisions	decision	NOUN
flr-250	45	14	are	be	AUX
flr-250	45	15	incorrect	incorrect	ADJ
flr-250	45	16	.	.	PUNCT
flr-250	46	1	2.1	2.1	NUM
flr-250	46	2	common	common	ADJ
flr-250	46	3	performance	performance	NOUN
flr-250	46	4	metrics	metric	NOUN
flr-250	46	5	suppose	suppose	VERB
flr-250	46	6	you	you	PRON
flr-250	46	7	have	have	VERB
flr-250	46	8	an	an	DET
flr-250	46	9	observer	observer	NOUN
flr-250	46	10	who	who	PRON
flr-250	46	11	is	be	AUX
flr-250	46	12	an	an	DET
flr-250	46	13	expert	expert	NOUN
flr-250	46	14	skilled	skilled	ADJ
flr-250	46	15	at	at	ADP
flr-250	46	16	visually	visually	ADV
flr-250	46	17	detecting	detect	VERB
flr-250	46	18	a	a	DET
flr-250	46	19	specific	specific	ADJ
flr-250	46	20	poisonous	poisonous	ADJ
flr-250	46	21	frog	frog	NOUN
flr-250	46	22	in	in	ADP
flr-250	46	23	the	the	DET
flr-250	46	24	jungle	jungle	NOUN
flr-250	46	25	versus	versus	ADP
flr-250	46	26	a	a	DET
flr-250	46	27	similar	similar	ADJ
flr-250	46	28	but	but	CCONJ
flr-250	46	29	non	non	ADJ
flr-250	46	30	-	-	ADJ
flr-250	46	31	poisonous	poisonous	ADJ
flr-250	46	32	frog	frog	NOUN
flr-250	46	33	.	.	PUNCT
flr-250	47	1	knowing	know	VERB
flr-250	47	2	that	that	SCONJ
flr-250	47	3	she	she	PRON
flr-250	47	4	can	can	AUX
flr-250	47	5	make	make	VERB
flr-250	47	6	the	the	DET
flr-250	47	7	correct	correct	ADJ
flr-250	47	8	decision	decision	NOUN
flr-250	47	9	is	be	AUX
flr-250	47	10	important	important	ADJ
flr-250	47	11	because	because	SCONJ
flr-250	47	12	she	she	PRON
flr-250	47	13	is	be	AUX
flr-250	47	14	your	your	PRON
flr-250	47	15	guide	guide	NOUN
flr-250	47	16	on	on	ADP
flr-250	47	17	an	an	DET
flr-250	47	18	expensive	expensive	ADJ
flr-250	47	19	eco	eco	NOUN
flr-250	47	20	-	-	PUNCT
flr-250	47	21	jungle	jungle	NOUN
flr-250	47	22	tour	tour	NOUN
flr-250	47	23	and	and	CCONJ
flr-250	47	24	if	if	SCONJ
flr-250	47	25	she	she	PRON
flr-250	47	26	says	say	VERB
flr-250	47	27	a	a	DET
flr-250	47	28	given	give	VERB
flr-250	47	29	cute	cute	ADJ
flr-250	47	30	little	little	ADJ
flr-250	47	31	frog	frog	NOUN
flr-250	47	32	is	be	AUX
flr-250	47	33	not	not	PART
flr-250	47	34	poisonous	poisonous	ADJ
flr-250	47	35	but	but	CCONJ
flr-250	47	36	in	in	ADP
flr-250	47	37	reality	reality	NOUN
flr-250	47	38	is	be	AUX
flr-250	47	39	,	,	PUNCT
flr-250	47	40	you	you	PRON
flr-250	47	41	might	might	AUX
flr-250	47	42	reach	reach	VERB
flr-250	47	43	out	out	ADP
flr-250	47	44	to	to	PART
flr-250	47	45	touch	touch	VERB
flr-250	47	46	it	it	PRON
flr-250	47	47	with	with	ADP
flr-250	47	48	potentially	potentially	ADV
flr-250	47	49	fatal	fatal	ADJ
flr-250	47	50	consequences	consequence	NOUN
flr-250	47	51	.	.	PUNCT
flr-250	48	1	in	in	ADP
flr-250	48	2	radiology	radiology	NOUN
flr-250	48	3	,	,	PUNCT
flr-250	48	4	one	one	NUM
flr-250	48	5	of	of	ADP
flr-250	48	6	the	the	DET
flr-250	48	7	common	common	ADJ
flr-250	48	8	sources	source	NOUN
flr-250	48	9	of	of	ADP
flr-250	48	10	litigation	litigation	NOUN
flr-250	48	11	is	be	AUX
flr-250	48	12	mammography	mammography	NOUN
flr-250	48	13	–	–	PUNCT
flr-250	48	14	mammographers	mammographer	NOUN
flr-250	48	15	either	either	CCONJ
flr-250	48	16	missing	miss	VERB
flr-250	48	17	potential	potential	ADJ
flr-250	48	18	breast	breast	NOUN
flr-250	48	19	cancers	cancer	NOUN
flr-250	48	20	or	or	CCONJ
flr-250	48	21	overcalling	overcalle	VERB
flr-250	48	22	benign	benign	ADJ
flr-250	48	23	conditions	condition	NOUN
flr-250	48	24	as	as	ADP
flr-250	48	25	malignant	malignant	ADJ
flr-250	48	26	causing	cause	VERB
flr-250	48	27	undo	undo	ADJ
flr-250	48	28	stress	stress	NOUN
flr-250	48	29	and	and	CCONJ
flr-250	48	30	anxiety	anxiety	NOUN
flr-250	48	31	.	.	PUNCT
flr-250	49	1	real	real	ADJ
flr-250	49	2	life	life	NOUN
flr-250	49	3	examples	example	NOUN
flr-250	49	4	of	of	ADP
flr-250	49	5	important	important	ADJ
flr-250	49	6	decision	decision	NOUN
flr-250	49	7	tasks	task	NOUN
flr-250	49	8	abound	abound	VERB
flr-250	49	9	,	,	PUNCT
flr-250	49	10	all	all	PRON
flr-250	49	11	of	of	ADP
flr-250	49	12	which	which	PRON
flr-250	49	13	require	require	VERB
flr-250	49	14	careful	careful	ADJ
flr-250	49	15	assessment	assessment	NOUN
flr-250	49	16	of	of	ADP
flr-250	49	17	correct	correct	ADJ
flr-250	49	18	and	and	CCONJ
flr-250	49	19	incorrect	incorrect	ADJ
flr-250	49	20	decisions	decision	NOUN
flr-250	49	21	.	.	PUNCT
flr-250	50	1	from	from	ADP
flr-250	50	2	the	the	DET
flr-250	50	3	basic	basic	ADJ
flr-250	50	4	2x2	2x2	NUM
flr-250	50	5	matrix	matrix	NOUN
flr-250	50	6	of	of	ADP
flr-250	50	7	4	4	NUM
flr-250	50	8	decisions	decision	NOUN
flr-250	50	9	described	describe	VERB
flr-250	50	10	above	above	ADV
flr-250	50	11	come	come	VERB
flr-250	50	12	some	some	DET
flr-250	50	13	key	key	ADJ
flr-250	50	14	metrics	metric	NOUN
flr-250	50	15	often	often	ADV
flr-250	50	16	used	use	VERB
flr-250	50	17	to	to	PART
flr-250	50	18	assess	assess	VERB
flr-250	50	19	performance	performance	NOUN
flr-250	50	20	in	in	ADP
flr-250	50	21	visual	visual	ADJ
flr-250	50	22	expertise	expertise	NOUN
flr-250	50	23	and	and	CCONJ
flr-250	50	24	other	other	ADJ
flr-250	50	25	observer	observer	NOUN
flr-250	50	26	performance	performance	NOUN
flr-250	50	27	studies	study	NOUN
flr-250	50	28	.	.	PUNCT
flr-250	51	1	the	the	DET
flr-250	51	2	two	two	NUM
flr-250	51	3	most	most	ADV
flr-250	51	4	commonly	commonly	ADV
flr-250	51	5	used	use	VERB
flr-250	51	6	are	be	AUX
flr-250	51	7	sensitivity	sensitivity	NOUN
flr-250	51	8	and	and	CCONJ
flr-250	51	9	specificity	specificity	NOUN
flr-250	51	10	.	.	PUNCT
flr-250	52	1	sensitivity	sensitivity	NOUN
flr-250	52	2	reflects	reflect	VERB
flr-250	52	3	the	the	DET
flr-250	52	4	ability	ability	NOUN
flr-250	52	5	of	of	ADP
flr-250	52	6	the	the	DET
flr-250	52	7	observer	observer	NOUN
flr-250	52	8	to	to	PART
flr-250	52	9	correctly	correctly	ADV
flr-250	52	10	classify	classify	VERB
flr-250	52	11	the	the	DET
flr-250	52	12	target	target	NOUN
flr-250	52	13	present	present	ADJ
flr-250	52	14	stimuli	stimulus	NOUN
flr-250	52	15	(	(	PUNCT
flr-250	52	16	e.g.	e.g.	ADV
flr-250	52	17	,	,	PUNCT
flr-250	52	18	x	x	NOUN
flr-250	52	19	-	-	NOUN
flr-250	52	20	ray	ray	NOUN
flr-250	52	21	or	or	CCONJ
flr-250	52	22	other	other	ADJ
flr-250	52	23	images	image	NOUN
flr-250	52	24	)	)	PUNCT
flr-250	52	25	and	and	CCONJ
flr-250	52	26	is	be	AUX
flr-250	52	27	calculated	calculate	VERB
flr-250	52	28	as	as	ADP
flr-250	52	29	:	:	PUNCT
flr-250	52	30	sensitivity	sensitivity	NOUN
flr-250	52	31	=	=	SYM
flr-250	52	32	tp/(tp	tp/(tp	PROPN
flr-250	52	33	+	+	CCONJ
flr-250	52	34	fn	fn	NOUN
flr-250	52	35	)	)	PUNCT
flr-250	52	36	(	(	PUNCT
flr-250	52	37	2.1	2.1	NUM
flr-250	52	38	)	)	PUNCT
flr-250	52	39	specificity	specificity	NOUN
flr-250	52	40	reflects	reflect	VERB
flr-250	52	41	the	the	DET
flr-250	52	42	ability	ability	NOUN
flr-250	52	43	of	of	ADP
flr-250	52	44	the	the	DET
flr-250	52	45	observer	observer	NOUN
flr-250	52	46	to	to	PART
flr-250	52	47	correctly	correctly	ADV
flr-250	52	48	classify	classify	VERB
flr-250	52	49	the	the	DET
flr-250	52	50	target	target	NOUN
flr-250	52	51	absent	absent	ADJ
flr-250	52	52	stimuli	stimulus	NOUN
flr-250	52	53	and	and	CCONJ
flr-250	52	54	is	be	AUX
flr-250	52	55	calculated	calculate	VERB
flr-250	52	56	as	as	ADP
flr-250	52	57	:	:	PUNCT
flr-250	52	58	specificity	specificity	NOUN
flr-250	52	59	=	=	SYM
flr-250	52	60	tn/(tn	tn/(tn	PROPN
flr-250	52	61	+	+	CCONJ
flr-250	52	62	fp	fp	X
flr-250	52	63	)	)	PUNCT
flr-250	52	64	(	(	PUNCT
flr-250	52	65	2.2	2.2	NUM
flr-250	52	66	)	)	PUNCT
flr-250	52	67	when	when	SCONJ
flr-250	52	68	you	you	PRON
flr-250	52	69	combine	combine	VERB
flr-250	52	70	these	these	DET
flr-250	52	71	decisions	decision	NOUN
flr-250	52	72	,	,	PUNCT
flr-250	52	73	a	a	DET
flr-250	52	74	measure	measure	NOUN
flr-250	52	75	of	of	ADP
flr-250	52	76	accuracy	accuracy	NOUN
flr-250	52	77	can	can	AUX
flr-250	52	78	be	be	AUX
flr-250	52	79	obtained	obtain	VERB
flr-250	52	80	as	as	ADP
flr-250	52	81	:	:	PUNCT
flr-250	52	82	accuracy	accuracy	NOUN
flr-250	52	83	=	=	SYM
flr-250	52	84	(	(	PUNCT
flr-250	52	85	tp	tp	X
flr-250	52	86	+	+	CCONJ
flr-250	52	87	tn)/	tn)/	PROPN
flr-250	52	88	(	(	PUNCT
flr-250	52	89	tp	tp	NOUN
flr-250	52	90	+	+	CCONJ
flr-250	52	91	fn	fn	PROPN
flr-250	52	92	+	+	NUM
flr-250	52	93	tn	tn	NOUN
flr-250	52	94	+	+	CCONJ
flr-250	52	95	fp	fp	X
flr-250	52	96	)	)	PUNCT
flr-250	52	97	(	(	PUNCT
flr-250	52	98	2.3	2.3	NUM
flr-250	52	99	)	)	PUNCT
flr-250	52	100	two	two	NUM
flr-250	52	101	other	other	ADJ
flr-250	52	102	common	common	ADJ
flr-250	52	103	metrics	metric	NOUN
flr-250	52	104	that	that	PRON
flr-250	52	105	are	be	AUX
flr-250	52	106	used	use	VERB
flr-250	52	107	in	in	ADP
flr-250	52	108	medicine	medicine	NOUN
flr-250	52	109	are	be	AUX
flr-250	52	110	positive	positive	ADJ
flr-250	52	111	and	and	CCONJ
flr-250	52	112	negative	negative	ADJ
flr-250	52	113	predictive	predictive	ADJ
flr-250	52	114	value	value	NOUN
flr-250	52	115	:	:	PUNCT
flr-250	52	116	positive	positive	ADJ
flr-250	52	117	predictive	predictive	ADJ
flr-250	52	118	value	value	NOUN
flr-250	52	119	(	(	PUNCT
flr-250	52	120	ppv	ppv	NOUN
flr-250	52	121	)	)	PUNCT
flr-250	53	1	=	=	SYM
flr-250	53	2	tp/(tp	tp/(tp	NOUN
flr-250	53	3	+	+	CCONJ
flr-250	53	4	fp	fp	X
flr-250	53	5	)	)	PUNCT
flr-250	53	6	(	(	PUNCT
flr-250	53	7	2.4	2.4	NUM
flr-250	53	8	)	)	PUNCT
flr-250	53	9	negative	negative	ADJ
flr-250	53	10	predictive	predictive	ADJ
flr-250	53	11	value	value	NOUN
flr-250	53	12	(	(	PUNCT
flr-250	53	13	npv	npv	NOUN
flr-250	53	14	)	)	PUNCT
flr-250	53	15	=	=	SYM
flr-250	53	16	tn/(tn	tn/(tn	PROPN
flr-250	53	17	+	+	CCONJ
flr-250	53	18	fn	fn	NOUN
flr-250	53	19	)	)	PUNCT
flr-250	53	20	(	(	PUNCT
flr-250	53	21	2.5	2.5	NUM
flr-250	53	22	)	)	PUNCT
flr-250	53	23	krupinski	krupinski	NOUN
flr-250	54	1	|	|	INTJ
flr-250	54	2	f	f	NOUN
flr-250	54	3	l	l	NOUN
flr-250	54	4	r	r	NOUN
flr-250	54	5	34	34	NUM
flr-250	54	6	in	in	ADP
flr-250	54	7	general	general	ADJ
flr-250	54	8	,	,	PUNCT
flr-250	54	9	there	there	PRON
flr-250	54	10	is	be	VERB
flr-250	54	11	a	a	DET
flr-250	54	12	trade	trade	NOUN
flr-250	54	13	-	-	PUNCT
flr-250	54	14	off	off	NOUN
flr-250	54	15	between	between	ADP
flr-250	54	16	sensitivity	sensitivity	NOUN
flr-250	54	17	and	and	CCONJ
flr-250	54	18	specificity	specificity	NOUN
flr-250	54	19	–	–	PUNCT
flr-250	54	20	as	as	SCONJ
flr-250	54	21	you	you	PRON
flr-250	54	22	increase	increase	VERB
flr-250	54	23	one	one	NUM
flr-250	54	24	you	you	PRON
flr-250	54	25	decrease	decrease	VERB
flr-250	54	26	the	the	DET
flr-250	54	27	other	other	ADJ
flr-250	54	28	.	.	PUNCT
flr-250	55	1	in	in	ADP
flr-250	55	2	other	other	ADJ
flr-250	55	3	words	word	NOUN
flr-250	55	4	,	,	PUNCT
flr-250	55	5	if	if	SCONJ
flr-250	55	6	you	you	PRON
flr-250	55	7	want	want	VERB
flr-250	55	8	to	to	PART
flr-250	55	9	detect	detect	VERB
flr-250	55	10	more	more	ADJ
flr-250	55	11	targets	target	NOUN
flr-250	55	12	(	(	PUNCT
flr-250	55	13	high	high	ADJ
flr-250	55	14	sensitivity	sensitivity	NOUN
flr-250	55	15	)	)	PUNCT
flr-250	55	16	it	it	PRON
flr-250	55	17	often	often	ADV
flr-250	55	18	occurs	occur	VERB
flr-250	55	19	as	as	ADP
flr-250	55	20	a	a	DET
flr-250	55	21	result	result	NOUN
flr-250	55	22	of	of	ADP
flr-250	55	23	making	make	VERB
flr-250	55	24	more	more	ADV
flr-250	55	25	false	false	ADJ
flr-250	55	26	positives	positive	NOUN
flr-250	55	27	(	(	PUNCT
flr-250	55	28	decreased	decrease	VERB
flr-250	55	29	specificity	specificity	NOUN
flr-250	55	30	)	)	PUNCT
flr-250	55	31	.	.	PUNCT
flr-250	56	1	why	why	SCONJ
flr-250	56	2	would	would	AUX
flr-250	56	3	you	you	PRON
flr-250	56	4	want	want	VERB
flr-250	56	5	to	to	PART
flr-250	56	6	use	use	VERB
flr-250	56	7	sensitivity	sensitivity	NOUN
flr-250	56	8	/	/	SYM
flr-250	56	9	specificity	specificity	NOUN
flr-250	56	10	versus	versus	ADP
flr-250	56	11	ppv	ppv	NOUN
flr-250	56	12	/	/	SYM
flr-250	56	13	npv	npv	NOUN
flr-250	56	14	?	?	PUNCT
flr-250	57	1	basically	basically	ADV
flr-250	57	2	,	,	PUNCT
flr-250	57	3	the	the	DET
flr-250	57	4	former	former	ADJ
flr-250	57	5	are	be	AUX
flr-250	57	6	independent	independent	ADJ
flr-250	57	7	of	of	ADP
flr-250	57	8	the	the	DET
flr-250	57	9	prevalence	prevalence	NOUN
flr-250	57	10	of	of	ADP
flr-250	57	11	targets	target	NOUN
flr-250	57	12	in	in	ADP
flr-250	57	13	the	the	DET
flr-250	57	14	case	case	NOUN
flr-250	57	15	sample	sample	NOUN
flr-250	57	16	while	while	SCONJ
flr-250	57	17	the	the	DET
flr-250	57	18	latter	latter	ADJ
flr-250	57	19	are	be	AUX
flr-250	57	20	not	not	PART
flr-250	57	21	.	.	PUNCT
flr-250	58	1	an	an	DET
flr-250	58	2	example	example	NOUN
flr-250	58	3	might	might	AUX
flr-250	58	4	be	be	AUX
flr-250	58	5	useful	useful	ADJ
flr-250	58	6	.	.	PUNCT
flr-250	59	1	suppose	suppose	VERB
flr-250	59	2	you	you	PRON
flr-250	59	3	have	have	VERB
flr-250	59	4	an	an	DET
flr-250	59	5	observer	observer	NOUN
flr-250	59	6	who	who	PRON
flr-250	59	7	is	be	AUX
flr-250	59	8	an	an	DET
flr-250	59	9	expert	expert	NOUN
flr-250	59	10	skilled	skilled	ADJ
flr-250	59	11	at	at	ADP
flr-250	59	12	visually	visually	ADV
flr-250	59	13	detecting	detect	VERB
flr-250	59	14	a	a	DET
flr-250	59	15	specific	specific	ADJ
flr-250	59	16	poisonous	poisonous	ADJ
flr-250	59	17	frog	frog	NOUN
flr-250	59	18	in	in	ADP
flr-250	59	19	the	the	DET
flr-250	59	20	jungle	jungle	NOUN
flr-250	59	21	versus	versus	ADP
flr-250	59	22	a	a	DET
flr-250	59	23	similar	similar	ADJ
flr-250	59	24	but	but	CCONJ
flr-250	59	25	non	non	ADJ
flr-250	59	26	-	-	ADJ
flr-250	59	27	poisonous	poisonous	ADJ
flr-250	59	28	frog	frog	NOUN
flr-250	59	29	and	and	CCONJ
flr-250	59	30	her	her	PRON
flr-250	59	31	sensitivity	sensitivity	NOUN
flr-250	59	32	is	be	AUX
flr-250	59	33	95	95	NUM
flr-250	59	34	%	%	NOUN
flr-250	59	35	and	and	CCONJ
flr-250	59	36	specificity	specificity	NOUN
flr-250	59	37	80	80	NUM
flr-250	59	38	%	%	NOUN
flr-250	59	39	.	.	PUNCT
flr-250	60	1	in	in	ADP
flr-250	60	2	jungle	jungle	NOUN
flr-250	60	3	#	#	SYM
flr-250	60	4	1	1	NUM
flr-250	60	5	there	there	PRON
flr-250	60	6	are	be	VERB
flr-250	60	7	1000	1000	NUM
flr-250	60	8	frogs	frog	NOUN
flr-250	60	9	total	total	ADJ
flr-250	60	10	with	with	ADP
flr-250	60	11	a	a	DET
flr-250	60	12	prevalence	prevalence	NOUN
flr-250	60	13	of	of	ADP
flr-250	60	14	50	50	NUM
flr-250	60	15	%	%	NOUN
flr-250	60	16	poisonous	poisonous	ADJ
flr-250	60	17	(	(	PUNCT
flr-250	60	18	n	n	NOUN
flr-250	60	19	=	=	SYM
flr-250	60	20	500	500	NUM
flr-250	60	21	)	)	PUNCT
flr-250	60	22	.	.	PUNCT
flr-250	61	1	in	in	ADP
flr-250	61	2	jungle	jungle	NOUN
flr-250	61	3	#	#	SYM
flr-250	61	4	2	2	NUM
flr-250	61	5	there	there	PRON
flr-250	61	6	are	be	VERB
flr-250	61	7	also	also	ADV
flr-250	61	8	1000	1000	NUM
flr-250	61	9	frogs	frog	NOUN
flr-250	61	10	total	total	ADJ
flr-250	61	11	but	but	CCONJ
flr-250	61	12	only	only	ADV
flr-250	61	13	25	25	NUM
flr-250	61	14	%	%	NOUN
flr-250	61	15	are	be	AUX
flr-250	61	16	poisonous	poisonous	ADJ
flr-250	61	17	(	(	PUNCT
flr-250	61	18	n	n	NOUN
flr-250	61	19	=	=	NUM
flr-250	61	20	250	250	NUM
flr-250	61	21	)	)	PUNCT
flr-250	61	22	.	.	PUNCT
flr-250	62	1	based	base	VERB
flr-250	62	2	on	on	ADP
flr-250	62	3	this	this	PRON
flr-250	62	4	,	,	PUNCT
flr-250	62	5	our	our	PRON
flr-250	62	6	observer	observer	NOUN
flr-250	62	7	has	have	VERB
flr-250	62	8	the	the	DET
flr-250	62	9	following	follow	VERB
flr-250	62	10	performance	performance	NOUN
flr-250	62	11	levels	level	NOUN
flr-250	62	12	.	.	PUNCT
flr-250	63	1	it	it	PRON
flr-250	63	2	can	can	AUX
flr-250	63	3	be	be	AUX
flr-250	63	4	seen	see	VERB
flr-250	63	5	that	that	SCONJ
flr-250	63	6	depending	depend	VERB
flr-250	63	7	on	on	ADP
flr-250	63	8	which	which	DET
flr-250	63	9	jungle	jungle	NOUN
flr-250	63	10	the	the	DET
flr-250	63	11	observer	observer	NOUN
flr-250	63	12	study	study	NOUN
flr-250	63	13	is	be	AUX
flr-250	63	14	conducted	conduct	VERB
flr-250	63	15	,	,	PUNCT
flr-250	63	16	the	the	DET
flr-250	63	17	performance	performance	NOUN
flr-250	63	18	even	even	ADV
flr-250	63	19	of	of	ADP
flr-250	63	20	an	an	DET
flr-250	63	21	expert	expert	ADJ
flr-250	63	22	observer	observer	NOUN
flr-250	63	23	will	will	AUX
flr-250	63	24	differ	differ	VERB
flr-250	63	25	.	.	PUNCT
flr-250	64	1	jungle	jungle	NOUN
flr-250	64	2	#	#	SYM
flr-250	64	3	1	1	NUM
flr-250	64	4	:	:	PUNCT
flr-250	64	5	tp	tp	ADP
flr-250	64	6	=	=	NOUN
flr-250	64	7	475	475	NUM
flr-250	64	8	fn	fn	NOUN
flr-250	64	9	=	=	SYM
flr-250	64	10	25	25	NUM
flr-250	64	11	fp	fp	NOUN
flr-250	64	12	=	=	NUM
flr-250	64	13	100	100	NUM
flr-250	64	14	tn	tn	NOUN
flr-250	64	15	=	=	SYM
flr-250	64	16	400	400	NUM
flr-250	64	17	accuracy	accuracy	NOUN
flr-250	64	18	=	=	SYM
flr-250	64	19	(	(	PUNCT
flr-250	64	20	475	475	NUM
flr-250	64	21	+	+	SYM
flr-250	64	22	400)/(475	400)/(475	NUM
flr-250	65	1	+	+	CCONJ
flr-250	65	2	25	25	NUM
flr-250	65	3	+	+	NUM
flr-250	65	4	100	100	NUM
flr-250	65	5	+	+	NUM
flr-250	65	6	400	400	NUM
flr-250	65	7	)	)	PUNCT
flr-250	66	1	=	=	SYM
flr-250	66	2	0.88	0.88	NUM
flr-250	66	3	or	or	CCONJ
flr-250	66	4	88	88	NUM
flr-250	66	5	%	%	NOUN
flr-250	66	6	ppv	ppv	NOUN
flr-250	66	7	=	=	SYM
flr-250	66	8	475/(475	475/(475	NUM
flr-250	66	9	+	+	CCONJ
flr-250	66	10	100	100	NUM
flr-250	66	11	)	)	PUNCT
flr-250	67	1	=	=	SYM
flr-250	67	2	0.83	0.83	NUM
flr-250	67	3	or	or	CCONJ
flr-250	67	4	83	83	NUM
flr-250	67	5	%	%	NOUN
flr-250	67	6	npv	npv	NOUN
flr-250	67	7	=	=	SYM
flr-250	67	8	400/(400	400/(400	NUM
flr-250	67	9	+	+	CCONJ
flr-250	67	10	25	25	NUM
flr-250	67	11	)	)	PUNCT
flr-250	67	12	=	=	SYM
flr-250	67	13	0.94	0.94	NUM
flr-250	67	14	or	or	CCONJ
flr-250	67	15	94	94	NUM
flr-250	67	16	%	%	NOUN
flr-250	67	17	jungle	jungle	NOUN
flr-250	67	18	#	#	SYM
flr-250	67	19	2	2	NUM
flr-250	67	20	:	:	PUNCT
flr-250	67	21	tp	tp	ADP
flr-250	67	22	=	=	NOUN
flr-250	68	1	238	238	NUM
flr-250	68	2	fn	fn	NOUN
flr-250	68	3	=	=	SYM
flr-250	68	4	12	12	NUM
flr-250	68	5	fp	fp	NOUN
flr-250	68	6	=	=	SYM
flr-250	68	7	150	150	NUM
flr-250	68	8	tn	tn	NOUN
flr-250	68	9	=	=	SYM
flr-250	68	10	600	600	NUM
flr-250	68	11	accuracy	accuracy	NOUN
flr-250	68	12	=	=	SYM
flr-250	68	13	(	(	PUNCT
flr-250	68	14	238	238	NUM
flr-250	68	15	+	+	NOUN
flr-250	68	16	600)/(238	600)/(238	NUM
flr-250	69	1	+	+	NUM
flr-250	69	2	12	12	NUM
flr-250	69	3	+	+	NUM
flr-250	69	4	150	150	NUM
flr-250	69	5	+	+	NUM
flr-250	69	6	600	600	NUM
flr-250	69	7	)	)	PUNCT
flr-250	70	1	=	=	SYM
flr-250	70	2	0.84	0.84	NUM
flr-250	70	3	or	or	CCONJ
flr-250	70	4	84	84	NUM
flr-250	71	1	%	%	NOUN
flr-250	71	2	ppv	ppv	NOUN
flr-250	71	3	=	=	SYM
flr-250	71	4	238/(238	238/(238	NUM
flr-250	71	5	+	+	CCONJ
flr-250	71	6	150	150	NUM
flr-250	71	7	)	)	PUNCT
flr-250	71	8	=	=	SYM
flr-250	71	9	0.61	0.61	NUM
flr-250	71	10	or	or	CCONJ
flr-250	71	11	61	61	NUM
flr-250	71	12	%	%	NOUN
flr-250	71	13	npv	npv	NOUN
flr-250	71	14	=	=	SYM
flr-250	71	15	600/(600	600/(600	NUM
flr-250	72	1	+	+	NUM
flr-250	72	2	12	12	NUM
flr-250	72	3	)	)	PUNCT
flr-250	73	1	=	=	NOUN
flr-250	73	2	0.98	0.98	NUM
flr-250	73	3	or	or	CCONJ
flr-250	73	4	98	98	NUM
flr-250	73	5	%	%	NOUN
flr-250	73	6	in	in	ADP
flr-250	73	7	many	many	ADJ
flr-250	73	8	cases	case	NOUN
flr-250	73	9	sensitivity	sensitivity	NOUN
flr-250	73	10	and	and	CCONJ
flr-250	73	11	specificity	specificity	NOUN
flr-250	73	12	are	be	AUX
flr-250	73	13	more	more	ADJ
flr-250	73	14	than	than	ADP
flr-250	73	15	adequate	adequate	ADJ
flr-250	73	16	measures	measure	NOUN
flr-250	73	17	of	of	ADP
flr-250	73	18	performance	performance	NOUN
flr-250	73	19	for	for	ADP
flr-250	73	20	visual	visual	ADJ
flr-250	73	21	search	search	NOUN
flr-250	73	22	tasks	task	NOUN
flr-250	73	23	,	,	PUNCT
flr-250	73	24	but	but	CCONJ
flr-250	73	25	it	it	PRON
flr-250	73	26	becomes	become	VERB
flr-250	73	27	complicated	complicated	ADJ
flr-250	73	28	when	when	SCONJ
flr-250	73	29	the	the	DET
flr-250	73	30	test	test	NOUN
flr-250	73	31	sets	set	NOUN
flr-250	73	32	contain	contain	VERB
flr-250	73	33	cases	case	NOUN
flr-250	73	34	with	with	ADP
flr-250	73	35	a	a	DET
flr-250	73	36	range	range	NOUN
flr-250	73	37	of	of	ADP
flr-250	73	38	difficulty	difficulty	NOUN
flr-250	73	39	levels	level	NOUN
flr-250	73	40	.	.	PUNCT
flr-250	74	1	for	for	ADP
flr-250	74	2	example	example	NOUN
flr-250	74	3	,	,	PUNCT
flr-250	74	4	in	in	ADP
flr-250	74	5	radiology	radiology	NOUN
flr-250	74	6	some	some	DET
flr-250	74	7	bone	bone	NOUN
flr-250	74	8	fractures	fracture	NOUN
flr-250	74	9	are	be	AUX
flr-250	74	10	very	very	ADV
flr-250	74	11	obvious	obvious	ADJ
flr-250	74	12	and	and	CCONJ
flr-250	74	13	thus	thus	ADV
flr-250	74	14	easy	easy	ADJ
flr-250	74	15	to	to	PART
flr-250	74	16	detect	detect	VERB
flr-250	74	17	but	but	CCONJ
flr-250	74	18	others	other	NOUN
flr-250	74	19	are	be	AUX
flr-250	74	20	quite	quite	ADV
flr-250	74	21	subtle	subtle	ADJ
flr-250	74	22	and	and	CCONJ
flr-250	74	23	can	can	AUX
flr-250	74	24	readily	readily	ADV
flr-250	74	25	be	be	AUX
flr-250	74	26	missed	miss	VERB
flr-250	74	27	.	.	PUNCT
flr-250	75	1	in	in	ADP
flr-250	75	2	the	the	DET
flr-250	75	3	frog	frog	NOUN
flr-250	75	4	example	example	NOUN
flr-250	75	5	,	,	PUNCT
flr-250	75	6	a	a	DET
flr-250	75	7	bright	bright	ADJ
flr-250	75	8	red	red	ADJ
flr-250	75	9	frog	frog	NOUN
flr-250	75	10	in	in	ADP
flr-250	75	11	a	a	DET
flr-250	75	12	green	green	ADJ
flr-250	75	13	jungle	jungle	NOUN
flr-250	75	14	is	be	AUX
flr-250	75	15	likely	likely	ADV
flr-250	75	16	easier	easy	ADJ
flr-250	75	17	to	to	PART
flr-250	75	18	detect	detect	VERB
flr-250	75	19	than	than	ADP
flr-250	75	20	a	a	DET
flr-250	75	21	light	light	ADJ
flr-250	75	22	green	green	ADJ
flr-250	75	23	frog	frog	NOUN
flr-250	75	24	in	in	ADP
flr-250	75	25	a	a	DET
flr-250	75	26	dark	dark	ADJ
flr-250	75	27	green	green	ADJ
flr-250	75	28	jungle	jungle	NOUN
flr-250	75	29	.	.	PUNCT
flr-250	76	1	in	in	ADP
flr-250	76	2	cases	case	NOUN
flr-250	76	3	that	that	PRON
flr-250	76	4	are	be	AUX
flr-250	76	5	not	not	PART
flr-250	76	6	obvious	obvious	ADJ
flr-250	76	7	the	the	DET
flr-250	76	8	decision	decision	NOUN
flr-250	76	9	as	as	ADP
flr-250	76	10	to	to	ADP
flr-250	76	11	whether	whether	SCONJ
flr-250	76	12	or	or	CCONJ
flr-250	76	13	not	not	PART
flr-250	76	14	the	the	DET
flr-250	76	15	target	target	NOUN
flr-250	76	16	is	be	AUX
flr-250	76	17	actually	actually	ADV
flr-250	76	18	there	there	PRON
flr-250	76	19	becomes	become	VERB
flr-250	76	20	less	less	ADV
flr-250	76	21	certain	certain	ADJ
flr-250	76	22	and	and	CCONJ
flr-250	76	23	observers	observer	NOUN
flr-250	76	24	may	may	AUX
flr-250	76	25	not	not	PART
flr-250	76	26	be	be	AUX
flr-250	76	27	willing	willing	ADJ
flr-250	76	28	to	to	PART
flr-250	76	29	give	give	VERB
flr-250	76	30	a	a	DET
flr-250	76	31	binary	binary	NOUN
flr-250	76	32	yes	yes	NOUN
flr-250	76	33	/	/	SYM
flr-250	76	34	no	no	DET
flr-250	76	35	present	present	ADJ
flr-250	76	36	/	/	SYM
flr-250	76	37	absent	absent	ADJ
flr-250	76	38	decision	decision	NOUN
flr-250	76	39	,	,	PUNCT
flr-250	76	40	but	but	CCONJ
flr-250	76	41	may	may	AUX
flr-250	76	42	be	be	AUX
flr-250	76	43	more	more	ADV
flr-250	76	44	willing	willing	ADJ
flr-250	76	45	to	to	PART
flr-250	76	46	report	report	VERB
flr-250	76	47	their	their	PRON
flr-250	76	48	decision	decision	NOUN
flr-250	76	49	as	as	ADP
flr-250	76	50	a	a	DET
flr-250	76	51	function	function	NOUN
flr-250	76	52	of	of	ADP
flr-250	76	53	confidence	confidence	NOUN
flr-250	76	54	,	,	PUNCT
flr-250	76	55	for	for	ADP
flr-250	76	56	example	example	NOUN
flr-250	76	57	reporting	report	VERB
flr-250	76	58	a	a	DET
flr-250	76	59	target	target	NOUN
flr-250	76	60	(	(	PUNCT
flr-250	76	61	or	or	CCONJ
flr-250	76	62	lack	lack	NOUN
flr-250	76	63	of	of	ADP
flr-250	76	64	target	target	NOUN
flr-250	76	65	)	)	PUNCT
flr-250	76	66	as	as	ADV
flr-250	76	67	definitely	definitely	ADV
flr-250	76	68	present	present	ADJ
flr-250	76	69	,	,	PUNCT
flr-250	76	70	probably	probably	ADV
flr-250	76	71	present	present	ADJ
flr-250	76	72	or	or	CCONJ
flr-250	76	73	possibly	possibly	ADV
flr-250	76	74	present	present	ADJ
flr-250	76	75	.	.	PUNCT
flr-250	77	1	in	in	ADP
flr-250	77	2	other	other	ADJ
flr-250	77	3	words	word	NOUN
flr-250	77	4	,	,	PUNCT
flr-250	77	5	the	the	DET
flr-250	77	6	observer	observer	NOUN
flr-250	77	7	’s	’s	PART
flr-250	77	8	decision	decision	NOUN
flr-250	77	9	threshold	threshold	NOUN
flr-250	77	10	for	for	ADP
flr-250	77	11	reporting	reporting	NOUN
flr-250	77	12	can	can	AUX
flr-250	77	13	change	change	VERB
flr-250	77	14	as	as	ADP
flr-250	77	15	a	a	DET
flr-250	77	16	function	function	NOUN
flr-250	77	17	of	of	ADP
flr-250	77	18	many	many	ADJ
flr-250	77	19	things	thing	NOUN
flr-250	77	20	,	,	PUNCT
flr-250	77	21	including	include	VERB
flr-250	77	22	but	but	CCONJ
flr-250	77	23	not	not	PART
flr-250	77	24	limited	limit	VERB
flr-250	77	25	to	to	ADP
flr-250	77	26	the	the	DET
flr-250	77	27	nature	nature	NOUN
flr-250	77	28	of	of	ADP
flr-250	77	29	the	the	DET
flr-250	77	30	target	target	NOUN
flr-250	77	31	,	,	PUNCT
flr-250	77	32	target	target	NOUN
flr-250	77	33	prevalence	prevalence	NOUN
flr-250	77	34	,	,	PUNCT
flr-250	77	35	background	background	NOUN
flr-250	77	36	complexity	complexity	NOUN
flr-250	77	37	within	within	ADP
flr-250	77	38	which	which	PRON
flr-250	77	39	the	the	DET
flr-250	77	40	target	target	NOUN
flr-250	77	41	is	be	AUX
flr-250	77	42	embedded	embed	VERB
flr-250	77	43	,	,	PUNCT
flr-250	77	44	number	number	NOUN
flr-250	77	45	and	and	CCONJ
flr-250	77	46	type	type	NOUN
flr-250	77	47	of	of	ADP
flr-250	77	48	targets	target	NOUN
flr-250	77	49	,	,	PUNCT
flr-250	77	50	and	and	CCONJ
flr-250	77	51	observer	observer	NOUN
flr-250	77	52	experience	experience	NOUN
flr-250	77	53	or	or	CCONJ
flr-250	77	54	expertise	expertise	NOUN
flr-250	77	55	.	.	PUNCT
flr-250	78	1	this	this	PRON
flr-250	78	2	is	be	AUX
flr-250	78	3	where	where	SCONJ
flr-250	78	4	roc	roc	PROPN
flr-250	78	5	analysis	analysis	NOUN
flr-250	78	6	becomes	become	VERB
flr-250	78	7	useful	useful	ADJ
flr-250	78	8	.	.	PUNCT
flr-250	79	1	2.2	2.2	NUM
flr-250	79	2	the	the	DET
flr-250	79	3	roc	roc	PROPN
flr-250	79	4	curve	curve	NOUN
flr-250	79	5	even	even	ADV
flr-250	79	6	the	the	DET
flr-250	79	7	visual	visual	ADJ
flr-250	79	8	expert	expert	NOUN
flr-250	79	9	may	may	AUX
flr-250	79	10	not	not	PART
flr-250	79	11	perform	perform	VERB
flr-250	79	12	as	as	SCONJ
flr-250	79	13	one	one	PRON
flr-250	79	14	would	would	AUX
flr-250	79	15	think	think	VERB
flr-250	79	16	without	without	ADP
flr-250	79	17	delving	delve	VERB
flr-250	79	18	further	far	ADV
flr-250	79	19	into	into	ADP
flr-250	79	20	the	the	DET
flr-250	79	21	nature	nature	NOUN
flr-250	79	22	of	of	ADP
flr-250	79	23	the	the	DET
flr-250	79	24	task	task	NOUN
flr-250	79	25	.	.	PUNCT
flr-250	80	1	for	for	ADP
flr-250	80	2	example	example	NOUN
flr-250	80	3	,	,	PUNCT
flr-250	80	4	in	in	ADP
flr-250	80	5	radiology	radiology	NOUN
flr-250	80	6	decision	decision	NOUN
flr-250	80	7	thresholds	threshold	NOUN
flr-250	80	8	can	can	AUX
flr-250	80	9	change	change	VERB
flr-250	80	10	within	within	ADP
flr-250	80	11	and	and	CCONJ
flr-250	80	12	between	between	ADP
flr-250	80	13	observers	observer	NOUN
flr-250	80	14	as	as	ADP
flr-250	80	15	a	a	DET
flr-250	80	16	function	function	NOUN
flr-250	80	17	of	of	ADP
flr-250	80	18	the	the	DET
flr-250	80	19	nature	nature	NOUN
flr-250	80	20	of	of	ADP
flr-250	80	21	the	the	DET
flr-250	80	22	task	task	NOUN
flr-250	80	23	and	and	CCONJ
flr-250	80	24	its	its	PRON
flr-250	80	25	consequences	consequence	NOUN
flr-250	80	26	.	.	PUNCT
flr-250	81	1	in	in	ADP
flr-250	81	2	chest	chest	NOUN
flr-250	81	3	ct	ct	PROPN
flr-250	81	4	exam	exam	NOUN
flr-250	81	5	interpretation	interpretation	NOUN
flr-250	81	6	a	a	DET
flr-250	81	7	radiologist	radiologist	NOUN
flr-250	81	8	may	may	AUX
flr-250	81	9	adopt	adopt	VERB
flr-250	81	10	a	a	DET
flr-250	81	11	very	very	ADV
flr-250	81	12	conservative	conservative	ADJ
flr-250	81	13	threshold	threshold	NOUN
flr-250	81	14	for	for	ADP
flr-250	81	15	reporting	report	VERB
flr-250	81	16	possible	possible	ADJ
flr-250	81	17	abnormalities	abnormality	NOUN
flr-250	81	18	in	in	ADP
flr-250	81	19	the	the	DET
flr-250	81	20	lungs	lung	NOUN
flr-250	81	21	that	that	PRON
flr-250	81	22	could	could	AUX
flr-250	81	23	be	be	AUX
flr-250	81	24	cancer	cancer	NOUN
flr-250	81	25	nodules	nodule	NOUN
flr-250	81	26	but	but	CCONJ
flr-250	81	27	are	be	AUX
flr-250	81	28	less	less	ADJ
flr-250	81	29	than	than	ADP
flr-250	81	30	5	5	NUM
flr-250	81	31	mm	mm	NOUN
flr-250	81	32	in	in	ADP
flr-250	81	33	size	size	NOUN
flr-250	81	34	because	because	SCONJ
flr-250	81	35	they	they	PRON
flr-250	81	36	know	know	VERB
flr-250	81	37	that	that	SCONJ
flr-250	81	38	ct	ct	PROPN
flr-250	81	39	is	be	AUX
flr-250	81	40	typically	typically	ADV
flr-250	81	41	the	the	DET
flr-250	81	42	best	good	ADJ
flr-250	81	43	imaging	imaging	NOUN
flr-250	81	44	exam	exam	NOUN
flr-250	81	45	to	to	PART
flr-250	81	46	do	do	VERB
flr-250	81	47	(	(	PUNCT
flr-250	81	48	i.e.	i.e.	X
flr-250	81	49	,	,	PUNCT
flr-250	81	50	no	no	DET
flr-250	81	51	other	other	ADJ
flr-250	81	52	follow	follow	VERB
flr-250	81	53	-	-	PUNCT
flr-250	81	54	up	up	ADP
flr-250	81	55	imaging	imaging	NOUN
flr-250	81	56	options	option	NOUN
flr-250	81	57	)	)	PUNCT
flr-250	81	58	and	and	CCONJ
flr-250	81	59	obtaining	obtain	VERB
flr-250	81	60	a	a	DET
flr-250	81	61	biopsy	biopsy	NOUN
flr-250	81	62	on	on	ADP
flr-250	81	63	such	such	DET
flr-250	81	64	a	a	DET
flr-250	81	65	small	small	ADJ
flr-250	81	66	target	target	NOUN
flr-250	81	67	is	be	AUX
flr-250	81	68	very	very	ADV
flr-250	81	69	difficult	difficult	ADJ
flr-250	81	70	(	(	PUNCT
flr-250	81	71	potentially	potentially	ADV
flr-250	81	72	leading	lead	VERB
flr-250	81	73	to	to	ADP
flr-250	81	74	a	a	DET
flr-250	81	75	pneumothorax	pneumothorax	NOUN
flr-250	81	76	or	or	CCONJ
flr-250	81	77	puncture	puncture	NOUN
flr-250	81	78	of	of	ADP
flr-250	81	79	the	the	DET
flr-250	81	80	lung	lung	NOUN
flr-250	81	81	)	)	PUNCT
flr-250	81	82	and	and	CCONJ
flr-250	81	83	unlikely	unlikely	ADJ
flr-250	81	84	to	to	PART
flr-250	81	85	yield	yield	VERB
flr-250	81	86	a	a	DET
flr-250	81	87	specimen	speciman	NOUN
flr-250	81	88	large	large	ADJ
flr-250	81	89	enough	enough	ADV
flr-250	81	90	to	to	PART
flr-250	81	91	get	get	VERB
flr-250	81	92	an	an	DET
flr-250	81	93	accurate	accurate	ADJ
flr-250	81	94	biopsy	biopsy	NOUN
flr-250	81	95	on	on	ADP
flr-250	81	96	.	.	PUNCT
flr-250	82	1	instead	instead	ADV
flr-250	82	2	of	of	ADP
flr-250	82	3	reporting	report	VERB
flr-250	82	4	it	it	PRON
flr-250	82	5	,	,	PUNCT
flr-250	82	6	the	the	DET
flr-250	82	7	radiologist	radiologist	NOUN
flr-250	82	8	may	may	AUX
flr-250	82	9	recommend	recommend	VERB
flr-250	82	10	a	a	DET
flr-250	82	11	6	6	NUM
flr-250	82	12	-	-	PUNCT
flr-250	82	13	month	month	NOUN
flr-250	82	14	watch	watch	NOUN
flr-250	82	15	period	period	NOUN
flr-250	82	16	and	and	CCONJ
flr-250	82	17	another	another	DET
flr-250	82	18	ct	ct	PROPN
flr-250	82	19	exam	exam	NOUN
flr-250	82	20	.	.	PUNCT
flr-250	83	1	in	in	ADP
flr-250	83	2	mammography	mammography	NOUN
flr-250	83	3	however	however	ADV
flr-250	83	4	,	,	PUNCT
flr-250	83	5	small	small	ADJ
flr-250	83	6	lesions	lesion	NOUN
flr-250	83	7	are	be	AUX
flr-250	83	8	easier	easy	ADJ
flr-250	83	9	to	to	PART
flr-250	83	10	biopsy	biopsy	VERB
flr-250	83	11	and	and	CCONJ
flr-250	83	12	there	there	PRON
flr-250	83	13	is	be	VERB
flr-250	83	14	no	no	DET
flr-250	83	15	risk	risk	NOUN
flr-250	83	16	of	of	ADP
flr-250	83	17	puncturing	puncture	VERB
flr-250	83	18	a	a	DET
flr-250	83	19	lung	lung	NOUN
flr-250	83	20	or	or	CCONJ
flr-250	83	21	other	other	ADJ
flr-250	83	22	vital	vital	ADJ
flr-250	83	23	structure	structure	NOUN
flr-250	83	24	,	,	PUNCT
flr-250	83	25	so	so	SCONJ
flr-250	83	26	mammographers	mammographer	NOUN
flr-250	83	27	tend	tend	VERB
flr-250	83	28	to	to	PART
flr-250	83	29	be	be	AUX
flr-250	83	30	more	more	ADV
flr-250	83	31	liberal	liberal	ADJ
flr-250	83	32	in	in	ADP
flr-250	83	33	their	their	PRON
flr-250	83	34	reporting	reporting	NOUN
flr-250	83	35	of	of	ADP
flr-250	83	36	potential	potential	ADJ
flr-250	83	37	cancerous	cancerous	ADJ
flr-250	83	38	findings	finding	NOUN
flr-250	83	39	.	.	PUNCT
flr-250	84	1	this	this	PRON
flr-250	84	2	comes	come	VERB
flr-250	84	3	at	at	ADP
flr-250	84	4	the	the	DET
flr-250	84	5	risk	risk	NOUN
flr-250	84	6	of	of	ADP
flr-250	84	7	more	more	ADV
flr-250	84	8	false	false	ADJ
flr-250	84	9	positives	positive	NOUN
flr-250	84	10	but	but	CCONJ
flr-250	84	11	before	before	ADP
flr-250	84	12	doing	do	VERB
flr-250	84	13	a	a	DET
flr-250	84	14	biopsy	biopsy	NOUN
flr-250	84	15	additional	additional	ADJ
flr-250	84	16	x	x	NOUN
flr-250	84	17	-	-	NOUN
flr-250	84	18	ray	ray	NOUN
flr-250	84	19	images	image	NOUN
flr-250	84	20	or	or	CCONJ
flr-250	84	21	an	an	DET
flr-250	84	22	ultrasound	ultrasound	NOUN
flr-250	84	23	is	be	AUX
flr-250	84	24	often	often	ADV
flr-250	84	25	recommended	recommend	VERB
flr-250	84	26	,	,	PUNCT
flr-250	84	27	reducing	reduce	VERB
flr-250	84	28	the	the	DET
flr-250	84	29	impact	impact	NOUN
flr-250	84	30	of	of	ADP
flr-250	84	31	a	a	DET
flr-250	84	32	false	false	ADJ
flr-250	84	33	positive	positive	ADJ
flr-250	84	34	even	even	ADV
flr-250	84	35	further	far	ADV
flr-250	84	36	.	.	PUNCT
flr-250	85	1	krupinski	krupinski	VERB
flr-250	85	2	|	|	ADV
flr-250	85	3	f	f	NOUN
flr-250	85	4	l	l	NOUN
flr-250	85	5	r	r	NOUN
flr-250	85	6	35	35	NUM
flr-250	85	7	roc	roc	NOUN
flr-250	85	8	analysis	analysis	NOUN
flr-250	85	9	and	and	CCONJ
flr-250	85	10	the	the	DET
flr-250	85	11	resulting	result	VERB
flr-250	85	12	roc	roc	PROPN
flr-250	85	13	curve	curve	NOUN
flr-250	85	14	is	be	AUX
flr-250	85	15	a	a	DET
flr-250	85	16	method	method	NOUN
flr-250	85	17	that	that	PRON
flr-250	85	18	captures	capture	VERB
flr-250	85	19	the	the	DET
flr-250	85	20	relationship	relationship	NOUN
flr-250	85	21	(	(	PUNCT
flr-250	85	22	i.e.	i.e.	X
flr-250	85	23	,	,	PUNCT
flr-250	85	24	trade	trade	NOUN
flr-250	85	25	-	-	PUNCT
flr-250	85	26	offs	off	NOUN
flr-250	85	27	)	)	PUNCT
flr-250	85	28	between	between	ADP
flr-250	85	29	sensitivity	sensitivity	NOUN
flr-250	85	30	and	and	CCONJ
flr-250	85	31	specificity	specificity	NOUN
flr-250	85	32	as	as	ADV
flr-250	85	33	well	well	ADV
flr-250	85	34	as	as	ADP
flr-250	85	35	the	the	DET
flr-250	85	36	range	range	NOUN
flr-250	85	37	of	of	ADP
flr-250	85	38	decision	decision	NOUN
flr-250	85	39	thresholds	threshold	VERB
flr-250	85	40	that	that	SCONJ
flr-250	85	41	every	every	DET
flr-250	85	42	observer	observer	NOUN
flr-250	85	43	has	have	AUX
flr-250	85	44	no	no	ADV
flr-250	85	45	matter	matter	ADV
flr-250	85	46	what	what	PRON
flr-250	85	47	their	their	PRON
flr-250	85	48	level	level	NOUN
flr-250	85	49	of	of	ADP
flr-250	85	50	expertise	expertise	NOUN
flr-250	85	51	and	and	CCONJ
flr-250	85	52	experience	experience	NOUN
flr-250	85	53	.	.	PUNCT
flr-250	86	1	the	the	DET
flr-250	86	2	roc	roc	PROPN
flr-250	86	3	curve	curve	NOUN
flr-250	86	4	(	(	PUNCT
flr-250	86	5	figure	figure	NOUN
flr-250	86	6	1	1	NUM
flr-250	86	7	)	)	PUNCT
flr-250	86	8	is	be	AUX
flr-250	86	9	a	a	DET
flr-250	86	10	graphical	graphical	ADJ
flr-250	86	11	representation	representation	NOUN
flr-250	86	12	of	of	ADP
flr-250	86	13	this	this	DET
flr-250	86	14	relationship	relationship	NOUN
flr-250	86	15	,	,	PUNCT
flr-250	86	16	plotting	plot	VERB
flr-250	86	17	sensitivity	sensitivity	NOUN
flr-250	86	18	(	(	PUNCT
flr-250	86	19	the	the	DET
flr-250	86	20	tp	tp	NOUN
flr-250	86	21	fraction	fraction	NOUN
flr-250	86	22	)	)	PUNCT
flr-250	86	23	versus	versus	ADP
flr-250	86	24	1	1	NUM
flr-250	86	25	-	-	PUNCT
flr-250	86	26	specificity	specificity	NOUN
flr-250	86	27	(	(	PUNCT
flr-250	86	28	the	the	DET
flr-250	86	29	fp	fp	PROPN
flr-250	86	30	fraction	fraction	NOUN
flr-250	86	31	or	or	CCONJ
flr-250	86	32	1	1	NUM
flr-250	86	33	–	–	PUNCT
flr-250	86	34	tn/(tn	tn/(tn	NOUN
flr-250	86	35	+	+	CCONJ
flr-250	86	36	fp	fp	X
flr-250	86	37	)	)	PUNCT
flr-250	86	38	=	=	SYM
flr-250	86	39	fp/(fp	fp/(fp	PROPN
flr-250	86	40	+	+	CCONJ
flr-250	86	41	tn	tn	NOUN
flr-250	86	42	)	)	PUNCT
flr-250	86	43	)	)	PUNCT
flr-250	86	44	for	for	ADP
flr-250	86	45	all	all	DET
flr-250	86	46	possible	possible	ADJ
flr-250	86	47	decision	decision	NOUN
flr-250	86	48	thresholds	threshold	NOUN
flr-250	86	49	.	.	PUNCT
flr-250	87	1	the	the	DET
flr-250	87	2	axes	axis	NOUN
flr-250	87	3	go	go	VERB
flr-250	87	4	from	from	ADP
flr-250	87	5	0	0	NUM
flr-250	87	6	to	to	ADP
flr-250	87	7	1	1	NUM
flr-250	87	8	since	since	SCONJ
flr-250	87	9	sensitivity	sensitivity	NOUN
flr-250	87	10	and	and	CCONJ
flr-250	87	11	specificity	specificity	NOUN
flr-250	87	12	are	be	AUX
flr-250	87	13	typically	typically	ADV
flr-250	87	14	represented	represent	VERB
flr-250	87	15	as	as	ADP
flr-250	87	16	proportions	proportion	NOUN
flr-250	87	17	.	.	PUNCT
flr-250	88	1	the	the	DET
flr-250	88	2	diagonal	diagonal	ADJ
flr-250	88	3	line	line	NOUN
flr-250	88	4	is	be	AUX
flr-250	88	5	chance	chance	NOUN
flr-250	88	6	performance	performance	NOUN
flr-250	88	7	or	or	CCONJ
flr-250	88	8	guessing	guessing	NOUN
flr-250	88	9	and	and	CCONJ
flr-250	88	10	the	the	DET
flr-250	88	11	curves	curve	NOUN
flr-250	88	12	indicate	indicate	VERB
flr-250	88	13	increasingly	increasingly	ADV
flr-250	88	14	better	well	ADJ
flr-250	88	15	performance	performance	NOUN
flr-250	88	16	moving	move	VERB
flr-250	88	17	to	to	ADP
flr-250	88	18	the	the	DET
flr-250	88	19	upper	upper	ADJ
flr-250	88	20	left	left	ADJ
flr-250	88	21	corner	corner	NOUN
flr-250	88	22	which	which	PRON
flr-250	88	23	represents	represent	VERB
flr-250	88	24	perfect	perfect	ADJ
flr-250	88	25	performance	performance	NOUN
flr-250	88	26	.	.	PUNCT
flr-250	89	1	thus	thus	ADV
flr-250	89	2	in	in	ADP
flr-250	89	3	terms	term	NOUN
flr-250	89	4	of	of	ADP
flr-250	89	5	visual	visual	ADJ
flr-250	89	6	expertise	expertise	NOUN
flr-250	89	7	,	,	PUNCT
flr-250	89	8	one	one	PRON
flr-250	89	9	would	would	AUX
flr-250	89	10	expect	expect	VERB
flr-250	89	11	that	that	SCONJ
flr-250	89	12	those	those	PRON
flr-250	89	13	with	with	ADP
flr-250	89	14	more	more	ADJ
flr-250	89	15	expertise	expertise	NOUN
flr-250	89	16	would	would	AUX
flr-250	89	17	tend	tend	VERB
flr-250	89	18	to	to	PART
flr-250	89	19	have	have	VERB
flr-250	89	20	curves	curve	NOUN
flr-250	89	21	closer	close	ADV
flr-250	89	22	to	to	ADP
flr-250	89	23	the	the	DET
flr-250	89	24	upper	upper	ADJ
flr-250	89	25	left	left	NOUN
flr-250	89	26	corner	corner	NOUN
flr-250	89	27	.	.	PUNCT
flr-250	90	1	the	the	DET
flr-250	90	2	fact	fact	NOUN
flr-250	90	3	that	that	SCONJ
flr-250	90	4	those	those	PRON
flr-250	90	5	with	with	ADP
flr-250	90	6	more	more	ADJ
flr-250	90	7	expertise	expertise	NOUN
flr-250	90	8	tend	tend	VERB
flr-250	90	9	to	to	PART
flr-250	90	10	have	have	VERB
flr-250	90	11	curves	curve	NOUN
flr-250	90	12	closer	close	ADV
flr-250	90	13	to	to	ADP
flr-250	90	14	the	the	DET
flr-250	90	15	upper	upper	ADJ
flr-250	90	16	left	left	ADJ
flr-250	90	17	corner	corner	NOUN
flr-250	90	18	is	be	AUX
flr-250	90	19	true	true	ADJ
flr-250	90	20	,	,	PUNCT
flr-250	90	21	but	but	CCONJ
flr-250	90	22	as	as	SCONJ
flr-250	90	23	noted	note	VERB
flr-250	90	24	above	above	ADP
flr-250	90	25	much	much	ADJ
flr-250	90	26	depends	depend	VERB
flr-250	90	27	on	on	ADP
flr-250	90	28	the	the	DET
flr-250	90	29	decision	decision	NOUN
flr-250	90	30	thresholds	threshold	VERB
flr-250	90	31	that	that	SCONJ
flr-250	90	32	the	the	DET
flr-250	90	33	individual	individual	ADJ
flr-250	90	34	observer	observer	NOUN
flr-250	90	35	has	have	VERB
flr-250	90	36	for	for	ADP
flr-250	90	37	a	a	DET
flr-250	90	38	given	give	VERB
flr-250	90	39	task	task	NOUN
flr-250	90	40	.	.	PUNCT
flr-250	91	1	each	each	DET
flr-250	91	2	point	point	NOUN
flr-250	91	3	on	on	ADP
flr-250	91	4	a	a	DET
flr-250	91	5	given	give	VERB
flr-250	91	6	curve	curve	NOUN
flr-250	91	7	represents	represent	VERB
flr-250	91	8	a	a	DET
flr-250	91	9	specific	specific	ADJ
flr-250	91	10	tp	tp	NOUN
flr-250	91	11	and	and	CCONJ
flr-250	91	12	fp	fp	X
flr-250	91	13	fraction	fraction	NOUN
flr-250	91	14	or	or	CCONJ
flr-250	91	15	decision	decision	NOUN
flr-250	91	16	threshold	threshold	NOUN
flr-250	91	17	setting	setting	NOUN
flr-250	91	18	.	.	PUNCT
flr-250	92	1	for	for	ADP
flr-250	92	2	example	example	NOUN
flr-250	92	3	,	,	PUNCT
flr-250	92	4	on	on	ADP
flr-250	92	5	the	the	DET
flr-250	92	6	“	"	PUNCT
flr-250	92	7	good	good	ADJ
flr-250	92	8	”	"	PUNCT
flr-250	92	9	curve	curve	NOUN
flr-250	92	10	,	,	PUNCT
flr-250	92	11	the	the	DET
flr-250	92	12	plus	plus	ADJ
flr-250	92	13	sign	sign	NOUN
flr-250	92	14	represents	represent	VERB
flr-250	92	15	a	a	DET
flr-250	92	16	rather	rather	ADV
flr-250	92	17	conservative	conservative	ADJ
flr-250	92	18	decision	decision	NOUN
flr-250	92	19	maker	maker	NOUN
flr-250	92	20	–	–	PUNCT
flr-250	92	21	not	not	PART
flr-250	92	22	reporting	report	VERB
flr-250	92	23	a	a	DET
flr-250	92	24	lot	lot	NOUN
flr-250	92	25	of	of	ADP
flr-250	92	26	targets	target	NOUN
flr-250	92	27	(	(	PUNCT
flr-250	92	28	low	low	ADJ
flr-250	92	29	sensitivity	sensitivity	NOUN
flr-250	92	30	)	)	PUNCT
flr-250	92	31	but	but	CCONJ
flr-250	92	32	also	also	ADV
flr-250	92	33	not	not	PART
flr-250	92	34	making	make	VERB
flr-250	92	35	a	a	DET
flr-250	92	36	lot	lot	NOUN
flr-250	92	37	of	of	ADP
flr-250	92	38	false	false	ADJ
flr-250	92	39	positives	positive	NOUN
flr-250	92	40	(	(	PUNCT
flr-250	92	41	high	high	ADJ
flr-250	92	42	specificity	specificity	NOUN
flr-250	92	43	or	or	CCONJ
flr-250	92	44	low	low	ADJ
flr-250	92	45	1specificity	1specificity	NUM
flr-250	92	46	)	)	PUNCT
flr-250	92	47	.	.	PUNCT
flr-250	93	1	the	the	DET
flr-250	93	2	cross	cross	NOUN
flr-250	93	3	sign	sign	NOUN
flr-250	93	4	represents	represent	VERB
flr-250	93	5	a	a	DET
flr-250	93	6	more	more	ADV
flr-250	93	7	liberal	liberal	ADJ
flr-250	93	8	decision	decision	NOUN
flr-250	93	9	maker	maker	NOUN
flr-250	93	10	–	–	PUNCT
flr-250	93	11	reporting	report	VERB
flr-250	93	12	a	a	DET
flr-250	93	13	lot	lot	NOUN
flr-250	93	14	of	of	ADP
flr-250	93	15	targets	target	NOUN
flr-250	93	16	(	(	PUNCT
flr-250	93	17	high	high	ADJ
flr-250	93	18	sensitivity	sensitivity	NOUN
flr-250	93	19	)	)	PUNCT
flr-250	93	20	but	but	CCONJ
flr-250	93	21	making	make	VERB
flr-250	93	22	a	a	DET
flr-250	93	23	lot	lot	NOUN
flr-250	93	24	of	of	ADP
flr-250	93	25	false	false	ADJ
flr-250	93	26	positives	positive	NOUN
flr-250	93	27	(	(	PUNCT
flr-250	93	28	low	low	ADJ
flr-250	93	29	specificity	specificity	NOUN
flr-250	93	30	or	or	CCONJ
flr-250	93	31	high	high	ADJ
flr-250	93	32	1	1	NUM
flr-250	93	33	-	-	PUNCT
flr-250	93	34	specificity	specificity	NOUN
flr-250	93	35	)	)	PUNCT
flr-250	93	36	.	.	PUNCT
flr-250	94	1	referring	refer	VERB
flr-250	94	2	back	back	ADV
flr-250	94	3	to	to	ADP
flr-250	94	4	the	the	DET
flr-250	94	5	example	example	NOUN
flr-250	94	6	of	of	ADP
flr-250	94	7	the	the	DET
flr-250	94	8	radiologist	radiologist	NOUN
flr-250	94	9	moving	move	VERB
flr-250	94	10	to	to	ADP
flr-250	94	11	arizona	arizona	PROPN
flr-250	94	12	and	and	CCONJ
flr-250	94	13	overcalling	overcalling	NOUN
flr-250	94	14	valley	valley	NOUN
flr-250	94	15	fever	fever	NOUN
flr-250	94	16	as	as	ADP
flr-250	94	17	cancer	cancer	NOUN
flr-250	94	18	,	,	PUNCT
flr-250	94	19	the	the	DET
flr-250	94	20	figure	figure	NOUN
flr-250	94	21	could	could	AUX
flr-250	94	22	just	just	ADV
flr-250	94	23	as	as	ADV
flr-250	94	24	well	well	ADV
flr-250	94	25	represent	represent	VERB
flr-250	94	26	his	his	PRON
flr-250	94	27	/	/	SYM
flr-250	94	28	her	her	PRON
flr-250	94	29	learning	learning	NOUN
flr-250	94	30	or	or	CCONJ
flr-250	94	31	adjustment	adjustment	NOUN
flr-250	94	32	period	period	NOUN
flr-250	94	33	,	,	PUNCT
flr-250	94	34	in	in	ADP
flr-250	94	35	which	which	PRON
flr-250	94	36	the	the	DET
flr-250	94	37	lower	low	ADJ
flr-250	94	38	curve	curve	NOUN
flr-250	94	39	is	be	AUX
flr-250	94	40	when	when	SCONJ
flr-250	94	41	they	they	PRON
flr-250	94	42	first	first	ADV
flr-250	94	43	move	move	VERB
flr-250	94	44	and	and	CCONJ
flr-250	94	45	overall	overall	ADJ
flr-250	94	46	cancers	cancer	NOUN
flr-250	94	47	and	and	CCONJ
flr-250	94	48	the	the	DET
flr-250	94	49	upper	upper	ADJ
flr-250	94	50	curve	curve	NOUN
flr-250	94	51	is	be	AUX
flr-250	94	52	their	their	PRON
flr-250	94	53	performance	performance	NOUN
flr-250	94	54	after	after	ADP
flr-250	94	55	a	a	DET
flr-250	94	56	few	few	ADJ
flr-250	94	57	months	month	NOUN
flr-250	94	58	of	of	ADP
flr-250	94	59	seeing	see	VERB
flr-250	94	60	more	more	ADJ
flr-250	94	61	exemplars	exemplar	NOUN
flr-250	94	62	of	of	ADP
flr-250	94	63	valley	valley	NOUN
flr-250	94	64	fever	fever	NOUN
flr-250	94	65	so	so	ADV
flr-250	94	66	better	well	ADJ
flr-250	94	67	discrimination	discrimination	NOUN
flr-250	94	68	is	be	AUX
flr-250	94	69	possible	possible	ADJ
flr-250	94	70	.	.	PUNCT
flr-250	95	1	figure	figure	NOUN
flr-250	95	2	1	1	NUM
flr-250	95	3	.	.	NOUN
flr-250	95	4	example	example	NOUN
flr-250	95	5	of	of	ADP
flr-250	95	6	a	a	DET
flr-250	95	7	typical	typical	ADJ
flr-250	95	8	roc	roc	PROPN
flr-250	95	9	curve	curve	NOUN
flr-250	95	10	plotting	plotting	NOUN
flr-250	95	11	sensitivity	sensitivity	NOUN
flr-250	95	12	vs	vs	ADP
flr-250	95	13	1	1	NUM
flr-250	95	14	-	-	PUNCT
flr-250	95	15	specificity	specificity	NOUN
flr-250	95	16	.	.	PUNCT
flr-250	96	1	the	the	DET
flr-250	96	2	diagonal	diagonal	ADJ
flr-250	96	3	line	line	NOUN
flr-250	96	4	is	be	AUX
flr-250	96	5	chance	chance	NOUN
flr-250	96	6	performance	performance	NOUN
flr-250	96	7	or	or	CCONJ
flr-250	96	8	guessing	guessing	NOUN
flr-250	96	9	and	and	CCONJ
flr-250	96	10	the	the	DET
flr-250	96	11	curves	curve	NOUN
flr-250	96	12	indicate	indicate	VERB
flr-250	96	13	increasingly	increasingly	ADV
flr-250	96	14	better	well	ADJ
flr-250	96	15	performance	performance	NOUN
flr-250	96	16	moving	move	VERB
flr-250	96	17	to	to	ADP
flr-250	96	18	the	the	DET
flr-250	96	19	upper	upper	ADJ
flr-250	96	20	left	left	ADJ
flr-250	96	21	corner	corner	NOUN
flr-250	96	22	which	which	PRON
flr-250	96	23	represents	represent	VERB
flr-250	96	24	perfect	perfect	ADJ
flr-250	96	25	performance	performance	NOUN
flr-250	96	26	.	.	PUNCT
flr-250	97	1	2.3	2.3	NUM
flr-250	97	2	how	how	SCONJ
flr-250	97	3	to	to	PART
flr-250	97	4	plot	plot	VERB
flr-250	97	5	an	an	DET
flr-250	97	6	roc	roc	PROPN
flr-250	97	7	curve	curve	NOUN
flr-250	97	8	it	it	PRON
flr-250	97	9	is	be	AUX
flr-250	97	10	rare	rare	ADJ
flr-250	97	11	that	that	SCONJ
flr-250	97	12	someone	someone	PRON
flr-250	97	13	actually	actually	ADV
flr-250	97	14	plots	plot	VERB
flr-250	97	15	an	an	DET
flr-250	97	16	roc	roc	PROPN
flr-250	97	17	curve	curve	NOUN
flr-250	97	18	by	by	ADP
flr-250	97	19	hand	hand	NOUN
flr-250	97	20	as	as	SCONJ
flr-250	97	21	there	there	PRON
flr-250	97	22	are	be	VERB
flr-250	97	23	a	a	DET
flr-250	97	24	number	number	NOUN
flr-250	97	25	of	of	ADP
flr-250	97	26	software	software	NOUN
flr-250	97	27	programs	program	NOUN
flr-250	97	28	available	available	ADJ
flr-250	97	29	both	both	PRON
flr-250	97	30	in	in	ADP
flr-250	97	31	commercial	commercial	ADJ
flr-250	97	32	statistical	statistical	ADJ
flr-250	97	33	packages	package	NOUN
flr-250	97	34	and	and	CCONJ
flr-250	97	35	freely	freely	ADV
flr-250	97	36	downloadable	downloadable	ADJ
flr-250	97	37	from	from	ADP
flr-250	97	38	research	research	NOUN
flr-250	97	39	web	web	NOUN
flr-250	97	40	sites	site	NOUN
flr-250	97	41	.	.	PUNCT
flr-250	98	1	however	however	ADV
flr-250	98	2	,	,	PUNCT
flr-250	98	3	it	it	PRON
flr-250	98	4	is	be	AUX
flr-250	98	5	useful	useful	ADJ
flr-250	98	6	to	to	PART
flr-250	98	7	see	see	VERB
flr-250	98	8	how	how	SCONJ
flr-250	98	9	the	the	DET
flr-250	98	10	confidence	confidence	NOUN
flr-250	98	11	ratings	rating	NOUN
flr-250	98	12	discussed	discuss	VERB
flr-250	98	13	above	above	ADP
flr-250	98	14	lead	lead	NOUN
flr-250	98	15	to	to	ADP
flr-250	98	16	the	the	DET
flr-250	98	17	generation	generation	NOUN
flr-250	98	18	of	of	ADP
flr-250	98	19	an	an	DET
flr-250	98	20	roc	roc	PROPN
flr-250	98	21	curve	curve	NOUN
flr-250	98	22	.	.	PUNCT
flr-250	99	1	thus	thus	ADV
flr-250	99	2	,	,	PUNCT
flr-250	99	3	suppose	suppose	VERB
flr-250	99	4	you	you	PRON
flr-250	99	5	ran	run	VERB
flr-250	99	6	a	a	DET
flr-250	99	7	visual	visual	ADJ
flr-250	99	8	detection	detection	NOUN
flr-250	99	9	experiment	experiment	NOUN
flr-250	99	10	and	and	CCONJ
flr-250	99	11	the	the	DET
flr-250	99	12	observer	observer	NOUN
flr-250	99	13	was	be	AUX
flr-250	99	14	required	require	VERB
flr-250	99	15	to	to	PART
flr-250	99	16	identify	identify	VERB
flr-250	99	17	whether	whether	SCONJ
flr-250	99	18	or	or	CCONJ
flr-250	99	19	not	not	PART
flr-250	99	20	a	a	DET
flr-250	99	21	given	give	VERB
flr-250	99	22	image	image	NOUN
flr-250	99	23	had	have	VERB
flr-250	99	24	a	a	DET
flr-250	99	25	target	target	NOUN
flr-250	99	26	(	(	PUNCT
flr-250	99	27	n	n	NOUN
flr-250	99	28	=	=	SYM
flr-250	99	29	50	50	NUM
flr-250	99	30	)	)	PUNCT
flr-250	99	31	or	or	CCONJ
flr-250	99	32	not	not	PART
flr-250	99	33	(	(	PUNCT
flr-250	99	34	n	n	NOUN
flr-250	99	35	=	=	SYM
flr-250	99	36	50	50	NUM
flr-250	99	37	)	)	PUNCT
flr-250	99	38	and	and	CCONJ
flr-250	99	39	then	then	ADV
flr-250	99	40	report	report	VERB
flr-250	99	41	confidence	confidence	NOUN
flr-250	99	42	in	in	ADP
flr-250	99	43	that	that	DET
flr-250	99	44	decision	decision	NOUN
flr-250	99	45	as	as	ADP
flr-250	99	46	definite	definite	ADJ
flr-250	99	47	,	,	PUNCT
flr-250	99	48	probable	probable	ADJ
flr-250	99	49	or	or	CCONJ
flr-250	99	50	possible	possible	ADJ
flr-250	99	51	.	.	PUNCT
flr-250	100	1	this	this	PRON
flr-250	100	2	yields	yield	VERB
flr-250	100	3	6	6	NUM
flr-250	100	4	categories	category	NOUN
flr-250	100	5	of	of	ADP
flr-250	100	6	responses	response	NOUN
flr-250	100	7	where	where	SCONJ
flr-250	100	8	1	1	NUM
flr-250	100	9	=	=	SYM
flr-250	100	10	absent	absent	ADJ
flr-250	100	11	,	,	PUNCT
flr-250	100	12	definite	definite	ADJ
flr-250	100	13	and	and	CCONJ
flr-250	100	14	6	6	NUM
flr-250	100	15	=	=	SYM
flr-250	100	16	present	present	ADJ
flr-250	100	17	,	,	PUNCT
flr-250	100	18	definite	definite	ADJ
flr-250	100	19	.	.	PUNCT
flr-250	101	1	you	you	PRON
flr-250	101	2	can	can	AUX
flr-250	101	3	then	then	ADV
flr-250	101	4	create	create	VERB
flr-250	101	5	a	a	DET
flr-250	101	6	table	table	NOUN
flr-250	101	7	(	(	PUNCT
flr-250	101	8	table	table	NOUN
flr-250	101	9	1	1	NUM
flr-250	101	10	)	)	PUNCT
flr-250	101	11	showing	show	VERB
flr-250	101	12	the	the	DET
flr-250	101	13	distribution	distribution	NOUN
flr-250	101	14	of	of	ADP
flr-250	101	15	responses	response	NOUN
flr-250	101	16	as	as	ADP
flr-250	101	17	a	a	DET
flr-250	101	18	function	function	NOUN
flr-250	101	19	of	of	ADP
flr-250	101	20	truth	truth	NOUN
flr-250	101	21	(	(	PUNCT
flr-250	101	22	whether	whether	SCONJ
flr-250	101	23	or	or	CCONJ
flr-250	101	24	not	not	PART
flr-250	101	25	the	the	DET
flr-250	101	26	image	image	NOUN
flr-250	101	27	actually	actually	ADV
flr-250	101	28	contains	contain	VERB
flr-250	101	29	a	a	DET
flr-250	101	30	target	target	NOUN
flr-250	101	31	)	)	PUNCT
flr-250	101	32	.	.	PUNCT
flr-250	102	1	it	it	PRON
flr-250	102	2	should	should	AUX
flr-250	102	3	be	be	AUX
flr-250	102	4	noted	note	VERB
flr-250	102	5	that	that	SCONJ
flr-250	102	6	continuous	continuous	ADJ
flr-250	102	7	rating	rating	NOUN
flr-250	102	8	scales	scale	NOUN
flr-250	102	9	(	(	PUNCT
flr-250	102	10	e.g.	e.g.	ADV
flr-250	102	11	,	,	PUNCT
flr-250	102	12	0	0	NUM
flr-250	102	13	–	–	PUNCT
flr-250	102	14	100	100	NUM
flr-250	102	15	)	)	PUNCT
flr-250	102	16	can	can	AUX
flr-250	102	17	be	be	AUX
flr-250	102	18	used	use	VERB
flr-250	102	19	as	as	ADV
flr-250	102	20	well	well	ADV
flr-250	102	21	and	and	CCONJ
flr-250	102	22	methods	method	NOUN
flr-250	102	23	exist	exist	VERB
flr-250	102	24	for	for	ADP
flr-250	102	25	generating	generate	VERB
flr-250	102	26	operating	operating	NOUN
flr-250	102	27	points	point	NOUN
flr-250	102	28	(	(	PUNCT
flr-250	102	29	decision	decision	NOUN
flr-250	102	30	thresholds	threshold	NOUN
flr-250	102	31	)	)	PUNCT
flr-250	102	32	and	and	CCONJ
flr-250	102	33	plotting	plot	VERB
flr-250	102	34	these	these	PRON
flr-250	102	35	as	as	ADV
flr-250	102	36	well	well	ADV
flr-250	102	37	[	[	X
flr-250	102	38	22	22	NUM
flr-250	102	39	]	]	PUNCT
flr-250	102	40	.	.	PUNCT
flr-250	103	1	krupinski	krupinski	NOUN
flr-250	103	2	|	|	ADV
flr-250	103	3	f	f	NOUN
flr-250	103	4	l	l	NOUN
flr-250	103	5	r	r	NOUN
flr-250	103	6	36	36	NUM
flr-250	103	7	table	table	NOUN
flr-250	103	8	1	1	NUM
flr-250	103	9	an	an	DET
flr-250	103	10	example	example	NOUN
flr-250	103	11	of	of	ADP
flr-250	103	12	the	the	DET
flr-250	103	13	distribution	distribution	NOUN
flr-250	103	14	of	of	ADP
flr-250	103	15	confidence	confidence	NOUN
flr-250	103	16	scores	score	NOUN
flr-250	103	17	for	for	ADP
flr-250	103	18	a	a	DET
flr-250	103	19	subject	subject	NOUN
flr-250	103	20	in	in	ADP
flr-250	103	21	an	an	DET
flr-250	103	22	observer	observer	NOUN
flr-250	103	23	performance	performance	NOUN
flr-250	103	24	study	study	NOUN
flr-250	103	25	with	with	ADP
flr-250	103	26	a	a	DET
flr-250	103	27	6point	6point	NUM
flr-250	103	28	confidence	confidence	NOUN
flr-250	103	29	scale	scale	NOUN
flr-250	103	30	and	and	CCONJ
flr-250	103	31	images	image	NOUN
flr-250	103	32	with	with	ADP
flr-250	103	33	a	a	DET
flr-250	103	34	target	target	NOUN
flr-250	103	35	present	present	ADJ
flr-250	103	36	or	or	CCONJ
flr-250	103	37	absent	absent	ADJ
flr-250	103	38	(	(	PUNCT
flr-250	103	39	truth	truth	NOUN
flr-250	103	40	)	)	PUNCT
flr-250	103	41	.	.	PUNCT
flr-250	104	1	truth	truth	NOUN
flr-250	104	2	1	1	NUM
flr-250	104	3	2	2	NUM
flr-250	104	4	3	3	NUM
flr-250	104	5	4	4	NUM
flr-250	104	6	5	5	NUM
flr-250	104	7	6	6	NUM
flr-250	104	8	present	present	ADJ
flr-250	104	9	2	2	NUM
flr-250	104	10	3	3	NUM
flr-250	104	11	2	2	NUM
flr-250	104	12	5	5	NUM
flr-250	104	13	20	20	NUM
flr-250	104	14	18	18	NUM
flr-250	104	15	absent	absent	ADJ
flr-250	104	16	16	16	NUM
flr-250	104	17	15	15	NUM
flr-250	104	18	10	10	NUM
flr-250	104	19	4	4	NUM
flr-250	104	20	3	3	NUM
flr-250	104	21	2	2	NUM
flr-250	104	22	the	the	DET
flr-250	104	23	sensitivity	sensitivity	NOUN
flr-250	104	24	,	,	PUNCT
flr-250	104	25	specificity	specificity	NOUN
flr-250	104	26	and	and	CCONJ
flr-250	104	27	fp	fp	PRON
flr-250	104	28	fraction	fraction	NOUN
flr-250	104	29	can	can	AUX
flr-250	104	30	then	then	ADV
flr-250	104	31	be	be	AUX
flr-250	104	32	determined	determine	VERB
flr-250	104	33	at	at	ADP
flr-250	104	34	each	each	DET
flr-250	104	35	threshold	threshold	NOUN
flr-250	104	36	or	or	CCONJ
flr-250	104	37	cutoff	cutoff	NOUN
flr-250	104	38	point	point	NOUN
flr-250	104	39	as	as	ADP
flr-250	104	40	in	in	ADP
flr-250	104	41	table	table	NOUN
flr-250	104	42	2	2	NUM
flr-250	104	43	.	.	PUNCT
flr-250	104	44	table	table	NOUN
flr-250	104	45	2	2	NUM
flr-250	104	46	sensitivity	sensitivity	NOUN
flr-250	104	47	,	,	PUNCT
flr-250	104	48	specificity	specificity	NOUN
flr-250	104	49	and	and	CCONJ
flr-250	104	50	fp	fp	PRON
flr-250	104	51	fraction	fraction	NOUN
flr-250	104	52	can	can	AUX
flr-250	104	53	then	then	ADV
flr-250	104	54	be	be	AUX
flr-250	104	55	determined	determine	VERB
flr-250	104	56	at	at	ADP
flr-250	104	57	each	each	DET
flr-250	104	58	threshold	threshold	NOUN
flr-250	104	59	or	or	CCONJ
flr-250	104	60	cutoff	cutoff	NOUN
flr-250	104	61	point	point	NOUN
flr-250	104	62	.	.	PUNCT
flr-250	105	1	result	result	VERB
flr-250	105	2	positive	positive	ADJ
flr-250	105	3	is	be	AUX
flr-250	105	4	>	>	X
flr-250	105	5	sensitivity	sensitivity	NOUN
flr-250	105	6	specificity	specificity	NOUN
flr-250	105	7	fp	fp	PROPN
flr-250	105	8	fraction	fraction	NOUN
flr-250	105	9	2	2	NUM
flr-250	105	10	probably	probably	ADV
flr-250	105	11	absent	absent	ADJ
flr-250	105	12	0.96	0.96	NUM
flr-250	105	13	(	(	PUNCT
flr-250	105	14	48/50	48/50	NUM
flr-250	105	15	)	)	PUNCT
flr-250	105	16	0.32	0.32	NUM
flr-250	105	17	(	(	PUNCT
flr-250	105	18	16/50	16/50	NUM
flr-250	105	19	)	)	PUNCT
flr-250	105	20	0.68	0.68	NUM
flr-250	105	21	3	3	NUM
flr-250	105	22	possibly	possibly	ADV
flr-250	105	23	absent	absent	ADJ
flr-250	105	24	0.90	0.90	NUM
flr-250	105	25	(	(	PUNCT
flr-250	105	26	45/50	45/50	NUM
flr-250	105	27	)	)	PUNCT
flr-250	105	28	0.62	0.62	NUM
flr-250	105	29	(	(	PUNCT
flr-250	105	30	31/50	31/50	NUM
flr-250	105	31	)	)	PUNCT
flr-250	105	32	0.38	0.38	NUM
flr-250	105	33	4	4	NUM
flr-250	105	34	possibly	possibly	ADV
flr-250	105	35	present	present	VERB
flr-250	105	36	0.86	0.86	NUM
flr-250	105	37	(	(	PUNCT
flr-250	105	38	43/50	43/50	NUM
flr-250	105	39	)	)	PUNCT
flr-250	105	40	0.82	0.82	NUM
flr-250	105	41	(	(	PUNCT
flr-250	105	42	41/50	41/50	NUM
flr-250	105	43	)	)	PUNCT
flr-250	105	44	0.18	0.18	NUM
flr-250	105	45	5	5	NUM
flr-250	105	46	probably	probably	ADV
flr-250	105	47	present	present	ADJ
flr-250	105	48	0.76	0.76	NUM
flr-250	105	49	(	(	PUNCT
flr-250	105	50	38/50	38/50	NUM
flr-250	105	51	)	)	PUNCT
flr-250	105	52	0.90	0.90	NUM
flr-250	105	53	(	(	PUNCT
flr-250	105	54	45/50	45/50	NUM
flr-250	105	55	)	)	PUNCT
flr-250	105	56	0.10	0.10	NUM
flr-250	105	57	6	6	NUM
flr-250	105	58	definitely	definitely	ADV
flr-250	105	59	present	present	ADJ
flr-250	105	60	0.36	0.36	NUM
flr-250	105	61	(	(	PUNCT
flr-250	105	62	18/50	18/50	NUM
flr-250	105	63	)	)	PUNCT
flr-250	105	64	0.96	0.96	NUM
flr-250	105	65	(	(	PUNCT
flr-250	105	66	48/50	48/50	NUM
flr-250	105	67	)	)	PUNCT
flr-250	105	68	0.04	0.04	NUM
flr-250	105	69	the	the	DET
flr-250	105	70	roc	roc	PROPN
flr-250	105	71	plot	plot	NOUN
flr-250	105	72	can	can	AUX
flr-250	105	73	then	then	ADV
flr-250	105	74	be	be	AUX
flr-250	105	75	generated	generate	VERB
flr-250	105	76	plotting	plot	VERB
flr-250	105	77	sensitivity	sensitivity	NOUN
flr-250	105	78	on	on	ADP
flr-250	105	79	the	the	DET
flr-250	105	80	y	y	NOUN
flr-250	105	81	-	-	PUNCT
flr-250	105	82	axis	axis	NOUN
flr-250	105	83	and	and	CCONJ
flr-250	105	84	1	1	NUM
flr-250	105	85	-	-	PUNCT
flr-250	105	86	specificity	specificity	NOUN
flr-250	105	87	(	(	PUNCT
flr-250	105	88	fp	fp	INTJ
flr-250	105	89	fraction	fraction	NOUN
flr-250	105	90	)	)	PUNCT
flr-250	105	91	on	on	ADP
flr-250	105	92	the	the	DET
flr-250	105	93	x	x	NOUN
flr-250	105	94	-	-	NOUN
flr-250	105	95	axis	axis	ADJ
flr-250	105	96	.	.	PUNCT
flr-250	106	1	figure	figure	NOUN
flr-250	106	2	2	2	NUM
flr-250	106	3	.	.	PUNCT
flr-250	107	1	roc	roc	PROPN
flr-250	107	2	curve	curve	NOUN
flr-250	107	3	generated	generate	VERB
flr-250	107	4	from	from	ADP
flr-250	107	5	the	the	DET
flr-250	107	6	data	datum	NOUN
flr-250	107	7	in	in	ADP
flr-250	107	8	table	table	NOUN
flr-250	107	9	2	2	NUM
flr-250	107	10	.	.	PUNCT
flr-250	108	1	in	in	ADP
flr-250	108	2	terms	term	NOUN
flr-250	108	3	of	of	ADP
flr-250	108	4	fitting	fit	VERB
flr-250	108	5	an	an	DET
flr-250	108	6	actual	actual	ADJ
flr-250	108	7	curve	curve	NOUN
flr-250	108	8	to	to	ADP
flr-250	108	9	the	the	DET
flr-250	108	10	data	data	NOUN
flr-250	108	11	points	point	NOUN
flr-250	108	12	in	in	ADP
flr-250	108	13	the	the	DET
flr-250	108	14	roc	roc	PROPN
flr-250	108	15	plot	plot	NOUN
flr-250	108	16	,	,	PUNCT
flr-250	108	17	there	there	PRON
flr-250	108	18	are	be	VERB
flr-250	108	19	a	a	DET
flr-250	108	20	variety	variety	NOUN
flr-250	108	21	of	of	ADP
flr-250	108	22	methods	method	NOUN
flr-250	108	23	.	.	PUNCT
flr-250	109	1	simply	simply	ADV
flr-250	109	2	“	"	PUNCT
flr-250	109	3	connecting	connect	VERB
flr-250	109	4	the	the	DET
flr-250	109	5	dots	dot	NOUN
flr-250	109	6	”	"	PUNCT
flr-250	109	7	is	be	AUX
flr-250	109	8	the	the	DET
flr-250	109	9	empirically	empirically	ADV
flr-250	109	10	based	base	VERB
flr-250	109	11	version	version	NOUN
flr-250	109	12	but	but	CCONJ
flr-250	109	13	it	it	PRON
flr-250	109	14	creates	create	VERB
flr-250	109	15	a	a	DET
flr-250	109	16	stepped	step	VERB
flr-250	109	17	or	or	CCONJ
flr-250	109	18	jagged	jagged	ADJ
flr-250	109	19	plot	plot	NOUN
flr-250	109	20	.	.	PUNCT
flr-250	110	1	a	a	DET
flr-250	110	2	smooth	smooth	ADJ
flr-250	110	3	curve	curve	NOUN
flr-250	110	4	reflecting	reflect	VERB
flr-250	110	5	the	the	DET
flr-250	110	6	theoretical	theoretical	ADJ
flr-250	110	7	“	"	PUNCT
flr-250	110	8	true	true	ADJ
flr-250	110	9	”	"	PUNCT
flr-250	110	10	curve	curve	NOUN
flr-250	110	11	is	be	AUX
flr-250	110	12	much	much	ADV
flr-250	110	13	more	more	ADV
flr-250	110	14	desirable	desirable	ADJ
flr-250	110	15	.	.	PUNCT
flr-250	111	1	there	there	PRON
flr-250	111	2	are	be	VERB
flr-250	111	3	basically	basically	ADV
flr-250	111	4	two	two	NUM
flr-250	111	5	ways	way	NOUN
flr-250	111	6	to	to	PART
flr-250	111	7	approach	approach	VERB
flr-250	111	8	generating	generate	VERB
flr-250	111	9	the	the	DET
flr-250	111	10	curve	curve	NOUN
flr-250	111	11	–	–	PUNCT
flr-250	111	12	parametric	parametric	ADJ
flr-250	111	13	and	and	CCONJ
flr-250	111	14	non	non	ADJ
flr-250	111	15	-	-	ADJ
flr-250	111	16	parametric	parametric	ADJ
flr-250	111	17	[	[	X
flr-250	111	18	22	22	NUM
flr-250	111	19	-	-	SYM
flr-250	111	20	28	28	NUM
flr-250	111	21	]	]	PUNCT
flr-250	111	22	.	.	PUNCT
flr-250	112	1	the	the	DET
flr-250	112	2	non	non	ADJ
flr-250	112	3	-	-	ADJ
flr-250	112	4	parametric	parametric	ADJ
flr-250	112	5	approach	approach	NOUN
flr-250	112	6	does	do	AUX
flr-250	112	7	not	not	PART
flr-250	112	8	have	have	VERB
flr-250	112	9	any	any	DET
flr-250	112	10	assumptions	assumption	NOUN
flr-250	112	11	about	about	ADP
flr-250	112	12	the	the	DET
flr-250	112	13	structure	structure	NOUN
flr-250	112	14	of	of	ADP
flr-250	112	15	the	the	DET
flr-250	112	16	underlying	underlie	VERB
flr-250	112	17	data	datum	NOUN
flr-250	112	18	distribution	distribution	NOUN
flr-250	112	19	and	and	CCONJ
flr-250	112	20	essentially	essentially	ADV
flr-250	112	21	smooths	smooth	VERB
flr-250	112	22	the	the	DET
flr-250	112	23	histograms	histogram	NOUN
flr-250	112	24	of	of	ADP
flr-250	112	25	the	the	DET
flr-250	112	26	output	output	NOUN
flr-250	112	27	data	datum	NOUN
flr-250	112	28	for	for	ADP
flr-250	112	29	the	the	DET
flr-250	112	30	two	two	NUM
flr-250	112	31	classes	class	NOUN
flr-250	112	32	.	.	PUNCT
flr-250	113	1	the	the	DET
flr-250	113	2	parametric	parametric	ADJ
flr-250	113	3	methods	method	NOUN
flr-250	113	4	do	do	AUX
flr-250	113	5	rely	rely	VERB
flr-250	113	6	on	on	ADP
flr-250	113	7	the	the	DET
flr-250	113	8	validity	validity	NOUN
flr-250	113	9	of	of	ADP
flr-250	113	10	the	the	DET
flr-250	113	11	underlying	underlie	VERB
flr-250	113	12	distribution	distribution	NOUN
flr-250	113	13	assumptions	assumption	NOUN
flr-250	113	14	.	.	PUNCT
flr-250	114	1	most	most	ADJ
flr-250	114	2	researchers	researcher	NOUN
flr-250	114	3	prefer	prefer	VERB
flr-250	114	4	the	the	DET
flr-250	114	5	parametric	parametric	ADJ
flr-250	114	6	approaches	approach	NOUN
flr-250	114	7	and	and	CCONJ
flr-250	114	8	much	much	ADJ
flr-250	114	9	of	of	ADP
flr-250	114	10	the	the	DET
flr-250	114	11	available	available	ADJ
flr-250	114	12	software	software	NOUN
flr-250	114	13	uses	use	VERB
flr-250	114	14	these	these	DET
flr-250	114	15	approaches	approach	NOUN
flr-250	114	16	as	as	ADV
flr-250	114	17	well	well	ADV
flr-250	114	18	.	.	PUNCT
flr-250	115	1	2.4	2.4	NUM
flr-250	115	2	interpreting	interpret	VERB
flr-250	115	3	the	the	DET
flr-250	115	4	roc	roc	PROPN
flr-250	115	5	curve	curve	NOUN
flr-250	115	6	there	there	PRON
flr-250	115	7	are	be	VERB
flr-250	115	8	a	a	DET
flr-250	115	9	few	few	ADJ
flr-250	115	10	key	key	ADJ
flr-250	115	11	metrics	metric	NOUN
flr-250	115	12	used	use	VERB
flr-250	115	13	to	to	PART
flr-250	115	14	interpret	interpret	VERB
flr-250	115	15	the	the	DET
flr-250	115	16	roc	roc	PROPN
flr-250	115	17	curve	curve	NOUN
flr-250	115	18	and	and	CCONJ
flr-250	115	19	characterize	characterize	VERB
flr-250	115	20	observer	observer	NOUN
flr-250	115	21	performance	performance	NOUN
flr-250	115	22	.	.	PUNCT
flr-250	116	1	the	the	DET
flr-250	116	2	most	most	ADV
flr-250	116	3	common	common	ADJ
flr-250	116	4	one	one	NUM
flr-250	116	5	is	be	AUX
flr-250	116	6	the	the	DET
flr-250	116	7	area	area	NOUN
flr-250	116	8	under	under	ADP
flr-250	116	9	the	the	DET
flr-250	116	10	curve	curve	NOUN
flr-250	116	11	(	(	PUNCT
flr-250	116	12	auc	auc	NOUN
flr-250	116	13	or	or	CCONJ
flr-250	116	14	az	az	PROPN
flr-250	116	15	)	)	PUNCT
flr-250	116	16	.	.	PUNCT
flr-250	117	1	as	as	SCONJ
flr-250	117	2	noted	note	VERB
flr-250	117	3	above	above	ADP
flr-250	117	4	the	the	DET
flr-250	117	5	diagonal	diagonal	ADJ
flr-250	117	6	line	line	NOUN
flr-250	117	7	in	in	ADP
flr-250	117	8	the	the	DET
flr-250	117	9	roc	roc	PROPN
flr-250	117	10	plot	plot	NOUN
flr-250	117	11	represents	represent	VERB
flr-250	117	12	chance	chance	NOUN
flr-250	117	13	or	or	CCONJ
flr-250	117	14	guessing	guess	VERB
flr-250	117	15	performance	performance	NOUN
flr-250	117	16	and	and	CCONJ
flr-250	117	17	it	it	PRON
flr-250	117	18	clearly	clearly	ADV
flr-250	117	19	divides	divide	VERB
flr-250	117	20	the	the	DET
flr-250	117	21	roc	roc	PROPN
flr-250	117	22	space	space	NOUN
flr-250	117	23	into	into	ADP
flr-250	117	24	two	two	NUM
flr-250	117	25	halves	half	NOUN
flr-250	117	26	thus	thus	ADV
flr-250	117	27	representing	represent	VERB
flr-250	117	28	an	an	DET
flr-250	117	29	auc	auc	NOUN
flr-250	117	30	of	of	ADP
flr-250	117	31	0.5	0.5	NUM
flr-250	117	32	.	.	PUNCT
flr-250	118	1	the	the	DET
flr-250	118	2	top	top	ADJ
flr-250	118	3	left	left	ADJ
flr-250	118	4	corner	corner	NOUN
flr-250	118	5	is	be	AUX
flr-250	118	6	perfect	perfect	ADJ
flr-250	118	7	performance	performance	NOUN
flr-250	118	8	and	and	CCONJ
flr-250	118	9	encompasses	encompass	VERB
flr-250	118	10	all	all	PRON
flr-250	118	11	of	of	ADP
flr-250	118	12	the	the	DET
flr-250	118	13	area	area	NOUN
flr-250	118	14	krupinski	krupinski	NOUN
flr-250	118	15	|	|	ADV
flr-250	118	16	f	f	NOUN
flr-250	118	17	l	l	NOUN
flr-250	118	18	r	r	NOUN
flr-250	118	19	37	37	NUM
flr-250	118	20	below	below	ADP
flr-250	118	21	it	it	PRON
flr-250	118	22	,	,	PUNCT
flr-250	118	23	thus	thus	ADV
flr-250	118	24	auc	auc	NOUN
flr-250	118	25	is	be	AUX
flr-250	118	26	1.0	1.0	NUM
flr-250	118	27	.	.	PUNCT
flr-250	119	1	any	any	DET
flr-250	119	2	curve	curve	NOUN
flr-250	119	3	lying	lie	VERB
flr-250	119	4	between	between	ADP
flr-250	119	5	chance	chance	NOUN
flr-250	119	6	and	and	CCONJ
flr-250	119	7	perfect	perfect	ADJ
flr-250	119	8	performance	performance	NOUN
flr-250	119	9	will	will	AUX
flr-250	119	10	have	have	VERB
flr-250	119	11	a	a	DET
flr-250	119	12	value	value	NOUN
flr-250	119	13	between	between	ADP
flr-250	119	14	0	0	NUM
flr-250	119	15	and	and	CCONJ
flr-250	119	16	1.0	1.0	NUM
flr-250	119	17	,	,	PUNCT
flr-250	119	18	with	with	ADP
flr-250	119	19	better	well	ADJ
flr-250	119	20	performance	performance	NOUN
flr-250	119	21	having	have	VERB
flr-250	119	22	values	value	NOUN
flr-250	119	23	closer	close	ADV
flr-250	119	24	to	to	ADP
flr-250	119	25	1.0	1.0	NUM
flr-250	119	26	.	.	PUNCT
flr-250	120	1	as	as	ADP
flr-250	120	2	with	with	ADP
flr-250	120	3	generation	generation	NOUN
flr-250	120	4	of	of	ADP
flr-250	120	5	the	the	DET
flr-250	120	6	curve	curve	NOUN
flr-250	120	7	itself	itself	PRON
flr-250	120	8	,	,	PUNCT
flr-250	120	9	there	there	PRON
flr-250	120	10	are	be	VERB
flr-250	120	11	a	a	DET
flr-250	120	12	variety	variety	NOUN
flr-250	120	13	of	of	ADP
flr-250	120	14	methods	method	NOUN
flr-250	120	15	to	to	PART
flr-250	120	16	calculate	calculate	VERB
flr-250	120	17	auc	auc	NOUN
flr-250	121	1	[	[	X
flr-250	121	2	22	22	NUM
flr-250	121	3	-	-	SYM
flr-250	121	4	29	29	NUM
flr-250	121	5	]	]	PUNCT
flr-250	121	6	and	and	CCONJ
flr-250	121	7	most	most	ADJ
flr-250	121	8	programs	program	NOUN
flr-250	121	9	use	use	VERB
flr-250	121	10	one	one	NUM
flr-250	121	11	of	of	ADP
flr-250	121	12	these	these	DET
flr-250	121	13	methods	method	NOUN
flr-250	121	14	.	.	PUNCT
flr-250	122	1	partial	partial	ADJ
flr-250	122	2	auc	auc	NOUN
flr-250	122	3	acknowledges	acknowledge	VERB
flr-250	122	4	that	that	SCONJ
flr-250	122	5	the	the	DET
flr-250	122	6	more	more	ADV
flr-250	122	7	traditional	traditional	ADJ
flr-250	122	8	auc	auc	NOUN
flr-250	122	9	is	be	AUX
flr-250	122	10	often	often	ADV
flr-250	122	11	not	not	PART
flr-250	122	12	appropriate	appropriate	ADJ
flr-250	122	13	,	,	PUNCT
flr-250	122	14	as	as	SCONJ
flr-250	122	15	not	not	PART
flr-250	122	16	all	all	DET
flr-250	122	17	decision	decision	NOUN
flr-250	122	18	thresholds	threshold	NOUN
flr-250	122	19	or	or	CCONJ
flr-250	122	20	operating	operating	NOUN
flr-250	122	21	points	point	NOUN
flr-250	122	22	are	be	AUX
flr-250	122	23	equally	equally	ADV
flr-250	122	24	important	important	ADJ
flr-250	123	1	[	[	X
flr-250	123	2	30	30	NUM
flr-250	123	3	-	-	SYM
flr-250	123	4	31	31	NUM
flr-250	123	5	]	]	PUNCT
flr-250	123	6	.	.	PUNCT
flr-250	124	1	in	in	ADP
flr-250	124	2	other	other	ADJ
flr-250	124	3	words	word	NOUN
flr-250	124	4	,	,	PUNCT
flr-250	124	5	in	in	ADP
flr-250	124	6	real	real	ADJ
flr-250	124	7	life	life	NOUN
flr-250	124	8	observers	observer	NOUN
flr-250	124	9	may	may	AUX
flr-250	124	10	not	not	PART
flr-250	124	11	actually	actually	ADV
flr-250	124	12	operate	operate	VERB
flr-250	124	13	at	at	ADP
flr-250	124	14	certain	certain	ADJ
flr-250	124	15	thresholds	threshold	NOUN
flr-250	124	16	for	for	ADP
flr-250	124	17	one	one	NUM
flr-250	124	18	reason	reason	NOUN
flr-250	124	19	or	or	CCONJ
flr-250	124	20	another	another	PRON
flr-250	124	21	.	.	PUNCT
flr-250	125	1	in	in	ADP
flr-250	125	2	medicine	medicine	NOUN
flr-250	125	3	for	for	ADP
flr-250	125	4	example	example	NOUN
flr-250	125	5	,	,	PUNCT
flr-250	125	6	a	a	DET
flr-250	125	7	diagnostic	diagnostic	ADJ
flr-250	125	8	test	test	NOUN
flr-250	125	9	with	with	ADP
flr-250	125	10	low	low	ADJ
flr-250	125	11	specificity	specificity	NOUN
flr-250	125	12	(	(	PUNCT
flr-250	125	13	a	a	DET
flr-250	125	14	high	high	ADJ
flr-250	125	15	false	false	ADJ
flr-250	125	16	positive	positive	ADJ
flr-250	125	17	rate	rate	NOUN
flr-250	125	18	)	)	PUNCT
flr-250	125	19	may	may	AUX
flr-250	125	20	not	not	PART
flr-250	125	21	be	be	AUX
flr-250	125	22	clinically	clinically	ADV
flr-250	125	23	acceptable	acceptable	ADJ
flr-250	125	24	.	.	PUNCT
flr-250	126	1	in	in	ADP
flr-250	126	2	this	this	DET
flr-250	126	3	case	case	NOUN
flr-250	126	4	it	it	PRON
flr-250	126	5	may	may	AUX
flr-250	126	6	be	be	AUX
flr-250	126	7	useful	useful	ADJ
flr-250	126	8	to	to	PART
flr-250	126	9	select	select	VERB
flr-250	126	10	a	a	DET
flr-250	126	11	given	give	VERB
flr-250	126	12	(	(	PUNCT
flr-250	126	13	acceptable	acceptable	ADJ
flr-250	126	14	)	)	PUNCT
flr-250	126	15	fp	fp	NOUN
flr-250	126	16	rate	rate	NOUN
flr-250	126	17	,	,	PUNCT
flr-250	126	18	determine	determine	VERB
flr-250	126	19	its	its	PRON
flr-250	126	20	associated	associated	ADJ
flr-250	126	21	sensitivity	sensitivity	NOUN
flr-250	126	22	(	(	PUNCT
flr-250	126	23	tp	tp	NOUN
flr-250	126	24	rate	rate	NOUN
flr-250	126	25	)	)	PUNCT
flr-250	126	26	,	,	PUNCT
flr-250	126	27	and	and	CCONJ
flr-250	126	28	then	then	ADV
flr-250	126	29	calculate	calculate	VERB
flr-250	126	30	the	the	DET
flr-250	126	31	area	area	NOUN
flr-250	126	32	under	under	ADP
flr-250	126	33	the	the	DET
flr-250	126	34	curve	curve	NOUN
flr-250	126	35	only	only	ADV
flr-250	126	36	up	up	ADP
flr-250	126	37	to	to	ADP
flr-250	126	38	that	that	DET
flr-250	126	39	operating	operating	NOUN
flr-250	126	40	point	point	NOUN
flr-250	126	41	(	(	PUNCT
flr-250	126	42	i.e.	i.e.	X
flr-250	126	43	,	,	PUNCT
flr-250	126	44	capturing	capture	VERB
flr-250	126	45	only	only	ADV
flr-250	126	46	a	a	DET
flr-250	126	47	part	part	NOUN
flr-250	126	48	of	of	ADP
flr-250	126	49	the	the	DET
flr-250	126	50	total	total	ADJ
flr-250	126	51	auc	auc	NOUN
flr-250	126	52	)	)	PUNCT
flr-250	126	53	.	.	PUNCT
flr-250	127	1	partial	partial	ADJ
flr-250	127	2	auc	auc	NOUN
flr-250	127	3	is	be	AUX
flr-250	127	4	very	very	ADV
flr-250	127	5	common	common	ADJ
flr-250	127	6	in	in	ADP
flr-250	127	7	the	the	DET
flr-250	127	8	development	development	NOUN
flr-250	127	9	of	of	ADP
flr-250	127	10	computer	computer	NOUN
flr-250	127	11	-	-	PUNCT
flr-250	127	12	aided	aid	VERB
flr-250	127	13	detection	detection	NOUN
flr-250	127	14	and	and	CCONJ
flr-250	127	15	discrimination	discrimination	NOUN
flr-250	127	16	algorithms	algorithm	NOUN
flr-250	127	17	for	for	ADP
flr-250	127	18	medical	medical	ADJ
flr-250	127	19	imaging	imaging	NOUN
flr-250	127	20	.	.	PUNCT
flr-250	128	1	other	other	ADJ
flr-250	128	2	metrics	metric	NOUN
flr-250	128	3	used	use	VERB
flr-250	128	4	less	less	ADV
flr-250	128	5	often	often	ADV
flr-250	128	6	are	be	AUX
flr-250	128	7	d	d	PRON
flr-250	128	8	´	´	NOUN
flr-250	128	9	,	,	PUNCT
flr-250	128	10	de	de	ADJ
flr-250	128	11	´	´	NOUN
flr-250	128	12	,	,	PUNCT
flr-250	128	13	m	m	ADV
flr-250	128	14	,	,	PUNCT
flr-250	128	15			NOUN
flr-250	128	16	and	and	CCONJ
flr-250	128	17	zk	zk	PROPN
flr-250	129	1	[	[	X
flr-250	129	2	32	32	NUM
flr-250	129	3	-	-	SYM
flr-250	129	4	33	33	NUM
flr-250	129	5	]	]	PUNCT
flr-250	129	6	.	.	PUNCT
flr-250	130	1	2.5	2.5	NUM
flr-250	130	2	comparing	compare	VERB
flr-250	130	3	roc	roc	PROPN
flr-250	130	4	curves	curve	NOUN
flr-250	130	5	although	although	SCONJ
flr-250	130	6	a	a	DET
flr-250	130	7	single	single	ADJ
flr-250	130	8	roc	roc	PROPN
flr-250	130	9	curve	curve	NOUN
flr-250	130	10	is	be	AUX
flr-250	130	11	common	common	ADJ
flr-250	130	12	,	,	PUNCT
flr-250	130	13	quite	quite	ADV
flr-250	130	14	often	often	ADV
flr-250	130	15	studies	study	NOUN
flr-250	130	16	are	be	AUX
flr-250	130	17	designed	design	VERB
flr-250	130	18	to	to	PART
flr-250	130	19	compare	compare	VERB
flr-250	130	20	performance	performance	NOUN
flr-250	130	21	,	,	PUNCT
flr-250	130	22	for	for	ADP
flr-250	130	23	example	example	NOUN
flr-250	130	24	between	between	ADP
flr-250	130	25	experts	expert	NOUN
flr-250	130	26	and	and	CCONJ
flr-250	130	27	novices	novice	NOUN
flr-250	130	28	on	on	ADP
flr-250	130	29	a	a	DET
flr-250	130	30	given	give	VERB
flr-250	130	31	visual	visual	ADJ
flr-250	130	32	task	task	NOUN
flr-250	130	33	.	.	PUNCT
flr-250	131	1	thus	thus	ADV
flr-250	131	2	there	there	PRON
flr-250	131	3	are	be	VERB
flr-250	131	4	two	two	NUM
flr-250	131	5	curves	curve	NOUN
flr-250	131	6	–	–	PUNCT
flr-250	131	7	one	one	NUM
flr-250	131	8	for	for	ADP
flr-250	131	9	experts	expert	NOUN
flr-250	131	10	and	and	CCONJ
flr-250	131	11	one	one	NUM
flr-250	131	12	for	for	ADP
flr-250	131	13	novices	novice	NOUN
flr-250	131	14	–	–	PUNCT
flr-250	131	15	and	and	CCONJ
flr-250	131	16	the	the	DET
flr-250	131	17	question	question	NOUN
flr-250	131	18	is	be	AUX
flr-250	131	19	whether	whether	SCONJ
flr-250	131	20	there	there	PRON
flr-250	131	21	is	be	VERB
flr-250	131	22	a	a	DET
flr-250	131	23	significant	significant	ADJ
flr-250	131	24	difference	difference	NOUN
flr-250	131	25	in	in	ADP
flr-250	131	26	performance	performance	NOUN
flr-250	131	27	(	(	PUNCT
flr-250	131	28	auc	auc	NOUN
flr-250	131	29	typically	typically	ADV
flr-250	131	30	)	)	PUNCT
flr-250	131	31	or	or	CCONJ
flr-250	131	32	not	not	PART
flr-250	131	33	.	.	PUNCT
flr-250	132	1	visually	visually	ADV
flr-250	132	2	it	it	PRON
flr-250	132	3	is	be	AUX
flr-250	132	4	not	not	PART
flr-250	132	5	always	always	ADV
flr-250	132	6	possible	possible	ADJ
flr-250	132	7	to	to	PART
flr-250	132	8	tell	tell	VERB
flr-250	132	9	if	if	SCONJ
flr-250	132	10	the	the	DET
flr-250	132	11	difference	difference	NOUN
flr-250	132	12	is	be	AUX
flr-250	132	13	significant	significant	ADJ
flr-250	132	14	.	.	PUNCT
flr-250	133	1	this	this	PRON
flr-250	133	2	is	be	AUX
flr-250	133	3	especially	especially	ADV
flr-250	133	4	true	true	ADJ
flr-250	133	5	when	when	SCONJ
flr-250	133	6	the	the	DET
flr-250	133	7	roc	roc	PROPN
flr-250	133	8	curves	curve	NOUN
flr-250	133	9	cross	cross	NOUN
flr-250	133	10	at	at	ADP
flr-250	133	11	some	some	DET
flr-250	133	12	point	point	NOUN
flr-250	133	13	(	(	PUNCT
flr-250	133	14	usually	usually	ADV
flr-250	133	15	the	the	DET
flr-250	133	16	upper	upper	ADJ
flr-250	133	17	right	right	ADJ
flr-250	133	18	quadrant	quadrant	NOUN
flr-250	133	19	/	/	SYM
flr-250	133	20	corner	corner	NOUN
flr-250	133	21	)	)	PUNCT
flr-250	133	22	as	as	ADP
flr-250	133	23	in	in	ADP
flr-250	133	24	figure	figure	NOUN
flr-250	133	25	3	3	NUM
flr-250	133	26	[	[	X
flr-250	133	27	3435	3435	NUM
flr-250	133	28	]	]	PUNCT
flr-250	133	29	.	.	PUNCT
flr-250	134	1	therefore	therefore	ADV
flr-250	134	2	,	,	PUNCT
flr-250	134	3	statistical	statistical	ADJ
flr-250	134	4	methods	method	NOUN
flr-250	134	5	have	have	AUX
flr-250	134	6	been	be	AUX
flr-250	134	7	developed	develop	VERB
flr-250	134	8	,	,	PUNCT
flr-250	134	9	some	some	PRON
flr-250	134	10	for	for	ADP
flr-250	134	11	comparing	compare	VERB
flr-250	134	12	only	only	ADV
flr-250	134	13	two	two	NUM
flr-250	134	14	curves	curve	NOUN
flr-250	134	15	and	and	CCONJ
flr-250	134	16	some	some	PRON
flr-250	134	17	for	for	ADP
flr-250	134	18	comparing	compare	VERB
flr-250	134	19	multiple	multiple	ADJ
flr-250	134	20	curves	curve	NOUN
flr-250	134	21	.	.	PUNCT
flr-250	135	1	again	again	ADV
flr-250	135	2	,	,	PUNCT
flr-250	135	3	there	there	PRON
flr-250	135	4	are	be	VERB
flr-250	135	5	parametric	parametric	ADJ
flr-250	135	6	and	and	CCONJ
flr-250	135	7	non	non	ADJ
flr-250	135	8	-	-	ADJ
flr-250	135	9	parametric	parametric	ADJ
flr-250	135	10	options	option	NOUN
flr-250	135	11	[	[	X
flr-250	135	12	22	22	NUM
flr-250	135	13	-	-	SYM
flr-250	135	14	23	23	NUM
flr-250	135	15	,	,	PUNCT
flr-250	135	16	36	36	NUM
flr-250	135	17	-	-	SYM
flr-250	135	18	40	40	NUM
flr-250	135	19	]	]	PUNCT
flr-250	135	20	.	.	PUNCT
flr-250	136	1	one	one	NUM
flr-250	136	2	of	of	ADP
flr-250	136	3	the	the	DET
flr-250	136	4	most	most	ADV
flr-250	136	5	common	common	ADJ
flr-250	136	6	methods	method	NOUN
flr-250	136	7	for	for	ADP
flr-250	136	8	comparing	compare	VERB
flr-250	136	9	multiple	multiple	ADJ
flr-250	136	10	observers	observer	NOUN
flr-250	136	11	and	and	CCONJ
flr-250	136	12	multiple	multiple	ADJ
flr-250	136	13	cases	case	NOUN
flr-250	136	14	is	be	AUX
flr-250	136	15	the	the	DET
flr-250	136	16	multi	multi	ADJ
flr-250	136	17	-	-	ADJ
flr-250	136	18	reader	reader	ADJ
flr-250	136	19	multicase	multicase	NOUN
flr-250	136	20	method	method	NOUN
flr-250	136	21	developed	develop	VERB
flr-250	136	22	by	by	ADP
flr-250	136	23	dorfman	dorfman	PROPN
flr-250	136	24	,	,	PUNCT
flr-250	136	25	berbaum	berbaum	NOUN
flr-250	136	26	and	and	CCONJ
flr-250	136	27	metz	metz	NOUN
flr-250	137	1	[	[	X
flr-250	137	2	39	39	NUM
flr-250	137	3	]	]	PUNCT
flr-250	137	4	.	.	PUNCT
flr-250	138	1	figure	figure	NOUN
flr-250	138	2	3	3	NUM
flr-250	138	3	.	.	NOUN
flr-250	138	4	example	example	NOUN
flr-250	138	5	of	of	ADP
flr-250	138	6	2	2	NUM
flr-250	138	7	roc	roc	PROPN
flr-250	138	8	curves	curve	NOUN
flr-250	138	9	that	that	SCONJ
flr-250	138	10	cross	cross	NOUN
flr-250	138	11	.	.	PROPN
flr-250	138	12	figure	figure	NOUN
flr-250	138	13	4	4	NUM
flr-250	138	14	is	be	AUX
flr-250	138	15	an	an	DET
flr-250	138	16	example	example	NOUN
flr-250	138	17	of	of	ADP
flr-250	138	18	the	the	DET
flr-250	138	19	output	output	NOUN
flr-250	138	20	from	from	ADP
flr-250	138	21	an	an	DET
flr-250	138	22	mrmc	mrmc	NOUN
flr-250	138	23	analysis	analysis	NOUN
flr-250	138	24	on	on	ADP
flr-250	138	25	a	a	DET
flr-250	138	26	study	study	NOUN
flr-250	138	27	that	that	PRON
flr-250	138	28	had	have	VERB
flr-250	138	29	6	6	NUM
flr-250	138	30	observers	observer	NOUN
flr-250	138	31	viewing	view	VERB
flr-250	138	32	a	a	DET
flr-250	138	33	series	series	NOUN
flr-250	138	34	of	of	ADP
flr-250	138	35	images	image	NOUN
flr-250	138	36	in	in	ADP
flr-250	138	37	2	2	NUM
flr-250	138	38	conditions	condition	NOUN
flr-250	138	39	(	(	PUNCT
flr-250	138	40	different	different	ADJ
flr-250	138	41	computer	computer	NOUN
flr-250	138	42	monitors	monitor	NOUN
flr-250	138	43	for	for	ADP
flr-250	138	44	displaying	display	VERB
flr-250	138	45	medical	medical	ADJ
flr-250	138	46	images	image	NOUN
flr-250	138	47	)	)	PUNCT
flr-250	138	48	.	.	PUNCT
flr-250	139	1	the	the	DET
flr-250	139	2	visual	visual	ADJ
flr-250	139	3	task	task	NOUN
flr-250	139	4	was	be	AUX
flr-250	139	5	to	to	PART
flr-250	139	6	search	search	VERB
flr-250	139	7	for	for	ADP
flr-250	139	8	subtle	subtle	ADJ
flr-250	139	9	fractures	fracture	NOUN
flr-250	139	10	in	in	ADP
flr-250	139	11	bone	bone	NOUN
flr-250	139	12	x	x	NOUN
flr-250	139	13	-	-	NOUN
flr-250	139	14	ray	ray	NOUN
flr-250	139	15	images	image	NOUN
flr-250	139	16	.	.	PUNCT
flr-250	140	1	readers	reader	NOUN
flr-250	140	2	1	1	NUM
flr-250	140	3	,	,	PUNCT
flr-250	140	4	3	3	NUM
flr-250	140	5	and	and	CCONJ
flr-250	140	6	5	5	NUM
flr-250	140	7	were	be	AUX
flr-250	140	8	expert	expert	ADJ
flr-250	140	9	radiologists	radiologist	NOUN
flr-250	140	10	and	and	CCONJ
flr-250	140	11	2	2	NUM
flr-250	140	12	,	,	PUNCT
flr-250	140	13	4	4	NUM
flr-250	140	14	and	and	CCONJ
flr-250	140	15	6	6	NUM
flr-250	140	16	were	be	AUX
flr-250	140	17	resident	resident	ADJ
flr-250	140	18	trainees	trainee	NOUN
flr-250	140	19	.	.	PUNCT
flr-250	141	1	the	the	DET
flr-250	141	2	upper	upper	ADJ
flr-250	141	3	portion	portion	NOUN
flr-250	141	4	provides	provide	VERB
flr-250	141	5	the	the	DET
flr-250	141	6	auc	auc	NOUN
flr-250	141	7	values	value	NOUN
flr-250	141	8	for	for	ADP
flr-250	141	9	each	each	DET
flr-250	141	10	observer	observer	NOUN
flr-250	141	11	in	in	ADP
flr-250	141	12	each	each	DET
flr-250	141	13	condition	condition	NOUN
flr-250	141	14	,	,	PUNCT
flr-250	141	15	followed	follow	VERB
flr-250	141	16	by	by	ADP
flr-250	141	17	the	the	DET
flr-250	141	18	difference	difference	NOUN
flr-250	141	19	between	between	ADP
flr-250	141	20	the	the	DET
flr-250	141	21	two	two	NUM
flr-250	141	22	conditions	condition	NOUN
flr-250	141	23	.	.	PUNCT
flr-250	142	1	auc	auc	NOUN
flr-250	142	2	values	value	NOUN
flr-250	142	3	are	be	AUX
flr-250	142	4	usually	usually	ADV
flr-250	142	5	reported	report	VERB
flr-250	142	6	in	in	ADP
flr-250	142	7	publications	publication	NOUN
flr-250	142	8	out	out	ADP
flr-250	142	9	to	to	PART
flr-250	142	10	three	three	NUM
flr-250	142	11	decimal	decimal	ADJ
flr-250	142	12	points	point	NOUN
flr-250	142	13	maximum	maximum	ADJ
flr-250	142	14	.	.	PUNCT
flr-250	143	1	the	the	DET
flr-250	143	2	lower	low	ADJ
flr-250	143	3	portion	portion	NOUN
flr-250	143	4	shows	show	VERB
flr-250	143	5	the	the	DET
flr-250	143	6	results	result	NOUN
flr-250	143	7	of	of	ADP
flr-250	143	8	the	the	DET
flr-250	143	9	analysis	analysis	NOUN
flr-250	143	10	of	of	ADP
flr-250	143	11	variance	variance	NOUN
flr-250	143	12	(	(	PUNCT
flr-250	143	13	anova	anova	PROPN
flr-250	143	14	)	)	PUNCT
flr-250	143	15	comparing	compare	VERB
flr-250	143	16	the	the	DET
flr-250	143	17	aucs	aucs	NOUN
flr-250	143	18	in	in	ADP
flr-250	143	19	the	the	DET
flr-250	143	20	standard	standard	ADJ
flr-250	143	21	anova	anova	PROPN
flr-250	143	22	output	output	NOUN
flr-250	143	23	format	format	NOUN
flr-250	143	24	.	.	PUNCT
flr-250	144	1	the	the	DET
flr-250	144	2	actual	actual	ADJ
flr-250	144	3	output	output	NOUN
flr-250	144	4	document	document	NOUN
flr-250	144	5	provides	provide	VERB
flr-250	144	6	more	more	ADJ
flr-250	144	7	information	information	NOUN
flr-250	144	8	than	than	SCONJ
flr-250	144	9	provided	provide	VERB
flr-250	144	10	here	here	ADV
flr-250	144	11	,	,	PUNCT
flr-250	144	12	such	such	ADJ
flr-250	144	13	as	as	ADP
flr-250	144	14	the	the	DET
flr-250	144	15	variance	variance	NOUN
flr-250	144	16	components	component	NOUN
flr-250	144	17	and	and	CCONJ
flr-250	144	18	errorcovariance	errorcovariance	NOUN
flr-250	144	19	estimates	estimate	NOUN
flr-250	144	20	,	,	PUNCT
flr-250	144	21	and	and	CCONJ
flr-250	144	22	different	different	ADJ
flr-250	144	23	ways	way	NOUN
flr-250	144	24	of	of	ADP
flr-250	144	25	treating	treat	VERB
flr-250	144	26	the	the	DET
flr-250	144	27	various	various	ADJ
flr-250	144	28	variables	variable	NOUN
flr-250	144	29	(	(	PUNCT
flr-250	144	30	e.g.	e.g.	ADV
flr-250	144	31	,	,	PUNCT
flr-250	144	32	random	random	ADJ
flr-250	144	33	readers	reader	NOUN
flr-250	144	34	and	and	CCONJ
flr-250	144	35	random	random	ADJ
flr-250	144	36	cases	case	NOUN
flr-250	144	37	,	,	PUNCT
flr-250	144	38	fixed	fix	VERB
flr-250	144	39	readers	reader	NOUN
flr-250	144	40	and	and	CCONJ
flr-250	144	41	random	random	ADJ
flr-250	144	42	cases	case	NOUN
flr-250	144	43	,	,	PUNCT
flr-250	144	44	random	random	ADJ
flr-250	144	45	readers	reader	NOUN
flr-250	144	46	and	and	CCONJ
flr-250	144	47	fixed	fix	VERB
flr-250	144	48	cases	case	NOUN
flr-250	144	49	)	)	PUNCT
flr-250	144	50	,	,	PUNCT
flr-250	144	51	but	but	CCONJ
flr-250	144	52	this	this	DET
flr-250	144	53	example	example	NOUN
flr-250	144	54	shows	show	VERB
flr-250	144	55	how	how	SCONJ
flr-250	144	56	many	many	ADJ
flr-250	144	57	available	available	ADJ
flr-250	144	58	programs	program	NOUN
flr-250	144	59	output	output	VERB
flr-250	144	60	relevant	relevant	ADJ
flr-250	144	61	data	datum	NOUN
flr-250	144	62	comparing	compare	VERB
flr-250	144	63	roc	roc	PROPN
flr-250	144	64	curves	curve	NOUN
flr-250	144	65	.	.	PUNCT
flr-250	145	1	the	the	DET
flr-250	145	2	analyses	analysis	NOUN
flr-250	145	3	can	can	AUX
flr-250	145	4	also	also	ADV
flr-250	145	5	take	take	VERB
flr-250	145	6	into	into	ADP
flr-250	145	7	account	account	NOUN
flr-250	145	8	krupinski	krupinski	NOUN
flr-250	146	1	|	|	INTJ
flr-250	146	2	f	f	NOUN
flr-250	146	3	l	l	NOUN
flr-250	146	4	r	r	NOUN
flr-250	146	5	38	38	NUM
flr-250	146	6	level	level	NOUN
flr-250	146	7	of	of	ADP
flr-250	146	8	expertise	expertise	NOUN
flr-250	146	9	by	by	ADP
flr-250	146	10	comparing	compare	VERB
flr-250	146	11	the	the	DET
flr-250	146	12	two	two	NUM
flr-250	146	13	groups	group	NOUN
flr-250	146	14	of	of	ADP
flr-250	146	15	observers	observer	NOUN
flr-250	146	16	as	as	SCONJ
flr-250	146	17	described	describe	VERB
flr-250	146	18	above	above	ADV
flr-250	146	19	(	(	PUNCT
flr-250	146	20	in	in	ADP
flr-250	146	21	this	this	DET
flr-250	146	22	case	case	NOUN
flr-250	146	23	there	there	PRON
flr-250	146	24	were	be	VERB
flr-250	146	25	differences	difference	NOUN
flr-250	146	26	but	but	CCONJ
flr-250	146	27	it	it	PRON
flr-250	146	28	did	do	AUX
flr-250	146	29	not	not	PART
flr-250	146	30	reach	reach	VERB
flr-250	146	31	statistical	statistical	ADJ
flr-250	146	32	significance	significance	NOUN
flr-250	146	33	)	)	PUNCT
flr-250	146	34	.	.	PUNCT
flr-250	147	1	figure	figure	VERB
flr-250	147	2	4	4	NUM
flr-250	147	3	.	.	PUNCT
flr-250	148	1	an	an	DET
flr-250	148	2	example	example	NOUN
flr-250	148	3	of	of	ADP
flr-250	148	4	the	the	DET
flr-250	148	5	output	output	NOUN
flr-250	148	6	from	from	ADP
flr-250	148	7	an	an	DET
flr-250	148	8	mrmc	mrmc	NOUN
flr-250	148	9	analysis	analysis	NOUN
flr-250	148	10	.	.	PUNCT
flr-250	149	1	3	3	X
flr-250	149	2	.	.	X
flr-250	149	3	other	other	ADJ
flr-250	149	4	types	type	NOUN
flr-250	149	5	of	of	ADP
flr-250	149	6	roc	roc	PROPN
flr-250	149	7	the	the	DET
flr-250	149	8	discussion	discussion	NOUN
flr-250	149	9	of	of	ADP
flr-250	149	10	roc	roc	PROPN
flr-250	149	11	analysis	analysis	NOUN
flr-250	149	12	up	up	ADP
flr-250	149	13	to	to	ADP
flr-250	149	14	this	this	DET
flr-250	149	15	point	point	NOUN
flr-250	149	16	has	have	AUX
flr-250	149	17	been	be	AUX
flr-250	149	18	about	about	ADP
flr-250	149	19	tasks	task	NOUN
flr-250	149	20	that	that	PRON
flr-250	149	21	involve	involve	VERB
flr-250	149	22	the	the	DET
flr-250	149	23	detection	detection	NOUN
flr-250	149	24	of	of	ADP
flr-250	149	25	one	one	NUM
flr-250	149	26	target	target	NOUN
flr-250	149	27	and	and	CCONJ
flr-250	149	28	for	for	ADP
flr-250	149	29	the	the	DET
flr-250	149	30	most	most	ADJ
flr-250	149	31	part	part	NOUN
flr-250	149	32	assess	assess	NOUN
flr-250	149	33	fps	fps	NOUN
flr-250	149	34	only	only	ADV
flr-250	149	35	as	as	SCONJ
flr-250	149	36	they	they	PRON
flr-250	149	37	occur	occur	VERB
flr-250	149	38	in	in	ADP
flr-250	149	39	target	target	NOUN
flr-250	149	40	absent	absent	ADJ
flr-250	149	41	images	image	NOUN
flr-250	149	42	(	(	PUNCT
flr-250	149	43	again	again	ADV
flr-250	149	44	1	1	NUM
flr-250	149	45	per	per	ADP
flr-250	149	46	)	)	PUNCT
flr-250	149	47	.	.	PUNCT
flr-250	150	1	in	in	ADP
flr-250	150	2	real	real	ADJ
flr-250	150	3	life	life	NOUN
flr-250	150	4	however	however	ADV
flr-250	150	5	,	,	PUNCT
flr-250	150	6	scenes	scene	NOUN
flr-250	150	7	and	and	CCONJ
flr-250	150	8	other	other	ADJ
flr-250	150	9	visual	visual	ADJ
flr-250	150	10	stimuli	stimulus	NOUN
flr-250	150	11	often	often	ADV
flr-250	150	12	contain	contain	VERB
flr-250	150	13	multiple	multiple	ADJ
flr-250	150	14	targets	target	NOUN
flr-250	150	15	and	and	CCONJ
flr-250	150	16	fps	fps	NOUN
flr-250	150	17	can	can	AUX
flr-250	150	18	occur	occur	VERB
flr-250	150	19	in	in	ADP
flr-250	150	20	both	both	DET
flr-250	150	21	target	target	NOUN
flr-250	150	22	present	present	ADJ
flr-250	150	23	and	and	CCONJ
flr-250	150	24	target	target	VERB
flr-250	150	25	absent	absent	ADJ
flr-250	150	26	stimuli	stimulus	NOUN
flr-250	150	27	.	.	PUNCT
flr-250	151	1	traditional	traditional	ADJ
flr-250	151	2	roc	roc	PROPN
flr-250	151	3	analysis	analysis	NOUN
flr-250	151	4	also	also	ADV
flr-250	151	5	typically	typically	ADV
flr-250	151	6	does	do	AUX
flr-250	151	7	not	not	PART
flr-250	151	8	ask	ask	VERB
flr-250	151	9	or	or	CCONJ
flr-250	151	10	require	require	VERB
flr-250	151	11	the	the	DET
flr-250	151	12	observer	observer	NOUN
flr-250	151	13	to	to	PART
flr-250	151	14	locate	locate	VERB
flr-250	151	15	the	the	DET
flr-250	151	16	target	target	NOUN
flr-250	151	17	once	once	SCONJ
flr-250	151	18	it	it	PRON
flr-250	151	19	has	have	AUX
flr-250	151	20	been	be	AUX
flr-250	151	21	detected	detect	VERB
flr-250	151	22	.	.	PUNCT
flr-250	152	1	thus	thus	ADV
flr-250	152	2	there	there	PRON
flr-250	152	3	is	be	VERB
flr-250	152	4	always	always	ADV
flr-250	152	5	some	some	DET
flr-250	152	6	question	question	NOUN
flr-250	152	7	(	(	PUNCT
flr-250	152	8	unless	unless	SCONJ
flr-250	152	9	the	the	DET
flr-250	152	10	target	target	NOUN
flr-250	152	11	appears	appear	VERB
flr-250	152	12	in	in	ADP
flr-250	152	13	a	a	DET
flr-250	152	14	specific	specific	ADJ
flr-250	152	15	location	location	NOUN
flr-250	152	16	(	(	PUNCT
flr-250	152	17	e.g.	e.g.	ADV
flr-250	152	18	,	,	PUNCT
flr-250	152	19	center	center	NOUN
flr-250	152	20	of	of	ADP
flr-250	152	21	the	the	DET
flr-250	152	22	display	display	NOUN
flr-250	152	23	)	)	PUNCT
flr-250	152	24	every	every	DET
flr-250	152	25	time	time	NOUN
flr-250	152	26	)	)	PUNCT
flr-250	152	27	as	as	ADP
flr-250	152	28	to	to	ADP
flr-250	152	29	whether	whether	SCONJ
flr-250	152	30	the	the	DET
flr-250	152	31	observer	observer	NOUN
flr-250	152	32	actually	actually	ADV
flr-250	152	33	detected	detect	VERB
flr-250	152	34	the	the	DET
flr-250	152	35	true	true	ADJ
flr-250	152	36	target	target	NOUN
flr-250	152	37	(	(	PUNCT
flr-250	152	38	tp	tp	NOUN
flr-250	152	39	)	)	PUNCT
flr-250	152	40	or	or	CCONJ
flr-250	152	41	called	call	VERB
flr-250	152	42	something	something	PRON
flr-250	152	43	else	else	ADV
flr-250	152	44	in	in	ADP
flr-250	152	45	the	the	DET
flr-250	152	46	image	image	NOUN
flr-250	152	47	(	(	PUNCT
flr-250	152	48	fp	fp	X
flr-250	152	49	)	)	PUNCT
flr-250	152	50	thereby	thereby	ADV
flr-250	152	51	actually	actually	ADV
flr-250	152	52	missing	miss	VERB
flr-250	152	53	the	the	DET
flr-250	152	54	true	true	ADJ
flr-250	152	55	target	target	NOUN
flr-250	152	56	(	(	PUNCT
flr-250	152	57	fn	fn	NOUN
flr-250	152	58	)	)	PUNCT
flr-250	152	59	.	.	PUNCT
flr-250	153	1	the	the	DET
flr-250	153	2	earliest	early	ADJ
flr-250	153	3	attempt	attempt	NOUN
flr-250	153	4	to	to	PART
flr-250	153	5	account	account	VERB
flr-250	153	6	for	for	ADP
flr-250	153	7	location	location	NOUN
flr-250	153	8	in	in	ADP
flr-250	153	9	roc	roc	PROPN
flr-250	153	10	tasks	task	NOUN
flr-250	153	11	was	be	AUX
flr-250	153	12	the	the	DET
flr-250	153	13	lroc	lroc	NOUN
flr-250	153	14	(	(	PUNCT
flr-250	153	15	location	location	PROPN
flr-250	153	16	roc	roc	PROPN
flr-250	153	17	)	)	PUNCT
flr-250	153	18	.	.	PUNCT
flr-250	154	1	in	in	ADP
flr-250	154	2	this	this	DET
flr-250	154	3	method	method	NOUN
flr-250	154	4	,	,	PUNCT
flr-250	154	5	the	the	DET
flr-250	154	6	observer	observer	NOUN
flr-250	154	7	provides	provide	VERB
flr-250	154	8	a	a	DET
flr-250	154	9	confidence	confidence	NOUN
flr-250	154	10	rating	rating	NOUN
flr-250	154	11	that	that	SCONJ
flr-250	154	12	somewhere	somewhere	ADV
flr-250	154	13	in	in	ADP
flr-250	154	14	the	the	DET
flr-250	154	15	image	image	NOUN
flr-250	154	16	there	there	PRON
flr-250	154	17	is	be	VERB
flr-250	154	18	a	a	DET
flr-250	154	19	target	target	NOUN
flr-250	154	20	,	,	PUNCT
flr-250	154	21	then	then	ADV
flr-250	154	22	marks	mark	VERB
flr-250	154	23	the	the	DET
flr-250	154	24	location	location	NOUN
flr-250	154	25	of	of	ADP
flr-250	154	26	the	the	DET
flr-250	154	27	most	most	ADV
flr-250	154	28	suspicious	suspicious	ADJ
flr-250	154	29	region	region	NOUN
flr-250	155	1	[	[	X
flr-250	155	2	41	41	NUM
flr-250	155	3	-	-	SYM
flr-250	155	4	42	42	NUM
flr-250	155	5	]	]	PUNCT
flr-250	155	6	.	.	PUNCT
flr-250	156	1	lroc	lroc	PROPN
flr-250	156	2	is	be	AUX
flr-250	156	3	an	an	DET
flr-250	156	4	advantage	advantage	NOUN
flr-250	156	5	over	over	ADP
flr-250	156	6	roc	roc	PROPN
flr-250	156	7	in	in	SCONJ
flr-250	156	8	that	that	SCONJ
flr-250	156	9	it	it	PRON
flr-250	156	10	takes	take	VERB
flr-250	156	11	location	location	NOUN
flr-250	156	12	into	into	ADP
flr-250	156	13	account	account	NOUN
flr-250	156	14	,	,	PUNCT
flr-250	156	15	but	but	CCONJ
flr-250	156	16	it	it	PRON
flr-250	156	17	still	still	ADV
flr-250	156	18	only	only	ADV
flr-250	156	19	allows	allow	VERB
flr-250	156	20	for	for	ADP
flr-250	156	21	a	a	DET
flr-250	156	22	single	single	ADJ
flr-250	156	23	target	target	NOUN
flr-250	156	24	.	.	PUNCT
flr-250	157	1	to	to	PART
flr-250	157	2	account	account	VERB
flr-250	157	3	for	for	ADP
flr-250	157	4	multiple	multiple	ADJ
flr-250	157	5	targets	target	NOUN
flr-250	157	6	,	,	PUNCT
flr-250	157	7	free	free	ADJ
flr-250	157	8	response	response	NOUN
flr-250	157	9	roc	roc	PROPN
flr-250	157	10	(	(	PUNCT
flr-250	157	11	froc	froc	NOUN
flr-250	157	12	)	)	PUNCT
flr-250	157	13	was	be	AUX
flr-250	157	14	developed	develop	VERB
flr-250	157	15	in	in	ADP
flr-250	157	16	which	which	PRON
flr-250	157	17	observers	observer	NOUN
flr-250	157	18	mark	mark	VERB
flr-250	157	19	different	different	ADJ
flr-250	157	20	locations	location	NOUN
flr-250	157	21	and	and	CCONJ
flr-250	157	22	provide	provide	VERB
flr-250	157	23	a	a	DET
flr-250	157	24	confidence	confidence	NOUN
flr-250	157	25	rating	rating	NOUN
flr-250	157	26	for	for	ADP
flr-250	157	27	each	each	DET
flr-250	157	28	mark	mark	NOUN
flr-250	158	1	[	[	X
flr-250	158	2	43	43	NUM
flr-250	158	3	-	-	SYM
flr-250	158	4	46	46	NUM
flr-250	158	5	]	]	PUNCT
flr-250	158	6	.	.	PUNCT
flr-250	159	1	the	the	DET
flr-250	159	2	problem	problem	NOUN
flr-250	159	3	with	with	ADP
flr-250	159	4	froc	froc	NOUN
flr-250	159	5	is	be	AUX
flr-250	159	6	that	that	SCONJ
flr-250	159	7	when	when	SCONJ
flr-250	159	8	the	the	DET
flr-250	159	9	roc	roc	PROPN
flr-250	159	10	is	be	AUX
flr-250	159	11	plotted	plot	VERB
flr-250	159	12	the	the	DET
flr-250	159	13	x	x	NOUN
flr-250	159	14	-	-	NOUN
flr-250	159	15	axis	axis	ADJ
flr-250	159	16	,	,	PUNCT
flr-250	159	17	rather	rather	ADV
flr-250	159	18	than	than	ADP
flr-250	159	19	going	go	VERB
flr-250	159	20	from	from	ADP
flr-250	159	21	0	0	NUM
flr-250	159	22	to	to	PART
flr-250	159	23	1	1	NUM
flr-250	159	24	like	like	ADP
flr-250	159	25	the	the	DET
flr-250	159	26	traditional	traditional	ADJ
flr-250	159	27	rfoc	rfoc	ADJ
flr-250	159	28	curve	curve	NOUN
flr-250	159	29	,	,	PUNCT
flr-250	159	30	goes	go	VERB
flr-250	159	31	from	from	ADP
flr-250	159	32	0	0	NUM
flr-250	159	33	to	to	ADP
flr-250	159	34	infinity	infinity	NOUN
flr-250	159	35	(	(	PUNCT
flr-250	159	36	based	base	VERB
flr-250	159	37	on	on	ADP
flr-250	159	38	however	however	ADV
flr-250	159	39	many	many	ADJ
flr-250	159	40	fps	fps	NOUN
flr-250	159	41	are	be	AUX
flr-250	159	42	reported	report	VERB
flr-250	159	43	)	)	PUNCT
flr-250	159	44	.	.	PUNCT
flr-250	160	1	this	this	PRON
flr-250	160	2	makes	make	VERB
flr-250	160	3	calculating	calculate	VERB
flr-250	160	4	the	the	DET
flr-250	160	5	area	area	NOUN
flr-250	160	6	under	under	ADP
flr-250	160	7	curve	curve	NOUN
flr-250	160	8	quite	quite	ADV
flr-250	160	9	difficult	difficult	ADJ
flr-250	160	10	and	and	CCONJ
flr-250	160	11	the	the	DET
flr-250	160	12	comparison	comparison	NOUN
flr-250	160	13	of	of	ADP
flr-250	160	14	two	two	NUM
flr-250	160	15	curves	curve	NOUN
flr-250	160	16	even	even	ADV
flr-250	160	17	more	more	ADV
flr-250	160	18	challenging	challenging	ADJ
flr-250	160	19	.	.	PUNCT
flr-250	161	1	the	the	DET
flr-250	161	2	alternative	alternative	ADJ
flr-250	161	3	froc	froc	ADJ
flr-250	161	4	(	(	PUNCT
flr-250	161	5	afroc	afroc	NOUN
flr-250	161	6	)	)	PUNCT
flr-250	161	7	method	method	NOUN
flr-250	161	8	was	be	AUX
flr-250	161	9	developed	develop	VERB
flr-250	161	10	to	to	PART
flr-250	161	11	address	address	VERB
flr-250	161	12	issue	issue	NOUN
flr-250	161	13	,	,	PUNCT
flr-250	161	14	creating	create	VERB
flr-250	161	15	a	a	DET
flr-250	161	16	plot	plot	NOUN
flr-250	161	17	that	that	PRON
flr-250	161	18	has	have	AUX
flr-250	161	19	both	both	DET
flr-250	161	20	axes	axis	NOUN
flr-250	161	21	going	go	VERB
flr-250	161	22	from	from	ADP
flr-250	161	23	0	0	NUM
flr-250	161	24	to	to	ADP
flr-250	161	25	1	1	NUM
flr-250	161	26	[	[	X
flr-250	161	27	47	47	NUM
flr-250	161	28	]	]	PUNCT
flr-250	161	29	.	.	PUNCT
flr-250	162	1	the	the	DET
flr-250	162	2	jackknife	jackknife	PROPN
flr-250	162	3	afroc	afroc	PROPN
flr-250	162	4	(	(	PUNCT
flr-250	162	5	jafroc	jafroc	ADJ
flr-250	162	6	)	)	PUNCT
flr-250	162	7	method	method	NOUN
flr-250	162	8	was	be	AUX
flr-250	162	9	then	then	ADV
flr-250	162	10	developed	develop	VERB
flr-250	162	11	to	to	PART
flr-250	162	12	allow	allow	VERB
flr-250	162	13	for	for	ADP
flr-250	162	14	generalization	generalization	NOUN
flr-250	162	15	to	to	ADP
flr-250	162	16	the	the	DET
flr-250	162	17	population	population	NOUN
flr-250	162	18	of	of	ADP
flr-250	162	19	readers	reader	NOUN
flr-250	162	20	and	and	CCONJ
flr-250	162	21	cases	case	NOUN
flr-250	162	22	,	,	PUNCT
flr-250	162	23	in	in	ADP
flr-250	162	24	the	the	DET
flr-250	162	25	same	same	ADJ
flr-250	162	26	way	way	NOUN
flr-250	162	27	that	that	PRON
flr-250	162	28	mrmc	mrmc	NOUN
flr-250	162	29	roc	roc	PROPN
flr-250	162	30	does	do	VERB
flr-250	162	31	[	[	X
flr-250	162	32	4446	4446	NUM
flr-250	162	33	]	]	PUNCT
flr-250	162	34	.	.	PUNCT
flr-250	163	1	krupinski	krupinski	NOUN
flr-250	163	2	|	|	ADV
flr-250	163	3	f	f	NOUN
flr-250	163	4	l	l	NOUN
flr-250	163	5	r	r	NOUN
flr-250	163	6	39	39	NUM
flr-250	163	7	4	4	NUM
flr-250	163	8	.	.	PUNCT
flr-250	164	1	other	other	ADJ
flr-250	164	2	considerations	consideration	NOUN
flr-250	164	3	in	in	ADP
flr-250	164	4	addition	addition	NOUN
flr-250	164	5	to	to	ADP
flr-250	164	6	deciding	decide	VERB
flr-250	164	7	which	which	DET
flr-250	164	8	type	type	NOUN
flr-250	164	9	of	of	ADP
flr-250	164	10	roc	roc	NOUN
flr-250	164	11	analysis	analysis	NOUN
flr-250	164	12	is	be	AUX
flr-250	164	13	best	good	ADJ
flr-250	164	14	(	(	PUNCT
flr-250	164	15	which	which	PRON
flr-250	164	16	really	really	ADV
flr-250	164	17	depends	depend	VERB
flr-250	164	18	on	on	ADP
flr-250	164	19	the	the	DET
flr-250	164	20	hypotheses	hypothesis	NOUN
flr-250	164	21	,	,	PUNCT
flr-250	164	22	nature	nature	NOUN
flr-250	164	23	of	of	ADP
flr-250	164	24	the	the	DET
flr-250	164	25	task	task	NOUN
flr-250	164	26	,	,	PUNCT
flr-250	164	27	types	type	NOUN
flr-250	164	28	and	and	CCONJ
flr-250	164	29	number	number	NOUN
flr-250	164	30	of	of	ADP
flr-250	164	31	images	image	NOUN
flr-250	164	32	and	and	CCONJ
flr-250	164	33	targets	target	NOUN
flr-250	164	34	)	)	PUNCT
flr-250	164	35	,	,	PUNCT
flr-250	164	36	there	there	PRON
flr-250	164	37	are	be	VERB
flr-250	164	38	two	two	NUM
flr-250	164	39	other	other	ADJ
flr-250	164	40	aspects	aspect	NOUN
flr-250	164	41	that	that	PRON
flr-250	164	42	are	be	AUX
flr-250	164	43	typically	typically	ADV
flr-250	164	44	important	important	ADJ
flr-250	164	45	.	.	PUNCT
flr-250	165	1	as	as	SCONJ
flr-250	165	2	already	already	ADV
flr-250	165	3	discussed	discuss	VERB
flr-250	165	4	above	above	ADV
flr-250	165	5	,	,	PUNCT
flr-250	165	6	the	the	DET
flr-250	165	7	truth	truth	NOUN
flr-250	165	8	or	or	CCONJ
flr-250	165	9	gold	gold	NOUN
flr-250	165	10	standard	standard	NOUN
flr-250	165	11	for	for	ADP
flr-250	165	12	cases	case	NOUN
flr-250	165	13	must	must	AUX
flr-250	165	14	be	be	AUX
flr-250	165	15	known	know	VERB
flr-250	165	16	in	in	ADP
flr-250	165	17	advance	advance	NOUN
flr-250	165	18	.	.	PUNCT
flr-250	166	1	with	with	ADP
flr-250	166	2	simple	simple	ADJ
flr-250	166	3	psychophysical	psychophysical	ADJ
flr-250	166	4	studies	study	NOUN
flr-250	166	5	this	this	PRON
flr-250	166	6	is	be	AUX
flr-250	166	7	quite	quite	ADV
flr-250	166	8	easy	easy	ADJ
flr-250	166	9	but	but	CCONJ
flr-250	166	10	for	for	ADP
flr-250	166	11	real	real	ADJ
flr-250	166	12	life	life	NOUN
flr-250	166	13	images	image	NOUN
flr-250	166	14	(	(	PUNCT
flr-250	166	15	scenes	scene	NOUN
flr-250	166	16	,	,	PUNCT
flr-250	166	17	medical	medical	ADJ
flr-250	166	18	images	image	NOUN
flr-250	166	19	,	,	PUNCT
flr-250	166	20	industrial	industrial	ADJ
flr-250	166	21	images	image	NOUN
flr-250	166	22	,	,	PUNCT
flr-250	166	23	satellite	satellite	NOUN
flr-250	166	24	images	image	NOUN
flr-250	166	25	)	)	PUNCT
flr-250	166	26	this	this	PRON
flr-250	166	27	can	can	AUX
flr-250	166	28	be	be	AUX
flr-250	166	29	more	more	ADV
flr-250	166	30	difficult	difficult	ADJ
flr-250	166	31	.	.	PUNCT
flr-250	167	1	other	other	ADJ
flr-250	167	2	considerations	consideration	NOUN
flr-250	167	3	when	when	SCONJ
flr-250	167	4	selecting	selecting	NOUN
flr-250	167	5	cases	case	NOUN
flr-250	167	6	include	include	VERB
flr-250	167	7	:	:	PUNCT
flr-250	167	8	how	how	SCONJ
flr-250	167	9	subtle	subtle	ADJ
flr-250	167	10	or	or	CCONJ
flr-250	167	11	obvious	obvious	ADJ
flr-250	167	12	the	the	DET
flr-250	167	13	targets	target	NOUN
flr-250	167	14	are	be	AUX
flr-250	167	15	,	,	PUNCT
flr-250	167	16	how	how	SCONJ
flr-250	167	17	much	much	ADJ
flr-250	167	18	and	and	CCONJ
flr-250	167	19	what	what	DET
flr-250	167	20	type	type	NOUN
flr-250	167	21	of	of	ADP
flr-250	167	22	background	background	NOUN
flr-250	167	23	“	"	PUNCT
flr-250	167	24	noise	noise	NOUN
flr-250	167	25	”	"	PUNCT
flr-250	167	26	is	be	AUX
flr-250	167	27	in	in	ADP
flr-250	167	28	the	the	DET
flr-250	167	29	image	image	NOUN
flr-250	167	30	,	,	PUNCT
flr-250	167	31	where	where	SCONJ
flr-250	167	32	the	the	DET
flr-250	167	33	targets	target	NOUN
flr-250	167	34	are	be	AUX
flr-250	167	35	located	locate	VERB
flr-250	167	36	(	(	PUNCT
flr-250	167	37	random	random	ADJ
flr-250	167	38	or	or	CCONJ
flr-250	167	39	in	in	ADP
flr-250	167	40	specified	specified	ADJ
flr-250	167	41	locations	location	NOUN
flr-250	167	42	)	)	PUNCT
flr-250	167	43	,	,	PUNCT
flr-250	167	44	the	the	DET
flr-250	167	45	size(s	size(s	NOUN
flr-250	167	46	)	)	PUNCT
flr-250	167	47	of	of	ADP
flr-250	167	48	the	the	DET
flr-250	167	49	targets	target	NOUN
flr-250	167	50	and	and	CCONJ
flr-250	167	51	how	how	SCONJ
flr-250	167	52	much	much	ADJ
flr-250	167	53	background	background	NOUN
flr-250	167	54	image	image	NOUN
flr-250	167	55	is	be	AUX
flr-250	167	56	included	include	VERB
flr-250	167	57	,	,	PUNCT
flr-250	167	58	how	how	SCONJ
flr-250	167	59	long	long	ADV
flr-250	167	60	the	the	DET
flr-250	167	61	images	image	NOUN
flr-250	167	62	will	will	AUX
flr-250	167	63	remain	remain	VERB
flr-250	167	64	available	available	ADJ
flr-250	167	65	for	for	ADP
flr-250	167	66	viewing	view	VERB
flr-250	167	67	,	,	PUNCT
flr-250	167	68	whether	whether	SCONJ
flr-250	167	69	the	the	DET
flr-250	167	70	images	image	NOUN
flr-250	167	71	can	can	AUX
flr-250	167	72	be	be	AUX
flr-250	167	73	manipulated	manipulate	VERB
flr-250	167	74	(	(	PUNCT
flr-250	167	75	e.g.	e.g.	ADV
flr-250	167	76	,	,	PUNCT
flr-250	167	77	zoom	zoom	NOUN
flr-250	167	78	/	/	SYM
flr-250	167	79	pan	pan	NOUN
flr-250	167	80	or	or	CCONJ
flr-250	167	81	window	window	NOUN
flr-250	167	82	/	/	SYM
flr-250	167	83	level	level	NOUN
flr-250	167	84	)	)	PUNCT
flr-250	167	85	by	by	ADP
flr-250	167	86	the	the	DET
flr-250	167	87	observers	observer	NOUN
flr-250	167	88	,	,	PUNCT
flr-250	167	89	and	and	CCONJ
flr-250	167	90	target	target	VERB
flr-250	167	91	prevalence	prevalence	NOUN
flr-250	167	92	as	as	SCONJ
flr-250	167	93	noted	note	VERB
flr-250	167	94	above	above	ADV
flr-250	167	95	.	.	PUNCT
flr-250	168	1	sample	sample	NOUN
flr-250	168	2	size	size	NOUN
flr-250	168	3	is	be	AUX
flr-250	168	4	the	the	DET
flr-250	168	5	other	other	ADJ
flr-250	168	6	key	key	ADJ
flr-250	168	7	issue	issue	NOUN
flr-250	168	8	with	with	ADP
flr-250	168	9	respect	respect	NOUN
flr-250	168	10	to	to	ADP
flr-250	168	11	setting	set	VERB
flr-250	168	12	up	up	ADP
flr-250	168	13	an	an	DET
flr-250	168	14	roc	roc	PROPN
flr-250	168	15	study	study	NOUN
flr-250	168	16	–	–	PUNCT
flr-250	168	17	how	how	SCONJ
flr-250	168	18	many	many	ADJ
flr-250	168	19	images	image	NOUN
flr-250	168	20	and	and	CCONJ
flr-250	168	21	observers	observer	NOUN
flr-250	168	22	are	be	AUX
flr-250	168	23	required	require	VERB
flr-250	168	24	to	to	PART
flr-250	168	25	achieve	achieve	VERB
flr-250	168	26	adequate	adequate	ADJ
flr-250	168	27	power	power	NOUN
flr-250	168	28	once	once	SCONJ
flr-250	168	29	the	the	DET
flr-250	168	30	study	study	NOUN
flr-250	168	31	is	be	AUX
flr-250	168	32	completed	complete	VERB
flr-250	168	33	.	.	PUNCT
flr-250	169	1	as	as	ADP
flr-250	169	2	with	with	ADP
flr-250	169	3	any	any	DET
flr-250	169	4	other	other	ADJ
flr-250	169	5	power	power	NOUN
flr-250	169	6	calculation	calculation	NOUN
flr-250	169	7	,	,	PUNCT
flr-250	169	8	sample	sample	NOUN
flr-250	169	9	size	size	NOUN
flr-250	169	10	will	will	AUX
flr-250	169	11	depend	depend	VERB
flr-250	169	12	on	on	ADP
flr-250	169	13	a	a	DET
flr-250	169	14	number	number	NOUN
flr-250	169	15	of	of	ADP
flr-250	169	16	factors	factor	NOUN
flr-250	169	17	including	include	VERB
flr-250	169	18	the	the	DET
flr-250	169	19	metric	metric	NOUN
flr-250	169	20	under	under	ADP
flr-250	169	21	consideration	consideration	NOUN
flr-250	169	22	(	(	PUNCT
flr-250	169	23	e.g.	e.g.	ADV
flr-250	169	24	,	,	PUNCT
flr-250	169	25	auc	auc	NOUN
flr-250	169	26	or	or	CCONJ
flr-250	169	27	partial	partial	ADJ
flr-250	169	28	auc	auc	NOUN
flr-250	169	29	)	)	PUNCT
flr-250	169	30	and	and	CCONJ
flr-250	169	31	the	the	DET
flr-250	169	32	design	design	NOUN
flr-250	169	33	(	(	PUNCT
flr-250	169	34	e.g.	e.g.	ADV
flr-250	169	35	,	,	PUNCT
flr-250	169	36	repeated	repeat	VERB
flr-250	169	37	measures	measure	NOUN
flr-250	169	38	with	with	ADP
flr-250	169	39	the	the	DET
flr-250	169	40	same	same	ADJ
flr-250	169	41	observers	observer	NOUN
flr-250	169	42	viewing	view	VERB
flr-250	169	43	the	the	DET
flr-250	169	44	same	same	ADJ
flr-250	169	45	images	image	NOUN
flr-250	169	46	in	in	ADP
flr-250	169	47	two	two	NUM
flr-250	169	48	or	or	CCONJ
flr-250	169	49	more	more	ADJ
flr-250	169	50	conditions	condition	NOUN
flr-250	169	51	or	or	CCONJ
flr-250	169	52	different	different	ADJ
flr-250	169	53	readers	reader	NOUN
flr-250	169	54	viewing	view	VERB
flr-250	169	55	the	the	DET
flr-250	169	56	images	image	NOUN
flr-250	169	57	in	in	ADP
flr-250	169	58	different	different	ADJ
flr-250	169	59	conditions	condition	NOUN
flr-250	169	60	)	)	PUNCT
flr-250	169	61	.	.	PUNCT
flr-250	170	1	there	there	PRON
flr-250	170	2	are	be	VERB
flr-250	170	3	a	a	DET
flr-250	170	4	number	number	NOUN
flr-250	170	5	of	of	ADP
flr-250	170	6	key	key	ADJ
flr-250	170	7	papers	paper	NOUN
flr-250	170	8	describing	describe	VERB
flr-250	170	9	methods	method	NOUN
flr-250	170	10	to	to	PART
flr-250	170	11	calculate	calculate	VERB
flr-250	170	12	sample	sample	NOUN
flr-250	170	13	sizes	size	NOUN
flr-250	170	14	for	for	ADP
flr-250	170	15	various	various	ADJ
flr-250	170	16	study	study	NOUN
flr-250	170	17	designs	design	NOUN
flr-250	170	18	,	,	PUNCT
flr-250	170	19	many	many	ADJ
flr-250	170	20	of	of	ADP
flr-250	170	21	which	which	PRON
flr-250	170	22	include	include	VERB
flr-250	170	23	representative	representative	ADJ
flr-250	170	24	tables	table	NOUN
flr-250	170	25	showing	show	VERB
flr-250	170	26	sample	sample	NOUN
flr-250	170	27	sizes	size	NOUN
flr-250	170	28	required	require	VERB
flr-250	170	29	for	for	ADP
flr-250	170	30	different	different	ADJ
flr-250	170	31	power	power	NOUN
flr-250	170	32	estimates	estimate	NOUN
flr-250	170	33	[	[	X
flr-250	170	34	48	48	NUM
flr-250	170	35	-	-	SYM
flr-250	170	36	51	51	NUM
flr-250	170	37	]	]	PUNCT
flr-250	170	38	.	.	PUNCT
flr-250	171	1	some	some	PRON
flr-250	171	2	of	of	ADP
flr-250	171	3	the	the	DET
flr-250	171	4	available	available	ADJ
flr-250	171	5	roc	roc	PROPN
flr-250	171	6	programs	program	NOUN
flr-250	171	7	also	also	ADV
flr-250	171	8	include	include	VERB
flr-250	171	9	a	a	DET
flr-250	171	10	power	power	NOUN
flr-250	171	11	calculator	calculator	NOUN
flr-250	171	12	and	and	CCONJ
flr-250	171	13	some	some	PRON
flr-250	171	14	will	will	AUX
flr-250	171	15	provide	provide	VERB
flr-250	171	16	power	power	NOUN
flr-250	171	17	in	in	ADP
flr-250	171	18	the	the	DET
flr-250	171	19	analysis	analysis	NOUN
flr-250	171	20	output	output	NOUN
flr-250	171	21	.	.	PUNCT
flr-250	172	1	5	5	X
flr-250	172	2	.	.	X
flr-250	172	3	software	software	NOUN
flr-250	172	4	programs	program	NOUN
flr-250	172	5	it	it	PRON
flr-250	172	6	is	be	AUX
flr-250	172	7	not	not	PART
flr-250	172	8	possible	possible	ADJ
flr-250	172	9	to	to	PART
flr-250	172	10	list	list	VERB
flr-250	172	11	all	all	PRON
flr-250	172	12	of	of	ADP
flr-250	172	13	the	the	DET
flr-250	172	14	available	available	ADJ
flr-250	172	15	software	software	NOUN
flr-250	172	16	programs	program	NOUN
flr-250	172	17	for	for	ADP
flr-250	172	18	roc	roc	PROPN
flr-250	172	19	analysis	analysis	NOUN
flr-250	172	20	as	as	SCONJ
flr-250	172	21	there	there	PRON
flr-250	172	22	are	be	VERB
flr-250	172	23	always	always	ADV
flr-250	172	24	new	new	ADJ
flr-250	172	25	ones	one	NOUN
flr-250	172	26	being	be	AUX
flr-250	172	27	released	release	VERB
flr-250	172	28	.	.	PUNCT
flr-250	173	1	there	there	PRON
flr-250	173	2	are	be	VERB
flr-250	173	3	however	however	ADV
flr-250	173	4	some	some	DET
flr-250	173	5	reliable	reliable	ADJ
flr-250	173	6	sites	site	NOUN
flr-250	173	7	where	where	SCONJ
flr-250	173	8	the	the	DET
flr-250	173	9	more	more	ADV
flr-250	173	10	commonly	commonly	ADV
flr-250	173	11	used	use	VERB
flr-250	173	12	programs	program	NOUN
flr-250	173	13	can	can	AUX
flr-250	173	14	be	be	AUX
flr-250	173	15	found	find	VERB
flr-250	173	16	.	.	PUNCT
flr-250	174	1	the	the	DET
flr-250	174	2	medical	medical	ADJ
flr-250	174	3	image	image	NOUN
flr-250	174	4	perception	perception	NOUN
flr-250	174	5	laboratory	laboratory	NOUN
flr-250	174	6	web	web	NOUN
flr-250	174	7	site	site	NOUN
flr-250	174	8	[	[	X
flr-250	174	9	52	52	NUM
flr-250	174	10	]	]	PUNCT
flr-250	174	11	at	at	ADP
flr-250	174	12	the	the	DET
flr-250	174	13	university	university	PROPN
flr-250	174	14	of	of	ADP
flr-250	174	15	iowa	iowa	PROPN
flr-250	174	16	and	and	CCONJ
flr-250	174	17	the	the	DET
flr-250	174	18	roc	roc	PROPN
flr-250	174	19	software	software	NOUN
flr-250	174	20	site	site	NOUN
flr-250	174	21	at	at	ADP
flr-250	174	22	the	the	DET
flr-250	174	23	university	university	NOUN
flr-250	174	24	of	of	ADP
flr-250	174	25	chicago	chicago	PROPN
flr-250	174	26	[	[	X
flr-250	174	27	53	53	NUM
flr-250	174	28	]	]	PUNCT
flr-250	174	29	contain	contain	VERB
flr-250	174	30	the	the	DET
flr-250	174	31	programs	program	NOUN
flr-250	174	32	developed	develop	VERB
flr-250	174	33	by	by	ADP
flr-250	174	34	that	that	DET
flr-250	174	35	team	team	NOUN
flr-250	174	36	(	(	PUNCT
flr-250	174	37	dorfman	dorfman	PROPN
flr-250	174	38	,	,	PUNCT
flr-250	174	39	berbaum	berbaum	PROPN
flr-250	174	40	,	,	PUNCT
flr-250	174	41	metz	metz	PROPN
flr-250	174	42	,	,	PUNCT
flr-250	174	43	hillis	hillis	PROPN
flr-250	174	44	)	)	PUNCT
flr-250	174	45	including	include	VERB
flr-250	174	46	the	the	DET
flr-250	174	47	mrmc	mrmc	PROPN
flr-250	174	48	roc	roc	PROPN
flr-250	174	49	,	,	PUNCT
flr-250	174	50	rocfit	rocfit	NOUN
flr-250	174	51	,	,	PUNCT
flr-250	174	52	labroc4	labroc4	PROPN
flr-250	174	53	,	,	PUNCT
flr-250	174	54	corroc	corroc	PROPN
flr-250	174	55	,	,	PUNCT
flr-250	174	56	indroc	indroc	ADJ
flr-250	174	57	,	,	PUNCT
flr-250	174	58	rocket	rocket	NOUN
flr-250	174	59	,	,	PUNCT
flr-250	174	60	labmrmc	labmrmc	NOUN
flr-250	174	61	,	,	PUNCT
flr-250	174	62	proproc	proproc	NOUN
flr-250	174	63	,	,	PUNCT
flr-250	174	64	rscore	rscore	NOUN
flr-250	174	65	,	,	PUNCT
flr-250	174	66	bigamma	bigamma	PROPN
flr-250	174	67	,	,	PUNCT
flr-250	174	68	rscore	rscore	NOUN
flr-250	174	69	-	-	PUNCT
flr-250	174	70	j	j	NOUN
flr-250	174	71	and	and	CCONJ
flr-250	174	72	sas	sa	VERB
flr-250	174	73	programs	program	NOUN
flr-250	174	74	to	to	PART
flr-250	174	75	perform	perform	VERB
flr-250	174	76	sample	sample	NOUN
flr-250	174	77	size	size	NOUN
flr-250	174	78	estimates	estimate	NOUN
flr-250	174	79	.	.	PUNCT
flr-250	175	1	software	software	NOUN
flr-250	175	2	for	for	ADP
flr-250	175	3	froc	froc	ADJ
flr-250	175	4	,	,	PUNCT
flr-250	175	5	afroc	afroc	NOUN
flr-250	175	6	and	and	CCONJ
flr-250	175	7	jafroc	jafroc	PROPN
flr-250	175	8	(	(	PUNCT
flr-250	175	9	chakraborty	chakraborty	NOUN
flr-250	175	10	)	)	PUNCT
flr-250	175	11	analyses	analysis	NOUN
flr-250	175	12	are	be	AUX
flr-250	175	13	available	available	ADJ
flr-250	175	14	as	as	ADV
flr-250	175	15	well	well	ADV
flr-250	175	16	[	[	X
flr-250	175	17	54	54	NUM
flr-250	175	18	]	]	PUNCT
flr-250	175	19	.	.	PUNCT
flr-250	176	1	some	some	DET
flr-250	176	2	commercial	commercial	ADJ
flr-250	176	3	statistical	statistical	ADJ
flr-250	176	4	software	software	NOUN
flr-250	176	5	also	also	ADV
flr-250	176	6	has	have	VERB
flr-250	176	7	modules	module	NOUN
flr-250	176	8	for	for	ADP
flr-250	176	9	roc	roc	NOUN
flr-250	176	10	analysis	analysis	NOUN
flr-250	176	11	[	[	X
flr-250	176	12	55	55	NUM
flr-250	176	13	-	-	SYM
flr-250	176	14	59	59	NUM
flr-250	176	15	]	]	PUNCT
flr-250	176	16	.	.	PUNCT
flr-250	177	1	keypoints	keypoint	NOUN
flr-250	177	2	roc	roc	PROPN
flr-250	177	3	analysis	analysis	NOUN
flr-250	177	4	provides	provide	VERB
flr-250	177	5	metrics	metric	NOUN
flr-250	177	6	of	of	ADP
flr-250	177	7	observer	observer	NOUN
flr-250	177	8	performance	performance	NOUN
flr-250	177	9	in	in	ADP
flr-250	177	10	visual	visual	ADJ
flr-250	177	11	detection	detection	NOUN
flr-250	177	12	and	and	CCONJ
flr-250	177	13	discrimination	discrimination	NOUN
flr-250	177	14	tasks	task	NOUN
flr-250	177	15	area	area	NOUN
flr-250	177	16	under	under	ADP
flr-250	177	17	the	the	DET
flr-250	177	18	curve	curve	NOUN
flr-250	177	19	(	(	PUNCT
flr-250	177	20	auc	auc	NOUN
flr-250	177	21	)	)	PUNCT
flr-250	177	22	is	be	AUX
flr-250	177	23	the	the	DET
flr-250	177	24	most	most	ADV
flr-250	177	25	commonly	commonly	ADV
flr-250	177	26	used	use	VERB
flr-250	177	27	metric	metric	NOUN
flr-250	177	28	of	of	ADP
flr-250	177	29	performance	performance	NOUN
flr-250	177	30	common	common	ADJ
flr-250	177	31	variants	variant	NOUN
flr-250	177	32	of	of	ADP
flr-250	177	33	roc	roc	PROPN
flr-250	177	34	analysis	analysis	NOUN
flr-250	177	35	allow	allow	VERB
flr-250	177	36	for	for	ADP
flr-250	177	37	multiple	multiple	ADJ
flr-250	177	38	targets	target	NOUN
flr-250	177	39	and	and	CCONJ
flr-250	177	40	location	location	NOUN
flr-250	177	41	accuracy	accuracy	NOUN
flr-250	177	42	key	key	ADJ
flr-250	177	43	study	study	NOUN
flr-250	177	44	design	design	NOUN
flr-250	177	45	issues	issue	NOUN
flr-250	177	46	include	include	VERB
flr-250	177	47	target	target	NOUN
flr-250	177	48	characteristics	characteristic	NOUN
flr-250	177	49	and	and	CCONJ
flr-250	177	50	establishing	establish	VERB
flr-250	177	51	a	a	DET
flr-250	177	52	gold	gold	ADJ
flr-250	177	53	standard	standard	ADJ
flr-250	177	54	software	software	NOUN
flr-250	177	55	is	be	AUX
flr-250	177	56	available	available	ADJ
flr-250	177	57	to	to	PART
flr-250	177	58	conduct	conduct	VERB
flr-250	177	59	roc	roc	PROPN
flr-250	177	60	analyses	analysis	NOUN
flr-250	177	61	krupinski	krupinski	NOUN
flr-250	178	1	|	|	ADV
flr-250	178	2	f	f	NOUN
flr-250	178	3	l	l	NOUN
flr-250	178	4	r	r	NOUN
flr-250	178	5	40	40	NUM
flr-250	178	6	references	reference	NOUN
flr-250	178	7	analyse	analyse	NOUN
flr-250	178	8	-	-	PUNCT
flr-250	178	9	it	it	PRON
flr-250	178	10	.	.	PUNCT
flr-250	179	1	http://analyse-it.com/docs/220/method_evaluation/roc_curve_plot.htm	http://analyse-it.com/docs/220/method_evaluation/roc_curve_plot.htm	PROPN
flr-250	179	2	last	last	ADV
flr-250	179	3	accessed	access	VERB
flr-250	179	4	april	april	PROPN
flr-250	179	5	13	13	NUM
flr-250	179	6	,	,	PUNCT
flr-250	179	7	2016	2016	NUM
flr-250	179	8	.	.	PUNCT
flr-250	180	1	birkelo	birkelo	PROPN
flr-250	180	2	,	,	PUNCT
flr-250	180	3	c.c	c.c	PROPN
flr-250	180	4	.	.	PROPN
flr-250	180	5	,	,	PUNCT
flr-250	180	6	chamberlain	chamberlain	PROPN
flr-250	180	7	,	,	PUNCT
flr-250	180	8	w.e	w.e	PROPN
flr-250	180	9	.	.	PROPN
flr-250	180	10	,	,	PUNCT
flr-250	180	11	phelps	phelps	PROPN
flr-250	180	12	,	,	PUNCT
flr-250	180	13	p.s	p.s	PROPN
flr-250	180	14	.	.	PUNCT
flr-250	180	15	(	(	PUNCT
flr-250	180	16	1947	1947	NUM
flr-250	180	17	)	)	PUNCT
flr-250	180	18	.	.	PUNCT
flr-250	181	1	tuberculosis	tuberculosis	NOUN
flr-250	181	2	case	case	NOUN
flr-250	181	3	finding	finding	NOUN
flr-250	181	4	.	.	PUNCT
flr-250	182	1	a	a	DET
flr-250	182	2	comparison	comparison	NOUN
flr-250	182	3	of	of	ADP
flr-250	182	4	the	the	DET
flr-250	182	5	effectiveness	effectiveness	NOUN
flr-250	182	6	of	of	ADP
flr-250	182	7	various	various	ADJ
flr-250	182	8	roentgenographic	roentgenographic	ADJ
flr-250	182	9	and	and	CCONJ
flr-250	182	10	photofluorographic	photofluorographic	ADJ
flr-250	182	11	methods	method	NOUN
flr-250	182	12	.	.	PUNCT
flr-250	183	1	jama	jama	PROPN
flr-250	183	2	,	,	PUNCT
flr-250	183	3	133	133	NUM
flr-250	183	4	,	,	PUNCT
flr-250	183	5	(	(	PUNCT
flr-250	183	6	6	6	NUM
flr-250	183	7	)	)	PUNCT
flr-250	183	8	,	,	PUNCT
flr-250	183	9	359	359	NUM
flr-250	183	10	-	-	SYM
flr-250	183	11	366	366	NUM
flr-250	183	12	.	.	PUNCT
flr-250	184	1	pmid	pmid	NOUN
flr-250	184	2	:	:	PUNCT
flr-250	184	3	20281873	20281873	NUM
flr-250	184	4	bunch	bunch	NOUN
flr-250	184	5	,	,	PUNCT
flr-250	184	6	p.c	p.c	PROPN
flr-250	184	7	.	.	PROPN
flr-250	184	8	,	,	PUNCT
flr-250	184	9	hamilton	hamilton	PROPN
flr-250	184	10	,	,	PUNCT
flr-250	184	11	j.f	j.f	PROPN
flr-250	184	12	.	.	PROPN
flr-250	184	13	,	,	PUNCT
flr-250	184	14	sanderson	sanderson	PROPN
flr-250	184	15	,	,	PUNCT
flr-250	184	16	g.k	g.k	PROPN
flr-250	184	17	.	.	PROPN
flr-250	184	18	simmons	simmons	PROPN
flr-250	184	19	,	,	PUNCT
flr-250	184	20	a.h	a.h	PROPN
flr-250	184	21	.	.	PROPN
flr-250	184	22	(	(	PUNCT
flr-250	184	23	1978	1978	NUM
flr-250	184	24	)	)	PUNCT
flr-250	184	25	.	.	PUNCT
flr-250	185	1	a	a	DET
flr-250	185	2	free	free	ADJ
flr-250	185	3	-	-	PUNCT
flr-250	185	4	response	response	NOUN
flr-250	185	5	approach	approach	NOUN
flr-250	185	6	to	to	ADP
flr-250	185	7	the	the	DET
flr-250	185	8	measurement	measurement	NOUN
flr-250	185	9	and	and	CCONJ
flr-250	185	10	characterization	characterization	NOUN
flr-250	185	11	of	of	ADP
flr-250	185	12	radiographic	radiographic	ADJ
flr-250	185	13	-	-	PUNCT
flr-250	185	14	observer	observer	NOUN
flr-250	185	15	performance	performance	NOUN
flr-250	185	16	.	.	PUNCT
flr-250	186	1	j	j	PROPN
flr-250	186	2	appl	appl	PROPN
flr-250	186	3	photogr	photogr	PROPN
flr-250	186	4	eng	eng	PROPN
flr-250	186	5	,	,	PUNCT
flr-250	186	6	4,166	4,166	NUM
flr-250	186	7	–	–	PUNCT
flr-250	186	8	171	171	NUM
flr-250	186	9	.	.	PUNCT
flr-250	187	1	chakraborty	chakraborty	PROPN
flr-250	187	2	,	,	PUNCT
flr-250	187	3	d.p	d.p	PROPN
flr-250	187	4	.	.	PROPN
flr-250	187	5	,	,	PUNCT
flr-250	187	6	berbaum	berbaum	PROPN
flr-250	187	7	,	,	PUNCT
flr-250	187	8	k.s	k.s	PROPN
flr-250	187	9	.	.	PROPN
flr-250	187	10	(	(	PUNCT
flr-250	187	11	2004	2004	NUM
flr-250	187	12	)	)	PUNCT
flr-250	187	13	.	.	PUNCT
flr-250	188	1	observer	observer	NOUN
flr-250	188	2	studies	study	NOUN
flr-250	188	3	involving	involve	VERB
flr-250	188	4	detection	detection	NOUN
flr-250	188	5	and	and	CCONJ
flr-250	188	6	localization	localization	NOUN
flr-250	188	7	:	:	PUNCT
flr-250	188	8	modeling	modeling	NOUN
flr-250	188	9	,	,	PUNCT
flr-250	188	10	analysis	analysis	NOUN
flr-250	188	11	,	,	PUNCT
flr-250	188	12	and	and	CCONJ
flr-250	188	13	validation	validation	NOUN
flr-250	188	14	.	.	PUNCT
flr-250	189	1	med	med	ADJ
flr-250	189	2	phys	phy	NOUN
flr-250	189	3	,	,	PUNCT
flr-250	189	4	31	31	NUM
flr-250	189	5	(	(	PUNCT
flr-250	189	6	8)	8)	NUM
flr-250	189	7	,	,	PUNCT
flr-250	189	8	2313	2313	NUM
flr-250	189	9	-	-	SYM
flr-250	189	10	2330	2330	NUM
flr-250	189	11	.	.	PUNCT
flr-250	190	1	doi	doi	NOUN
flr-250	190	2	:	:	PUNCT
flr-250	190	3	10.1118/1.1769352	10.1118/1.1769352	NUM
flr-250	190	4	chakraborty	chakraborty	NOUN
flr-250	190	5	,	,	PUNCT
flr-250	190	6	d.p	d.p	PROPN
flr-250	190	7	.	.	PUNCT
flr-250	191	1	(	(	PUNCT
flr-250	191	2	2005	2005	NUM
flr-250	191	3	)	)	PUNCT
flr-250	191	4	.	.	PUNCT
flr-250	192	1	recent	recent	ADJ
flr-250	192	2	advances	advance	NOUN
flr-250	192	3	in	in	ADP
flr-250	192	4	observer	observer	NOUN
flr-250	192	5	performance	performance	NOUN
flr-250	192	6	methodology	methodology	NOUN
flr-250	192	7	:	:	PUNCT
flr-250	192	8	jackknife	jackknife	NOUN
flr-250	192	9	free	free	ADJ
flr-250	192	10	-	-	PUNCT
flr-250	192	11	response	response	NOUN
flr-250	192	12	roc	roc	PROPN
flr-250	192	13	(	(	PUNCT
flr-250	192	14	jafroc	jafroc	ADJ
flr-250	192	15	)	)	PUNCT
flr-250	192	16	.	.	PUNCT
flr-250	193	1	rad	rad	PROPN
flr-250	193	2	protect	protect	VERB
flr-250	193	3	dosim	dosim	NOUN
flr-250	193	4	,	,	PUNCT
flr-250	193	5	114	114	NUM
flr-250	193	6	(	(	PUNCT
flr-250	193	7	1	1	NUM
flr-250	193	8	)	)	PUNCT
flr-250	193	9	,	,	PUNCT
flr-250	193	10	26	26	NUM
flr-250	193	11	-	-	SYM
flr-250	193	12	31	31	NUM
flr-250	193	13	.	.	PUNCT
flr-250	194	1	doi	doi	NOUN
flr-250	194	2	:	:	PUNCT
flr-250	194	3	10.1093	10.1093	NUM
flr-250	194	4	/	/	SYM
flr-250	194	5	rpd	rpd	PROPN
flr-250	194	6	/	/	SYM
flr-250	194	7	nch512	nch512	PROPN
flr-250	194	8	chakraborty	chakraborty	PROPN
flr-250	194	9	,	,	PUNCT
flr-250	194	10	d.p	d.p	PROPN
flr-250	194	11	.	.	PUNCT
flr-250	194	12	(	(	PUNCT
flr-250	194	13	2006	2006	NUM
flr-250	194	14	)	)	PUNCT
flr-250	194	15	.	.	PUNCT
flr-250	195	1	analysis	analysis	NOUN
flr-250	195	2	of	of	ADP
flr-250	195	3	location	location	NOUN
flr-250	195	4	specific	specific	ADJ
flr-250	195	5	observer	observer	NOUN
flr-250	195	6	performance	performance	NOUN
flr-250	195	7	data	datum	NOUN
flr-250	195	8	:	:	PUNCT
flr-250	195	9	validated	validate	VERB
flr-250	195	10	extensions	extension	NOUN
flr-250	195	11	of	of	ADP
flr-250	195	12	the	the	DET
flr-250	195	13	jackknife	jackknife	NOUN
flr-250	195	14	free	free	ADJ
flr-250	195	15	-	-	PUNCT
flr-250	195	16	response	response	NOUN
flr-250	195	17	(	(	PUNCT
flr-250	195	18	jafroc	jafroc	ADJ
flr-250	195	19	)	)	PUNCT
flr-250	195	20	method	method	NOUN
flr-250	195	21	.	.	PUNCT
flr-250	196	1	acad	acad	PROPN
flr-250	196	2	radiol	radiol	NOUN
flr-250	196	3	,	,	PUNCT
flr-250	196	4	13	13	NUM
flr-250	196	5	(	(	PUNCT
flr-250	196	6	10	10	NUM
flr-250	196	7	)	)	PUNCT
flr-250	196	8	,	,	PUNCT
flr-250	196	9	1187	1187	NUM
flr-250	196	10	-	-	SYM
flr-250	196	11	1193	1193	NUM
flr-250	196	12	.	.	PUNCT
flr-250	197	1	doi	doi	NOUN
flr-250	197	2	:	:	PUNCT
flr-250	197	3	10.1016	10.1016	NUM
flr-250	197	4	/	/	SYM
flr-250	197	5	j.acra.2006.06.016	j.acra.2006.06.016	PROPN
flr-250	197	6	chakraborty	chakraborty	PROPN
flr-250	197	7	,	,	PUNCT
flr-250	197	8	d.p	d.p	PROPN
flr-250	197	9	.	.	PROPN
flr-250	197	10	,	,	PUNCT
flr-250	197	11	winter	winter	NOUN
flr-250	197	12	,	,	PUNCT
flr-250	197	13	l.h.l	l.h.l	NOUN
flr-250	197	14	.	.	PUNCT
flr-250	197	15	(	(	PUNCT
flr-250	197	16	1990	1990	NUM
flr-250	197	17	)	)	PUNCT
flr-250	197	18	.	.	PUNCT
flr-250	198	1	free	free	ADJ
flr-250	198	2	-	-	PUNCT
flr-250	198	3	response	response	NOUN
flr-250	198	4	methodology	methodology	NOUN
flr-250	198	5	:	:	PUNCT
flr-250	198	6	alternate	alternate	VERB
flr-250	198	7	analysis	analysis	NOUN
flr-250	198	8	and	and	CCONJ
flr-250	198	9	the	the	DET
flr-250	198	10	new	new	ADJ
flr-250	198	11	observer	observer	NOUN
flr-250	198	12	-	-	PUNCT
flr-250	198	13	performance	performance	NOUN
flr-250	198	14	experiment	experiment	NOUN
flr-250	198	15	.	.	PUNCT
flr-250	199	1	radiol	radiol	NOUN
flr-250	199	2	,	,	PUNCT
flr-250	199	3	174	174	NUM
flr-250	199	4	(	(	PUNCT
flr-250	199	5	3	3	NUM
flr-250	199	6	)	)	PUNCT
flr-250	199	7	,	,	PUNCT
flr-250	199	8	873	873	NUM
flr-250	199	9	-	-	SYM
flr-250	199	10	881	881	NUM
flr-250	199	11	.	.	PUNCT
flr-250	200	1	doi	doi	NOUN
flr-250	200	2	:	:	PUNCT
flr-250	200	3	10.1148	10.1148	NUM
flr-250	200	4	/	/	SYM
flr-250	200	5	radiology.174.3.2305073	radiology.174.3.2305073	PROPN
flr-250	200	6	delong	delong	PROPN
flr-250	200	7	,	,	PUNCT
flr-250	200	8	e.r	e.r	PROPN
flr-250	200	9	.	.	PROPN
flr-250	200	10	,	,	PUNCT
flr-250	200	11	delong	delong	PROPN
flr-250	200	12	,	,	PUNCT
flr-250	200	13	d.m	d.m	PROPN
flr-250	200	14	.	.	PROPN
flr-250	200	15	,	,	PUNCT
flr-250	200	16	clarke	clarke	PROPN
flr-250	200	17	-	-	PUNCT
flr-250	200	18	pearson	pearson	PROPN
flr-250	200	19	,	,	PUNCT
flr-250	200	20	d.l	d.l	PROPN
flr-250	200	21	.	.	PROPN
flr-250	200	22	(	(	PUNCT
flr-250	200	23	19880	19880	NUM
flr-250	200	24	.	.	PUNCT
flr-250	201	1	comparing	compare	VERB
flr-250	201	2	the	the	DET
flr-250	201	3	areas	area	NOUN
flr-250	201	4	under	under	ADP
flr-250	201	5	two	two	NUM
flr-250	201	6	or	or	CCONJ
flr-250	201	7	more	more	ADV
flr-250	201	8	correlated	correlate	VERB
flr-250	201	9	receiver	receiver	ADJ
flr-250	201	10	operating	operate	VERB
flr-250	201	11	characteristics	characteristic	NOUN
flr-250	201	12	curves	curve	NOUN
flr-250	201	13	:	:	PUNCT
flr-250	201	14	a	a	DET
flr-250	201	15	non	non	ADJ
flr-250	201	16	-	-	ADJ
flr-250	201	17	parametric	parametric	ADJ
flr-250	201	18	approach	approach	NOUN
flr-250	201	19	.	.	PUNCT
flr-250	202	1	biometrics	biometric	NOUN
flr-250	202	2	,	,	PUNCT
flr-250	202	3	44	44	NUM
flr-250	202	4	(	(	PUNCT
flr-250	202	5	3	3	NUM
flr-250	202	6	)	)	PUNCT
flr-250	202	7	,	,	PUNCT
flr-250	202	8	837	837	NUM
flr-250	202	9	-	-	SYM
flr-250	202	10	845	845	NUM
flr-250	202	11	.	.	PUNCT
flr-250	203	1	pmid	pmid	NOUN
flr-250	203	2	:	:	PUNCT
flr-250	203	3	3203132	3203132	NUM
flr-250	203	4	dev	dev	PROPN
flr-250	203	5	chakraborty	chakraborty	PROPN
flr-250	203	6	’s	’s	PART
flr-250	203	7	froc	froc	PROPN
flr-250	203	8	web	web	NOUN
flr-250	203	9	site	site	NOUN
flr-250	203	10	.	.	PUNCT
flr-250	204	1	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/defaul	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/defaul	NOUN
flr-250	204	2	t.aspx	t.aspx	NOUN
flr-250	204	3	last	last	ADV
flr-250	204	4	accessed	access	VERB
flr-250	204	5	april	april	PROPN
flr-250	204	6	13	13	NUM
flr-250	204	7	,	,	PUNCT
flr-250	204	8	2016	2016	NUM
flr-250	204	9	.	.	PUNCT
flr-250	205	1	dorfmam	dorfmam	NOUN
flr-250	205	2	,	,	PUNCT
flr-250	205	3	d.d	d.d	PROPN
flr-250	205	4	.	.	PROPN
flr-250	205	5	,	,	PUNCT
flr-250	205	6	alf	alf	PROPN
flr-250	205	7	,	,	PUNCT
flr-250	205	8	e.	e.	PROPN
flr-250	205	9	(	(	PUNCT
flr-250	205	10	1968	1968	NUM
flr-250	205	11	)	)	PUNCT
flr-250	205	12	.	.	PUNCT
flr-250	206	1	maximum	maximum	ADJ
flr-250	206	2	likelihood	likelihood	NOUN
flr-250	206	3	estimation	estimation	NOUN
flr-250	206	4	of	of	ADP
flr-250	206	5	parameters	parameter	NOUN
flr-250	206	6	of	of	ADP
flr-250	206	7	signal	signal	ADJ
flr-250	206	8	detection	detection	NOUN
flr-250	206	9	theory	theory	NOUN
flr-250	206	10	:	:	PUNCT
flr-250	206	11	a	a	DET
flr-250	206	12	direct	direct	ADJ
flr-250	206	13	solution	solution	NOUN
flr-250	206	14	.	.	PUNCT
flr-250	207	1	psychometrika	psychometrika	NOUN
flr-250	207	2	,	,	PUNCT
flr-250	207	3	33	33	NUM
flr-250	207	4	(	(	PUNCT
flr-250	207	5	1	1	NUM
flr-250	207	6	)	)	PUNCT
flr-250	207	7	,	,	PUNCT
flr-250	207	8	117	117	NUM
flr-250	207	9	-	-	SYM
flr-250	207	10	124	124	NUM
flr-250	207	11	.	.	PUNCT
flr-250	208	1	doi	doi	NOUN
flr-250	208	2	:	:	PUNCT
flr-250	208	3	10.1007	10.1007	NUM
flr-250	208	4	/	/	SYM
flr-250	208	5	bf02289677	bf02289677	ADJ
flr-250	208	6	dorfman	dorfman	PROPN
flr-250	208	7	,	,	PUNCT
flr-250	208	8	d.d	d.d	PROPN
flr-250	208	9	.	.	PROPN
flr-250	208	10	,	,	PUNCT
flr-250	208	11	alf	alf	PROPN
flr-250	208	12	,	,	PUNCT
flr-250	208	13	e.	e.	PROPN
flr-250	208	14	(	(	PUNCT
flr-250	208	15	1969	1969	NUM
flr-250	208	16	)	)	PUNCT
flr-250	208	17	.	.	PUNCT
flr-250	209	1	maximum	maximum	ADJ
flr-250	209	2	likelihood	likelihood	NOUN
flr-250	209	3	estimation	estimation	NOUN
flr-250	209	4	of	of	ADP
flr-250	209	5	parameters	parameter	NOUN
flr-250	209	6	of	of	ADP
flr-250	209	7	signal	signal	ADJ
flr-250	209	8	detection	detection	NOUN
flr-250	209	9	theory	theory	NOUN
flr-250	209	10	and	and	CCONJ
flr-250	209	11	determination	determination	NOUN
flr-250	209	12	of	of	ADP
flr-250	209	13	confidence	confidence	NOUN
flr-250	209	14	intervals	interval	NOUN
flr-250	209	15	–	–	PUNCT
flr-250	209	16	rating	rating	NOUN
flr-250	209	17	method	method	NOUN
flr-250	209	18	data	datum	NOUN
flr-250	209	19	.	.	PUNCT
flr-250	210	1	j	j	PROPN
flr-250	210	2	math	math	PROPN
flr-250	210	3	psychol	psychol	PROPN
flr-250	210	4	,	,	PUNCT
flr-250	210	5	6	6	NUM
flr-250	210	6	(	(	PUNCT
flr-250	210	7	3	3	NUM
flr-250	210	8	)	)	PUNCT
flr-250	210	9	,	,	PUNCT
flr-250	210	10	487	487	NUM
flr-250	210	11	-	-	SYM
flr-250	210	12	496	496	NUM
flr-250	210	13	.	.	PUNCT
flr-250	211	1	doi	doi	NOUN
flr-250	211	2	:	:	PUNCT
flr-250	211	3	http://dx.doi.org/10.1016/0022-2496(69)90019-4	http://dx.doi.org/10.1016/0022-2496(69)90019-4	PROPN
flr-250	211	4	dorfman	dorfman	PROPN
flr-250	211	5	,	,	PUNCT
flr-250	211	6	d.d	d.d	PROPN
flr-250	211	7	.	.	PROPN
flr-250	211	8	,	,	PUNCT
flr-250	211	9	berbaum	berbaum	PROPN
flr-250	211	10	,	,	PUNCT
flr-250	211	11	k.s	k.s	PROPN
flr-250	211	12	.	.	PROPN
flr-250	211	13	,	,	PUNCT
flr-250	211	14	metz	metz	PROPN
flr-250	211	15	,	,	PUNCT
flr-250	211	16	c.e	c.e	PROPN
flr-250	211	17	.	.	PROPN
flr-250	211	18	(	(	PUNCT
flr-250	211	19	1992	1992	NUM
flr-250	211	20	)	)	PUNCT
flr-250	211	21	.	.	PUNCT
flr-250	212	1	receiver	receiver	ADJ
flr-250	212	2	operating	operate	VERB
flr-250	212	3	characteristic	characteristic	ADJ
flr-250	212	4	rating	rating	NOUN
flr-250	212	5	analysis	analysis	NOUN
flr-250	212	6	:	:	PUNCT
flr-250	212	7	generalization	generalization	NOUN
flr-250	212	8	to	to	ADP
flr-250	212	9	the	the	DET
flr-250	212	10	population	population	NOUN
flr-250	212	11	of	of	ADP
flr-250	212	12	readers	reader	NOUN
flr-250	212	13	and	and	CCONJ
flr-250	212	14	patients	patient	NOUN
flr-250	212	15	with	with	ADP
flr-250	212	16	the	the	DET
flr-250	212	17	jackknife	jackknife	NOUN
flr-250	212	18	method	method	NOUN
flr-250	212	19	.	.	PUNCT
flr-250	213	1	invest	invest	VERB
flr-250	213	2	radiol	radiol	NOUN
flr-250	213	3	,	,	PUNCT
flr-250	213	4	27(9	27(9	NOUN
flr-250	213	5	)	)	PUNCT
flr-250	213	6	,	,	PUNCT
flr-250	213	7	723731	723731	NUM
flr-250	213	8	.	.	PUNCT
flr-250	214	1	pmid	pmid	NOUN
flr-250	214	2	:	:	PUNCT
flr-250	214	3	1399456	1399456	NUM
flr-250	214	4	dorfman	dorfman	PROPN
flr-250	214	5	,	,	PUNCT
flr-250	214	6	d.d	d.d	PROPN
flr-250	214	7	.	.	PROPN
flr-250	214	8	,	,	PUNCT
flr-250	214	9	berbaum	berbaum	PROPN
flr-250	214	10	,	,	PUNCT
flr-250	214	11	k.s	k.s	PROPN
flr-250	214	12	.	.	PROPN
flr-250	214	13	,	,	PUNCT
flr-250	214	14	metz	metz	PROPN
flr-250	214	15	.	.	PUNCT
flr-250	215	1	c.e	c.e	PROPN
flr-250	215	2	.	.	PROPN
flr-250	215	3	,	,	PUNCT
flr-250	215	4	lenth	lenth	PROPN
flr-250	215	5	,	,	PUNCT
flr-250	215	6	r.v	r.v	PROPN
flr-250	215	7	.	.	PROPN
flr-250	215	8	,	,	PUNCT
flr-250	215	9	hanley	hanley	PROPN
flr-250	215	10	,	,	PUNCT
flr-250	215	11	j.a	j.a	PROPN
flr-250	215	12	.	.	PROPN
flr-250	215	13	,	,	PUNCT
flr-250	215	14	dagga	dagga	PROPN
flr-250	215	15	,	,	PUNCT
flr-250	215	16	h.a	h.a	PROPN
flr-250	215	17	.	.	PROPN
flr-250	215	18	(	(	PUNCT
flr-250	215	19	1997	1997	NUM
flr-250	215	20	)	)	PUNCT
flr-250	215	21	.	.	PUNCT
flr-250	216	1	proper	proper	ADJ
flr-250	216	2	receiver	receiver	NOUN
flr-250	216	3	operating	operate	VERB
flr-250	216	4	characteristic	characteristic	ADJ
flr-250	216	5	analysis	analysis	NOUN
flr-250	216	6	:	:	PUNCT
flr-250	216	7	the	the	DET
flr-250	216	8	bigamma	bigamma	PROPN
flr-250	216	9	model	model	PROPN
flr-250	216	10	.	.	PUNCT
flr-250	217	1	acad	acad	PROPN
flr-250	217	2	radiol	radiol	NOUN
flr-250	217	3	,	,	PUNCT
flr-250	217	4	4	4	NUM
flr-250	217	5	(	(	PUNCT
flr-250	217	6	2	2	NUM
flr-250	217	7	)	)	PUNCT
flr-250	217	8	,	,	PUNCT
flr-250	217	9	138	138	NUM
flr-250	217	10	-	-	SYM
flr-250	217	11	149	149	NUM
flr-250	217	12	.	.	PUNCT
flr-250	218	1	pmid	pmid	NOUN
flr-250	218	2	:	:	PUNCT
flr-250	218	3	9061087	9061087	NUM
flr-250	218	4	edwards	edwards	PROPN
flr-250	218	5	,	,	PUNCT
flr-250	218	6	d.c	d.c	PROPN
flr-250	218	7	.	.	PROPN
flr-250	218	8	(	(	PUNCT
flr-250	218	9	2013	2013	NUM
flr-250	218	10	)	)	PUNCT
flr-250	218	11	.	.	PUNCT
flr-250	219	1	validation	validation	NOUN
flr-250	219	2	of	of	ADP
flr-250	219	3	monte	monte	PROPN
flr-250	219	4	carlo	carlo	PROPN
flr-250	219	5	estimates	estimate	NOUN
flr-250	219	6	of	of	ADP
flr-250	219	7	three	three	NUM
flr-250	219	8	-	-	PUNCT
flr-250	219	9	class	class	NOUN
flr-250	219	10	ideal	ideal	ADJ
flr-250	219	11	observer	observer	NOUN
flr-250	219	12	operating	operating	NOUN
flr-250	219	13	points	point	NOUN
flr-250	219	14	for	for	ADP
flr-250	219	15	normal	normal	ADJ
flr-250	219	16	data	datum	NOUN
flr-250	219	17	.	.	PUNCT
flr-250	220	1	acad	acad	PROPN
flr-250	220	2	radiol	radiol	NOUN
flr-250	220	3	,	,	PUNCT
flr-250	220	4	20	20	NUM
flr-250	220	5	(	(	PUNCT
flr-250	220	6	7	7	NUM
flr-250	220	7	)	)	PUNCT
flr-250	220	8	,	,	PUNCT
flr-250	220	9	908	908	NUM
flr-250	220	10	-	-	SYM
flr-250	220	11	914	914	NUM
flr-250	220	12	.	.	PUNCT
flr-250	221	1	doi	doi	NOUN
flr-250	221	2	:	:	PUNCT
flr-250	221	3	10.1016	10.1016	NUM
flr-250	221	4	/	/	SYM
flr-250	221	5	j.acra.2013.04.002	j.acra.2013.04.002	PROPN
flr-250	221	6	egan	egan	PROPN
flr-250	221	7	,	,	PUNCT
flr-250	221	8	j.p	j.p	PROPN
flr-250	221	9	.	.	PROPN
flr-250	221	10	(	(	PUNCT
flr-250	221	11	1975	1975	NUM
flr-250	221	12	)	)	PUNCT
flr-250	221	13	.	.	PUNCT
flr-250	222	1	signal	signal	PROPN
flr-250	222	2	detection	detection	NOUN
flr-250	222	3	theory	theory	NOUN
flr-250	222	4	and	and	CCONJ
flr-250	222	5	roc	roc	PROPN
flr-250	222	6	analysis	analysis	NOUN
flr-250	222	7	.	.	PUNCT
flr-250	223	1	new	new	PROPN
flr-250	223	2	york	york	PROPN
flr-250	223	3	,	,	PUNCT
flr-250	223	4	ny	ny	PROPN
flr-250	223	5	:	:	PUNCT
flr-250	223	6	academic	academic	ADJ
flr-250	223	7	press	press	NOUN
flr-250	223	8	.	.	PUNCT
flr-250	224	1	faraggi	faraggi	PROPN
flr-250	224	2	,	,	PUNCT
flr-250	224	3	d.	d.	PROPN
flr-250	224	4	,	,	PUNCT
flr-250	224	5	reiser	reiser	PROPN
flr-250	224	6	,	,	PUNCT
flr-250	224	7	b.	b.	PROPN
flr-250	224	8	(	(	PUNCT
flr-250	224	9	2002	2002	NUM
flr-250	224	10	)	)	PUNCT
flr-250	224	11	.	.	PUNCT
flr-250	225	1	estimation	estimation	NOUN
flr-250	225	2	of	of	ADP
flr-250	225	3	the	the	DET
flr-250	225	4	area	area	NOUN
flr-250	225	5	under	under	ADP
flr-250	225	6	the	the	DET
flr-250	225	7	roc	roc	PROPN
flr-250	225	8	curve	curve	NOUN
flr-250	225	9	.	.	PUNCT
flr-250	226	1	stats	stat	NOUN
flr-250	226	2	med	med	ADJ
flr-250	226	3	,	,	PUNCT
flr-250	226	4	21	21	NUM
flr-250	226	5	(	(	PUNCT
flr-250	226	6	20	20	NUM
flr-250	226	7	)	)	PUNCT
flr-250	226	8	,	,	PUNCT
flr-250	226	9	3093	3093	NUM
flr-250	226	10	-	-	SYM
flr-250	226	11	3106	3106	NUM
flr-250	226	12	.	.	PUNCT
flr-250	227	1	doi	doi	NOUN
flr-250	227	2	:	:	PUNCT
flr-250	227	3	10.1002	10.1002	NUM
flr-250	227	4	/	/	SYM
flr-250	227	5	sim.1228	sim.1228	PROPN
flr-250	227	6	garland	garland	PROPN
flr-250	227	7	,	,	PUNCT
flr-250	227	8	l.h	l.h	PROPN
flr-250	227	9	.	.	PROPN
flr-250	227	10	(	(	PUNCT
flr-250	227	11	1949	1949	NUM
flr-250	227	12	)	)	PUNCT
flr-250	227	13	.	.	PUNCT
flr-250	228	1	on	on	ADP
flr-250	228	2	the	the	DET
flr-250	228	3	scientific	scientific	ADJ
flr-250	228	4	evaluation	evaluation	NOUN
flr-250	228	5	of	of	ADP
flr-250	228	6	diagnostic	diagnostic	ADJ
flr-250	228	7	procedures	procedure	NOUN
flr-250	228	8	.	.	PUNCT
flr-250	229	1	radiol	radiol	NOUN
flr-250	229	2	,	,	PUNCT
flr-250	229	3	52	52	NUM
flr-250	229	4	,	,	PUNCT
flr-250	229	5	(	(	PUNCT
flr-250	229	6	3	3	NUM
flr-250	229	7	)	)	PUNCT
flr-250	229	8	,	,	PUNCT
flr-250	229	9	309	309	NUM
flr-250	229	10	-	-	SYM
flr-250	229	11	328	328	NUM
flr-250	229	12	.	.	PUNCT
flr-250	230	1	doi	doi	NOUN
flr-250	230	2	:	:	PUNCT
flr-250	230	3	http://dx.doi.org/10.1148/52.3.309	http://dx.doi.org/10.1148/52.3.309	NOUN
flr-250	230	4	green	green	PROPN
flr-250	230	5	,	,	PUNCT
flr-250	230	6	d.m	d.m	PROPN
flr-250	230	7	.	.	PROPN
flr-250	230	8	,	,	PUNCT
flr-250	230	9	swets	swet	NOUN
flr-250	230	10	,	,	PUNCT
flr-250	230	11	j.a	j.a	PROPN
flr-250	230	12	.	.	PROPN
flr-250	230	13	(	(	PUNCT
flr-250	230	14	1974	1974	NUM
flr-250	230	15	)	)	PUNCT
flr-250	230	16	.	.	PUNCT
flr-250	231	1	signal	signal	PROPN
flr-250	231	2	detection	detection	NOUN
flr-250	231	3	theory	theory	NOUN
flr-250	231	4	and	and	CCONJ
flr-250	231	5	psychophysics	psychophysic	NOUN
flr-250	231	6	.	.	PUNCT
flr-250	232	1	huntington	huntington	PROPN
flr-250	232	2	,	,	PUNCT
flr-250	232	3	ny	ny	PROPN
flr-250	232	4	:	:	PUNCT
flr-250	232	5	krieger	krieger	PROPN
flr-250	232	6	publishers	publishers	PROPN
flr-250	232	7	.	.	PUNCT
flr-250	233	1	hajian	hajian	ADJ
flr-250	233	2	-	-	PUNCT
flr-250	233	3	tilaki	tilaki	PROPN
flr-250	233	4	,	,	PUNCT
flr-250	233	5	k.o	k.o	PROPN
flr-250	233	6	.	.	PROPN
flr-250	233	7	,	,	PUNCT
flr-250	233	8	hanley	hanley	PROPN
flr-250	233	9	,	,	PUNCT
flr-250	233	10	j.a	j.a	PROPN
flr-250	233	11	.	.	PROPN
flr-250	233	12	,	,	PUNCT
flr-250	233	13	joseph	joseph	PROPN
flr-250	233	14	,	,	PUNCT
flr-250	233	15	l.	l.	PROPN
flr-250	233	16	,	,	PUNCT
flr-250	233	17	collet	collet	PROPN
flr-250	233	18	,	,	PUNCT
flr-250	233	19	j.p	j.p	PROPN
flr-250	233	20	.	.	PROPN
flr-250	233	21	(	(	PUNCT
flr-250	233	22	1997	1997	NUM
flr-250	233	23	)	)	PUNCT
flr-250	233	24	.	.	PUNCT
flr-250	234	1	a	a	DET
flr-250	234	2	comparison	comparison	NOUN
flr-250	234	3	of	of	ADP
flr-250	234	4	parametric	parametric	ADJ
flr-250	234	5	and	and	CCONJ
flr-250	234	6	nonparametric	nonparametric	NOUN
flr-250	234	7	approaches	approach	NOUN
flr-250	234	8	to	to	ADP
flr-250	234	9	roc	roc	NOUN
flr-250	234	10	analysis	analysis	NOUN
flr-250	234	11	of	of	ADP
flr-250	234	12	quantitative	quantitative	ADJ
flr-250	234	13	diagnostic	diagnostic	ADJ
flr-250	234	14	tests	test	NOUN
flr-250	234	15	.	.	PUNCT
flr-250	235	1	med	med	PROPN
flr-250	235	2	decis	decis	PROPN
flr-250	235	3	making	making	NOUN
flr-250	235	4	,	,	PUNCT
flr-250	235	5	17	17	NUM
flr-250	235	6	(	(	PUNCT
flr-250	235	7	1	1	NUM
flr-250	235	8	)	)	PUNCT
flr-250	235	9	,	,	PUNCT
flr-250	235	10	94102	94102	NUM
flr-250	235	11	.	.	PUNCT
flr-250	236	1	doi:10.1177/0272989x9701700111	doi:10.1177/0272989x9701700111	PROPN
flr-250	236	2	hanley	hanley	PROPN
flr-250	236	3	,	,	PUNCT
flr-250	236	4	j.a	j.a	PROPN
flr-250	236	5	.	.	PROPN
flr-250	236	6	(	(	PUNCT
flr-250	236	7	1988	1988	NUM
flr-250	236	8	)	)	PUNCT
flr-250	236	9	.	.	PUNCT
flr-250	237	1	the	the	DET
flr-250	237	2	robustness	robustness	NOUN
flr-250	237	3	of	of	ADP
flr-250	237	4	the	the	DET
flr-250	237	5	“	"	PUNCT
flr-250	237	6	binormal	binormal	NOUN
flr-250	237	7	”	"	PUNCT
flr-250	237	8	assumptions	assumption	NOUN
flr-250	237	9	used	use	VERB
flr-250	237	10	in	in	ADP
flr-250	237	11	fitting	fitting	ADJ
flr-250	237	12	roc	roc	PROPN
flr-250	237	13	curves	curve	NOUN
flr-250	237	14	.	.	PUNCT
flr-250	238	1	med	med	PROPN
flr-250	238	2	decis	decis	PROPN
flr-250	238	3	making	making	NOUN
flr-250	238	4	,	,	PUNCT
flr-250	238	5	8	8	NUM
flr-250	238	6	(	(	PUNCT
flr-250	238	7	3	3	NUM
flr-250	238	8	)	)	PUNCT
flr-250	238	9	,	,	PUNCT
flr-250	238	10	197	197	NUM
flr-250	238	11	-	-	SYM
flr-250	238	12	203	203	NUM
flr-250	238	13	.	.	PUNCT
flr-250	239	1	doi	doi	NOUN
flr-250	239	2	:	:	PUNCT
flr-250	239	3	10.1177/0272989x8800800308	10.1177/0272989x8800800308	PROPN
flr-250	239	4	hanley	hanley	PROPN
flr-250	239	5	,	,	PUNCT
flr-250	239	6	j.a	j.a	PROPN
flr-250	239	7	.	.	PROPN
flr-250	239	8	,	,	PUNCT
flr-250	239	9	mcneil	mcneil	PROPN
flr-250	239	10	,	,	PUNCT
flr-250	239	11	b.j	b.j	PROPN
flr-250	239	12	.	.	PROPN
flr-250	239	13	(	(	PUNCT
flr-250	239	14	1983	1983	NUM
flr-250	239	15	)	)	PUNCT
flr-250	239	16	.	.	PUNCT
flr-250	240	1	a	a	DET
flr-250	240	2	method	method	NOUN
flr-250	240	3	for	for	ADP
flr-250	240	4	comparing	compare	VERB
flr-250	240	5	the	the	DET
flr-250	240	6	areas	area	NOUN
flr-250	240	7	under	under	ADP
flr-250	240	8	receiver	receiver	NOUN
flr-250	240	9	operating	operate	VERB
flr-250	240	10	characteristic	characteristic	ADJ
flr-250	240	11	curves	curve	NOUN
flr-250	240	12	derived	derive	VERB
flr-250	240	13	from	from	ADP
flr-250	240	14	the	the	DET
flr-250	240	15	same	same	ADJ
flr-250	240	16	cases	case	NOUN
flr-250	240	17	.	.	PUNCT
flr-250	241	1	radiol	radiol	NOUN
flr-250	241	2	,	,	PUNCT
flr-250	241	3	148	148	NUM
flr-250	241	4	(	(	PUNCT
flr-250	241	5	3	3	NUM
flr-250	241	6	)	)	PUNCT
flr-250	241	7	,	,	PUNCT
flr-250	241	8	839	839	NUM
flr-250	241	9	-	-	SYM
flr-250	241	10	843	843	NUM
flr-250	241	11	.	.	PUNCT
flr-250	242	1	doi	doi	NOUN
flr-250	242	2	:	:	PUNCT
flr-250	242	3	10.1148	10.1148	NUM
flr-250	242	4	/	/	SYM
flr-250	242	5	radiology.148.3.6878708	radiology.148.3.6878708	NOUN
flr-250	242	6	http://analyse-it.com/docs/220/method_evaluation/roc_curve_plot.htm	http://analyse-it.com/docs/220/method_evaluation/roc_curve_plot.htm	NOUN
flr-250	242	7	https://doi.org/10.1118/1.1769352	https://doi.org/10.1118/1.1769352	PROPN
flr-250	243	1	http://dx.doi.org/10.1093/rpd/nch512	http://dx.doi.org/10.1093/rpd/nch512	PROPN
flr-250	243	2	https://doi.org/10.1016/j.acra.2006.06.016	https://doi.org/10.1016/j.acra.2006.06.016	PROPN
flr-250	243	3	https://doi.org/10.1148/radiology.174.3.2305073	https://doi.org/10.1148/radiology.174.3.2305073	NOUN
flr-250	243	4	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/default.aspx	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/default.aspx	NOUN
flr-250	243	5	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/default.aspx	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/default.aspx	NOUN
flr-250	244	1	http://dx.doi.org/10.1016/0022-2496%2869%2990019-4	http://dx.doi.org/10.1016/0022-2496%2869%2990019-4	PRON
flr-250	244	2	https://doi.org/10.1002/sim.1228	https://doi.org/10.1002/sim.1228	PROPN
flr-250	244	3	http://dx.doi.org/10.1148/52.3.309	http://dx.doi.org/10.1148/52.3.309	PROPN
flr-250	244	4	https://doi.org/10.1177/0272989x9701700111	https://doi.org/10.1177/0272989x9701700111	PROPN
flr-250	244	5	https://doi.org/10.1177/0272989x8800800308	https://doi.org/10.1177/0272989x8800800308	PROPN
flr-250	244	6	https://doi.org/10.1148/radiology.148.3.6878708	https://doi.org/10.1148/radiology.148.3.6878708	ADJ
flr-250	244	7	krupinski	krupinski	NOUN
flr-250	245	1	|	|	INTJ
flr-250	245	2	f	f	NOUN
flr-250	245	3	l	l	NOUN
flr-250	245	4	r	r	NOUN
flr-250	245	5	41	41	NUM
flr-250	245	6	hanley	hanley	NOUN
flr-250	245	7	,	,	PUNCT
flr-250	245	8	j.a	j.a	PROPN
flr-250	245	9	.	.	PROPN
flr-250	245	10	,	,	PUNCT
flr-250	245	11	mcneil	mcneil	PROPN
flr-250	245	12	,	,	PUNCT
flr-250	245	13	b.j	b.j	PROPN
flr-250	245	14	.	.	PROPN
flr-250	245	15	(	(	PUNCT
flr-250	245	16	1982	1982	NUM
flr-250	245	17	)	)	PUNCT
flr-250	245	18	.	.	PUNCT
flr-250	246	1	the	the	DET
flr-250	246	2	meaning	meaning	NOUN
flr-250	246	3	and	and	CCONJ
flr-250	246	4	use	use	NOUN
flr-250	246	5	of	of	ADP
flr-250	246	6	the	the	DET
flr-250	246	7	area	area	NOUN
flr-250	246	8	under	under	ADP
flr-250	246	9	a	a	DET
flr-250	246	10	receiver	receiver	ADJ
flr-250	246	11	operating	operate	VERB
flr-250	246	12	characteristic	characteristic	NOUN
flr-250	246	13	(	(	PUNCT
flr-250	246	14	roc	roc	PROPN
flr-250	246	15	)	)	PUNCT
flr-250	246	16	curve	curve	NOUN
flr-250	246	17	.	.	PUNCT
flr-250	247	1	radiol	radiol	NOUN
flr-250	247	2	,	,	PUNCT
flr-250	247	3	143	143	NUM
flr-250	247	4	(	(	PUNCT
flr-250	247	5	1	1	NUM
flr-250	247	6	)	)	PUNCT
flr-250	247	7	,	,	PUNCT
flr-250	247	8	29	29	NUM
flr-250	247	9	-	-	SYM
flr-250	247	10	36	36	NUM
flr-250	247	11	.	.	PUNCT
flr-250	248	1	doi	doi	PROPN
flr-250	248	2	:	:	PUNCT
flr-250	248	3	http://dx.doi.org/10.1148/radiology.143.1.7063747	http://dx.doi.org/10.1148/radiology.143.1.7063747	PROPN
flr-250	248	4	institute	institute	PROPN
flr-250	248	5	of	of	ADP
flr-250	248	6	medicine	medicine	NOUN
flr-250	248	7	.	.	PUNCT
flr-250	249	1	(	(	PUNCT
flr-250	249	2	1999	1999	NUM
flr-250	249	3	)	)	PUNCT
flr-250	249	4	.	.	PUNCT
flr-250	250	1	to	to	ADP
flr-250	250	2	err	err	NOUN
flr-250	250	3	is	be	AUX
flr-250	250	4	human	human	ADJ
flr-250	250	5	:	:	PUNCT
flr-250	250	6	building	build	VERB
flr-250	250	7	a	a	DET
flr-250	250	8	safer	safe	ADJ
flr-250	250	9	health	health	NOUN
flr-250	250	10	care	care	NOUN
flr-250	250	11	system	system	NOUN
flr-250	250	12	.	.	PUNCT
flr-250	251	1	washington	washington	PROPN
flr-250	251	2	,	,	PUNCT
flr-250	251	3	dc	dc	PROPN
flr-250	251	4	:	:	PUNCT
flr-250	251	5	national	national	PROPN
flr-250	251	6	academy	academy	PROPN
flr-250	251	7	press	press	PROPN
flr-250	251	8	.	.	PUNCT
flr-250	252	1	jiang	jiang	PROPN
flr-250	252	2	,	,	PUNCT
flr-250	252	3	y.	y.	PROPN
flr-250	252	4	,	,	PUNCT
flr-250	252	5	metz	metz	PROPN
flr-250	252	6	,	,	PUNCT
flr-250	252	7	c.e	c.e	PROPN
flr-250	252	8	.	.	PROPN
flr-250	252	9	,	,	PUNCT
flr-250	252	10	nishikawa	nishikawa	PROPN
flr-250	252	11	,	,	PUNCT
flr-250	252	12	r.m	r.m	PROPN
flr-250	252	13	.	.	PROPN
flr-250	252	14	(	(	PUNCT
flr-250	252	15	1996	1996	NUM
flr-250	252	16	)	)	PUNCT
flr-250	252	17	.	.	PUNCT
flr-250	253	1	a	a	DET
flr-250	253	2	receiver	receiver	ADV
flr-250	253	3	operating	operate	VERB
flr-250	253	4	characteristic	characteristic	ADJ
flr-250	253	5	partial	partial	ADJ
flr-250	253	6	area	area	NOUN
flr-250	253	7	index	index	NOUN
flr-250	253	8	for	for	ADP
flr-250	253	9	highly	highly	ADV
flr-250	253	10	sensitive	sensitive	ADJ
flr-250	253	11	diagnostic	diagnostic	ADJ
flr-250	253	12	tests	test	NOUN
flr-250	253	13	.	.	PUNCT
flr-250	254	1	radiol	radiol	NOUN
flr-250	254	2	,	,	PUNCT
flr-250	254	3	201	201	NUM
flr-250	254	4	(	(	PUNCT
flr-250	254	5	3	3	NUM
flr-250	254	6	)	)	PUNCT
flr-250	254	7	,	,	PUNCT
flr-250	254	8	745	745	NUM
flr-250	254	9	-	-	SYM
flr-250	254	10	750	750	NUM
flr-250	254	11	.	.	PUNCT
flr-250	255	1	doi	doi	NOUN
flr-250	255	2	:	:	PUNCT
flr-250	255	3	10.1148	10.1148	NUM
flr-250	255	4	/	/	SYM
flr-250	255	5	radiology.201.3.8939225	radiology.201.3.8939225	PROPN
flr-250	255	6	kundel	kundel	NOUN
flr-250	255	7	,	,	PUNCT
flr-250	255	8	h.l	h.l	PROPN
flr-250	255	9	.	.	PROPN
flr-250	255	10	,	,	PUNCT
flr-250	255	11	polansky	polansky	NOUN
flr-250	255	12	,	,	PUNCT
flr-250	255	13	m.	m.	NOUN
flr-250	255	14	(	(	PUNCT
flr-250	255	15	1997	1997	NUM
flr-250	255	16	)	)	PUNCT
flr-250	255	17	.	.	PUNCT
flr-250	256	1	mixture	mixture	NOUN
flr-250	256	2	distribution	distribution	NOUN
flr-250	256	3	and	and	CCONJ
flr-250	256	4	receiver	receiver	NOUN
flr-250	256	5	operating	operate	VERB
flr-250	256	6	characteristic	characteristic	ADJ
flr-250	256	7	analysis	analysis	NOUN
flr-250	256	8	of	of	ADP
flr-250	256	9	bedside	bedside	NOUN
flr-250	256	10	chest	chest	NOUN
flr-250	256	11	imaging	imaging	NOUN
flr-250	256	12	with	with	ADP
flr-250	256	13	screen	screen	NOUN
flr-250	256	14	-	-	PUNCT
flr-250	256	15	film	film	NOUN
flr-250	256	16	and	and	CCONJ
flr-250	256	17	computed	compute	VERB
flr-250	256	18	radiography	radiography	NOUN
flr-250	256	19	.	.	PUNCT
flr-250	257	1	acad	acad	PROPN
flr-250	257	2	radiol	radiol	NOUN
flr-250	257	3	,	,	PUNCT
flr-250	257	4	4	4	NUM
flr-250	257	5	(	(	PUNCT
flr-250	257	6	1	1	NUM
flr-250	257	7	)	)	PUNCT
flr-250	257	8	,	,	PUNCT
flr-250	257	9	1	1	NUM
flr-250	257	10	-	-	SYM
flr-250	257	11	7	7	NUM
flr-250	257	12	.	.	PUNCT
flr-250	258	1	pmid	pmid	NOUN
flr-250	258	2	:	:	PUNCT
flr-250	258	3	904086	904086	NUM
flr-250	258	4	.	.	PUNCT
flr-250	259	1	lusted	lust	VERB
flr-250	259	2	,	,	PUNCT
flr-250	259	3	l.b	l.b	PROPN
flr-250	259	4	.	.	PROPN
flr-250	260	1	(	(	PUNCT
flr-250	260	2	1960	1960	NUM
flr-250	260	3	)	)	PUNCT
flr-250	260	4	.	.	PUNCT
flr-250	261	1	logical	logical	ADJ
flr-250	261	2	analysis	analysis	NOUN
flr-250	261	3	in	in	ADP
flr-250	261	4	roentgen	roentgen	NOUN
flr-250	261	5	diagnosis	diagnosis	NOUN
flr-250	261	6	.	.	PUNCT
flr-250	262	1	radiol	radiol	NOUN
flr-250	262	2	,	,	PUNCT
flr-250	262	3	74,178	74,178	NUM
flr-250	262	4	-	-	SYM
flr-250	262	5	193	193	NUM
flr-250	262	6	.	.	PUNCT
flr-250	263	1	doi	doi	NOUN
flr-250	263	2	:	:	PUNCT
flr-250	263	3	http://dx.doi.org/10.1148/74.2.178	http://dx.doi.org/10.1148/74.2.178	PROPN
flr-250	263	4	lusted	lust	VERB
flr-250	263	5	,	,	PUNCT
flr-250	263	6	l.b	l.b	PROPN
flr-250	263	7	.	.	PROPN
flr-250	264	1	(	(	PUNCT
flr-250	264	2	1968	1968	NUM
flr-250	264	3	)	)	PUNCT
flr-250	264	4	.	.	PUNCT
flr-250	265	1	introduction	introduction	NOUN
flr-250	265	2	to	to	ADP
flr-250	265	3	medical	medical	ADJ
flr-250	265	4	decision	decision	NOUN
flr-250	265	5	making	making	NOUN
flr-250	265	6	.	.	PUNCT
flr-250	266	1	springfield	springfield	PROPN
flr-250	266	2	,	,	PUNCT
flr-250	266	3	il	il	PROPN
flr-250	266	4	:	:	PUNCT
flr-250	266	5	charles	charles	PROPN
flr-250	266	6	c.	c.	PROPN
flr-250	266	7	thomas	thomas	PROPN
flr-250	266	8	publishers	publishers	PROPN
flr-250	266	9	.	.	PUNCT
flr-250	267	1	lusted	lust	VERB
flr-250	267	2	,	,	PUNCT
flr-250	267	3	l.b	l.b	PROPN
flr-250	267	4	.	.	PROPN
flr-250	268	1	(	(	PUNCT
flr-250	268	2	1969	1969	NUM
flr-250	268	3	)	)	PUNCT
flr-250	268	4	.	.	PUNCT
flr-250	269	1	perception	perception	NOUN
flr-250	269	2	of	of	ADP
flr-250	269	3	the	the	DET
flr-250	269	4	roentgen	roentgen	NOUN
flr-250	269	5	image	image	NOUN
flr-250	269	6	:	:	PUNCT
flr-250	269	7	applications	application	NOUN
flr-250	269	8	of	of	ADP
flr-250	269	9	signal	signal	ADJ
flr-250	269	10	detection	detection	NOUN
flr-250	269	11	theory	theory	NOUN
flr-250	269	12	.	.	PUNCT
flr-250	270	1	rad	rad	PROPN
flr-250	270	2	clin	clin	PROPN
flr-250	270	3	n	n	PRON
flr-250	270	4	am	be	AUX
flr-250	270	5	,	,	PUNCT
flr-250	270	6	7	7	NUM
flr-250	270	7	,	,	PUNCT
flr-250	270	8	435	435	NUM
flr-250	270	9	-	-	SYM
flr-250	270	10	459	459	NUM
flr-250	270	11	.	.	PUNCT
flr-250	271	1	lusted	lust	VERB
flr-250	271	2	,	,	PUNCT
flr-250	271	3	l.b	l.b	PROPN
flr-250	271	4	.	.	PROPN
flr-250	272	1	(	(	PUNCT
flr-250	272	2	1971	1971	NUM
flr-250	272	3	)	)	PUNCT
flr-250	272	4	.	.	PUNCT
flr-250	273	1	signal	signal	ADJ
flr-250	273	2	detectability	detectability	NOUN
flr-250	273	3	and	and	CCONJ
flr-250	273	4	medical	medical	ADJ
flr-250	273	5	decision	decision	NOUN
flr-250	273	6	making	making	NOUN
flr-250	273	7	.	.	PUNCT
flr-250	274	1	science	science	NOUN
flr-250	274	2	,	,	PUNCT
flr-250	274	3	171	171	NUM
flr-250	274	4	,	,	PUNCT
flr-250	274	5	(	(	PUNCT
flr-250	274	6	3977	3977	NUM
flr-250	274	7	)	)	PUNCT
flr-250	274	8	,	,	PUNCT
flr-250	274	9	1217	1217	NUM
flr-250	274	10	-	-	SYM
flr-250	274	11	1219	1219	NUM
flr-250	274	12	.	.	PUNCT
flr-250	275	1	doi	doi	NOUN
flr-250	275	2	:	:	PUNCT
flr-250	275	3	10.1126	10.1126	NUM
flr-250	275	4	/	/	SYM
flr-250	275	5	science.171.3977.1217	science.171.3977.1217	PROPN
flr-250	275	6	mcclish	mcclish	PROPN
flr-250	275	7	,	,	PUNCT
flr-250	275	8	d.k	d.k	PROPN
flr-250	275	9	.	.	PROPN
flr-250	275	10	(	(	PUNCT
flr-250	275	11	1989	1989	NUM
flr-250	275	12	)	)	PUNCT
flr-250	275	13	.	.	PUNCT
flr-250	276	1	analyzing	analyze	VERB
flr-250	276	2	a	a	DET
flr-250	276	3	portion	portion	NOUN
flr-250	276	4	of	of	ADP
flr-250	276	5	the	the	DET
flr-250	276	6	roc	roc	PROPN
flr-250	276	7	curve	curve	NOUN
flr-250	276	8	.	.	PUNCT
flr-250	277	1	med	med	PROPN
flr-250	277	2	decis	decis	PROPN
flr-250	277	3	making	making	NOUN
flr-250	277	4	,	,	PUNCT
flr-250	277	5	9	9	NUM
flr-250	277	6	(	(	PUNCT
flr-250	277	7	3	3	NUM
flr-250	277	8	)	)	PUNCT
flr-250	277	9	,	,	PUNCT
flr-250	277	10	190	190	NUM
flr-250	277	11	-	-	SYM
flr-250	277	12	195	195	NUM
flr-250	277	13	.	.	PUNCT
flr-250	278	1	doi	doi	NOUN
flr-250	278	2	:	:	PUNCT
flr-250	278	3	10.1177/0272989x8900900307	10.1177/0272989x8900900307	NUM
flr-250	278	4	mcneil	mcneil	PROPN
flr-250	278	5	,	,	PUNCT
flr-250	278	6	b.j	b.j	PROPN
flr-250	278	7	.	.	PROPN
flr-250	278	8	,	,	PUNCT
flr-250	278	9	adelstein	adelstein	PROPN
flr-250	278	10	,	,	PUNCT
flr-250	278	11	s.j	s.j	PROPN
flr-250	278	12	.	.	PROPN
flr-250	278	13	(	(	PUNCT
flr-250	278	14	1976	1976	NUM
flr-250	278	15	)	)	PUNCT
flr-250	278	16	.	.	PUNCT
flr-250	279	1	determining	determine	VERB
flr-250	279	2	the	the	DET
flr-250	279	3	value	value	NOUN
flr-250	279	4	of	of	ADP
flr-250	279	5	diagnostic	diagnostic	ADJ
flr-250	279	6	and	and	CCONJ
flr-250	279	7	screening	screen	VERB
flr-250	279	8	tests	test	NOUN
flr-250	279	9	.	.	PUNCT
flr-250	280	1	j	j	PROPN
flr-250	280	2	nuc	nuc	PROPN
flr-250	280	3	med	med	PROPN
flr-250	280	4	,	,	PUNCT
flr-250	280	5	17	17	NUM
flr-250	280	6	,	,	PUNCT
flr-250	280	7	(	(	PUNCT
flr-250	280	8	6	6	NUM
flr-250	280	9	)	)	PUNCT
flr-250	280	10	,	,	PUNCT
flr-250	280	11	439	439	NUM
flr-250	280	12	-	-	SYM
flr-250	280	13	448	448	NUM
flr-250	280	14	.	.	PUNCT
flr-250	281	1	pmid:1262961	pmid:1262961	PROPN
flr-250	281	2	mcneil	mcneil	PROPN
flr-250	281	3	,	,	PUNCT
flr-250	281	4	b.j	b.j	PROPN
flr-250	281	5	.	.	PROPN
flr-250	281	6	,	,	PUNCT
flr-250	281	7	hanley	hanley	PROPN
flr-250	281	8	,	,	PUNCT
flr-250	281	9	j.a	j.a	PROPN
flr-250	281	10	.	.	PROPN
flr-250	281	11	(	(	PUNCT
flr-250	281	12	1984	1984	NUM
flr-250	281	13	)	)	PUNCT
flr-250	281	14	.	.	PUNCT
flr-250	282	1	statistical	statistical	ADJ
flr-250	282	2	approaches	approach	NOUN
flr-250	282	3	to	to	ADP
flr-250	282	4	the	the	DET
flr-250	282	5	analysis	analysis	NOUN
flr-250	282	6	of	of	ADP
flr-250	282	7	receiver	receiver	NOUN
flr-250	282	8	operating	operate	VERB
flr-250	282	9	characteristic	characteristic	NOUN
flr-250	282	10	(	(	PUNCT
flr-250	282	11	roc	roc	NOUN
flr-250	282	12	)	)	PUNCT
flr-250	282	13	curves	curve	NOUN
flr-250	282	14	.	.	PUNCT
flr-250	283	1	med	med	ADJ
flr-250	283	2	dec	dec	PROPN
flr-250	283	3	making	making	NOUN
flr-250	283	4	,	,	PUNCT
flr-250	283	5	4	4	NUM
flr-250	283	6	,	,	PUNCT
flr-250	283	7	(	(	PUNCT
flr-250	283	8	2	2	NUM
flr-250	283	9	)	)	PUNCT
flr-250	283	10	,	,	PUNCT
flr-250	283	11	137	137	NUM
flr-250	283	12	-	-	SYM
flr-250	283	13	150	150	NUM
flr-250	283	14	.	.	PUNCT
flr-250	284	1	doi:10.1177/0272989x8400400203	doi:10.1177/0272989x8400400203	PROPN
flr-250	284	2	mcneil	mcneil	PROPN
flr-250	284	3	,	,	PUNCT
flr-250	284	4	b.j	b.j	PROPN
flr-250	284	5	.	.	PROPN
flr-250	284	6	,	,	PUNCT
flr-250	284	7	keeler	keeler	PROPN
flr-250	284	8	,	,	PUNCT
flr-250	284	9	e.	e.	PROPN
flr-250	284	10	,	,	PUNCT
flr-250	284	11	adelstein	adelstein	PROPN
flr-250	284	12	,	,	PUNCT
flr-250	284	13	s.j	s.j	PROPN
flr-250	284	14	.	.	PROPN
flr-250	284	15	(	(	PUNCT
flr-250	284	16	1975	1975	NUM
flr-250	284	17	)	)	PUNCT
flr-250	284	18	.	.	PUNCT
flr-250	285	1	primer	primer	NOUN
flr-250	285	2	on	on	ADP
flr-250	285	3	certain	certain	ADJ
flr-250	285	4	elements	element	NOUN
flr-250	285	5	of	of	ADP
flr-250	285	6	medical	medical	ADJ
flr-250	285	7	decision	decision	NOUN
flr-250	285	8	making	making	NOUN
flr-250	285	9	.	.	PUNCT
flr-250	286	1	ne	ne	PROPN
flr-250	286	2	j	j	PROPN
flr-250	286	3	med	med	PROPN
flr-250	286	4	,	,	PUNCT
flr-250	286	5	293	293	NUM
flr-250	286	6	,	,	PUNCT
flr-250	286	7	(	(	PUNCT
flr-250	286	8	5	5	NUM
flr-250	286	9	)	)	PUNCT
flr-250	286	10	,	,	PUNCT
flr-250	286	11	211	211	NUM
flr-250	286	12	-	-	SYM
flr-250	286	13	215	215	NUM
flr-250	286	14	.	.	PUNCT
flr-250	287	1	doi	doi	NOUN
flr-250	287	2	:	:	PUNCT
flr-250	287	3	10.1056	10.1056	NUM
flr-250	287	4	/	/	SYM
flr-250	287	5	nejm197507312930501	nejm197507312930501	NOUN
flr-250	287	6	metz	metz	NOUN
flr-250	287	7	,	,	PUNCT
flr-250	287	8	c.e	c.e	PROPN
flr-250	287	9	.	.	PROPN
flr-250	287	10	,	,	PUNCT
flr-250	287	11	herman	herman	PROPN
flr-250	287	12	,	,	PUNCT
flr-250	287	13	b.a	b.a	PROPN
flr-250	287	14	.	.	PROPN
flr-250	287	15	,	,	PUNCT
flr-250	287	16	shen	shen	PROPN
flr-250	287	17	,	,	PUNCT
flr-250	287	18	j.h	j.h	PROPN
flr-250	287	19	.	.	PROPN
flr-250	287	20	(	(	PUNCT
flr-250	287	21	1998	1998	NUM
flr-250	287	22	)	)	PUNCT
flr-250	287	23	.	.	PUNCT
flr-250	288	1	maximum	maximum	ADJ
flr-250	288	2	-	-	PUNCT
flr-250	288	3	likelihood	likelihood	NOUN
flr-250	288	4	estimation	estimation	NOUN
flr-250	288	5	of	of	ADP
flr-250	288	6	roc	roc	PROPN
flr-250	288	7	curves	curve	NOUN
flr-250	288	8	from	from	ADP
flr-250	288	9	continuously	continuously	ADV
flr-250	288	10	-	-	PUNCT
flr-250	288	11	distributed	distribute	VERB
flr-250	288	12	data	datum	NOUN
flr-250	288	13	.	.	PUNCT
flr-250	289	1	stats	stat	NOUN
flr-250	289	2	med	med	ADJ
flr-250	289	3	,	,	PUNCT
flr-250	289	4	17	17	NUM
flr-250	289	5	(	(	PUNCT
flr-250	289	6	9	9	NUM
flr-250	289	7	)	)	PUNCT
flr-250	289	8	,	,	PUNCT
flr-250	289	9	1033	1033	NUM
flr-250	289	10	-	-	SYM
flr-250	289	11	1053	1053	NUM
flr-250	289	12	.	.	PUNCT
flr-250	290	1	pmid	pmid	NOUN
flr-250	290	2	:	:	PUNCT
flr-250	290	3	9612889	9612889	NUM
flr-250	290	4	metz	metz	PROPN
flr-250	290	5	,	,	PUNCT
flr-250	290	6	c.e	c.e	PROPN
flr-250	290	7	.	.	PROPN
flr-250	290	8	,	,	PUNCT
flr-250	290	9	kronman	kronman	PROPN
flr-250	290	10	,	,	PUNCT
flr-250	290	11	h.b	h.b	PROPN
flr-250	290	12	.	.	PROPN
flr-250	290	13	(	(	PUNCT
flr-250	290	14	1980	1980	NUM
flr-250	290	15	)	)	PUNCT
flr-250	290	16	.	.	PUNCT
flr-250	291	1	statistical	statistical	ADJ
flr-250	291	2	significance	significance	NOUN
flr-250	291	3	tests	test	VERB
flr-250	291	4	for	for	ADP
flr-250	291	5	binormal	binormal	ADJ
flr-250	291	6	roc	roc	PROPN
flr-250	291	7	curves	curve	NOUN
flr-250	291	8	.	.	PUNCT
flr-250	292	1	j	j	PROPN
flr-250	292	2	math	math	PROPN
flr-250	292	3	psych	psych	PROPN
flr-250	292	4	,	,	PUNCT
flr-250	292	5	22	22	NUM
flr-250	292	6	(	(	PUNCT
flr-250	292	7	3	3	NUM
flr-250	292	8	)	)	PUNCT
flr-250	292	9	,	,	PUNCT
flr-250	292	10	218	218	NUM
flr-250	292	11	-	-	SYM
flr-250	292	12	243	243	NUM
flr-250	292	13	.	.	PUNCT
flr-250	293	1	doi	doi	NOUN
flr-250	293	2	:	:	PUNCT
flr-250	293	3	http://dx.doi.org/10.1016/0022-2496(80)90020-6	http://dx.doi.org/10.1016/0022-2496(80)90020-6	PUNCT
flr-250	294	1	metz	metz	PROPN
flr-250	294	2	,	,	PUNCT
flr-250	294	3	c.e	c.e	PROPN
flr-250	294	4	.	.	PROPN
flr-250	294	5	,	,	PUNCT
flr-250	294	6	pan	pan	PROPN
flr-250	294	7	,	,	PUNCT
flr-250	294	8	x.	x.	NOUN
flr-250	294	9	(	(	PUNCT
flr-250	294	10	1999	1999	NUM
flr-250	294	11	)	)	PUNCT
flr-250	294	12	.	.	PUNCT
flr-250	295	1	“	"	PUNCT
flr-250	295	2	proper	proper	ADJ
flr-250	295	3	”	"	PUNCT
flr-250	295	4	binormal	binormal	ADJ
flr-250	295	5	roc	roc	PROPN
flr-250	295	6	curves	curve	NOUN
flr-250	295	7	:	:	PUNCT
flr-250	295	8	theory	theory	NOUN
flr-250	295	9	and	and	CCONJ
flr-250	295	10	maximum	maximum	ADJ
flr-250	295	11	-	-	PUNCT
flr-250	295	12	likelihood	likelihood	NOUN
flr-250	295	13	estimation	estimation	NOUN
flr-250	295	14	.	.	PUNCT
flr-250	296	1	j	j	PROPN
flr-250	296	2	math	math	PROPN
flr-250	296	3	psych	psych	PROPN
flr-250	296	4	,	,	PUNCT
flr-250	296	5	43	43	NUM
flr-250	296	6	(	(	PUNCT
flr-250	296	7	1	1	NUM
flr-250	296	8	)	)	PUNCT
flr-250	296	9	,	,	PUNCT
flr-250	296	10	1	1	NUM
flr-250	296	11	-	-	SYM
flr-250	296	12	33	33	NUM
flr-250	296	13	.	.	PUNCT
flr-250	297	1	doi	doi	NOUN
flr-250	297	2	:	:	PUNCT
flr-250	297	3	10.1006	10.1006	NUM
flr-250	297	4	/	/	SYM
flr-250	297	5	jmps.1998.1218	jmps.1998.1218	PROPN
flr-250	297	6	nakas	nakas	PROPN
flr-250	297	7	,	,	PUNCT
flr-250	297	8	c.t	c.t	PROPN
flr-250	297	9	.	.	PROPN
flr-250	297	10	(	(	PUNCT
flr-250	297	11	2014	2014	NUM
flr-250	297	12	)	)	PUNCT
flr-250	297	13	.	.	PUNCT
flr-250	298	1	developments	development	NOUN
flr-250	298	2	in	in	ADP
flr-250	298	3	roc	roc	PROPN
flr-250	298	4	surface	surface	NOUN
flr-250	298	5	analysis	analysis	NOUN
flr-250	298	6	and	and	CCONJ
flr-250	298	7	assessment	assessment	NOUN
flr-250	298	8	of	of	ADP
flr-250	298	9	diagnostic	diagnostic	ADJ
flr-250	298	10	markers	marker	NOUN
flr-250	298	11	in	in	ADP
flr-250	298	12	threeclass	threeclass	ADJ
flr-250	298	13	classification	classification	NOUN
flr-250	298	14	problems	problem	NOUN
flr-250	298	15	.	.	PUNCT
flr-250	299	1	revstat	revstat	NOUN
flr-250	299	2	–	–	PUNCT
flr-250	299	3	stat	stat	PROPN
flr-250	299	4	j	j	PROPN
flr-250	299	5	,	,	PUNCT
flr-250	299	6	12	12	NUM
flr-250	299	7	(	(	PUNCT
flr-250	299	8	1	1	NUM
flr-250	299	9	)	)	PUNCT
flr-250	299	10	,	,	PUNCT
flr-250	299	11	43	43	NUM
flr-250	299	12	-	-	SYM
flr-250	299	13	65	65	NUM
flr-250	299	14	.	.	PUNCT
flr-250	300	1	ncss	ncss	PROPN
flr-250	300	2	statistical	statistical	ADJ
flr-250	300	3	software	software	NOUN
flr-250	300	4	.	.	PUNCT
flr-250	301	1	http://www.ncss.com/software/ncss/procedures/	http://www.ncss.com/software/ncss/procedures/	PROPN
flr-250	301	2	last	last	ADV
flr-250	301	3	accessed	access	VERB
flr-250	301	4	april	april	PROPN
flr-250	301	5	13	13	NUM
flr-250	301	6	,	,	PUNCT
flr-250	301	7	2016	2016	NUM
flr-250	301	8	.	.	PUNCT
flr-250	302	1	obuchowski	obuchowski	PROPN
flr-250	302	2	,	,	PUNCT
flr-250	302	3	n.a	n.a	PROPN
flr-250	302	4	.	.	PROPN
flr-250	303	1	(	(	PUNCT
flr-250	303	2	1997	1997	NUM
flr-250	303	3	)	)	PUNCT
flr-250	303	4	.	.	PUNCT
flr-250	304	1	testing	test	VERB
flr-250	304	2	for	for	ADP
flr-250	304	3	equivalence	equivalence	NOUN
flr-250	304	4	of	of	ADP
flr-250	304	5	diagnostic	diagnostic	ADJ
flr-250	304	6	tests	test	NOUN
flr-250	304	7	.	.	PUNCT
flr-250	305	1	am	be	AUX
flr-250	305	2	j	j	PROPN
flr-250	305	3	roentgen	roentgen	PROPN
flr-250	305	4	,	,	PUNCT
flr-250	305	5	168	168	NUM
flr-250	305	6	(	(	PUNCT
flr-250	305	7	1	1	NUM
flr-250	305	8	)	)	PUNCT
flr-250	305	9	,	,	PUNCT
flr-250	305	10	13	13	NUM
flr-250	305	11	-	-	SYM
flr-250	305	12	17	17	NUM
flr-250	305	13	.	.	PUNCT
flr-250	306	1	doi	doi	NOUN
flr-250	306	2	:	:	PUNCT
flr-250	306	3	10.2214	10.2214	NUM
flr-250	306	4	/	/	SYM
flr-250	306	5	ajr.168.1.8976911	ajr.168.1.8976911	NUM
flr-250	306	6	obuchowski	obuchowski	NOUN
flr-250	306	7	,	,	PUNCT
flr-250	306	8	n.a	n.a	PROPN
flr-250	306	9	.	.	PROPN
flr-250	306	10	(	(	PUNCT
flr-250	306	11	1994	1994	NUM
flr-250	306	12	)	)	PUNCT
flr-250	306	13	.	.	PUNCT
flr-250	307	1	computing	compute	VERB
flr-250	307	2	sample	sample	NOUN
flr-250	307	3	size	size	NOUN
flr-250	307	4	for	for	ADP
flr-250	307	5	receiver	receiver	NOUN
flr-250	307	6	operating	operate	VERB
flr-250	307	7	characteristic	characteristic	ADJ
flr-250	307	8	studies	study	NOUN
flr-250	307	9	.	.	PUNCT
flr-250	308	1	invest	invest	VERB
flr-250	308	2	radiol	radiol	NOUN
flr-250	308	3	,	,	PUNCT
flr-250	308	4	29	29	NUM
flr-250	308	5	(	(	PUNCT
flr-250	308	6	2	2	NUM
flr-250	308	7	)	)	PUNCT
flr-250	308	8	,	,	PUNCT
flr-250	308	9	238	238	NUM
flr-250	308	10	-	-	SYM
flr-250	308	11	243	243	NUM
flr-250	308	12	.	.	PUNCT
flr-250	309	1	doi:10.2214	doi:10.2214	PROPN
flr-250	309	2	/	/	SYM
flr-250	309	3	ajr.175.3.1750603	ajr.175.3.1750603	PROPN
flr-250	309	4	obuchowski	obuchowski	NOUN
flr-250	309	5	,	,	PUNCT
flr-250	309	6	n.a	n.a	PROPN
flr-250	309	7	.	.	PROPN
flr-250	309	8	(	(	PUNCT
flr-250	309	9	2000	2000	NUM
flr-250	309	10	)	)	PUNCT
flr-250	309	11	.	.	PUNCT
flr-250	310	1	sample	sample	NOUN
flr-250	310	2	size	size	NOUN
flr-250	310	3	tables	table	NOUN
flr-250	310	4	for	for	ADP
flr-250	310	5	receiver	receiver	NOUN
flr-250	310	6	operating	operate	VERB
flr-250	310	7	characteristic	characteristic	ADJ
flr-250	310	8	studies	study	NOUN
flr-250	310	9	.	.	PUNCT
flr-250	311	1	am	be	AUX
flr-250	311	2	j	j	PROPN
flr-250	311	3	roentgen	roentgen	PROPN
flr-250	311	4	,	,	PUNCT
flr-250	311	5	175	175	NUM
flr-250	311	6	(	(	PUNCT
flr-250	311	7	3	3	NUM
flr-250	311	8	)	)	PUNCT
flr-250	311	9	,	,	PUNCT
flr-250	311	10	603	603	NUM
flr-250	311	11	-	-	SYM
flr-250	311	12	608	608	NUM
flr-250	311	13	.	.	PUNCT
flr-250	312	1	doi:10.2214	doi:10.2214	PROPN
flr-250	312	2	/	/	SYM
flr-250	312	3	ajr.175.3.1750603	ajr.175.3.1750603	PROPN
flr-250	312	4	obuchowski	obuchowski	NOUN
flr-250	312	5	,	,	PUNCT
flr-250	312	6	n.a	n.a	PROPN
flr-250	312	7	.	.	PROPN
flr-250	313	1	(	(	PUNCT
flr-250	313	2	2004	2004	NUM
flr-250	313	3	)	)	PUNCT
flr-250	313	4	.	.	PUNCT
flr-250	314	1	how	how	SCONJ
flr-250	314	2	many	many	ADJ
flr-250	314	3	observers	observer	NOUN
flr-250	314	4	care	care	AUX
flr-250	314	5	needed	need	VERB
flr-250	314	6	in	in	ADP
flr-250	314	7	clinical	clinical	ADJ
flr-250	314	8	studies	study	NOUN
flr-250	314	9	of	of	ADP
flr-250	314	10	medical	medical	ADJ
flr-250	314	11	imaging	imaging	NOUN
flr-250	314	12	?	?	PUNCT
flr-250	315	1	am	be	AUX
flr-250	315	2	j	j	PROPN
flr-250	315	3	roentgen	roentgen	NOUN
flr-250	315	4	,	,	PUNCT
flr-250	315	5	182	182	NUM
flr-250	315	6	(	(	PUNCT
flr-250	315	7	4	4	NUM
flr-250	315	8	)	)	PUNCT
flr-250	315	9	,	,	PUNCT
flr-250	315	10	867	867	NUM
flr-250	315	11	-	-	SYM
flr-250	315	12	869	869	NUM
flr-250	315	13	.	.	PUNCT
flr-250	316	1	doi	doi	NOUN
flr-250	316	2	:	:	PUNCT
flr-250	316	3	10.2214	10.2214	NUM
flr-250	316	4	/	/	SYM
flr-250	316	5	ajr.182.4.1820867	ajr.182.4.1820867	PROPN
flr-250	316	6	peterson	peterson	PROPN
flr-250	316	7	,	,	PUNCT
flr-250	316	8	w.w	w.w	PROPN
flr-250	316	9	.	.	PROPN
flr-250	316	10	,	,	PUNCT
flr-250	316	11	birdsall	birdsall	PROPN
flr-250	316	12	,	,	PUNCT
flr-250	316	13	t.l	t.l	PROPN
flr-250	316	14	.	.	PROPN
flr-250	316	15	,	,	PUNCT
flr-250	316	16	fox	fox	PROPN
flr-250	316	17	,	,	PUNCT
flr-250	316	18	w.c	w.c	PROPN
flr-250	316	19	.	.	PROPN
flr-250	316	20	(	(	PUNCT
flr-250	316	21	1954	1954	NUM
flr-250	316	22	)	)	PUNCT
flr-250	316	23	.	.	PUNCT
flr-250	317	1	the	the	DET
flr-250	317	2	theory	theory	NOUN
flr-250	317	3	of	of	ADP
flr-250	317	4	signal	signal	ADJ
flr-250	317	5	detectability	detectability	NOUN
flr-250	317	6	.	.	PUNCT
flr-250	318	1	ire	ire	ADJ
flr-250	318	2	prof	prof	NOUN
flr-250	318	3	gp	gp	NOUN
flr-250	318	4	in	in	ADP
flr-250	318	5	theory	theory	NOUN
flr-250	318	6	trans	trans	NOUN
flr-250	318	7	pgit	pgit	NOUN
flr-250	318	8	,	,	PUNCT
flr-250	318	9	4	4	NUM
flr-250	318	10	,	,	PUNCT
flr-250	318	11	(	(	PUNCT
flr-250	318	12	4	4	NUM
flr-250	318	13	)	)	PUNCT
flr-250	318	14	,	,	PUNCT
flr-250	318	15	171	171	NUM
flr-250	318	16	-	-	SYM
flr-250	318	17	212	212	NUM
flr-250	318	18	.	.	PUNCT
flr-250	319	1	doi	doi	NOUN
flr-250	319	2	:	:	PUNCT
flr-250	319	3	10.1109	10.1109	NUM
flr-250	319	4	/	/	SYM
flr-250	319	5	tit.1954.1057460	tit.1954.1057460	NUM
flr-250	319	6	petrick	petrick	NOUN
flr-250	319	7	,	,	PUNCT
flr-250	319	8	n.	n.	NOUN
flr-250	319	9	,	,	PUNCT
flr-250	319	10	gallas	gallas	PROPN
flr-250	319	11	,	,	PUNCT
flr-250	319	12	b.d	b.d	PROPN
flr-250	319	13	.	.	PROPN
flr-250	319	14	,	,	PUNCT
flr-250	319	15	samuelson	samuelson	PROPN
flr-250	319	16	,	,	PUNCT
flr-250	319	17	f.w	f.w	PROPN
flr-250	319	18	.	.	PROPN
flr-250	319	19	,	,	PUNCT
flr-250	319	20	wagner	wagner	PROPN
flr-250	319	21	,	,	PUNCT
flr-250	319	22	r.f	r.f	PROPN
flr-250	319	23	.	.	PROPN
flr-250	319	24	,	,	PUNCT
flr-250	319	25	myers	myers	PROPN
flr-250	319	26	,	,	PUNCT
flr-250	319	27	k.j	k.j	PROPN
flr-250	319	28	.	.	PROPN
flr-250	319	29	(	(	PUNCT
flr-250	319	30	2005	2005	NUM
flr-250	319	31	)	)	PUNCT
flr-250	319	32	.	.	PUNCT
flr-250	320	1	influence	influence	NOUN
flr-250	320	2	of	of	ADP
flr-250	320	3	panel	panel	NOUN
flr-250	320	4	size	size	NOUN
flr-250	320	5	and	and	CCONJ
flr-250	320	6	expert	expert	NOUN
flr-250	320	7	skill	skill	NOUN
flr-250	320	8	on	on	ADP
flr-250	320	9	truth	truth	NOUN
flr-250	320	10	panel	panel	NOUN
flr-250	320	11	performance	performance	NOUN
flr-250	320	12	when	when	SCONJ
flr-250	320	13	combining	combine	VERB
flr-250	320	14	expert	expert	ADJ
flr-250	320	15	ratings	rating	NOUN
flr-250	320	16	.	.	PUNCT
flr-250	321	1	proc	proc	PROPN
flr-250	321	2	spie	spie	PROPN
flr-250	321	3	med	med	PROPN
flr-250	321	4	imag	imag	NOUN
flr-250	321	5	,	,	PUNCT
flr-250	321	6	5749	5749	NUM
flr-250	321	7	,	,	PUNCT
flr-250	321	8	596286	596286	NUM
flr-250	321	9	.	.	PUNCT
flr-250	322	1	doi	doi	NOUN
flr-250	322	2	:	:	PUNCT
flr-250	322	3	10.1117/12.596286	10.1117/12.596286	PROPN
flr-250	322	4	schulman	schulman	PROPN
flr-250	322	5	,	,	PUNCT
flr-250	322	6	k.a	k.a	PROPN
flr-250	322	7	.	.	PROPN
flr-250	322	8	,	,	PUNCT
flr-250	322	9	kim	kim	PROPN
flr-250	322	10	,	,	PUNCT
flr-250	322	11	j.j	j.j	PROPN
flr-250	322	12	.	.	PROPN
flr-250	322	13	(	(	PUNCT
flr-250	322	14	2000	2000	NUM
flr-250	322	15	)	)	PUNCT
flr-250	322	16	.	.	PUNCT
flr-250	323	1	medical	medical	ADJ
flr-250	323	2	errors	error	NOUN
flr-250	323	3	:	:	PUNCT
flr-250	323	4	how	how	SCONJ
flr-250	323	5	the	the	DET
flr-250	323	6	us	us	PROPN
flr-250	323	7	government	government	NOUN
flr-250	323	8	is	be	AUX
flr-250	323	9	addressing	address	VERB
flr-250	323	10	the	the	DET
flr-250	323	11	problem	problem	NOUN
flr-250	323	12	.	.	PUNCT
flr-250	324	1	curr	curr	PROPN
flr-250	324	2	control	control	PROPN
flr-250	324	3	trials	trial	NOUN
flr-250	324	4	cardiovasc	cardiovasc	PROPN
flr-250	324	5	med	me	VERB
flr-250	324	6	,	,	PUNCT
flr-250	324	7	1(1	1(1	NUM
flr-250	324	8	)	)	PUNCT
flr-250	324	9	,	,	PUNCT
flr-250	324	10	35	35	NUM
flr-250	324	11	-	-	SYM
flr-250	324	12	37	37	NUM
flr-250	324	13	.	.	PUNCT
flr-250	325	1	doi	doi	NOUN
flr-250	325	2	:	:	PUNCT
flr-250	325	3	10.1186	10.1186	NUM
flr-250	325	4	/	/	SYM
flr-250	325	5	cvm-1	cvm-1	NOUN
flr-250	325	6	-	-	PUNCT
flr-250	325	7	1	1	NUM
flr-250	325	8	-	-	PUNCT
flr-250	325	9	035	035	NUM
flr-250	325	10	spss	spss	PROPN
flr-250	325	11	statistics	statistic	NOUN
flr-250	325	12	.	.	PUNCT
flr-250	326	1	http://www-03.ibm.com/software/products/en/spss-statistics	http://www-03.ibm.com/software/products/en/spss-statistic	NOUN
flr-250	326	2	last	last	ADV
flr-250	326	3	accessed	access	VERB
flr-250	326	4	april	april	PROPN
flr-250	326	5	13	13	NUM
flr-250	326	6	,	,	PUNCT
flr-250	326	7	2016	2016	NUM
flr-250	326	8	.	.	PUNCT
flr-250	327	1	starr	starr	PROPN
flr-250	327	2	,	,	PUNCT
flr-250	327	3	s.j	s.j	PROPN
flr-250	327	4	.	.	PROPN
flr-250	327	5	,	,	PUNCT
flr-250	327	6	metz	metz	PROPN
flr-250	327	7	,	,	PUNCT
flr-250	327	8	c.e	c.e	PROPN
flr-250	327	9	.	.	PROPN
flr-250	327	10	,	,	PUNCT
flr-250	327	11	lusted	lust	VERB
flr-250	327	12	,	,	PUNCT
flr-250	327	13	l.b	l.b	PROPN
flr-250	327	14	.	.	PROPN
flr-250	327	15	,	,	PUNCT
flr-250	327	16	goodenough	goodenough	PROPN
flr-250	327	17	,	,	PUNCT
flr-250	327	18	d.j	d.j	PROPN
flr-250	327	19	.	.	PROPN
flr-250	327	20	(	(	PUNCT
flr-250	327	21	1975	1975	NUM
flr-250	327	22	)	)	PUNCT
flr-250	327	23	.	.	PUNCT
flr-250	328	1	visual	visual	ADJ
flr-250	328	2	detection	detection	NOUN
flr-250	328	3	and	and	CCONJ
flr-250	328	4	localization	localization	NOUN
flr-250	328	5	of	of	ADP
flr-250	328	6	http://dx.doi.org/10.1148/radiology.143.1.7063747	http://dx.doi.org/10.1148/radiology.143.1.7063747	PROPN
flr-250	329	1	https://doi.org/10.1148/radiology.201.3.8939225	https://doi.org/10.1148/radiology.201.3.8939225	PROPN
flr-250	329	2	http://dx.doi.org/10.1148/74.2.178	http://dx.doi.org/10.1148/74.2.178	VERB
flr-250	329	3	https://doi.org/10.1177/0272989x8900900307	https://doi.org/10.1177/0272989x8900900307	NOUN
flr-250	329	4	https://doi.org/10.1177/0272989x8400400203	https://doi.org/10.1177/0272989x8400400203	DET
flr-250	329	5	http://dx.doi.org/10.1016/0022-2496%2880%2990020-6	http://dx.doi.org/10.1016/0022-2496%2880%2990020-6	PROPN
flr-250	329	6	https://doi.org/10.1006/jmps.1998.1218	https://doi.org/10.1006/jmps.1998.1218	PROPN
flr-250	329	7	http://www.ncss.com/software/ncss/procedures/	http://www.ncss.com/software/ncss/procedures/	PROPN
flr-250	329	8	https://doi.org/10.2214/ajr.168.1.8976911	https://doi.org/10.2214/ajr.168.1.8976911	PROPN
flr-250	329	9	https://doi.org/10.2214/ajr.175.3.1750603	https://doi.org/10.2214/ajr.175.3.1750603	PROPN
flr-250	329	10	https://doi.org/10.2214/ajr.175.3.1750603	https://doi.org/10.2214/ajr.175.3.1750603	PROPN
flr-250	329	11	https://doi.org/10.1109/tit.1954.1057460	https://doi.org/10.1109/tit.1954.1057460	VERB
flr-250	329	12	https://dx.doi.org/10.1186%2fcvm-1-1-035	https://dx.doi.org/10.1186%2fcvm-1-1-035	VERB
flr-250	330	1	http://www-03.ibm.com/software/products/en/spss-statistics	http://www-03.ibm.com/software/products/en/spss-statistics	PROPN
flr-250	330	2	krupinski	krupinski	VERB
flr-250	330	3	|	|	ADV
flr-250	330	4	f	f	NOUN
flr-250	330	5	l	l	NOUN
flr-250	330	6	r	r	NOUN
flr-250	330	7	42	42	NUM
flr-250	330	8	radiographic	radiographic	ADJ
flr-250	330	9	images	image	NOUN
flr-250	330	10	.	.	PUNCT
flr-250	331	1	radiol	radiol	NOUN
flr-250	331	2	,	,	PUNCT
flr-250	331	3	116	116	NUM
flr-250	331	4	(	(	PUNCT
flr-250	331	5	3	3	NUM
flr-250	331	6	)	)	PUNCT
flr-250	331	7	,	,	PUNCT
flr-250	331	8	533	533	NUM
flr-250	331	9	-	-	SYM
flr-250	331	10	538	538	NUM
flr-250	331	11	.	.	PUNCT
flr-250	332	1	doi:10.1148/116.3.533	doi:10.1148/116.3.533	VERB
flr-250	332	2	stata	stata	PROPN
flr-250	332	3	data	datum	NOUN
flr-250	332	4	analysis	analysis	NOUN
flr-250	332	5	and	and	CCONJ
flr-250	332	6	statistical	statistical	ADJ
flr-250	332	7	software	software	NOUN
flr-250	332	8	.	.	PUNCT
flr-250	333	1	http://www.stata.com/features/overview/receiver-operatingcharacteristic/	http://www.stata.com/features/overview/receiver-operatingcharacteristic/	PROPN
flr-250	333	2	last	last	ADV
flr-250	333	3	accessed	access	VERB
flr-250	333	4	april	april	PROPN
flr-250	333	5	13	13	NUM
flr-250	333	6	,	,	PUNCT
flr-250	333	7	2016	2016	NUM
flr-250	333	8	swensson	swensson	NOUN
flr-250	333	9	,	,	PUNCT
flr-250	333	10	r.g	r.g	PROPN
flr-250	333	11	.	.	PROPN
flr-250	333	12	(	(	PUNCT
flr-250	333	13	1996	1996	NUM
flr-250	333	14	)	)	PUNCT
flr-250	333	15	.	.	PUNCT
flr-250	334	1	unified	unified	ADJ
flr-250	334	2	measurement	measurement	NOUN
flr-250	334	3	of	of	ADP
flr-250	334	4	observer	observer	NOUN
flr-250	334	5	performance	performance	NOUN
flr-250	334	6	in	in	ADP
flr-250	334	7	detecting	detect	VERB
flr-250	334	8	and	and	CCONJ
flr-250	334	9	localizing	localize	VERB
flr-250	334	10	target	target	NOUN
flr-250	334	11	objects	object	NOUN
flr-250	334	12	on	on	ADP
flr-250	334	13	images	image	NOUN
flr-250	334	14	.	.	PUNCT
flr-250	335	1	med	med	ADJ
flr-250	335	2	phys	phy	NOUN
flr-250	335	3	,	,	PUNCT
flr-250	335	4	23	23	NUM
flr-250	335	5	(	(	PUNCT
flr-250	335	6	10	10	NUM
flr-250	335	7	)	)	PUNCT
flr-250	335	8	,	,	PUNCT
flr-250	335	9	1709	1709	NUM
flr-250	335	10	-	-	SYM
flr-250	335	11	1725	1725	NUM
flr-250	335	12	.	.	PUNCT
flr-250	336	1	doi:10.1118/1.597758	doi:10.1118/1.597758	NOUN
flr-250	336	2	swets	swet	NOUN
flr-250	336	3	,	,	PUNCT
flr-250	336	4	j.a	j.a	PROPN
flr-250	336	5	.	.	PROPN
flr-250	336	6	(	(	PUNCT
flr-250	336	7	1979	1979	NUM
flr-250	336	8	)	)	PUNCT
flr-250	336	9	.	.	PUNCT
flr-250	337	1	roc	roc	NOUN
flr-250	337	2	analysis	analysis	NOUN
flr-250	337	3	applied	apply	VERB
flr-250	337	4	to	to	ADP
flr-250	337	5	the	the	DET
flr-250	337	6	evaluation	evaluation	NOUN
flr-250	337	7	of	of	ADP
flr-250	337	8	medical	medical	ADJ
flr-250	337	9	imaging	imaging	NOUN
flr-250	337	10	techniques	technique	NOUN
flr-250	337	11	.	.	PUNCT
flr-250	338	1	radiol	radiol	NOUN
flr-250	338	2	,	,	PUNCT
flr-250	338	3	14	14	NUM
flr-250	338	4	(	(	PUNCT
flr-250	338	5	2	2	NUM
flr-250	338	6	)	)	PUNCT
flr-250	338	7	,	,	PUNCT
flr-250	338	8	109	109	NUM
flr-250	338	9	-	-	SYM
flr-250	338	10	121	121	NUM
flr-250	338	11	.	.	PUNCT
flr-250	339	1	pmid	pmid	NOUN
flr-250	339	2	:	:	PUNCT
flr-250	339	3	478799	478799	NUM
flr-250	339	4	swets	swet	NOUN
flr-250	339	5	,	,	PUNCT
flr-250	339	6	j.a	j.a	PROPN
flr-250	339	7	.	.	PROPN
flr-250	339	8	,	,	PUNCT
flr-250	339	9	dawes	dawes	PROPN
flr-250	339	10	,	,	PUNCT
flr-250	339	11	r.m	r.m	PROPN
flr-250	339	12	.	.	PROPN
flr-250	339	13	,	,	PUNCT
flr-250	339	14	monahan	monahan	PROPN
flr-250	339	15	,	,	PUNCT
flr-250	339	16	j.	j.	PROPN
flr-250	339	17	(	(	PUNCT
flr-250	339	18	2000	2000	NUM
flr-250	339	19	)	)	PUNCT
flr-250	339	20	.	.	PUNCT
flr-250	340	1	psychological	psychological	ADJ
flr-250	340	2	science	science	NOUN
flr-250	340	3	can	can	AUX
flr-250	340	4	improve	improve	VERB
flr-250	340	5	diagnostic	diagnostic	ADJ
flr-250	340	6	decisions	decision	NOUN
flr-250	340	7	.	.	PUNCT
flr-250	341	1	psych	psych	PROPN
flr-250	341	2	sci	sci	PROPN
flr-250	341	3	public	public	ADJ
flr-250	341	4	interest	interest	NOUN
flr-250	341	5	,	,	PUNCT
flr-250	341	6	1	1	NUM
flr-250	341	7	(	(	PUNCT
flr-250	341	8	1	1	NUM
flr-250	341	9	)	)	PUNCT
flr-250	341	10	,	,	PUNCT
flr-250	341	11	1	1	NUM
flr-250	341	12	-	-	SYM
flr-250	341	13	26	26	NUM
flr-250	341	14	.	.	PUNCT
flr-250	342	1	doi	doi	NOUN
flr-250	342	2	:	:	PUNCT
flr-250	342	3	10.1111/1529	10.1111/1529	NUM
flr-250	342	4	-	-	SYM
flr-250	342	5	1006.001	1006.001	NUM
flr-250	342	6	swets	swet	NOUN
flr-250	342	7	,	,	PUNCT
flr-250	342	8	j.a	j.a	PROPN
flr-250	342	9	.	.	PROPN
flr-250	342	10	,	,	PUNCT
flr-250	342	11	pickett	pickett	PROPN
flr-250	342	12	,	,	PUNCT
flr-250	342	13	r.m	r.m	PROPN
flr-250	342	14	.	.	PROPN
flr-250	342	15	(	(	PUNCT
flr-250	342	16	1982	1982	NUM
flr-250	342	17	)	)	PUNCT
flr-250	342	18	.	.	PUNCT
flr-250	343	1	evaluation	evaluation	NOUN
flr-250	343	2	of	of	ADP
flr-250	343	3	diagnostic	diagnostic	ADJ
flr-250	343	4	systems	system	NOUN
flr-250	343	5	.	.	PUNCT
flr-250	344	1	methods	method	NOUN
flr-250	344	2	from	from	ADP
flr-250	344	3	signal	signal	ADJ
flr-250	344	4	detection	detection	NOUN
flr-250	344	5	theory	theory	NOUN
flr-250	344	6	.	.	PUNCT
flr-250	345	1	new	new	PROPN
flr-250	345	2	york	york	PROPN
flr-250	345	3	,	,	PUNCT
flr-250	345	4	ny	ny	PROPN
flr-250	345	5	:	:	PUNCT
flr-250	345	6	academic	academic	ADJ
flr-250	345	7	press	press	NOUN
flr-250	345	8	.	.	PUNCT
flr-250	346	1	tanner	tanner	NOUN
flr-250	346	2	,	,	PUNCT
flr-250	346	3	w.p	w.p	PROPN
flr-250	346	4	.	.	PROPN
flr-250	346	5	,	,	PUNCT
flr-250	346	6	swets	swet	NOUN
flr-250	346	7	,	,	PUNCT
flr-250	346	8	j.a	j.a	PROPN
flr-250	346	9	.	.	PROPN
flr-250	346	10	(	(	PUNCT
flr-250	346	11	1954	1954	NUM
flr-250	346	12	)	)	PUNCT
flr-250	346	13	.	.	PUNCT
flr-250	347	1	a	a	DET
flr-250	347	2	decision	decision	NOUN
flr-250	347	3	-	-	PUNCT
flr-250	347	4	making	make	VERB
flr-250	347	5	theory	theory	NOUN
flr-250	347	6	of	of	ADP
flr-250	347	7	visual	visual	ADJ
flr-250	347	8	detection	detection	NOUN
flr-250	347	9	.	.	PUNCT
flr-250	348	1	psych	psych	PROPN
flr-250	348	2	rev	rev	VERB
flr-250	348	3	,	,	PUNCT
flr-250	348	4	61	61	NUM
flr-250	348	5	,	,	PUNCT
flr-250	348	6	(	(	PUNCT
flr-250	348	7	6	6	NUM
flr-250	348	8	)	)	PUNCT
flr-250	348	9	,	,	PUNCT
flr-250	348	10	401	401	NUM
flr-250	348	11	-	-	SYM
flr-250	348	12	409	409	NUM
flr-250	348	13	.	.	PUNCT
flr-250	349	1	pmid	pmid	NOUN
flr-250	349	2	:	:	PUNCT
flr-250	349	3	13215690	13215690	NUM
flr-250	349	4	university	university	PROPN
flr-250	349	5	of	of	ADP
flr-250	349	6	iowa	iowa	PROPN
flr-250	349	7	medical	medical	PROPN
flr-250	349	8	image	image	NOUN
flr-250	349	9	perception	perception	NOUN
flr-250	349	10	roc	roc	PROPN
flr-250	349	11	software	software	NOUN
flr-250	349	12	.	.	PUNCT
flr-250	350	1	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/defaul	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/defaul	NOUN
flr-250	350	2	t.aspx	t.aspx	NOUN
flr-250	350	3	last	last	ADV
flr-250	350	4	accessed	access	VERB
flr-250	350	5	april	april	PROPN
flr-250	350	6	13	13	NUM
flr-250	350	7	,	,	PUNCT
flr-250	350	8	2016	2016	NUM
flr-250	350	9	.	.	PUNCT
flr-250	351	1	university	university	NOUN
flr-250	351	2	of	of	ADP
flr-250	351	3	chicago	chicago	PROPN
flr-250	351	4	roc	roc	PROPN
flr-250	351	5	software	software	PROPN
flr-250	351	6	.	.	PUNCT
flr-250	352	1	http://metz-roc.uchicago.edu/	http://metz-roc.uchicago.edu/	PROPN
flr-250	352	2	last	last	ADV
flr-250	352	3	accessed	access	VERB
flr-250	352	4	april	april	PROPN
flr-250	352	5	13	13	NUM
flr-250	352	6	,	,	PUNCT
flr-250	352	7	2016	2016	NUM
flr-250	352	8	.	.	PUNCT
flr-250	353	1	wald	wald	X
flr-250	353	2	,	,	PUNCT
flr-250	353	3	a.	a.	NOUN
flr-250	353	4	(	(	PUNCT
flr-250	353	5	1950	1950	NUM
flr-250	353	6	)	)	PUNCT
flr-250	353	7	.	.	PUNCT
flr-250	354	1	statistical	statistical	ADJ
flr-250	354	2	decision	decision	NOUN
flr-250	354	3	functions	function	NOUN
flr-250	354	4	.	.	PUNCT
flr-250	355	1	new	new	PROPN
flr-250	355	2	york	york	PROPN
flr-250	355	3	,	,	PUNCT
flr-250	355	4	ny	ny	PROPN
flr-250	355	5	:	:	PUNCT
flr-250	355	6	wiley	wiley	PROPN
flr-250	355	7	,	,	PUNCT
flr-250	355	8	inc	inc	PROPN
flr-250	355	9	.	.	PROPN
flr-250	356	1	zou	zou	PROPN
flr-250	356	2	,	,	PUNCT
flr-250	356	3	k.h	k.h	PROPN
flr-250	356	4	.	.	PROPN
flr-250	356	5	,	,	PUNCT
flr-250	356	6	hall	hall	PROPN
flr-250	356	7	,	,	PUNCT
flr-250	356	8	w.j	w.j	PROPN
flr-250	356	9	.	.	PROPN
flr-250	356	10	,	,	PUNCT
flr-250	356	11	shapiro	shapiro	PROPN
flr-250	356	12	,	,	PUNCT
flr-250	356	13	d.e	d.e	PROPN
flr-250	356	14	.	.	PROPN
flr-250	356	15	(	(	PUNCT
flr-250	356	16	1997	1997	NUM
flr-250	356	17	)	)	PUNCT
flr-250	356	18	.	.	PUNCT
flr-250	357	1	smooth	smooth	ADJ
flr-250	357	2	non	non	ADJ
flr-250	357	3	-	-	ADJ
flr-250	357	4	parametric	parametric	ADJ
flr-250	357	5	receiver	receiver	NOUN
flr-250	357	6	operating	operate	VERB
flr-250	357	7	characteristic	characteristic	NOUN
flr-250	357	8	(	(	PUNCT
flr-250	357	9	roc	roc	PROPN
flr-250	357	10	)	)	PUNCT
flr-250	357	11	curves	curve	NOUN
flr-250	357	12	for	for	ADP
flr-250	357	13	continuous	continuous	ADJ
flr-250	357	14	diagnostic	diagnostic	ADJ
flr-250	357	15	tests	test	NOUN
flr-250	357	16	.	.	PUNCT
flr-250	358	1	stats	stat	VERB
flr-250	358	2	med	med	ADJ
flr-250	358	3	,	,	PUNCT
flr-250	358	4	16	16	NUM
flr-250	358	5	(	(	PUNCT
flr-250	358	6	19	19	NUM
flr-250	358	7	)	)	PUNCT
flr-250	358	8	,	,	PUNCT
flr-250	358	9	2143	2143	NUM
flr-250	358	10	-	-	SYM
flr-250	358	11	2156	2156	NUM
flr-250	358	12	.	.	PUNCT
flr-250	359	1	pmid	pmid	NOUN
flr-250	359	2	:	:	PUNCT
flr-250	359	3	9330425	9330425	NUM
flr-250	359	4	zhou	zhou	X
flr-250	359	5	,	,	PUNCT
flr-250	359	6	x.h	x.h	PROPN
flr-250	359	7	.	.	PROPN
flr-250	359	8	,	,	PUNCT
flr-250	359	9	obuchowski	obuchowski	VERB
flr-250	359	10	,	,	PUNCT
flr-250	359	11	n.a	n.a	PROPN
flr-250	359	12	.	.	PROPN
flr-250	359	13	,	,	PUNCT
flr-250	359	14	mcclish	mcclish	PROPN
flr-250	359	15	,	,	PUNCT
flr-250	359	16	d.k	d.k	PROPN
flr-250	359	17	.	.	PROPN
flr-250	360	1	(	(	PUNCT
flr-250	360	2	2002	2002	NUM
flr-250	360	3	)	)	PUNCT
flr-250	360	4	.	.	PUNCT
flr-250	361	1	statistical	statistical	ADJ
flr-250	361	2	methods	method	NOUN
flr-250	361	3	in	in	ADP
flr-250	361	4	diagnostic	diagnostic	ADJ
flr-250	361	5	medicine	medicine	NOUN
flr-250	361	6	.	.	PUNCT
flr-250	362	1	new	new	PROPN
flr-250	362	2	york	york	PROPN
flr-250	362	3	,	,	PUNCT
flr-250	362	4	ny	ny	PROPN
flr-250	362	5	:	:	PUNCT
flr-250	362	6	wiley	wiley	PROPN
flr-250	362	7	.	.	PUNCT
flr-250	363	1	medcalc	medcalc	PROPN
flr-250	363	2	statistical	statistical	ADJ
flr-250	363	3	software	software	NOUN
flr-250	363	4	.	.	PUNCT
flr-250	364	1	https://www.medcalc.org/manual/roc-curves.php	https://www.medcalc.org/manual/roc-curves.php	PROPN
flr-250	364	2	last	last	ADJ
flr-250	364	3	accessed	access	VERB
flr-250	364	4	april	april	PROPN
flr-250	364	5	13	13	NUM
flr-250	364	6	,	,	PUNCT
flr-250	364	7	2016	2016	NUM
flr-250	364	8	.	.	PUNCT
flr-250	365	1	https://doi.org/10.1148/116.3.533	https://doi.org/10.1148/116.3.533	PROPN
flr-250	365	2	http://www.stata.com/features/overview/receiver-operating-characteristic/	http://www.stata.com/features/overview/receiver-operating-characteristic/	NOUN
flr-250	365	3	http://www.stata.com/features/overview/receiver-operating-characteristic/	http://www.stata.com/features/overview/receiver-operating-characteristic/	PROPN
flr-250	365	4	https://doi.org/10.1118/1.597758	https://doi.org/10.1118/1.597758	PROPN
flr-250	365	5	https://doi.org/10.1111/1529-1006.001	https://doi.org/10.1111/1529-1006.001	VERB
flr-250	365	6	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/default.aspx	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/default.aspx	NOUN
flr-250	365	7	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/default.aspx	http://perception.radiology.uiowa.edu/software/receiveroperatingcharacteristicroc/tabid/120/default.aspx	VERB
flr-250	365	8	http://metz-roc.uchicago.edu/	http://metz-roc.uchicago.edu/	X
flr-250	366	1	https://www.medcalc.org/manual/roc-curves.php	https://www.medcalc.org/manual/roc-curves.php	ADJ