id	sid	tid	token	lemma	pos
spir-13471	1	1	niemann	niemann	PROPN
spir-13471	1	2	-	-	PUNCT
spir-13471	1	3	lenz	lenz	PROPN
spir-13471	1	4	-	-	PUNCT
spir-13471	1	5	with	with	ADP
spir-13471	1	6	or	or	CCONJ
spir-13471	1	7	without	without	ADP
spir-13471	1	8	the	the	DET
spir-13471	1	9	crowd	crowd	NOUN
spir-13471	1	10	the	the	DET
spir-13471	1	11	influence	influence	NOUN
spir-13471	1	12	of	of	ADP
spir-13471	1	13	coder	coder	NOUN
spir-13471	1	14	characteristics	characteristic	NOUN
spir-13471	1	15	-	-	PUNCT
spir-13471	1	16	with	with	ADP
spir-13471	1	17	or	or	CCONJ
spir-13471	1	18	without	without	ADP
spir-13471	1	19	the	the	DET
spir-13471	1	20	crowd	crowd	NOUN
spir-13471	1	21	the	the	DET
spir-13471	1	22	selected	select	VERB
spir-13471	1	23	papers	paper	NOUN
spir-13471	1	24	of	of	ADP
spir-13471	1	25	#	#	SYM
spir-13471	1	26	aoir2023	aoir2023	PROPN
spir-13471	1	27	:	:	PUNCT
spir-13471	1	28	the	the	DET
spir-13471	1	29	24th	24th	ADJ
spir-13471	1	30	annual	annual	ADJ
spir-13471	1	31	conference	conference	NOUN
spir-13471	1	32	of	of	ADP
spir-13471	1	33	the	the	DET
spir-13471	1	34	association	association	NOUN
spir-13471	1	35	of	of	ADP
spir-13471	1	36	internet	internet	NOUN
spir-13471	1	37	researchers	researcher	NOUN
spir-13471	1	38	philadelphia	philadelphia	PROPN
spir-13471	1	39	,	,	PUNCT
spir-13471	1	40	pa	pa	PROPN
spir-13471	1	41	,	,	PUNCT
spir-13471	1	42	usa	usa	PROPN
spir-13471	1	43	/	/	SYM
spir-13471	1	44	18	18	NUM
spir-13471	1	45	-	-	SYM
spir-13471	1	46	21	21	NUM
spir-13471	1	47	oct	oct	NOUN
spir-13471	1	48	2023	2023	NUM
spir-13471	1	49	with	with	ADP
spir-13471	1	50	or	or	CCONJ
spir-13471	1	51	without	without	ADP
spir-13471	1	52	the	the	DET
spir-13471	1	53	crowd	crowd	NOUN
spir-13471	1	54	?	?	PUNCT
spir-13471	2	1	the	the	DET
spir-13471	2	2	influence	influence	NOUN
spir-13471	2	3	of	of	ADP
spir-13471	2	4	coder	coder	ADJ
spir-13471	2	5	characteristics	characteristic	NOUN
spir-13471	2	6	on	on	ADP
spir-13471	2	7	coding	code	VERB
spir-13471	2	8	decisions	decision	NOUN
spir-13471	2	9	comparing	compare	VERB
spir-13471	2	10	crowdworkers	crowdworker	NOUN
spir-13471	2	11	and	and	CCONJ
spir-13471	2	12	traditional	traditional	ADJ
spir-13471	2	13	coders	coder	NOUN
spir-13471	2	14	.	.	PUNCT
spir-13471	3	1	julia	julia	PROPN
spir-13471	3	2	niemann	niemann	PROPN
spir-13471	3	3	-	-	PUNCT
spir-13471	3	4	lenz	lenz	PROPN
spir-13471	3	5	university	university	PROPN
spir-13471	3	6	of	of	ADP
spir-13471	3	7	hamburg	hamburg	PROPN
spir-13471	3	8	anja	anja	PROPN
spir-13471	3	9	dittrich	dittrich	PROPN
spir-13471	3	10	hanover	hanover	PROPN
spir-13471	3	11	university	university	PROPN
spir-13471	3	12	of	of	ADP
spir-13471	3	13	music	music	NOUN
spir-13471	3	14	,	,	PUNCT
spir-13471	3	15	drama	drama	NOUN
spir-13471	3	16	,	,	PUNCT
spir-13471	3	17	&	&	CCONJ
spir-13471	3	18	media	media	PROPN
spir-13471	3	19	jule	jule	PROPN
spir-13471	3	20	scheper	scheper	PROPN
spir-13471	3	21	hanover	hanover	PROPN
spir-13471	3	22	university	university	PROPN
spir-13471	3	23	of	of	ADP
spir-13471	3	24	music	music	NOUN
spir-13471	3	25	,	,	PUNCT
spir-13471	3	26	drama	drama	NOUN
spir-13471	3	27	,	,	PUNCT
spir-13471	3	28	&	&	CCONJ
spir-13471	3	29	media	media	PROPN
spir-13471	3	30	research	research	PROPN
spir-13471	3	31	objective	objective	ADJ
spir-13471	3	32	standardized	standardize	VERB
spir-13471	3	33	content	content	NOUN
spir-13471	3	34	analysis	analysis	NOUN
spir-13471	3	35	is	be	AUX
spir-13471	3	36	a	a	DET
spir-13471	3	37	prominent	prominent	ADJ
spir-13471	3	38	method	method	NOUN
spir-13471	3	39	in	in	ADP
spir-13471	3	40	internet	internet	NOUN
spir-13471	3	41	research	research	NOUN
spir-13471	3	42	;	;	PUNCT
spir-13471	3	43	it	it	PRON
spir-13471	3	44	is	be	AUX
spir-13471	3	45	used	use	VERB
spir-13471	3	46	,	,	PUNCT
spir-13471	3	47	for	for	ADP
spir-13471	3	48	example	example	NOUN
spir-13471	3	49	,	,	PUNCT
spir-13471	3	50	to	to	PART
spir-13471	3	51	evaluate	evaluate	VERB
spir-13471	3	52	topics	topic	NOUN
spir-13471	3	53	,	,	PUNCT
spir-13471	3	54	tendencies	tendency	NOUN
spir-13471	3	55	,	,	PUNCT
spir-13471	3	56	and	and	CCONJ
spir-13471	3	57	sentiment	sentiment	NOUN
spir-13471	3	58	of	of	ADP
spir-13471	3	59	textual	textual	ADJ
spir-13471	3	60	digital	digital	ADJ
spir-13471	3	61	trace	trace	NOUN
spir-13471	3	62	data	datum	NOUN
spir-13471	3	63	,	,	PUNCT
spir-13471	3	64	such	such	ADJ
spir-13471	3	65	as	as	ADP
spir-13471	3	66	social	social	ADJ
spir-13471	3	67	media	medium	NOUN
spir-13471	3	68	posts	post	NOUN
spir-13471	3	69	or	or	CCONJ
spir-13471	3	70	comments	comment	NOUN
spir-13471	3	71	.	.	PUNCT
spir-13471	4	1	the	the	DET
spir-13471	4	2	importance	importance	NOUN
spir-13471	4	3	of	of	ADP
spir-13471	4	4	the	the	DET
spir-13471	4	5	method	method	NOUN
spir-13471	4	6	has	have	AUX
spir-13471	4	7	even	even	ADV
spir-13471	4	8	increased	increase	VERB
spir-13471	4	9	due	due	ADP
spir-13471	4	10	to	to	ADP
spir-13471	4	11	the	the	DET
spir-13471	4	12	demand	demand	NOUN
spir-13471	4	13	of	of	ADP
spir-13471	4	14	high	high	ADJ
spir-13471	4	15	-	-	PUNCT
spir-13471	4	16	quality	quality	NOUN
spir-13471	4	17	training	training	NOUN
spir-13471	4	18	data	datum	NOUN
spir-13471	4	19	for	for	ADP
spir-13471	4	20	text	text	NOUN
spir-13471	4	21	mining	mining	NOUN
spir-13471	4	22	approaches	approach	NOUN
spir-13471	4	23	based	base	VERB
spir-13471	4	24	upon	upon	SCONJ
spir-13471	4	25	supervised	supervised	ADJ
spir-13471	4	26	machine	machine	NOUN
spir-13471	4	27	learning	learning	NOUN
spir-13471	4	28	.	.	PUNCT
spir-13471	5	1	however	however	ADV
spir-13471	5	2	,	,	PUNCT
spir-13471	5	3	standardized	standardized	ADJ
spir-13471	5	4	content	content	NOUN
spir-13471	5	5	analyses	analysis	NOUN
spir-13471	5	6	are	be	AUX
spir-13471	5	7	very	very	ADV
spir-13471	5	8	time	time	NOUN
spir-13471	5	9	and	and	CCONJ
spir-13471	5	10	resource	resource	NOUN
spir-13471	5	11	-	-	PUNCT
spir-13471	5	12	intensive	intensive	ADJ
spir-13471	5	13	,	,	PUNCT
spir-13471	5	14	mainly	mainly	ADV
spir-13471	5	15	due	due	ADP
spir-13471	5	16	to	to	ADP
spir-13471	5	17	the	the	DET
spir-13471	5	18	high	high	ADJ
spir-13471	5	19	demand	demand	NOUN
spir-13471	5	20	for	for	ADP
spir-13471	5	21	human	human	ADJ
spir-13471	5	22	resources	resource	NOUN
spir-13471	5	23	.	.	PUNCT
spir-13471	6	1	here	here	ADV
spir-13471	6	2	,	,	PUNCT
spir-13471	6	3	the	the	DET
spir-13471	6	4	process	process	NOUN
spir-13471	6	5	of	of	ADP
spir-13471	6	6	coding1	coding1	NOUN
spir-13471	6	7	is	be	AUX
spir-13471	6	8	particularly	particularly	ADV
spir-13471	6	9	critical	critical	ADJ
spir-13471	6	10	:	:	PUNCT
spir-13471	6	11	to	to	PART
spir-13471	6	12	obtain	obtain	VERB
spir-13471	6	13	sufficient	sufficient	ADJ
spir-13471	6	14	reliability	reliability	NOUN
spir-13471	6	15	,	,	PUNCT
spir-13471	6	16	it	it	PRON
spir-13471	6	17	is	be	AUX
spir-13471	6	18	essential	essential	ADJ
spir-13471	6	19	that	that	SCONJ
spir-13471	6	20	all	all	DET
spir-13471	6	21	coders	coder	NOUN
spir-13471	6	22	use	use	VERB
spir-13471	6	23	the	the	DET
spir-13471	6	24	codebook	codebook	NOUN
spir-13471	6	25	in	in	ADP
spir-13471	6	26	the	the	DET
spir-13471	6	27	same	same	ADJ
spir-13471	6	28	way	way	NOUN
spir-13471	6	29	(	(	PUNCT
spir-13471	6	30	krippendorff	krippendorff	NOUN
spir-13471	6	31	,	,	PUNCT
spir-13471	6	32	2002	2002	NUM
spir-13471	6	33	)	)	PUNCT
spir-13471	6	34	.	.	PUNCT
spir-13471	7	1	however	however	ADV
spir-13471	7	2	,	,	PUNCT
spir-13471	7	3	coders	coder	NOUN
spir-13471	7	4	code	code	VERB
spir-13471	7	5	under	under	ADP
spir-13471	7	6	the	the	DET
spir-13471	7	7	conditions	condition	NOUN
spir-13471	7	8	of	of	ADP
spir-13471	7	9	a	a	DET
spir-13471	7	10	specific	specific	ADJ
spir-13471	7	11	context	context	NOUN
spir-13471	7	12	,	,	PUNCT
spir-13471	7	13	which	which	PRON
spir-13471	7	14	is	be	AUX
spir-13471	7	15	shaped	shape	VERB
spir-13471	7	16	by	by	ADP
spir-13471	7	17	their	their	PRON
spir-13471	7	18	individual	individual	ADJ
spir-13471	7	19	characteristics	characteristic	NOUN
spir-13471	7	20	,	,	PUNCT
spir-13471	7	21	the	the	DET
spir-13471	7	22	coding	code	VERB
spir-13471	7	23	situation	situation	NOUN
spir-13471	7	24	,	,	PUNCT
spir-13471	7	25	and	and	CCONJ
spir-13471	7	26	the	the	DET
spir-13471	7	27	material	material	NOUN
spir-13471	7	28	to	to	PART
spir-13471	7	29	be	be	AUX
spir-13471	7	30	coded	code	VERB
spir-13471	7	31	(	(	PUNCT
spir-13471	7	32	degen	degen	NOUN
spir-13471	7	33	,	,	PUNCT
spir-13471	7	34	2015	2015	NUM
spir-13471	7	35	)	)	PUNCT
spir-13471	7	36	.	.	PUNCT
spir-13471	8	1	notwithstanding	notwithstanding	ADP
spir-13471	8	2	the	the	DET
spir-13471	8	3	importance	importance	NOUN
spir-13471	8	4	of	of	ADP
spir-13471	8	5	the	the	DET
spir-13471	8	6	coding	code	VERB
spir-13471	8	7	process	process	NOUN
spir-13471	8	8	,	,	PUNCT
spir-13471	8	9	little	little	ADJ
spir-13471	8	10	research	research	NOUN
spir-13471	8	11	has	have	AUX
spir-13471	8	12	been	be	AUX
spir-13471	8	13	done	do	VERB
spir-13471	8	14	on	on	ADP
spir-13471	8	15	how	how	SCONJ
spir-13471	8	16	coding	coding	NOUN
spir-13471	8	17	should	should	AUX
spir-13471	8	18	be	be	AUX
spir-13471	8	19	optimally	optimally	ADV
spir-13471	8	20	designed	design	VERB
spir-13471	8	21	and	and	CCONJ
spir-13471	8	22	how	how	SCONJ
spir-13471	8	23	personal	personal	ADJ
spir-13471	8	24	characteristics	characteristic	NOUN
spir-13471	8	25	of	of	ADP
spir-13471	8	26	the	the	DET
spir-13471	8	27	coders	coder	NOUN
spir-13471	8	28	influence	influence	VERB
spir-13471	8	29	the	the	DET
spir-13471	8	30	coding	code	VERB
spir-13471	8	31	results	result	NOUN
spir-13471	8	32	.	.	PUNCT
spir-13471	9	1	1	1	NUM
spir-13471	9	2	we	we	PRON
spir-13471	9	3	use	use	VERB
spir-13471	9	4	the	the	DET
spir-13471	9	5	term	term	NOUN
spir-13471	9	6	"	"	PUNCT
spir-13471	9	7	coding	code	VERB
spir-13471	9	8	"	"	PUNCT
spir-13471	9	9	because	because	SCONJ
spir-13471	9	10	it	it	PRON
spir-13471	9	11	is	be	AUX
spir-13471	9	12	common	common	ADJ
spir-13471	9	13	terminology	terminology	NOUN
spir-13471	9	14	in	in	ADP
spir-13471	9	15	literature	literature	NOUN
spir-13471	9	16	(	(	PUNCT
spir-13471	9	17	e.g.	e.g.	ADV
spir-13471	9	18	krippendorff	krippendorff	NOUN
spir-13471	9	19	2002	2002	NUM
spir-13471	9	20	)	)	PUNCT
spir-13471	9	21	.	.	PUNCT
spir-13471	10	1	however	however	ADV
spir-13471	10	2	,	,	PUNCT
spir-13471	10	3	the	the	DET
spir-13471	10	4	considerations	consideration	NOUN
spir-13471	10	5	and	and	CCONJ
spir-13471	10	6	results	result	NOUN
spir-13471	10	7	provided	provide	VERB
spir-13471	10	8	by	by	ADP
spir-13471	10	9	our	our	PRON
spir-13471	10	10	study	study	NOUN
spir-13471	10	11	are	be	AUX
spir-13471	10	12	applicable	applicable	ADJ
spir-13471	10	13	to	to	ADP
spir-13471	10	14	"	"	PUNCT
spir-13471	10	15	annotating	annotate	VERB
spir-13471	10	16	"	"	PUNCT
spir-13471	10	17	as	as	ADV
spir-13471	10	18	well	well	ADV
spir-13471	10	19	,	,	PUNCT
spir-13471	10	20	which	which	PRON
spir-13471	10	21	is	be	AUX
spir-13471	10	22	a	a	DET
spir-13471	10	23	corresponding	corresponding	ADJ
spir-13471	10	24	task	task	NOUN
spir-13471	10	25	in	in	ADP
spir-13471	10	26	computer	computer	NOUN
spir-13471	10	27	science	science	NOUN
spir-13471	10	28	.	.	PUNCT
spir-13471	11	1	suggested	suggest	VERB
spir-13471	11	2	citation	citation	NOUN
spir-13471	11	3	(	(	PUNCT
spir-13471	11	4	apa	apa	PROPN
spir-13471	11	5	):	):	PUNCT
spir-13471	11	6	niemann	niemann	PROPN
spir-13471	11	7	-	-	PUNCT
spir-13471	11	8	lenz	lenz	PROPN
spir-13471	11	9	,	,	PUNCT
spir-13471	11	10	j.	j.	PROPN
spir-13471	11	11	,	,	PUNCT
spir-13471	11	12	dittrich	dittrich	PROPN
spir-13471	11	13	,	,	PUNCT
spir-13471	11	14	a.	a.	NOUN
spir-13471	11	15	,	,	PUNCT
spir-13471	11	16	scheper	scheper	PROPN
spir-13471	11	17	,	,	PUNCT
spir-13471	11	18	j.	j.	PROPN
spir-13471	11	19	(	(	PUNCT
spir-13471	11	20	2023	2023	NUM
spir-13471	11	21	,	,	PUNCT
spir-13471	11	22	october	october	PROPN
spir-13471	11	23	)	)	PUNCT
spir-13471	11	24	.	.	PUNCT
spir-13471	12	1	with	with	ADP
spir-13471	12	2	or	or	CCONJ
spir-13471	12	3	without	without	ADP
spir-13471	12	4	the	the	DET
spir-13471	12	5	crowd	crowd	NOUN
spir-13471	12	6	?	?	PUNCT
spir-13471	13	1	the	the	DET
spir-13471	13	2	influence	influence	NOUN
spir-13471	13	3	of	of	ADP
spir-13471	13	4	coder	coder	ADJ
spir-13471	13	5	characteristics	characteristic	NOUN
spir-13471	13	6	on	on	ADP
spir-13471	13	7	coding	code	VERB
spir-13471	13	8	decisions	decision	NOUN
spir-13471	13	9	comparing	compare	VERB
spir-13471	13	10	crowdworkers	crowdworker	NOUN
spir-13471	13	11	and	and	CCONJ
spir-13471	13	12	traditional	traditional	ADJ
spir-13471	13	13	coders	coder	NOUN
spir-13471	13	14	.	.	PUNCT
spir-13471	14	1	paper	paper	NOUN
spir-13471	14	2	presented	present	VERB
spir-13471	14	3	at	at	ADP
spir-13471	14	4	aoir2023	aoir2023	PROPN
spir-13471	14	5	:	:	PUNCT
spir-13471	14	6	the	the	DET
spir-13471	14	7	24th	24th	ADJ
spir-13471	14	8	annual	annual	ADJ
spir-13471	14	9	conference	conference	NOUN
spir-13471	14	10	of	of	ADP
spir-13471	14	11	the	the	DET
spir-13471	14	12	association	association	NOUN
spir-13471	14	13	of	of	ADP
spir-13471	14	14	internet	internet	NOUN
spir-13471	14	15	researchers	researcher	NOUN
spir-13471	14	16	.	.	PUNCT
spir-13471	15	1	philadelphia	philadelphia	PROPN
spir-13471	15	2	,	,	PUNCT
spir-13471	15	3	pa	pa	PROPN
spir-13471	15	4	,	,	PUNCT
spir-13471	15	5	usa	usa	PROPN
spir-13471	15	6	:	:	PUNCT
spir-13471	15	7	aoir	aoir	NOUN
spir-13471	15	8	.	.	PUNCT
spir-13471	15	9	retrieved	retrieve	VERB
spir-13471	15	10	from	from	ADP
spir-13471	15	11	http://spir.aoir.org	http://spir.aoir.org	PROPN
spir-13471	15	12	.	.	PUNCT
spir-13471	16	1	the	the	DET
spir-13471	16	2	issue	issue	NOUN
spir-13471	16	3	raised	raise	VERB
spir-13471	16	4	above	above	ADV
spir-13471	16	5	is	be	AUX
spir-13471	16	6	intensified	intensify	VERB
spir-13471	16	7	by	by	ADP
spir-13471	16	8	the	the	DET
spir-13471	16	9	fact	fact	NOUN
spir-13471	16	10	that	that	SCONJ
spir-13471	16	11	an	an	DET
spir-13471	16	12	increasing	increase	VERB
spir-13471	16	13	number	number	NOUN
spir-13471	16	14	of	of	ADP
spir-13471	16	15	studies	study	NOUN
spir-13471	16	16	rely	rely	VERB
spir-13471	16	17	on	on	ADP
spir-13471	16	18	crowdworkers	crowdworker	NOUN
spir-13471	16	19	as	as	ADP
spir-13471	16	20	coders	coder	NOUN
spir-13471	16	21	(	(	PUNCT
spir-13471	16	22	e.g.	e.g.	ADV
spir-13471	16	23	,	,	PUNCT
spir-13471	16	24	boxmann	boxmann	NOUN
spir-13471	16	25	-	-	PUNCT
spir-13471	16	26	shabtai	shabtai	ADJ
spir-13471	16	27	2021	2021	NUM
spir-13471	16	28	;	;	PUNCT
spir-13471	16	29	budak	budak	PROPN
spir-13471	16	30	,	,	PUNCT
spir-13471	16	31	goel	goel	PROPN
spir-13471	16	32	&	&	CCONJ
spir-13471	16	33	rao	rao	PROPN
spir-13471	16	34	2016	2016	NUM
spir-13471	16	35	;	;	PUNCT
spir-13471	16	36	hornik	hornik	X
spir-13471	16	37	et	et	PROPN
spir-13471	16	38	al	al	PROPN
spir-13471	16	39	.	.	PROPN
spir-13471	16	40	2022	2022	NUM
spir-13471	16	41	)	)	PUNCT
spir-13471	16	42	.	.	PUNCT
spir-13471	17	1	this	this	PRON
spir-13471	17	2	is	be	AUX
spir-13471	17	3	reasonable	reasonable	ADJ
spir-13471	17	4	,	,	PUNCT
spir-13471	17	5	for	for	ADP
spir-13471	17	6	crowdworkers	crowdworker	NOUN
spir-13471	17	7	are	be	AUX
spir-13471	17	8	cost	cost	NOUN
spir-13471	17	9	-	-	PUNCT
spir-13471	17	10	effective	effective	ADJ
spir-13471	17	11	and	and	CCONJ
spir-13471	17	12	flexible	flexible	ADJ
spir-13471	17	13	in	in	ADP
spir-13471	17	14	employment	employment	NOUN
spir-13471	17	15	.	.	PUNCT
spir-13471	18	1	however	however	ADV
spir-13471	18	2	,	,	PUNCT
spir-13471	18	3	in	in	ADP
spir-13471	18	4	content	content	NOUN
spir-13471	18	5	analysis	analysis	NOUN
spir-13471	18	6	they	they	PRON
spir-13471	18	7	bear	bear	VERB
spir-13471	18	8	substantial	substantial	ADJ
spir-13471	18	9	disadvantages	disadvantage	NOUN
spir-13471	18	10	:	:	PUNCT
spir-13471	18	11	in	in	ADP
spir-13471	18	12	addition	addition	NOUN
spir-13471	18	13	to	to	ADP
spir-13471	18	14	ethical	ethical	ADJ
spir-13471	18	15	aspects	aspect	NOUN
spir-13471	18	16	—	—	PUNCT
spir-13471	18	17	such	such	ADJ
spir-13471	18	18	as	as	ADP
spir-13471	18	19	the	the	DET
spir-13471	18	20	question	question	NOUN
spir-13471	18	21	of	of	ADP
spir-13471	18	22	appropriate	appropriate	ADJ
spir-13471	18	23	compensation	compensation	NOUN
spir-13471	18	24	for	for	ADP
spir-13471	18	25	research	research	NOUN
spir-13471	18	26	work	work	NOUN
spir-13471	18	27	—	—	PUNCT
spir-13471	18	28	the	the	DET
spir-13471	18	29	use	use	NOUN
spir-13471	18	30	of	of	ADP
spir-13471	18	31	crowdworkers	crowdworker	NOUN
spir-13471	18	32	also	also	ADV
spir-13471	18	33	exacerbate	exacerbate	VERB
spir-13471	18	34	blind	blind	ADJ
spir-13471	18	35	spots	spot	NOUN
spir-13471	18	36	in	in	ADP
spir-13471	18	37	the	the	DET
spir-13471	18	38	research	research	NOUN
spir-13471	18	39	process	process	NOUN
spir-13471	18	40	.	.	PUNCT
spir-13471	19	1	in	in	ADP
spir-13471	19	2	particular	particular	ADJ
spir-13471	19	3	,	,	PUNCT
spir-13471	19	4	the	the	DET
spir-13471	19	5	selection	selection	NOUN
spir-13471	19	6	of	of	ADP
spir-13471	19	7	suitable	suitable	ADJ
spir-13471	19	8	coders	coder	NOUN
spir-13471	19	9	and	and	CCONJ
spir-13471	19	10	the	the	DET
spir-13471	19	11	poor	poor	ADJ
spir-13471	19	12	control	control	NOUN
spir-13471	19	13	of	of	ADP
spir-13471	19	14	the	the	DET
spir-13471	19	15	coding	code	VERB
spir-13471	19	16	process	process	NOUN
spir-13471	19	17	are	be	AUX
spir-13471	19	18	to	to	PART
spir-13471	19	19	be	be	AUX
spir-13471	19	20	mentioned	mention	VERB
spir-13471	19	21	.	.	PUNCT
spir-13471	20	1	the	the	DET
spir-13471	20	2	employment	employment	NOUN
spir-13471	20	3	of	of	ADP
spir-13471	20	4	crowdworkers	crowdworker	NOUN
spir-13471	20	5	as	as	ADP
spir-13471	20	6	coders	coder	NOUN
spir-13471	20	7	should	should	AUX
spir-13471	20	8	therefore	therefore	ADV
spir-13471	20	9	not	not	PART
spir-13471	20	10	be	be	AUX
spir-13471	20	11	undertaken	undertake	VERB
spir-13471	20	12	without	without	ADP
spir-13471	20	13	reflection	reflection	NOUN
spir-13471	20	14	.	.	PUNCT
spir-13471	21	1	a	a	DET
spir-13471	21	2	first	first	ADJ
spir-13471	21	3	study	study	NOUN
spir-13471	21	4	evaluating	evaluate	VERB
spir-13471	21	5	the	the	DET
spir-13471	21	6	use	use	NOUN
spir-13471	21	7	of	of	ADP
spir-13471	21	8	crowdworkers	crowdworker	NOUN
spir-13471	21	9	concludes	conclude	VERB
spir-13471	21	10	that	that	SCONJ
spir-13471	21	11	they	they	PRON
spir-13471	21	12	offer	offer	VERB
spir-13471	21	13	a	a	DET
spir-13471	21	14	high	high	ADJ
spir-13471	21	15	potential	potential	NOUN
spir-13471	21	16	for	for	ADP
spir-13471	21	17	content	content	NOUN
spir-13471	21	18	analysis	analysis	NOUN
spir-13471	21	19	and	and	CCONJ
spir-13471	21	20	that	that	SCONJ
spir-13471	21	21	they	they	PRON
spir-13471	21	22	do	do	AUX
spir-13471	21	23	not	not	PART
spir-13471	21	24	perform	perform	VERB
spir-13471	21	25	significantly	significantly	ADV
spir-13471	21	26	worse	bad	ADJ
spir-13471	21	27	than	than	ADP
spir-13471	21	28	classic	classic	ADJ
spir-13471	21	29	offline	offline	ADJ
spir-13471	21	30	coders	coder	NOUN
spir-13471	21	31	(	(	PUNCT
spir-13471	21	32	lind	lind	PROPN
spir-13471	21	33	et	et	PROPN
spir-13471	21	34	al	al	PROPN
spir-13471	21	35	.	.	PROPN
spir-13471	21	36	2017	2017	NUM
spir-13471	21	37	)	)	PUNCT
spir-13471	21	38	.	.	PUNCT
spir-13471	22	1	however	however	ADV
spir-13471	22	2	,	,	PUNCT
spir-13471	22	3	the	the	DET
spir-13471	22	4	reliability	reliability	NOUN
spir-13471	22	5	coefficients	coefficient	NOUN
spir-13471	22	6	reported	report	VERB
spir-13471	22	7	there	there	PRON
spir-13471	22	8	are	be	VERB
spir-13471	22	9	by	by	ADP
spir-13471	22	10	no	no	DET
spir-13471	22	11	means	mean	NOUN
spir-13471	22	12	satisfactory	satisfactory	ADJ
spir-13471	22	13	according	accord	VERB
spir-13471	22	14	to	to	ADP
spir-13471	22	15	common	common	ADJ
spir-13471	22	16	criteria	criterion	NOUN
spir-13471	22	17	.	.	PUNCT
spir-13471	23	1	against	against	ADP
spir-13471	23	2	this	this	DET
spir-13471	23	3	background	background	NOUN
spir-13471	23	4	,	,	PUNCT
spir-13471	23	5	we	we	PRON
spir-13471	23	6	investigate	investigate	VERB
spir-13471	23	7	comparatively	comparatively	ADV
spir-13471	23	8	,	,	PUNCT
spir-13471	23	9	in	in	ADP
spir-13471	23	10	a	a	DET
spir-13471	23	11	sample	sample	NOUN
spir-13471	23	12	of	of	ADP
spir-13471	23	13	crowdworkers	crowdworker	NOUN
spir-13471	23	14	and	and	CCONJ
spir-13471	23	15	student	student	NOUN
spir-13471	23	16	coders	coder	NOUN
spir-13471	23	17	,	,	PUNCT
spir-13471	23	18	which	which	DET
spir-13471	23	19	influence	influence	NOUN
spir-13471	23	20	coder	coder	NOUN
spir-13471	23	21	characteristics	characteristic	NOUN
spir-13471	23	22	have	have	VERB
spir-13471	23	23	on	on	ADP
spir-13471	23	24	the	the	DET
spir-13471	23	25	validity	validity	NOUN
spir-13471	23	26	and	and	CCONJ
spir-13471	23	27	reliability	reliability	NOUN
spir-13471	23	28	of	of	ADP
spir-13471	23	29	the	the	DET
spir-13471	23	30	coding	code	VERB
spir-13471	23	31	results	result	NOUN
spir-13471	23	32	.	.	PUNCT
spir-13471	24	1	specifically	specifically	ADV
spir-13471	24	2	,	,	PUNCT
spir-13471	24	3	we	we	PRON
spir-13471	24	4	ask	ask	VERB
spir-13471	24	5	(	(	PUNCT
spir-13471	24	6	a	a	X
spir-13471	24	7	)	)	PUNCT
spir-13471	24	8	how	how	SCONJ
spir-13471	24	9	the	the	DET
spir-13471	24	10	quality	quality	NOUN
spir-13471	24	11	of	of	ADP
spir-13471	24	12	coding	code	VERB
spir-13471	24	13	with	with	ADP
spir-13471	24	14	crowdworkers	crowdworker	NOUN
spir-13471	24	15	compares	compare	VERB
spir-13471	24	16	to	to	ADP
spir-13471	24	17	offline	offline	ADJ
spir-13471	24	18	coders	coder	NOUN
spir-13471	24	19	,	,	PUNCT
spir-13471	24	20	and	and	CCONJ
spir-13471	24	21	(	(	PUNCT
spir-13471	24	22	b	b	NOUN
spir-13471	24	23	)	)	PUNCT
spir-13471	24	24	how	how	SCONJ
spir-13471	24	25	coder	coder	ADJ
spir-13471	24	26	characteristics	characteristic	NOUN
spir-13471	24	27	affect	affect	VERB
spir-13471	24	28	the	the	DET
spir-13471	24	29	quality	quality	NOUN
spir-13471	24	30	of	of	ADP
spir-13471	24	31	the	the	DET
spir-13471	24	32	data	datum	NOUN
spir-13471	24	33	.	.	PUNCT
spir-13471	25	1	our	our	PRON
spir-13471	25	2	hypotheses	hypothesis	NOUN
spir-13471	25	3	relate	relate	VERB
spir-13471	25	4	to	to	ADP
spir-13471	25	5	the	the	DET
spir-13471	25	6	difference	difference	NOUN
spir-13471	25	7	between	between	ADP
spir-13471	25	8	traditional	traditional	ADJ
spir-13471	25	9	coders	coder	NOUN
spir-13471	25	10	and	and	CCONJ
spir-13471	25	11	crowdworkers	crowdworker	NOUN
spir-13471	25	12	and	and	CCONJ
spir-13471	25	13	to	to	ADP
spir-13471	25	14	the	the	DET
spir-13471	25	15	influence	influence	NOUN
spir-13471	25	16	of	of	ADP
spir-13471	25	17	socio	socio	NOUN
spir-13471	25	18	-	-	ADJ
spir-13471	25	19	demographic	demographic	ADJ
spir-13471	25	20	characteristics	characteristic	NOUN
spir-13471	25	21	(	(	PUNCT
spir-13471	25	22	age	age	NOUN
spir-13471	25	23	,	,	PUNCT
spir-13471	25	24	gender	gender	NOUN
spir-13471	25	25	,	,	PUNCT
spir-13471	25	26	and	and	CCONJ
spir-13471	25	27	formal	formal	ADJ
spir-13471	25	28	education	education	NOUN
spir-13471	25	29	)	)	PUNCT
spir-13471	25	30	,	,	PUNCT
spir-13471	25	31	personality	personality	NOUN
spir-13471	25	32	traits	trait	NOUN
spir-13471	25	33	(	(	PUNCT
spir-13471	25	34	need	need	VERB
spir-13471	25	35	for	for	ADP
spir-13471	25	36	cognition	cognition	NOUN
spir-13471	25	37	and	and	CCONJ
spir-13471	25	38	emotional	emotional	ADJ
spir-13471	25	39	sensitivity	sensitivity	NOUN
spir-13471	25	40	)	)	PUNCT
spir-13471	25	41	,	,	PUNCT
spir-13471	25	42	and	and	CCONJ
spir-13471	25	43	(	(	PUNCT
spir-13471	25	44	3	3	X
spir-13471	25	45	)	)	PUNCT
spir-13471	25	46	coding	code	VERB
spir-13471	25	47	tasks	task	NOUN
spir-13471	25	48	of	of	ADP
spir-13471	25	49	varying	vary	VERB
spir-13471	25	50	difficulty	difficulty	NOUN
spir-13471	25	51	.	.	PUNCT
spir-13471	26	1	the	the	DET
spir-13471	26	2	results	result	NOUN
spir-13471	26	3	of	of	ADP
spir-13471	26	4	the	the	DET
spir-13471	26	5	study	study	NOUN
spir-13471	26	6	help	help	NOUN
spir-13471	26	7	to	to	PART
spir-13471	26	8	eliminate	eliminate	VERB
spir-13471	26	9	the	the	DET
spir-13471	26	10	influence	influence	NOUN
spir-13471	26	11	of	of	ADP
spir-13471	26	12	confounding	confound	VERB
spir-13471	26	13	factors	factor	NOUN
spir-13471	26	14	,	,	PUNCT
spir-13471	26	15	e.g.	e.g.	ADV
spir-13471	26	16	,	,	PUNCT
spir-13471	26	17	by	by	ADP
spir-13471	26	18	weighting	weight	VERB
spir-13471	26	19	or	or	CCONJ
spir-13471	26	20	selecting	select	VERB
spir-13471	26	21	coders	coder	NOUN
spir-13471	26	22	to	to	PART
spir-13471	26	23	minimize	minimize	VERB
spir-13471	26	24	systematic	systematic	ADJ
spir-13471	26	25	errors	error	NOUN
spir-13471	26	26	in	in	ADP
spir-13471	26	27	the	the	DET
spir-13471	26	28	future	future	NOUN
spir-13471	26	29	.	.	PUNCT
spir-13471	27	1	method	method	NOUN
spir-13471	27	2	to	to	PART
spir-13471	27	3	answer	answer	VERB
spir-13471	27	4	our	our	PRON
spir-13471	27	5	research	research	NOUN
spir-13471	27	6	questions	question	NOUN
spir-13471	27	7	,	,	PUNCT
spir-13471	27	8	we	we	PRON
spir-13471	27	9	employed	employ	VERB
spir-13471	27	10	a	a	DET
spir-13471	27	11	mixed	mixed	ADJ
spir-13471	27	12	method	method	NOUN
spir-13471	27	13	design	design	NOUN
spir-13471	27	14	combining	combine	VERB
spir-13471	27	15	content	content	NOUN
spir-13471	27	16	analysis	analysis	NOUN
spir-13471	27	17	and	and	CCONJ
spir-13471	27	18	survey	survey	NOUN
spir-13471	27	19	.	.	PUNCT
spir-13471	28	1	for	for	ADP
spir-13471	28	2	the	the	DET
spir-13471	28	3	content	content	NOUN
spir-13471	28	4	analysis	analysis	NOUN
spir-13471	28	5	we	we	PRON
spir-13471	28	6	picked	pick	VERB
spir-13471	28	7	a	a	DET
spir-13471	28	8	task	task	NOUN
spir-13471	28	9	that	that	PRON
spir-13471	28	10	is	be	AUX
spir-13471	28	11	common	common	ADJ
spir-13471	28	12	in	in	ADP
spir-13471	28	13	internet	internet	NOUN
spir-13471	28	14	research	research	NOUN
spir-13471	28	15	:	:	PUNCT
spir-13471	28	16	the	the	DET
spir-13471	28	17	analysis	analysis	NOUN
spir-13471	28	18	of	of	ADP
spir-13471	28	19	tweets	tweet	NOUN
spir-13471	28	20	.	.	PUNCT
spir-13471	29	1	the	the	DET
spir-13471	29	2	example	example	NOUN
spir-13471	29	3	topic	topic	NOUN
spir-13471	29	4	focused	focus	VERB
spir-13471	29	5	on	on	ADP
spir-13471	29	6	the	the	DET
spir-13471	29	7	current	current	ADJ
spir-13471	29	8	debate	debate	NOUN
spir-13471	29	9	about	about	ADP
spir-13471	29	10	the	the	DET
spir-13471	29	11	legalization	legalization	NOUN
spir-13471	29	12	of	of	ADP
spir-13471	29	13	abortion	abortion	NOUN
spir-13471	29	14	in	in	ADP
spir-13471	29	15	germany	germany	PROPN
spir-13471	29	16	.	.	PUNCT
spir-13471	30	1	in	in	ADP
spir-13471	30	2	addition	addition	NOUN
spir-13471	30	3	to	to	ADP
spir-13471	30	4	its	its	PRON
spir-13471	30	5	typicality	typicality	NOUN
spir-13471	30	6	,	,	PUNCT
spir-13471	30	7	relevance	relevance	NOUN
spir-13471	30	8	and	and	CCONJ
spir-13471	30	9	topicality	topicality	NOUN
spir-13471	30	10	,	,	PUNCT
spir-13471	30	11	this	this	DET
spir-13471	30	12	topic	topic	NOUN
spir-13471	30	13	has	have	VERB
spir-13471	30	14	further	further	ADJ
spir-13471	30	15	merits	merit	NOUN
spir-13471	30	16	:	:	PUNCT
spir-13471	30	17	it	it	PRON
spir-13471	30	18	is	be	AUX
spir-13471	30	19	controversial	controversial	ADJ
spir-13471	30	20	,	,	PUNCT
spir-13471	30	21	so	so	CCONJ
spir-13471	30	22	it	it	PRON
spir-13471	30	23	allows	allow	VERB
spir-13471	30	24	for	for	ADP
spir-13471	30	25	differing	differ	VERB
spir-13471	30	26	opinions	opinion	NOUN
spir-13471	30	27	,	,	PUNCT
spir-13471	30	28	and	and	CCONJ
spir-13471	30	29	it	it	PRON
spir-13471	30	30	is	be	AUX
spir-13471	30	31	complex	complex	ADJ
spir-13471	30	32	so	so	SCONJ
spir-13471	30	33	that	that	SCONJ
spir-13471	30	34	variables	variable	NOUN
spir-13471	30	35	with	with	ADP
spir-13471	30	36	varying	vary	VERB
spir-13471	30	37	degrees	degree	NOUN
spir-13471	30	38	of	of	ADP
spir-13471	30	39	complexity	complexity	NOUN
spir-13471	30	40	can	can	AUX
spir-13471	30	41	be	be	AUX
spir-13471	30	42	elicited	elicit	VERB
spir-13471	30	43	.	.	PUNCT
spir-13471	31	1	moreover	moreover	ADV
spir-13471	31	2	,	,	PUNCT
spir-13471	31	3	the	the	DET
spir-13471	31	4	analysis	analysis	NOUN
spir-13471	31	5	of	of	ADP
spir-13471	31	6	the	the	DET
spir-13471	31	7	rather	rather	ADV
spir-13471	31	8	short	short	ADJ
spir-13471	31	9	posts	post	NOUN
spir-13471	31	10	is	be	AUX
spir-13471	31	11	a	a	DET
spir-13471	31	12	task	task	NOUN
spir-13471	31	13	that	that	PRON
spir-13471	31	14	crowdworkers	crowdworker	NOUN
spir-13471	31	15	typically	typically	ADV
spir-13471	31	16	do	do	VERB
spir-13471	31	17	as	as	ADP
spir-13471	31	18	part	part	NOUN
spir-13471	31	19	of	of	ADP
spir-13471	31	20	their	their	PRON
spir-13471	31	21	job	job	NOUN
spir-13471	31	22	.	.	PUNCT
spir-13471	32	1	the	the	DET
spir-13471	32	2	content	content	NOUN
spir-13471	32	3	analysis	analysis	NOUN
spir-13471	32	4	sample	sample	NOUN
spir-13471	32	5	consists	consist	VERB
spir-13471	32	6	of	of	ADP
spir-13471	32	7	300	300	NUM
spir-13471	32	8	tweets	tweet	NOUN
spir-13471	32	9	that	that	PRON
spir-13471	32	10	contain	contain	VERB
spir-13471	32	11	the	the	DET
spir-13471	32	12	term	term	NOUN
spir-13471	32	13	"	"	PUNCT
spir-13471	32	14	abortion	abortion	NOUN
spir-13471	32	15	"	"	PUNCT
spir-13471	32	16	and	and	CCONJ
spir-13471	32	17	actually	actually	ADV
spir-13471	32	18	relate	relate	VERB
spir-13471	32	19	to	to	ADP
spir-13471	32	20	the	the	DET
spir-13471	32	21	topic	topic	NOUN
spir-13471	32	22	.	.	PUNCT
spir-13471	33	1	the	the	DET
spir-13471	33	2	(	(	PUNCT
spir-13471	33	3	very	very	ADV
spir-13471	33	4	short	short	ADJ
spir-13471	33	5	)	)	PUNCT
spir-13471	33	6	codebook	codebook	NOUN
spir-13471	33	7	includes	include	VERB
spir-13471	33	8	variables	variable	NOUN
spir-13471	33	9	that	that	PRON
spir-13471	33	10	differ	differ	VERB
spir-13471	33	11	in	in	ADP
spir-13471	33	12	terms	term	NOUN
spir-13471	33	13	of	of	ADP
spir-13471	33	14	the	the	DET
spir-13471	33	15	interpretive	interpretive	ADJ
spir-13471	33	16	effort	effort	NOUN
spir-13471	33	17	to	to	PART
spir-13471	33	18	be	be	AUX
spir-13471	33	19	made	make	VERB
spir-13471	33	20	:	:	PUNCT
spir-13471	33	21	(	(	PUNCT
spir-13471	33	22	1	1	X
spir-13471	33	23	)	)	PUNCT
spir-13471	33	24	positioning	positioning	NOUN
spir-13471	33	25	(	(	PUNCT
spir-13471	33	26	pro	pro	ADJ
spir-13471	33	27	-	-	ADJ
spir-13471	33	28	abortion	abortion	ADJ
spir-13471	33	29	vs.	vs.	ADP
spir-13471	33	30	anti	anti	ADJ
spir-13471	33	31	-	-	ADJ
spir-13471	33	32	abortion	abortion	ADJ
spir-13471	33	33	or	or	CCONJ
spir-13471	33	34	neutral	neutral	ADJ
spir-13471	33	35	)	)	PUNCT
spir-13471	33	36	,	,	PUNCT
spir-13471	33	37	(	(	PUNCT
spir-13471	33	38	2	2	X
spir-13471	33	39	)	)	PUNCT
spir-13471	33	40	emotional	emotional	ADJ
spir-13471	33	41	valence	valence	NOUN
spir-13471	33	42	,	,	PUNCT
spir-13471	33	43	(	(	PUNCT
spir-13471	33	44	3	3	X
spir-13471	33	45	)	)	PUNCT
spir-13471	33	46	references	reference	NOUN
spir-13471	33	47	to	to	ADP
spir-13471	33	48	the	the	DET
spir-13471	33	49	current	current	ADJ
spir-13471	33	50	legal	legal	ADJ
spir-13471	33	51	situation	situation	NOUN
spir-13471	33	52	,	,	PUNCT
spir-13471	33	53	and	and	CCONJ
spir-13471	33	54	finally	finally	ADV
spir-13471	33	55	,	,	PUNCT
spir-13471	33	56	as	as	ADP
spir-13471	33	57	more	more	ADV
spir-13471	33	58	formal	formal	ADJ
spir-13471	33	59	aspects	aspect	NOUN
spir-13471	33	60	of	of	ADP
spir-13471	33	61	the	the	DET
spir-13471	33	62	texts	text	NOUN
spir-13471	33	63	,	,	PUNCT
spir-13471	33	64	the	the	DET
spir-13471	33	65	number	number	NOUN
spir-13471	33	66	of	of	ADP
spir-13471	33	67	(	(	PUNCT
spir-13471	33	68	4	4	NUM
spir-13471	33	69	)	)	PUNCT
spir-13471	33	70	question	question	NOUN
spir-13471	33	71	marks	mark	NOUN
spir-13471	33	72	and	and	CCONJ
spir-13471	33	73	(	(	PUNCT
spir-13471	33	74	5	5	NUM
spir-13471	33	75	)	)	PUNCT
spir-13471	33	76	exclamation	exclamation	NOUN
spir-13471	33	77	marks	mark	NOUN
spir-13471	33	78	.	.	PUNCT
spir-13471	34	1	the	the	DET
spir-13471	34	2	codebook	codebook	NOUN
spir-13471	34	3	was	be	AUX
spir-13471	34	4	pretested	preteste	VERB
spir-13471	34	5	and	and	CCONJ
spir-13471	34	6	optimized	optimize	VERB
spir-13471	34	7	in	in	ADP
spir-13471	34	8	terms	term	NOUN
spir-13471	34	9	of	of	ADP
spir-13471	34	10	wording	wording	NOUN
spir-13471	34	11	.	.	PUNCT
spir-13471	35	1	it	it	PRON
spir-13471	35	2	was	be	AUX
spir-13471	35	3	intentionally	intentionally	ADV
spir-13471	35	4	designed	design	VERB
spir-13471	35	5	to	to	PART
spir-13471	35	6	be	be	AUX
spir-13471	35	7	challenging	challenge	VERB
spir-13471	35	8	in	in	ADP
spir-13471	35	9	order	order	NOUN
spir-13471	35	10	to	to	PART
spir-13471	35	11	generate	generate	VERB
spir-13471	35	12	variance	variance	NOUN
spir-13471	35	13	in	in	ADP
spir-13471	35	14	the	the	DET
spir-13471	35	15	coding	coding	NOUN
spir-13471	35	16	.	.	PUNCT
spir-13471	36	1	initially	initially	ADV
spir-13471	36	2	,	,	PUNCT
spir-13471	36	3	all	all	DET
spir-13471	36	4	tweets	tweet	NOUN
spir-13471	36	5	were	be	AUX
spir-13471	36	6	coded	code	VERB
spir-13471	36	7	by	by	ADP
spir-13471	36	8	two	two	NUM
spir-13471	36	9	researchers	researcher	NOUN
spir-13471	36	10	who	who	PRON
spir-13471	36	11	discussed	discuss	VERB
spir-13471	36	12	and	and	CCONJ
spir-13471	36	13	reconciled	reconcile	VERB
spir-13471	36	14	their	their	PRON
spir-13471	36	15	coding	code	VERB
spir-13471	36	16	decisions	decision	NOUN
spir-13471	36	17	.	.	PUNCT
spir-13471	37	1	this	this	DET
spir-13471	37	2	reference	reference	NOUN
spir-13471	37	3	data	datum	NOUN
spir-13471	37	4	served	serve	VERB
spir-13471	37	5	as	as	ADP
spir-13471	37	6	the	the	DET
spir-13471	37	7	gold	gold	NOUN
spir-13471	37	8	standard	standard	NOUN
spir-13471	37	9	.	.	PUNCT
spir-13471	38	1	data	datum	NOUN
spir-13471	38	2	was	be	AUX
spir-13471	38	3	collected	collect	VERB
spir-13471	38	4	using	use	VERB
spir-13471	38	5	an	an	DET
spir-13471	38	6	online	online	ADJ
spir-13471	38	7	questionnaire	questionnaire	NOUN
spir-13471	38	8	as	as	ADP
spir-13471	38	9	an	an	DET
spir-13471	38	10	input	input	NOUN
spir-13471	38	11	mask	mask	NOUN
spir-13471	38	12	.	.	PUNCT
spir-13471	39	1	the	the	DET
spir-13471	39	2	survey	survey	NOUN
spir-13471	39	3	contained	contain	VERB
spir-13471	39	4	questions	question	NOUN
spir-13471	39	5	on	on	ADP
spir-13471	39	6	coder	coder	NOUN
spir-13471	39	7	characteristics	characteristic	NOUN
spir-13471	39	8	,	,	PUNCT
spir-13471	39	9	short	short	ADJ
spir-13471	39	10	briefings	briefing	NOUN
spir-13471	39	11	,	,	PUNCT
spir-13471	39	12	the	the	DET
spir-13471	39	13	codebook	codebook	NOUN
spir-13471	39	14	,	,	PUNCT
spir-13471	39	15	and	and	CCONJ
spir-13471	39	16	the	the	DET
spir-13471	39	17	tweets	tweet	NOUN
spir-13471	39	18	.	.	PUNCT
spir-13471	40	1	the	the	DET
spir-13471	40	2	comparative	comparative	ADJ
spir-13471	40	3	study	study	NOUN
spir-13471	40	4	design	design	NOUN
spir-13471	40	5	involved	involve	VERB
spir-13471	40	6	conducting	conduct	VERB
spir-13471	40	7	the	the	DET
spir-13471	40	8	same	same	ADJ
spir-13471	40	9	content	content	NOUN
spir-13471	40	10	analysis	analysis	NOUN
spir-13471	40	11	twice	twice	ADV
spir-13471	40	12	:	:	PUNCT
spir-13471	40	13	in	in	ADP
spir-13471	40	14	the	the	DET
spir-13471	40	15	first	first	ADJ
spir-13471	40	16	condition	condition	NOUN
spir-13471	40	17	,	,	PUNCT
spir-13471	40	18	four	four	NUM
spir-13471	40	19	students	student	NOUN
spir-13471	40	20	who	who	PRON
spir-13471	40	21	received	receive	VERB
spir-13471	40	22	a	a	DET
spir-13471	40	23	training	training	NOUN
spir-13471	40	24	coded	code	VERB
spir-13471	40	25	the	the	DET
spir-13471	40	26	whole	whole	ADJ
spir-13471	40	27	sample	sample	NOUN
spir-13471	40	28	of	of	ADP
spir-13471	40	29	300	300	NUM
spir-13471	40	30	tweets	tweet	NOUN
spir-13471	40	31	.	.	PUNCT
spir-13471	41	1	in	in	ADP
spir-13471	41	2	the	the	DET
spir-13471	41	3	second	second	ADJ
spir-13471	41	4	condition	condition	NOUN
spir-13471	41	5	,	,	PUNCT
spir-13471	41	6	150	150	NUM
spir-13471	41	7	crowdworkers	crowdworker	NOUN
spir-13471	41	8	were	be	AUX
spir-13471	41	9	recruited	recruit	VERB
spir-13471	41	10	via	via	ADP
spir-13471	41	11	the	the	DET
spir-13471	41	12	platform	platform	NOUN
spir-13471	41	13	clickworker	clickworker	NOUN
spir-13471	41	14	.	.	PUNCT
spir-13471	42	1	they	they	PRON
spir-13471	42	2	coded	code	VERB
spir-13471	42	3	30	30	NUM
spir-13471	42	4	tweets	tweet	NOUN
spir-13471	42	5	each	each	PRON
spir-13471	42	6	without	without	ADP
spir-13471	42	7	a	a	DET
spir-13471	42	8	training	training	NOUN
spir-13471	42	9	.	.	PUNCT
spir-13471	43	1	depending	depend	VERB
spir-13471	43	2	on	on	ADP
spir-13471	43	3	their	their	PRON
spir-13471	43	4	coding	coding	NOUN
spir-13471	43	5	,	,	PUNCT
spir-13471	43	6	the	the	DET
spir-13471	43	7	all	all	DET
spir-13471	43	8	coders	coder	NOUN
spir-13471	43	9	received	receive	VERB
spir-13471	43	10	feedback	feedback	NOUN
spir-13471	43	11	every	every	DET
spir-13471	43	12	fifth	fifth	ADJ
spir-13471	43	13	tweet	tweet	NOUN
spir-13471	43	14	to	to	PART
spir-13471	43	15	ensure	ensure	VERB
spir-13471	43	16	higher	high	ADJ
spir-13471	43	17	quality	quality	NOUN
spir-13471	43	18	of	of	ADP
spir-13471	43	19	coding	code	VERB
spir-13471	43	20	decisions	decision	NOUN
spir-13471	43	21	.	.	PUNCT
spir-13471	44	1	in	in	ADP
spir-13471	44	2	total	total	NOUN
spir-13471	44	3	,	,	PUNCT
spir-13471	44	4	the	the	DET
spir-13471	44	5	study	study	NOUN
spir-13471	44	6	includes	include	VERB
spir-13471	44	7	28,500	28,500	NUM
spir-13471	44	8	coding	code	VERB
spir-13471	44	9	decisions	decision	NOUN
spir-13471	44	10	.	.	PUNCT
spir-13471	45	1	all	all	DET
spir-13471	45	2	coders	coder	NOUN
spir-13471	45	3	ere	ere	PROPN
spir-13471	45	4	adequatly	adequatly	ADV
spir-13471	45	5	compensated	compensate	VERB
spir-13471	45	6	,	,	PUNCT
spir-13471	45	7	either	either	CCONJ
spir-13471	45	8	in	in	ADP
spir-13471	45	9	form	form	NOUN
spir-13471	45	10	of	of	ADP
spir-13471	45	11	credit	credit	NOUN
spir-13471	45	12	points	point	NOUN
spir-13471	45	13	(	(	PUNCT
spir-13471	45	14	student	student	NOUN
spir-13471	45	15	coders	coder	NOUN
spir-13471	45	16	)	)	PUNCT
spir-13471	45	17	or	or	CCONJ
spir-13471	45	18	with	with	ADP
spir-13471	45	19	a	a	DET
spir-13471	45	20	monetary	monetary	ADJ
spir-13471	45	21	incentive	incentive	NOUN
spir-13471	45	22	(	(	PUNCT
spir-13471	45	23	crowdworker	crowdworker	NOUN
spir-13471	45	24	)	)	PUNCT
spir-13471	45	25	.	.	PUNCT
spir-13471	46	1	results	result	VERB
spir-13471	46	2	our	our	PRON
spir-13471	46	3	data	data	NOUN
spir-13471	46	4	analysis	analysis	NOUN
spir-13471	46	5	includes	include	VERB
spir-13471	46	6	systematic	systematic	ADJ
spir-13471	46	7	comparison	comparison	NOUN
spir-13471	46	8	of	of	ADP
spir-13471	46	9	established	establish	VERB
spir-13471	46	10	reliability	reliability	NOUN
spir-13471	46	11	coefficients	coefficient	NOUN
spir-13471	46	12	(	(	PUNCT
spir-13471	46	13	inter	inter	NOUN
spir-13471	46	14	coder	coder	NOUN
spir-13471	46	15	agreement	agreement	NOUN
spir-13471	46	16	and	and	CCONJ
spir-13471	46	17	krippendorff	krippendorff	NOUN
spir-13471	46	18	's	's	PART
spir-13471	46	19	alpha	alpha	NOUN
spir-13471	46	20	)	)	PUNCT
spir-13471	46	21	to	to	PART
spir-13471	46	22	obtain	obtain	VERB
spir-13471	46	23	a	a	DET
spir-13471	46	24	comprehensive	comprehensive	ADJ
spir-13471	46	25	picture	picture	NOUN
spir-13471	46	26	of	of	ADP
spir-13471	46	27	the	the	DET
spir-13471	46	28	quality	quality	NOUN
spir-13471	46	29	of	of	ADP
spir-13471	46	30	coding	code	VERB
spir-13471	46	31	in	in	ADP
spir-13471	46	32	the	the	DET
spir-13471	46	33	different	different	ADJ
spir-13471	46	34	conditions	condition	NOUN
spir-13471	46	35	.	.	PUNCT
spir-13471	47	1	in	in	ADP
spir-13471	47	2	addition	addition	NOUN
spir-13471	47	3	,	,	PUNCT
spir-13471	47	4	we	we	PRON
spir-13471	47	5	matched	match	VERB
spir-13471	47	6	the	the	DET
spir-13471	47	7	coding	coding	NOUN
spir-13471	47	8	with	with	ADP
spir-13471	47	9	the	the	DET
spir-13471	47	10	gold	gold	ADJ
spir-13471	47	11	standard	standard	NOUN
spir-13471	47	12	.	.	PUNCT
spir-13471	48	1	further	far	ADV
spir-13471	48	2	,	,	PUNCT
spir-13471	48	3	we	we	PRON
spir-13471	48	4	tested	test	VERB
spir-13471	48	5	the	the	DET
spir-13471	48	6	influence	influence	NOUN
spir-13471	48	7	of	of	ADP
spir-13471	48	8	the	the	DET
spir-13471	48	9	coders	coder	NOUN
spir-13471	48	10	'	'	PART
spir-13471	48	11	characteristics	characteristic	NOUN
spir-13471	48	12	on	on	ADP
spir-13471	48	13	the	the	DET
spir-13471	48	14	number	number	NOUN
spir-13471	48	15	of	of	ADP
spir-13471	48	16	correct	correct	ADJ
spir-13471	48	17	coding	code	VERB
spir-13471	48	18	decisions	decision	NOUN
spir-13471	48	19	by	by	ADP
spir-13471	48	20	regression	regression	NOUN
spir-13471	48	21	analysis	analysis	NOUN
spir-13471	48	22	.	.	PUNCT
spir-13471	49	1	notably	notably	ADV
spir-13471	49	2	,	,	PUNCT
spir-13471	49	3	the	the	DET
spir-13471	49	4	data	data	NOUN
spir-13471	49	5	comprises	comprise	VERB
spir-13471	49	6	a	a	DET
spir-13471	49	7	complex	complex	ADJ
spir-13471	49	8	dependency	dependency	NOUN
spir-13471	49	9	structure	structure	NOUN
spir-13471	49	10	.	.	PUNCT
spir-13471	50	1	the	the	DET
spir-13471	50	2	codings	coding	NOUN
spir-13471	50	3	are	be	AUX
spir-13471	50	4	nested	nest	VERB
spir-13471	50	5	in	in	ADP
spir-13471	50	6	both	both	PRON
spir-13471	50	7	:	:	PUNCT
spir-13471	50	8	coders	coder	NOUN
spir-13471	50	9	as	as	ADV
spir-13471	50	10	well	well	ADV
spir-13471	50	11	as	as	ADP
spir-13471	50	12	tweets	tweet	NOUN
spir-13471	50	13	.	.	PUNCT
spir-13471	51	1	hence	hence	ADV
spir-13471	51	2	,	,	PUNCT
spir-13471	51	3	multi	multi	ADJ
spir-13471	51	4	-	-	ADJ
spir-13471	51	5	level	level	ADJ
spir-13471	51	6	analysis	analysis	NOUN
spir-13471	51	7	was	be	AUX
spir-13471	51	8	applied	apply	VERB
spir-13471	51	9	.	.	PUNCT
spir-13471	52	1	descriptive	descriptive	ADJ
spir-13471	52	2	findings	finding	NOUN
spir-13471	52	3	imply	imply	VERB
spir-13471	52	4	that	that	SCONJ
spir-13471	52	5	student	student	NOUN
spir-13471	52	6	coders	coder	NOUN
spir-13471	52	7	and	and	CCONJ
spir-13471	52	8	crowdworkers	crowdworker	NOUN
spir-13471	52	9	differ	differ	VERB
spir-13471	52	10	regarding	regard	VERB
spir-13471	52	11	validity	validity	NOUN
spir-13471	52	12	and	and	CCONJ
spir-13471	52	13	reliability	reliability	NOUN
spir-13471	52	14	.	.	PUNCT
spir-13471	53	1	student	student	NOUN
spir-13471	53	2	coders	coder	NOUN
spir-13471	53	3	performed	perform	VERB
spir-13471	53	4	better	well	ADV
spir-13471	53	5	when	when	SCONJ
spir-13471	53	6	compared	compare	VERB
spir-13471	53	7	to	to	ADP
spir-13471	53	8	the	the	DET
spir-13471	53	9	gold	gold	NOUN
spir-13471	53	10	standard	standard	NOUN
spir-13471	53	11	,	,	PUNCT
spir-13471	53	12	especially	especially	ADV
spir-13471	53	13	for	for	ADP
spir-13471	53	14	the	the	DET
spir-13471	53	15	pro	pro	ADJ
spir-13471	53	16	-	-	ADJ
spir-13471	53	17	abortion	abortion	ADJ
spir-13471	53	18	vs.	vs.	ADP
spir-13471	53	19	anti	anti	ADJ
spir-13471	53	20	-	-	ADJ
spir-13471	53	21	abortion	abortion	ADJ
spir-13471	53	22	categorization	categorization	NOUN
spir-13471	53	23	(	(	PUNCT
spir-13471	53	24	83	83	NUM
spir-13471	53	25	vs.	vs.	CCONJ
spir-13471	53	26	74	74	NUM
spir-13471	53	27	percent	percent	NOUN
spir-13471	53	28	compliance	compliance	NOUN
spir-13471	53	29	)	)	PUNCT
spir-13471	53	30	.	.	PUNCT
spir-13471	54	1	overall	overall	ADJ
spir-13471	54	2	,	,	PUNCT
spir-13471	54	3	however	however	ADV
spir-13471	54	4	,	,	PUNCT
spir-13471	54	5	coding	code	VERB
spir-13471	54	6	was	be	AUX
spir-13471	54	7	found	find	VERB
spir-13471	54	8	to	to	PART
spir-13471	54	9	be	be	AUX
spir-13471	54	10	difficult	difficult	ADJ
spir-13471	54	11	regarding	regard	VERB
spir-13471	54	12	validity	validity	NOUN
spir-13471	54	13	,	,	PUNCT
spir-13471	54	14	as	as	SCONJ
spir-13471	54	15	agreement	agreement	NOUN
spir-13471	54	16	with	with	ADP
spir-13471	54	17	the	the	DET
spir-13471	54	18	gold	gold	NOUN
spir-13471	54	19	standard	standard	NOUN
spir-13471	54	20	is	be	AUX
spir-13471	54	21	rather	rather	ADV
spir-13471	54	22	low	low	ADJ
spir-13471	54	23	in	in	ADP
spir-13471	54	24	both	both	DET
spir-13471	54	25	groups	group	NOUN
spir-13471	54	26	(	(	PUNCT
spir-13471	54	27	average	average	ADJ
spir-13471	54	28	62	62	NUM
spir-13471	54	29	percent	percent	NOUN
spir-13471	54	30	)	)	PUNCT
spir-13471	54	31	.	.	PUNCT
spir-13471	55	1	in	in	ADP
spir-13471	55	2	contrast	contrast	NOUN
spir-13471	55	3	,	,	PUNCT
spir-13471	55	4	reliability	reliability	NOUN
spir-13471	55	5	is	be	AUX
spir-13471	55	6	generally	generally	ADV
spir-13471	55	7	higher	high	ADJ
spir-13471	55	8	(	(	PUNCT
spir-13471	55	9	average	average	ADJ
spir-13471	55	10	inter	inter	ADJ
spir-13471	55	11	coder	coder	PROPN
spir-13471	55	12	agreement	agreement	NOUN
spir-13471	55	13	:	:	PUNCT
spir-13471	55	14	.78	.78	NUM
spir-13471	55	15	;	;	PUNCT
spir-13471	55	16	average	average	ADJ
spir-13471	55	17	krippendoff	krippendoff	NOUN
spir-13471	55	18	's	's	PART
spir-13471	55	19	alpha	alpha	NOUN
spir-13471	55	20	=	=	NOUN
spir-13471	55	21	.73	.73	NUM
spir-13471	55	22	)	)	PUNCT
spir-13471	55	23	.	.	PUNCT
spir-13471	56	1	notably	notably	ADV
spir-13471	56	2	,	,	PUNCT
spir-13471	56	3	for	for	ADP
spir-13471	56	4	the	the	DET
spir-13471	56	5	inter	inter	ADJ
spir-13471	56	6	coder	coder	NOUN
spir-13471	56	7	agreement	agreement	NOUN
spir-13471	56	8	and	and	CCONJ
spir-13471	56	9	even	even	ADV
spir-13471	56	10	more	more	ADJ
spir-13471	56	11	for	for	ADP
spir-13471	56	12	krippendorff	krippendorff	NOUN
spir-13471	56	13	's	's	PART
spir-13471	56	14	alpha	alpha	NOUN
spir-13471	56	15	,	,	PUNCT
spir-13471	56	16	student	student	NOUN
spir-13471	56	17	coders	coder	NOUN
spir-13471	56	18	outperform	outperform	VERB
spir-13471	56	19	crowdworkers	crowdworker	NOUN
spir-13471	56	20	(	(	PUNCT
spir-13471	56	21	krippendoff	krippendoff	PROPN
spir-13471	56	22	's	's	PART
spir-13471	56	23	alphastudents	alphastudent	NOUN
spir-13471	56	24	=	=	SYM
spir-13471	57	1	.83	.83	NUM
spir-13471	57	2	vs.	vs.	X
spir-13471	57	3	krippendoff	krippendoff	PROPN
spir-13471	57	4	's	's	PART
spir-13471	57	5	alphacrowdworker	alphacrowdworker	NOUN
spir-13471	57	6	=	=	SYM
spir-13471	57	7	.64	.64	NUM
spir-13471	57	8	)	)	PUNCT
spir-13471	57	9	.	.	PUNCT
spir-13471	58	1	in	in	ADP
spir-13471	58	2	conclusion	conclusion	NOUN
spir-13471	58	3	,	,	PUNCT
spir-13471	58	4	the	the	DET
spir-13471	58	5	results	result	NOUN
spir-13471	58	6	advocate	advocate	VERB
spir-13471	58	7	the	the	DET
spir-13471	58	8	use	use	NOUN
spir-13471	58	9	of	of	ADP
spir-13471	58	10	traditional	traditional	ADJ
spir-13471	58	11	coders	coder	NOUN
spir-13471	58	12	for	for	ADP
spir-13471	58	13	greater	great	ADJ
spir-13471	58	14	control	control	NOUN
spir-13471	58	15	of	of	ADP
spir-13471	58	16	the	the	DET
spir-13471	58	17	coding	code	VERB
spir-13471	58	18	process	process	NOUN
spir-13471	58	19	as	as	ADV
spir-13471	58	20	well	well	ADV
spir-13471	58	21	as	as	ADP
spir-13471	58	22	intensive	intensive	ADJ
spir-13471	58	23	coding	code	VERB
spir-13471	58	24	training	training	NOUN
spir-13471	58	25	.	.	PUNCT
spir-13471	59	1	moreover	moreover	ADV
spir-13471	59	2	,	,	PUNCT
spir-13471	59	3	multivariate	multivariate	NOUN
spir-13471	59	4	regression	regression	NOUN
spir-13471	59	5	analysis	analysis	NOUN
spir-13471	59	6	indicate	indicate	VERB
spir-13471	59	7	that	that	SCONJ
spir-13471	59	8	the	the	DET
spir-13471	59	9	influence	influence	NOUN
spir-13471	59	10	of	of	ADP
spir-13471	59	11	the	the	DET
spir-13471	59	12	categories	category	NOUN
spir-13471	59	13	or	or	CCONJ
spir-13471	59	14	variable	variable	ADJ
spir-13471	59	15	types	type	NOUN
spir-13471	59	16	of	of	ADP
spir-13471	59	17	varying	vary	VERB
spir-13471	59	18	difficulty	difficulty	NOUN
spir-13471	59	19	is	be	AUX
spir-13471	59	20	much	much	ADV
spir-13471	59	21	stronger	strong	ADJ
spir-13471	59	22	than	than	ADP
spir-13471	59	23	the	the	DET
spir-13471	59	24	influence	influence	NOUN
spir-13471	59	25	of	of	ADP
spir-13471	59	26	the	the	DET
spir-13471	59	27	coder	coder	PROPN
spir-13471	59	28	characteristics	characteristic	NOUN
spir-13471	59	29	.	.	PUNCT
spir-13471	60	1	this	this	PRON
spir-13471	60	2	applies	apply	VERB
spir-13471	60	3	both	both	PRON
spir-13471	60	4	to	to	PART
spir-13471	60	5	agreement	agreement	VERB
spir-13471	60	6	with	with	ADP
spir-13471	60	7	the	the	DET
spir-13471	60	8	gold	gold	NOUN
spir-13471	60	9	standard	standard	NOUN
spir-13471	60	10	and	and	CCONJ
spir-13471	60	11	to	to	ADP
spir-13471	60	12	inter	inter	ADJ
spir-13471	60	13	-	-	ADJ
spir-13471	60	14	coder	coder	ADJ
spir-13471	60	15	agreement	agreement	NOUN
spir-13471	60	16	.	.	PUNCT
spir-13471	61	1	this	this	DET
spir-13471	61	2	result	result	NOUN
spir-13471	61	3	highlights	highlight	VERB
spir-13471	61	4	the	the	DET
spir-13471	61	5	importance	importance	NOUN
spir-13471	61	6	of	of	ADP
spir-13471	61	7	conducting	conduct	VERB
spir-13471	61	8	simple	simple	ADJ
spir-13471	61	9	coding	coding	NOUN
spir-13471	61	10	tasks	task	NOUN
spir-13471	61	11	,	,	PUNCT
spir-13471	61	12	formulating	formulate	VERB
spir-13471	61	13	categories	category	NOUN
spir-13471	61	14	precisely	precisely	ADV
spir-13471	61	15	and	and	CCONJ
spir-13471	61	16	choosing	choose	VERB
spir-13471	61	17	concise	concise	ADJ
spir-13471	61	18	operationalizations	operationalization	NOUN
spir-13471	61	19	—	—	PUNCT
spir-13471	61	20	especially	especially	ADV
spir-13471	61	21	when	when	SCONJ
spir-13471	61	22	working	work	VERB
spir-13471	61	23	with	with	ADP
spir-13471	61	24	crowdworkers	crowdworker	NOUN
spir-13471	61	25	.	.	PUNCT
spir-13471	62	1	nevertheless	nevertheless	ADV
spir-13471	62	2	,	,	PUNCT
spir-13471	62	3	certain	certain	ADJ
spir-13471	62	4	coder	coder	NOUN
spir-13471	62	5	characteristics	characteristic	NOUN
spir-13471	62	6	also	also	ADV
spir-13471	62	7	positively	positively	ADV
spir-13471	62	8	determine	determine	VERB
spir-13471	62	9	coding	code	VERB
spir-13471	62	10	results	result	NOUN
spir-13471	62	11	,	,	PUNCT
spir-13471	62	12	this	this	PRON
spir-13471	62	13	should	should	AUX
spir-13471	62	14	be	be	AUX
spir-13471	62	15	considered	consider	VERB
spir-13471	62	16	in	in	ADP
spir-13471	62	17	the	the	DET
spir-13471	62	18	recruitment	recruitment	NOUN
spir-13471	62	19	of	of	ADP
spir-13471	62	20	coders	coder	NOUN
spir-13471	62	21	.	.	PUNCT
spir-13471	63	1	individuals	individual	NOUN
spir-13471	63	2	with	with	ADP
spir-13471	63	3	a	a	DET
spir-13471	63	4	high	high	ADJ
spir-13471	63	5	level	level	NOUN
spir-13471	63	6	of	of	ADP
spir-13471	63	7	formal	formal	ADJ
spir-13471	63	8	education	education	NOUN
spir-13471	63	9	and	and	CCONJ
spir-13471	63	10	emotional	emotional	ADJ
spir-13471	63	11	sensitivity	sensitivity	NOUN
spir-13471	63	12	are	be	AUX
spir-13471	63	13	to	to	PART
spir-13471	63	14	be	be	AUX
spir-13471	63	15	preferred	prefer	VERB
spir-13471	63	16	.	.	PUNCT
spir-13471	64	1	further	further	ADJ
spir-13471	64	2	implications	implication	NOUN
spir-13471	64	3	and	and	CCONJ
spir-13471	64	4	recommendations	recommendation	NOUN
spir-13471	64	5	were	be	AUX
spir-13471	64	6	discussed	discuss	VERB
spir-13471	64	7	at	at	ADP
spir-13471	64	8	the	the	DET
spir-13471	64	9	conference	conference	NOUN
spir-13471	64	10	.	.	PUNCT
spir-13471	65	1	references	reference	NOUN
spir-13471	65	2	boxman	boxman	PROPN
spir-13471	65	3	-	-	PUNCT
spir-13471	65	4	shabtai	shabtai	PROPN
spir-13471	65	5	,	,	PUNCT
spir-13471	65	6	l.	l.	PROPN
spir-13471	65	7	(	(	PUNCT
spir-13471	65	8	2021	2021	NUM
spir-13471	65	9	)	)	PUNCT
spir-13471	65	10	.	.	PUNCT
spir-13471	66	1	encoding	encode	VERB
spir-13471	66	2	polysemy	polysemy	NOUN
spir-13471	66	3	in	in	ADP
spir-13471	66	4	the	the	DET
spir-13471	66	5	news	news	NOUN
spir-13471	66	6	.	.	PUNCT
spir-13471	67	1	journalism	journalism	NOUN
spir-13471	67	2	,	,	PUNCT
spir-13471	67	3	14648849211045964	14648849211045964	NUM
spir-13471	67	4	.	.	PUNCT
spir-13471	68	1	https://doi.org/10.1177/14648849211045963	https://doi.org/10.1177/14648849211045963	PROPN
spir-13471	69	1	budak	budak	PROPN
spir-13471	69	2	,	,	PUNCT
spir-13471	69	3	c.	c.	PROPN
spir-13471	69	4	,	,	PUNCT
spir-13471	69	5	goel	goel	PROPN
spir-13471	69	6	,	,	PUNCT
spir-13471	69	7	s.	s.	PROPN
spir-13471	69	8	,	,	PUNCT
spir-13471	69	9	&	&	CCONJ
spir-13471	69	10	rao	rao	PROPN
spir-13471	69	11	,	,	PUNCT
spir-13471	69	12	j.	j.	PROPN
spir-13471	69	13	m.	m.	PROPN
spir-13471	69	14	(	(	PUNCT
spir-13471	69	15	2016	2016	NUM
spir-13471	69	16	)	)	PUNCT
spir-13471	69	17	.	.	PUNCT
spir-13471	70	1	fair	fair	ADJ
spir-13471	70	2	and	and	CCONJ
spir-13471	70	3	balanced	balanced	ADJ
spir-13471	70	4	?	?	PUNCT
spir-13471	71	1	quantifying	quantify	VERB
spir-13471	71	2	media	medium	NOUN
spir-13471	71	3	bias	bias	NOUN
spir-13471	71	4	through	through	ADP
spir-13471	71	5	crowdsourced	crowdsourced	ADJ
spir-13471	71	6	content	content	NOUN
spir-13471	71	7	analysis	analysis	NOUN
spir-13471	71	8	.	.	PUNCT
spir-13471	72	1	public	public	ADJ
spir-13471	72	2	opinion	opinion	NOUN
spir-13471	72	3	quarterly	quarterly	ADV
spir-13471	72	4	,	,	PUNCT
spir-13471	72	5	80(1	80(1	NUM
spir-13471	72	6	)	)	PUNCT
spir-13471	72	7	,	,	PUNCT
spir-13471	72	8	250	250	NUM
spir-13471	72	9	-	-	SYM
spir-13471	72	10	271	271	NUM
spir-13471	72	11	.	.	PUNCT
spir-13471	73	1	https://doi.org/10.1093/poq/nfw007	https://doi.org/10.1093/poq/nfw007	PROPN
spir-13471	73	2	degen	degen	NOUN
spir-13471	73	3	,	,	PUNCT
spir-13471	73	4	m.	m.	NOUN
spir-13471	73	5	(	(	PUNCT
spir-13471	73	6	2015	2015	NUM
spir-13471	73	7	)	)	PUNCT
spir-13471	73	8	.	.	PUNCT
spir-13471	74	1	codierer	codierer	NOUN
spir-13471	74	2	-	-	PUNCT
spir-13471	74	3	effekte	effekte	ADJ
spir-13471	74	4	in	in	ADP
spir-13471	74	5	inhaltsanalysen	inhaltsanalysen	NOUN
spir-13471	74	6	–	–	PUNCT
spir-13471	74	7	ein	ein	PROPN
spir-13471	74	8	vernachlässigtes	vernachlässigte	NOUN
spir-13471	74	9	forschungsfeld	forschungsfeld	VERB
spir-13471	74	10	[	[	X
spir-13471	74	11	coder	coder	X
spir-13471	74	12	effects	effect	NOUN
spir-13471	74	13	in	in	ADP
spir-13471	74	14	content	content	NOUN
spir-13471	74	15	analyses	analysis	NOUN
spir-13471	74	16	---	---	PUNCT
spir-13471	74	17	a	a	DET
spir-13471	74	18	neglected	neglect	VERB
spir-13471	74	19	field	field	NOUN
spir-13471	74	20	of	of	ADP
spir-13471	74	21	research	research	NOUN
spir-13471	74	22	]	]	PUNCT
spir-13471	74	23	.	.	PUNCT
spir-13471	75	1	in	in	ADP
spir-13471	75	2	w.	w.	PROPN
spir-13471	75	3	wirth	wirth	PROPN
spir-13471	75	4	,	,	PUNCT
spir-13471	75	5	k.	k.	PROPN
spir-13471	75	6	sommer	sommer	PROPN
spir-13471	75	7	,	,	PUNCT
spir-13471	75	8	m.	m.	NOUN
spir-13471	75	9	wettstein	wettstein	PROPN
spir-13471	75	10	&	&	CCONJ
spir-13471	75	11	j.	j.	PROPN
spir-13471	75	12	matthes	matthes	PROPN
spir-13471	75	13	(	(	PUNCT
spir-13471	75	14	eds	ed	NOUN
spir-13471	75	15	.	.	PUNCT
spir-13471	75	16	)	)	PUNCT
spir-13471	75	17	,	,	PUNCT
spir-13471	75	18	qualitätskriterien	qualitätskriterien	NOUN
spir-13471	75	19	in	in	ADP
spir-13471	75	20	der	der	ADJ
spir-13471	75	21	inhaltsanalyse	inhaltsanalyse	NOUN
spir-13471	75	22	[	[	X
spir-13471	75	23	quality	quality	NOUN
spir-13471	75	24	criteria	criterion	NOUN
spir-13471	75	25	in	in	ADP
spir-13471	75	26	content	content	NOUN
spir-13471	75	27	analysis	analysis	NOUN
spir-13471	75	28	]	]	PUNCT
spir-13471	75	29	(	(	PUNCT
spir-13471	75	30	pp	pp	ADJ
spir-13471	75	31	.	.	PUNCT
spir-13471	76	1	78	78	NUM
spir-13471	76	2	-	-	SYM
spir-13471	76	3	95	95	NUM
spir-13471	76	4	)	)	PUNCT
spir-13471	76	5	.	.	PUNCT
spir-13471	77	1	herbert	herbert	PROPN
spir-13471	77	2	von	von	PROPN
spir-13471	77	3	halem	halem	PROPN
spir-13471	77	4	.	.	PUNCT
spir-13471	78	1	hornik	hornik	PROPN
spir-13471	78	2	,	,	PUNCT
spir-13471	78	3	r.	r.	PROPN
spir-13471	78	4	,	,	PUNCT
spir-13471	78	5	binns	binn	NOUN
spir-13471	78	6	,	,	PUNCT
spir-13471	78	7	s.	s.	PROPN
spir-13471	78	8	,	,	PUNCT
spir-13471	78	9	emery	emery	PROPN
spir-13471	78	10	,	,	PUNCT
spir-13471	78	11	s.	s.	PROPN
spir-13471	78	12	,	,	PUNCT
spir-13471	78	13	epstein	epstein	PROPN
spir-13471	78	14	,	,	PUNCT
spir-13471	78	15	v.	v.	ADP
spir-13471	78	16	m.	m.	NOUN
spir-13471	78	17	,	,	PUNCT
spir-13471	78	18	jeong	jeong	PROPN
spir-13471	78	19	,	,	PUNCT
spir-13471	78	20	m.	m.	NOUN
spir-13471	78	21	,	,	PUNCT
spir-13471	78	22	kim	kim	PROPN
spir-13471	78	23	,	,	PUNCT
spir-13471	78	24	k.	k.	PROPN
spir-13471	78	25	,	,	PUNCT
spir-13471	78	26	kim	kim	PROPN
spir-13471	78	27	,	,	PUNCT
spir-13471	78	28	y.	y.	PROPN
spir-13471	78	29	,	,	PUNCT
spir-13471	78	30	kranzler	kranzler	NOUN
spir-13471	78	31	,	,	PUNCT
spir-13471	78	32	e.	e.	PROPN
spir-13471	78	33	c.	c.	PROPN
spir-13471	78	34	,	,	PUNCT
spir-13471	78	35	jesch	jesch	PROPN
spir-13471	78	36	,	,	PUNCT
spir-13471	78	37	e.	e.	PROPN
spir-13471	78	38	,	,	PUNCT
spir-13471	78	39	lee	lee	PROPN
spir-13471	78	40	,	,	PUNCT
spir-13471	78	41	s.	s.	PROPN
spir-13471	78	42	j.	j.	PROPN
spir-13471	78	43	,	,	PUNCT
spir-13471	78	44	levin	levin	PROPN
spir-13471	78	45	,	,	PUNCT
spir-13471	78	46	a.	a.	PROPN
spir-13471	78	47	v.	v.	PROPN
spir-13471	78	48	,	,	PUNCT
spir-13471	78	49	liu	liu	PROPN
spir-13471	78	50	,	,	PUNCT
spir-13471	78	51	j.	j.	PROPN
spir-13471	78	52	,	,	PUNCT
spir-13471	78	53	o’donnell	o’donnell	PROPN
spir-13471	78	54	,	,	PUNCT
spir-13471	78	55	m.	m.	PROPN
spir-13471	78	56	b.	b.	PROPN
spir-13471	78	57	,	,	PUNCT
spir-13471	78	58	siegel	siegel	PROPN
spir-13471	78	59	,	,	PUNCT
spir-13471	78	60	l.	l.	PROPN
spir-13471	78	61	,	,	PUNCT
spir-13471	78	62	tran	tran	PROPN
spir-13471	78	63	,	,	PUNCT
spir-13471	78	64	h.	h.	PROPN
spir-13471	78	65	,	,	PUNCT
spir-13471	78	66	williams	williams	PROPN
spir-13471	78	67	,	,	PUNCT
spir-13471	78	68	s.	s.	PROPN
spir-13471	78	69	,	,	PUNCT
spir-13471	78	70	yang	yang	PROPN
spir-13471	78	71	,	,	PUNCT
spir-13471	78	72	q.	q.	PROPN
spir-13471	78	73	,	,	PUNCT
spir-13471	78	74	&	&	CCONJ
spir-13471	78	75	gibson	gibson	PROPN
spir-13471	78	76	,	,	PUNCT
spir-13471	78	77	l.	l.	PROPN
spir-13471	78	78	a.	a.	PROPN
spir-13471	78	79	(	(	PUNCT
spir-13471	78	80	2022	2022	NUM
spir-13471	78	81	)	)	PUNCT
spir-13471	78	82	.	.	PUNCT
spir-13471	79	1	the	the	DET
spir-13471	79	2	effects	effect	NOUN
spir-13471	79	3	of	of	ADP
spir-13471	79	4	tobacco	tobacco	NOUN
spir-13471	79	5	coverage	coverage	NOUN
spir-13471	79	6	in	in	ADP
spir-13471	79	7	the	the	DET
spir-13471	79	8	public	public	ADJ
spir-13471	79	9	communication	communication	NOUN
spir-13471	79	10	environment	environment	NOUN
spir-13471	79	11	on	on	ADP
spir-13471	79	12	young	young	ADJ
spir-13471	79	13	people	people	NOUN
spir-13471	79	14	’s	’s	PART
spir-13471	79	15	decisions	decision	NOUN
spir-13471	79	16	to	to	PART
spir-13471	79	17	smoke	smoke	VERB
spir-13471	79	18	combustible	combustible	ADJ
spir-13471	79	19	cigarettes	cigarette	NOUN
spir-13471	79	20	.	.	PUNCT
spir-13471	80	1	journal	journal	NOUN
spir-13471	80	2	of	of	ADP
spir-13471	80	3	communication	communication	NOUN
spir-13471	80	4	,	,	PUNCT
spir-13471	80	5	72(2	72(2	NOUN
spir-13471	80	6	)	)	PUNCT
spir-13471	80	7	,	,	PUNCT
spir-13471	80	8	187–213	187–213	NUM
spir-13471	80	9	.	.	PUNCT
spir-13471	81	1	https://doi.org/10.1093/joc/jqab052	https://doi.org/10.1093/joc/jqab052	PROPN
spir-13471	81	2	krippendorff	krippendorff	PROPN
spir-13471	81	3	,	,	PUNCT
spir-13471	81	4	k.	k.	PROPN
spir-13471	81	5	(	(	PUNCT
spir-13471	81	6	2002	2002	NUM
spir-13471	81	7	)	)	PUNCT
spir-13471	81	8	.	.	PUNCT
spir-13471	82	1	content	content	NOUN
spir-13471	82	2	analysis	analysis	NOUN
spir-13471	82	3	.	.	PUNCT
spir-13471	83	1	an	an	DET
spir-13471	83	2	introduction	introduction	NOUN
spir-13471	83	3	to	to	ADP
spir-13471	83	4	its	its	PRON
spir-13471	83	5	methodology	methodology	NOUN
spir-13471	83	6	(	(	PUNCT
spir-13471	83	7	2nd	2nd	ADJ
spir-13471	83	8	.	.	PUNCT
spir-13471	84	1	ed	ed	NOUN
spir-13471	84	2	.	.	PUNCT
spir-13471	84	3	)	)	PUNCT
spir-13471	84	4	.	.	PUNCT
spir-13471	85	1	sage	sage	NOUN
spir-13471	85	2	.	.	PUNCT
spir-13471	86	1	lind	lind	PROPN
spir-13471	86	2	,	,	PUNCT
spir-13471	86	3	f.	f.	PROPN
spir-13471	86	4	,	,	PUNCT
spir-13471	86	5	gruber	gruber	PROPN
spir-13471	86	6	,	,	PUNCT
spir-13471	86	7	m.	m.	NOUN
spir-13471	86	8	,	,	PUNCT
spir-13471	86	9	&	&	CCONJ
spir-13471	86	10	boomgaarden	boomgaarden	PROPN
spir-13471	86	11	,	,	PUNCT
spir-13471	86	12	h.	h.	PROPN
spir-13471	86	13	g.	g.	PROPN
spir-13471	86	14	(	(	PUNCT
spir-13471	86	15	2017	2017	NUM
spir-13471	86	16	)	)	PUNCT
spir-13471	86	17	.	.	PUNCT
spir-13471	87	1	content	content	NOUN
spir-13471	87	2	analysis	analysis	NOUN
spir-13471	87	3	by	by	ADP
spir-13471	87	4	the	the	DET
spir-13471	87	5	crowd	crowd	NOUN
spir-13471	87	6	:	:	PUNCT
spir-13471	87	7	assessing	assess	VERB
spir-13471	87	8	the	the	DET
spir-13471	87	9	usability	usability	NOUN
spir-13471	87	10	of	of	ADP
spir-13471	87	11	crowdsourcing	crowdsource	VERB
spir-13471	87	12	for	for	ADP
spir-13471	87	13	coding	code	VERB
spir-13471	87	14	latent	latent	NOUN
spir-13471	87	15	constructs	construct	NOUN
spir-13471	87	16	.	.	PUNCT
spir-13471	88	1	communication	communication	NOUN
spir-13471	88	2	methods	method	NOUN
spir-13471	88	3	and	and	CCONJ
spir-13471	88	4	measures	measure	NOUN
spir-13471	88	5	,	,	PUNCT
spir-13471	88	6	11(3	11(3	NUM
spir-13471	88	7	)	)	PUNCT
spir-13471	88	8	,	,	PUNCT
spir-13471	88	9	191–209	191–209	NUM
spir-13471	88	10	.	.	PUNCT
spir-13471	89	1	https://doi.org/10.1080/19312458.2017.1317338	https://doi.org/10.1080/19312458.2017.1317338	PUNCT