id	sid	tid	token	lemma	pos
spir-13991	1	1	selected	select	VERB
spir-13991	1	2	papers	paper	NOUN
spir-13991	1	3	of	of	ADP
spir-13991	1	4	#	#	SYM
spir-13991	1	5	aoir2024	aoir2024	PROPN
spir-13991	1	6	:	:	PUNCT
spir-13991	1	7	the	the	DET
spir-13991	1	8	25th	25th	ADJ
spir-13991	1	9	annual	annual	ADJ
spir-13991	1	10	conference	conference	NOUN
spir-13991	1	11	of	of	ADP
spir-13991	1	12	the	the	DET
spir-13991	1	13	association	association	NOUN
spir-13991	1	14	of	of	ADP
spir-13991	1	15	internet	internet	NOUN
spir-13991	1	16	researchers	researcher	NOUN
spir-13991	1	17	sheffield	sheffield	PROPN
spir-13991	1	18	,	,	PUNCT
spir-13991	1	19	uk	uk	PROPN
spir-13991	1	20	/	/	SYM
spir-13991	1	21	30	30	NUM
spir-13991	1	22	oct	oct	NOUN
spir-13991	1	23	2	2	NUM
spir-13991	1	24	nov	nov	PROPN
spir-13991	1	25	2024	2024	NUM
spir-13991	1	26	suggested	suggest	VERB
spir-13991	1	27	citation	citation	NOUN
spir-13991	1	28	(	(	PUNCT
spir-13991	1	29	apa	apa	PROPN
spir-13991	1	30	):	):	PUNCT
spir-13991	1	31	hansen	hansen	PROPN
spir-13991	1	32	,	,	PUNCT
spir-13991	1	33	a.	a.	PROPN
spir-13991	1	34	s.	s.	PROPN
spir-13991	1	35	,	,	PUNCT
spir-13991	1	36	luitse	luitse	PROPN
spir-13991	1	37	,	,	PUNCT
spir-13991	1	38	d.	d.	PROPN
spir-13991	1	39	(	(	PUNCT
spir-13991	1	40	2024	2024	NUM
spir-13991	1	41	,	,	PUNCT
spir-13991	1	42	october	october	PROPN
spir-13991	1	43	)	)	PUNCT
spir-13991	1	44	.	.	PUNCT
spir-13991	2	1	the	the	DET
spir-13991	2	2	politics	politic	NOUN
spir-13991	2	3	of	of	ADP
spir-13991	2	4	machine	machine	NOUN
spir-13991	2	5	-	-	PUNCT
spir-13991	2	6	learning	learn	VERB
spir-13991	2	7	evaluation	evaluation	NOUN
spir-13991	2	8	:	:	PUNCT
spir-13991	2	9	from	from	ADP
spir-13991	2	10	lab	lab	NOUN
spir-13991	2	11	to	to	ADP
spir-13991	2	12	industry	industry	NOUN
spir-13991	2	13	.	.	PUNCT
spir-13991	3	1	paper	paper	NOUN
spir-13991	3	2	presented	present	VERB
spir-13991	3	3	at	at	ADP
spir-13991	3	4	aoir2024	aoir2024	PROPN
spir-13991	3	5	:	:	PUNCT
spir-13991	3	6	the	the	DET
spir-13991	3	7	25th	25th	ADJ
spir-13991	3	8	annual	annual	ADJ
spir-13991	3	9	conference	conference	NOUN
spir-13991	3	10	of	of	ADP
spir-13991	3	11	the	the	DET
spir-13991	3	12	association	association	NOUN
spir-13991	3	13	of	of	ADP
spir-13991	3	14	internet	internet	NOUN
spir-13991	3	15	researchers	researcher	NOUN
spir-13991	3	16	.	.	PUNCT
spir-13991	4	1	sheffield	sheffield	PROPN
spir-13991	4	2	,	,	PUNCT
spir-13991	4	3	uk	uk	PROPN
spir-13991	4	4	:	:	PUNCT
spir-13991	4	5	aoir	aoir	NOUN
spir-13991	4	6	.	.	PUNCT
spir-13991	4	7	retrieved	retrieve	VERB
spir-13991	4	8	from	from	ADP
spir-13991	4	9	http://spir.aoir.org	http://spir.aoir.org	PROPN
spir-13991	4	10	.	.	PUNCT
spir-13991	5	1	the	the	DET
spir-13991	5	2	politics	politic	NOUN
spir-13991	5	3	of	of	ADP
spir-13991	5	4	machine	machine	NOUN
spir-13991	5	5	-	-	PUNCT
spir-13991	5	6	learning	learn	VERB
spir-13991	5	7	evaluation	evaluation	NOUN
spir-13991	5	8	:	:	PUNCT
spir-13991	5	9	from	from	ADP
spir-13991	5	10	lab	lab	NOUN
spir-13991	5	11	to	to	ADP
spir-13991	5	12	industry	industry	NOUN
spir-13991	5	13	anna	anna	PROPN
spir-13991	5	14	schjøtt	schjøtt	PROPN
spir-13991	5	15	hansen	hansen	PROPN
spir-13991	5	16	university	university	PROPN
spir-13991	5	17	of	of	ADP
spir-13991	5	18	amsterdam	amsterdam	PROPN
spir-13991	5	19	dieuwertje	dieuwertje	PROPN
spir-13991	5	20	luitse	luitse	PROPN
spir-13991	5	21	university	university	PROPN
spir-13991	5	22	of	of	ADP
spir-13991	5	23	amsterdam	amsterdam	PROPN
spir-13991	5	24	introduction	introduction	PROPN
spir-13991	5	25	artificial	artificial	ADJ
spir-13991	5	26	intelligence	intelligence	NOUN
spir-13991	5	27	(	(	PUNCT
spir-13991	5	28	ai	ai	NOUN
spir-13991	5	29	)	)	PUNCT
spir-13991	5	30	applications	application	NOUN
spir-13991	5	31	are	be	AUX
spir-13991	5	32	today	today	NOUN
spir-13991	5	33	implemented	implement	VERB
spir-13991	5	34	across	across	ADP
spir-13991	5	35	various	various	ADJ
spir-13991	5	36	societal	societal	ADJ
spir-13991	5	37	sectors	sector	NOUN
spir-13991	5	38	,	,	PUNCT
spir-13991	5	39	ranging	range	VERB
spir-13991	5	40	from	from	ADP
spir-13991	5	41	health	health	NOUN
spir-13991	5	42	care	care	NOUN
spir-13991	5	43	and	and	CCONJ
spir-13991	5	44	security	security	NOUN
spir-13991	5	45	to	to	ADP
spir-13991	5	46	taking	take	VERB
spir-13991	5	47	part	part	NOUN
spir-13991	5	48	in	in	ADP
spir-13991	5	49	shaping	shape	VERB
spir-13991	5	50	the	the	DET
spir-13991	5	51	media	medium	NOUN
spir-13991	5	52	environment	environment	NOUN
spir-13991	5	53	we	we	PRON
spir-13991	5	54	encounter	encounter	VERB
spir-13991	5	55	online	online	ADV
spir-13991	5	56	.	.	PUNCT
spir-13991	6	1	in	in	ADP
spir-13991	6	2	the	the	DET
spir-13991	6	3	last	last	ADJ
spir-13991	6	4	decade	decade	NOUN
spir-13991	6	5	there	there	PRON
spir-13991	6	6	has	have	AUX
spir-13991	6	7	been	be	AUX
spir-13991	6	8	a	a	DET
spir-13991	6	9	significant	significant	ADJ
spir-13991	6	10	shift	shift	NOUN
spir-13991	6	11	in	in	ADP
spir-13991	6	12	the	the	DET
spir-13991	6	13	field	field	NOUN
spir-13991	6	14	of	of	ADP
spir-13991	6	15	ai	ai	NOUN
spir-13991	6	16	,	,	PUNCT
spir-13991	6	17	as	as	SCONJ
spir-13991	6	18	the	the	DET
spir-13991	6	19	development	development	NOUN
spir-13991	6	20	of	of	ADP
spir-13991	6	21	ai	ai	PROPN
spir-13991	6	22	applications	application	NOUN
spir-13991	6	23	is	be	AUX
spir-13991	6	24	no	no	ADV
spir-13991	6	25	longer	long	ADV
spir-13991	6	26	confined	confine	VERB
spir-13991	6	27	to	to	ADP
spir-13991	6	28	the	the	DET
spir-13991	6	29	laboratory	laboratory	NOUN
spir-13991	6	30	,	,	PUNCT
spir-13991	6	31	but	but	CCONJ
spir-13991	6	32	rather	rather	ADV
spir-13991	6	33	widely	widely	ADV
spir-13991	6	34	used	use	VERB
spir-13991	6	35	and	and	CCONJ
spir-13991	6	36	tested	test	VERB
spir-13991	6	37	in	in	ADP
spir-13991	6	38	and	and	CCONJ
spir-13991	6	39	on	on	ADP
spir-13991	6	40	societies	society	NOUN
spir-13991	6	41	(	(	PUNCT
spir-13991	6	42	whitaker	whitaker	PROPN
spir-13991	6	43	et	et	PROPN
spir-13991	6	44	al	al	PROPN
spir-13991	6	45	.	.	PROPN
spir-13991	6	46	,	,	PUNCT
spir-13991	6	47	2018	2018	NUM
spir-13991	6	48	;	;	PUNCT
spir-13991	6	49	seaver	seaver	PROPN
spir-13991	6	50	,	,	PUNCT
spir-13991	6	51	2019	2019	NUM
spir-13991	6	52	:	:	PUNCT
spir-13991	6	53	marres	marre	NOUN
spir-13991	6	54	and	and	CCONJ
spir-13991	6	55	stark	stark	ADJ
spir-13991	6	56	,	,	PUNCT
spir-13991	6	57	2020	2020	NUM
spir-13991	6	58	;	;	PUNCT
spir-13991	6	59	sheik	sheik	PROPN
spir-13991	6	60	et	et	PROPN
spir-13991	6	61	al	al	PROPN
spir-13991	6	62	.	.	PROPN
spir-13991	6	63	,	,	PUNCT
spir-13991	6	64	2023	2023	NUM
spir-13991	6	65	;	;	PUNCT
spir-13991	6	66	van	van	PROPN
spir-13991	6	67	der	der	NOUN
spir-13991	6	68	vlist	vlist	VERB
spir-13991	6	69	et	et	PROPN
spir-13991	6	70	al	al	PROPN
spir-13991	6	71	.	.	PROPN
spir-13991	6	72	2024	2024	NUM
spir-13991	6	73	)	)	PUNCT
spir-13991	6	74	.	.	PUNCT
spir-13991	7	1	with	with	ADP
spir-13991	7	2	this	this	DET
spir-13991	7	3	rapid	rapid	ADJ
spir-13991	7	4	industrialisation	industrialisation	NOUN
spir-13991	7	5	of	of	ADP
spir-13991	7	6	ai	ai	NOUN
spir-13991	7	7	,	,	PUNCT
spir-13991	7	8	there	there	PRON
spir-13991	7	9	is	be	VERB
spir-13991	7	10	an	an	DET
spir-13991	7	11	increased	increase	VERB
spir-13991	7	12	need	need	NOUN
spir-13991	7	13	to	to	PART
spir-13991	7	14	understand	understand	VERB
spir-13991	7	15	the	the	DET
spir-13991	7	16	implications	implication	NOUN
spir-13991	7	17	of	of	ADP
spir-13991	7	18	both	both	CCONJ
spir-13991	7	19	the	the	DET
spir-13991	7	20	development	development	NOUN
spir-13991	7	21	and	and	CCONJ
spir-13991	7	22	deployment	deployment	NOUN
spir-13991	7	23	of	of	ADP
spir-13991	7	24	these	these	DET
spir-13991	7	25	systems	system	NOUN
spir-13991	7	26	.	.	PUNCT
spir-13991	8	1	while	while	SCONJ
spir-13991	8	2	critical	critical	ADJ
spir-13991	8	3	scholars	scholar	NOUN
spir-13991	8	4	have	have	AUX
spir-13991	8	5	started	start	VERB
spir-13991	8	6	to	to	PART
spir-13991	8	7	scrutinize	scrutinize	VERB
spir-13991	8	8	different	different	ADJ
spir-13991	8	9	components	component	NOUN
spir-13991	8	10	of	of	ADP
spir-13991	8	11	ai	ai	PROPN
spir-13991	8	12	development	development	NOUN
spir-13991	8	13	,	,	PUNCT
spir-13991	8	14	such	such	ADJ
spir-13991	8	15	as	as	ADP
spir-13991	8	16	dataset	dataset	ADJ
spir-13991	8	17	construction	construction	NOUN
spir-13991	8	18	and	and	CCONJ
spir-13991	8	19	annotation	annotation	NOUN
spir-13991	8	20	practices	practice	NOUN
spir-13991	8	21	(	(	PUNCT
spir-13991	8	22	miceli	miceli	PROPN
spir-13991	8	23	et	et	PROPN
spir-13991	8	24	al	al	PROPN
spir-13991	8	25	.	.	PROPN
spir-13991	8	26	,	,	PUNCT
spir-13991	8	27	2021	2021	NUM
spir-13991	8	28	;	;	PUNCT
spir-13991	8	29	miceli	miceli	PROPN
spir-13991	8	30	and	and	CCONJ
spir-13991	8	31	posada	posada	PROPN
spir-13991	8	32	,	,	PUNCT
spir-13991	8	33	2021	2021	NUM
spir-13991	8	34	;	;	PUNCT
spir-13991	8	35	paullada	paullada	PROPN
spir-13991	8	36	et	et	PROPN
spir-13991	8	37	al	al	PROPN
spir-13991	8	38	.	.	PROPN
spir-13991	8	39	,	,	PUNCT
spir-13991	8	40	2021	2021	NUM
spir-13991	8	41	;	;	PUNCT
spir-13991	8	42	orr	orr	PROPN
spir-13991	8	43	and	and	CCONJ
spir-13991	8	44	crawford	crawford	PROPN
spir-13991	8	45	,	,	PUNCT
spir-13991	8	46	2023	2023	NUM
spir-13991	8	47	)	)	PUNCT
spir-13991	8	48	,	,	PUNCT
spir-13991	8	49	the	the	DET
spir-13991	8	50	study	study	NOUN
spir-13991	8	51	of	of	ADP
spir-13991	8	52	evaluative	evaluative	ADJ
spir-13991	8	53	practices	practice	NOUN
spir-13991	8	54	in	in	ADP
spir-13991	8	55	ai	ai	NOUN
spir-13991	8	56	has	have	AUX
spir-13991	8	57	received	receive	VERB
spir-13991	8	58	limited	limited	ADJ
spir-13991	8	59	attention	attention	NOUN
spir-13991	8	60	.	.	PUNCT
spir-13991	9	1	a	a	DET
spir-13991	9	2	few	few	ADJ
spir-13991	9	3	studies	study	NOUN
spir-13991	9	4	have	have	AUX
spir-13991	9	5	highlighted	highlight	VERB
spir-13991	9	6	the	the	DET
spir-13991	9	7	importance	importance	NOUN
spir-13991	9	8	of	of	ADP
spir-13991	9	9	benchmarking	benchmarke	VERB
spir-13991	9	10	practices	practice	NOUN
spir-13991	9	11	and	and	CCONJ
spir-13991	9	12	how	how	SCONJ
spir-13991	9	13	these	these	DET
spir-13991	9	14	methods	method	NOUN
spir-13991	9	15	become	become	VERB
spir-13991	9	16	integral	integral	ADJ
spir-13991	9	17	in	in	ADP
spir-13991	9	18	establishing	establish	VERB
spir-13991	9	19	the	the	DET
spir-13991	9	20	validity	validity	NOUN
spir-13991	9	21	of	of	ADP
spir-13991	9	22	the	the	DET
spir-13991	9	23	system	system	NOUN
spir-13991	9	24	and	and	CCONJ
spir-13991	9	25	its	its	PRON
spir-13991	9	26	success	success	NOUN
spir-13991	9	27	,	,	PUNCT
spir-13991	9	28	which	which	PRON
spir-13991	9	29	then	then	ADV
spir-13991	9	30	enables	enable	VERB
spir-13991	9	31	widespread	widespread	ADJ
spir-13991	9	32	application	application	NOUN
spir-13991	9	33	(	(	PUNCT
spir-13991	9	34	jaton	jaton	NOUN
spir-13991	9	35	,	,	PUNCT
spir-13991	9	36	2017	2017	NUM
spir-13991	9	37	;	;	PUNCT
spir-13991	9	38	2023	2023	NUM
spir-13991	9	39	;	;	PUNCT
spir-13991	9	40	raji	raji	PROPN
spir-13991	9	41	et	et	PROPN
spir-13991	9	42	al	al	PROPN
spir-13991	9	43	.	.	PROPN
spir-13991	9	44	,	,	PUNCT
spir-13991	9	45	2021	2021	NUM
spir-13991	9	46	;	;	PUNCT
spir-13991	9	47	grill	grill	NOUN
spir-13991	9	48	,	,	PUNCT
spir-13991	9	49	2022	2022	NUM
spir-13991	9	50	)	)	PUNCT
spir-13991	9	51	.	.	PUNCT
spir-13991	10	1	this	this	DET
spir-13991	10	2	paper	paper	NOUN
spir-13991	10	3	presents	present	VERB
spir-13991	10	4	a	a	DET
spir-13991	10	5	research	research	NOUN
spir-13991	10	6	agenda	agenda	NOUN
spir-13991	10	7	that	that	PRON
spir-13991	10	8	outlines	outline	VERB
spir-13991	10	9	how	how	SCONJ
spir-13991	10	10	to	to	PART
spir-13991	10	11	study	study	VERB
spir-13991	10	12	machine	machine	NOUN
spir-13991	10	13	-	-	PUNCT
spir-13991	10	14	learning	learn	VERB
spir-13991	10	15	evaluation	evaluation	NOUN
spir-13991	10	16	practices	practice	NOUN
spir-13991	10	17	that	that	PRON
spir-13991	10	18	move	move	VERB
spir-13991	10	19	beyond	beyond	ADP
spir-13991	10	20	the	the	DET
spir-13991	10	21	laboratory	laboratory	NOUN
spir-13991	10	22	into	into	ADP
spir-13991	10	23	industry	industry	NOUN
spir-13991	10	24	applications	application	NOUN
spir-13991	10	25	and	and	CCONJ
spir-13991	10	26	standardised	standardise	VERB
spir-13991	10	27	validation	validation	NOUN
spir-13991	10	28	practices	practice	NOUN
spir-13991	10	29	.	.	PUNCT
spir-13991	11	1	based	base	VERB
spir-13991	11	2	on	on	ADP
spir-13991	11	3	emerging	emerge	VERB
spir-13991	11	4	research	research	NOUN
spir-13991	11	5	and	and	CCONJ
spir-13991	11	6	illustrative	illustrative	ADJ
spir-13991	11	7	empirical	empirical	ADJ
spir-13991	11	8	examples	example	NOUN
spir-13991	11	9	from	from	ADP
spir-13991	11	10	recent	recent	ADJ
spir-13991	11	11	fieldwork	fieldwork	NOUN
spir-13991	11	12	,	,	PUNCT
spir-13991	11	13	we	we	PRON
spir-13991	11	14	argue	argue	VERB
spir-13991	11	15	to	to	PART
spir-13991	11	16	study	study	VERB
spir-13991	11	17	machine	machine	NOUN
spir-13991	11	18	-	-	PUNCT
spir-13991	11	19	learning	learn	VERB
spir-13991	11	20	evaluation	evaluation	NOUN
spir-13991	11	21	as	as	ADP
spir-13991	11	22	a	a	DET
spir-13991	11	23	sociotechnical	sociotechnical	ADJ
spir-13991	11	24	and	and	CCONJ
spir-13991	11	25	political	political	ADJ
spir-13991	11	26	phenomenon	phenomenon	NOUN
spir-13991	11	27	that	that	PRON
spir-13991	11	28	requires	require	VERB
spir-13991	11	29	multi	multi	ADJ
spir-13991	11	30	-	-	ADJ
spir-13991	11	31	level	level	ADJ
spir-13991	11	32	scrutiny	scrutiny	NOUN
spir-13991	11	33	.	.	PUNCT
spir-13991	12	1	therefore	therefore	ADV
spir-13991	12	2	,	,	PUNCT
spir-13991	12	3	we	we	PRON
spir-13991	12	4	provide	provide	VERB
spir-13991	12	5	three	three	NUM
spir-13991	12	6	analytical	analytical	ADJ
spir-13991	12	7	entry	entry	NOUN
spir-13991	12	8	points	point	NOUN
spir-13991	12	9	for	for	ADP
spir-13991	12	10	future	future	ADJ
spir-13991	12	11	research	research	NOUN
spir-13991	12	12	that	that	PRON
spir-13991	12	13	address	address	VERB
spir-13991	12	14	the	the	DET
spir-13991	12	15	political	political	ADJ
spir-13991	12	16	dynamics	dynamic	NOUN
spir-13991	12	17	of	of	ADP
spir-13991	12	18	(	(	PUNCT
spir-13991	12	19	1	1	NUM
spir-13991	12	20	)	)	PUNCT
spir-13991	12	21	standardised	standardise	VERB
spir-13991	12	22	validation	validation	NOUN
spir-13991	12	23	infrastructures	infrastructure	NOUN
spir-13991	12	24	,	,	PUNCT
spir-13991	12	25	(	(	PUNCT
spir-13991	12	26	2	2	X
spir-13991	12	27	)	)	PUNCT
spir-13991	12	28	the	the	DET
spir-13991	12	29	circulation	circulation	NOUN
spir-13991	12	30	of	of	ADP
spir-13991	12	31	evaluation	evaluation	NOUN
spir-13991	12	32	methods	method	NOUN
spir-13991	12	33	and	and	CCONJ
spir-13991	12	34	(	(	PUNCT
spir-13991	12	35	3	3	X
spir-13991	12	36	)	)	PUNCT
spir-13991	12	37	the	the	DET
spir-13991	12	38	situated	situated	ADJ
spir-13991	12	39	enactment	enactment	NOUN
spir-13991	12	40	of	of	ADP
spir-13991	12	41	evaluation	evaluation	NOUN
spir-13991	12	42	in	in	ADP
spir-13991	12	43	practice	practice	NOUN
spir-13991	12	44	.	.	PUNCT
spir-13991	13	1	evaluation	evaluation	NOUN
spir-13991	13	2	is	be	AUX
spir-13991	13	3	a	a	DET
spir-13991	13	4	foundational	foundational	ADJ
spir-13991	13	5	part	part	NOUN
spir-13991	13	6	of	of	ADP
spir-13991	13	7	the	the	DET
spir-13991	13	8	development	development	NOUN
spir-13991	13	9	of	of	ADP
spir-13991	13	10	ai	ai	PROPN
spir-13991	13	11	systems	system	NOUN
spir-13991	13	12	because	because	SCONJ
spir-13991	13	13	it	it	PRON
spir-13991	13	14	requires	require	VERB
spir-13991	13	15	“	"	PUNCT
spir-13991	13	16	reflection	reflection	NOUN
spir-13991	13	17	on	on	ADP
spir-13991	13	18	what	what	PRON
spir-13991	13	19	the	the	DET
spir-13991	13	20	objectives	objective	NOUN
spir-13991	13	21	of	of	ADP
spir-13991	13	22	an	an	DET
spir-13991	13	23	effective	effective	ADJ
spir-13991	13	24	model	model	NOUN
spir-13991	13	25	involve	involve	NOUN
spir-13991	13	26	,	,	PUNCT
spir-13991	13	27	in	in	ADP
spir-13991	13	28	addition	addition	NOUN
spir-13991	13	29	to	to	ADP
spir-13991	13	30	conscious	conscious	ADJ
spir-13991	13	31	decision	decision	NOUN
spir-13991	13	32	making	make	VERB
spir-13991	13	33	on	on	ADP
spir-13991	13	34	how	how	SCONJ
spir-13991	13	35	to	to	PART
spir-13991	13	36	best	well	ADV
spir-13991	13	37	represent	represent	VERB
spir-13991	13	38	these	these	DET
spir-13991	13	39	desired	desire	VERB
spir-13991	13	40	outcomes	outcome	NOUN
spir-13991	13	41	in	in	ADP
spir-13991	13	42	evaluation	evaluation	NOUN
spir-13991	13	43	metrics	metric	NOUN
spir-13991	13	44	,	,	PUNCT
spir-13991	13	45	data	datum	NOUN
spir-13991	13	46	and	and	CCONJ
spir-13991	13	47	methodology	methodology	NOUN
spir-13991	13	48	”	"	PUNCT
spir-13991	13	49	(	(	PUNCT
spir-13991	13	50	raji	raji	PROPN
spir-13991	13	51	et	et	PROPN
spir-13991	13	52	al	al	PROPN
spir-13991	13	53	.	.	PROPN
spir-13991	13	54	,	,	PUNCT
spir-13991	13	55	2021	2021	NUM
spir-13991	13	56	,	,	PUNCT
spir-13991	13	57	p.	p.	NOUN
spir-13991	13	58	3	3	NUM
spir-13991	13	59	)	)	PUNCT
spir-13991	13	60	.	.	PUNCT
spir-13991	14	1	in	in	ADP
spir-13991	14	2	practice	practice	NOUN
spir-13991	14	3	,	,	PUNCT
spir-13991	14	4	this	this	PRON
spir-13991	14	5	often	often	ADV
spir-13991	14	6	involves	involve	VERB
spir-13991	14	7	developing	develop	VERB
spir-13991	14	8	metrics	metric	NOUN
spir-13991	14	9	that	that	PRON
spir-13991	14	10	can	can	AUX
spir-13991	14	11	best	well	ADV
spir-13991	14	12	approximate	approximate	VERB
spir-13991	14	13	and	and	CCONJ
spir-13991	14	14	measure	measure	VERB
spir-13991	14	15	whether	whether	SCONJ
spir-13991	14	16	the	the	DET
spir-13991	14	17	system	system	NOUN
spir-13991	14	18	achieves	achieve	VERB
spir-13991	14	19	the	the	DET
spir-13991	14	20	anticipated	anticipated	ADJ
spir-13991	14	21	behaviour	behaviour	NOUN
spir-13991	14	22	(	(	PUNCT
spir-13991	14	23	aradau	aradau	NOUN
spir-13991	14	24	and	and	CCONJ
spir-13991	14	25	blanke	blanke	ADJ
spir-13991	14	26	,	,	PUNCT
spir-13991	14	27	2021	2021	NUM
spir-13991	14	28	;	;	PUNCT
spir-13991	14	29	raji	raji	PROPN
spir-13991	14	30	et	et	PROPN
spir-13991	14	31	al	al	PROPN
spir-13991	14	32	.	.	PROPN
spir-13991	14	33	,	,	PUNCT
spir-13991	14	34	2021	2021	NUM
spir-13991	14	35	)	)	PUNCT
spir-13991	14	36	.	.	PUNCT
spir-13991	15	1	as	as	SCONJ
spir-13991	15	2	also	also	ADV
spir-13991	15	3	highlighted	highlight	VERB
spir-13991	15	4	by	by	ADP
spir-13991	15	5	thomas	thomas	PROPN
spir-13991	15	6	and	and	CCONJ
spir-13991	15	7	uminski	uminski	ADJ
spir-13991	15	8	,	,	PUNCT
spir-13991	15	9	(	(	PUNCT
spir-13991	15	10	2020	2020	NUM
spir-13991	15	11	):	):	PUNCT
spir-13991	15	12	‘	'	PUNCT
spir-13991	15	13	what	what	PRON
spir-13991	15	14	most	most	ADV
spir-13991	15	15	current	current	ADJ
spir-13991	15	16	ai	ai	NOUN
spir-13991	15	17	approaches	approach	NOUN
spir-13991	15	18	do	do	AUX
spir-13991	15	19	is	be	AUX
spir-13991	15	20	to	to	PART
spir-13991	15	21	optimize	optimize	VERB
spir-13991	15	22	metrics	metric	NOUN
spir-13991	15	23	’	'	PUNCT
spir-13991	15	24	(	(	PUNCT
spir-13991	15	25	p.	p.	NOUN
spir-13991	15	26	1	1	NUM
spir-13991	15	27	)	)	PUNCT
spir-13991	15	28	.	.	PUNCT
spir-13991	16	1	taken	take	VERB
spir-13991	16	2	together	together	ADV
spir-13991	16	3	,	,	PUNCT
spir-13991	16	4	this	this	PRON
spir-13991	16	5	illustrates	illustrate	VERB
spir-13991	16	6	how	how	SCONJ
spir-13991	16	7	evaluation	evaluation	NOUN
spir-13991	16	8	is	be	AUX
spir-13991	16	9	a	a	DET
spir-13991	16	10	highly	highly	ADV
spir-13991	16	11	political	political	ADJ
spir-13991	16	12	practice	practice	NOUN
spir-13991	16	13	,	,	PUNCT
spir-13991	16	14	where	where	SCONJ
spir-13991	16	15	the	the	DET
spir-13991	16	16	decisions	decision	NOUN
spir-13991	16	17	will	will	AUX
spir-13991	16	18	often	often	ADV
spir-13991	16	19	favour	favour	VERB
spir-13991	16	20	‘	'	PUNCT
spir-13991	16	21	good	good	ADJ
spir-13991	16	22	enough	enough	ADJ
spir-13991	16	23	’	'	PUNCT
spir-13991	16	24	optimisation	optimisation	NOUN
spir-13991	16	25	methodologies	methodology	NOUN
spir-13991	16	26	(	(	PUNCT
spir-13991	16	27	amoore	amoore	ADJ
spir-13991	16	28	,	,	PUNCT
spir-13991	16	29	2020	2020	NUM
spir-13991	16	30	;	;	PUNCT
spir-13991	16	31	thomas	thomas	PROPN
spir-13991	16	32	and	and	CCONJ
spir-13991	16	33	uminsky	uminsky	ADJ
spir-13991	16	34	,	,	PUNCT
spir-13991	16	35	2021	2021	NUM
spir-13991	16	36	)	)	PUNCT
spir-13991	16	37	.	.	PUNCT
spir-13991	17	1	the	the	DET
spir-13991	17	2	political	political	ADJ
spir-13991	17	3	nature	nature	NOUN
spir-13991	17	4	of	of	ADP
spir-13991	17	5	‘	'	PUNCT
spir-13991	17	6	good	good	ADJ
spir-13991	17	7	enough	enough	ADJ
spir-13991	17	8	’	'	PUNCT
spir-13991	17	9	optimisations	optimisation	NOUN
spir-13991	17	10	has	have	AUX
spir-13991	17	11	been	be	AUX
spir-13991	17	12	highlighted	highlight	VERB
spir-13991	17	13	by	by	ADP
spir-13991	17	14	authors	author	NOUN
spir-13991	17	15	,	,	PUNCT
spir-13991	17	16	pointing	point	VERB
spir-13991	17	17	out	out	ADP
spir-13991	17	18	how	how	SCONJ
spir-13991	17	19	this	this	PRON
spir-13991	17	20	enables	enable	VERB
spir-13991	17	21	the	the	DET
spir-13991	17	22	construction	construction	NOUN
spir-13991	17	23	of	of	ADP
spir-13991	17	24	validity	validity	NOUN
spir-13991	17	25	(	(	PUNCT
spir-13991	17	26	raji	raji	PROPN
spir-13991	17	27	et	et	PROPN
spir-13991	17	28	al	al	PROPN
spir-13991	17	29	.	.	PROPN
spir-13991	17	30	,	,	PUNCT
spir-13991	17	31	2021	2021	NUM
spir-13991	17	32	)	)	PUNCT
spir-13991	17	33	and	and	CCONJ
spir-13991	17	34	the	the	DET
spir-13991	17	35	production	production	NOUN
spir-13991	17	36	of	of	ADP
spir-13991	17	37	certainty	certainty	NOUN
spir-13991	17	38	through	through	ADP
spir-13991	17	39	accuracy	accuracy	NOUN
spir-13991	17	40	measurements	measurement	NOUN
spir-13991	17	41	(	(	PUNCT
spir-13991	17	42	grill	grill	NOUN
spir-13991	17	43	,	,	PUNCT
spir-13991	17	44	2022	2022	NUM
spir-13991	17	45	)	)	PUNCT
spir-13991	17	46	.	.	PUNCT
spir-13991	18	1	in	in	ADP
spir-13991	18	2	addition	addition	NOUN
spir-13991	18	3	,	,	PUNCT
spir-13991	18	4	thakkar	thakkar	NOUN
spir-13991	18	5	et	et	PROPN
spir-13991	18	6	al	al	PROPN
spir-13991	18	7	.	.	PROPN
spir-13991	18	8	,	,	PUNCT
spir-13991	18	9	2022	2022	NUM
spir-13991	18	10	highlight	highlight	VERB
spir-13991	18	11	how	how	SCONJ
spir-13991	18	12	the	the	DET
spir-13991	18	13	construction	construction	NOUN
spir-13991	18	14	of	of	ADP
spir-13991	18	15	machine	machine	NOUN
spir-13991	18	16	-	-	PUNCT
spir-13991	18	17	learning	learn	VERB
spir-13991	18	18	datasets	dataset	NOUN
spir-13991	18	19	is	be	AUX
spir-13991	18	20	also	also	ADV
spir-13991	18	21	guided	guide	VERB
spir-13991	18	22	by	by	ADP
spir-13991	18	23	questions	question	NOUN
spir-13991	18	24	of	of	ADP
spir-13991	18	25	‘	'	PUNCT
spir-13991	18	26	goodenough	goodenough	NOUN
spir-13991	18	27	’	'	PUNCT
spir-13991	18	28	,	,	PUNCT
spir-13991	18	29	while	while	SCONJ
spir-13991	18	30	jaton	jaton	PROPN
spir-13991	18	31	(	(	PUNCT
spir-13991	18	32	2017	2017	NUM
spir-13991	18	33	)	)	PUNCT
spir-13991	18	34	has	have	AUX
spir-13991	18	35	focused	focus	VERB
spir-13991	18	36	on	on	ADP
spir-13991	18	37	the	the	DET
spir-13991	18	38	key	key	ADJ
spir-13991	18	39	role	role	NOUN
spir-13991	18	40	of	of	ADP
spir-13991	18	41	so	so	ADV
spir-13991	18	42	-	-	PUNCT
spir-13991	18	43	called	call	VERB
spir-13991	18	44	‘	'	PUNCT
spir-13991	18	45	ground	ground	NOUN
spir-13991	18	46	truths	truth	NOUN
spir-13991	18	47	,	,	PUNCT
spir-13991	18	48	’	'	PUNCT
spir-13991	18	49	in	in	ADP
spir-13991	18	50	both	both	PRON
spir-13991	18	51	training	training	NOUN
spir-13991	18	52	and	and	CCONJ
spir-13991	18	53	evaluating	evaluate	VERB
spir-13991	18	54	ai	ai	VERB
spir-13991	18	55	applications	application	NOUN
spir-13991	18	56	through	through	ADP
spir-13991	18	57	randomized	randomized	ADJ
spir-13991	18	58	tests	test	NOUN
spir-13991	18	59	to	to	PART
spir-13991	18	60	assess	assess	VERB
spir-13991	18	61	their	their	PRON
spir-13991	18	62	performance	performance	NOUN
spir-13991	18	63	.	.	PUNCT
spir-13991	19	1	however	however	ADV
spir-13991	19	2	,	,	PUNCT
spir-13991	19	3	these	these	DET
spir-13991	19	4	studies	study	NOUN
spir-13991	19	5	have	have	AUX
spir-13991	19	6	been	be	AUX
spir-13991	19	7	predominately	predominately	ADV
spir-13991	19	8	concerned	concerned	ADJ
spir-13991	19	9	with	with	ADP
spir-13991	19	10	the	the	DET
spir-13991	19	11	established	establish	VERB
spir-13991	19	12	‘	'	PUNCT
spir-13991	19	13	scientific	scientific	ADJ
spir-13991	19	14	’	'	PUNCT
spir-13991	19	15	evaluative	evaluative	ADJ
spir-13991	19	16	practices	practice	NOUN
spir-13991	19	17	of	of	ADP
spir-13991	19	18	ai	ai	PRON
spir-13991	19	19	research	research	NOUN
spir-13991	19	20	and	and	CCONJ
spir-13991	19	21	development	development	NOUN
spir-13991	19	22	and	and	CCONJ
spir-13991	19	23	do	do	AUX
spir-13991	19	24	not	not	PART
spir-13991	19	25	consider	consider	VERB
spir-13991	19	26	that	that	PRON
spir-13991	19	27	ai	ai	VERB
spir-13991	19	28	applications	application	NOUN
spir-13991	19	29	and	and	CCONJ
spir-13991	19	30	their	their	PRON
spir-13991	19	31	evaluative	evaluative	ADJ
spir-13991	19	32	methods	method	NOUN
spir-13991	19	33	are	be	AUX
spir-13991	19	34	increasingly	increasingly	ADV
spir-13991	19	35	becoming	become	VERB
spir-13991	19	36	industrialised	industrialise	VERB
spir-13991	19	37	.	.	PUNCT
spir-13991	20	1	in	in	ADP
spir-13991	20	2	light	light	NOUN
spir-13991	20	3	of	of	ADP
spir-13991	20	4	these	these	DET
spir-13991	20	5	developments	development	NOUN
spir-13991	20	6	,	,	PUNCT
spir-13991	20	7	this	this	DET
spir-13991	20	8	paper	paper	NOUN
spir-13991	20	9	aims	aim	VERB
spir-13991	20	10	to	to	PART
spir-13991	20	11	expand	expand	VERB
spir-13991	20	12	the	the	DET
spir-13991	20	13	current	current	ADJ
spir-13991	20	14	research	research	NOUN
spir-13991	20	15	on	on	ADP
spir-13991	20	16	evaluation	evaluation	NOUN
spir-13991	20	17	by	by	ADP
spir-13991	20	18	building	build	VERB
spir-13991	20	19	on	on	ADP
spir-13991	20	20	emerging	emerge	VERB
spir-13991	20	21	insights	insight	NOUN
spir-13991	20	22	from	from	ADP
spir-13991	20	23	the	the	DET
spir-13991	20	24	cited	cite	VERB
spir-13991	20	25	critical	critical	ADJ
spir-13991	20	26	enquires	enquire	NOUN
spir-13991	20	27	and	and	CCONJ
spir-13991	20	28	illustrative	illustrative	ADJ
spir-13991	20	29	empirical	empirical	ADJ
spir-13991	20	30	examples	example	NOUN
spir-13991	20	31	from	from	ADP
spir-13991	20	32	ongoing	ongoing	ADJ
spir-13991	20	33	fieldwork	fieldwork	NOUN
spir-13991	20	34	.	.	PUNCT
spir-13991	21	1	for	for	ADP
spir-13991	21	2	the	the	DET
spir-13991	21	3	latter	latter	ADJ
spir-13991	21	4	,	,	PUNCT
spir-13991	21	5	we	we	PRON
spir-13991	21	6	draw	draw	VERB
spir-13991	21	7	on	on	ADP
spir-13991	21	8	examples	example	NOUN
spir-13991	21	9	from	from	ADP
spir-13991	21	10	a	a	DET
spir-13991	21	11	recent	recent	ADJ
spir-13991	21	12	ethnographic	ethnographic	ADJ
spir-13991	21	13	enquiry	enquiry	NOUN
spir-13991	21	14	within	within	ADP
spir-13991	21	15	the	the	DET
spir-13991	21	16	bbc	bbc	PROPN
spir-13991	21	17	’s	’s	PROPN
spir-13991	21	18	data	data	PROPN
spir-13991	21	19	science	science	PROPN
spir-13991	21	20	department	department	PROPN
spir-13991	21	21	(	(	PUNCT
spir-13991	21	22	conducted	conduct	VERB
spir-13991	21	23	by	by	ADP
spir-13991	21	24	author	author	NOUN
spir-13991	21	25	1	1	NUM
spir-13991	21	26	)	)	PUNCT
spir-13991	21	27	and	and	CCONJ
spir-13991	21	28	ongoing	ongoing	ADJ
spir-13991	21	29	fieldwork	fieldwork	NOUN
spir-13991	21	30	on	on	ADP
spir-13991	21	31	the	the	DET
spir-13991	21	32	development	development	NOUN
spir-13991	21	33	of	of	ADP
spir-13991	21	34	ai	ai	PROPN
spir-13991	21	35	applications	application	NOUN
spir-13991	21	36	at	at	ADP
spir-13991	21	37	a	a	DET
spir-13991	21	38	dutch	dutch	ADJ
spir-13991	21	39	medical	medical	ADJ
spir-13991	21	40	centre	centre	NOUN
spir-13991	21	41	(	(	PUNCT
spir-13991	21	42	conducted	conduct	VERB
spir-13991	21	43	by	by	ADP
spir-13991	21	44	author	author	NOUN
spir-13991	21	45	2	2	NUM
spir-13991	21	46	)	)	PUNCT
spir-13991	21	47	.	.	PUNCT
spir-13991	22	1	taking	take	VERB
spir-13991	22	2	a	a	DET
spir-13991	22	3	critical	critical	ADJ
spir-13991	22	4	ai	ai	NOUN
spir-13991	22	5	studies	study	NOUN
spir-13991	22	6	and	and	CCONJ
spir-13991	22	7	science	science	NOUN
spir-13991	22	8	and	and	CCONJ
spir-13991	22	9	technology	technology	NOUN
spir-13991	22	10	studies	study	NOUN
spir-13991	22	11	(	(	PUNCT
spir-13991	22	12	sts	st	NOUN
spir-13991	22	13	)	)	PUNCT
spir-13991	22	14	research	research	NOUN
spir-13991	22	15	approach	approach	NOUN
spir-13991	22	16	,	,	PUNCT
spir-13991	22	17	we	we	PRON
spir-13991	22	18	outline	outline	VERB
spir-13991	22	19	how	how	SCONJ
spir-13991	22	20	to	to	PART
spir-13991	22	21	study	study	VERB
spir-13991	22	22	what	what	PRON
spir-13991	22	23	we	we	PRON
spir-13991	22	24	call	call	VERB
spir-13991	22	25	industrialised	industrialised	ADJ
spir-13991	22	26	accounts	account	NOUN
spir-13991	22	27	of	of	ADP
spir-13991	22	28	ai	ai	ADJ
spir-13991	22	29	evaluation	evaluation	NOUN
spir-13991	22	30	beyond	beyond	ADP
spir-13991	22	31	the	the	DET
spir-13991	22	32	laboratory	laboratory	NOUN
spir-13991	22	33	.	.	PUNCT
spir-13991	23	1	concretely	concretely	ADV
spir-13991	23	2	,	,	PUNCT
spir-13991	23	3	we	we	PRON
spir-13991	23	4	present	present	VERB
spir-13991	23	5	three	three	NUM
spir-13991	23	6	analytical	analytical	ADJ
spir-13991	23	7	entry	entry	NOUN
spir-13991	23	8	points	point	NOUN
spir-13991	23	9	that	that	PRON
spir-13991	23	10	address	address	VERB
spir-13991	23	11	the	the	DET
spir-13991	23	12	sociotechnical	sociotechnical	ADJ
spir-13991	23	13	and	and	CCONJ
spir-13991	23	14	political	political	ADJ
spir-13991	23	15	nature	nature	NOUN
spir-13991	23	16	of	of	ADP
spir-13991	23	17	ai	ai	PROPN
spir-13991	23	18	evaluation	evaluation	NOUN
spir-13991	23	19	.	.	PUNCT
spir-13991	24	1	these	these	DET
spir-13991	24	2	entry	entry	NOUN
spir-13991	24	3	points	point	NOUN
spir-13991	24	4	focus	focus	VERB
spir-13991	24	5	on	on	ADP
spir-13991	24	6	different	different	ADJ
spir-13991	24	7	levels	level	NOUN
spir-13991	24	8	of	of	ADP
spir-13991	24	9	ai	ai	ADJ
spir-13991	24	10	evaluation	evaluation	NOUN
spir-13991	24	11	in	in	ADP
spir-13991	24	12	industry	industry	NOUN
spir-13991	24	13	settings	setting	NOUN
spir-13991	24	14	that	that	PRON
spir-13991	24	15	include	include	VERB
spir-13991	24	16	wider	wide	ADJ
spir-13991	24	17	attempts	attempt	NOUN
spir-13991	24	18	of	of	ADP
spir-13991	24	19	standardised	standardised	ADJ
spir-13991	24	20	evaluation	evaluation	NOUN
spir-13991	24	21	infrastructures	infrastructure	NOUN
spir-13991	24	22	,	,	PUNCT
spir-13991	24	23	the	the	DET
spir-13991	24	24	circulation	circulation	NOUN
spir-13991	24	25	of	of	ADP
spir-13991	24	26	evaluative	evaluative	ADJ
spir-13991	24	27	methods	method	NOUN
spir-13991	24	28	and	and	CCONJ
spir-13991	24	29	the	the	DET
spir-13991	24	30	situated	situated	ADJ
spir-13991	24	31	practice	practice	NOUN
spir-13991	24	32	of	of	ADP
spir-13991	24	33	evaluating	evaluate	VERB
spir-13991	24	34	emerging	emerge	VERB
spir-13991	24	35	ai	ai	NOUN
spir-13991	24	36	applications	application	NOUN
spir-13991	24	37	within	within	ADP
spir-13991	24	38	localised	localised	ADJ
spir-13991	24	39	contexts	contexts	NOUN
spir-13991	24	40	.	.	PUNCT
spir-13991	25	1	in	in	ADP
spir-13991	25	2	what	what	PRON
spir-13991	25	3	follows	follow	VERB
spir-13991	25	4	,	,	PUNCT
spir-13991	25	5	we	we	PRON
spir-13991	25	6	briefly	briefly	ADV
spir-13991	25	7	discuss	discuss	VERB
spir-13991	25	8	these	these	DET
spir-13991	25	9	entry	entry	NOUN
spir-13991	25	10	points	point	NOUN
spir-13991	25	11	and	and	CCONJ
spir-13991	25	12	substantiate	substantiate	VERB
spir-13991	25	13	them	they	PRON
spir-13991	25	14	with	with	ADP
spir-13991	25	15	empirical	empirical	ADJ
spir-13991	25	16	illustrations	illustration	NOUN
spir-13991	25	17	from	from	ADP
spir-13991	25	18	existing	exist	VERB
spir-13991	25	19	research	research	NOUN
spir-13991	25	20	and	and	CCONJ
spir-13991	25	21	our	our	PRON
spir-13991	25	22	ongoing	ongoing	ADJ
spir-13991	25	23	fieldwork	fieldwork	NOUN
spir-13991	25	24	.	.	PUNCT
spir-13991	26	1	the	the	DET
spir-13991	26	2	political	political	ADJ
spir-13991	26	3	dynamics	dynamic	NOUN
spir-13991	26	4	of	of	ADP
spir-13991	26	5	standardised	standardised	ADJ
spir-13991	26	6	validation	validation	NOUN
spir-13991	26	7	infrastructures	infrastructure	NOUN
spir-13991	26	8	the	the	DET
spir-13991	26	9	first	first	ADJ
spir-13991	26	10	entry	entry	NOUN
spir-13991	26	11	point	point	NOUN
spir-13991	26	12	engages	engage	VERB
spir-13991	26	13	with	with	ADP
spir-13991	26	14	the	the	DET
spir-13991	26	15	emergence	emergence	NOUN
spir-13991	26	16	of	of	ADP
spir-13991	26	17	wider	wide	ADJ
spir-13991	26	18	attempts	attempt	NOUN
spir-13991	26	19	at	at	ADP
spir-13991	26	20	standardisation	standardisation	NOUN
spir-13991	26	21	of	of	ADP
spir-13991	26	22	evaluation	evaluation	NOUN
spir-13991	26	23	in	in	ADP
spir-13991	26	24	the	the	DET
spir-13991	26	25	ai	ai	NOUN
spir-13991	26	26	industry	industry	NOUN
spir-13991	26	27	via	via	ADP
spir-13991	26	28	large	large	ADJ
spir-13991	26	29	-	-	PUNCT
spir-13991	26	30	scale	scale	NOUN
spir-13991	26	31	infrastructural	infrastructural	ADJ
spir-13991	26	32	agents	agent	NOUN
spir-13991	26	33	that	that	PRON
spir-13991	26	34	shape	shape	VERB
spir-13991	26	35	ai	ai	VERB
spir-13991	26	36	development	development	NOUN
spir-13991	26	37	and	and	CCONJ
spir-13991	26	38	evaluation	evaluation	NOUN
spir-13991	26	39	regimes	regime	NOUN
spir-13991	26	40	.	.	PUNCT
spir-13991	27	1	one	one	NUM
spir-13991	27	2	example	example	NOUN
spir-13991	27	3	of	of	ADP
spir-13991	27	4	such	such	DET
spir-13991	27	5	an	an	DET
spir-13991	27	6	infrastructure	infrastructure	NOUN
spir-13991	27	7	is	be	AUX
spir-13991	27	8	the	the	DET
spir-13991	27	9	ai	ai	PROPN
spir-13991	27	10	competition	competition	NOUN
spir-13991	27	11	platform	platform	NOUN
spir-13991	27	12	kaggle	kaggle	PROPN
spir-13991	27	13	,	,	PUNCT
spir-13991	27	14	which	which	PRON
spir-13991	27	15	facilitates	facilitate	VERB
spir-13991	27	16	the	the	DET
spir-13991	27	17	evaluation	evaluation	NOUN
spir-13991	27	18	of	of	ADP
spir-13991	27	19	new	new	ADJ
spir-13991	27	20	models	model	NOUN
spir-13991	27	21	across	across	ADP
spir-13991	27	22	domains	domain	NOUN
spir-13991	27	23	.	.	PUNCT
spir-13991	28	1	such	such	ADJ
spir-13991	28	2	competition	competition	NOUN
spir-13991	28	3	platforms	platform	NOUN
spir-13991	28	4	participate	participate	VERB
spir-13991	28	5	in	in	ADP
spir-13991	28	6	setting	set	VERB
spir-13991	28	7	the	the	DET
spir-13991	28	8	conditions	condition	NOUN
spir-13991	28	9	for	for	ADP
spir-13991	28	10	evaluation	evaluation	NOUN
spir-13991	28	11	by	by	ADP
spir-13991	28	12	providing	provide	VERB
spir-13991	28	13	a	a	DET
spir-13991	28	14	pre	pre	ADJ
spir-13991	28	15	-	-	ADJ
spir-13991	28	16	specified	specified	ADJ
spir-13991	28	17	list	list	NOUN
spir-13991	28	18	of	of	ADP
spir-13991	28	19	metrics	metric	NOUN
spir-13991	28	20	for	for	ADP
spir-13991	28	21	competitions	competition	NOUN
spir-13991	28	22	hosted	host	VERB
spir-13991	28	23	on	on	ADP
spir-13991	28	24	their	their	PRON
spir-13991	28	25	platforms	platform	NOUN
spir-13991	28	26	.	.	PUNCT
spir-13991	29	1	as	as	SCONJ
spir-13991	29	2	these	these	DET
spir-13991	29	3	competition	competition	NOUN
spir-13991	29	4	platforms	platform	NOUN
spir-13991	29	5	are	be	AUX
spir-13991	29	6	gaining	gain	VERB
spir-13991	29	7	increased	increase	VERB
spir-13991	29	8	popularity	popularity	NOUN
spir-13991	29	9	this	this	PRON
spir-13991	29	10	homogenises	homogenise	VERB
spir-13991	29	11	the	the	DET
spir-13991	29	12	types	type	NOUN
spir-13991	29	13	of	of	ADP
spir-13991	29	14	tasks	task	NOUN
spir-13991	29	15	and	and	CCONJ
spir-13991	29	16	associated	associated	ADJ
spir-13991	29	17	metrics	metric	NOUN
spir-13991	29	18	that	that	PRON
spir-13991	29	19	are	be	AUX
spir-13991	29	20	used	use	VERB
spir-13991	29	21	to	to	PART
spir-13991	29	22	develop	develop	VERB
spir-13991	29	23	new	new	ADJ
spir-13991	29	24	ai	ai	NOUN
spir-13991	29	25	applications	application	NOUN
spir-13991	29	26	,	,	PUNCT
spir-13991	29	27	while	while	SCONJ
spir-13991	29	28	at	at	ADP
spir-13991	29	29	the	the	DET
spir-13991	29	30	same	same	ADJ
spir-13991	29	31	time	time	NOUN
spir-13991	29	32	concentrating	concentrate	VERB
spir-13991	29	33	power	power	NOUN
spir-13991	29	34	amongst	amongst	ADP
spir-13991	29	35	a	a	DET
spir-13991	29	36	small	small	ADJ
spir-13991	29	37	group	group	NOUN
spir-13991	29	38	of	of	ADP
spir-13991	29	39	actors	actor	NOUN
spir-13991	29	40	(	(	PUNCT
spir-13991	29	41	luitse	luitse	VERB
spir-13991	29	42	et	et	PROPN
spir-13991	29	43	al	al	PROPN
spir-13991	29	44	.	.	PROPN
spir-13991	29	45	,	,	PUNCT
spir-13991	29	46	2024	2024	NUM
spir-13991	29	47	)	)	PUNCT
spir-13991	29	48	.	.	PUNCT
spir-13991	30	1	political	political	ADJ
spir-13991	30	2	dynamics	dynamic	NOUN
spir-13991	30	3	of	of	ADP
spir-13991	30	4	circulating	circulate	VERB
spir-13991	30	5	methods	method	NOUN
spir-13991	30	6	of	of	ADP
spir-13991	30	7	evaluation	evaluation	NOUN
spir-13991	30	8	with	with	ADP
spir-13991	30	9	the	the	DET
spir-13991	30	10	second	second	ADJ
spir-13991	30	11	entry	entry	NOUN
spir-13991	30	12	point	point	NOUN
spir-13991	30	13	,	,	PUNCT
spir-13991	30	14	we	we	PRON
spir-13991	30	15	wish	wish	VERB
spir-13991	30	16	to	to	PART
spir-13991	30	17	bring	bring	VERB
spir-13991	30	18	attention	attention	NOUN
spir-13991	30	19	to	to	ADP
spir-13991	30	20	the	the	DET
spir-13991	30	21	role	role	NOUN
spir-13991	30	22	of	of	ADP
spir-13991	30	23	specific	specific	ADJ
spir-13991	30	24	methods	method	NOUN
spir-13991	30	25	of	of	ADP
spir-13991	30	26	evaluation	evaluation	NOUN
spir-13991	30	27	that	that	PRON
spir-13991	30	28	circulate	circulate	VERB
spir-13991	30	29	.	.	PUNCT
spir-13991	31	1	for	for	ADP
spir-13991	31	2	example	example	NOUN
spir-13991	31	3	,	,	PUNCT
spir-13991	31	4	via	via	ADP
spir-13991	31	5	‘	'	PUNCT
spir-13991	31	6	best	good	ADJ
spir-13991	31	7	practice	practice	NOUN
spir-13991	31	8	’	'	PUNCT
spir-13991	31	9	metrics	metric	NOUN
spir-13991	31	10	for	for	ADP
spir-13991	31	11	certain	certain	ADJ
spir-13991	31	12	tasks	task	NOUN
spir-13991	31	13	or	or	CCONJ
spir-13991	31	14	the	the	DET
spir-13991	31	15	functionalities	functionality	NOUN
spir-13991	31	16	of	of	ADP
spir-13991	31	17	ai	ai	PROPN
spir-13991	31	18	development	development	NOUN
spir-13991	31	19	platforms	platform	NOUN
spir-13991	31	20	and	and	CCONJ
spir-13991	31	21	development	development	NOUN
spir-13991	31	22	frameworks	framework	NOUN
spir-13991	31	23	,	,	PUNCT
spir-13991	31	24	such	such	ADJ
spir-13991	31	25	as	as	ADP
spir-13991	31	26	sagemaker	sagemaker	NOUN
spir-13991	31	27	or	or	CCONJ
spir-13991	31	28	tensorflow	tensorflow	NOUN
spir-13991	31	29	.	.	PUNCT
spir-13991	32	1	building	build	VERB
spir-13991	32	2	on	on	ADP
spir-13991	32	3	the	the	DET
spir-13991	32	4	work	work	NOUN
spir-13991	32	5	of	of	ADP
spir-13991	32	6	selbst	selbst	PROPN
spir-13991	32	7	et	et	PROPN
spir-13991	32	8	al	al	PROPN
spir-13991	32	9	.	.	PROPN
spir-13991	33	1	(	(	PUNCT
spir-13991	33	2	2019	2019	NUM
spir-13991	33	3	)	)	PUNCT
spir-13991	33	4	,	,	PUNCT
spir-13991	33	5	we	we	PRON
spir-13991	33	6	argue	argue	VERB
spir-13991	33	7	for	for	ADP
spir-13991	33	8	the	the	DET
spir-13991	33	9	need	need	NOUN
spir-13991	33	10	to	to	PART
spir-13991	33	11	study	study	VERB
spir-13991	33	12	the	the	DET
spir-13991	33	13	‘	'	PUNCT
spir-13991	33	14	portability	portability	NOUN
spir-13991	33	15	’	'	PUNCT
spir-13991	33	16	of	of	ADP
spir-13991	33	17	metrics	metric	NOUN
spir-13991	33	18	into	into	ADP
spir-13991	33	19	different	different	ADJ
spir-13991	33	20	domains	domain	NOUN
spir-13991	33	21	and	and	CCONJ
spir-13991	33	22	how	how	SCONJ
spir-13991	33	23	that	that	PRON
spir-13991	33	24	shapes	shape	VERB
spir-13991	33	25	the	the	DET
spir-13991	33	26	development	development	NOUN
spir-13991	33	27	and	and	CCONJ
spir-13991	33	28	evaluation	evaluation	NOUN
spir-13991	33	29	of	of	ADP
spir-13991	33	30	ai	ai	PROPN
spir-13991	33	31	applications	application	NOUN
spir-13991	33	32	.	.	PUNCT
spir-13991	34	1	luchs	luchs	PROPN
spir-13991	34	2	et	et	PROPN
spir-13991	34	3	al	al	PROPN
spir-13991	34	4	.	.	PROPN
spir-13991	35	1	(	(	PUNCT
spir-13991	35	2	2023	2023	NUM
spir-13991	35	3	)	)	PUNCT
spir-13991	35	4	,	,	PUNCT
spir-13991	35	5	for	for	ADP
spir-13991	35	6	example	example	NOUN
spir-13991	35	7	,	,	PUNCT
spir-13991	35	8	illustrate	illustrate	VERB
spir-13991	35	9	how	how	SCONJ
spir-13991	35	10	machine	machine	NOUN
spir-13991	35	11	learning	learn	VERB
spir-13991	35	12	frameworks	framework	NOUN
spir-13991	35	13	such	such	ADJ
spir-13991	35	14	as	as	ADP
spir-13991	35	15	tensorflow	tensorflow	NOUN
spir-13991	35	16	produce	produce	NOUN
spir-13991	35	17	‘	'	PUNCT
spir-13991	35	18	step	step	NOUN
spir-13991	35	19	-	-	PUNCT
spir-13991	35	20	by	by	ADP
spir-13991	35	21	-	-	PUNCT
spir-13991	35	22	step	step	NOUN
spir-13991	35	23	'	'	PUNCT
spir-13991	35	24	approaches	approach	NOUN
spir-13991	35	25	to	to	PART
spir-13991	35	26	ai	ai	VERB
spir-13991	35	27	development	development	NOUN
spir-13991	35	28	including	include	VERB
spir-13991	35	29	methodologies	methodology	NOUN
spir-13991	35	30	for	for	ADP
spir-13991	35	31	iterative	iterative	NOUN
spir-13991	35	32	and	and	CCONJ
spir-13991	35	33	gradual	gradual	ADJ
spir-13991	35	34	optimisation	optimisation	NOUN
spir-13991	35	35	of	of	ADP
spir-13991	35	36	the	the	DET
spir-13991	35	37	model	model	NOUN
spir-13991	35	38	(	(	PUNCT
spir-13991	35	39	i.e.	i.e.	X
spir-13991	35	40	evaluation	evaluation	NOUN
spir-13991	35	41	)	)	PUNCT
spir-13991	35	42	.	.	PUNCT
spir-13991	36	1	in	in	ADP
spir-13991	36	2	addition	addition	NOUN
spir-13991	36	3	,	,	PUNCT
spir-13991	36	4	developers	developer	NOUN
spir-13991	36	5	at	at	ADP
spir-13991	36	6	the	the	DET
spir-13991	36	7	dutch	dutch	PROPN
spir-13991	36	8	medical	medical	ADJ
spir-13991	36	9	centre	centre	NOUN
spir-13991	36	10	,	,	PUNCT
spir-13991	36	11	highlighted	highlight	VERB
spir-13991	36	12	that	that	SCONJ
spir-13991	36	13	the	the	DET
spir-13991	36	14	commonly	commonly	ADV
spir-13991	36	15	used	use	VERB
spir-13991	36	16	developer	developer	NOUN
spir-13991	36	17	framework	framework	NOUN
spir-13991	36	18	pytorch	pytorch	NOUN
spir-13991	36	19	comes	come	VERB
spir-13991	36	20	with	with	ADP
spir-13991	36	21	specific	specific	ADJ
spir-13991	36	22	machine	machine	NOUN
spir-13991	36	23	-	-	PUNCT
spir-13991	36	24	learning	learn	VERB
spir-13991	36	25	pipeline	pipeline	NOUN
spir-13991	36	26	requirements	requirement	NOUN
spir-13991	36	27	,	,	PUNCT
spir-13991	36	28	which	which	PRON
spir-13991	36	29	again	again	ADV
spir-13991	36	30	shapes	shape	VERB
spir-13991	36	31	how	how	SCONJ
spir-13991	36	32	the	the	DET
spir-13991	36	33	application	application	NOUN
spir-13991	36	34	is	be	AUX
spir-13991	36	35	developed	develop	VERB
spir-13991	36	36	and	and	CCONJ
spir-13991	36	37	evaluated	evaluate	VERB
spir-13991	36	38	.	.	PUNCT
spir-13991	37	1	the	the	DET
spir-13991	37	2	political	political	ADJ
spir-13991	37	3	dynamics	dynamic	NOUN
spir-13991	37	4	of	of	ADP
spir-13991	37	5	situated	situated	ADJ
spir-13991	37	6	enactments	enactment	NOUN
spir-13991	37	7	of	of	ADP
spir-13991	37	8	evaluation	evaluation	NOUN
spir-13991	37	9	with	with	ADP
spir-13991	37	10	the	the	DET
spir-13991	37	11	third	third	ADJ
spir-13991	37	12	entry	entry	NOUN
spir-13991	37	13	point	point	NOUN
spir-13991	37	14	,	,	PUNCT
spir-13991	37	15	we	we	PRON
spir-13991	37	16	follow	follow	VERB
spir-13991	37	17	jaton	jaton	PROPN
spir-13991	37	18	and	and	CCONJ
spir-13991	37	19	sormani	sormani	PROPN
spir-13991	37	20	’s	’s	PART
spir-13991	37	21	(	(	PUNCT
spir-13991	37	22	2023	2023	NUM
spir-13991	37	23	)	)	PUNCT
spir-13991	37	24	recent	recent	ADJ
spir-13991	37	25	call	call	NOUN
spir-13991	37	26	for	for	ADP
spir-13991	37	27	situated	situated	ADJ
spir-13991	37	28	accounts	account	NOUN
spir-13991	37	29	of	of	ADP
spir-13991	37	30	ai	ai	ADJ
spir-13991	37	31	development	development	NOUN
spir-13991	37	32	and	and	CCONJ
spir-13991	37	33	highlight	highlight	VERB
spir-13991	37	34	the	the	DET
spir-13991	37	35	need	need	NOUN
spir-13991	37	36	to	to	PART
spir-13991	37	37	study	study	VERB
spir-13991	37	38	the	the	DET
spir-13991	37	39	localised	localised	ADJ
spir-13991	37	40	enactments	enactment	NOUN
spir-13991	37	41	of	of	ADP
spir-13991	37	42	evaluation	evaluation	NOUN
spir-13991	37	43	.	.	PUNCT
spir-13991	38	1	such	such	ADJ
spir-13991	38	2	concrete	concrete	ADJ
spir-13991	38	3	case	case	NOUN
spir-13991	38	4	studies	study	NOUN
spir-13991	38	5	(	(	PUNCT
spir-13991	38	6	flyvbjerg	flyvbjerg	PROPN
spir-13991	38	7	,	,	PUNCT
spir-13991	38	8	2011	2011	NUM
spir-13991	38	9	)	)	PUNCT
spir-13991	38	10	can	can	AUX
spir-13991	38	11	help	help	VERB
spir-13991	38	12	illustrate	illustrate	VERB
spir-13991	38	13	the	the	DET
spir-13991	38	14	tensions	tension	NOUN
spir-13991	38	15	between	between	ADP
spir-13991	38	16	metrics	metric	NOUN
spir-13991	38	17	and	and	CCONJ
spir-13991	38	18	contextual	contextual	ADJ
spir-13991	38	19	understandings	understanding	NOUN
spir-13991	38	20	of	of	ADP
spir-13991	38	21	what	what	PRON
spir-13991	38	22	makes	make	VERB
spir-13991	38	23	the	the	DET
spir-13991	38	24	system	system	NOUN
spir-13991	38	25	‘	'	PUNCT
spir-13991	38	26	good	good	ADJ
spir-13991	38	27	enough	enough	ADV
spir-13991	38	28	’	'	PUNCT
spir-13991	38	29	to	to	PART
spir-13991	38	30	be	be	AUX
spir-13991	38	31	deployed	deploy	VERB
spir-13991	38	32	.	.	PUNCT
spir-13991	39	1	schjøtt	schjøtt	NOUN
spir-13991	39	2	and	and	CCONJ
spir-13991	39	3	birkbak	birkbak	NOUN
spir-13991	39	4	(	(	PUNCT
spir-13991	39	5	2022	2022	NUM
spir-13991	39	6	)	)	PUNCT
spir-13991	39	7	,	,	PUNCT
spir-13991	39	8	for	for	ADP
spir-13991	39	9	example	example	NOUN
spir-13991	39	10	,	,	PUNCT
spir-13991	39	11	illustrate	illustrate	VERB
spir-13991	39	12	that	that	SCONJ
spir-13991	39	13	the	the	DET
spir-13991	39	14	failure	failure	NOUN
spir-13991	39	15	of	of	ADP
spir-13991	39	16	a	a	DET
spir-13991	39	17	recommender	recommender	NOUN
spir-13991	39	18	system	system	NOUN
spir-13991	39	19	for	for	ADP
spir-13991	39	20	news	news	NOUN
spir-13991	39	21	could	could	AUX
spir-13991	39	22	not	not	PART
spir-13991	39	23	solely	solely	ADV
spir-13991	39	24	be	be	AUX
spir-13991	39	25	ascribed	ascribe	VERB
spir-13991	39	26	to	to	ADP
spir-13991	39	27	its	its	PRON
spir-13991	39	28	lack	lack	NOUN
spir-13991	39	29	of	of	ADP
spir-13991	39	30	providing	provide	VERB
spir-13991	39	31	accurate	accurate	ADJ
spir-13991	39	32	predictions	prediction	NOUN
spir-13991	39	33	,	,	PUNCT
spir-13991	39	34	but	but	CCONJ
spir-13991	39	35	rather	rather	ADV
spir-13991	39	36	the	the	DET
spir-13991	39	37	opposite	opposite	NOUN
spir-13991	39	38	.	.	PUNCT
spir-13991	40	1	instead	instead	ADV
spir-13991	40	2	,	,	PUNCT
spir-13991	40	3	it	it	PRON
spir-13991	40	4	was	be	AUX
spir-13991	40	5	the	the	DET
spir-13991	40	6	editor	editor	NOUN
spir-13991	40	7	’s	’s	PART
spir-13991	40	8	need	need	NOUN
spir-13991	40	9	to	to	PART
spir-13991	40	10	protect	protect	VERB
spir-13991	40	11	the	the	DET
spir-13991	40	12	public	public	ADJ
spir-13991	40	13	interest	interest	NOUN
spir-13991	40	14	that	that	PRON
spir-13991	40	15	ultimately	ultimately	ADV
spir-13991	40	16	led	lead	VERB
spir-13991	40	17	to	to	ADP
spir-13991	40	18	the	the	DET
spir-13991	40	19	decision	decision	NOUN
spir-13991	40	20	not	not	PART
spir-13991	40	21	to	to	PART
spir-13991	40	22	deploy	deploy	VERB
spir-13991	40	23	it	it	PRON
spir-13991	40	24	on	on	ADP
spir-13991	40	25	the	the	DET
spir-13991	40	26	news	news	NOUN
spir-13991	40	27	sites	site	NOUN
spir-13991	40	28	.	.	PUNCT
spir-13991	41	1	the	the	DET
spir-13991	41	2	fieldwork	fieldwork	NOUN
spir-13991	41	3	at	at	ADP
spir-13991	41	4	the	the	DET
spir-13991	41	5	bbc	bbc	PROPN
spir-13991	41	6	also	also	ADV
spir-13991	41	7	illustrated	illustrate	VERB
spir-13991	41	8	how	how	SCONJ
spir-13991	41	9	many	many	ADJ
spir-13991	41	10	local	local	ADJ
spir-13991	41	11	methods	method	NOUN
spir-13991	41	12	of	of	ADP
spir-13991	41	13	evaluation	evaluation	NOUN
spir-13991	41	14	were	be	AUX
spir-13991	41	15	put	put	VERB
spir-13991	41	16	in	in	ADP
spir-13991	41	17	place	place	NOUN
spir-13991	41	18	,	,	PUNCT
spir-13991	41	19	such	such	ADJ
spir-13991	41	20	as	as	ADP
spir-13991	41	21	visualisation	visualisation	NOUN
spir-13991	41	22	tools	tool	NOUN
spir-13991	41	23	,	,	PUNCT
spir-13991	41	24	to	to	PART
spir-13991	41	25	enable	enable	VERB
spir-13991	41	26	qualitative	qualitative	ADJ
spir-13991	41	27	feedback	feedback	NOUN
spir-13991	41	28	from	from	ADP
spir-13991	41	29	editorial	editorial	ADJ
spir-13991	41	30	staff	staff	NOUN
spir-13991	41	31	.	.	PUNCT
spir-13991	42	1	however	however	ADV
spir-13991	42	2	,	,	PUNCT
spir-13991	42	3	these	these	DET
spir-13991	42	4	tools	tool	NOUN
spir-13991	42	5	also	also	ADV
spir-13991	42	6	configure	configure	VERB
spir-13991	42	7	the	the	DET
spir-13991	42	8	development	development	NOUN
spir-13991	42	9	of	of	ADP
spir-13991	42	10	ai	ai	PROPN
spir-13991	42	11	systems	system	NOUN
spir-13991	42	12	by	by	ADP
spir-13991	42	13	making	make	VERB
spir-13991	42	14	certain	certain	ADJ
spir-13991	42	15	features	feature	NOUN
spir-13991	42	16	of	of	ADP
spir-13991	42	17	the	the	DET
spir-13991	42	18	system	system	NOUN
spir-13991	42	19	salient	salient	NOUN
spir-13991	42	20	,	,	PUNCT
spir-13991	42	21	while	while	SCONJ
spir-13991	42	22	also	also	ADV
spir-13991	42	23	abstracting	abstract	VERB
spir-13991	42	24	away	away	ADP
spir-13991	42	25	other	other	ADJ
spir-13991	42	26	ways	way	NOUN
spir-13991	42	27	of	of	ADP
spir-13991	42	28	understanding	understand	VERB
spir-13991	42	29	the	the	DET
spir-13991	42	30	system	system	NOUN
spir-13991	42	31	.	.	PUNCT
spir-13991	43	1	references	reference	NOUN
spir-13991	43	2	amoore	amoore	VERB
spir-13991	43	3	l	l	NOUN
spir-13991	43	4	(	(	PUNCT
spir-13991	43	5	2020	2020	NUM
spir-13991	43	6	)	)	PUNCT
spir-13991	43	7	cloud	cloud	NOUN
spir-13991	43	8	ethics	ethic	NOUN
spir-13991	43	9	:	:	PUNCT
spir-13991	43	10	algorithms	algorithm	NOUN
spir-13991	43	11	and	and	CCONJ
spir-13991	43	12	the	the	DET
spir-13991	43	13	attributes	attribute	NOUN
spir-13991	43	14	of	of	ADP
spir-13991	43	15	ourselves	ourselves	PRON
spir-13991	43	16	and	and	CCONJ
spir-13991	43	17	others	other	NOUN
spir-13991	43	18	.	.	PUNCT
spir-13991	44	1	duke	duke	PROPN
spir-13991	44	2	university	university	PROPN
spir-13991	44	3	press	press	NOUN
spir-13991	44	4	.	.	PUNCT
spir-13991	45	1	aradau	aradau	NOUN
spir-13991	45	2	c	c	PROPN
spir-13991	45	3	and	and	CCONJ
spir-13991	45	4	blanke	blanke	PROPN
spir-13991	45	5	t	t	PROPN
spir-13991	45	6	(	(	PUNCT
spir-13991	45	7	2021	2021	NUM
spir-13991	45	8	)	)	PUNCT
spir-13991	45	9	algorithmic	algorithmic	ADJ
spir-13991	45	10	surveillance	surveillance	NOUN
spir-13991	45	11	and	and	CCONJ
spir-13991	45	12	the	the	DET
spir-13991	45	13	political	political	ADJ
spir-13991	45	14	life	life	NOUN
spir-13991	45	15	of	of	ADP
spir-13991	45	16	error	error	NOUN
spir-13991	45	17	.	.	PUNCT
spir-13991	46	1	journal	journal	PROPN
spir-13991	46	2	for	for	ADP
spir-13991	46	3	the	the	DET
spir-13991	46	4	history	history	NOUN
spir-13991	46	5	of	of	ADP
spir-13991	46	6	knowledge	knowledge	NOUN
spir-13991	46	7	2(1	2(1	NUM
spir-13991	46	8	)	)	PUNCT
spir-13991	46	9	.	.	PUNCT
spir-13991	47	1	1	1	NUM
spir-13991	47	2	:	:	SYM
spir-13991	47	3	10–10	10–10	NUM
spir-13991	47	4	.	.	PUNCT
spir-13991	48	1	flyvbjerg	flyvbjerg	PROPN
spir-13991	48	2	b	b	PROPN
spir-13991	48	3	(	(	PUNCT
spir-13991	48	4	2011	2011	NUM
spir-13991	48	5	)	)	PUNCT
spir-13991	48	6	‘	'	PUNCT
spir-13991	48	7	case	case	NOUN
spir-13991	48	8	study	study	NOUN
spir-13991	48	9	’	'	PUNCT
spir-13991	48	10	,	,	PUNCT
spir-13991	48	11	in	in	ADP
spir-13991	48	12	n.k	n.k	PROPN
spir-13991	48	13	.	.	PROPN
spir-13991	48	14	denzin	denzin	PROPN
spir-13991	48	15	and	and	CCONJ
spir-13991	48	16	y.s	y.s	PROPN
spir-13991	48	17	.	.	PROPN
spir-13991	48	18	lincoln	lincoln	PROPN
spir-13991	48	19	(	(	PUNCT
spir-13991	48	20	eds	eds	PROPN
spir-13991	48	21	)	)	PUNCT
spir-13991	48	22	the	the	DET
spir-13991	48	23	sage	sage	NOUN
spir-13991	48	24	handbook	handbook	NOUN
spir-13991	48	25	of	of	ADP
spir-13991	48	26	qualitative	qualitative	ADJ
spir-13991	48	27	research	research	NOUN
spir-13991	48	28	.	.	PUNCT
spir-13991	49	1	thousand	thousand	NUM
spir-13991	49	2	oaks	oak	NOUN
spir-13991	49	3	:	:	PUNCT
spir-13991	49	4	sage	sage	NOUN
spir-13991	49	5	,	,	PUNCT
spir-13991	49	6	pp	pp	ADJ
spir-13991	49	7	.	.	PUNCT
spir-13991	50	1	301–316	301–316	NUM
spir-13991	50	2	.	.	NOUN
spir-13991	50	3	grill	grill	NOUN
spir-13991	50	4	g	g	PROPN
spir-13991	50	5	(	(	PUNCT
spir-13991	50	6	2022	2022	NUM
spir-13991	50	7	)	)	PUNCT
spir-13991	50	8	constructing	construct	VERB
spir-13991	50	9	certainty	certainty	NOUN
spir-13991	50	10	in	in	ADP
spir-13991	50	11	machine	machine	NOUN
spir-13991	50	12	learning	learning	NOUN
spir-13991	50	13	:	:	PUNCT
spir-13991	50	14	on	on	ADP
spir-13991	50	15	the	the	DET
spir-13991	50	16	performativity	performativity	NOUN
spir-13991	50	17	of	of	ADP
spir-13991	50	18	testing	testing	NOUN
spir-13991	50	19	and	and	CCONJ
spir-13991	50	20	its	its	PRON
spir-13991	50	21	hold	hold	NOUN
spir-13991	50	22	on	on	ADP
spir-13991	50	23	the	the	DET
spir-13991	50	24	future	future	NOUN
spir-13991	50	25	.	.	PUNCT
spir-13991	51	1	osf	osf	NOUN
spir-13991	51	2	preprints	preprint	NOUN
spir-13991	51	3	.	.	PUNCT
spir-13991	52	1	available	available	ADJ
spir-13991	52	2	from	from	ADP
spir-13991	52	3	:	:	PUNCT
spir-13991	52	4	osf.io/zekqv	osf.io/zekqv	NOUN
spir-13991	52	5	.	.	PUNCT
spir-13991	53	1	jaton	jaton	PROPN
spir-13991	53	2	f	f	PROPN
spir-13991	53	3	(	(	PUNCT
spir-13991	53	4	2017	2017	NUM
spir-13991	53	5	)	)	PUNCT
spir-13991	53	6	we	we	PRON
spir-13991	53	7	get	get	VERB
spir-13991	53	8	the	the	DET
spir-13991	53	9	algorithms	algorithm	NOUN
spir-13991	53	10	of	of	ADP
spir-13991	53	11	our	our	PRON
spir-13991	53	12	ground	ground	NOUN
spir-13991	53	13	truths	truth	NOUN
spir-13991	53	14	:	:	PUNCT
spir-13991	53	15	designing	design	VERB
spir-13991	53	16	referential	referential	ADJ
spir-13991	53	17	databases	database	NOUN
spir-13991	53	18	in	in	ADP
spir-13991	53	19	digital	digital	ADJ
spir-13991	53	20	image	image	NOUN
spir-13991	53	21	processing	processing	NOUN
spir-13991	53	22	.	.	PUNCT
spir-13991	54	1	social	social	ADJ
spir-13991	54	2	studies	study	NOUN
spir-13991	54	3	of	of	ADP
spir-13991	54	4	science	science	NOUN
spir-13991	54	5	47(6	47(6	NUM
spir-13991	54	6	):	):	PUNCT
spir-13991	54	7	811–840	811–840	NUM
spir-13991	54	8	.	.	PUNCT
spir-13991	55	1	jaton	jaton	PROPN
spir-13991	55	2	f	f	PROPN
spir-13991	55	3	and	and	CCONJ
spir-13991	55	4	sormani	sormani	PROPN
spir-13991	55	5	p	p	PROPN
spir-13991	55	6	(	(	PUNCT
spir-13991	55	7	2023	2023	NUM
spir-13991	55	8	)	)	PUNCT
spir-13991	55	9	enabling	enable	VERB
spir-13991	55	10	‘	'	PUNCT
spir-13991	55	11	ai	ai	VERB
spir-13991	55	12	’	'	PUNCT
spir-13991	55	13	?	?	PUNCT
spir-13991	56	1	the	the	DET
spir-13991	56	2	situated	situated	ADJ
spir-13991	56	3	production	production	NOUN
spir-13991	56	4	of	of	ADP
spir-13991	56	5	commensurabilities	commensurabilitie	NOUN
spir-13991	56	6	.	.	PUNCT
spir-13991	57	1	social	social	ADJ
spir-13991	57	2	studies	study	NOUN
spir-13991	57	3	of	of	ADP
spir-13991	57	4	science	science	NOUN
spir-13991	57	5	53(5	53(5	NUM
spir-13991	57	6	):	):	PUNCT
spir-13991	57	7	625–634	625–634	NUM
spir-13991	57	8	.	.	PUNCT
spir-13991	57	9	marres	marre	NOUN
spir-13991	57	10	n	n	CCONJ
spir-13991	57	11	and	and	CCONJ
spir-13991	57	12	stark	stark	ADJ
spir-13991	57	13	d	d	PROPN
spir-13991	57	14	(	(	PUNCT
spir-13991	57	15	2020	2020	NUM
spir-13991	57	16	)	)	PUNCT
spir-13991	57	17	put	put	VERB
spir-13991	57	18	to	to	ADP
spir-13991	57	19	the	the	DET
spir-13991	57	20	test	test	NOUN
spir-13991	57	21	:	:	PUNCT
spir-13991	57	22	for	for	ADP
spir-13991	57	23	a	a	DET
spir-13991	57	24	new	new	ADJ
spir-13991	57	25	sociology	sociology	NOUN
spir-13991	57	26	of	of	ADP
spir-13991	57	27	testing	testing	NOUN
spir-13991	57	28	.	.	PUNCT
spir-13991	58	1	the	the	DET
spir-13991	58	2	british	british	ADJ
spir-13991	58	3	journal	journal	PROPN
spir-13991	58	4	of	of	ADP
spir-13991	58	5	sociology	sociology	NOUN
spir-13991	58	6	71(3	71(3	NUM
spir-13991	58	7	):	):	PUNCT
spir-13991	58	8	423–443	423–443	NUM
spir-13991	58	9	.	.	PUNCT
spir-13991	59	1	miceli	miceli	PROPN
spir-13991	59	2	m	m	PROPN
spir-13991	59	3	and	and	CCONJ
spir-13991	59	4	posada	posada	PROPN
spir-13991	59	5	j	j	PROPN
spir-13991	59	6	(	(	PUNCT
spir-13991	59	7	2021	2021	NUM
spir-13991	59	8	)	)	PUNCT
spir-13991	59	9	wisdom	wisdom	NOUN
spir-13991	59	10	for	for	ADP
spir-13991	59	11	the	the	DET
spir-13991	59	12	crowd	crowd	NOUN
spir-13991	59	13	:	:	PUNCT
spir-13991	59	14	discoursive	discoursive	ADJ
spir-13991	59	15	power	power	NOUN
spir-13991	59	16	in	in	ADP
spir-13991	59	17	annotation	annotation	NOUN
spir-13991	59	18	instructions	instruction	NOUN
spir-13991	59	19	for	for	ADP
spir-13991	59	20	computer	computer	NOUN
spir-13991	59	21	vision	vision	NOUN
spir-13991	59	22	.	.	PUNCT
spir-13991	60	1	arxiv:2105.10990	arxiv:2105.10990	PROPN
spir-13991	60	2	.	.	PUNCT
spir-13991	61	1	arxiv	arxiv	PROPN
spir-13991	61	2	.	.	PUNCT
spir-13991	61	3	available	available	ADJ
spir-13991	61	4	at	at	ADP
spir-13991	61	5	:	:	PUNCT
spir-13991	61	6	http://arxiv.org/abs/2105.10990	http://arxiv.org/abs/2105.10990	PROPN
spir-13991	61	7	(	(	PUNCT
spir-13991	61	8	accessed	access	VERB
spir-13991	61	9	26	26	NUM
spir-13991	61	10	february	february	NOUN
spir-13991	61	11	2024	2024	NUM
spir-13991	61	12	)	)	PUNCT
spir-13991	61	13	.	.	PUNCT
spir-13991	62	1	miceli	miceli	PROPN
spir-13991	62	2	m	m	PROPN
spir-13991	62	3	,	,	PUNCT
spir-13991	62	4	yang	yang	PROPN
spir-13991	62	5	t	t	PROPN
spir-13991	62	6	,	,	PUNCT
spir-13991	62	7	naudts	naudts	ADP
spir-13991	62	8	l	l	PROPN
spir-13991	62	9	,	,	PUNCT
spir-13991	62	10	et	et	PROPN
spir-13991	62	11	al	al	PROPN
spir-13991	62	12	.	.	PROPN
spir-13991	63	1	(	(	PUNCT
spir-13991	63	2	2021	2021	NUM
spir-13991	63	3	)	)	PUNCT
spir-13991	63	4	documenting	document	VERB
spir-13991	63	5	computer	computer	NOUN
spir-13991	63	6	vision	vision	NOUN
spir-13991	63	7	datasets	dataset	NOUN
spir-13991	63	8	:	:	PUNCT
spir-13991	63	9	an	an	DET
spir-13991	63	10	invitation	invitation	NOUN
spir-13991	63	11	to	to	PART
spir-13991	63	12	reflexive	reflexive	VERB
spir-13991	63	13	data	datum	NOUN
spir-13991	63	14	practices	practice	NOUN
spir-13991	63	15	.	.	PUNCT
spir-13991	64	1	in	in	ADP
spir-13991	64	2	:	:	PUNCT
spir-13991	64	3	proceedings	proceeding	NOUN
spir-13991	64	4	of	of	ADP
spir-13991	64	5	the	the	DET
spir-13991	64	6	2021	2021	NUM
spir-13991	64	7	acm	acm	NOUN
spir-13991	64	8	conference	conference	NOUN
spir-13991	64	9	on	on	ADP
spir-13991	64	10	fairness	fairness	NOUN
spir-13991	64	11	,	,	PUNCT
spir-13991	64	12	accountability	accountability	NOUN
spir-13991	64	13	,	,	PUNCT
spir-13991	64	14	and	and	CCONJ
spir-13991	64	15	transparency	transparency	NOUN
spir-13991	64	16	,	,	PUNCT
spir-13991	64	17	virtual	virtual	ADJ
spir-13991	64	18	event	event	NOUN
spir-13991	64	19	canada	canada	PROPN
spir-13991	64	20	,	,	PUNCT
spir-13991	64	21	3	3	NUM
spir-13991	64	22	march	march	NOUN
spir-13991	64	23	2021	2021	NUM
spir-13991	64	24	,	,	PUNCT
spir-13991	64	25	pp	pp	ADP
spir-13991	64	26	.	.	PUNCT
spir-13991	65	1	161–172	161–172	NUM
spir-13991	65	2	.	.	PUNCT
spir-13991	66	1	acm	acm	PROPN
spir-13991	66	2	.	.	PUNCT
spir-13991	66	3	available	available	ADJ
spir-13991	66	4	at	at	ADP
spir-13991	66	5	:	:	PUNCT
spir-13991	66	6	https://dl.acm.org/doi/10.1145/3442188.3445880	https://dl.acm.org/doi/10.1145/3442188.3445880	NUM
spir-13991	66	7	luchs	luchs	ADJ
spir-13991	66	8	i	i	PRON
spir-13991	66	9	,	,	PUNCT
spir-13991	66	10	apprich	apprich	PROPN
spir-13991	66	11	c	c	PROPN
spir-13991	66	12	and	and	CCONJ
spir-13991	66	13	broersma	broersma	PROPN
spir-13991	66	14	m	m	PROPN
spir-13991	66	15	(	(	PUNCT
spir-13991	66	16	2023	2023	NUM
spir-13991	66	17	)	)	PUNCT
spir-13991	66	18	learning	learn	VERB
spir-13991	66	19	machine	machine	NOUN
spir-13991	66	20	learning	learning	NOUN
spir-13991	66	21	:	:	PUNCT
spir-13991	66	22	on	on	ADP
spir-13991	66	23	the	the	DET
spir-13991	66	24	political	political	ADJ
spir-13991	66	25	economy	economy	NOUN
spir-13991	66	26	of	of	ADP
spir-13991	66	27	big	big	ADJ
spir-13991	66	28	tech	tech	NOUN
spir-13991	66	29	’s	’s	PART
spir-13991	66	30	online	online	ADJ
spir-13991	66	31	ai	ai	PROPN
spir-13991	66	32	courses	course	NOUN
spir-13991	66	33	.	.	PUNCT
spir-13991	67	1	big	big	ADJ
spir-13991	67	2	data	datum	NOUN
spir-13991	67	3	&	&	CCONJ
spir-13991	67	4	society	society	NOUN
spir-13991	67	5	10(1	10(1	NUM
spir-13991	67	6	)	)	PUNCT
spir-13991	67	7	.	.	PUNCT
spir-13991	68	1	doi	doi	NOUN
spir-13991	68	2	:	:	PUNCT
spir-13991	68	3	10.1177/20539517231153806	10.1177/20539517231153806	NUM
spir-13991	68	4	.	.	PUNCT
spir-13991	69	1	luitse	luitse	PROPN
spir-13991	69	2	d	d	PROPN
spir-13991	69	3	,	,	PUNCT
spir-13991	69	4	blanke	blanke	PROPN
spir-13991	69	5	t	t	PROPN
spir-13991	69	6	,	,	PUNCT
spir-13991	69	7	and	and	CCONJ
spir-13991	69	8	poell	poell	NOUN
spir-13991	69	9	t.	t.	PROPN
spir-13991	69	10	(	(	PUNCT
spir-13991	69	11	2024	2024	NUM
spir-13991	69	12	)	)	PUNCT
spir-13991	69	13	.	.	PUNCT
spir-13991	70	1	ai	ai	VERB
spir-13991	70	2	competitions	competition	NOUN
spir-13991	70	3	as	as	ADP
spir-13991	70	4	infrastructures	infrastructure	NOUN
spir-13991	70	5	of	of	ADP
spir-13991	70	6	power	power	NOUN
spir-13991	70	7	in	in	ADP
spir-13991	70	8	medical	medical	ADJ
spir-13991	70	9	imaging	imaging	NOUN
spir-13991	70	10	.	.	PUNCT
spir-13991	71	1	information	information	NOUN
spir-13991	71	2	,	,	PUNCT
spir-13991	71	3	communication	communication	NOUN
spir-13991	71	4	&	&	CCONJ
spir-13991	71	5	society	society	NOUN
spir-13991	71	6	,	,	PUNCT
spir-13991	71	7	1–22	1–22	PROPN
spir-13991	71	8	.	.	PUNCT
spir-13991	72	1	https://doi.org/10.1080/1369118x.2024.2334393	https://doi.org/10.1080/1369118x.2024.2334393	AUX
spir-13991	72	2	http://arxiv.org/abs/2105.10990	http://arxiv.org/abs/2105.10990	VERB
spir-13991	72	3	https://dl.acm.org/doi/10.1145/3442188.3445880	https://dl.acm.org/doi/10.1145/3442188.3445880	PROPN
spir-13991	72	4	https://doi.org/10.1177/20539517231153806	https://doi.org/10.1177/20539517231153806	NOUN
spir-13991	72	5	https://doi.org/10.1080/1369118x.2024.2334393	https://doi.org/10.1080/1369118x.2024.2334393	PROPN
spir-13991	72	6	orr	orr	PROPN
spir-13991	72	7	w	w	PROPN
spir-13991	72	8	and	and	CCONJ
spir-13991	72	9	crawford	crawford	PROPN
spir-13991	72	10	k	k	PROPN
spir-13991	72	11	(	(	PUNCT
spir-13991	72	12	2023	2023	NUM
spir-13991	72	13	)	)	PUNCT
spir-13991	72	14	the	the	DET
spir-13991	72	15	social	social	ADJ
spir-13991	72	16	construction	construction	NOUN
spir-13991	72	17	of	of	ADP
spir-13991	72	18	datasets	dataset	NOUN
spir-13991	72	19	:	:	PUNCT
spir-13991	72	20	on	on	ADP
spir-13991	72	21	the	the	DET
spir-13991	72	22	practices	practice	NOUN
spir-13991	72	23	,	,	PUNCT
spir-13991	72	24	processes	process	NOUN
spir-13991	72	25	and	and	CCONJ
spir-13991	72	26	challenges	challenge	NOUN
spir-13991	72	27	of	of	ADP
spir-13991	72	28	dataset	dataset	ADJ
spir-13991	72	29	creation	creation	NOUN
spir-13991	72	30	for	for	ADP
spir-13991	72	31	machine	machine	NOUN
spir-13991	72	32	learning	learning	NOUN
spir-13991	72	33	.	.	PUNCT
spir-13991	73	1	socarxiv	socarxiv	PROPN
spir-13991	73	2	.	.	PUNCT
spir-13991	74	1	available	available	ADJ
spir-13991	74	2	from	from	ADP
spir-13991	74	3	:	:	PUNCT
spir-13991	74	4	osf.io/preprints/socarxiv/8c9uh	osf.io/preprints/socarxiv/8c9uh	PROPN
spir-13991	74	5	.	.	PUNCT
spir-13991	74	6	paullada	paullada	PROPN
spir-13991	74	7	a	a	PROPN
spir-13991	74	8	,	,	PUNCT
spir-13991	74	9	raji	raji	PROPN
spir-13991	74	10	i	i	PROPN
spir-13991	74	11	d	d	PROPN
spir-13991	74	12	,	,	PUNCT
spir-13991	74	13	bender	bender	NOUN
spir-13991	74	14	em	em	PRON
spir-13991	74	15	,	,	PUNCT
spir-13991	74	16	et	et	PROPN
spir-13991	74	17	al	al	PROPN
spir-13991	74	18	.	.	PROPN
spir-13991	75	1	(	(	PUNCT
spir-13991	75	2	2021	2021	NUM
spir-13991	75	3	)	)	PUNCT
spir-13991	75	4	data	datum	NOUN
spir-13991	75	5	and	and	CCONJ
spir-13991	75	6	its	its	PRON
spir-13991	75	7	(	(	PUNCT
spir-13991	75	8	dis)contents	dis)content	NOUN
spir-13991	75	9	:	:	PUNCT
spir-13991	75	10	a	a	DET
spir-13991	75	11	survey	survey	NOUN
spir-13991	75	12	of	of	ADP
spir-13991	75	13	dataset	dataset	ADJ
spir-13991	75	14	development	development	NOUN
spir-13991	75	15	and	and	CCONJ
spir-13991	75	16	use	use	NOUN
spir-13991	75	17	in	in	ADP
spir-13991	75	18	machine	machine	NOUN
spir-13991	75	19	learning	learn	VERB
spir-13991	75	20	research	research	NOUN
spir-13991	75	21	.	.	PUNCT
spir-13991	76	1	patterns	pattern	NOUN
spir-13991	76	2	2(11	2(11	NUM
spir-13991	76	3	)	)	PUNCT
spir-13991	76	4	.	.	PUNCT
spir-13991	77	1	11	11	NUM
spir-13991	77	2	:	:	SYM
spir-13991	77	3	100336	100336	NUM
spir-13991	77	4	.	.	PUNCT
spir-13991	78	1	doi	doi	NOUN
spir-13991	78	2	:	:	PUNCT
spir-13991	78	3	10.1016	10.1016	NUM
spir-13991	78	4	/	/	SYM
spir-13991	78	5	j.patter.2021.100336	j.patter.2021.100336	PROPN
spir-13991	78	6	.	.	PUNCT
spir-13991	79	1	raji	raji	PROPN
spir-13991	79	2	d	d	PROPN
spir-13991	79	3	,	,	PUNCT
spir-13991	79	4	denton	denton	PROPN
spir-13991	79	5	e	e	PROPN
spir-13991	79	6	,	,	PUNCT
spir-13991	79	7	bender	bender	VERB
spir-13991	79	8	em	em	PRON
spir-13991	79	9	,	,	PUNCT
spir-13991	79	10	et	et	PROPN
spir-13991	79	11	al	al	PROPN
spir-13991	79	12	.	.	PROPN
spir-13991	80	1	(	(	PUNCT
spir-13991	80	2	2021	2021	NUM
spir-13991	80	3	)	)	PUNCT
spir-13991	80	4	ai	ai	VERB
spir-13991	80	5	and	and	CCONJ
spir-13991	80	6	the	the	DET
spir-13991	80	7	everything	everything	PRON
spir-13991	80	8	in	in	ADP
spir-13991	80	9	the	the	DET
spir-13991	80	10	whole	whole	ADJ
spir-13991	80	11	wide	wide	ADJ
spir-13991	80	12	world	world	NOUN
spir-13991	80	13	benchmark	benchmark	NOUN
spir-13991	80	14	.	.	PUNCT
spir-13991	81	1	in	in	ADP
spir-13991	81	2	:	:	PUNCT
spir-13991	81	3	proceedings	proceeding	NOUN
spir-13991	81	4	of	of	ADP
spir-13991	81	5	the	the	DET
spir-13991	81	6	neural	neural	ADJ
spir-13991	81	7	information	information	NOUN
spir-13991	81	8	processing	processing	NOUN
spir-13991	81	9	systems	system	NOUN
spir-13991	81	10	track	track	NOUN
spir-13991	81	11	on	on	ADP
spir-13991	81	12	datasets	dataset	NOUN
spir-13991	81	13	and	and	CCONJ
spir-13991	81	14	benchmarks	benchmark	NOUN
spir-13991	81	15	(	(	PUNCT
spir-13991	81	16	eds	ed	NOUN
spir-13991	81	17	j	j	PROPN
spir-13991	81	18	vanschoren	vanschoren	NOUN
spir-13991	81	19	and	and	CCONJ
spir-13991	81	20	s	s	PROPN
spir-13991	81	21	yeung	yeung	PROPN
spir-13991	81	22	)	)	PUNCT
spir-13991	81	23	,	,	PUNCT
spir-13991	81	24	2021	2021	NUM
spir-13991	81	25	.	.	PUNCT
spir-13991	82	1	curran	curran	NOUN
spir-13991	82	2	.	.	PUNCT
spir-13991	83	1	available	available	ADJ
spir-13991	83	2	at	at	ADP
spir-13991	83	3	:	:	PUNCT
spir-13991	83	4	https://datasets-benchmarksproceedings.neurips.cc/paper_files/paper/2021/file/084b6fbb10729ed4da8c3d3f5a3ae7	https://datasets-benchmarksproceedings.neurips.cc/paper_files/paper/2021/file/084b6fbb10729ed4da8c3d3f5a3ae7	PROPN
spir-13991	83	5	c9-paper-round2.pdf	c9-paper-round2.pdf	NOUN
spir-13991	83	6	.	.	PUNCT
spir-13991	84	1	seaver	seaver	PROPN
spir-13991	84	2	n	n	PROPN
spir-13991	84	3	(	(	PUNCT
spir-13991	84	4	2019	2019	NUM
spir-13991	84	5	)	)	PUNCT
spir-13991	84	6	knowing	know	VERB
spir-13991	84	7	algorithms	algorithm	NOUN
spir-13991	84	8	.	.	PUNCT
spir-13991	85	1	in	in	ADP
spir-13991	85	2	:	:	PUNCT
spir-13991	85	3	knowing	know	VERB
spir-13991	85	4	algorithms	algorithm	NOUN
spir-13991	85	5	.	.	PUNCT
spir-13991	86	1	princeton	princeton	PROPN
spir-13991	86	2	university	university	PROPN
spir-13991	86	3	press	press	NOUN
spir-13991	86	4	,	,	PUNCT
spir-13991	86	5	pp	pp	ADJ
spir-13991	86	6	.	.	PUNCT
spir-13991	87	1	412–422	412–422	NUM
spir-13991	87	2	.	.	PUNCT
spir-13991	87	3	available	available	ADJ
spir-13991	87	4	at	at	ADP
spir-13991	87	5	:	:	PUNCT
spir-13991	87	6	https://www.degruyter.com/document/doi/10.1515/9780691190600-028/html	https://www.degruyter.com/document/doi/10.1515/9780691190600-028/html	PROPN
spir-13991	87	7	.	.	PUNCT
spir-13991	88	1	selbst	selbst	PROPN
spir-13991	88	2	ad	ad	NOUN
spir-13991	88	3	,	,	PUNCT
spir-13991	88	4	boyd	boyd	PROPN
spir-13991	88	5	d	d	PROPN
spir-13991	88	6	,	,	PUNCT
spir-13991	88	7	friedler	friedler	PROPN
spir-13991	88	8	sa	sa	PROPN
spir-13991	88	9	,	,	PUNCT
spir-13991	88	10	et	et	PROPN
spir-13991	88	11	al	al	PROPN
spir-13991	88	12	.	.	PROPN
spir-13991	89	1	(	(	PUNCT
spir-13991	89	2	2019	2019	NUM
spir-13991	89	3	)	)	PUNCT
spir-13991	89	4	fairness	fairness	NOUN
spir-13991	89	5	and	and	CCONJ
spir-13991	89	6	abstraction	abstraction	NOUN
spir-13991	89	7	in	in	ADP
spir-13991	89	8	sociotechnical	sociotechnical	ADJ
spir-13991	89	9	systems	system	NOUN
spir-13991	89	10	.	.	PUNCT
spir-13991	90	1	in	in	ADP
spir-13991	90	2	:	:	PUNCT
spir-13991	90	3	proceedings	proceeding	NOUN
spir-13991	90	4	of	of	ADP
spir-13991	90	5	the	the	DET
spir-13991	90	6	conference	conference	NOUN
spir-13991	90	7	on	on	ADP
spir-13991	90	8	fairness	fairness	NOUN
spir-13991	90	9	,	,	PUNCT
spir-13991	90	10	accountability	accountability	NOUN
spir-13991	90	11	,	,	PUNCT
spir-13991	90	12	and	and	CCONJ
spir-13991	90	13	transparency	transparency	NOUN
spir-13991	90	14	,	,	PUNCT
spir-13991	90	15	atlanta	atlanta	PROPN
spir-13991	90	16	ga	ga	PROPN
spir-13991	90	17	usa	usa	PROPN
spir-13991	90	18	,	,	PUNCT
spir-13991	90	19	29	29	NUM
spir-13991	90	20	january	january	NOUN
spir-13991	90	21	2019	2019	NUM
spir-13991	90	22	,	,	PUNCT
spir-13991	90	23	pp	pp	ADJ
spir-13991	90	24	.	.	PUNCT
spir-13991	91	1	59–68	59–68	NUM
spir-13991	91	2	.	.	X
spir-13991	91	3	acm	acm	PROPN
spir-13991	91	4	.	.	PUNCT
spir-13991	92	1	doi	doi	NOUN
spir-13991	92	2	:	:	PUNCT
spir-13991	92	3	10.1145/3287560.3287598	10.1145/3287560.3287598	NUM
spir-13991	92	4	.	.	PUNCT
spir-13991	93	1	sheikh	sheikh	PROPN
spir-13991	93	2	h	h	PROPN
spir-13991	93	3	,	,	PUNCT
spir-13991	93	4	prins	prins	PROPN
spir-13991	93	5	c	c	PROPN
spir-13991	93	6	and	and	CCONJ
spir-13991	93	7	schrijvers	schrijvers	PROPN
spir-13991	93	8	e	e	X
spir-13991	93	9	(	(	PUNCT
spir-13991	93	10	2023	2023	NUM
spir-13991	93	11	)	)	PUNCT
spir-13991	93	12	ai	ai	AUX
spir-13991	93	13	is	be	AUX
spir-13991	93	14	leaving	leave	VERB
spir-13991	93	15	the	the	DET
spir-13991	93	16	lab	lab	NOUN
spir-13991	93	17	and	and	CCONJ
spir-13991	93	18	entering	enter	VERB
spir-13991	93	19	society	society	NOUN
spir-13991	93	20	.	.	PUNCT
spir-13991	94	1	in	in	ADP
spir-13991	94	2	:	:	PUNCT
spir-13991	94	3	sheikh	sheikh	PROPN
spir-13991	94	4	h	h	PROPN
spir-13991	94	5	,	,	PUNCT
spir-13991	94	6	prins	prin	NOUN
spir-13991	94	7	c	c	NOUN
spir-13991	94	8	,	,	PUNCT
spir-13991	94	9	and	and	CCONJ
spir-13991	94	10	schrijvers	schrijvers	PROPN
spir-13991	94	11	e	e	PROPN
spir-13991	94	12	(	(	PUNCT
spir-13991	94	13	eds	ed	NOUN
spir-13991	94	14	)	)	PUNCT
spir-13991	94	15	mission	mission	NOUN
spir-13991	94	16	ai	ai	VERB
spir-13991	94	17	:	:	PUNCT
spir-13991	94	18	the	the	DET
spir-13991	94	19	new	new	ADJ
spir-13991	94	20	system	system	NOUN
spir-13991	94	21	technology	technology	NOUN
spir-13991	94	22	.	.	PUNCT
spir-13991	95	1	research	research	NOUN
spir-13991	95	2	for	for	ADP
spir-13991	95	3	policy	policy	NOUN
spir-13991	95	4	.	.	PUNCT
spir-13991	96	1	cham	cham	PROPN
spir-13991	96	2	:	:	PUNCT
spir-13991	96	3	springer	springer	NOUN
spir-13991	96	4	international	international	ADJ
spir-13991	96	5	publishing	publishing	NOUN
spir-13991	96	6	,	,	PUNCT
spir-13991	96	7	pp	pp	ADJ
spir-13991	96	8	.	.	PUNCT
spir-13991	97	1	43–84	43–84	NOUN
spir-13991	97	2	.	.	PUNCT
spir-13991	97	3	available	available	ADJ
spir-13991	97	4	at	at	ADP
spir-13991	97	5	:	:	PUNCT
spir-13991	97	6	https://doi.org/10.1007/978-3-031-21448-6_3	https://doi.org/10.1007/978-3-031-21448-6_3	PRON
spir-13991	97	7	(	(	PUNCT
spir-13991	97	8	accessed	access	VERB
spir-13991	97	9	28	28	NUM
spir-13991	97	10	february	february	NOUN
spir-13991	97	11	2024	2024	NUM
spir-13991	97	12	)	)	PUNCT
spir-13991	97	13	.	.	PUNCT
spir-13991	98	1	schjøtt	schjøtt	NOUN
spir-13991	98	2	a	a	DET
spir-13991	98	3	and	and	CCONJ
spir-13991	98	4	birkbak	birkbak	PROPN
spir-13991	98	5	a	a	DET
spir-13991	98	6	(	(	PUNCT
spir-13991	98	7	2022	2022	NUM
spir-13991	98	8	)	)	PUNCT
spir-13991	98	9	you	you	PRON
spir-13991	98	10	can	can	AUX
spir-13991	98	11	not	not	PART
spir-13991	98	12	make	make	VERB
spir-13991	98	13	the	the	DET
spir-13991	98	14	algorithm	algorithm	NOUN
spir-13991	98	15	do	do	VERB
spir-13991	98	16	something	something	PRON
spir-13991	98	17	it	it	PRON
spir-13991	98	18	does	do	AUX
spir-13991	98	19	not	not	PART
spir-13991	98	20	want	want	VERB
spir-13991	98	21	to	to	PART
spir-13991	98	22	do	do	VERB
spir-13991	98	23	:	:	PUNCT
spir-13991	98	24	exploring	explore	VERB
spir-13991	98	25	the	the	DET
spir-13991	98	26	agency	agency	NOUN
spir-13991	98	27	and	and	CCONJ
spir-13991	98	28	power	power	NOUN
spir-13991	98	29	of	of	ADP
spir-13991	98	30	ai	ai	PROPN
spir-13991	98	31	systems	system	NOUN
spir-13991	98	32	in	in	ADP
spir-13991	98	33	practice	practice	NOUN
spir-13991	98	34	.	.	PUNCT
spir-13991	99	1	in	in	ADP
spir-13991	99	2	:	:	PUNCT
spir-13991	99	3	easst	easst	NOUN
spir-13991	99	4	2022	2022	NUM
spir-13991	99	5	,	,	PUNCT
spir-13991	99	6	2022	2022	NUM
spir-13991	99	7	.	.	PUNCT
spir-13991	99	8	available	available	ADJ
spir-13991	99	9	at	at	ADP
spir-13991	99	10	:	:	PUNCT
spir-13991	99	11	https://easst2022.org/programpreliminary7.asp	https://easst2022.org/programpreliminary7.asp	NOUN
spir-13991	99	12	thakkar	thakkar	PROPN
spir-13991	99	13	d	d	PROPN
spir-13991	99	14	,	,	PUNCT
spir-13991	99	15	ismail	ismail	PROPN
spir-13991	99	16	a	a	X
spir-13991	99	17	,	,	PUNCT
spir-13991	99	18	kumar	kumar	PROPN
spir-13991	99	19	p	p	PROPN
spir-13991	99	20	,	,	PUNCT
spir-13991	99	21	et	et	PROPN
spir-13991	99	22	al	al	PROPN
spir-13991	99	23	.	.	PROPN
spir-13991	99	24	(	(	PUNCT
spir-13991	99	25	2022	2022	NUM
spir-13991	99	26	)	)	PUNCT
spir-13991	99	27	when	when	SCONJ
spir-13991	99	28	is	be	AUX
spir-13991	99	29	machine	machine	NOUN
spir-13991	99	30	learning	learn	VERB
spir-13991	99	31	data	datum	NOUN
spir-13991	99	32	good	good	ADJ
spir-13991	99	33	?	?	PUNCT
spir-13991	99	34	:	:	PUNCT
spir-13991	99	35	valuing	value	VERB
spir-13991	99	36	in	in	ADP
spir-13991	99	37	public	public	ADJ
spir-13991	99	38	health	health	NOUN
spir-13991	99	39	datafication	datafication	NOUN
spir-13991	99	40	.	.	PUNCT
spir-13991	100	1	in	in	ADP
spir-13991	100	2	:	:	PUNCT
spir-13991	100	3	chi	chi	PROPN
spir-13991	100	4	conference	conference	NOUN
spir-13991	100	5	on	on	ADP
spir-13991	100	6	human	human	ADJ
spir-13991	100	7	factors	factor	NOUN
spir-13991	100	8	in	in	ADP
spir-13991	100	9	computing	computing	NOUN
spir-13991	100	10	systems	system	NOUN
spir-13991	100	11	,	,	PUNCT
spir-13991	100	12	new	new	PROPN
spir-13991	100	13	orleans	orleans	PROPN
spir-13991	100	14	la	la	PROPN
spir-13991	100	15	usa	usa	PROPN
spir-13991	100	16	,	,	PUNCT
spir-13991	100	17	29	29	NUM
spir-13991	100	18	april	april	PROPN
spir-13991	100	19	2022	2022	NUM
spir-13991	100	20	,	,	PUNCT
spir-13991	100	21	pp	pp	ADP
spir-13991	100	22	.	.	PUNCT
spir-13991	101	1	1–16	1–16	PROPN
spir-13991	101	2	.	.	PUNCT
spir-13991	102	1	acm	acm	PROPN
spir-13991	102	2	.	.	PUNCT
spir-13991	102	3	available	available	ADJ
spir-13991	102	4	at	at	ADP
spir-13991	102	5	:	:	PUNCT
spir-13991	102	6	https://dl.acm.org/doi/10.1145/3491102.3501868	https://dl.acm.org/doi/10.1145/3491102.3501868	PROPN
spir-13991	102	7	.	.	PUNCT
spir-13991	103	1	thomas	thomas	PROPN
spir-13991	103	2	rl	rl	PROPN
spir-13991	103	3	and	and	CCONJ
spir-13991	103	4	uminsky	uminsky	ADJ
spir-13991	103	5	d	d	PROPN
spir-13991	103	6	(	(	PUNCT
spir-13991	103	7	2022	2022	NUM
spir-13991	103	8	)	)	PUNCT
spir-13991	103	9	reliance	reliance	NOUN
spir-13991	103	10	on	on	ADP
spir-13991	103	11	metrics	metric	NOUN
spir-13991	103	12	is	be	AUX
spir-13991	103	13	a	a	DET
spir-13991	103	14	fundamental	fundamental	ADJ
spir-13991	103	15	challenge	challenge	NOUN
spir-13991	103	16	for	for	ADP
spir-13991	103	17	ai	ai	NOUN
spir-13991	103	18	.	.	PUNCT
spir-13991	103	19	patterns	pattern	NOUN
spir-13991	103	20	3(5	3(5	NUM
spir-13991	103	21	)	)	PUNCT
spir-13991	103	22	.	.	PUNCT
spir-13991	104	1	elsevier	elsevier	NOUN
spir-13991	104	2	.	.	PUNCT
spir-13991	105	1	doi	doi	PROPN
spir-13991	105	2	:	:	PUNCT
spir-13991	105	3	10.1016	10.1016	NUM
spir-13991	105	4	/	/	SYM
spir-13991	105	5	j.patter.2022.100476	j.patter.2022.100476	PROPN
spir-13991	105	6	van	van	PROPN
spir-13991	105	7	der	der	PROPN
spir-13991	105	8	vlist	vlist	VERB
spir-13991	105	9	f	f	PROPN
spir-13991	105	10	,	,	PUNCT
spir-13991	105	11	helmond	helmond	ADV
spir-13991	105	12	a	a	X
spir-13991	105	13	,	,	PUNCT
spir-13991	105	14	and	and	CCONJ
spir-13991	105	15	ferrari	ferrari	PROPN
spir-13991	105	16	f.	f.	PROPN
spir-13991	105	17	(	(	PUNCT
spir-13991	105	18	2024	2024	NUM
spir-13991	105	19	)	)	PUNCT
spir-13991	105	20	.	.	PUNCT
spir-13991	106	1	big	big	ADJ
spir-13991	106	2	ai	ai	NOUN
spir-13991	106	3	:	:	PUNCT
spir-13991	106	4	cloud	cloud	ADJ
spir-13991	106	5	infrastructure	infrastructure	NOUN
spir-13991	106	6	dependence	dependence	NOUN
spir-13991	106	7	and	and	CCONJ
spir-13991	106	8	the	the	DET
spir-13991	106	9	industrialisation	industrialisation	NOUN
spir-13991	106	10	of	of	ADP
spir-13991	106	11	artificial	artificial	ADJ
spir-13991	106	12	intelligence	intelligence	NOUN
spir-13991	106	13	.	.	PUNCT
spir-13991	107	1	big	big	ADJ
spir-13991	107	2	data	datum	NOUN
spir-13991	107	3	&	&	CCONJ
spir-13991	107	4	society	society	NOUN
spir-13991	107	5	,	,	PUNCT
spir-13991	107	6	11(1	11(1	NUM
spir-13991	107	7	)	)	PUNCT
spir-13991	107	8	.	.	PUNCT
spir-13991	108	1	https://doi.org/10.1177/20539517241232630	https://doi.org/10.1177/20539517241232630	PROPN
spir-13991	108	2	whittaker	whittaker	PROPN
spir-13991	108	3	m	m	PROPN
spir-13991	108	4	,	,	PUNCT
spir-13991	108	5	crawford	crawford	PROPN
spir-13991	108	6	k	k	PROPN
spir-13991	108	7	,	,	PUNCT
spir-13991	108	8	dobbe	dobbe	NOUN
spir-13991	108	9	r	r	NOUN
spir-13991	108	10	,	,	PUNCT
spir-13991	108	11	et	et	PROPN
spir-13991	108	12	al	al	PROPN
spir-13991	108	13	.	.	PUNCT
spir-13991	109	1	(	(	PUNCT
spir-13991	109	2	2018	2018	NUM
spir-13991	109	3	)	)	PUNCT
spir-13991	109	4	ai	ai	AUX
spir-13991	109	5	now	now	ADV
spir-13991	109	6	report	report	VERB
spir-13991	109	7	2018	2018	NUM
spir-13991	109	8	.	.	PUNCT
spir-13991	110	1	available	available	ADJ
spir-13991	110	2	at	at	ADP
spir-13991	110	3	:	:	PUNCT
spir-13991	110	4	https://ainowinstitute.org/aiareport2018.pdf	https://ainowinstitute.org/aiareport2018.pdf	PROPN
spir-13991	110	5	https://doi.org/10.1016/j.patter.2021.100336	https://doi.org/10.1016/j.patter.2021.100336	ADJ
spir-13991	110	6	https://datasets-benchmarks-proceedings.neurips.cc/paper_files/paper/2021/file/084b6fbb10729ed4da8c3d3f5a3ae7c9-paper-round2.pdf	https://datasets-benchmarks-proceedings.neurips.cc/paper_files/paper/2021/file/084b6fbb10729ed4da8c3d3f5a3ae7c9-paper-round2.pdf	X
spir-13991	110	7	https://datasets-benchmarks-proceedings.neurips.cc/paper_files/paper/2021/file/084b6fbb10729ed4da8c3d3f5a3ae7c9-paper-round2.pdf	https://datasets-benchmarks-proceedings.neurips.cc/paper_files/paper/2021/file/084b6fbb10729ed4da8c3d3f5a3ae7c9-paper-round2.pdf	X
spir-13991	110	8	https://datasets-benchmarks-proceedings.neurips.cc/paper_files/paper/2021/file/084b6fbb10729ed4da8c3d3f5a3ae7c9-paper-round2.pdf	https://datasets-benchmarks-proceedings.neurips.cc/paper_files/paper/2021/file/084b6fbb10729ed4da8c3d3f5a3ae7c9-paper-round2.pdf	PROPN
spir-13991	110	9	https://www.degruyter.com/document/doi/10.1515/9780691190600-028/html	https://www.degruyter.com/document/doi/10.1515/9780691190600-028/html	NOUN
spir-13991	110	10	.	.	PUNCT
spir-13991	111	1	https://doi.org/10.1145/3287560.3287598	https://doi.org/10.1145/3287560.3287598	ADV
spir-13991	111	2	https://doi.org/10.1007/978-3-031-21448-6_3	https://doi.org/10.1007/978-3-031-21448-6_3	PUNCT
spir-13991	111	3	https://easst2022.org/programpreliminary7.asp	https://easst2022.org/programpreliminary7.asp	NOUN
spir-13991	111	4	https://dl.acm.org/doi/10.1145/3491102.3501868	https://dl.acm.org/doi/10.1145/3491102.3501868	X
spir-13991	111	5	https://doi.org/10.1016/j.patter.2022.100476	https://doi.org/10.1016/j.patter.2022.100476	NOUN
spir-13991	111	6	https://doi.org/10.1177/20539517241232630	https://doi.org/10.1177/20539517241232630	NOUN
spir-13991	111	7	https://ainowinstitute.org/aiareport2018.pdf	https://ainowinstitute.org/aiareport2018.pdf	PUNCT