id	sid	tid	token	lemma	pos
fcis-13113	1	1	frontiers	frontier	NOUN
fcis-13113	1	2	in	in	ADP
fcis-13113	1	3	computing	computing	NOUN
fcis-13113	1	4	and	and	CCONJ
fcis-13113	1	5	intelligent	intelligent	ADJ
fcis-13113	1	6	systems	system	NOUN
fcis-13113	1	7	issn	issn	VERB
fcis-13113	1	8	:	:	PUNCT
fcis-13113	1	9	2832	2832	NUM
fcis-13113	1	10	-	-	SYM
fcis-13113	1	11	6024	6024	NUM
fcis-13113	1	12	|	|	NOUN
fcis-13113	1	13	vol	vol	NOUN
fcis-13113	1	14	.	.	PROPN
fcis-13113	2	1	5	5	NUM
fcis-13113	2	2	,	,	PUNCT
fcis-13113	2	3	no	no	INTJ
fcis-13113	2	4	.	.	NOUN
fcis-13113	2	5	2	2	NUM
fcis-13113	2	6	,	,	PUNCT
fcis-13113	2	7	2023	2023	NUM
fcis-13113	2	8	108	108	NUM
fcis-13113	2	9	a	a	DET
fcis-13113	2	10	review	review	NOUN
fcis-13113	2	11	of	of	ADP
fcis-13113	2	12	machine	machine	NOUN
fcis-13113	2	13	translation	translation	NOUN
fcis-13113	2	14	quality	quality	NOUN
fcis-13113	2	15	assessment	assessment	NOUN
fcis-13113	2	16	methods	method	NOUN
fcis-13113	2	17	wenbo	wenbo	PROPN
fcis-13113	2	18	wang	wang	PROPN
fcis-13113	2	19	*	*	PUNCT
fcis-13113	2	20	school	school	NOUN
fcis-13113	2	21	of	of	ADP
fcis-13113	2	22	computing	computing	NOUN
fcis-13113	2	23	,	,	PUNCT
fcis-13113	2	24	central	central	ADJ
fcis-13113	2	25	china	china	PROPN
fcis-13113	2	26	normal	normal	ADJ
fcis-13113	2	27	university	university	PROPN
fcis-13113	2	28	,	,	PUNCT
fcis-13113	2	29	wuhan	wuhan	PROPN
fcis-13113	2	30	,	,	PUNCT
fcis-13113	2	31	hubei	hubei	PROPN
fcis-13113	2	32	430079	430079	NUM
fcis-13113	2	33	,	,	PUNCT
fcis-13113	2	34	china	china	PROPN
fcis-13113	2	35	*	*	PUNCT
fcis-13113	2	36	corresponding	correspond	VERB
fcis-13113	2	37	author	author	NOUN
fcis-13113	2	38	email	email	NOUN
fcis-13113	2	39	:	:	PUNCT
fcis-13113	2	40	xiaobo.0412@outlook.com	xiaobo.0412@outlook.com	NUM
fcis-13113	2	41	abstract	abstract	ADJ
fcis-13113	2	42	:	:	PUNCT
fcis-13113	2	43	machine	machine	NOUN
fcis-13113	2	44	translation	translation	NOUN
fcis-13113	2	45	quality	quality	NOUN
fcis-13113	2	46	assessment	assessment	NOUN
fcis-13113	2	47	plays	play	VERB
fcis-13113	2	48	a	a	DET
fcis-13113	2	49	crucial	crucial	ADJ
fcis-13113	2	50	role	role	NOUN
fcis-13113	2	51	in	in	ADP
fcis-13113	2	52	enhancing	enhance	VERB
fcis-13113	2	53	the	the	DET
fcis-13113	2	54	performance	performance	NOUN
fcis-13113	2	55	of	of	ADP
fcis-13113	2	56	machine	machine	NOUN
fcis-13113	2	57	translation	translation	NOUN
fcis-13113	2	58	systems	system	NOUN
fcis-13113	2	59	.	.	PUNCT
fcis-13113	3	1	this	this	DET
fcis-13113	3	2	review	review	NOUN
fcis-13113	3	3	aims	aim	VERB
fcis-13113	3	4	to	to	PART
fcis-13113	3	5	survey	survey	VERB
fcis-13113	3	6	and	and	CCONJ
fcis-13113	3	7	outline	outline	VERB
fcis-13113	3	8	the	the	DET
fcis-13113	3	9	current	current	ADJ
fcis-13113	3	10	major	major	ADJ
fcis-13113	3	11	methods	method	NOUN
fcis-13113	3	12	used	use	VERB
fcis-13113	3	13	for	for	ADP
fcis-13113	3	14	assessing	assess	VERB
fcis-13113	3	15	machine	machine	NOUN
fcis-13113	3	16	translation	translation	NOUN
fcis-13113	3	17	quality	quality	NOUN
fcis-13113	3	18	,	,	PUNCT
fcis-13113	3	19	which	which	PRON
fcis-13113	3	20	can	can	AUX
fcis-13113	3	21	be	be	AUX
fcis-13113	3	22	broadly	broadly	ADV
fcis-13113	3	23	categorized	categorize	VERB
fcis-13113	3	24	into	into	ADP
fcis-13113	3	25	manual	manual	ADJ
fcis-13113	3	26	and	and	CCONJ
fcis-13113	3	27	automatic	automatic	ADJ
fcis-13113	3	28	assessment	assessment	NOUN
fcis-13113	3	29	methods	method	NOUN
fcis-13113	3	30	.	.	PUNCT
fcis-13113	4	1	upon	upon	SCONJ
fcis-13113	4	2	analyzing	analyze	VERB
fcis-13113	4	3	the	the	DET
fcis-13113	4	4	existing	exist	VERB
fcis-13113	4	5	literature	literature	NOUN
fcis-13113	4	6	,	,	PUNCT
fcis-13113	4	7	it	it	PRON
fcis-13113	4	8	becomes	become	VERB
fcis-13113	4	9	evident	evident	ADJ
fcis-13113	4	10	that	that	SCONJ
fcis-13113	4	11	while	while	SCONJ
fcis-13113	4	12	manual	manual	ADJ
fcis-13113	4	13	evaluation	evaluation	NOUN
fcis-13113	4	14	methods	method	NOUN
fcis-13113	4	15	yield	yield	VERB
fcis-13113	4	16	high	high	ADJ
fcis-13113	4	17	-	-	PUNCT
fcis-13113	4	18	quality	quality	NOUN
fcis-13113	4	19	results	result	NOUN
fcis-13113	4	20	,	,	PUNCT
fcis-13113	4	21	they	they	PRON
fcis-13113	4	22	are	be	AUX
fcis-13113	4	23	time	time	NOUN
fcis-13113	4	24	-	-	PUNCT
fcis-13113	4	25	consuming	consume	VERB
fcis-13113	4	26	and	and	CCONJ
fcis-13113	4	27	labor	labor	NOUN
fcis-13113	4	28	-	-	PUNCT
fcis-13113	4	29	intensive	intensive	ADJ
fcis-13113	4	30	.	.	PUNCT
fcis-13113	5	1	on	on	ADP
fcis-13113	5	2	the	the	DET
fcis-13113	5	3	other	other	ADJ
fcis-13113	5	4	hand	hand	NOUN
fcis-13113	5	5	,	,	PUNCT
fcis-13113	5	6	automatic	automatic	ADJ
fcis-13113	5	7	evaluation	evaluation	NOUN
fcis-13113	5	8	methods	method	NOUN
fcis-13113	5	9	are	be	AUX
fcis-13113	5	10	cost	cost	NOUN
fcis-13113	5	11	-	-	PUNCT
fcis-13113	5	12	effective	effective	ADJ
fcis-13113	5	13	and	and	CCONJ
fcis-13113	5	14	rapid	rapid	ADJ
fcis-13113	5	15	,	,	PUNCT
fcis-13113	5	16	but	but	CCONJ
fcis-13113	5	17	their	their	PRON
fcis-13113	5	18	evaluation	evaluation	NOUN
fcis-13113	5	19	results	result	NOUN
fcis-13113	5	20	do	do	AUX
fcis-13113	5	21	not	not	PART
fcis-13113	5	22	match	match	VERB
fcis-13113	5	23	the	the	DET
fcis-13113	5	24	expertise	expertise	NOUN
fcis-13113	5	25	of	of	ADP
fcis-13113	5	26	human	human	ADJ
fcis-13113	5	27	evaluators	evaluator	NOUN
fcis-13113	5	28	.	.	PUNCT
fcis-13113	6	1	the	the	DET
fcis-13113	6	2	objective	objective	NOUN
fcis-13113	6	3	of	of	ADP
fcis-13113	6	4	this	this	DET
fcis-13113	6	5	paper	paper	NOUN
fcis-13113	6	6	is	be	AUX
fcis-13113	6	7	to	to	PART
fcis-13113	6	8	furnish	furnish	VERB
fcis-13113	6	9	researchers	researcher	NOUN
fcis-13113	6	10	with	with	ADP
fcis-13113	6	11	a	a	DET
fcis-13113	6	12	comprehensive	comprehensive	ADJ
fcis-13113	6	13	overview	overview	NOUN
fcis-13113	6	14	of	of	ADP
fcis-13113	6	15	machine	machine	NOUN
fcis-13113	6	16	translation	translation	NOUN
fcis-13113	6	17	quality	quality	NOUN
fcis-13113	6	18	assessment	assessment	NOUN
fcis-13113	6	19	methods	method	NOUN
fcis-13113	6	20	,	,	PUNCT
fcis-13113	6	21	enabling	enable	VERB
fcis-13113	6	22	them	they	PRON
fcis-13113	6	23	to	to	PART
fcis-13113	6	24	select	select	VERB
fcis-13113	6	25	appropriate	appropriate	ADJ
fcis-13113	6	26	approaches	approach	NOUN
fcis-13113	6	27	based	base	VERB
fcis-13113	6	28	on	on	ADP
fcis-13113	6	29	their	their	PRON
fcis-13113	6	30	specific	specific	ADJ
fcis-13113	6	31	experimental	experimental	ADJ
fcis-13113	6	32	requirements	requirement	NOUN
fcis-13113	6	33	.	.	PUNCT
fcis-13113	7	1	additionally	additionally	ADV
fcis-13113	7	2	,	,	PUNCT
fcis-13113	7	3	we	we	PRON
fcis-13113	7	4	aspire	aspire	VERB
fcis-13113	7	5	to	to	PART
fcis-13113	7	6	offer	offer	VERB
fcis-13113	7	7	valuable	valuable	ADJ
fcis-13113	7	8	insights	insight	NOUN
fcis-13113	7	9	and	and	CCONJ
fcis-13113	7	10	novel	novel	ADJ
fcis-13113	7	11	perspectives	perspective	NOUN
fcis-13113	7	12	for	for	ADP
fcis-13113	7	13	further	further	ADJ
fcis-13113	7	14	advancements	advancement	NOUN
fcis-13113	7	15	in	in	ADP
fcis-13113	7	16	the	the	DET
fcis-13113	7	17	field	field	NOUN
fcis-13113	7	18	of	of	ADP
fcis-13113	7	19	machine	machine	NOUN
fcis-13113	7	20	translation	translation	NOUN
fcis-13113	7	21	quality	quality	NOUN
fcis-13113	7	22	assessment	assessment	NOUN
fcis-13113	7	23	methods	method	NOUN
fcis-13113	7	24	.	.	PUNCT
fcis-13113	8	1	keywords	keyword	NOUN
fcis-13113	8	2	:	:	PUNCT
fcis-13113	8	3	machine	machine	NOUN
fcis-13113	8	4	translation	translation	NOUN
fcis-13113	8	5	;	;	PUNCT
fcis-13113	8	6	automatic	automatic	ADJ
fcis-13113	8	7	evaluation	evaluation	NOUN
fcis-13113	8	8	metric	metric	NOUN
fcis-13113	8	9	;	;	PUNCT
fcis-13113	8	10	human	human	ADJ
fcis-13113	8	11	evaluation	evaluation	NOUN
fcis-13113	8	12	methods	method	NOUN
fcis-13113	8	13	.	.	PUNCT
fcis-13113	9	1	1	1	X
fcis-13113	9	2	.	.	X
fcis-13113	9	3	introduction	introduction	NOUN
fcis-13113	9	4	as	as	SCONJ
fcis-13113	9	5	the	the	DET
fcis-13113	9	6	demand	demand	NOUN
fcis-13113	9	7	for	for	ADP
fcis-13113	9	8	translation	translation	NOUN
fcis-13113	9	9	continues	continue	VERB
fcis-13113	9	10	to	to	PART
fcis-13113	9	11	rise	rise	VERB
fcis-13113	9	12	,	,	PUNCT
fcis-13113	9	13	traditional	traditional	ADJ
fcis-13113	9	14	human	human	ADJ
fcis-13113	9	15	translation	translation	NOUN
fcis-13113	9	16	alone	alone	ADV
fcis-13113	9	17	is	be	AUX
fcis-13113	9	18	no	no	ADV
fcis-13113	9	19	longer	long	ADV
fcis-13113	9	20	sufficient	sufficient	ADJ
fcis-13113	9	21	,	,	PUNCT
fcis-13113	9	22	leading	lead	VERB
fcis-13113	9	23	to	to	ADP
fcis-13113	9	24	the	the	DET
fcis-13113	9	25	emergence	emergence	NOUN
fcis-13113	9	26	of	of	ADP
fcis-13113	9	27	machine	machine	NOUN
fcis-13113	9	28	translation	translation	NOUN
fcis-13113	9	29	(	(	PUNCT
fcis-13113	9	30	mt	mt	PROPN
fcis-13113	9	31	)	)	PUNCT
fcis-13113	9	32	.	.	PUNCT
fcis-13113	10	1	the	the	DET
fcis-13113	10	2	evaluation	evaluation	NOUN
fcis-13113	10	3	of	of	ADP
fcis-13113	10	4	machine	machine	NOUN
fcis-13113	10	5	translation	translation	NOUN
fcis-13113	10	6	quality	quality	NOUN
fcis-13113	10	7	has	have	AUX
fcis-13113	10	8	played	play	VERB
fcis-13113	10	9	a	a	DET
fcis-13113	10	10	pivotal	pivotal	ADJ
fcis-13113	10	11	role	role	NOUN
fcis-13113	10	12	in	in	ADP
fcis-13113	10	13	advancing	advance	VERB
fcis-13113	10	14	the	the	DET
fcis-13113	10	15	development	development	NOUN
fcis-13113	10	16	of	of	ADP
fcis-13113	10	17	mt	mt	PROPN
fcis-13113	10	18	systems	system	NOUN
fcis-13113	10	19	.	.	PUNCT
fcis-13113	11	1	over	over	ADP
fcis-13113	11	2	time	time	NOUN
fcis-13113	11	3	,	,	PUNCT
fcis-13113	11	4	assessment	assessment	NOUN
fcis-13113	11	5	methods	method	NOUN
fcis-13113	11	6	have	have	AUX
fcis-13113	11	7	transitioned	transition	VERB
fcis-13113	11	8	from	from	ADP
fcis-13113	11	9	traditional	traditional	ADJ
fcis-13113	11	10	manual	manual	ADJ
fcis-13113	11	11	assessment	assessment	NOUN
fcis-13113	11	12	to	to	ADP
fcis-13113	11	13	current	current	ADJ
fcis-13113	11	14	automatic	automatic	ADJ
fcis-13113	11	15	assessment	assessment	NOUN
fcis-13113	11	16	methods	method	NOUN
fcis-13113	11	17	.	.	PUNCT
fcis-13113	12	1	however	however	ADV
fcis-13113	12	2	,	,	PUNCT
fcis-13113	12	3	the	the	DET
fcis-13113	12	4	existing	exist	VERB
fcis-13113	12	5	literature	literature	NOUN
fcis-13113	12	6	on	on	ADP
fcis-13113	12	7	mt	mt	PROPN
fcis-13113	12	8	quality	quality	NOUN
fcis-13113	12	9	assessment	assessment	NOUN
fcis-13113	12	10	methods	method	NOUN
fcis-13113	12	11	is	be	AUX
fcis-13113	12	12	lacking	lack	VERB
fcis-13113	12	13	,	,	PUNCT
fcis-13113	12	14	necessitating	necessitate	VERB
fcis-13113	12	15	a	a	DET
fcis-13113	12	16	comprehensive	comprehensive	ADJ
fcis-13113	12	17	summary	summary	NOUN
fcis-13113	12	18	of	of	ADP
fcis-13113	12	19	this	this	DET
fcis-13113	12	20	field	field	NOUN
fcis-13113	12	21	.	.	PUNCT
fcis-13113	13	1	the	the	DET
fcis-13113	13	2	purpose	purpose	NOUN
fcis-13113	13	3	of	of	ADP
fcis-13113	13	4	this	this	DET
fcis-13113	13	5	review	review	NOUN
fcis-13113	13	6	is	be	AUX
fcis-13113	13	7	to	to	PART
fcis-13113	13	8	examine	examine	VERB
fcis-13113	13	9	and	and	CCONJ
fcis-13113	13	10	summarize	summarize	VERB
fcis-13113	13	11	the	the	DET
fcis-13113	13	12	currently	currently	ADV
fcis-13113	13	13	available	available	ADJ
fcis-13113	13	14	methods	method	NOUN
fcis-13113	13	15	for	for	ADP
fcis-13113	13	16	assessing	assess	VERB
fcis-13113	13	17	the	the	DET
fcis-13113	13	18	quality	quality	NOUN
fcis-13113	13	19	of	of	ADP
fcis-13113	13	20	mt	mt	PROPN
fcis-13113	13	21	,	,	PUNCT
fcis-13113	13	22	while	while	SCONJ
fcis-13113	13	23	discussing	discuss	VERB
fcis-13113	13	24	their	their	PRON
fcis-13113	13	25	respective	respective	ADJ
fcis-13113	13	26	strengths	strength	NOUN
fcis-13113	13	27	and	and	CCONJ
fcis-13113	13	28	weaknesses	weakness	NOUN
fcis-13113	13	29	.	.	PUNCT
fcis-13113	14	1	the	the	DET
fcis-13113	14	2	review	review	NOUN
fcis-13113	14	3	begins	begin	VERB
fcis-13113	14	4	by	by	ADP
fcis-13113	14	5	exploring	explore	VERB
fcis-13113	14	6	traditional	traditional	ADJ
fcis-13113	14	7	manual	manual	ADJ
fcis-13113	14	8	assessment	assessment	NOUN
fcis-13113	14	9	methods	method	NOUN
fcis-13113	14	10	and	and	CCONJ
fcis-13113	14	11	then	then	ADV
fcis-13113	14	12	delves	delve	VERB
fcis-13113	14	13	into	into	ADP
fcis-13113	14	14	supervised	supervised	ADJ
fcis-13113	14	15	and	and	CCONJ
fcis-13113	14	16	unsupervised	unsupervised	ADJ
fcis-13113	14	17	learning	learning	NOUN
fcis-13113	14	18	-	-	PUNCT
fcis-13113	14	19	based	base	VERB
fcis-13113	14	20	assessment	assessment	NOUN
fcis-13113	14	21	methods	method	NOUN
fcis-13113	14	22	.	.	PUNCT
fcis-13113	15	1	by	by	ADP
fcis-13113	15	2	presenting	present	VERB
fcis-13113	15	3	a	a	DET
fcis-13113	15	4	comprehensive	comprehensive	ADJ
fcis-13113	15	5	overview	overview	NOUN
fcis-13113	15	6	,	,	PUNCT
fcis-13113	15	7	this	this	DET
fcis-13113	15	8	review	review	NOUN
fcis-13113	15	9	aims	aim	VERB
fcis-13113	15	10	to	to	PART
fcis-13113	15	11	assist	assist	VERB
fcis-13113	15	12	researchers	researcher	NOUN
fcis-13113	15	13	in	in	ADP
fcis-13113	15	14	selecting	select	VERB
fcis-13113	15	15	the	the	DET
fcis-13113	15	16	most	most	ADV
fcis-13113	15	17	suitable	suitable	ADJ
fcis-13113	15	18	mt	mt	PROPN
fcis-13113	15	19	quality	quality	NOUN
fcis-13113	15	20	assessment	assessment	NOUN
fcis-13113	15	21	methods	method	NOUN
fcis-13113	15	22	for	for	ADP
fcis-13113	15	23	their	their	PRON
fcis-13113	15	24	future	future	ADJ
fcis-13113	15	25	work	work	NOUN
fcis-13113	15	26	.	.	PUNCT
fcis-13113	16	1	additionally	additionally	ADV
fcis-13113	16	2	,	,	PUNCT
fcis-13113	16	3	this	this	DET
fcis-13113	16	4	paper	paper	NOUN
fcis-13113	16	5	will	will	AUX
fcis-13113	16	6	address	address	VERB
fcis-13113	16	7	the	the	DET
fcis-13113	16	8	challenges	challenge	NOUN
fcis-13113	16	9	associated	associate	VERB
fcis-13113	16	10	with	with	ADP
fcis-13113	16	11	mt	mt	PROPN
fcis-13113	16	12	quality	quality	NOUN
fcis-13113	16	13	assessment	assessment	NOUN
fcis-13113	16	14	and	and	CCONJ
fcis-13113	16	15	propose	propose	VERB
fcis-13113	16	16	directions	direction	NOUN
fcis-13113	16	17	for	for	ADP
fcis-13113	16	18	future	future	ADJ
fcis-13113	16	19	research	research	NOUN
fcis-13113	16	20	,	,	PUNCT
fcis-13113	16	21	providing	provide	VERB
fcis-13113	16	22	researchers	researcher	NOUN
fcis-13113	16	23	with	with	ADP
fcis-13113	16	24	new	new	ADJ
fcis-13113	16	25	perspectives	perspective	NOUN
fcis-13113	16	26	and	and	CCONJ
fcis-13113	16	27	insights	insight	NOUN
fcis-13113	16	28	to	to	PART
fcis-13113	16	29	enhance	enhance	VERB
fcis-13113	16	30	their	their	PRON
fcis-13113	16	31	techniques	technique	NOUN
fcis-13113	16	32	in	in	ADP
fcis-13113	16	33	this	this	DET
fcis-13113	16	34	domain	domain	NOUN
fcis-13113	16	35	.	.	PUNCT
fcis-13113	17	1	2	2	X
fcis-13113	17	2	.	.	X
fcis-13113	17	3	classification	classification	NOUN
fcis-13113	17	4	of	of	ADP
fcis-13113	17	5	mt	mt	PROPN
fcis-13113	17	6	quality	quality	PROPN
fcis-13113	17	7	assessment	assessment	NOUN
fcis-13113	17	8	methods	method	NOUN
fcis-13113	17	9	mt	mt	PROPN
fcis-13113	17	10	quality	quality	PROPN
fcis-13113	17	11	assessment	assessment	NOUN
fcis-13113	17	12	(	(	PUNCT
fcis-13113	17	13	mtqa	mtqa	NOUN
fcis-13113	17	14	)	)	PUNCT
fcis-13113	17	15	methods	method	NOUN
fcis-13113	17	16	can	can	AUX
fcis-13113	17	17	be	be	AUX
fcis-13113	17	18	broadly	broadly	ADV
fcis-13113	17	19	categorized	categorize	VERB
fcis-13113	17	20	into	into	ADP
fcis-13113	17	21	manual	manual	ADJ
fcis-13113	17	22	assessment	assessment	NOUN
fcis-13113	17	23	methods	method	NOUN
fcis-13113	17	24	and	and	CCONJ
fcis-13113	17	25	automatic	automatic	ADJ
fcis-13113	17	26	assessment	assessment	NOUN
fcis-13113	17	27	methods	method	NOUN
fcis-13113	17	28	.	.	PUNCT
fcis-13113	18	1	manual	manual	ADJ
fcis-13113	18	2	assessment	assessment	NOUN
fcis-13113	18	3	methods	method	NOUN
fcis-13113	18	4	involve	involve	VERB
fcis-13113	18	5	human	human	ADJ
fcis-13113	18	6	assessors	assessor	NOUN
fcis-13113	18	7	,	,	PUNCT
fcis-13113	18	8	while	while	SCONJ
fcis-13113	18	9	automatic	automatic	ADJ
fcis-13113	18	10	assessment	assessment	NOUN
fcis-13113	18	11	methods	method	NOUN
fcis-13113	18	12	utilize	utilize	VERB
fcis-13113	18	13	mathematical	mathematical	ADJ
fcis-13113	18	14	formulas	formula	NOUN
fcis-13113	18	15	or	or	CCONJ
fcis-13113	18	16	deep	deep	ADJ
fcis-13113	18	17	learning	learning	NOUN
fcis-13113	18	18	algorithms	algorithm	NOUN
fcis-13113	18	19	to	to	PART
fcis-13113	18	20	quantify	quantify	VERB
fcis-13113	18	21	and	and	CCONJ
fcis-13113	18	22	measure	measure	VERB
fcis-13113	18	23	translation	translation	NOUN
fcis-13113	18	24	quality	quality	NOUN
fcis-13113	18	25	.	.	PUNCT
fcis-13113	19	1	manual	manual	ADJ
fcis-13113	19	2	evaluation	evaluation	NOUN
fcis-13113	19	3	methods	method	NOUN
fcis-13113	19	4	are	be	AUX
fcis-13113	19	5	effective	effective	ADJ
fcis-13113	19	6	in	in	ADP
fcis-13113	19	7	assessing	assess	VERB
fcis-13113	19	8	translation	translation	NOUN
fcis-13113	19	9	quality	quality	NOUN
fcis-13113	19	10	.	.	PUNCT
fcis-13113	20	1	bilingual	bilingual	ADJ
fcis-13113	20	2	assessors	assessors	PROPN
fcis-13113	20	3	,	,	PUNCT
fcis-13113	20	4	with	with	ADP
fcis-13113	20	5	their	their	PRON
fcis-13113	20	6	deep	deep	ADJ
fcis-13113	20	7	understanding	understanding	NOUN
fcis-13113	20	8	of	of	ADP
fcis-13113	20	9	language	language	NOUN
fcis-13113	20	10	,	,	PUNCT
fcis-13113	20	11	can	can	AUX
fcis-13113	20	12	evaluate	evaluate	VERB
fcis-13113	20	13	translations	translation	NOUN
fcis-13113	20	14	based	base	VERB
fcis-13113	20	15	on	on	ADP
fcis-13113	20	16	accuracy	accuracy	NOUN
fcis-13113	20	17	,	,	PUNCT
fcis-13113	20	18	fluency	fluency	NOUN
fcis-13113	20	19	,	,	PUNCT
fcis-13113	20	20	expressive	expressive	ADJ
fcis-13113	20	21	integrity	integrity	NOUN
fcis-13113	20	22	,	,	PUNCT
fcis-13113	20	23	and	and	CCONJ
fcis-13113	20	24	the	the	DET
fcis-13113	20	25	ability	ability	NOUN
fcis-13113	20	26	to	to	PART
fcis-13113	20	27	capture	capture	VERB
fcis-13113	20	28	nuances	nuance	NOUN
fcis-13113	20	29	in	in	ADP
fcis-13113	20	30	complex	complex	ADJ
fcis-13113	20	31	texts	text	NOUN
fcis-13113	20	32	.	.	PUNCT
fcis-13113	21	1	however	however	ADV
fcis-13113	21	2	,	,	PUNCT
fcis-13113	21	3	manual	manual	ADJ
fcis-13113	21	4	assessment	assessment	NOUN
fcis-13113	21	5	methods	method	NOUN
fcis-13113	21	6	have	have	VERB
fcis-13113	21	7	drawbacks	drawback	NOUN
fcis-13113	21	8	.	.	PUNCT
fcis-13113	22	1	they	they	PRON
fcis-13113	22	2	are	be	AUX
fcis-13113	22	3	timeconsuming	timeconsuming	ADJ
fcis-13113	22	4	,	,	PUNCT
fcis-13113	22	5	labor	labor	NOUN
fcis-13113	22	6	-	-	PUNCT
fcis-13113	22	7	intensive	intensive	ADJ
fcis-13113	22	8	,	,	PUNCT
fcis-13113	22	9	and	and	CCONJ
fcis-13113	22	10	reliant	reliant	ADJ
fcis-13113	22	11	on	on	ADP
fcis-13113	22	12	finding	find	VERB
fcis-13113	22	13	trustworthy	trustworthy	ADJ
fcis-13113	22	14	bilingual	bilingual	ADJ
fcis-13113	22	15	evaluators	evaluator	NOUN
fcis-13113	22	16	.	.	PUNCT
fcis-13113	23	1	additionally	additionally	ADV
fcis-13113	23	2	,	,	PUNCT
fcis-13113	23	3	subjective	subjective	ADJ
fcis-13113	23	4	differences	difference	NOUN
fcis-13113	23	5	among	among	ADP
fcis-13113	23	6	evaluators	evaluator	NOUN
fcis-13113	23	7	can	can	AUX
fcis-13113	23	8	lead	lead	VERB
fcis-13113	23	9	to	to	ADP
fcis-13113	23	10	varying	vary	VERB
fcis-13113	23	11	assessment	assessment	NOUN
fcis-13113	23	12	results	result	NOUN
fcis-13113	23	13	for	for	ADP
fcis-13113	23	14	the	the	DET
fcis-13113	23	15	same	same	ADJ
fcis-13113	23	16	translation	translation	NOUN
fcis-13113	23	17	.	.	PUNCT
fcis-13113	24	1	to	to	PART
fcis-13113	24	2	enhance	enhance	VERB
fcis-13113	24	3	reliability	reliability	NOUN
fcis-13113	24	4	,	,	PUNCT
fcis-13113	24	5	averaging	average	VERB
fcis-13113	24	6	scores	score	NOUN
fcis-13113	24	7	from	from	ADP
fcis-13113	24	8	multiple	multiple	ADJ
fcis-13113	24	9	assessors	assessor	NOUN
fcis-13113	24	10	is	be	AUX
fcis-13113	24	11	commonly	commonly	ADV
fcis-13113	24	12	employed	employ	VERB
fcis-13113	24	13	.	.	PUNCT
fcis-13113	25	1	to	to	PART
fcis-13113	25	2	overcome	overcome	VERB
fcis-13113	25	3	the	the	DET
fcis-13113	25	4	limitations	limitation	NOUN
fcis-13113	25	5	of	of	ADP
fcis-13113	25	6	manual	manual	ADJ
fcis-13113	25	7	assessment	assessment	NOUN
fcis-13113	25	8	methods	method	NOUN
fcis-13113	25	9	,	,	PUNCT
fcis-13113	25	10	researchers	researcher	NOUN
fcis-13113	25	11	have	have	AUX
fcis-13113	25	12	turned	turn	VERB
fcis-13113	25	13	to	to	ADP
fcis-13113	25	14	automated	automate	VERB
fcis-13113	25	15	assessment	assessment	NOUN
fcis-13113	25	16	methods	method	NOUN
fcis-13113	25	17	.	.	PUNCT
fcis-13113	26	1	they	they	PRON
fcis-13113	26	2	employ	employ	VERB
fcis-13113	26	3	objective	objective	ADJ
fcis-13113	26	4	evaluation	evaluation	NOUN
fcis-13113	26	5	metrics	metric	NOUN
fcis-13113	26	6	through	through	ADP
fcis-13113	26	7	mathematical	mathematical	ADJ
fcis-13113	26	8	formulas	formula	NOUN
fcis-13113	26	9	or	or	CCONJ
fcis-13113	26	10	deep	deep	ADJ
fcis-13113	26	11	learning	learning	NOUN
fcis-13113	26	12	algorithms	algorithm	NOUN
fcis-13113	26	13	.	.	PUNCT
fcis-13113	27	1	however	however	ADV
fcis-13113	27	2	,	,	PUNCT
fcis-13113	27	3	these	these	DET
fcis-13113	27	4	metrics	metric	NOUN
fcis-13113	27	5	may	may	AUX
fcis-13113	27	6	not	not	PART
fcis-13113	27	7	fully	fully	ADV
fcis-13113	27	8	capture	capture	VERB
fcis-13113	27	9	human	human	ADJ
fcis-13113	27	10	understanding	understanding	NOUN
fcis-13113	27	11	of	of	ADP
fcis-13113	27	12	translation	translation	NOUN
fcis-13113	27	13	.	.	PUNCT
fcis-13113	28	1	traditional	traditional	ADJ
fcis-13113	28	2	automatic	automatic	ADJ
fcis-13113	28	3	evaluation	evaluation	NOUN
fcis-13113	28	4	metrics	metric	NOUN
fcis-13113	28	5	,	,	PUNCT
fcis-13113	28	6	such	such	ADJ
fcis-13113	28	7	as	as	ADP
fcis-13113	28	8	bleu	bleu	NOUN
fcis-13113	28	9	,	,	PUNCT
fcis-13113	28	10	focus	focus	VERB
fcis-13113	28	11	on	on	ADP
fcis-13113	28	12	lexical	lexical	ADJ
fcis-13113	28	13	similarity	similarity	NOUN
fcis-13113	28	14	and	and	CCONJ
fcis-13113	28	15	often	often	ADV
fcis-13113	28	16	overlook	overlook	VERB
fcis-13113	28	17	contextual	contextual	ADJ
fcis-13113	28	18	information	information	NOUN
fcis-13113	28	19	,	,	PUNCT
fcis-13113	28	20	resulting	result	VERB
fcis-13113	28	21	in	in	ADP
fcis-13113	28	22	inaccurate	inaccurate	ADJ
fcis-13113	28	23	evaluations	evaluation	NOUN
fcis-13113	28	24	[	[	X
fcis-13113	28	25	1	1	NUM
fcis-13113	28	26	]	]	PUNCT
fcis-13113	28	27	.	.	PUNCT
fcis-13113	29	1	to	to	PART
fcis-13113	29	2	address	address	VERB
fcis-13113	29	3	this	this	PRON
fcis-13113	29	4	,	,	PUNCT
fcis-13113	29	5	researchers	researcher	NOUN
fcis-13113	29	6	have	have	AUX
fcis-13113	29	7	proposed	propose	VERB
fcis-13113	29	8	automatic	automatic	ADJ
fcis-13113	29	9	evaluation	evaluation	NOUN
fcis-13113	29	10	methods	method	NOUN
fcis-13113	29	11	based	base	VERB
fcis-13113	29	12	on	on	ADP
fcis-13113	29	13	transformer	transformer	NOUN
fcis-13113	29	14	-	-	PUNCT
fcis-13113	29	15	based	base	VERB
fcis-13113	29	16	language	language	NOUN
fcis-13113	29	17	models	model	NOUN
fcis-13113	29	18	[	[	X
fcis-13113	29	19	2	2	NUM
fcis-13113	29	20	]	]	PUNCT
fcis-13113	29	21	.	.	PUNCT
fcis-13113	30	1	these	these	DET
fcis-13113	30	2	methods	method	NOUN
fcis-13113	30	3	consider	consider	VERB
fcis-13113	30	4	contextual	contextual	ADJ
fcis-13113	30	5	relationships	relationship	NOUN
fcis-13113	30	6	and	and	CCONJ
fcis-13113	30	7	semantic	semantic	ADJ
fcis-13113	30	8	features	feature	NOUN
fcis-13113	30	9	,	,	PUNCT
fcis-13113	30	10	making	make	VERB
fcis-13113	30	11	them	they	PRON
fcis-13113	30	12	closer	close	ADV
fcis-13113	30	13	to	to	ADP
fcis-13113	30	14	human	human	ADJ
fcis-13113	30	15	evaluation	evaluation	NOUN
fcis-13113	30	16	[	[	X
fcis-13113	30	17	3	3	NUM
fcis-13113	30	18	]	]	PUNCT
fcis-13113	30	19	.	.	PUNCT
fcis-13113	31	1	3	3	X
fcis-13113	31	2	.	.	X
fcis-13113	31	3	manual	manual	ADJ
fcis-13113	31	4	assessment	assessment	NOUN
fcis-13113	31	5	methods	method	NOUN
fcis-13113	31	6	human	human	ADJ
fcis-13113	31	7	evaluation	evaluation	NOUN
fcis-13113	31	8	methods	method	NOUN
fcis-13113	31	9	are	be	AUX
fcis-13113	31	10	mainly	mainly	ADV
fcis-13113	31	11	divided	divide	VERB
fcis-13113	31	12	into	into	ADP
fcis-13113	31	13	methods	method	NOUN
fcis-13113	31	14	based	base	VERB
fcis-13113	31	15	on	on	ADP
fcis-13113	31	16	direct	direct	ADJ
fcis-13113	31	17	judgement	judgement	NOUN
fcis-13113	31	18	and	and	CCONJ
fcis-13113	31	19	methods	method	NOUN
fcis-13113	31	20	not	not	PART
fcis-13113	31	21	based	base	VERB
fcis-13113	31	22	on	on	ADP
fcis-13113	31	23	direct	direct	ADJ
fcis-13113	31	24	judgement	judgement	NOUN
fcis-13113	31	25	[	[	X
fcis-13113	31	26	4	4	NUM
fcis-13113	31	27	]	]	PUNCT
fcis-13113	31	28	.	.	PUNCT
fcis-13113	32	1	the	the	DET
fcis-13113	32	2	direct	direct	ADJ
fcis-13113	32	3	judgement	judgement	NOUN
fcis-13113	32	4	-	-	PUNCT
fcis-13113	32	5	based	base	VERB
fcis-13113	32	6	assessment	assessment	NOUN
fcis-13113	32	7	method	method	NOUN
fcis-13113	32	8	was	be	AUX
fcis-13113	32	9	still	still	ADV
fcis-13113	32	10	used	use	VERB
fcis-13113	32	11	as	as	ADP
fcis-13113	32	12	the	the	DET
fcis-13113	32	13	main	main	ADJ
fcis-13113	32	14	method	method	NOUN
fcis-13113	32	15	for	for	ADP
fcis-13113	32	16	evaluating	evaluate	VERB
fcis-13113	32	17	english	english	ADJ
fcis-13113	32	18	translations	translation	NOUN
fcis-13113	32	19	at	at	ADP
fcis-13113	32	20	the	the	DET
fcis-13113	32	21	mt	mt	PROPN
fcis-13113	32	22	society	society	NOUN
fcis-13113	32	23	2022	2022	NUM
fcis-13113	33	1	[	[	X
fcis-13113	33	2	5	5	NUM
fcis-13113	33	3	]	]	PUNCT
fcis-13113	33	4	.	.	PUNCT
fcis-13113	34	1	the	the	DET
fcis-13113	34	2	evaluator	evaluator	NOUN
fcis-13113	34	3	reads	read	VERB
fcis-13113	34	4	and	and	CCONJ
fcis-13113	34	5	evaluates	evaluate	VERB
fcis-13113	34	6	the	the	DET
fcis-13113	34	7	translation	translation	NOUN
fcis-13113	34	8	output	output	NOUN
fcis-13113	34	9	by	by	ADP
fcis-13113	34	10	the	the	DET
fcis-13113	34	11	system	system	NOUN
fcis-13113	34	12	and	and	CCONJ
fcis-13113	34	13	compares	compare	VERB
fcis-13113	34	14	its	its	PRON
fcis-13113	34	15	meaning	meaning	NOUN
fcis-13113	34	16	with	with	ADP
fcis-13113	34	17	an	an	DET
fcis-13113	34	18	english	english	ADJ
fcis-13113	34	19	reference	reference	NOUN
fcis-13113	34	20	translation	translation	NOUN
fcis-13113	34	21	manually	manually	ADV
fcis-13113	34	22	translated	translate	VERB
fcis-13113	34	23	by	by	ADP
fcis-13113	34	24	a	a	DET
fcis-13113	34	25	human	human	ADJ
fcis-13113	34	26	translator	translator	NOUN
fcis-13113	34	27	.	.	PUNCT
fcis-13113	35	1	the	the	DET
fcis-13113	35	2	evaluator	evaluator	NOUN
fcis-13113	35	3	is	be	AUX
fcis-13113	35	4	asked	ask	VERB
fcis-13113	35	5	to	to	PART
fcis-13113	35	6	rate	rate	VERB
fcis-13113	35	7	the	the	DET
fcis-13113	35	8	mt	mt	PROPN
fcis-13113	35	9	result	result	NOUN
fcis-13113	35	10	on	on	ADP
fcis-13113	35	11	a	a	DET
fcis-13113	35	12	scale	scale	NOUN
fcis-13113	35	13	such	such	ADJ
fcis-13113	35	14	as	as	ADP
fcis-13113	35	15	0	0	NUM
fcis-13113	35	16	-	-	SYM
fcis-13113	35	17	5	5	NUM
fcis-13113	35	18	or	or	CCONJ
fcis-13113	35	19	0	0	NUM
fcis-13113	35	20	-	-	SYM
fcis-13113	35	21	100	100	NUM
fcis-13113	35	22	based	base	VERB
fcis-13113	35	23	on	on	ADP
fcis-13113	35	24	its	its	PRON
fcis-13113	35	25	accuracy	accuracy	NOUN
fcis-13113	35	26	,	,	PUNCT
fcis-13113	35	27	fluency	fluency	NOUN
fcis-13113	35	28	,	,	PUNCT
fcis-13113	35	29	and	and	CCONJ
fcis-13113	35	30	similarity	similarity	NOUN
fcis-13113	35	31	to	to	ADP
fcis-13113	35	32	the	the	DET
fcis-13113	35	33	reference	reference	NOUN
fcis-13113	35	34	translation	translation	NOUN
fcis-13113	35	35	.	.	PUNCT
fcis-13113	36	1	the	the	DET
fcis-13113	36	2	advantage	advantage	NOUN
fcis-13113	36	3	of	of	ADP
fcis-13113	36	4	a	a	DET
fcis-13113	36	5	direct	direct	ADJ
fcis-13113	36	6	judgement	judgement	NOUN
fcis-13113	36	7	based	base	VERB
fcis-13113	36	8	assessment	assessment	NOUN
fcis-13113	36	9	method	method	NOUN
fcis-13113	36	10	is	be	AUX
fcis-13113	36	11	that	that	SCONJ
fcis-13113	36	12	it	it	PRON
fcis-13113	36	13	is	be	AUX
fcis-13113	36	14	possible	possible	ADJ
fcis-13113	36	15	to	to	PART
fcis-13113	36	16	see	see	VERB
fcis-13113	36	17	from	from	ADP
fcis-13113	36	18	the	the	DET
fcis-13113	36	19	scoring	scoring	NOUN
fcis-13113	36	20	results	result	NOUN
fcis-13113	36	21	not	not	PART
fcis-13113	36	22	only	only	ADV
fcis-13113	36	23	how	how	SCONJ
fcis-13113	36	24	one	one	NUM
fcis-13113	36	25	translation	translation	NOUN
fcis-13113	36	26	result	result	NOUN
fcis-13113	36	27	is	be	AUX
fcis-13113	36	28	better	well	ADJ
fcis-13113	36	29	than	than	ADP
fcis-13113	36	30	another	another	PRON
fcis-13113	36	31	,	,	PUNCT
fcis-13113	36	32	but	but	CCONJ
fcis-13113	36	33	also	also	ADV
fcis-13113	36	34	how	how	SCONJ
fcis-13113	36	35	far	far	ADV
fcis-13113	36	36	that	that	DET
fcis-13113	36	37	result	result	NOUN
fcis-13113	36	38	is	be	AUX
fcis-13113	36	39	better	well	ADJ
fcis-13113	36	40	than	than	ADP
fcis-13113	36	41	the	the	DET
fcis-13113	36	42	other	other	ADJ
fcis-13113	36	43	[	[	X
fcis-13113	36	44	6	6	NUM
fcis-13113	36	45	]	]	PUNCT
fcis-13113	36	46	.	.	PUNCT
fcis-13113	37	1	109	109	NUM
fcis-13113	37	2	the	the	DET
fcis-13113	37	3	non	non	ADJ
fcis-13113	37	4	-	-	ADJ
fcis-13113	37	5	direct	direct	ADJ
fcis-13113	37	6	judgement	judgement	NOUN
fcis-13113	37	7	based	base	VERB
fcis-13113	37	8	approach	approach	NOUN
fcis-13113	37	9	assesses	assess	VERB
fcis-13113	37	10	the	the	DET
fcis-13113	37	11	translation	translation	NOUN
fcis-13113	37	12	quality	quality	NOUN
fcis-13113	37	13	by	by	ADP
fcis-13113	37	14	asking	ask	VERB
fcis-13113	37	15	the	the	DET
fcis-13113	37	16	assessor	assessor	NOUN
fcis-13113	37	17	to	to	PART
fcis-13113	37	18	perform	perform	VERB
fcis-13113	37	19	tasks	task	NOUN
fcis-13113	37	20	related	relate	VERB
fcis-13113	37	21	to	to	ADP
fcis-13113	37	22	the	the	DET
fcis-13113	37	23	mt	mt	PROPN
fcis-13113	37	24	results	result	NOUN
fcis-13113	37	25	,	,	PUNCT
fcis-13113	37	26	such	such	ADJ
fcis-13113	37	27	as	as	ADP
fcis-13113	37	28	reading	read	VERB
fcis-13113	37	29	comprehension	comprehension	NOUN
fcis-13113	37	30	,	,	PUNCT
fcis-13113	37	31	information	information	NOUN
fcis-13113	37	32	retrieval	retrieval	NOUN
fcis-13113	37	33	or	or	CCONJ
fcis-13113	37	34	gap	gap	NOUN
fcis-13113	37	35	filling	fill	VERB
fcis-13113	37	36	[	[	X
fcis-13113	37	37	7	7	NUM
fcis-13113	37	38	]	]	PUNCT
fcis-13113	37	39	,	,	PUNCT
fcis-13113	37	40	and	and	CCONJ
fcis-13113	37	41	assesses	assess	VERB
fcis-13113	37	42	the	the	DET
fcis-13113	37	43	translation	translation	NOUN
fcis-13113	37	44	quality	quality	NOUN
fcis-13113	37	45	based	base	VERB
fcis-13113	37	46	on	on	ADP
fcis-13113	37	47	the	the	DET
fcis-13113	37	48	results	result	NOUN
fcis-13113	37	49	of	of	ADP
fcis-13113	37	50	the	the	DET
fcis-13113	37	51	tasks	task	NOUN
fcis-13113	37	52	.	.	PUNCT
fcis-13113	38	1	this	this	DET
fcis-13113	38	2	assessment	assessment	NOUN
fcis-13113	38	3	method	method	NOUN
fcis-13113	38	4	can	can	AUX
fcis-13113	38	5	assess	assess	VERB
fcis-13113	38	6	the	the	DET
fcis-13113	38	7	mt	mt	PROPN
fcis-13113	38	8	results	result	VERB
fcis-13113	38	9	more	more	ADV
fcis-13113	38	10	comprehensively	comprehensively	ADV
fcis-13113	38	11	and	and	CCONJ
fcis-13113	38	12	is	be	AUX
fcis-13113	38	13	more	more	ADV
fcis-13113	38	14	objective	objective	ADJ
fcis-13113	38	15	than	than	ADP
fcis-13113	38	16	the	the	DET
fcis-13113	38	17	direct	direct	ADJ
fcis-13113	38	18	judgement	judgement	NOUN
fcis-13113	38	19	-	-	PUNCT
fcis-13113	38	20	based	base	VERB
fcis-13113	38	21	assessment	assessment	NOUN
fcis-13113	38	22	method	method	NOUN
fcis-13113	38	23	,	,	PUNCT
fcis-13113	38	24	but	but	CCONJ
fcis-13113	38	25	it	it	PRON
fcis-13113	38	26	requires	require	VERB
fcis-13113	38	27	more	more	ADJ
fcis-13113	38	28	time	time	NOUN
fcis-13113	38	29	and	and	CCONJ
fcis-13113	38	30	manpower	manpower	NOUN
fcis-13113	38	31	costs	cost	NOUN
fcis-13113	38	32	and	and	CCONJ
fcis-13113	38	33	has	have	VERB
fcis-13113	38	34	higher	high	ADJ
fcis-13113	38	35	requirements	requirement	NOUN
fcis-13113	38	36	for	for	ADP
fcis-13113	38	37	the	the	DET
fcis-13113	38	38	assessors	assessor	NOUN
fcis-13113	38	39	.	.	PUNCT
fcis-13113	39	1	4	4	X
fcis-13113	39	2	.	.	NUM
fcis-13113	39	3	automated	automate	VERB
fcis-13113	39	4	assessment	assessment	NOUN
fcis-13113	39	5	methodology	methodology	NOUN
fcis-13113	39	6	so	so	ADV
fcis-13113	39	7	far	far	ADV
fcis-13113	39	8	automatic	automatic	ADJ
fcis-13113	39	9	evaluation	evaluation	NOUN
fcis-13113	39	10	methods	method	NOUN
fcis-13113	39	11	can	can	AUX
fcis-13113	39	12	be	be	AUX
fcis-13113	39	13	mainly	mainly	ADV
fcis-13113	39	14	classified	classify	VERB
fcis-13113	39	15	into	into	ADP
fcis-13113	39	16	three	three	NUM
fcis-13113	39	17	main	main	ADJ
fcis-13113	39	18	categories	category	NOUN
fcis-13113	39	19	:	:	PUNCT
fcis-13113	39	20	methods	method	NOUN
fcis-13113	39	21	based	base	VERB
fcis-13113	39	22	on	on	ADP
fcis-13113	39	23	reference	reference	NOUN
fcis-13113	39	24	translation	translation	NOUN
fcis-13113	39	25	,	,	PUNCT
fcis-13113	39	26	methods	method	NOUN
fcis-13113	39	27	based	base	VERB
fcis-13113	39	28	on	on	ADP
fcis-13113	39	29	unsupervised	unsupervised	ADJ
fcis-13113	39	30	learning	learning	NOUN
fcis-13113	39	31	,	,	PUNCT
fcis-13113	39	32	and	and	CCONJ
fcis-13113	39	33	methods	method	NOUN
fcis-13113	39	34	based	base	VERB
fcis-13113	39	35	on	on	ADP
fcis-13113	39	36	supervised	supervised	ADJ
fcis-13113	39	37	learning	learning	NOUN
fcis-13113	39	38	.	.	PUNCT
fcis-13113	40	1	reference	reference	NOUN
fcis-13113	40	2	translation	translation	NOUN
fcis-13113	40	3	-	-	PUNCT
fcis-13113	40	4	based	base	VERB
fcis-13113	40	5	methods	method	NOUN
fcis-13113	40	6	mainly	mainly	ADV
fcis-13113	40	7	assess	assess	VERB
fcis-13113	40	8	translation	translation	NOUN
fcis-13113	40	9	quality	quality	NOUN
fcis-13113	40	10	by	by	ADP
fcis-13113	40	11	comparing	compare	VERB
fcis-13113	40	12	mt	mt	PROPN
fcis-13113	40	13	results	result	NOUN
fcis-13113	40	14	with	with	ADP
fcis-13113	40	15	reference	reference	NOUN
fcis-13113	40	16	translations	translation	NOUN
fcis-13113	40	17	,	,	PUNCT
fcis-13113	40	18	and	and	CCONJ
fcis-13113	40	19	measure	measure	VERB
fcis-13113	40	20	the	the	DET
fcis-13113	40	21	degree	degree	NOUN
fcis-13113	40	22	of	of	ADP
fcis-13113	40	23	similarity	similarity	NOUN
fcis-13113	40	24	between	between	ADP
fcis-13113	40	25	mt	mt	PROPN
fcis-13113	40	26	results	result	NOUN
fcis-13113	40	27	and	and	CCONJ
fcis-13113	40	28	reference	reference	NOUN
fcis-13113	40	29	translations	translation	NOUN
fcis-13113	40	30	through	through	ADP
fcis-13113	40	31	mathematical	mathematical	ADJ
fcis-13113	40	32	formulas	formula	NOUN
fcis-13113	40	33	[	[	X
fcis-13113	40	34	8	8	NUM
fcis-13113	40	35	]	]	PUNCT
fcis-13113	40	36	,	,	PUNCT
fcis-13113	40	37	etc	etc	X
fcis-13113	40	38	.	.	X
fcis-13113	41	1	unsupervised	unsupervised	ADJ
fcis-13113	41	2	learning	learning	NOUN
fcis-13113	41	3	-	-	PUNCT
fcis-13113	41	4	based	base	VERB
fcis-13113	41	5	approaches	approach	NOUN
fcis-13113	41	6	learn	learn	VERB
fcis-13113	41	7	the	the	DET
fcis-13113	41	8	criteria	criterion	NOUN
fcis-13113	41	9	for	for	ADP
fcis-13113	41	10	assessing	assess	VERB
fcis-13113	41	11	translation	translation	NOUN
fcis-13113	41	12	quality	quality	NOUN
fcis-13113	41	13	from	from	ADP
fcis-13113	41	14	the	the	DET
fcis-13113	41	15	data	datum	NOUN
fcis-13113	41	16	itself	itself	PRON
fcis-13113	41	17	by	by	ADP
fcis-13113	41	18	using	use	VERB
fcis-13113	41	19	datasets	dataset	NOUN
fcis-13113	41	20	that	that	PRON
fcis-13113	41	21	do	do	AUX
fcis-13113	41	22	not	not	PART
fcis-13113	41	23	require	require	VERB
fcis-13113	41	24	human	human	ADJ
fcis-13113	41	25	labelling	labelling	NOUN
fcis-13113	41	26	to	to	PART
fcis-13113	41	27	learn	learn	VERB
fcis-13113	41	28	models	model	NOUN
fcis-13113	41	29	or	or	CCONJ
fcis-13113	41	30	compute	compute	NOUN
fcis-13113	41	31	metrics	metric	NOUN
fcis-13113	41	32	.	.	PUNCT
fcis-13113	42	1	supervised	supervise	VERB
fcis-13113	42	2	learning	learning	NOUN
fcis-13113	42	3	-	-	PUNCT
fcis-13113	42	4	based	base	VERB
fcis-13113	42	5	approaches	approach	NOUN
fcis-13113	42	6	learn	learn	VERB
fcis-13113	42	7	models	model	NOUN
fcis-13113	42	8	or	or	CCONJ
fcis-13113	42	9	compute	compute	NOUN
fcis-13113	42	10	metrics	metric	NOUN
fcis-13113	42	11	by	by	ADP
fcis-13113	42	12	training	train	VERB
fcis-13113	42	13	with	with	ADP
fcis-13113	42	14	human	human	ADJ
fcis-13113	42	15	labelled	label	VERB
fcis-13113	42	16	datasets	dataset	NOUN
fcis-13113	42	17	,	,	PUNCT
fcis-13113	42	18	which	which	PRON
fcis-13113	42	19	are	be	AUX
fcis-13113	42	20	usually	usually	ADV
fcis-13113	42	21	human	human	ADJ
fcis-13113	42	22	labelled	label	VERB
fcis-13113	42	23	reference	reference	NOUN
fcis-13113	42	24	translations	translation	NOUN
fcis-13113	42	25	.	.	PUNCT
fcis-13113	43	1	4.1	4.1	NUM
fcis-13113	43	2	.	.	PUNCT
fcis-13113	43	3	methodology	methodology	NOUN
fcis-13113	43	4	based	base	VERB
fcis-13113	43	5	on	on	ADP
fcis-13113	43	6	reference	reference	NOUN
fcis-13113	43	7	translations	translation	NOUN
fcis-13113	43	8	4.1.1	4.1.1	NUM
fcis-13113	43	9	.	.	PUNCT
fcis-13113	43	10	bleu	bleu	NOUN
fcis-13113	43	11	the	the	DET
fcis-13113	43	12	bleu	bleu	NOUN
fcis-13113	43	13	metric	metric	NOUN
fcis-13113	43	14	(	(	PUNCT
fcis-13113	43	15	bilingual	bilingual	ADJ
fcis-13113	43	16	evaluation	evaluation	NOUN
fcis-13113	43	17	understudy	understudy	NOUN
fcis-13113	43	18	)	)	PUNCT
fcis-13113	43	19	was	be	AUX
fcis-13113	43	20	proposed	propose	VERB
fcis-13113	43	21	by	by	ADP
fcis-13113	43	22	kishore	kishore	PROPN
fcis-13113	43	23	papineni	papineni	PROPN
fcis-13113	43	24	et	et	PROPN
fcis-13113	43	25	al	al	PROPN
fcis-13113	43	26	.	.	PROPN
fcis-13113	43	27	bleu	bleu	PROPN
fcis-13113	43	28	uses	use	VERB
fcis-13113	43	29	an	an	DET
fcis-13113	43	30	n	n	CCONJ
fcis-13113	43	31	-	-	PUNCT
fcis-13113	43	32	gram	gram	NOUN
fcis-13113	43	33	matching	matching	NOUN
fcis-13113	43	34	rule	rule	NOUN
fcis-13113	43	35	to	to	PART
fcis-13113	43	36	compare	compare	VERB
fcis-13113	43	37	the	the	DET
fcis-13113	43	38	percentage	percentage	NOUN
fcis-13113	43	39	of	of	ADP
fcis-13113	43	40	similarity	similarity	NOUN
fcis-13113	43	41	of	of	ADP
fcis-13113	43	42	n	n	PRON
fcis-13113	43	43	groups	group	NOUN
fcis-13113	43	44	of	of	ADP
fcis-13113	43	45	words	word	NOUN
fcis-13113	43	46	between	between	ADP
fcis-13113	43	47	mt	mt	PROPN
fcis-13113	43	48	results	result	NOUN
fcis-13113	43	49	and	and	CCONJ
fcis-13113	43	50	reference	reference	NOUN
fcis-13113	43	51	translations	translation	NOUN
fcis-13113	43	52	.	.	PUNCT
fcis-13113	44	1	1	1	NUM
fcis-13113	44	2	-	-	PUNCT
fcis-13113	44	3	gram	gram	NOUN
fcis-13113	44	4	results	result	NOUN
fcis-13113	44	5	tend	tend	VERB
fcis-13113	44	6	to	to	PART
fcis-13113	44	7	represent	represent	VERB
fcis-13113	44	8	adequacy	adequacy	NOUN
fcis-13113	44	9	,	,	PUNCT
fcis-13113	44	10	while	while	SCONJ
fcis-13113	44	11	2	2	NUM
fcis-13113	44	12	-	-	PUNCT
fcis-13113	44	13	gram	gram	NOUN
fcis-13113	44	14	and	and	CCONJ
fcis-13113	44	15	above	above	ADP
fcis-13113	44	16	matching	matching	NOUN
fcis-13113	44	17	results	result	NOUN
fcis-13113	44	18	represent	represent	VERB
fcis-13113	44	19	fluency	fluency	NOUN
fcis-13113	44	20	and	and	CCONJ
fcis-13113	44	21	text	text	NOUN
fcis-13113	44	22	readability	readability	NOUN
fcis-13113	44	23	.	.	PUNCT
fcis-13113	45	1	[	[	X
fcis-13113	45	2	1	1	X
fcis-13113	45	3	]	]	PUNCT
fcis-13113	45	4	the	the	DET
fcis-13113	45	5	advantages	advantage	NOUN
fcis-13113	45	6	of	of	ADP
fcis-13113	45	7	this	this	DET
fcis-13113	45	8	method	method	NOUN
fcis-13113	45	9	are	be	AUX
fcis-13113	45	10	:	:	PUNCT
fcis-13113	45	11	(	(	PUNCT
fcis-13113	45	12	1	1	X
fcis-13113	45	13	)	)	PUNCT
fcis-13113	45	14	fast	fast	ADJ
fcis-13113	45	15	calculation	calculation	NOUN
fcis-13113	45	16	speed	speed	NOUN
fcis-13113	45	17	;	;	PUNCT
fcis-13113	45	18	(	(	PUNCT
fcis-13113	45	19	2	2	X
fcis-13113	45	20	)	)	PUNCT
fcis-13113	45	21	low	low	ADJ
fcis-13113	45	22	resource	resource	NOUN
fcis-13113	45	23	languages	language	NOUN
fcis-13113	45	24	are	be	AUX
fcis-13113	45	25	also	also	ADV
fcis-13113	45	26	applicable	applicable	ADJ
fcis-13113	45	27	;	;	PUNCT
fcis-13113	45	28	(	(	PUNCT
fcis-13113	45	29	3	3	X
fcis-13113	45	30	)	)	PUNCT
fcis-13113	45	31	the	the	DET
fcis-13113	45	32	calculation	calculation	NOUN
fcis-13113	45	33	method	method	NOUN
fcis-13113	45	34	is	be	AUX
fcis-13113	45	35	easy	easy	ADJ
fcis-13113	45	36	to	to	PART
fcis-13113	45	37	understand	understand	VERB
fcis-13113	45	38	.	.	PUNCT
fcis-13113	46	1	but	but	CCONJ
fcis-13113	46	2	it	it	PRON
fcis-13113	46	3	also	also	ADV
fcis-13113	46	4	has	have	VERB
fcis-13113	46	5	disadvantages	disadvantage	NOUN
fcis-13113	46	6	,	,	PUNCT
fcis-13113	46	7	in	in	ADP
fcis-13113	46	8	the	the	DET
fcis-13113	46	9	process	process	NOUN
fcis-13113	46	10	of	of	ADP
fcis-13113	46	11	matching	match	VERB
fcis-13113	46	12	the	the	DET
fcis-13113	46	13	translation	translation	NOUN
fcis-13113	46	14	result	result	NOUN
fcis-13113	46	15	must	must	AUX
fcis-13113	46	16	match	match	VERB
fcis-13113	46	17	the	the	DET
fcis-13113	46	18	reference	reference	NOUN
fcis-13113	46	19	translation	translation	NOUN
fcis-13113	46	20	exactly	exactly	ADV
fcis-13113	46	21	ignoring	ignore	VERB
fcis-13113	46	22	the	the	DET
fcis-13113	46	23	syntactic	syntactic	ADJ
fcis-13113	46	24	semantics	semantic	NOUN
fcis-13113	46	25	,	,	PUNCT
fcis-13113	46	26	and	and	CCONJ
fcis-13113	46	27	even	even	ADV
fcis-13113	46	28	manual	manual	ADJ
fcis-13113	46	29	translation	translation	NOUN
fcis-13113	46	30	is	be	AUX
fcis-13113	46	31	impossible	impossible	ADJ
fcis-13113	46	32	to	to	PART
fcis-13113	46	33	do	do	VERB
fcis-13113	46	34	this	this	PRON
fcis-13113	46	35	.	.	PUNCT
fcis-13113	47	1	4.1.2	4.1.2	X
fcis-13113	47	2	.	.	X
fcis-13113	47	3	meteor	meteor	NOUN
fcis-13113	47	4	to	to	PART
fcis-13113	47	5	address	address	VERB
fcis-13113	47	6	the	the	DET
fcis-13113	47	7	shortcomings	shortcoming	NOUN
fcis-13113	47	8	of	of	ADP
fcis-13113	47	9	bleu	bleu	NOUN
fcis-13113	47	10	,	,	PUNCT
fcis-13113	47	11	the	the	DET
fcis-13113	47	12	researchers	researcher	NOUN
fcis-13113	47	13	proposed	propose	VERB
fcis-13113	47	14	the	the	DET
fcis-13113	47	15	meteor	meteor	NOUN
fcis-13113	47	16	(	(	PUNCT
fcis-13113	47	17	metric	metric	ADJ
fcis-13113	47	18	for	for	ADP
fcis-13113	47	19	evaluation	evaluation	NOUN
fcis-13113	47	20	of	of	ADP
fcis-13113	47	21	translation	translation	NOUN
fcis-13113	47	22	with	with	ADP
fcis-13113	47	23	explicit	explicit	ADJ
fcis-13113	47	24	ordering	ordering	NOUN
fcis-13113	47	25	)	)	PUNCT
fcis-13113	47	26	method	method	NOUN
fcis-13113	47	27	.	.	PUNCT
fcis-13113	48	1	meteor	meteor	NOUN
fcis-13113	48	2	takes	take	VERB
fcis-13113	48	3	into	into	ADP
fcis-13113	48	4	account	account	NOUN
fcis-13113	48	5	both	both	DET
fcis-13113	48	6	accuracy	accuracy	NOUN
fcis-13113	48	7	and	and	CCONJ
fcis-13113	48	8	recall	recall	VERB
fcis-13113	48	9	and	and	CCONJ
fcis-13113	48	10	proposes	propose	VERB
fcis-13113	48	11	an	an	DET
fcis-13113	48	12	automatic	automatic	ADJ
fcis-13113	48	13	evaluation	evaluation	NOUN
fcis-13113	48	14	based	base	VERB
fcis-13113	48	15	on	on	ADP
fcis-13113	48	16	the	the	DET
fcis-13113	48	17	weighted	weight	VERB
fcis-13113	48	18	reconciled	reconcile	VERB
fcis-13113	48	19	mean	mean	NOUN
fcis-13113	48	20	of	of	ADP
fcis-13113	48	21	the	the	DET
fcis-13113	48	22	single	single	ADJ
fcis-13113	48	23	-	-	PUNCT
fcis-13113	48	24	precision	precision	NOUN
fcis-13113	48	25	and	and	CCONJ
fcis-13113	48	26	single	single	ADJ
fcis-13113	48	27	-	-	PUNCT
fcis-13113	48	28	word	word	NOUN
fcis-13113	48	29	recall	recall	NOUN
fcis-13113	48	30	metrics	metric	NOUN
fcis-13113	48	31	[	[	X
fcis-13113	48	32	9	9	NUM
fcis-13113	48	33	]	]	PUNCT
fcis-13113	48	34	.	.	PUNCT
fcis-13113	49	1	meanwhile	meanwhile	ADV
fcis-13113	49	2	,	,	PUNCT
fcis-13113	49	3	it	it	PRON
fcis-13113	49	4	expands	expand	VERB
fcis-13113	49	5	the	the	DET
fcis-13113	49	6	synonym	synonym	NOUN
fcis-13113	49	7	set	set	VERB
fcis-13113	49	8	with	with	ADP
fcis-13113	49	9	knowledge	knowledge	NOUN
fcis-13113	49	10	sources	source	NOUN
fcis-13113	49	11	such	such	ADJ
fcis-13113	49	12	as	as	ADP
fcis-13113	49	13	wordnet	wordnet	NOUN
fcis-13113	49	14	,	,	PUNCT
fcis-13113	49	15	while	while	SCONJ
fcis-13113	49	16	considering	consider	VERB
fcis-13113	49	17	the	the	DET
fcis-13113	49	18	lexical	lexical	ADJ
fcis-13113	49	19	properties	property	NOUN
fcis-13113	49	20	of	of	ADP
fcis-13113	49	21	words	word	NOUN
fcis-13113	49	22	.	.	PUNCT
fcis-13113	50	1	in	in	ADP
fcis-13113	50	2	summary	summary	NOUN
fcis-13113	50	3	,	,	PUNCT
fcis-13113	50	4	compared	compare	VERB
fcis-13113	50	5	with	with	ADP
fcis-13113	50	6	bleu	bleu	NOUN
fcis-13113	50	7	and	and	CCONJ
fcis-13113	50	8	nist	nist	NOUN
fcis-13113	50	9	,	,	PUNCT
fcis-13113	50	10	meteor	meteor	NOUN
fcis-13113	50	11	considers	consider	VERB
fcis-13113	50	12	more	more	ADV
fcis-13113	50	13	semantic	semantic	ADJ
fcis-13113	50	14	and	and	CCONJ
fcis-13113	50	15	lexical	lexical	ADJ
fcis-13113	50	16	information	information	NOUN
fcis-13113	50	17	and	and	CCONJ
fcis-13113	50	18	provides	provide	VERB
fcis-13113	50	19	a	a	DET
fcis-13113	50	20	more	more	ADV
fcis-13113	50	21	accurate	accurate	ADJ
fcis-13113	50	22	assessment	assessment	NOUN
fcis-13113	50	23	of	of	ADP
fcis-13113	50	24	mt	mt	PROPN
fcis-13113	50	25	quality	quality	NOUN
fcis-13113	50	26	.	.	PUNCT
fcis-13113	51	1	4.1.3	4.1.3	X
fcis-13113	51	2	.	.	PUNCT
fcis-13113	51	3	other	other	ADJ
fcis-13113	51	4	methods	method	NOUN
fcis-13113	51	5	based	base	VERB
fcis-13113	51	6	on	on	ADP
fcis-13113	51	7	reference	reference	NOUN
fcis-13113	51	8	translation	translation	NOUN
fcis-13113	51	9	in	in	ADP
fcis-13113	51	10	wmt19	wmt19	NOUN
fcis-13113	51	11	,	,	PUNCT
fcis-13113	51	12	which	which	PRON
fcis-13113	51	13	is	be	AUX
fcis-13113	51	14	also	also	ADV
fcis-13113	51	15	a	a	DET
fcis-13113	51	16	reference	reference	NOUN
fcis-13113	51	17	translation	translation	NOUN
fcis-13113	51	18	-	-	PUNCT
fcis-13113	51	19	based	base	VERB
fcis-13113	51	20	method	method	NOUN
fcis-13113	51	21	,	,	PUNCT
fcis-13113	51	22	character	character	NOUN
fcis-13113	51	23	n	n	CCONJ
fcis-13113	51	24	-	-	PUNCT
fcis-13113	51	25	gram	gram	NOUN
fcis-13113	51	26	f	f	NOUN
fcis-13113	51	27	-	-	PUNCT
fcis-13113	51	28	score	score	NOUN
fcis-13113	51	29	(	(	PUNCT
fcis-13113	51	30	chrf	chrf	NOUN
fcis-13113	51	31	)	)	PUNCT
fcis-13113	51	32	demonstrates	demonstrate	VERB
fcis-13113	51	33	a	a	DET
fcis-13113	51	34	higher	high	ADJ
fcis-13113	51	35	correlation	correlation	NOUN
fcis-13113	51	36	with	with	ADP
fcis-13113	51	37	human	human	ADJ
fcis-13113	51	38	judgement	judgement	NOUN
fcis-13113	51	39	than	than	SCONJ
fcis-13113	51	40	bleu.chrf	bleu.chrf	PRON
fcis-13113	51	41	is	be	AUX
fcis-13113	51	42	based	base	VERB
fcis-13113	51	43	on	on	ADP
fcis-13113	51	44	character	character	NOUN
fcis-13113	51	45	level	level	NOUN
fcis-13113	51	46	n	n	CCONJ
fcis-13113	51	47	-	-	PUNCT
fcis-13113	51	48	gram	gram	NOUN
fcis-13113	51	49	matching	matching	NOUN
fcis-13113	51	50	to	to	PART
fcis-13113	51	51	calculate	calculate	VERB
fcis-13113	51	52	and	and	CCONJ
fcis-13113	51	53	evaluate	evaluate	VERB
fcis-13113	51	54	the	the	DET
fcis-13113	51	55	quality	quality	NOUN
fcis-13113	51	56	of	of	ADP
fcis-13113	51	57	mt	mt	PROPN
fcis-13113	51	58	results	result	NOUN
fcis-13113	51	59	.	.	PUNCT
fcis-13113	52	1	firstly	firstly	ADV
fcis-13113	52	2	,	,	PUNCT
fcis-13113	52	3	the	the	DET
fcis-13113	52	4	mt	mt	PROPN
fcis-13113	52	5	results	result	NOUN
fcis-13113	52	6	and	and	CCONJ
fcis-13113	52	7	reference	reference	NOUN
fcis-13113	52	8	translations	translation	NOUN
fcis-13113	52	9	are	be	AUX
fcis-13113	52	10	segmented	segment	VERB
fcis-13113	52	11	at	at	ADP
fcis-13113	52	12	the	the	DET
fcis-13113	52	13	character	character	NOUN
fcis-13113	52	14	level	level	NOUN
fcis-13113	52	15	,	,	PUNCT
fcis-13113	52	16	and	and	CCONJ
fcis-13113	52	17	then	then	ADV
fcis-13113	52	18	the	the	DET
fcis-13113	52	19	n	n	CCONJ
fcis-13113	52	20	-	-	PUNCT
fcis-13113	52	21	gram	gram	NOUN
fcis-13113	52	22	match	match	NOUN
fcis-13113	52	23	is	be	AUX
fcis-13113	52	24	calculated	calculate	VERB
fcis-13113	52	25	.	.	PUNCT
fcis-13113	53	1	finally	finally	ADV
fcis-13113	53	2	,	,	PUNCT
fcis-13113	53	3	the	the	DET
fcis-13113	53	4	precision	precision	NOUN
fcis-13113	53	5	rate	rate	NOUN
fcis-13113	53	6	,	,	PUNCT
fcis-13113	53	7	recall	recall	NOUN
fcis-13113	53	8	rate	rate	NOUN
fcis-13113	53	9	and	and	CCONJ
fcis-13113	53	10	f	f	X
fcis-13113	53	11	-	-	PUNCT
fcis-13113	53	12	score	score	NOUN
fcis-13113	53	13	are	be	AUX
fcis-13113	53	14	calculated	calculate	VERB
fcis-13113	53	15	based	base	VERB
fcis-13113	53	16	on	on	ADP
fcis-13113	53	17	the	the	DET
fcis-13113	53	18	matching	matching	NOUN
fcis-13113	53	19	degree	degree	NOUN
fcis-13113	53	20	.	.	PUNCT
fcis-13113	54	1	compared	compare	VERB
fcis-13113	54	2	with	with	ADP
fcis-13113	54	3	bleu	bleu	NOUN
fcis-13113	54	4	,	,	PUNCT
fcis-13113	54	5	chrf	chrf	NOUN
fcis-13113	54	6	is	be	AUX
fcis-13113	54	7	more	more	ADV
fcis-13113	54	8	suitable	suitable	ADJ
fcis-13113	54	9	for	for	ADP
fcis-13113	54	10	evaluating	evaluate	VERB
fcis-13113	54	11	some	some	DET
fcis-13113	54	12	non	non	ADJ
fcis-13113	54	13	-	-	ADJ
fcis-13113	54	14	standardized	standardized	ADJ
fcis-13113	54	15	texts	text	NOUN
fcis-13113	54	16	,	,	PUNCT
fcis-13113	54	17	such	such	ADJ
fcis-13113	54	18	as	as	ADP
fcis-13113	54	19	the	the	DET
fcis-13113	54	20	spoken	spoken	ADJ
fcis-13113	54	21	language	language	NOUN
fcis-13113	54	22	domain	domain	NOUN
fcis-13113	54	23	.	.	PUNCT
fcis-13113	55	1	there	there	PRON
fcis-13113	55	2	are	be	VERB
fcis-13113	55	3	many	many	ADJ
fcis-13113	55	4	other	other	ADJ
fcis-13113	55	5	methods	method	NOUN
fcis-13113	55	6	based	base	VERB
fcis-13113	55	7	on	on	ADP
fcis-13113	55	8	reference	reference	NOUN
fcis-13113	55	9	translation	translation	NOUN
fcis-13113	55	10	,	,	PUNCT
fcis-13113	55	11	which	which	PRON
fcis-13113	55	12	are	be	AUX
fcis-13113	55	13	not	not	PART
fcis-13113	55	14	listed	list	VERB
fcis-13113	55	15	here	here	ADV
fcis-13113	55	16	.	.	PUNCT
fcis-13113	56	1	the	the	DET
fcis-13113	56	2	central	central	ADJ
fcis-13113	56	3	idea	idea	NOUN
fcis-13113	56	4	of	of	ADP
fcis-13113	56	5	the	the	DET
fcis-13113	56	6	automatic	automatic	ADJ
fcis-13113	56	7	evaluation	evaluation	NOUN
fcis-13113	56	8	method	method	NOUN
fcis-13113	56	9	based	base	VERB
fcis-13113	56	10	on	on	ADP
fcis-13113	56	11	reference	reference	NOUN
fcis-13113	56	12	translation	translation	NOUN
fcis-13113	56	13	is	be	AUX
fcis-13113	56	14	to	to	PART
fcis-13113	56	15	evaluate	evaluate	VERB
fcis-13113	56	16	the	the	DET
fcis-13113	56	17	quality	quality	NOUN
fcis-13113	56	18	of	of	ADP
fcis-13113	56	19	mt	mt	PROPN
fcis-13113	56	20	by	by	ADP
fcis-13113	56	21	comparing	compare	VERB
fcis-13113	56	22	the	the	DET
fcis-13113	56	23	result	result	NOUN
fcis-13113	56	24	of	of	ADP
fcis-13113	56	25	mt	mt	PROPN
fcis-13113	56	26	with	with	ADP
fcis-13113	56	27	the	the	DET
fcis-13113	56	28	reference	reference	NOUN
fcis-13113	56	29	translation	translation	NOUN
fcis-13113	56	30	,	,	PUNCT
fcis-13113	56	31	which	which	PRON
fcis-13113	56	32	can	can	AUX
fcis-13113	56	33	not	not	PART
fcis-13113	56	34	be	be	AUX
fcis-13113	56	35	used	use	VERB
fcis-13113	56	36	if	if	SCONJ
fcis-13113	56	37	there	there	PRON
fcis-13113	56	38	is	be	VERB
fcis-13113	56	39	no	no	DET
fcis-13113	56	40	reference	reference	NOUN
fcis-13113	56	41	translation	translation	NOUN
fcis-13113	56	42	.	.	PUNCT
fcis-13113	57	1	4.2	4.2	NUM
fcis-13113	57	2	.	.	PUNCT
fcis-13113	58	1	unsupervised	unsupervised	ADJ
fcis-13113	58	2	learning	learning	NOUN
fcis-13113	58	3	based	base	VERB
fcis-13113	58	4	approach	approach	NOUN
fcis-13113	58	5	yisi	yisi	VERB
fcis-13113	58	6	is	be	AUX
fcis-13113	58	7	an	an	DET
fcis-13113	58	8	automatic	automatic	ADJ
fcis-13113	58	9	evaluation	evaluation	NOUN
fcis-13113	58	10	framework	framework	NOUN
fcis-13113	58	11	that	that	PRON
fcis-13113	58	12	can	can	AUX
fcis-13113	58	13	be	be	AUX
fcis-13113	58	14	applied	apply	VERB
fcis-13113	58	15	to	to	ADP
fcis-13113	58	16	different	different	ADJ
fcis-13113	58	17	resource	resource	NOUN
fcis-13113	58	18	languages	language	NOUN
fcis-13113	58	19	,	,	PUNCT
fcis-13113	58	20	which	which	PRON
fcis-13113	58	21	utilises	utilise	VERB
fcis-13113	58	22	two	two	NUM
fcis-13113	58	23	measurement	measurement	NOUN
fcis-13113	58	24	models	model	NOUN
fcis-13113	58	25	and	and	CCONJ
fcis-13113	58	26	provides	provide	VERB
fcis-13113	58	27	the	the	DET
fcis-13113	58	28	option	option	NOUN
fcis-13113	58	29	to	to	PART
fcis-13113	58	30	fall	fall	VERB
fcis-13113	58	31	back	back	ADV
fcis-13113	58	32	to	to	ADP
fcis-13113	58	33	a	a	DET
fcis-13113	58	34	lexically	lexically	ADV
fcis-13113	58	35	-	-	PUNCT
fcis-13113	58	36	based	base	VERB
fcis-13113	58	37	evaluation	evaluation	NOUN
fcis-13113	58	38	method	method	NOUN
fcis-13113	58	39	such	such	ADJ
fcis-13113	58	40	as	as	ADP
fcis-13113	58	41	bleu	bleu	NOUN
fcis-13113	58	42	when	when	SCONJ
fcis-13113	58	43	a	a	DET
fcis-13113	58	44	semantic	semantic	ADJ
fcis-13113	58	45	model	model	NOUN
fcis-13113	58	46	of	of	ADP
fcis-13113	58	47	the	the	DET
fcis-13113	58	48	evaluated	evaluated	ADJ
fcis-13113	58	49	language	language	NOUN
fcis-13113	58	50	is	be	AUX
fcis-13113	58	51	not	not	PART
fcis-13113	58	52	available	available	ADJ
fcis-13113	58	53	.	.	PUNCT
fcis-13113	59	1	yisi	yisi	PROPN
fcis-13113	59	2	was	be	AUX
fcis-13113	59	3	first	first	ADV
fcis-13113	59	4	used	use	VERB
fcis-13113	59	5	in	in	ADP
fcis-13113	59	6	the	the	DET
fcis-13113	59	7	wmt18	wmt18	NOUN
fcis-13113	59	8	shared	share	VERB
fcis-13113	59	9	tasks	task	NOUN
fcis-13113	59	10	and	and	CCONJ
fcis-13113	59	11	performed	perform	VERB
fcis-13113	59	12	well	well	ADV
fcis-13113	59	13	and	and	CCONJ
fcis-13113	59	14	consistently	consistently	ADV
fcis-13113	59	15	in	in	ADP
fcis-13113	59	16	tests	test	NOUN
fcis-13113	59	17	related	relate	VERB
fcis-13113	59	18	to	to	ADP
fcis-13113	59	19	human	human	ADJ
fcis-13113	59	20	judgement	judgement	NOUN
fcis-13113	59	21	.	.	PUNCT
fcis-13113	60	1	there	there	PRON
fcis-13113	60	2	are	be	VERB
fcis-13113	60	3	three	three	NUM
fcis-13113	60	4	versions	version	NOUN
fcis-13113	60	5	of	of	ADP
fcis-13113	60	6	yisi	yisi	NOUN
fcis-13113	60	7	:	:	PUNCT
fcis-13113	60	8	yisi-0	yisi-0	PROPN
fcis-13113	60	9	,	,	PUNCT
fcis-13113	60	10	yisi-1	yisi-1	PROPN
fcis-13113	60	11	,	,	PUNCT
fcis-13113	60	12	and	and	CCONJ
fcis-13113	60	13	yisi-2	yisi-2	NUM
fcis-13113	60	14	.	.	PUNCT
fcis-13113	61	1	yisi-0	yisi-0	PROPN
fcis-13113	61	2	is	be	AUX
fcis-13113	61	3	a	a	DET
fcis-13113	61	4	resource	resource	NOUN
fcis-13113	61	5	-	-	PUNCT
fcis-13113	61	6	free	free	ADJ
fcis-13113	61	7	variant	variant	NOUN
fcis-13113	61	8	of	of	ADP
fcis-13113	61	9	yisi	yisi	NOUN
fcis-13113	61	10	that	that	PRON
fcis-13113	61	11	uses	use	VERB
fcis-13113	61	12	the	the	DET
fcis-13113	61	13	longest	long	ADJ
fcis-13113	61	14	common	common	ADJ
fcis-13113	61	15	character	character	NOUN
fcis-13113	61	16	substring	substre	VERB
fcis-13113	61	17	accuracy	accuracy	NOUN
fcis-13113	61	18	to	to	PART
fcis-13113	61	19	evaluate	evaluate	VERB
fcis-13113	61	20	the	the	DET
fcis-13113	61	21	lexical	lexical	ADJ
fcis-13113	61	22	similarity	similarity	NOUN
fcis-13113	61	23	between	between	ADP
fcis-13113	61	24	the	the	DET
fcis-13113	61	25	mt	mt	PROPN
fcis-13113	61	26	result	result	NOUN
fcis-13113	61	27	and	and	CCONJ
fcis-13113	61	28	the	the	DET
fcis-13113	61	29	reference	reference	NOUN
fcis-13113	61	30	translation	translation	NOUN
fcis-13113	61	31	.	.	PUNCT
fcis-13113	62	1	yisi-1	yisi-1	NUM
fcis-13113	62	2	is	be	AUX
fcis-13113	62	3	a	a	DET
fcis-13113	62	4	monolingual	monolingual	ADJ
fcis-13113	62	5	variant	variant	NOUN
fcis-13113	62	6	of	of	ADP
fcis-13113	62	7	yisi	yisi	NOUN
fcis-13113	62	8	that	that	PRON
fcis-13113	62	9	uses	use	VERB
fcis-13113	62	10	an	an	DET
fcis-13113	62	11	embedding	embed	VERB
fcis-13113	62	12	model	model	NOUN
fcis-13113	62	13	to	to	PART
fcis-13113	62	14	evaluate	evaluate	VERB
fcis-13113	62	15	lexical	lexical	ADJ
fcis-13113	62	16	semantic	semantic	ADJ
fcis-13113	62	17	similarity	similarity	NOUN
fcis-13113	62	18	and	and	CCONJ
fcis-13113	62	19	a	a	DET
fcis-13113	62	20	semantic	semantic	ADJ
fcis-13113	62	21	role	role	NOUN
fcis-13113	62	22	tagger	tagger	NOUN
fcis-13113	62	23	to	to	PART
fcis-13113	62	24	evaluate	evaluate	VERB
fcis-13113	62	25	structural	structural	ADJ
fcis-13113	62	26	semantic	semantic	ADJ
fcis-13113	62	27	similarity	similarity	NOUN
fcis-13113	62	28	.	.	PUNCT
fcis-13113	63	1	yisi-2	yisi-2	NUM
fcis-13113	63	2	is	be	AUX
fcis-13113	63	3	a	a	DET
fcis-13113	63	4	cross	cross	ADJ
fcis-13113	63	5	-	-	ADJ
fcis-13113	63	6	linguistic	linguistic	ADJ
fcis-13113	63	7	variant	variant	NOUN
fcis-13113	63	8	of	of	ADP
fcis-13113	63	9	yisi	yisi	NOUN
fcis-13113	63	10	,	,	PUNCT
fcis-13113	63	11	which	which	PRON
fcis-13113	63	12	uses	use	VERB
fcis-13113	63	13	a	a	DET
fcis-13113	63	14	cross	cross	ADJ
fcis-13113	63	15	-	-	ADJ
fcis-13113	63	16	linguistic	linguistic	ADJ
fcis-13113	63	17	embedding	embed	VERB
fcis-13113	63	18	model	model	NOUN
fcis-13113	63	19	to	to	PART
fcis-13113	63	20	evaluate	evaluate	VERB
fcis-13113	63	21	crosslinguistic	crosslinguistic	ADJ
fcis-13113	63	22	lexical	lexical	ADJ
fcis-13113	63	23	semantic	semantic	ADJ
fcis-13113	63	24	similarity	similarity	NOUN
fcis-13113	63	25	,	,	PUNCT
fcis-13113	63	26	along	along	ADP
fcis-13113	63	27	with	with	ADP
fcis-13113	63	28	a	a	DET
fcis-13113	63	29	semantic	semantic	ADJ
fcis-13113	63	30	role	role	NOUN
fcis-13113	63	31	tagger	tagger	NOUN
fcis-13113	63	32	to	to	PART
fcis-13113	63	33	evaluate	evaluate	VERB
fcis-13113	63	34	structural	structural	ADJ
fcis-13113	63	35	semantic	semantic	ADJ
fcis-13113	63	36	similarity	similarity	NOUN
fcis-13113	63	37	.	.	PUNCT
fcis-13113	64	1	4.3	4.3	NUM
fcis-13113	64	2	.	.	PUNCT
fcis-13113	64	3	supervised	supervised	ADJ
fcis-13113	64	4	learning	learning	NOUN
fcis-13113	64	5	based	base	VERB
fcis-13113	64	6	approach	approach	NOUN
fcis-13113	64	7	4.3.1	4.3.1	NUM
fcis-13113	64	8	.	.	PUNCT
fcis-13113	65	1	beer	beer	NOUN
fcis-13113	65	2	beer	beer	NOUN
fcis-13113	65	3	(	(	PUNCT
fcis-13113	65	4	better	well	ADJ
fcis-13113	65	5	evaluation	evaluation	NOUN
fcis-13113	65	6	as	as	ADP
fcis-13113	65	7	ranking	ranking	NOUN
fcis-13113	65	8	)	)	PUNCT
fcis-13113	65	9	is	be	AUX
fcis-13113	65	10	a	a	DET
fcis-13113	65	11	supervised	supervised	ADJ
fcis-13113	65	12	learning	learning	NOUN
fcis-13113	65	13	-	-	PUNCT
fcis-13113	65	14	based	base	VERB
fcis-13113	65	15	method	method	NOUN
fcis-13113	65	16	for	for	ADP
fcis-13113	65	17	automatically	automatically	ADV
fcis-13113	65	18	evaluating	evaluate	VERB
fcis-13113	65	19	the	the	DET
fcis-13113	65	20	quality	quality	NOUN
fcis-13113	65	21	of	of	ADP
fcis-13113	65	22	mt	mt	PROPN
fcis-13113	65	23	that	that	PRON
fcis-13113	65	24	can	can	AUX
fcis-13113	65	25	combine	combine	VERB
fcis-13113	65	26	a	a	DET
fcis-13113	65	27	large	large	ADJ
fcis-13113	65	28	number	number	NOUN
fcis-13113	65	29	of	of	ADP
fcis-13113	65	30	features	feature	NOUN
fcis-13113	65	31	into	into	ADP
fcis-13113	65	32	a	a	DET
fcis-13113	65	33	linear	linear	ADJ
fcis-13113	65	34	model	model	NOUN
fcis-13113	65	35	.	.	PUNCT
fcis-13113	66	1	beer	beer	PROPN
fcis-13113	66	2	uses	use	VERB
fcis-13113	66	3	character	character	NOUN
fcis-13113	66	4	-	-	PUNCT
fcis-13113	66	5	level	level	NOUN
fcis-13113	66	6	n	n	CCONJ
fcis-13113	66	7	-	-	PUNCT
fcis-13113	66	8	grams	gram	NOUN
fcis-13113	66	9	to	to	PART
fcis-13113	66	10	compute	compute	VERB
fcis-13113	66	11	similarity	similarity	NOUN
fcis-13113	66	12	and	and	CCONJ
fcis-13113	66	13	alignment	alignment	NOUN
fcis-13113	66	14	trees	tree	NOUN
fcis-13113	66	15	to	to	PART
fcis-13113	66	16	compute	compute	VERB
fcis-13113	66	17	fluency	fluency	NOUN
fcis-13113	66	18	of	of	ADP
fcis-13113	66	19	input	input	NOUN
fcis-13113	66	20	sentences	sentence	NOUN
fcis-13113	66	21	.	.	PUNCT
fcis-13113	67	1	the	the	DET
fcis-13113	67	2	model	model	NOUN
fcis-13113	67	3	is	be	AUX
fcis-13113	67	4	also	also	ADV
fcis-13113	67	5	trained	train	VERB
fcis-13113	67	6	using	use	VERB
fcis-13113	67	7	labelled	label	VERB
fcis-13113	67	8	data	datum	NOUN
fcis-13113	67	9	to	to	PART
fcis-13113	67	10	increase	increase	VERB
fcis-13113	67	11	the	the	DET
fcis-13113	67	12	similarity	similarity	NOUN
fcis-13113	67	13	with	with	ADP
fcis-13113	67	14	human	human	ADJ
fcis-13113	67	15	rankings	ranking	NOUN
fcis-13113	67	16	.	.	PUNCT
fcis-13113	68	1	4.3.2	4.3.2	X
fcis-13113	68	2	.	.	PUNCT
fcis-13113	69	1	comet	comet	PROPN
fcis-13113	69	2	cross	cross	PROPN
fcis-13113	69	3	-	-	ADJ
fcis-13113	69	4	lingual	lingual	ADJ
fcis-13113	69	5	optimised	optimise	VERB
fcis-13113	69	6	metric	metric	NOUN
fcis-13113	69	7	for	for	ADP
fcis-13113	69	8	evaluation	evaluation	NOUN
fcis-13113	69	9	of	of	ADP
fcis-13113	69	10	translation	translation	NOUN
fcis-13113	69	11	(	(	PUNCT
fcis-13113	69	12	comet	comet	PROPN
fcis-13113	69	13	)	)	PUNCT
fcis-13113	69	14	is	be	AUX
fcis-13113	69	15	a	a	DET
fcis-13113	69	16	neural	neural	ADJ
fcis-13113	69	17	network	network	NOUN
fcis-13113	69	18	based	base	VERB
fcis-13113	69	19	on	on	ADP
fcis-13113	69	20	pytorch	pytorch	NOUN
fcis-13113	69	21	framework	framework	NOUN
fcis-13113	69	22	for	for	ADP
fcis-13113	69	23	training	train	VERB
fcis-13113	69	24	multilingual	multilingual	ADJ
fcis-13113	69	25	mt	mt	PROPN
fcis-13113	69	26	evaluation	evaluation	NOUN
fcis-13113	69	27	models	model	NOUN
fcis-13113	69	28	.	.	PUNCT
fcis-13113	70	1	comet	comet	PROPN
fcis-13113	70	2	supports	support	VERB
fcis-13113	70	3	two	two	NUM
fcis-13113	70	4	different	different	ADJ
fcis-13113	70	5	architectures	architecture	NOUN
fcis-13113	70	6	:	:	PUNCT
fcis-13113	70	7	the	the	DET
fcis-13113	70	8	estimator	estimator	NOUN
fcis-13113	70	9	model	model	NOUN
fcis-13113	70	10	and	and	CCONJ
fcis-13113	70	11	the	the	DET
fcis-13113	70	12	translation	translation	NOUN
fcis-13113	70	13	ranking	ranking	NOUN
fcis-13113	70	14	model	model	NOUN
fcis-13113	70	15	.	.	PUNCT
fcis-13113	71	1	the	the	DET
fcis-13113	71	2	fundamental	fundamental	ADJ
fcis-13113	71	3	difference	difference	NOUN
fcis-13113	71	4	between	between	ADP
fcis-13113	71	5	these	these	DET
fcis-13113	71	6	two	two	NUM
fcis-13113	71	7	models	model	NOUN
fcis-13113	71	8	lies	lie	VERB
fcis-13113	71	9	in	in	ADP
fcis-13113	71	10	the	the	DET
fcis-13113	71	11	training	training	NOUN
fcis-13113	71	12	objective	objective	NOUN
fcis-13113	71	13	,	,	PUNCT
fcis-13113	71	14	which	which	PRON
fcis-13113	71	15	is	be	AUX
fcis-13113	71	16	to	to	PART
fcis-13113	71	17	regress	regress	VERB
fcis-13113	71	18	the	the	DET
fcis-13113	71	19	quality	quality	NOUN
fcis-13113	71	20	scores	score	NOUN
fcis-13113	71	21	directly	directly	ADV
fcis-13113	71	22	,	,	PUNCT
fcis-13113	71	23	while	while	SCONJ
fcis-13113	71	24	the	the	DET
fcis-13113	71	25	training	training	NOUN
fcis-13113	71	26	objective	objective	NOUN
fcis-13113	71	27	of	of	ADP
fcis-13113	71	28	the	the	DET
fcis-13113	71	29	latter	latter	NOUN
fcis-13113	71	30	is	be	AUX
fcis-13113	71	31	to	to	PART
fcis-13113	71	32	minimise	minimise	VERB
fcis-13113	71	33	the	the	DET
fcis-13113	71	34	distance	distance	NOUN
fcis-13113	71	35	between	between	ADP
fcis-13113	71	36	the	the	DET
fcis-13113	71	37	"	"	PUNCT
fcis-13113	71	38	good	good	ADJ
fcis-13113	71	39	"	"	PUNCT
fcis-13113	71	40	mt	mt	PROPN
fcis-13113	71	41	results	result	NOUN
fcis-13113	71	42	and	and	CCONJ
fcis-13113	71	43	the	the	DET
fcis-13113	71	44	reference	reference	NOUN
fcis-13113	71	45	translation	translation	NOUN
fcis-13113	71	46	.	.	PUNCT
fcis-13113	72	1	both	both	DET
fcis-13113	72	2	models	model	NOUN
fcis-13113	72	3	are	be	AUX
fcis-13113	72	4	composed	compose	VERB
fcis-13113	72	5	of	of	ADP
fcis-13113	72	6	a	a	DET
fcis-13113	72	7	cross	cross	ADJ
fcis-13113	72	8	-	-	ADJ
fcis-13113	72	9	lingual	lingual	ADJ
fcis-13113	72	10	encoder	encoder	NOUN
fcis-13113	72	11	and	and	CCONJ
fcis-13113	72	12	a	a	DET
fcis-13113	72	13	pooling	pool	VERB
fcis-13113	72	14	layer	layer	NOUN
fcis-13113	72	15	,	,	PUNCT
fcis-13113	72	16	and	and	CCONJ
fcis-13113	72	17	comet	comet	NOUN
fcis-13113	72	18	has	have	VERB
fcis-13113	72	19	the	the	DET
fcis-13113	72	20	advantage	advantage	NOUN
fcis-13113	72	21	of	of	ADP
fcis-13113	72	22	being	be	AUX
fcis-13113	72	23	easily	easily	ADV
fcis-13113	72	24	adapted	adapt	VERB
fcis-13113	72	25	to	to	ADP
fcis-13113	72	26	and	and	CCONJ
fcis-13113	72	27	optimised	optimise	VERB
fcis-13113	72	28	for	for	ADP
fcis-13113	72	29	different	different	ADJ
fcis-13113	72	30	types	type	NOUN
fcis-13113	72	31	of	of	ADP
fcis-13113	72	32	human	human	ADJ
fcis-13113	72	33	judgements	judgement	NOUN
fcis-13113	72	34	on	on	ADP
fcis-13113	72	35	the	the	DET
fcis-13113	72	36	quality	quality	NOUN
fcis-13113	72	37	of	of	ADP
fcis-13113	72	38	mts	mts	NOUN
fcis-13113	73	1	[	[	X
fcis-13113	73	2	9	9	NUM
fcis-13113	73	3	]	]	PUNCT
fcis-13113	73	4	.	.	PUNCT
fcis-13113	74	1	4.3.3	4.3.3	X
fcis-13113	74	2	.	.	PUNCT
fcis-13113	75	1	other	other	ADJ
fcis-13113	75	2	supervised	supervised	ADJ
fcis-13113	75	3	learning	learning	NOUN
fcis-13113	75	4	based	base	VERB
fcis-13113	75	5	methods	method	NOUN
fcis-13113	75	6	with	with	ADP
fcis-13113	75	7	the	the	DET
fcis-13113	75	8	development	development	NOUN
fcis-13113	75	9	of	of	ADP
fcis-13113	75	10	deep	deep	ADJ
fcis-13113	75	11	learning	learning	NOUN
fcis-13113	75	12	and	and	CCONJ
fcis-13113	75	13	neural	neural	ADJ
fcis-13113	75	14	networks	network	NOUN
fcis-13113	75	15	,	,	PUNCT
fcis-13113	75	16	there	there	PRON
fcis-13113	75	17	are	be	VERB
fcis-13113	75	18	many	many	ADJ
fcis-13113	75	19	supervised	supervised	ADJ
fcis-13113	75	20	learning	learning	NOUN
fcis-13113	75	21	based	base	VERB
fcis-13113	75	22	automatic	automatic	ADJ
fcis-13113	75	23	methods	method	NOUN
fcis-13113	75	24	for	for	ADP
fcis-13113	75	25	evaluating	evaluate	VERB
fcis-13113	75	26	the	the	DET
fcis-13113	75	27	quality	quality	NOUN
fcis-13113	75	28	of	of	ADP
fcis-13113	75	29	mt	mt	PROPN
fcis-13113	75	30	such	such	ADJ
fcis-13113	75	31	as	as	ADP
fcis-13113	75	32	ruse	ruse	NOUN
fcis-13113	75	33	,	,	PUNCT
fcis-13113	75	34	blend	blend	ADJ
fcis-13113	75	35	,	,	PUNCT
fcis-13113	75	36	bleurt	bleurt	ADJ
fcis-13113	75	37	and	and	CCONJ
fcis-13113	75	38	so	so	ADV
fcis-13113	75	39	on	on	ADV
fcis-13113	75	40	in	in	ADP
fcis-13113	75	41	addition	addition	NOUN
fcis-13113	75	42	to	to	ADP
fcis-13113	75	43	beer	beer	NOUN
fcis-13113	75	44	and	and	CCONJ
fcis-13113	75	45	comet	comet	NOUN
fcis-13113	75	46	.	.	PUNCT
fcis-13113	76	1	the	the	DET
fcis-13113	76	2	main	main	ADJ
fcis-13113	76	3	idea	idea	NOUN
fcis-13113	76	4	of	of	ADP
fcis-13113	76	5	these	these	DET
fcis-13113	76	6	methods	method	NOUN
fcis-13113	76	7	is	be	AUX
fcis-13113	76	8	to	to	PART
fcis-13113	76	9	assess	assess	VERB
fcis-13113	76	10	the	the	DET
fcis-13113	76	11	quality	quality	NOUN
fcis-13113	76	12	of	of	ADP
fcis-13113	76	13	mt	mt	PROPN
fcis-13113	76	14	results	result	NOUN
fcis-13113	76	15	by	by	ADP
fcis-13113	76	16	using	use	VERB
fcis-13113	76	17	manually	manually	ADV
fcis-13113	76	18	labelled	label	VERB
fcis-13113	76	19	translation	translation	NOUN
fcis-13113	76	20	110	110	NUM
fcis-13113	76	21	scores	score	NOUN
fcis-13113	76	22	or	or	CCONJ
fcis-13113	76	23	datasets	dataset	NOUN
fcis-13113	76	24	for	for	ADP
fcis-13113	76	25	prediction	prediction	NOUN
fcis-13113	76	26	.	.	PUNCT
fcis-13113	77	1	5	5	X
fcis-13113	77	2	.	.	X
fcis-13113	77	3	challenge	challenge	VERB
fcis-13113	77	4	a	a	DET
fcis-13113	77	5	large	large	ADJ
fcis-13113	77	6	number	number	NOUN
fcis-13113	77	7	of	of	ADP
fcis-13113	77	8	researchers	researcher	NOUN
fcis-13113	77	9	have	have	AUX
fcis-13113	77	10	designed	design	VERB
fcis-13113	77	11	experiments	experiment	NOUN
fcis-13113	77	12	to	to	PART
fcis-13113	77	13	prove	prove	VERB
fcis-13113	77	14	that	that	SCONJ
fcis-13113	77	15	neural	neural	ADJ
fcis-13113	77	16	network	network	NOUN
fcis-13113	77	17	-	-	PUNCT
fcis-13113	77	18	based	base	VERB
fcis-13113	77	19	methods	method	NOUN
fcis-13113	77	20	for	for	ADP
fcis-13113	77	21	automatically	automatically	ADV
fcis-13113	77	22	assessing	assess	VERB
fcis-13113	77	23	the	the	DET
fcis-13113	77	24	quality	quality	NOUN
fcis-13113	77	25	of	of	ADP
fcis-13113	77	26	mt	mt	PROPN
fcis-13113	77	27	perform	perform	VERB
fcis-13113	77	28	better	well	ADV
fcis-13113	77	29	than	than	ADP
fcis-13113	77	30	traditional	traditional	ADJ
fcis-13113	77	31	assessment	assessment	NOUN
fcis-13113	77	32	methods	method	NOUN
fcis-13113	77	33	.	.	PUNCT
fcis-13113	78	1	however	however	ADV
fcis-13113	78	2	,	,	PUNCT
fcis-13113	78	3	the	the	DET
fcis-13113	78	4	most	most	ADV
fcis-13113	78	5	commonly	commonly	ADV
fcis-13113	78	6	used	use	VERB
fcis-13113	78	7	evaluation	evaluation	NOUN
fcis-13113	78	8	methods	method	NOUN
fcis-13113	78	9	nowadays	nowadays	ADV
fcis-13113	78	10	are	be	AUX
fcis-13113	78	11	still	still	ADV
fcis-13113	78	12	the	the	DET
fcis-13113	78	13	most	most	ADV
fcis-13113	78	14	traditional	traditional	ADJ
fcis-13113	78	15	methods	method	NOUN
fcis-13113	78	16	such	such	ADJ
fcis-13113	78	17	as	as	ADP
fcis-13113	78	18	bleu	bleu	NOUN
fcis-13113	78	19	.	.	PUNCT
fcis-13113	79	1	so	so	ADV
fcis-13113	79	2	the	the	DET
fcis-13113	79	3	challenge	challenge	NOUN
fcis-13113	79	4	now	now	ADV
fcis-13113	79	5	is	be	AUX
fcis-13113	79	6	to	to	PART
fcis-13113	79	7	find	find	VERB
fcis-13113	79	8	out	out	ADP
fcis-13113	79	9	why	why	SCONJ
fcis-13113	79	10	the	the	DET
fcis-13113	79	11	use	use	NOUN
fcis-13113	79	12	of	of	ADP
fcis-13113	79	13	higher	high	ADJ
fcis-13113	79	14	performance	performance	NOUN
fcis-13113	79	15	automated	automate	VERB
fcis-13113	79	16	evaluation	evaluation	NOUN
fcis-13113	79	17	methods	method	NOUN
fcis-13113	79	18	is	be	AUX
fcis-13113	79	19	not	not	PART
fcis-13113	79	20	widespread	widespread	ADJ
fcis-13113	79	21	and	and	CCONJ
fcis-13113	79	22	why	why	SCONJ
fcis-13113	79	23	people	people	NOUN
fcis-13113	79	24	prefer	prefer	VERB
fcis-13113	79	25	to	to	PART
fcis-13113	79	26	go	go	VERB
fcis-13113	79	27	for	for	ADP
fcis-13113	79	28	traditional	traditional	ADJ
fcis-13113	79	29	automated	automate	VERB
fcis-13113	79	30	evaluation	evaluation	NOUN
fcis-13113	79	31	metrics	metric	NOUN
fcis-13113	79	32	.	.	PUNCT
fcis-13113	80	1	there	there	PRON
fcis-13113	80	2	is	be	VERB
fcis-13113	80	3	also	also	ADV
fcis-13113	80	4	the	the	DET
fcis-13113	80	5	fact	fact	NOUN
fcis-13113	80	6	that	that	SCONJ
fcis-13113	80	7	human	human	ADJ
fcis-13113	80	8	evaluation	evaluation	NOUN
fcis-13113	80	9	is	be	AUX
fcis-13113	80	10	still	still	ADV
fcis-13113	80	11	used	use	VERB
fcis-13113	80	12	as	as	ADP
fcis-13113	80	13	a	a	DET
fcis-13113	80	14	benchmark	benchmark	NOUN
fcis-13113	80	15	for	for	ADP
fcis-13113	80	16	evaluating	evaluate	VERB
fcis-13113	80	17	translation	translation	NOUN
fcis-13113	80	18	results	result	NOUN
fcis-13113	80	19	for	for	ADP
fcis-13113	80	20	different	different	ADJ
fcis-13113	80	21	tasks	task	NOUN
fcis-13113	80	22	in	in	ADP
fcis-13113	80	23	wmt22	wmt22	PROPN
fcis-13113	80	24	,	,	PUNCT
fcis-13113	80	25	and	and	CCONJ
fcis-13113	80	26	how	how	SCONJ
fcis-13113	80	27	to	to	PART
fcis-13113	80	28	improve	improve	VERB
fcis-13113	80	29	the	the	DET
fcis-13113	80	30	automatic	automatic	ADJ
fcis-13113	80	31	evaluation	evaluation	NOUN
fcis-13113	80	32	method	method	NOUN
fcis-13113	80	33	so	so	SCONJ
fcis-13113	80	34	that	that	SCONJ
fcis-13113	80	35	it	it	PRON
fcis-13113	80	36	can	can	AUX
fcis-13113	80	37	be	be	AUX
fcis-13113	80	38	aligned	align	VERB
fcis-13113	80	39	with	with	ADP
fcis-13113	80	40	the	the	DET
fcis-13113	80	41	quality	quality	NOUN
fcis-13113	80	42	of	of	ADP
fcis-13113	80	43	human	human	ADJ
fcis-13113	80	44	evaluation	evaluation	NOUN
fcis-13113	80	45	should	should	AUX
fcis-13113	80	46	also	also	ADV
fcis-13113	80	47	be	be	AUX
fcis-13113	80	48	a	a	DET
fcis-13113	80	49	key	key	ADJ
fcis-13113	80	50	area	area	NOUN
fcis-13113	80	51	of	of	ADP
fcis-13113	80	52	focus	focus	NOUN
fcis-13113	80	53	for	for	ADP
fcis-13113	80	54	researchers	researcher	NOUN
fcis-13113	80	55	in	in	ADP
fcis-13113	80	56	the	the	DET
fcis-13113	80	57	future	future	NOUN
fcis-13113	80	58	.	.	PUNCT
fcis-13113	81	1	6	6	X
fcis-13113	81	2	.	.	X
fcis-13113	81	3	conclusion	conclusion	NOUN
fcis-13113	81	4	and	and	CCONJ
fcis-13113	81	5	recommendations	recommendation	NOUN
fcis-13113	81	6	finally	finally	ADV
fcis-13113	81	7	,	,	PUNCT
fcis-13113	81	8	in	in	ADP
fcis-13113	81	9	this	this	DET
fcis-13113	81	10	paper	paper	NOUN
fcis-13113	81	11	we	we	PRON
fcis-13113	81	12	review	review	VERB
fcis-13113	81	13	common	common	ADJ
fcis-13113	81	14	mt	mt	PROPN
fcis-13113	81	15	quality	quality	NOUN
fcis-13113	81	16	assessment	assessment	NOUN
fcis-13113	81	17	methods	method	NOUN
fcis-13113	81	18	regarding	regard	VERB
fcis-13113	81	19	manual	manual	NOUN
fcis-13113	81	20	and	and	CCONJ
fcis-13113	81	21	automatic	automatic	ADJ
fcis-13113	81	22	assessment	assessment	NOUN
fcis-13113	81	23	.	.	PUNCT
fcis-13113	82	1	traditional	traditional	ADJ
fcis-13113	82	2	manual	manual	ADJ
fcis-13113	82	3	evaluation	evaluation	NOUN
fcis-13113	82	4	methods	method	NOUN
fcis-13113	82	5	evaluate	evaluate	VERB
fcis-13113	82	6	mt	mt	PROPN
fcis-13113	82	7	results	result	NOUN
fcis-13113	82	8	by	by	ADP
fcis-13113	82	9	human	human	ADJ
fcis-13113	82	10	experts	expert	NOUN
fcis-13113	82	11	,	,	PUNCT
fcis-13113	82	12	which	which	PRON
fcis-13113	82	13	can	can	AUX
fcis-13113	82	14	get	get	VERB
fcis-13113	82	15	high	high	ADJ
fcis-13113	82	16	quality	quality	NOUN
fcis-13113	82	17	evaluation	evaluation	NOUN
fcis-13113	82	18	results	result	NOUN
fcis-13113	82	19	,	,	PUNCT
fcis-13113	82	20	but	but	CCONJ
fcis-13113	82	21	they	they	PRON
fcis-13113	82	22	are	be	AUX
fcis-13113	82	23	time	time	NOUN
fcis-13113	82	24	-	-	PUNCT
fcis-13113	82	25	consuming	consume	VERB
fcis-13113	82	26	and	and	CCONJ
fcis-13113	82	27	costly	costly	ADJ
fcis-13113	82	28	with	with	ADP
fcis-13113	82	29	a	a	DET
fcis-13113	82	30	certain	certain	ADJ
fcis-13113	82	31	degree	degree	NOUN
fcis-13113	82	32	of	of	ADP
fcis-13113	82	33	subjectivity	subjectivity	NOUN
fcis-13113	82	34	.	.	PUNCT
fcis-13113	83	1	the	the	DET
fcis-13113	83	2	automatic	automatic	ADJ
fcis-13113	83	3	evaluation	evaluation	NOUN
fcis-13113	83	4	method	method	NOUN
fcis-13113	83	5	evaluates	evaluate	VERB
fcis-13113	83	6	the	the	DET
fcis-13113	83	7	mt	mt	PROPN
fcis-13113	83	8	results	result	VERB
fcis-13113	83	9	through	through	ADP
fcis-13113	83	10	mathematical	mathematical	ADJ
fcis-13113	83	11	formulae	formulae	NOUN
fcis-13113	83	12	or	or	CCONJ
fcis-13113	83	13	computer	computer	NOUN
fcis-13113	83	14	algorithms	algorithm	NOUN
fcis-13113	83	15	,	,	PUNCT
fcis-13113	83	16	which	which	PRON
fcis-13113	83	17	is	be	AUX
fcis-13113	83	18	fast	fast	ADJ
fcis-13113	83	19	and	and	CCONJ
fcis-13113	83	20	low	low	ADJ
fcis-13113	83	21	-	-	PUNCT
fcis-13113	83	22	cost	cost	NOUN
fcis-13113	83	23	,	,	PUNCT
fcis-13113	83	24	but	but	CCONJ
fcis-13113	83	25	can	can	AUX
fcis-13113	83	26	not	not	PART
fcis-13113	83	27	provide	provide	VERB
fcis-13113	83	28	high	high	ADJ
fcis-13113	83	29	-	-	PUNCT
fcis-13113	83	30	quality	quality	NOUN
fcis-13113	83	31	evaluation	evaluation	NOUN
fcis-13113	83	32	results	result	NOUN
fcis-13113	83	33	like	like	ADP
fcis-13113	83	34	the	the	DET
fcis-13113	83	35	manual	manual	ADJ
fcis-13113	83	36	evaluation	evaluation	NOUN
fcis-13113	83	37	method	method	NOUN
fcis-13113	83	38	.	.	PUNCT
fcis-13113	84	1	in	in	ADP
fcis-13113	84	2	order	order	NOUN
fcis-13113	84	3	to	to	PART
fcis-13113	84	4	further	far	ADV
fcis-13113	84	5	develop	develop	VERB
fcis-13113	84	6	the	the	DET
fcis-13113	84	7	mt	mt	PROPN
fcis-13113	84	8	quality	quality	NOUN
fcis-13113	84	9	assessment	assessment	NOUN
fcis-13113	84	10	methods	method	NOUN
fcis-13113	84	11	,	,	PUNCT
fcis-13113	84	12	it	it	PRON
fcis-13113	84	13	is	be	AUX
fcis-13113	84	14	suggested	suggest	VERB
fcis-13113	84	15	to	to	PART
fcis-13113	84	16	combine	combine	VERB
fcis-13113	84	17	the	the	DET
fcis-13113	84	18	advantages	advantage	NOUN
fcis-13113	84	19	of	of	ADP
fcis-13113	84	20	manual	manual	ADJ
fcis-13113	84	21	assessment	assessment	NOUN
fcis-13113	84	22	and	and	CCONJ
fcis-13113	84	23	automatic	automatic	ADJ
fcis-13113	84	24	assessment	assessment	NOUN
fcis-13113	84	25	.	.	PUNCT
fcis-13113	85	1	it	it	PRON
fcis-13113	85	2	is	be	AUX
fcis-13113	85	3	also	also	ADV
fcis-13113	85	4	suggested	suggest	VERB
fcis-13113	85	5	that	that	SCONJ
fcis-13113	85	6	the	the	DET
fcis-13113	85	7	development	development	NOUN
fcis-13113	85	8	of	of	ADP
fcis-13113	85	9	automatic	automatic	ADJ
fcis-13113	85	10	assessment	assessment	NOUN
fcis-13113	85	11	methods	method	NOUN
fcis-13113	85	12	should	should	AUX
fcis-13113	85	13	focus	focus	VERB
fcis-13113	85	14	on	on	ADP
fcis-13113	85	15	improving	improve	VERB
fcis-13113	85	16	neural	neural	ADJ
fcis-13113	85	17	network	network	NOUN
fcis-13113	85	18	-	-	PUNCT
fcis-13113	85	19	based	base	VERB
fcis-13113	85	20	assessment	assessment	NOUN
fcis-13113	85	21	methods	method	NOUN
fcis-13113	85	22	so	so	SCONJ
fcis-13113	85	23	that	that	SCONJ
fcis-13113	85	24	such	such	ADJ
fcis-13113	85	25	highperformance	highperformance	NOUN
fcis-13113	85	26	assessment	assessment	NOUN
fcis-13113	85	27	methods	method	NOUN
fcis-13113	85	28	can	can	AUX
fcis-13113	85	29	be	be	AUX
fcis-13113	85	30	used	use	VERB
fcis-13113	85	31	universally	universally	ADV
fcis-13113	85	32	.	.	PUNCT
fcis-13113	86	1	references	reference	NOUN
fcis-13113	86	2	[	[	X
fcis-13113	86	3	1	1	NUM
fcis-13113	86	4	]	]	X
fcis-13113	86	5	papineni	papineni	PROPN
fcis-13113	86	6	k.	k.	PROPN
fcis-13113	86	7	,	,	PUNCT
fcis-13113	86	8	roukos	roukos	ADJ
fcis-13113	86	9	s.	s.	PROPN
fcis-13113	86	10	,	,	PUNCT
fcis-13113	86	11	ward	ward	NOUN
fcis-13113	86	12	t.	t.	PROPN
fcis-13113	86	13	and	and	CCONJ
fcis-13113	86	14	zhu	zhu	PROPN
fcis-13113	86	15	w.j	w.j	PROPN
fcis-13113	86	16	.	.	PROPN
fcis-13113	86	17	(	(	PUNCT
fcis-13113	86	18	2002	2002	NUM
fcis-13113	86	19	)	)	PUNCT
fcis-13113	86	20	.	.	PUNCT
fcis-13113	87	1	bleu	bleu	NOUN
fcis-13113	87	2	:	:	PUNCT
fcis-13113	87	3	a	a	DET
fcis-13113	87	4	method	method	NOUN
fcis-13113	87	5	for	for	ADP
fcis-13113	87	6	automatic	automatic	ADJ
fcis-13113	87	7	evaluation	evaluation	NOUN
fcis-13113	87	8	of	of	ADP
fcis-13113	87	9	machine	machine	NOUN
fcis-13113	87	10	translation	translation	NOUN
fcis-13113	87	11	.	.	PUNCT
fcis-13113	88	1	in	in	ADP
fcis-13113	88	2	:	:	PUNCT
fcis-13113	88	3	proceedings	proceeding	NOUN
fcis-13113	88	4	of	of	ADP
fcis-13113	88	5	acl-2002	acl-2002	NOUN
fcis-13113	88	6	:	:	PUNCT
fcis-13113	88	7	40th	40th	ADJ
fcis-13113	88	8	annual	annual	ADJ
fcis-13113	88	9	meeting	meeting	NOUN
fcis-13113	88	10	of	of	ADP
fcis-13113	88	11	the	the	DET
fcis-13113	88	12	association	association	NOUN
fcis-13113	88	13	for	for	ADP
fcis-13113	88	14	computational	computational	ADJ
fcis-13113	88	15	linguistics	linguistic	NOUN
fcis-13113	88	16	,	,	PUNCT
fcis-13113	88	17	philadelphia	philadelphia	PROPN
fcis-13113	88	18	,	,	PUNCT
fcis-13113	88	19	pennsylvania	pennsylvania	PROPN
fcis-13113	88	20	,	,	PUNCT
fcis-13113	88	21	pp.311	pp.311	PROPN
fcis-13113	88	22	-	-	PUNCT
fcis-13113	88	23	318	318	NUM
fcis-13113	88	24	.	.	PUNCT
fcis-13113	89	1	[	[	X
fcis-13113	89	2	2	2	NUM
fcis-13113	89	3	]	]	PUNCT
fcis-13113	89	4	devlin	devlin	NOUN
fcis-13113	89	5	,	,	PUNCT
fcis-13113	89	6	j.	j.	PROPN
fcis-13113	89	7	,	,	PUNCT
fcis-13113	89	8	chang	chang	PROPN
fcis-13113	89	9	,	,	PUNCT
fcis-13113	89	10	m.w	m.w	PROPN
fcis-13113	89	11	.	.	PROPN
fcis-13113	89	12	,	,	PUNCT
fcis-13113	89	13	lee	lee	PROPN
fcis-13113	89	14	,	,	PUNCT
fcis-13113	89	15	k.	k.	PROPN
fcis-13113	89	16	,	,	PUNCT
fcis-13113	89	17	toutanova	toutanova	PROPN
fcis-13113	89	18	,	,	PUNCT
fcis-13113	89	19	k.	k.	PROPN
fcis-13113	89	20	bert	bert	PROPN
fcis-13113	89	21	.	.	PUNCT
fcis-13113	90	1	(	(	PUNCT
fcis-13113	90	2	2018	2018	NUM
fcis-13113	90	3	)	)	PUNCT
fcis-13113	90	4	pre	pre	ADJ
fcis-13113	90	5	-	-	NOUN
fcis-13113	90	6	training	training	NOUN
fcis-13113	90	7	of	of	ADP
fcis-13113	90	8	deep	deep	ADJ
fcis-13113	90	9	bidirectional	bidirectional	ADJ
fcis-13113	90	10	transformers	transformer	NOUN
fcis-13113	90	11	for	for	ADP
fcis-13113	90	12	language	language	NOUN
fcis-13113	90	13	understanding	understanding	NOUN
fcis-13113	90	14	.	.	PUNCT
fcis-13113	91	1	arxiv:1810.04805	arxiv:1810.04805	PROPN
fcis-13113	91	2	.	.	PUNCT
fcis-13113	92	1	[	[	X
fcis-13113	92	2	3	3	NUM
fcis-13113	92	3	]	]	X
fcis-13113	92	4	freitag	freitag	NOUN
fcis-13113	92	5	,	,	PUNCT
fcis-13113	92	6	m.	m.	NOUN
fcis-13113	92	7	,	,	PUNCT
fcis-13113	92	8	rei	rei	PROPN
fcis-13113	92	9	,	,	PUNCT
fcis-13113	92	10	r.	r.	PROPN
fcis-13113	92	11	,	,	PUNCT
fcis-13113	92	12	mathur	mathur	PROPN
fcis-13113	92	13	,	,	PUNCT
fcis-13113	92	14	n.	n.	PROPN
fcis-13113	92	15	,	,	PUNCT
fcis-13113	92	16	lo	lo	PROPN
fcis-13113	92	17	,	,	PUNCT
fcis-13113	92	18	c.k	c.k	PROPN
fcis-13113	92	19	.	.	PROPN
fcis-13113	92	20	,	,	PUNCT
fcis-13113	92	21	stewart	stewart	PROPN
fcis-13113	92	22	,	,	PUNCT
fcis-13113	92	23	c.	c.	PROPN
fcis-13113	92	24	,	,	PUNCT
fcis-13113	92	25	foster	foster	PROPN
fcis-13113	92	26	,	,	PUNCT
fcis-13113	92	27	g.	g.	PROPN
fcis-13113	92	28	,	,	PUNCT
fcis-13113	92	29	lavie	lavie	PROPN
fcis-13113	92	30	,	,	PUNCT
fcis-13113	92	31	a.	a.	PROPN
fcis-13113	92	32	,	,	PUNCT
fcis-13113	92	33	bojar	bojar	NOUN
fcis-13113	92	34	,	,	PUNCT
fcis-13113	92	35	o.	o.	NOUN
fcis-13113	92	36	(	(	PUNCT
fcis-13113	92	37	2021	2021	NUM
fcis-13113	92	38	)	)	PUNCT
fcis-13113	92	39	results	result	NOUN
fcis-13113	92	40	of	of	ADP
fcis-13113	92	41	the	the	DET
fcis-13113	92	42	wmt21	wmt21	ADJ
fcis-13113	92	43	metrics	metric	NOUN
fcis-13113	92	44	shared	share	VERB
fcis-13113	92	45	task	task	NOUN
fcis-13113	92	46	:	:	PUNCT
fcis-13113	92	47	evaluating	evaluate	VERB
fcis-13113	92	48	metrics	metric	NOUN
fcis-13113	92	49	with	with	ADP
fcis-13113	92	50	expert	expert	NOUN
fcis-13113	92	51	-	-	PUNCT
fcis-13113	92	52	based	base	VERB
fcis-13113	92	53	human	human	ADJ
fcis-13113	92	54	evaluations	evaluation	NOUN
fcis-13113	92	55	on	on	ADP
fcis-13113	92	56	ted	ted	NOUN
fcis-13113	92	57	and	and	CCONJ
fcis-13113	92	58	news	news	NOUN
fcis-13113	92	59	domain	domain	NOUN
fcis-13113	92	60	.	.	PUNCT
fcis-13113	93	1	in	in	ADP
fcis-13113	93	2	proceedings	proceeding	NOUN
fcis-13113	93	3	of	of	ADP
fcis-13113	93	4	the	the	DET
fcis-13113	93	5	sixth	sixth	ADJ
fcis-13113	93	6	conference	conference	NOUN
fcis-13113	93	7	on	on	ADP
fcis-13113	93	8	machine	machine	NOUN
fcis-13113	93	9	translation	translation	NOUN
fcis-13113	93	10	,	,	PUNCT
fcis-13113	93	11	online	online	INTJ
fcis-13113	93	12	,	,	PUNCT
fcis-13113	93	13	pp	pp	ADJ
fcis-13113	93	14	.	.	PUNCT
fcis-13113	94	1	733	733	NUM
fcis-13113	94	2	-	-	SYM
fcis-13113	94	3	774	774	NUM
fcis-13113	94	4	.	.	PUNCT
fcis-13113	95	1	[	[	X
fcis-13113	95	2	4	4	X
fcis-13113	95	3	]	]	X
fcis-13113	95	4	chatzikoumi	chatzikoumi	PROPN
fcis-13113	95	5	e.	e.	PROPN
fcis-13113	95	6	(	(	PUNCT
fcis-13113	95	7	2020	2020	NUM
fcis-13113	95	8	)	)	PUNCT
fcis-13113	95	9	how	how	SCONJ
fcis-13113	95	10	to	to	PART
fcis-13113	95	11	evaluate	evaluate	VERB
fcis-13113	95	12	machine	machine	NOUN
fcis-13113	95	13	translation	translation	NOUN
fcis-13113	95	14	:	:	PUNCT
fcis-13113	95	15	a	a	DET
fcis-13113	95	16	review	review	NOUN
fcis-13113	95	17	of	of	ADP
fcis-13113	95	18	automated	automate	VERB
fcis-13113	95	19	and	and	CCONJ
fcis-13113	95	20	human	human	ADJ
fcis-13113	95	21	metrics	metric	NOUN
fcis-13113	95	22	.	.	PUNCT
fcis-13113	95	23	natural	natural	ADJ
fcis-13113	95	24	language	language	NOUN
fcis-13113	95	25	engineering	engineering	NOUN
fcis-13113	95	26	.	.	PUNCT
fcis-13113	96	1	26(2):137	26(2):137	NUM
fcis-13113	96	2	-	-	SYM
fcis-13113	96	3	161	161	NUM
fcis-13113	96	4	.	.	PUNCT
fcis-13113	97	1	[	[	X
fcis-13113	97	2	5	5	NUM
fcis-13113	97	3	]	]	SYM
fcis-13113	97	4	zerva	zerva	PROPN
fcis-13113	97	5	,	,	PUNCT
fcis-13113	97	6	c.	c.	PROPN
fcis-13113	97	7	,	,	PUNCT
fcis-13113	97	8	blain	blain	PROPN
fcis-13113	97	9	,	,	PUNCT
fcis-13113	97	10	f.	f.	PROPN
fcis-13113	97	11	,	,	PUNCT
fcis-13113	97	12	rei	rei	PROPN
fcis-13113	97	13	,	,	PUNCT
fcis-13113	97	14	r.	r.	PROPN
fcis-13113	97	15	,	,	PUNCT
fcis-13113	97	16	lertvittayakumjorn	lertvittayakumjorn	VERB
fcis-13113	97	17	,	,	PUNCT
fcis-13113	97	18	p.	p.	NOUN
fcis-13113	97	19	,	,	PUNCT
fcis-13113	97	20	de	de	PROPN
fcis-13113	97	21	souza	souza	PROPN
fcis-13113	97	22	,	,	PUNCT
fcis-13113	97	23	j.	j.	PROPN
fcis-13113	97	24	g.	g.	PROPN
fcis-13113	97	25	,	,	PUNCT
fcis-13113	97	26	eger	eger	PROPN
fcis-13113	97	27	,	,	PUNCT
fcis-13113	97	28	s.	s.	PROPN
fcis-13113	97	29	,	,	PUNCT
fcis-13113	97	30	...	...	PUNCT
fcis-13113	97	31	&	&	CCONJ
fcis-13113	97	32	specia	specia	PROPN
fcis-13113	97	33	,	,	PUNCT
fcis-13113	97	34	l.	l.	PROPN
fcis-13113	97	35	(	(	PUNCT
fcis-13113	97	36	2022	2022	NUM
fcis-13113	97	37	)	)	PUNCT
fcis-13113	97	38	findings	finding	NOUN
fcis-13113	97	39	of	of	ADP
fcis-13113	97	40	the	the	DET
fcis-13113	97	41	wmt	wmt	PROPN
fcis-13113	97	42	2022	2022	NUM
fcis-13113	97	43	shared	share	VERB
fcis-13113	97	44	task	task	NOUN
fcis-13113	97	45	on	on	ADP
fcis-13113	97	46	quality	quality	NOUN
fcis-13113	97	47	estimation	estimation	NOUN
fcis-13113	97	48	.	.	PUNCT
fcis-13113	98	1	in	in	ADP
fcis-13113	98	2	:	:	PUNCT
fcis-13113	98	3	proceedings	proceeding	NOUN
fcis-13113	98	4	of	of	ADP
fcis-13113	98	5	the	the	DET
fcis-13113	98	6	seventh	seventh	ADJ
fcis-13113	98	7	conference	conference	NOUN
fcis-13113	98	8	on	on	ADP
fcis-13113	98	9	machine	machine	NOUN
fcis-13113	98	10	translation	translation	NOUN
fcis-13113	98	11	(	(	PUNCT
fcis-13113	98	12	wmt	wmt	PROPN
fcis-13113	98	13	)	)	PUNCT
fcis-13113	98	14	.	.	PUNCT
fcis-13113	99	1	pp.69	pp.69	NOUN
fcis-13113	99	2	-	-	PUNCT
fcis-13113	99	3	99	99	NUM
fcis-13113	99	4	.	.	PUNCT
fcis-13113	100	1	[	[	X
fcis-13113	100	2	6	6	NUM
fcis-13113	100	3	]	]	X
fcis-13113	100	4	graham	graham	PROPN
fcis-13113	100	5	,	,	PUNCT
fcis-13113	100	6	y.	y.	PROPN
fcis-13113	100	7	,	,	PUNCT
fcis-13113	100	8	baldwin	baldwin	PROPN
fcis-13113	100	9	,	,	PUNCT
fcis-13113	100	10	t.	t.	PROPN
fcis-13113	100	11	,	,	PUNCT
fcis-13113	100	12	moffat	moffat	PROPN
fcis-13113	100	13	,	,	PUNCT
fcis-13113	100	14	a.	a.	PROPN
fcis-13113	100	15	,	,	PUNCT
fcis-13113	100	16	&	&	CCONJ
fcis-13113	100	17	zobel	zobel	PROPN
fcis-13113	100	18	,	,	PUNCT
fcis-13113	100	19	j.	j.	PROPN
fcis-13113	100	20	(	(	PUNCT
fcis-13113	100	21	2013	2013	NUM
fcis-13113	100	22	)	)	PUNCT
fcis-13113	100	23	continuous	continuous	ADJ
fcis-13113	100	24	measurement	measurement	NOUN
fcis-13113	100	25	scales	scale	NOUN
fcis-13113	100	26	in	in	ADP
fcis-13113	100	27	human	human	ADJ
fcis-13113	100	28	evaluation	evaluation	NOUN
fcis-13113	100	29	of	of	ADP
fcis-13113	100	30	machine	machine	NOUN
fcis-13113	100	31	translation	translation	NOUN
fcis-13113	100	32	.	.	PUNCT
fcis-13113	101	1	in	in	ADP
fcis-13113	101	2	:	:	PUNCT
fcis-13113	101	3	proceedings	proceeding	NOUN
fcis-13113	101	4	of	of	ADP
fcis-13113	101	5	the	the	DET
fcis-13113	101	6	7th	7th	ADJ
fcis-13113	101	7	linguistic	linguistic	ADJ
fcis-13113	101	8	annotation	annotation	NOUN
fcis-13113	101	9	workshop	workshop	NOUN
fcis-13113	101	10	and	and	CCONJ
fcis-13113	101	11	interoperability	interoperability	NOUN
fcis-13113	101	12	with	with	ADP
fcis-13113	101	13	discourse	discourse	NOUN
fcis-13113	101	14	.	.	PUNCT
fcis-13113	102	1	pp.33	pp.33	ADJ
fcis-13113	102	2	-	-	PUNCT
fcis-13113	102	3	41	41	NUM
fcis-13113	102	4	.	.	PUNCT
fcis-13113	103	1	[	[	X
fcis-13113	103	2	7	7	X
fcis-13113	103	3	]	]	X
fcis-13113	103	4	ageeva	ageeva	PROPN
fcis-13113	103	5	e.	e.	PROPN
fcis-13113	103	6	,	,	PUNCT
fcis-13113	103	7	tyers	tyers	PROPN
fcis-13113	103	8	f.	f.	PROPN
fcis-13113	103	9	,	,	PUNCT
fcis-13113	103	10	forcada	forcada	PROPN
fcis-13113	103	11	m.	m.	PROPN
fcis-13113	103	12	and	and	CCONJ
fcis-13113	103	13	perez	perez	PROPN
fcis-13113	103	14	-	-	PUNCT
fcis-13113	103	15	ortiz	ortiz	PROPN
fcis-13113	103	16	j.	j.	PROPN
fcis-13113	103	17	(	(	PUNCT
fcis-13113	103	18	2015	2015	NUM
fcis-13113	103	19	)	)	PUNCT
fcis-13113	103	20	evaluating	evaluate	VERB
fcis-13113	103	21	machine	machine	NOUN
fcis-13113	103	22	translation	translation	NOUN
fcis-13113	103	23	for	for	ADP
fcis-13113	103	24	assimilation	assimilation	NOUN
fcis-13113	103	25	via	via	ADP
fcis-13113	103	26	a	a	DET
fcis-13113	103	27	gapfilling	gapfille	VERB
fcis-13113	103	28	task	task	NOUN
fcis-13113	103	29	.	.	PUNCT
fcis-13113	104	1	in	in	ADP
fcis-13113	104	2	:	:	PUNCT
fcis-13113	104	3	proceedings	proceeding	NOUN
fcis-13113	104	4	of	of	ADP
fcis-13113	104	5	the	the	DET
fcis-13113	104	6	conference	conference	NOUN
fcis-13113	104	7	of	of	ADP
fcis-13113	104	8	the	the	DET
fcis-13113	104	9	european	european	PROPN
fcis-13113	104	10	association	association	PROPN
fcis-13113	104	11	for	for	ADP
fcis-13113	104	12	machine	machine	NOUN
fcis-13113	104	13	translation	translation	NOUN
fcis-13113	104	14	,	,	PUNCT
fcis-13113	104	15	antalya	antalya	PROPN
fcis-13113	104	16	,	,	PUNCT
fcis-13113	104	17	turkey	turkey	PROPN
fcis-13113	104	18	,	,	PUNCT
fcis-13113	104	19	pp	pp	ADJ
fcis-13113	104	20	.	.	PUNCT
fcis-13113	104	21	137	137	NUM
fcis-13113	104	22	-	-	SYM
fcis-13113	104	23	144	144	NUM
fcis-13113	104	24	.	.	PUNCT
fcis-13113	105	1	[	[	X
fcis-13113	105	2	8	8	NUM
fcis-13113	105	3	]	]	X
fcis-13113	105	4	lee	lee	PROPN
fcis-13113	105	5	,	,	PUNCT
fcis-13113	105	6	s.	s.	PROPN
fcis-13113	105	7	,	,	PUNCT
fcis-13113	105	8	lee	lee	PROPN
fcis-13113	105	9	,	,	PUNCT
fcis-13113	105	10	j.	j.	PROPN
fcis-13113	105	11	,	,	PUNCT
fcis-13113	105	12	moon	moon	PROPN
fcis-13113	105	13	,	,	PUNCT
fcis-13113	105	14	h.	h.	PROPN
fcis-13113	105	15	,	,	PUNCT
fcis-13113	105	16	park	park	PROPN
fcis-13113	105	17	,	,	PUNCT
fcis-13113	105	18	c.	c.	PROPN
fcis-13113	105	19	,	,	PUNCT
fcis-13113	105	20	seo	seo	PROPN
fcis-13113	105	21	,	,	PUNCT
fcis-13113	105	22	j.	j.	PROPN
fcis-13113	105	23	,	,	PUNCT
fcis-13113	105	24	eo	eo	PROPN
fcis-13113	105	25	,	,	PUNCT
fcis-13113	105	26	s.	s.	PROPN
fcis-13113	105	27	,	,	PUNCT
fcis-13113	105	28	...	...	PUNCT
fcis-13113	105	29	&	&	CCONJ
fcis-13113	105	30	lim	lim	PROPN
fcis-13113	105	31	,	,	PUNCT
fcis-13113	105	32	h.	h.	PROPN
fcis-13113	105	33	(	(	PUNCT
fcis-13113	105	34	2023	2023	NUM
fcis-13113	105	35	)	)	PUNCT
fcis-13113	105	36	.	.	PUNCT
fcis-13113	106	1	a	a	DET
fcis-13113	106	2	survey	survey	NOUN
fcis-13113	106	3	on	on	ADP
fcis-13113	106	4	evaluation	evaluation	NOUN
fcis-13113	106	5	metrics	metric	NOUN
fcis-13113	106	6	for	for	ADP
fcis-13113	106	7	machine	machine	NOUN
fcis-13113	106	8	translation	translation	NOUN
fcis-13113	106	9	.	.	PUNCT
fcis-13113	107	1	mathematics	mathematic	NOUN
fcis-13113	107	2	,	,	PUNCT
fcis-13113	107	3	11(4	11(4	NUM
fcis-13113	107	4	):	):	PUNCT
fcis-13113	107	5	1006	1006	NUM
fcis-13113	107	6	.	.	PUNCT
fcis-13113	108	1	[	[	X
fcis-13113	108	2	9	9	NUM
fcis-13113	108	3	]	]	X
fcis-13113	108	4	banerjee	banerjee	PROPN
fcis-13113	108	5	,	,	PUNCT
fcis-13113	108	6	s.	s.	PROPN
fcis-13113	108	7	,	,	PUNCT
fcis-13113	108	8	&	&	CCONJ
fcis-13113	108	9	lavie	lavie	PROPN
fcis-13113	108	10	,	,	PUNCT
fcis-13113	108	11	a.	a.	NOUN
fcis-13113	108	12	(	(	PUNCT
fcis-13113	108	13	2005	2005	NUM
fcis-13113	108	14	,	,	PUNCT
fcis-13113	108	15	june	june	PROPN
fcis-13113	108	16	)	)	PUNCT
fcis-13113	108	17	.	.	PUNCT
fcis-13113	109	1	meteor	meteor	NOUN
fcis-13113	109	2	:	:	PUNCT
fcis-13113	109	3	an	an	DET
fcis-13113	109	4	automatic	automatic	ADJ
fcis-13113	109	5	metric	metric	NOUN
fcis-13113	109	6	for	for	ADP
fcis-13113	109	7	mt	mt	PROPN
fcis-13113	109	8	evaluation	evaluation	NOUN
fcis-13113	109	9	with	with	ADP
fcis-13113	109	10	improved	improved	ADJ
fcis-13113	109	11	correlation	correlation	NOUN
fcis-13113	109	12	with	with	ADP
fcis-13113	109	13	human	human	ADJ
fcis-13113	109	14	judgments	judgment	NOUN
fcis-13113	109	15	.	.	PUNCT
fcis-13113	110	1	in	in	ADP
fcis-13113	110	2	:	:	PUNCT
fcis-13113	110	3	proceedings	proceeding	NOUN
fcis-13113	110	4	of	of	ADP
fcis-13113	110	5	the	the	DET
fcis-13113	110	6	acl	acl	PROPN
fcis-13113	110	7	workshop	workshop	NOUN
fcis-13113	110	8	on	on	ADP
fcis-13113	110	9	intrinsic	intrinsic	ADJ
fcis-13113	110	10	and	and	CCONJ
fcis-13113	110	11	extrinsic	extrinsic	ADJ
fcis-13113	110	12	evaluation	evaluation	NOUN
fcis-13113	110	13	measures	measure	NOUN
fcis-13113	110	14	for	for	ADP
fcis-13113	110	15	machine	machine	NOUN
fcis-13113	110	16	translation	translation	NOUN
fcis-13113	110	17	and/or	and/or	CCONJ
fcis-13113	110	18	summarization	summarization	NOUN
fcis-13113	110	19	.	.	PUNCT
fcis-13113	111	1	pp.65	pp.65	NOUN
fcis-13113	111	2	-	-	PUNCT
fcis-13113	111	3	72	72	NUM
fcis-13113	111	4	.	.	PUNCT