id	sid	tid	token	lemma	pos
ajema-7508	1	1	152	152	NUM
ajema-7508	1	2	journal	journal	NOUN
ajema-7508	1	3	of	of	ADP
ajema-7508	1	4	engineering	engineering	NOUN
ajema-7508	1	5	,	,	PUNCT
ajema-7508	1	6	mechanics	mechanic	NOUN
ajema-7508	1	7	and	and	CCONJ
ajema-7508	1	8	architecture	architecture	NOUN
ajema-7508	1	9	www	www	NOUN
ajema-7508	1	10	.	.	PUNCT
ajema-7508	2	1	grnjournal.us	grnjournal.us	PROPN
ajema-7508	2	2	american	american	PROPN
ajema-7508	2	3	journal	journal	PROPN
ajema-7508	2	4	of	of	ADP
ajema-7508	2	5	engineering	engineering	NOUN
ajema-7508	2	6	,	,	PUNCT
ajema-7508	2	7	mechanics	mechanic	NOUN
ajema-7508	2	8	and	and	CCONJ
ajema-7508	2	9	architecture	architecture	NOUN
ajema-7508	2	10	volume	volume	NOUN
ajema-7508	2	11	3	3	NUM
ajema-7508	2	12	,	,	PUNCT
ajema-7508	2	13	issue	issue	NOUN
ajema-7508	2	14	4	4	NUM
ajema-7508	2	15	,	,	PUNCT
ajema-7508	2	16	2025	2025	NUM
ajema-7508	2	17	issn	issn	X
ajema-7508	2	18	(	(	PUNCT
ajema-7508	2	19	e	e	NOUN
ajema-7508	2	20	):	):	PUNCT
ajema-7508	2	21	2993	2993	NUM
ajema-7508	2	22	-	-	SYM
ajema-7508	2	23	2637	2637	NUM
ajema-7508	2	24	data	datum	NOUN
ajema-7508	2	25	analysis	analysis	NOUN
ajema-7508	2	26	based	base	VERB
ajema-7508	2	27	on	on	ADP
ajema-7508	2	28	the	the	DET
ajema-7508	2	29	bert	bert	PROPN
ajema-7508	2	30	model	model	PROPN
ajema-7508	2	31	eshankulov	eshankulov	PROPN
ajema-7508	2	32	hamza	hamza	PROPN
ajema-7508	2	33	ilhomovich	ilhomovich	PROPN
ajema-7508	2	34	professor	professor	NOUN
ajema-7508	2	35	of	of	ADP
ajema-7508	2	36	the	the	DET
ajema-7508	2	37	department	department	NOUN
ajema-7508	2	38	of	of	ADP
ajema-7508	2	39	applied	apply	VERB
ajema-7508	2	40	mathematics	mathematic	NOUN
ajema-7508	2	41	and	and	CCONJ
ajema-7508	2	42	programming	programming	NOUN
ajema-7508	2	43	technologies	technology	NOUN
ajema-7508	2	44	at	at	ADP
ajema-7508	2	45	bsu	bsu	PROPN
ajema-7508	2	46	x.i.eshonkulov@buxdu.uz	x.i.eshonkulov@buxdu.uz	PROPN
ajema-7508	2	47	soliyeva	soliyeva	PROPN
ajema-7508	2	48	dilsora	dilsora	PROPN
ajema-7508	2	49	alisher	alisher	PROPN
ajema-7508	2	50	qizi	qizi	PROPN
ajema-7508	2	51	master	master	PROPN
ajema-7508	2	52	's	's	PART
ajema-7508	2	53	degree	degree	NOUN
ajema-7508	2	54	holder	holder	NOUN
ajema-7508	2	55	at	at	ADP
ajema-7508	2	56	the	the	DET
ajema-7508	2	57	department	department	NOUN
ajema-7508	2	58	of	of	ADP
ajema-7508	2	59	applied	apply	VERB
ajema-7508	2	60	mathematics	mathematic	NOUN
ajema-7508	2	61	and	and	CCONJ
ajema-7508	2	62	programming	programming	NOUN
ajema-7508	2	63	technologies	technology	NOUN
ajema-7508	2	64	at	at	ADP
ajema-7508	2	65	bsu	bsu	NOUN
ajema-7508	2	66	soliyeva.dilsora.2020@gmail.com	soliyeva.dilsora.2020@gmail.com	X
ajema-7508	2	67	abstract	abstract	NOUN
ajema-7508	2	68	:	:	PUNCT
ajema-7508	2	69	this	this	DET
ajema-7508	2	70	article	article	NOUN
ajema-7508	2	71	provides	provide	VERB
ajema-7508	2	72	an	an	DET
ajema-7508	2	73	in	in	ADP
ajema-7508	2	74	-	-	PUNCT
ajema-7508	2	75	depth	depth	NOUN
ajema-7508	2	76	analysis	analysis	NOUN
ajema-7508	2	77	of	of	ADP
ajema-7508	2	78	the	the	DET
ajema-7508	2	79	capabilities	capability	NOUN
ajema-7508	2	80	of	of	ADP
ajema-7508	2	81	the	the	DET
ajema-7508	2	82	bert	bert	PROPN
ajema-7508	2	83	(	(	PUNCT
ajema-7508	2	84	bidirectional	bidirectional	ADJ
ajema-7508	2	85	encoder	encoder	NOUN
ajema-7508	2	86	representations	representation	VERB
ajema-7508	2	87	from	from	ADP
ajema-7508	2	88	transformers	transformer	NOUN
ajema-7508	2	89	)	)	PUNCT
ajema-7508	2	90	model	model	NOUN
ajema-7508	2	91	,	,	PUNCT
ajema-7508	2	92	which	which	PRON
ajema-7508	2	93	holds	hold	VERB
ajema-7508	2	94	a	a	DET
ajema-7508	2	95	significant	significant	ADJ
ajema-7508	2	96	place	place	NOUN
ajema-7508	2	97	in	in	ADP
ajema-7508	2	98	the	the	DET
ajema-7508	2	99	field	field	NOUN
ajema-7508	2	100	of	of	ADP
ajema-7508	2	101	natural	natural	ADJ
ajema-7508	2	102	language	language	NOUN
ajema-7508	2	103	processing	processing	NOUN
ajema-7508	2	104	(	(	PUNCT
ajema-7508	2	105	nlp	nlp	NOUN
ajema-7508	2	106	)	)	PUNCT
ajema-7508	2	107	,	,	PUNCT
ajema-7508	2	108	for	for	ADP
ajema-7508	2	109	performing	perform	VERB
ajema-7508	2	110	deep	deep	ADJ
ajema-7508	2	111	,	,	PUNCT
ajema-7508	2	112	contextual	contextual	ADJ
ajema-7508	2	113	,	,	PUNCT
ajema-7508	2	114	and	and	CCONJ
ajema-7508	2	115	semantic	semantic	ADJ
ajema-7508	2	116	data	datum	NOUN
ajema-7508	2	117	analysis	analysis	NOUN
ajema-7508	2	118	.	.	PUNCT
ajema-7508	3	1	the	the	DET
ajema-7508	3	2	internal	internal	ADJ
ajema-7508	3	3	architecture	architecture	NOUN
ajema-7508	3	4	and	and	CCONJ
ajema-7508	3	5	working	working	NOUN
ajema-7508	3	6	mechanisms	mechanism	NOUN
ajema-7508	3	7	of	of	ADP
ajema-7508	3	8	the	the	DET
ajema-7508	3	9	bert	bert	PROPN
ajema-7508	3	10	model	model	NOUN
ajema-7508	3	11	are	be	AUX
ajema-7508	3	12	explored	explore	VERB
ajema-7508	3	13	,	,	PUNCT
ajema-7508	3	14	along	along	ADP
ajema-7508	3	15	with	with	ADP
ajema-7508	3	16	advanced	advanced	ADJ
ajema-7508	3	17	approaches	approach	NOUN
ajema-7508	3	18	such	such	ADJ
ajema-7508	3	19	as	as	ADP
ajema-7508	3	20	distilbert	distilbert	NOUN
ajema-7508	3	21	and	and	CCONJ
ajema-7508	3	22	topic	topic	NOUN
ajema-7508	3	23	bert	bert	NOUN
ajema-7508	3	24	-	-	PUNCT
ajema-7508	3	25	bigru	bigru	NOUN
ajema-7508	3	26	,	,	PUNCT
ajema-7508	3	27	focusing	focus	VERB
ajema-7508	3	28	on	on	ADP
ajema-7508	3	29	methodologies	methodology	NOUN
ajema-7508	3	30	for	for	ADP
ajema-7508	3	31	accurately	accurately	ADV
ajema-7508	3	32	classifying	classify	VERB
ajema-7508	3	33	both	both	CCONJ
ajema-7508	3	34	complex	complex	ADJ
ajema-7508	3	35	and	and	CCONJ
ajema-7508	3	36	short	short	ADJ
ajema-7508	3	37	texts[5	texts[5	NOUN
ajema-7508	3	38	]	]	PUNCT
ajema-7508	3	39	.	.	PUNCT
ajema-7508	4	1	the	the	DET
ajema-7508	4	2	paper	paper	NOUN
ajema-7508	4	3	also	also	ADV
ajema-7508	4	4	presents	present	VERB
ajema-7508	4	5	a	a	DET
ajema-7508	4	6	comparative	comparative	ADJ
ajema-7508	4	7	analysis	analysis	NOUN
ajema-7508	4	8	with	with	ADP
ajema-7508	4	9	traditional	traditional	ADJ
ajema-7508	4	10	models	model	NOUN
ajema-7508	4	11	,	,	PUNCT
ajema-7508	4	12	clearly	clearly	ADV
ajema-7508	4	13	outlining	outline	VERB
ajema-7508	4	14	the	the	DET
ajema-7508	4	15	advantages	advantage	NOUN
ajema-7508	4	16	of	of	ADP
ajema-7508	4	17	modern	modern	ADJ
ajema-7508	4	18	approaches	approach	NOUN
ajema-7508	4	19	.	.	PUNCT
ajema-7508	5	1	[	[	X
ajema-7508	5	2	1	1	NUM
ajema-7508	5	3	]	]	X
ajema-7508	5	4	main	main	ADJ
ajema-7508	5	5	results	result	NOUN
ajema-7508	5	6	:	:	PUNCT
ajema-7508	5	7	according	accord	VERB
ajema-7508	5	8	to	to	ADP
ajema-7508	5	9	the	the	DET
ajema-7508	5	10	research	research	NOUN
ajema-7508	5	11	findings	finding	NOUN
ajema-7508	5	12	,	,	PUNCT
ajema-7508	5	13	approaches	approach	NOUN
ajema-7508	5	14	based	base	VERB
ajema-7508	5	15	on	on	ADP
ajema-7508	5	16	the	the	DET
ajema-7508	5	17	bert	bert	PROPN
ajema-7508	5	18	model	model	NOUN
ajema-7508	5	19	demonstrate	demonstrate	VERB
ajema-7508	5	20	high	high	ADJ
ajema-7508	5	21	accuracy	accuracy	NOUN
ajema-7508	5	22	in	in	ADP
ajema-7508	5	23	contextual	contextual	ADJ
ajema-7508	5	24	and	and	CCONJ
ajema-7508	5	25	semantic	semantic	ADJ
ajema-7508	5	26	text	text	NOUN
ajema-7508	5	27	analysis	analysis	NOUN
ajema-7508	5	28	.	.	PUNCT
ajema-7508	6	1	specifically	specifically	ADV
ajema-7508	6	2	,	,	PUNCT
ajema-7508	6	3	the	the	DET
ajema-7508	6	4	topic	topic	NOUN
ajema-7508	6	5	bert	bert	NOUN
ajema-7508	6	6	-	-	PUNCT
ajema-7508	6	7	bigru	bigru	NOUN
ajema-7508	6	8	model	model	NOUN
ajema-7508	6	9	stood	stand	VERB
ajema-7508	6	10	out	out	ADP
ajema-7508	6	11	with	with	ADP
ajema-7508	6	12	an	an	DET
ajema-7508	6	13	f1	f1	NOUN
ajema-7508	6	14	-	-	PUNCT
ajema-7508	6	15	score	score	NOUN
ajema-7508	6	16	of	of	ADP
ajema-7508	6	17	86.91	86.91	NUM
ajema-7508	6	18	%	%	NOUN
ajema-7508	6	19	on	on	ADP
ajema-7508	6	20	short	short	ADJ
ajema-7508	6	21	texts	text	NOUN
ajema-7508	6	22	.	.	PUNCT
ajema-7508	7	1	distilbert	distilbert	PROPN
ajema-7508	7	2	outperformed	outperform	VERB
ajema-7508	7	3	the	the	DET
ajema-7508	7	4	classic	classic	ADJ
ajema-7508	7	5	bert	bert	NOUN
ajema-7508	7	6	model	model	NOUN
ajema-7508	7	7	by	by	ADP
ajema-7508	7	8	0.8–1.2	0.8–1.2	NOUN
ajema-7508	7	9	%	%	NOUN
ajema-7508	7	10	in	in	ADP
ajema-7508	7	11	accuracy	accuracy	NOUN
ajema-7508	7	12	on	on	ADP
ajema-7508	7	13	smaller	small	ADJ
ajema-7508	7	14	datasets	dataset	NOUN
ajema-7508	7	15	.	.	PUNCT
ajema-7508	8	1	transformer	transformer	NOUN
ajema-7508	8	2	-	-	PUNCT
ajema-7508	8	3	based	base	VERB
ajema-7508	8	4	classification	classification	NOUN
ajema-7508	8	5	systems	system	NOUN
ajema-7508	8	6	,	,	PUNCT
ajema-7508	8	7	especially	especially	ADV
ajema-7508	8	8	when	when	SCONJ
ajema-7508	8	9	combined	combine	VERB
ajema-7508	8	10	with	with	ADP
ajema-7508	8	11	the	the	DET
ajema-7508	8	12	random	random	ADJ
ajema-7508	8	13	forest	forest	NOUN
ajema-7508	8	14	classifier	classifier	NOUN
ajema-7508	8	15	,	,	PUNCT
ajema-7508	8	16	achieved	achieve	VERB
ajema-7508	8	17	over	over	ADP
ajema-7508	8	18	90	90	NUM
ajema-7508	8	19	%	%	NOUN
ajema-7508	8	20	accuracy	accuracy	NOUN
ajema-7508	8	21	.	.	PUNCT
ajema-7508	9	1	these	these	DET
ajema-7508	9	2	approaches	approach	NOUN
ajema-7508	9	3	confirm	confirm	VERB
ajema-7508	9	4	the	the	DET
ajema-7508	9	5	high	high	ADJ
ajema-7508	9	6	efficiency	efficiency	NOUN
ajema-7508	9	7	of	of	ADP
ajema-7508	9	8	the	the	DET
ajema-7508	9	9	bert	bert	PROPN
ajema-7508	9	10	model	model	NOUN
ajema-7508	9	11	in	in	ADP
ajema-7508	9	12	real	real	ADJ
ajema-7508	9	13	-	-	PUNCT
ajema-7508	9	14	world	world	NOUN
ajema-7508	9	15	applications	application	NOUN
ajema-7508	9	16	.	.	PUNCT
ajema-7508	10	1	keywords	keyword	NOUN
ajema-7508	10	2	:	:	PUNCT
ajema-7508	10	3	bert	bert	PROPN
ajema-7508	10	4	,	,	PUNCT
ajema-7508	10	5	transformer	transformer	NOUN
ajema-7508	10	6	,	,	PUNCT
ajema-7508	10	7	bigru	bigru	NOUN
ajema-7508	10	8	,	,	PUNCT
ajema-7508	10	9	topic	topic	NOUN
ajema-7508	10	10	modeling	modeling	NOUN
ajema-7508	10	11	,	,	PUNCT
ajema-7508	10	12	nlp	nlp	ADJ
ajema-7508	10	13	,	,	PUNCT
ajema-7508	10	14	semantic	semantic	ADJ
ajema-7508	10	15	analysis	analysis	NOUN
ajema-7508	10	16	,	,	PUNCT
ajema-7508	10	17	text	text	NOUN
ajema-7508	10	18	classification	classification	NOUN
ajema-7508	10	19	.	.	PUNCT
ajema-7508	11	1	i.	i.	NOUN
ajema-7508	11	2	introduction	introduction	NOUN
ajema-7508	11	3	in	in	ADP
ajema-7508	11	4	the	the	DET
ajema-7508	11	5	era	era	NOUN
ajema-7508	11	6	of	of	ADP
ajema-7508	11	7	modern	modern	ADJ
ajema-7508	11	8	information	information	NOUN
ajema-7508	11	9	technologies	technology	NOUN
ajema-7508	11	10	,	,	PUNCT
ajema-7508	11	11	the	the	DET
ajema-7508	11	12	volume	volume	NOUN
ajema-7508	11	13	of	of	ADP
ajema-7508	11	14	textual	textual	ADJ
ajema-7508	11	15	data	datum	NOUN
ajema-7508	11	16	obtained	obtain	VERB
ajema-7508	11	17	from	from	ADP
ajema-7508	11	18	the	the	DET
ajema-7508	11	19	internet	internet	NOUN
ajema-7508	11	20	,	,	PUNCT
ajema-7508	11	21	social	social	ADJ
ajema-7508	11	22	networks	network	NOUN
ajema-7508	11	23	,	,	PUNCT
ajema-7508	11	24	and	and	CCONJ
ajema-7508	11	25	online	online	ADJ
ajema-7508	11	26	platforms	platform	NOUN
ajema-7508	11	27	is	be	AUX
ajema-7508	11	28	rapidly	rapidly	ADV
ajema-7508	11	29	increasing	increase	VERB
ajema-7508	11	30	.	.	PUNCT
ajema-7508	12	1	the	the	DET
ajema-7508	12	2	need	need	NOUN
ajema-7508	12	3	to	to	PART
ajema-7508	12	4	automatically	automatically	ADV
ajema-7508	12	5	analyze	analyze	VERB
ajema-7508	12	6	,	,	PUNCT
ajema-7508	12	7	classify	classify	VERB
ajema-7508	12	8	,	,	PUNCT
ajema-7508	12	9	and	and	CCONJ
ajema-7508	12	10	draw	draw	VERB
ajema-7508	12	11	logical	logical	ADJ
ajema-7508	12	12	conclusions	conclusion	NOUN
ajema-7508	12	13	from	from	ADP
ajema-7508	12	14	this	this	DET
ajema-7508	12	15	data	data	NOUN
ajema-7508	12	16	has	have	AUX
ajema-7508	12	17	become	become	VERB
ajema-7508	12	18	a	a	DET
ajema-7508	12	19	strategic	strategic	ADJ
ajema-7508	12	20	direction	direction	NOUN
ajema-7508	12	21	in	in	ADP
ajema-7508	12	22	the	the	DET
ajema-7508	12	23	field	field	NOUN
ajema-7508	12	24	of	of	ADP
ajema-7508	12	25	natural	natural	ADJ
ajema-7508	12	26	language	language	NOUN
ajema-7508	12	27	processing	processing	NOUN
ajema-7508	12	28	(	(	PUNCT
ajema-7508	12	29	nlp	nlp	NOUN
ajema-7508	12	30	)	)	PUNCT
ajema-7508	12	31	.	.	PUNCT
ajema-7508	13	1	traditional	traditional	ADJ
ajema-7508	13	2	approaches	approach	NOUN
ajema-7508	13	3	,	,	PUNCT
ajema-7508	13	4	including	include	VERB
ajema-7508	13	5	bag	bag	NOUN
ajema-7508	13	6	-	-	PUNCT
ajema-7508	13	7	ofwords	ofword	NOUN
ajema-7508	13	8	(	(	PUNCT
ajema-7508	13	9	bow	bow	PROPN
ajema-7508	13	10	)	)	PUNCT
ajema-7508	13	11	,	,	PUNCT
ajema-7508	13	12	tf	tf	PROPN
ajema-7508	13	13	-	-	PUNCT
ajema-7508	13	14	idf	idf	PROPN
ajema-7508	13	15	,	,	PUNCT
ajema-7508	13	16	and	and	CCONJ
ajema-7508	13	17	word2vec	word2vec	PROPN
ajema-7508	13	18	models	model	NOUN
ajema-7508	13	19	,	,	PUNCT
ajema-7508	13	20	have	have	VERB
ajema-7508	13	21	key	key	ADJ
ajema-7508	13	22	limitations	limitation	NOUN
ajema-7508	13	23	—	—	PUNCT
ajema-7508	13	24	primarily	primarily	ADV
ajema-7508	13	25	their	their	PRON
ajema-7508	13	26	inability	inability	NOUN
ajema-7508	13	27	to	to	PART
ajema-7508	13	28	fully	fully	ADV
ajema-7508	13	29	capture	capture	VERB
ajema-7508	13	30	the	the	DET
ajema-7508	13	31	semantic	semantic	ADJ
ajema-7508	13	32	relationships	relationship	NOUN
ajema-7508	13	33	between	between	ADP
ajema-7508	13	34	words	word	NOUN
ajema-7508	13	35	and	and	CCONJ
ajema-7508	13	36	to	to	PART
ajema-7508	13	37	sufficiently	sufficiently	ADV
ajema-7508	13	38	understand	understand	VERB
ajema-7508	13	39	context	context	NOUN
ajema-7508	13	40	.	.	PUNCT
ajema-7508	14	1	[	[	X
ajema-7508	14	2	6	6	NUM
ajema-7508	14	3	]	]	X
ajema-7508	14	4	bert	bert	NOUN
ajema-7508	14	5	,	,	PUNCT
ajema-7508	14	6	on	on	ADP
ajema-7508	14	7	the	the	DET
ajema-7508	14	8	other	other	ADJ
ajema-7508	14	9	hand	hand	NOUN
ajema-7508	14	10	,	,	PUNCT
ajema-7508	14	11	was	be	AUX
ajema-7508	14	12	developed	develop	VERB
ajema-7508	14	13	precisely	precisely	ADV
ajema-7508	14	14	to	to	PART
ajema-7508	14	15	address	address	VERB
ajema-7508	14	16	these	these	DET
ajema-7508	14	17	issues	issue	NOUN
ajema-7508	14	18	,	,	PUNCT
ajema-7508	14	19	offering	offer	VERB
ajema-7508	14	20	an	an	DET
ajema-7508	14	21	advanced	advanced	ADJ
ajema-7508	14	22	approach	approach	NOUN
ajema-7508	14	23	that	that	PRON
ajema-7508	14	24	deeply	deeply	ADV
ajema-7508	14	25	understands	understand	VERB
ajema-7508	14	26	texts	text	NOUN
ajema-7508	14	27	through	through	ADP
ajema-7508	14	28	bidirectional	bidirectional	ADJ
ajema-7508	14	29	encoding	encoding	NOUN
ajema-7508	14	30	,	,	PUNCT
ajema-7508	14	31	self	self	NOUN
ajema-7508	14	32	-	-	PUNCT
ajema-7508	14	33	attention	attention	NOUN
ajema-7508	14	34	,	,	PUNCT
ajema-7508	14	35	and	and	CCONJ
ajema-7508	14	36	pre	pre	ADJ
ajema-7508	14	37	-	-	ADJ
ajema-7508	14	38	training	training	ADJ
ajema-7508	14	39	mechanisms	mechanism	NOUN
ajema-7508	14	40	.	.	PUNCT
ajema-7508	15	1	[	[	X
ajema-7508	15	2	5	5	NUM
ajema-7508	15	3	]	]	PUNCT
ajema-7508	15	4	in	in	ADP
ajema-7508	15	5	today	today	NOUN
ajema-7508	15	6	's	's	PART
ajema-7508	15	7	digital	digital	ADJ
ajema-7508	15	8	world	world	NOUN
ajema-7508	15	9	,	,	PUNCT
ajema-7508	15	10	the	the	DET
ajema-7508	15	11	volume	volume	NOUN
ajema-7508	15	12	of	of	ADP
ajema-7508	15	13	textual	textual	ADJ
ajema-7508	15	14	content	content	NOUN
ajema-7508	15	15	on	on	ADP
ajema-7508	15	16	the	the	DET
ajema-7508	15	17	internet	internet	NOUN
ajema-7508	15	18	is	be	AUX
ajema-7508	15	19	growing	grow	VERB
ajema-7508	15	20	at	at	ADP
ajema-7508	15	21	an	an	DET
ajema-7508	15	22	exponential	exponential	ADJ
ajema-7508	15	23	rate	rate	NOUN
ajema-7508	15	24	.	.	PUNCT
ajema-7508	16	1	at	at	ADP
ajema-7508	16	2	the	the	DET
ajema-7508	16	3	same	same	ADJ
ajema-7508	16	4	time	time	NOUN
ajema-7508	16	5	,	,	PUNCT
ajema-7508	16	6	users	user	NOUN
ajema-7508	16	7	’	'	PUNCT
ajema-7508	16	8	language	language	NOUN
ajema-7508	16	9	-	-	PUNCT
ajema-7508	16	10	related	relate	VERB
ajema-7508	16	11	needs	need	NOUN
ajema-7508	16	12	—	—	PUNCT
ajema-7508	16	13	such	such	ADJ
ajema-7508	16	14	as	as	ADP
ajema-7508	16	15	automatic	automatic	ADJ
ajema-7508	16	16	translation	translation	NOUN
ajema-7508	16	17	,	,	PUNCT
ajema-7508	16	18	153	153	NUM
ajema-7508	16	19	journal	journal	NOUN
ajema-7508	16	20	of	of	ADP
ajema-7508	16	21	engineering	engineering	NOUN
ajema-7508	16	22	,	,	PUNCT
ajema-7508	16	23	mechanics	mechanic	NOUN
ajema-7508	16	24	and	and	CCONJ
ajema-7508	16	25	architecture	architecture	NOUN
ajema-7508	16	26	www	www	NOUN
ajema-7508	16	27	.	.	PUNCT
ajema-7508	17	1	grnjournal.us	grnjournal.us	X
ajema-7508	17	2	text	text	NOUN
ajema-7508	17	3	summarization	summarization	NOUN
ajema-7508	17	4	,	,	PUNCT
ajema-7508	17	5	chatbots	chatbot	NOUN
ajema-7508	17	6	,	,	PUNCT
ajema-7508	17	7	recommendation	recommendation	NOUN
ajema-7508	17	8	systems	system	NOUN
ajema-7508	17	9	,	,	PUNCT
ajema-7508	17	10	and	and	CCONJ
ajema-7508	17	11	fake	fake	ADJ
ajema-7508	17	12	news	news	NOUN
ajema-7508	17	13	detection	detection	NOUN
ajema-7508	17	14	—	—	PUNCT
ajema-7508	17	15	are	be	AUX
ajema-7508	17	16	also	also	ADV
ajema-7508	17	17	rapidly	rapidly	ADV
ajema-7508	17	18	increasing	increase	VERB
ajema-7508	17	19	.	.	PUNCT
ajema-7508	18	1	traditional	traditional	ADJ
ajema-7508	18	2	methods	method	NOUN
ajema-7508	18	3	have	have	AUX
ajema-7508	18	4	proven	prove	VERB
ajema-7508	18	5	inadequate	inadequate	ADJ
ajema-7508	18	6	when	when	SCONJ
ajema-7508	18	7	dealing	deal	VERB
ajema-7508	18	8	with	with	ADP
ajema-7508	18	9	such	such	ADJ
ajema-7508	18	10	complex	complex	ADJ
ajema-7508	18	11	and	and	CCONJ
ajema-7508	18	12	multi	multi	ADJ
ajema-7508	18	13	-	-	ADJ
ajema-7508	18	14	layered	layered	ADJ
ajema-7508	18	15	texts	text	NOUN
ajema-7508	18	16	.	.	PUNCT
ajema-7508	19	1	in	in	ADP
ajema-7508	19	2	this	this	DET
ajema-7508	19	3	context	context	NOUN
ajema-7508	19	4	,	,	PUNCT
ajema-7508	19	5	transformer	transformer	NOUN
ajema-7508	19	6	models	model	NOUN
ajema-7508	19	7	based	base	VERB
ajema-7508	19	8	on	on	ADP
ajema-7508	19	9	deep	deep	ADJ
ajema-7508	19	10	learning	learning	NOUN
ajema-7508	19	11	,	,	PUNCT
ajema-7508	19	12	such	such	ADJ
ajema-7508	19	13	as	as	ADP
ajema-7508	19	14	bert	bert	NOUN
ajema-7508	19	15	,	,	PUNCT
ajema-7508	19	16	have	have	AUX
ajema-7508	19	17	gained	gain	VERB
ajema-7508	19	18	even	even	ADV
ajema-7508	19	19	greater	great	ADJ
ajema-7508	19	20	significance	significance	NOUN
ajema-7508	19	21	.	.	PUNCT
ajema-7508	20	1	the	the	DET
ajema-7508	20	2	bert	bert	PROPN
ajema-7508	20	3	model	model	NOUN
ajema-7508	20	4	has	have	AUX
ajema-7508	20	5	become	become	VERB
ajema-7508	20	6	a	a	DET
ajema-7508	20	7	revolutionary	revolutionary	ADJ
ajema-7508	20	8	solution	solution	NOUN
ajema-7508	20	9	for	for	ADP
ajema-7508	20	10	understanding	understand	VERB
ajema-7508	20	11	contextual	contextual	ADJ
ajema-7508	20	12	semantics	semantic	NOUN
ajema-7508	20	13	,	,	PUNCT
ajema-7508	20	14	detecting	detect	VERB
ajema-7508	20	15	polysemy	polysemy	NOUN
ajema-7508	20	16	,	,	PUNCT
ajema-7508	20	17	and	and	CCONJ
ajema-7508	20	18	capturing	capture	VERB
ajema-7508	20	19	complex	complex	ADJ
ajema-7508	20	20	relationships	relationship	NOUN
ajema-7508	20	21	between	between	ADP
ajema-7508	20	22	linguistic	linguistic	ADJ
ajema-7508	20	23	units	unit	NOUN
ajema-7508	20	24	.	.	PUNCT
ajema-7508	21	1	therefore	therefore	ADV
ajema-7508	21	2	,	,	PUNCT
ajema-7508	21	3	this	this	DET
ajema-7508	21	4	topic	topic	NOUN
ajema-7508	21	5	is	be	AUX
ajema-7508	21	6	considered	consider	VERB
ajema-7508	21	7	highly	highly	ADV
ajema-7508	21	8	relevant	relevant	ADJ
ajema-7508	21	9	not	not	PART
ajema-7508	21	10	only	only	ADV
ajema-7508	21	11	from	from	ADP
ajema-7508	21	12	a	a	DET
ajema-7508	21	13	scientific	scientific	ADJ
ajema-7508	21	14	perspective	perspective	NOUN
ajema-7508	21	15	but	but	CCONJ
ajema-7508	21	16	also	also	ADV
ajema-7508	21	17	in	in	ADP
ajema-7508	21	18	practical	practical	ADJ
ajema-7508	21	19	applications	application	NOUN
ajema-7508	21	20	.	.	PUNCT
ajema-7508	22	1	the	the	DET
ajema-7508	22	2	main	main	ADJ
ajema-7508	22	3	goal	goal	NOUN
ajema-7508	22	4	of	of	ADP
ajema-7508	22	5	this	this	DET
ajema-7508	22	6	study	study	NOUN
ajema-7508	22	7	is	be	AUX
ajema-7508	22	8	to	to	PART
ajema-7508	22	9	thoroughly	thoroughly	ADV
ajema-7508	22	10	explore	explore	VERB
ajema-7508	22	11	the	the	DET
ajema-7508	22	12	role	role	NOUN
ajema-7508	22	13	of	of	ADP
ajema-7508	22	14	the	the	DET
ajema-7508	22	15	bert	bert	PROPN
ajema-7508	22	16	model	model	NOUN
ajema-7508	22	17	in	in	ADP
ajema-7508	22	18	natural	natural	ADJ
ajema-7508	22	19	language	language	NOUN
ajema-7508	22	20	processing	processing	NOUN
ajema-7508	22	21	,	,	PUNCT
ajema-7508	22	22	particularly	particularly	ADV
ajema-7508	22	23	in	in	ADP
ajema-7508	22	24	text	text	NOUN
ajema-7508	22	25	classification	classification	NOUN
ajema-7508	22	26	and	and	CCONJ
ajema-7508	22	27	analysis	analysis	NOUN
ajema-7508	22	28	,	,	PUNCT
ajema-7508	22	29	to	to	PART
ajema-7508	22	30	examine	examine	VERB
ajema-7508	22	31	its	its	PRON
ajema-7508	22	32	technical	technical	ADJ
ajema-7508	22	33	mechanisms	mechanism	NOUN
ajema-7508	22	34	,	,	PUNCT
ajema-7508	22	35	and	and	CCONJ
ajema-7508	22	36	to	to	PART
ajema-7508	22	37	scientifically	scientifically	ADV
ajema-7508	22	38	compare	compare	VERB
ajema-7508	22	39	the	the	DET
ajema-7508	22	40	advantages	advantage	NOUN
ajema-7508	22	41	of	of	ADP
ajema-7508	22	42	improved	improved	ADJ
ajema-7508	22	43	approaches	approach	NOUN
ajema-7508	22	44	such	such	ADJ
ajema-7508	22	45	as	as	ADP
ajema-7508	22	46	distilbert	distilbert	NOUN
ajema-7508	22	47	and	and	CCONJ
ajema-7508	22	48	topic	topic	NOUN
ajema-7508	22	49	bert	bert	NOUN
ajema-7508	22	50	-	-	PUNCT
ajema-7508	22	51	bigru	bigru	NOUN
ajema-7508	22	52	.	.	PUNCT
ajema-7508	23	1	this	this	DET
ajema-7508	23	2	article	article	NOUN
ajema-7508	23	3	analyzes	analyze	VERB
ajema-7508	23	4	the	the	DET
ajema-7508	23	5	performance	performance	NOUN
ajema-7508	23	6	,	,	PUNCT
ajema-7508	23	7	capabilities	capability	NOUN
ajema-7508	23	8	,	,	PUNCT
ajema-7508	23	9	and	and	CCONJ
ajema-7508	23	10	limitations	limitation	NOUN
ajema-7508	23	11	of	of	ADP
ajema-7508	23	12	bert	bert	NOUN
ajema-7508	23	13	-	-	PUNCT
ajema-7508	23	14	based	base	VERB
ajema-7508	23	15	models	model	NOUN
ajema-7508	23	16	by	by	ADP
ajema-7508	23	17	applying	apply	VERB
ajema-7508	23	18	them	they	PRON
ajema-7508	23	19	to	to	ADP
ajema-7508	23	20	different	different	ADJ
ajema-7508	23	21	types	type	NOUN
ajema-7508	23	22	of	of	ADP
ajema-7508	23	23	text	text	NOUN
ajema-7508	23	24	corpora	corpus	NOUN
ajema-7508	23	25	.	.	PUNCT
ajema-7508	24	1	as	as	ADP
ajema-7508	24	2	a	a	DET
ajema-7508	24	3	result	result	NOUN
ajema-7508	24	4	,	,	PUNCT
ajema-7508	24	5	a	a	DET
ajema-7508	24	6	scientific	scientific	ADJ
ajema-7508	24	7	and	and	CCONJ
ajema-7508	24	8	methodological	methodological	ADJ
ajema-7508	24	9	foundation	foundation	NOUN
ajema-7508	24	10	is	be	AUX
ajema-7508	24	11	established	establish	VERB
ajema-7508	24	12	for	for	ADP
ajema-7508	24	13	the	the	DET
ajema-7508	24	14	practical	practical	ADJ
ajema-7508	24	15	application	application	NOUN
ajema-7508	24	16	of	of	ADP
ajema-7508	24	17	the	the	DET
ajema-7508	24	18	bert	bert	PROPN
ajema-7508	24	19	model	model	PROPN
ajema-7508	24	20	.	.	PUNCT
ajema-7508	25	1	ii	ii	PROPN
ajema-7508	25	2	bert	bert	PROPN
ajema-7508	25	3	architecture	architecture	NOUN
ajema-7508	25	4	and	and	CCONJ
ajema-7508	25	5	its	its	PRON
ajema-7508	25	6	application	application	NOUN
ajema-7508	25	7	in	in	ADP
ajema-7508	25	8	text	text	NOUN
ajema-7508	25	9	classification	classification	NOUN
ajema-7508	25	10	2.1	2.1	NUM
ajema-7508	25	11	.	.	PUNCT
ajema-7508	26	1	internal	internal	ADJ
ajema-7508	26	2	structure	structure	NOUN
ajema-7508	26	3	and	and	CCONJ
ajema-7508	26	4	working	working	NOUN
ajema-7508	26	5	principle	principle	NOUN
ajema-7508	26	6	of	of	ADP
ajema-7508	26	7	the	the	DET
ajema-7508	26	8	bert	bert	PROPN
ajema-7508	26	9	model.the	model.the	DET
ajema-7508	26	10	bert	bert	PROPN
ajema-7508	26	11	model	model	NOUN
ajema-7508	26	12	was	be	AUX
ajema-7508	26	13	introduced	introduce	VERB
ajema-7508	26	14	by	by	ADP
ajema-7508	26	15	google	google	PROPN
ajema-7508	26	16	ai	ai	PROPN
ajema-7508	26	17	in	in	ADP
ajema-7508	26	18	2018	2018	NUM
ajema-7508	26	19	.	.	PUNCT
ajema-7508	27	1	its	its	PRON
ajema-7508	27	2	main	main	ADJ
ajema-7508	27	3	advantage	advantage	NOUN
ajema-7508	27	4	lies	lie	VERB
ajema-7508	27	5	in	in	ADP
ajema-7508	27	6	the	the	DET
ajema-7508	27	7	ability	ability	NOUN
ajema-7508	27	8	to	to	PART
ajema-7508	27	9	understand	understand	VERB
ajema-7508	27	10	the	the	DET
ajema-7508	27	11	meaning	meaning	NOUN
ajema-7508	27	12	of	of	ADP
ajema-7508	27	13	each	each	DET
ajema-7508	27	14	word	word	NOUN
ajema-7508	27	15	not	not	PART
ajema-7508	27	16	only	only	ADV
ajema-7508	27	17	in	in	ADP
ajema-7508	27	18	the	the	DET
ajema-7508	27	19	context	context	NOUN
ajema-7508	27	20	of	of	ADP
ajema-7508	27	21	preceding	precede	VERB
ajema-7508	27	22	words	word	NOUN
ajema-7508	27	23	but	but	CCONJ
ajema-7508	27	24	also	also	ADV
ajema-7508	27	25	in	in	ADP
ajema-7508	27	26	the	the	DET
ajema-7508	27	27	context	context	NOUN
ajema-7508	27	28	of	of	ADP
ajema-7508	27	29	subsequent	subsequent	ADJ
ajema-7508	27	30	ones	one	NOUN
ajema-7508	27	31	.	.	PUNCT
ajema-7508	28	1	this	this	PRON
ajema-7508	28	2	is	be	AUX
ajema-7508	28	3	made	make	VERB
ajema-7508	28	4	possible	possible	ADJ
ajema-7508	28	5	through	through	ADP
ajema-7508	28	6	two	two	NUM
ajema-7508	28	7	pre	pre	ADJ
ajema-7508	28	8	-	-	ADJ
ajema-7508	28	9	training	training	ADJ
ajema-7508	28	10	strategies	strategy	NOUN
ajema-7508	28	11	:	:	PUNCT
ajema-7508	28	12	masked	mask	VERB
ajema-7508	28	13	language	language	NOUN
ajema-7508	28	14	modeling	modeling	NOUN
ajema-7508	28	15	(	(	PUNCT
ajema-7508	28	16	mlm	mlm	NOUN
ajema-7508	28	17	)	)	PUNCT
ajema-7508	28	18	and	and	CCONJ
ajema-7508	28	19	next	next	ADJ
ajema-7508	28	20	sentence	sentence	NOUN
ajema-7508	28	21	prediction	prediction	NOUN
ajema-7508	28	22	(	(	PUNCT
ajema-7508	28	23	nsp	nsp	PROPN
ajema-7508	28	24	)	)	PUNCT
ajema-7508	28	25	.	.	PUNCT
ajema-7508	29	1	built	build	VERB
ajema-7508	29	2	on	on	ADP
ajema-7508	29	3	the	the	DET
ajema-7508	29	4	transformer	transformer	NOUN
ajema-7508	29	5	architecture	architecture	NOUN
ajema-7508	29	6	,	,	PUNCT
ajema-7508	29	7	bert	bert	PROPN
ajema-7508	29	8	represents	represent	VERB
ajema-7508	29	9	words	word	NOUN
ajema-7508	29	10	as	as	ADP
ajema-7508	29	11	vectors	vector	NOUN
ajema-7508	29	12	using	use	VERB
ajema-7508	29	13	complex	complex	ADJ
ajema-7508	29	14	attention	attention	NOUN
ajema-7508	29	15	mechanisms	mechanism	NOUN
ajema-7508	29	16	,	,	PUNCT
ajema-7508	29	17	where	where	SCONJ
ajema-7508	29	18	these	these	DET
ajema-7508	29	19	vectors	vector	NOUN
ajema-7508	29	20	encapsulate	encapsulate	VERB
ajema-7508	29	21	deep	deep	ADJ
ajema-7508	29	22	contextual	contextual	ADJ
ajema-7508	29	23	semantic	semantic	ADJ
ajema-7508	29	24	information	information	NOUN
ajema-7508	29	25	.	.	PUNCT
ajema-7508	30	1	[	[	X
ajema-7508	30	2	4	4	NUM
ajema-7508	30	3	]	]	X
ajema-7508	30	4	bert	bert	NOUN
ajema-7508	30	5	models	model	NOUN
ajema-7508	30	6	have	have	AUX
ajema-7508	30	7	demonstrated	demonstrate	VERB
ajema-7508	30	8	state	state	NOUN
ajema-7508	30	9	-	-	PUNCT
ajema-7508	30	10	ofthe	ofthe	NOUN
ajema-7508	30	11	-	-	PUNCT
ajema-7508	30	12	art	art	NOUN
ajema-7508	30	13	performance	performance	NOUN
ajema-7508	30	14	in	in	ADP
ajema-7508	30	15	various	various	ADJ
ajema-7508	30	16	nlp	nlp	NOUN
ajema-7508	30	17	tasks	task	NOUN
ajema-7508	30	18	,	,	PUNCT
ajema-7508	30	19	such	such	ADJ
ajema-7508	30	20	as	as	ADP
ajema-7508	30	21	question	question	NOUN
ajema-7508	30	22	answering	answering	NOUN
ajema-7508	30	23	,	,	PUNCT
ajema-7508	30	24	text	text	NOUN
ajema-7508	30	25	classification	classification	NOUN
ajema-7508	30	26	,	,	PUNCT
ajema-7508	30	27	and	and	CCONJ
ajema-7508	30	28	named	name	VERB
ajema-7508	30	29	entity	entity	NOUN
ajema-7508	30	30	recognition	recognition	NOUN
ajema-7508	30	31	.	.	PUNCT
ajema-7508	31	1	bert	bert	PROPN
ajema-7508	31	2	is	be	AUX
ajema-7508	31	3	a	a	DET
ajema-7508	31	4	deep	deep	ADJ
ajema-7508	31	5	neural	neural	ADJ
ajema-7508	31	6	network	network	NOUN
ajema-7508	31	7	-	-	PUNCT
ajema-7508	31	8	based	base	VERB
ajema-7508	31	9	natural	natural	ADJ
ajema-7508	31	10	language	language	NOUN
ajema-7508	31	11	understanding	understanding	NOUN
ajema-7508	31	12	model	model	NOUN
ajema-7508	31	13	that	that	PRON
ajema-7508	31	14	can	can	AUX
ajema-7508	31	15	learn	learn	VERB
ajema-7508	31	16	the	the	DET
ajema-7508	31	17	semantics	semantic	NOUN
ajema-7508	31	18	and	and	CCONJ
ajema-7508	31	19	structure	structure	NOUN
ajema-7508	31	20	of	of	ADP
ajema-7508	31	21	language	language	NOUN
ajema-7508	31	22	from	from	ADP
ajema-7508	31	23	large	large	ADJ
ajema-7508	31	24	volumes	volume	NOUN
ajema-7508	31	25	of	of	ADP
ajema-7508	31	26	unlabeled	unlabeled	ADJ
ajema-7508	31	27	text	text	NOUN
ajema-7508	31	28	.	.	PUNCT
ajema-7508	32	1	[	[	X
ajema-7508	32	2	5	5	NUM
ajema-7508	32	3	]	]	PUNCT
ajema-7508	32	4	one	one	NUM
ajema-7508	32	5	of	of	ADP
ajema-7508	32	6	bert	bert	PROPN
ajema-7508	32	7	’s	’s	PART
ajema-7508	32	8	core	core	NOUN
ajema-7508	32	9	innovations	innovation	NOUN
ajema-7508	32	10	is	be	AUX
ajema-7508	32	11	the	the	DET
ajema-7508	32	12	use	use	NOUN
ajema-7508	32	13	of	of	ADP
ajema-7508	32	14	a	a	DET
ajema-7508	32	15	bidirectional	bidirectional	ADJ
ajema-7508	32	16	transformer	transformer	NOUN
ajema-7508	32	17	encoder	encoder	NOUN
ajema-7508	32	18	,	,	PUNCT
ajema-7508	32	19	which	which	PRON
ajema-7508	32	20	considers	consider	VERB
ajema-7508	32	21	both	both	PRON
ajema-7508	32	22	left	left	ADJ
ajema-7508	32	23	and	and	CCONJ
ajema-7508	32	24	right	right	ADJ
ajema-7508	32	25	contexts	context	NOUN
ajema-7508	32	26	to	to	PART
ajema-7508	32	27	extract	extract	VERB
ajema-7508	32	28	richer	rich	ADJ
ajema-7508	32	29	linguistic	linguistic	ADJ
ajema-7508	32	30	features	feature	NOUN
ajema-7508	32	31	.	.	PUNCT
ajema-7508	33	1	the	the	DET
ajema-7508	33	2	model	model	NOUN
ajema-7508	33	3	’s	’s	PART
ajema-7508	33	4	pre	pre	ADJ
ajema-7508	33	5	-	-	ADJ
ajema-7508	33	6	training	training	ADJ
ajema-7508	33	7	tasks	task	NOUN
ajema-7508	33	8	—	—	PUNCT
ajema-7508	33	9	mlm	mlm	PROPN
ajema-7508	33	10	and	and	CCONJ
ajema-7508	33	11	nsp	nsp	PROPN
ajema-7508	33	12	—	—	PUNCT
ajema-7508	33	13	are	be	AUX
ajema-7508	33	14	designed	design	VERB
ajema-7508	33	15	to	to	PART
ajema-7508	33	16	learn	learn	VERB
ajema-7508	33	17	representations	representation	NOUN
ajema-7508	33	18	at	at	ADP
ajema-7508	33	19	the	the	DET
ajema-7508	33	20	vocabulary	vocabulary	ADJ
ajema-7508	33	21	and	and	CCONJ
ajema-7508	33	22	sentence	sentence	NOUN
ajema-7508	33	23	levels	level	NOUN
ajema-7508	33	24	,	,	PUNCT
ajema-7508	33	25	respectively	respectively	ADV
ajema-7508	33	26	.	.	PUNCT
ajema-7508	34	1	this	this	DET
ajema-7508	34	2	model	model	NOUN
ajema-7508	34	3	utilizes	utilize	VERB
ajema-7508	34	4	only	only	ADV
ajema-7508	34	5	the	the	DET
ajema-7508	34	6	encoder	encoder	NOUN
ajema-7508	34	7	part	part	NOUN
ajema-7508	34	8	of	of	ADP
ajema-7508	34	9	the	the	DET
ajema-7508	34	10	transformer	transformer	NOUN
ajema-7508	34	11	architecture	architecture	NOUN
ajema-7508	34	12	and	and	CCONJ
ajema-7508	34	13	operates	operate	VERB
ajema-7508	34	14	by	by	ADP
ajema-7508	34	15	incorporating	incorporate	VERB
ajema-7508	34	16	bidirectional	bidirectional	ADJ
ajema-7508	34	17	context	context	NOUN
ajema-7508	34	18	.	.	PUNCT
ajema-7508	35	1	(	(	PUNCT
ajema-7508	35	2	see	see	VERB
ajema-7508	35	3	figure	figure	NOUN
ajema-7508	35	4	1	1	NUM
ajema-7508	35	5	)	)	PUNCT
ajema-7508	35	6	input	input	NOUN
ajema-7508	35	7	layer	layer	NOUN
ajema-7508	35	8	:	:	PUNCT
ajema-7508	35	9	when	when	SCONJ
ajema-7508	35	10	text	text	NOUN
ajema-7508	35	11	is	be	AUX
ajema-7508	35	12	input	input	NOUN
ajema-7508	35	13	into	into	ADP
ajema-7508	35	14	the	the	DET
ajema-7508	35	15	model	model	NOUN
ajema-7508	35	16	,	,	PUNCT
ajema-7508	35	17	it	it	PRON
ajema-7508	35	18	is	be	AUX
ajema-7508	35	19	converted	convert	VERB
ajema-7508	35	20	into	into	ADP
ajema-7508	35	21	a	a	DET
ajema-7508	35	22	numerical	numerical	ADJ
ajema-7508	35	23	representation	representation	NOUN
ajema-7508	35	24	using	use	VERB
ajema-7508	35	25	three	three	NUM
ajema-7508	35	26	types	type	NOUN
ajema-7508	35	27	of	of	ADP
ajema-7508	35	28	embeddings	embedding	NOUN
ajema-7508	35	29	:	:	PUNCT
ajema-7508	35	30			VERB
ajema-7508	35	31	token	token	ADJ
ajema-7508	35	32	embedding	embed	VERB
ajema-7508	35	33	–	–	PUNCT
ajema-7508	35	34	a	a	DET
ajema-7508	35	35	unique	unique	ADJ
ajema-7508	35	36	vector	vector	NOUN
ajema-7508	35	37	for	for	ADP
ajema-7508	35	38	each	each	PRON
ajema-7508	35	39	token	token	ADJ
ajema-7508	35	40	;	;	PUNCT
ajema-7508	35	41			ADJ
ajema-7508	35	42	segment	segment	NOUN
ajema-7508	35	43	embedding	embed	VERB
ajema-7508	35	44	–	–	PUNCT
ajema-7508	35	45	used	use	VERB
ajema-7508	35	46	to	to	PART
ajema-7508	35	47	distinguish	distinguish	VERB
ajema-7508	35	48	between	between	ADP
ajema-7508	35	49	different	different	ADJ
ajema-7508	35	50	text	text	NOUN
ajema-7508	35	51	segments	segment	NOUN
ajema-7508	35	52	;	;	PUNCT
ajema-7508	35	53			ADJ
ajema-7508	35	54	position	position	NOUN
ajema-7508	35	55	embedding	embed	VERB
ajema-7508	35	56	–	–	PUNCT
ajema-7508	35	57	represents	represent	VERB
ajema-7508	35	58	the	the	DET
ajema-7508	35	59	position	position	NOUN
ajema-7508	35	60	of	of	ADP
ajema-7508	35	61	tokens	token	NOUN
ajema-7508	35	62	within	within	ADP
ajema-7508	35	63	the	the	DET
ajema-7508	35	64	text	text	NOUN
ajema-7508	35	65	.	.	PUNCT
ajema-7508	36	1	these	these	DET
ajema-7508	36	2	embeddings	embedding	NOUN
ajema-7508	36	3	are	be	AUX
ajema-7508	36	4	combined	combine	VERB
ajema-7508	36	5	as	as	SCONJ
ajema-7508	36	6	follows	follow	VERB
ajema-7508	36	7	:	:	PUNCT
ajema-7508	36	8	(	(	PUNCT
ajema-7508	36	9	1	1	X
ajema-7508	36	10	)	)	PUNCT
ajema-7508	36	11	transformer	transformer	NOUN
ajema-7508	36	12	encoder	encoder	NOUN
ajema-7508	36	13	blocks	block	NOUN
ajema-7508	36	14	:	:	PUNCT
ajema-7508	36	15	the	the	DET
ajema-7508	36	16	bert	bert	PROPN
ajema-7508	36	17	model	model	NOUN
ajema-7508	36	18	consists	consist	VERB
ajema-7508	36	19	of	of	ADP
ajema-7508	36	20	multiple	multiple	ADJ
ajema-7508	36	21	encoder	encoder	NOUN
ajema-7508	36	22	blocks	block	NOUN
ajema-7508	36	23	.	.	PUNCT
ajema-7508	37	1	each	each	DET
ajema-7508	37	2	block	block	NOUN
ajema-7508	37	3	includes	include	VERB
ajema-7508	37	4	the	the	DET
ajema-7508	37	5	following	follow	VERB
ajema-7508	37	6	components	component	NOUN
ajema-7508	37	7	:	:	PUNCT
ajema-7508	37	8			ADJ
ajema-7508	37	9	multi	multi	ADJ
ajema-7508	37	10	-	-	ADJ
ajema-7508	37	11	head	head	ADJ
ajema-7508	37	12	self	self	NOUN
ajema-7508	37	13	-	-	PUNCT
ajema-7508	37	14	attention	attention	NOUN
ajema-7508	37	15	:	:	PUNCT
ajema-7508	37	16	each	each	DET
ajema-7508	37	17	token	token	VERB
ajema-7508	37	18	is	be	AUX
ajema-7508	37	19	connected	connect	VERB
ajema-7508	37	20	with	with	ADP
ajema-7508	37	21	all	all	DET
ajema-7508	37	22	other	other	ADJ
ajema-7508	37	23	tokens	token	NOUN
ajema-7508	37	24	in	in	ADP
ajema-7508	37	25	the	the	DET
ajema-7508	37	26	input	input	NOUN
ajema-7508	37	27	sequence	sequence	NOUN
ajema-7508	37	28	,	,	PUNCT
ajema-7508	37	29	allowing	allow	VERB
ajema-7508	37	30	the	the	DET
ajema-7508	37	31	model	model	NOUN
ajema-7508	37	32	to	to	PART
ajema-7508	37	33	capture	capture	VERB
ajema-7508	37	34	contextual	contextual	ADJ
ajema-7508	37	35	relationships	relationship	NOUN
ajema-7508	37	36	.	.	PUNCT
ajema-7508	38	1			VERB
ajema-7508	38	2	feed	feed	NOUN
ajema-7508	38	3	forward	forward	ADV
ajema-7508	38	4	neural	neural	ADJ
ajema-7508	38	5	network	network	NOUN
ajema-7508	38	6	(	(	PUNCT
ajema-7508	38	7	ffnn	ffnn	NOUN
ajema-7508	38	8	):	):	PUNCT
ajema-7508	38	9	independently	independently	ADV
ajema-7508	38	10	processes	process	VERB
ajema-7508	38	11	each	each	DET
ajema-7508	38	12	token	token	VERB
ajema-7508	38	13	through	through	ADP
ajema-7508	38	14	a	a	DET
ajema-7508	38	15	fully	fully	ADV
ajema-7508	38	16	connected	connect	VERB
ajema-7508	38	17	neural	neural	ADJ
ajema-7508	38	18	network	network	NOUN
ajema-7508	38	19	.	.	PUNCT
ajema-7508	39	1			VERB
ajema-7508	39	2	residual	residual	ADJ
ajema-7508	39	3	connections	connection	NOUN
ajema-7508	39	4	and	and	CCONJ
ajema-7508	39	5	layer	layer	NOUN
ajema-7508	39	6	normalization	normalization	NOUN
ajema-7508	39	7	:	:	PUNCT
ajema-7508	39	8	these	these	DET
ajema-7508	39	9	components	component	NOUN
ajema-7508	39	10	ensure	ensure	VERB
ajema-7508	39	11	the	the	DET
ajema-7508	39	12	stability	stability	NOUN
ajema-7508	39	13	and	and	CCONJ
ajema-7508	39	14	efficiency	efficiency	NOUN
ajema-7508	39	15	of	of	ADP
ajema-7508	39	16	the	the	DET
ajema-7508	39	17	model	model	NOUN
ajema-7508	39	18	during	during	ADP
ajema-7508	39	19	training	training	NOUN
ajema-7508	39	20	by	by	ADP
ajema-7508	39	21	preserving	preserve	VERB
ajema-7508	39	22	gradients	gradient	NOUN
ajema-7508	39	23	and	and	CCONJ
ajema-7508	39	24	normalizing	normalizing	ADJ
ajema-7508	39	25	activations	activation	NOUN
ajema-7508	39	26	.	.	PUNCT
ajema-7508	40	1	154	154	NUM
ajema-7508	40	2	journal	journal	NOUN
ajema-7508	40	3	of	of	ADP
ajema-7508	40	4	engineering	engineering	NOUN
ajema-7508	40	5	,	,	PUNCT
ajema-7508	40	6	mechanics	mechanic	NOUN
ajema-7508	40	7	and	and	CCONJ
ajema-7508	40	8	architecture	architecture	NOUN
ajema-7508	40	9	www	www	NOUN
ajema-7508	40	10	.	.	PUNCT
ajema-7508	41	1	grnjournal.us	grnjournal.us	X
ajema-7508	41	2	each	each	DET
ajema-7508	41	3	encoder	encoder	NOUN
ajema-7508	41	4	block	block	NOUN
ajema-7508	41	5	can	can	AUX
ajema-7508	41	6	be	be	AUX
ajema-7508	41	7	represented	represent	VERB
ajema-7508	41	8	by	by	ADP
ajema-7508	41	9	the	the	DET
ajema-7508	41	10	following	follow	VERB
ajema-7508	41	11	formulas	formula	NOUN
ajema-7508	41	12	:	:	PUNCT
ajema-7508	41	13	(	(	PUNCT
ajema-7508	41	14	2	2	X
ajema-7508	41	15	)	)	PUNCT
ajema-7508	41	16	(	(	PUNCT
ajema-7508	41	17	3	3	X
ajema-7508	41	18	)	)	PUNCT
ajema-7508	41	19	here	here	ADV
ajema-7508	41	20	:	:	PUNCT
ajema-7508	41	21			ADJ
ajema-7508	41	22	x	x	ADJ
ajema-7508	41	23	–	–	PUNCT
ajema-7508	41	24	input	input	NOUN
ajema-7508	41	25	vector	vector	NOUN
ajema-7508	41	26			PROPN
ajema-7508	41	27	z	z	NOUN
ajema-7508	41	28	–	–	PUNCT
ajema-7508	41	29	result	result	NOUN
ajema-7508	41	30	of	of	ADP
ajema-7508	41	31	the	the	DET
ajema-7508	41	32	self	self	NOUN
ajema-7508	41	33	-	-	PUNCT
ajema-7508	41	34	attention	attention	NOUN
ajema-7508	41	35	mechanism	mechanism	NOUN
ajema-7508	41	36			VERB
ajema-7508	41	37	h	h	NOUN
ajema-7508	41	38	–	–	PUNCT
ajema-7508	41	39	output	output	NOUN
ajema-7508	41	40	of	of	ADP
ajema-7508	41	41	the	the	DET
ajema-7508	41	42	encoder	encoder	NOUN
ajema-7508	41	43	block	block	NOUN
ajema-7508	41	44	output	output	NOUN
ajema-7508	41	45	layer	layer	NOUN
ajema-7508	41	46	:	:	PUNCT
ajema-7508	41	47	at	at	ADP
ajema-7508	41	48	the	the	DET
ajema-7508	41	49	output	output	NOUN
ajema-7508	41	50	of	of	ADP
ajema-7508	41	51	the	the	DET
ajema-7508	41	52	model	model	NOUN
ajema-7508	41	53	,	,	PUNCT
ajema-7508	41	54	the	the	DET
ajema-7508	41	55	vector	vector	NOUN
ajema-7508	41	56	corresponding	correspond	VERB
ajema-7508	41	57	to	to	ADP
ajema-7508	41	58	the	the	DET
ajema-7508	41	59	[	[	X
ajema-7508	41	60	cls	cls	X
ajema-7508	41	61	]	]	X
ajema-7508	41	62	token	token	NOUN
ajema-7508	41	63	is	be	AUX
ajema-7508	41	64	taken	take	VERB
ajema-7508	41	65	as	as	ADP
ajema-7508	41	66	a	a	DET
ajema-7508	41	67	general	general	ADJ
ajema-7508	41	68	representation	representation	NOUN
ajema-7508	41	69	of	of	ADP
ajema-7508	41	70	the	the	DET
ajema-7508	41	71	entire	entire	ADJ
ajema-7508	41	72	input	input	NOUN
ajema-7508	41	73	text	text	NOUN
ajema-7508	41	74	.	.	PUNCT
ajema-7508	42	1	this	this	DET
ajema-7508	42	2	vector	vector	NOUN
ajema-7508	42	3	is	be	AUX
ajema-7508	42	4	then	then	ADV
ajema-7508	42	5	passed	pass	VERB
ajema-7508	42	6	through	through	ADP
ajema-7508	42	7	the	the	DET
ajema-7508	42	8	following	follow	VERB
ajema-7508	42	9	classification	classification	NOUN
ajema-7508	42	10	layer	layer	NOUN
ajema-7508	42	11	:	:	PUNCT
ajema-7508	42	12	(	(	PUNCT
ajema-7508	42	13	4	4	X
ajema-7508	42	14	)	)	PUNCT
ajema-7508	42	15	here	here	ADV
ajema-7508	42	16	:	:	PUNCT
ajema-7508	42	17			ADJ
ajema-7508	42	18	h	h	NOUN
ajema-7508	42	19	–	–	PUNCT
ajema-7508	42	20	hidden	hide	VERB
ajema-7508	42	21	vector	vector	NOUN
ajema-7508	42	22	of	of	ADP
ajema-7508	42	23	the	the	DET
ajema-7508	42	24	[	[	X
ajema-7508	42	25	cls	cls	X
ajema-7508	42	26	]	]	X
ajema-7508	42	27	token	token	VERB
ajema-7508	42	28			ADJ
ajema-7508	42	29	w	w	PROPN
ajema-7508	42	30	–	–	PUNCT
ajema-7508	42	31	weight	weight	NOUN
ajema-7508	42	32	matrix	matrix	NOUN
ajema-7508	42	33			VERB
ajema-7508	42	34	b	b	PROPN
ajema-7508	42	35	–	–	PUNCT
ajema-7508	42	36	bias	bias	NOUN
ajema-7508	42	37	vector	vector	NOUN
ajema-7508	42	38			PROPN
ajema-7508	42	39	y	y	PROPN
ajema-7508	42	40	–	–	PUNCT
ajema-7508	42	41	output	output	NOUN
ajema-7508	42	42	probabilities	probability	NOUN
ajema-7508	42	43	figure	figure	VERB
ajema-7508	42	44	1	1	NUM
ajema-7508	42	45	.	.	SYM
ajema-7508	42	46	155	155	NUM
ajema-7508	42	47	journal	journal	NOUN
ajema-7508	42	48	of	of	ADP
ajema-7508	42	49	engineering	engineering	NOUN
ajema-7508	42	50	,	,	PUNCT
ajema-7508	42	51	mechanics	mechanic	NOUN
ajema-7508	42	52	and	and	CCONJ
ajema-7508	42	53	architecture	architecture	NOUN
ajema-7508	42	54	www	www	NOUN
ajema-7508	42	55	.	.	PUNCT
ajema-7508	43	1	grnjournal.us	grnjournal.us	X
ajema-7508	43	2	2.2	2.2	NUM
ajema-7508	43	3	.	.	PUNCT
ajema-7508	44	1	text	text	NOUN
ajema-7508	44	2	classification	classification	NOUN
ajema-7508	44	3	based	base	VERB
ajema-7508	44	4	on	on	ADP
ajema-7508	44	5	bert	bert	PROPN
ajema-7508	44	6	.	.	PUNCT
ajema-7508	45	1	the	the	DET
ajema-7508	45	2	process	process	NOUN
ajema-7508	45	3	of	of	ADP
ajema-7508	45	4	text	text	NOUN
ajema-7508	45	5	classification	classification	NOUN
ajema-7508	45	6	using	use	VERB
ajema-7508	45	7	the	the	DET
ajema-7508	45	8	bert	bert	PROPN
ajema-7508	45	9	model	model	NOUN
ajema-7508	45	10	consists	consist	VERB
ajema-7508	45	11	of	of	ADP
ajema-7508	45	12	two	two	NUM
ajema-7508	45	13	stages	stage	NOUN
ajema-7508	45	14	:	:	PUNCT
ajema-7508	45	15	pre	pre	ADJ
ajema-7508	45	16	-	-	ADJ
ajema-7508	45	17	training	training	ADJ
ajema-7508	45	18	and	and	CCONJ
ajema-7508	45	19	fine	fine	ADV
ajema-7508	45	20	-	-	PUNCT
ajema-7508	45	21	tuning	tuning	NOUN
ajema-7508	45	22	.	.	PUNCT
ajema-7508	46	1	during	during	ADP
ajema-7508	46	2	the	the	DET
ajema-7508	46	3	pre	pre	ADJ
ajema-7508	46	4	-	-	ADJ
ajema-7508	46	5	training	training	ADJ
ajema-7508	46	6	phase	phase	NOUN
ajema-7508	46	7	,	,	PUNCT
ajema-7508	46	8	the	the	DET
ajema-7508	46	9	model	model	NOUN
ajema-7508	46	10	learns	learn	VERB
ajema-7508	46	11	general	general	ADJ
ajema-7508	46	12	language	language	NOUN
ajema-7508	46	13	understanding	understanding	NOUN
ajema-7508	46	14	from	from	ADP
ajema-7508	46	15	large	large	ADJ
ajema-7508	46	16	corpora	corpora	NOUN
ajema-7508	46	17	of	of	ADP
ajema-7508	46	18	text	text	NOUN
ajema-7508	46	19	.	.	PUNCT
ajema-7508	47	1	in	in	ADP
ajema-7508	47	2	the	the	DET
ajema-7508	47	3	fine	fine	ADV
ajema-7508	47	4	-	-	PUNCT
ajema-7508	47	5	tuning	tuning	NOUN
ajema-7508	47	6	phase	phase	NOUN
ajema-7508	47	7	,	,	PUNCT
ajema-7508	47	8	the	the	DET
ajema-7508	47	9	model	model	NOUN
ajema-7508	47	10	is	be	AUX
ajema-7508	47	11	adapted	adapt	VERB
ajema-7508	47	12	to	to	ADP
ajema-7508	47	13	a	a	DET
ajema-7508	47	14	specific	specific	ADJ
ajema-7508	47	15	task	task	NOUN
ajema-7508	47	16	.	.	PUNCT
ajema-7508	48	1	for	for	ADP
ajema-7508	48	2	classification	classification	NOUN
ajema-7508	48	3	,	,	PUNCT
ajema-7508	48	4	the	the	DET
ajema-7508	48	5	[	[	X
ajema-7508	48	6	cls	cls	X
ajema-7508	48	7	]	]	X
ajema-7508	48	8	token	token	NOUN
ajema-7508	48	9	is	be	AUX
ajema-7508	48	10	taken	take	VERB
ajema-7508	48	11	as	as	ADP
ajema-7508	48	12	the	the	DET
ajema-7508	48	13	main	main	ADJ
ajema-7508	48	14	semantic	semantic	ADJ
ajema-7508	48	15	vector	vector	NOUN
ajema-7508	48	16	,	,	PUNCT
ajema-7508	48	17	and	and	CCONJ
ajema-7508	48	18	the	the	DET
ajema-7508	48	19	class	class	NOUN
ajema-7508	48	20	is	be	AUX
ajema-7508	48	21	determined	determine	VERB
ajema-7508	48	22	through	through	ADP
ajema-7508	48	23	a	a	DET
ajema-7508	48	24	softmax	softmax	NOUN
ajema-7508	48	25	function	function	NOUN
ajema-7508	48	26	.	.	PUNCT
ajema-7508	49	1	this	this	DET
ajema-7508	49	2	approach	approach	NOUN
ajema-7508	49	3	demonstrates	demonstrate	VERB
ajema-7508	49	4	high	high	ADJ
ajema-7508	49	5	performance	performance	NOUN
ajema-7508	49	6	,	,	PUNCT
ajema-7508	49	7	especially	especially	ADV
ajema-7508	49	8	when	when	SCONJ
ajema-7508	49	9	dealing	deal	VERB
ajema-7508	49	10	with	with	ADP
ajema-7508	49	11	texts	text	NOUN
ajema-7508	49	12	containing	contain	VERB
ajema-7508	49	13	polysemous	polysemous	ADJ
ajema-7508	49	14	words	word	NOUN
ajema-7508	49	15	and	and	CCONJ
ajema-7508	49	16	complex	complex	ADJ
ajema-7508	49	17	sentence	sentence	NOUN
ajema-7508	49	18	structures	structure	NOUN
ajema-7508	49	19	.	.	PUNCT
ajema-7508	50	1	iii	iii	X
ajema-7508	50	2	.	.	PUNCT
ajema-7508	50	3	advanced	advanced	ADJ
ajema-7508	50	4	models	model	NOUN
ajema-7508	50	5	based	base	VERB
ajema-7508	50	6	on	on	ADP
ajema-7508	50	7	bert	bert	PROPN
ajema-7508	50	8	and	and	CCONJ
ajema-7508	50	9	their	their	PRON
ajema-7508	50	10	practical	practical	ADJ
ajema-7508	50	11	performance	performance	NOUN
ajema-7508	50	12	distilbert	distilbert	NOUN
ajema-7508	50	13	:	:	PUNCT
ajema-7508	50	14	the	the	DET
ajema-7508	50	15	distilbert	distilbert	ADJ
ajema-7508	50	16	model	model	NOUN
ajema-7508	50	17	enriches	enrich	VERB
ajema-7508	50	18	the	the	DET
ajema-7508	50	19	semantic	semantic	ADJ
ajema-7508	50	20	analysis	analysis	NOUN
ajema-7508	50	21	capabilities	capability	NOUN
ajema-7508	50	22	of	of	ADP
ajema-7508	50	23	bert	bert	NOUN
ajema-7508	50	24	by	by	ADP
ajema-7508	50	25	incorporating	incorporate	VERB
ajema-7508	50	26	rule	rule	NOUN
ajema-7508	50	27	-	-	PUNCT
ajema-7508	50	28	based	base	VERB
ajema-7508	50	29	logic	logic	NOUN
ajema-7508	50	30	based	base	VERB
ajema-7508	50	31	on	on	ADP
ajema-7508	50	32	an	an	DET
ajema-7508	50	33	additional	additional	ADJ
ajema-7508	50	34	lexicon	lexicon	NOUN
ajema-7508	50	35	.	.	PUNCT
ajema-7508	51	1	[	[	X
ajema-7508	51	2	6	6	NUM
ajema-7508	51	3	]	]	PUNCT
ajema-7508	51	4	the	the	DET
ajema-7508	51	5	core	core	ADJ
ajema-7508	51	6	idea	idea	NOUN
ajema-7508	51	7	of	of	ADP
ajema-7508	51	8	the	the	DET
ajema-7508	51	9	model	model	NOUN
ajema-7508	51	10	is	be	AUX
ajema-7508	51	11	that	that	SCONJ
ajema-7508	51	12	if	if	SCONJ
ajema-7508	51	13	the	the	DET
ajema-7508	51	14	ratio	ratio	NOUN
ajema-7508	51	15	of	of	ADP
ajema-7508	51	16	positive	positive	ADJ
ajema-7508	51	17	-	-	PUNCT
ajema-7508	51	18	to	to	ADP
ajema-7508	51	19	-	-	PUNCT
ajema-7508	51	20	negative	negative	ADJ
ajema-7508	51	21	probabilities	probability	NOUN
ajema-7508	51	22	in	in	ADP
ajema-7508	51	23	the	the	DET
ajema-7508	51	24	bert	bert	PROPN
ajema-7508	51	25	output	output	NOUN
ajema-7508	51	26	is	be	AUX
ajema-7508	51	27	low	low	ADJ
ajema-7508	51	28	,	,	PUNCT
ajema-7508	51	29	it	it	PRON
ajema-7508	51	30	indicates	indicate	VERB
ajema-7508	51	31	that	that	SCONJ
ajema-7508	51	32	the	the	DET
ajema-7508	51	33	model	model	NOUN
ajema-7508	51	34	has	have	AUX
ajema-7508	51	35	produced	produce	VERB
ajema-7508	51	36	unreliable	unreliable	ADJ
ajema-7508	51	37	results	result	NOUN
ajema-7508	51	38	.	.	PUNCT
ajema-7508	52	1	in	in	ADP
ajema-7508	52	2	this	this	DET
ajema-7508	52	3	case	case	NOUN
ajema-7508	52	4	,	,	PUNCT
ajema-7508	52	5	a	a	DET
ajema-7508	52	6	sentiment	sentiment	NOUN
ajema-7508	52	7	lexicon	lexicon	NOUN
ajema-7508	52	8	-	-	PUNCT
ajema-7508	52	9	based	base	VERB
ajema-7508	52	10	rule	rule	NOUN
ajema-7508	52	11	is	be	AUX
ajema-7508	52	12	applied	apply	VERB
ajema-7508	52	13	to	to	PART
ajema-7508	52	14	improve	improve	VERB
ajema-7508	52	15	the	the	DET
ajema-7508	52	16	result	result	NOUN
ajema-7508	52	17	’s	’s	PART
ajema-7508	52	18	accuracy	accuracy	NOUN
ajema-7508	52	19	.	.	PUNCT
ajema-7508	53	1	this	this	DET
ajema-7508	53	2	approach	approach	NOUN
ajema-7508	53	3	,	,	PUNCT
ajema-7508	53	4	especially	especially	ADV
ajema-7508	53	5	in	in	ADP
ajema-7508	53	6	small	small	ADJ
ajema-7508	53	7	datasets	dataset	NOUN
ajema-7508	53	8	,	,	PUNCT
ajema-7508	53	9	provides	provide	VERB
ajema-7508	53	10	an	an	DET
ajema-7508	53	11	improvement	improvement	NOUN
ajema-7508	53	12	of	of	ADP
ajema-7508	53	13	0.8–1.2	0.8–1.2	NOUN
ajema-7508	53	14	%	%	NOUN
ajema-7508	53	15	over	over	ADP
ajema-7508	53	16	traditional	traditional	ADJ
ajema-7508	53	17	bert	bert	NOUN
ajema-7508	53	18	.	.	PUNCT
ajema-7508	54	1	topic	topic	NOUN
ajema-7508	54	2	bert	bert	NOUN
ajema-7508	54	3	-	-	PUNCT
ajema-7508	54	4	bigru	bigru	NOUN
ajema-7508	54	5	:	:	PUNCT
ajema-7508	54	6	short	short	ADJ
ajema-7508	54	7	texts	text	NOUN
ajema-7508	54	8	,	,	PUNCT
ajema-7508	54	9	particularly	particularly	ADV
ajema-7508	54	10	those	those	PRON
ajema-7508	54	11	in	in	ADP
ajema-7508	54	12	titles	title	NOUN
ajema-7508	54	13	,	,	PUNCT
ajema-7508	54	14	comments	comment	NOUN
ajema-7508	54	15	,	,	PUNCT
ajema-7508	54	16	and	and	CCONJ
ajema-7508	54	17	tweets	tweet	NOUN
ajema-7508	54	18	,	,	PUNCT
ajema-7508	54	19	reveal	reveal	VERB
ajema-7508	54	20	the	the	DET
ajema-7508	54	21	limitations	limitation	NOUN
ajema-7508	54	22	of	of	ADP
ajema-7508	54	23	the	the	DET
ajema-7508	54	24	bert	bert	PROPN
ajema-7508	54	25	model	model	NOUN
ajema-7508	54	26	.	.	PUNCT
ajema-7508	55	1	this	this	PRON
ajema-7508	55	2	is	be	AUX
ajema-7508	55	3	because	because	SCONJ
ajema-7508	55	4	the	the	DET
ajema-7508	55	5	masked	masked	ADJ
ajema-7508	55	6	language	language	NOUN
ajema-7508	55	7	modeling	modeling	NOUN
ajema-7508	55	8	(	(	PUNCT
ajema-7508	55	9	mlm	mlm	ADJ
ajema-7508	55	10	)	)	PUNCT
ajema-7508	55	11	strategy	strategy	NOUN
ajema-7508	55	12	can	can	AUX
ajema-7508	55	13	lead	lead	VERB
ajema-7508	55	14	to	to	ADP
ajema-7508	55	15	the	the	DET
ajema-7508	55	16	loss	loss	NOUN
ajema-7508	55	17	of	of	ADP
ajema-7508	55	18	semantic	semantic	ADJ
ajema-7508	55	19	richness	richness	NOUN
ajema-7508	55	20	when	when	SCONJ
ajema-7508	55	21	masking	masking	NOUN
ajema-7508	55	22	keywords	keyword	NOUN
ajema-7508	55	23	in	in	ADP
ajema-7508	55	24	short	short	ADJ
ajema-7508	55	25	texts	text	NOUN
ajema-7508	55	26	.	.	PUNCT
ajema-7508	56	1	to	to	PART
ajema-7508	56	2	address	address	VERB
ajema-7508	56	3	this	this	PRON
ajema-7508	56	4	,	,	PUNCT
ajema-7508	56	5	the	the	DET
ajema-7508	56	6	topic	topic	NOUN
ajema-7508	56	7	bert	bert	NOUN
ajema-7508	56	8	-	-	PUNCT
ajema-7508	56	9	bigru	bigru	NOUN
ajema-7508	56	10	model	model	NOUN
ajema-7508	56	11	was	be	AUX
ajema-7508	56	12	developed	develop	VERB
ajema-7508	56	13	,	,	PUNCT
ajema-7508	56	14	which	which	PRON
ajema-7508	56	15	enhances	enhance	VERB
ajema-7508	56	16	bert	bert	NOUN
ajema-7508	56	17	embeddings	embedding	NOUN
ajema-7508	56	18	with	with	ADP
ajema-7508	56	19	biterm	biterm	NOUN
ajema-7508	56	20	topic	topic	NOUN
ajema-7508	56	21	model	model	NOUN
ajema-7508	56	22	(	(	PUNCT
ajema-7508	56	23	btm	btm	PROPN
ajema-7508	56	24	)	)	PUNCT
ajema-7508	56	25	topic	topic	NOUN
ajema-7508	56	26	vectors	vector	NOUN
ajema-7508	56	27	.	.	PUNCT
ajema-7508	57	1	[	[	X
ajema-7508	57	2	7	7	NUM
ajema-7508	57	3	]	]	PUNCT
ajema-7508	57	4	bigru	bigru	NOUN
ajema-7508	57	5	captures	capture	VERB
ajema-7508	57	6	semantic	semantic	ADJ
ajema-7508	57	7	dependencies	dependency	NOUN
ajema-7508	57	8	in	in	ADP
ajema-7508	57	9	both	both	DET
ajema-7508	57	10	directions	direction	NOUN
ajema-7508	57	11	of	of	ADP
ajema-7508	57	12	a	a	DET
ajema-7508	57	13	sequence	sequence	NOUN
ajema-7508	57	14	and	and	CCONJ
ajema-7508	57	15	extracts	extract	NOUN
ajema-7508	57	16	deep	deep	ADJ
ajema-7508	57	17	features	feature	NOUN
ajema-7508	57	18	.	.	PUNCT
ajema-7508	58	1	the	the	DET
ajema-7508	58	2	model	model	NOUN
ajema-7508	58	3	's	's	PART
ajema-7508	58	4	performance	performance	NOUN
ajema-7508	58	5	is	be	AUX
ajema-7508	58	6	demonstrated	demonstrate	VERB
ajema-7508	58	7	by	by	ADP
ajema-7508	58	8	an	an	DET
ajema-7508	58	9	f1	f1	ADJ
ajema-7508	58	10	score	score	NOUN
ajema-7508	58	11	of	of	ADP
ajema-7508	58	12	86.91	86.91	NUM
ajema-7508	58	13	%	%	NOUN
ajema-7508	58	14	on	on	ADP
ajema-7508	58	15	the	the	DET
ajema-7508	58	16	thucnews	thucnews	NOUN
ajema-7508	58	17	dataset	dataset	NOUN
ajema-7508	58	18	.	.	PUNCT
ajema-7508	59	1	iv	iv	X
ajema-7508	59	2	.	.	PUNCT
ajema-7508	59	3	model	model	NOUN
ajema-7508	59	4	results	result	NOUN
ajema-7508	59	5	and	and	CCONJ
ajema-7508	59	6	analytical	analytical	ADJ
ajema-7508	59	7	comparison	comparison	NOUN
ajema-7508	59	8	intellectual	intellectual	ADJ
ajema-7508	59	9	analysis	analysis	NOUN
ajema-7508	59	10	:	:	PUNCT
ajema-7508	59	11	the	the	DET
ajema-7508	59	12	classification	classification	NOUN
ajema-7508	59	13	system	system	NOUN
ajema-7508	59	14	based	base	VERB
ajema-7508	59	15	on	on	ADP
ajema-7508	59	16	a	a	DET
ajema-7508	59	17	transformer	transformer	ADJ
ajema-7508	59	18	model	model	NOUN
ajema-7508	59	19	classified	classify	VERB
ajema-7508	59	20	using	use	VERB
ajema-7508	59	21	bert	bert	PROPN
ajema-7508	59	22	has	have	AUX
ajema-7508	59	23	achieved	achieve	VERB
ajema-7508	59	24	high	high	ADJ
ajema-7508	59	25	accuracy	accuracy	NOUN
ajema-7508	59	26	on	on	ADP
ajema-7508	59	27	large	large	ADJ
ajema-7508	59	28	text	text	NOUN
ajema-7508	59	29	corpora	corpora	NOUN
ajema-7508	59	30	in	in	ADP
ajema-7508	59	31	real	real	ADJ
ajema-7508	59	32	-	-	PUNCT
ajema-7508	59	33	life	life	NOUN
ajema-7508	59	34	scenarios	scenario	NOUN
ajema-7508	59	35	,	,	PUNCT
ajema-7508	59	36	including	include	VERB
ajema-7508	59	37	the	the	DET
ajema-7508	59	38	reuter	reuter	NOUN
ajema-7508	59	39	and	and	CCONJ
ajema-7508	59	40	daily	daily	ADJ
ajema-7508	59	41	star	star	NOUN
ajema-7508	59	42	datasets	dataset	NOUN
ajema-7508	59	43	.	.	PUNCT
ajema-7508	60	1	specifically	specifically	ADV
ajema-7508	60	2	,	,	PUNCT
ajema-7508	60	3	when	when	SCONJ
ajema-7508	60	4	combined	combine	VERB
ajema-7508	60	5	with	with	ADP
ajema-7508	60	6	the	the	DET
ajema-7508	60	7	random	random	ADJ
ajema-7508	60	8	forest	forest	NOUN
ajema-7508	60	9	classifier	classifier	NOUN
ajema-7508	60	10	,	,	PUNCT
ajema-7508	60	11	the	the	DET
ajema-7508	60	12	bert	bert	PROPN
ajema-7508	60	13	model	model	NOUN
ajema-7508	60	14	's	's	PART
ajema-7508	60	15	embeddings	embedding	NOUN
ajema-7508	60	16	have	have	AUX
ajema-7508	60	17	shown	show	VERB
ajema-7508	60	18	accuracy	accuracy	NOUN
ajema-7508	60	19	exceeding	exceed	VERB
ajema-7508	60	20	90	90	NUM
ajema-7508	60	21	%	%	NOUN
ajema-7508	60	22	.	.	PUNCT
ajema-7508	61	1	[	[	X
ajema-7508	61	2	2	2	X
ajema-7508	61	3	]	]	PUNCT
ajema-7508	61	4	this	this	PRON
ajema-7508	61	5	demonstrates	demonstrate	VERB
ajema-7508	61	6	the	the	DET
ajema-7508	61	7	practical	practical	ADJ
ajema-7508	61	8	advantages	advantage	NOUN
ajema-7508	61	9	of	of	ADP
ajema-7508	61	10	the	the	DET
ajema-7508	61	11	bert	bert	PROPN
ajema-7508	61	12	model	model	NOUN
ajema-7508	61	13	based	base	VERB
ajema-7508	61	14	on	on	ADP
ajema-7508	61	15	contextual	contextual	ADJ
ajema-7508	61	16	semantics	semantic	NOUN
ajema-7508	61	17	.	.	PUNCT
ajema-7508	62	1	№	№	NOUN
ajema-7508	62	2	model	model	NOUN
ajema-7508	62	3	name	name	NOUN
ajema-7508	62	4	dataset	dataset	NOUN
ajema-7508	62	5	accuracy	accuracy	NOUN
ajema-7508	62	6	(	(	PUNCT
ajema-7508	62	7	%	%	NOUN
ajema-7508	62	8	)	)	PUNCT
ajema-7508	62	9	/f1	/f1	PUNCT
ajema-7508	62	10	-	-	PUNCT
ajema-7508	62	11	score	score	NOUN
ajema-7508	62	12	1	1	NUM
ajema-7508	62	13	bert	bert	NOUN
ajema-7508	62	14	thucnews	thucnews	NOUN
ajema-7508	62	15	83,29	83,29	NUM
ajema-7508	62	16	%	%	NOUN
ajema-7508	62	17	2	2	NUM
ajema-7508	62	18	bert+bigru	bert+bigru	ADJ
ajema-7508	62	19	thucnews	thucnews	NOUN
ajema-7508	62	20	83,95	83,95	NUM
ajema-7508	62	21	%	%	NOUN
ajema-7508	62	22	3	3	NUM
ajema-7508	62	23	topic	topic	NOUN
ajema-7508	62	24	bert	bert	NOUN
ajema-7508	62	25	-	-	PUNCT
ajema-7508	62	26	bigru	bigru	NOUN
ajema-7508	62	27	thucnews	thucnews	NOUN
ajema-7508	62	28	86,91	86,91	NUM
ajema-7508	62	29	%	%	NOUN
ajema-7508	62	30	4	4	NUM
ajema-7508	62	31	distilbert	distilbert	NOUN
ajema-7508	62	32	sentiment	sentiment	NOUN
ajema-7508	62	33	dataset	dataset	VERB
ajema-7508	62	34	95,20	95,20	NUM
ajema-7508	62	35	%	%	NOUN
ajema-7508	62	36	5	5	NUM
ajema-7508	62	37	bert	bert	NOUN
ajema-7508	62	38	sentiment	sentiment	NOUN
ajema-7508	62	39	dataset	dataset	VERB
ajema-7508	62	40	94,38	94,38	NUM
ajema-7508	62	41	%	%	NOUN
ajema-7508	62	42	6	6	NUM
ajema-7508	62	43	bert+random	bert+random	PROPN
ajema-7508	62	44	forest	forest	NOUN
ajema-7508	62	45	sentiment	sentiment	NOUN
ajema-7508	62	46	dataset	dataset	VERB
ajema-7508	62	47	90	90	NUM
ajema-7508	62	48	%	%	NOUN
ajema-7508	62	49	the	the	DET
ajema-7508	62	50	research	research	NOUN
ajema-7508	62	51	results	result	NOUN
ajema-7508	62	52	show	show	VERB
ajema-7508	62	53	that	that	SCONJ
ajema-7508	62	54	models	model	NOUN
ajema-7508	62	55	developed	develop	VERB
ajema-7508	62	56	based	base	VERB
ajema-7508	62	57	on	on	ADP
ajema-7508	62	58	bert	bert	PROPN
ajema-7508	62	59	demonstrate	demonstrate	VERB
ajema-7508	62	60	varying	vary	VERB
ajema-7508	62	61	levels	level	NOUN
ajema-7508	62	62	of	of	ADP
ajema-7508	62	63	effectiveness	effectiveness	NOUN
ajema-7508	62	64	across	across	ADP
ajema-7508	62	65	different	different	ADJ
ajema-7508	62	66	types	type	NOUN
ajema-7508	62	67	of	of	ADP
ajema-7508	62	68	text	text	NOUN
ajema-7508	62	69	.	.	PUNCT
ajema-7508	63	1	specifically	specifically	ADV
ajema-7508	63	2	,	,	PUNCT
ajema-7508	63	3	the	the	DET
ajema-7508	63	4	topic	topic	NOUN
ajema-7508	63	5	bert	bert	NOUN
ajema-7508	63	6	-	-	PUNCT
ajema-7508	63	7	bigru	bigru	NOUN
ajema-7508	63	8	model	model	NOUN
ajema-7508	63	9	shows	show	VERB
ajema-7508	63	10	significant	significant	ADJ
ajema-7508	63	11	superiority	superiority	NOUN
ajema-7508	63	12	in	in	ADP
ajema-7508	63	13	working	work	VERB
ajema-7508	63	14	with	with	ADP
ajema-7508	63	15	short	short	ADJ
ajema-7508	63	16	texts	text	NOUN
ajema-7508	63	17	,	,	PUNCT
ajema-7508	63	18	recording	record	VERB
ajema-7508	63	19	an	an	DET
ajema-7508	63	20	f1	f1	NOUN
ajema-7508	63	21	-	-	PUNCT
ajema-7508	63	22	score	score	NOUN
ajema-7508	63	23	of	of	ADP
ajema-7508	63	24	86.91	86.91	NUM
ajema-7508	63	25	%	%	NOUN
ajema-7508	63	26	on	on	ADP
ajema-7508	63	27	the	the	DET
ajema-7508	63	28	thucnews	thucnews	NOUN
ajema-7508	63	29	headline	headline	NOUN
ajema-7508	63	30	dataset	dataset	VERB
ajema-7508	63	31	.	.	PUNCT
ajema-7508	64	1	this	this	PRON
ajema-7508	64	2	is	be	AUX
ajema-7508	64	3	3.62	3.62	NUM
ajema-7508	64	4	%	%	NOUN
ajema-7508	64	5	higher	high	ADJ
ajema-7508	64	6	than	than	ADP
ajema-7508	64	7	the	the	DET
ajema-7508	64	8	standard	standard	ADJ
ajema-7508	64	9	bert	bert	PROPN
ajema-7508	64	10	model	model	PROPN
ajema-7508	64	11	,	,	PUNCT
ajema-7508	64	12	demonstrating	demonstrate	VERB
ajema-7508	64	13	the	the	DET
ajema-7508	64	14	effectiveness	effectiveness	NOUN
ajema-7508	64	15	of	of	ADP
ajema-7508	64	16	combining	combine	VERB
ajema-7508	64	17	topic	topic	NOUN
ajema-7508	64	18	vectors	vector	NOUN
ajema-7508	64	19	and	and	CCONJ
ajema-7508	64	20	bidirectional	bidirectional	ADJ
ajema-7508	64	21	sequences	sequence	NOUN
ajema-7508	64	22	in	in	ADP
ajema-7508	64	23	the	the	DET
ajema-7508	64	24	context	context	NOUN
ajema-7508	64	25	of	of	ADP
ajema-7508	64	26	short	short	ADJ
ajema-7508	64	27	texts	text	NOUN
ajema-7508	64	28	.	.	PUNCT
ajema-7508	65	1	the	the	DET
ajema-7508	65	2	distilbert	distilbert	ADJ
ajema-7508	65	3	model	model	NOUN
ajema-7508	65	4	,	,	PUNCT
ajema-7508	65	5	on	on	ADP
ajema-7508	65	6	the	the	DET
ajema-7508	65	7	other	other	ADJ
ajema-7508	65	8	hand	hand	NOUN
ajema-7508	65	9	,	,	PUNCT
ajema-7508	65	10	showed	show	VERB
ajema-7508	65	11	high	high	ADJ
ajema-7508	65	12	accuracy	accuracy	NOUN
ajema-7508	65	13	on	on	ADP
ajema-7508	65	14	small	small	ADJ
ajema-7508	65	15	-	-	PUNCT
ajema-7508	65	16	sized	sized	ADJ
ajema-7508	65	17	and	and	CCONJ
ajema-7508	65	18	sentimentbased	sentimentbase	VERB
ajema-7508	65	19	datasets	dataset	NOUN
ajema-7508	65	20	.	.	PUNCT
ajema-7508	66	1	by	by	ADP
ajema-7508	66	2	evaluating	evaluate	VERB
ajema-7508	66	3	the	the	DET
ajema-7508	66	4	reliability	reliability	NOUN
ajema-7508	66	5	of	of	ADP
ajema-7508	66	6	bert	bert	PROPN
ajema-7508	66	7	outputs	output	NOUN
ajema-7508	66	8	,	,	PUNCT
ajema-7508	66	9	the	the	DET
ajema-7508	66	10	model	model	NOUN
ajema-7508	66	11	refers	refer	VERB
ajema-7508	66	12	to	to	ADP
ajema-7508	66	13	an	an	DET
ajema-7508	66	14	adaptive	adaptive	ADJ
ajema-7508	66	15	lexicon	lexicon	NOUN
ajema-7508	66	16	when	when	SCONJ
ajema-7508	66	17	necessary	necessary	ADJ
ajema-7508	66	18	.	.	PUNCT
ajema-7508	67	1	this	this	DET
ajema-7508	67	2	approach	approach	NOUN
ajema-7508	67	3	provided	provide	VERB
ajema-7508	67	4	a	a	DET
ajema-7508	67	5	high	high	ADJ
ajema-7508	67	6	performance	performance	NOUN
ajema-7508	67	7	with	with	ADP
ajema-7508	67	8	an	an	DET
ajema-7508	67	9	f1	f1	NOUN
ajema-7508	67	10	-	-	PUNCT
ajema-7508	67	11	score	score	NOUN
ajema-7508	67	12	of	of	ADP
ajema-7508	67	13	95.20	95.20	NUM
ajema-7508	67	14	%	%	NOUN
ajema-7508	67	15	.	.	PUNCT
ajema-7508	68	1	additionally	additionally	ADV
ajema-7508	68	2	,	,	PUNCT
ajema-7508	68	3	the	the	DET
ajema-7508	68	4	integration	integration	NOUN
ajema-7508	68	5	of	of	ADP
ajema-7508	68	6	the	the	DET
ajema-7508	68	7	bert	bert	PROPN
ajema-7508	68	8	model	model	NOUN
ajema-7508	68	9	with	with	ADP
ajema-7508	68	10	classical	classical	ADJ
ajema-7508	68	11	classifiers	classifier	NOUN
ajema-7508	68	12	has	have	AUX
ajema-7508	68	13	demonstrated	demonstrate	VERB
ajema-7508	68	14	high	high	ADJ
ajema-7508	68	15	performance	performance	NOUN
ajema-7508	68	16	in	in	ADP
ajema-7508	68	17	classification	classification	NOUN
ajema-7508	68	18	tasks	task	NOUN
ajema-7508	68	19	across	across	ADP
ajema-7508	68	20	broader	broad	ADJ
ajema-7508	68	21	domains	domain	NOUN
ajema-7508	68	22	.	.	PUNCT
ajema-7508	69	1	in	in	ADP
ajema-7508	69	2	systems	system	NOUN
ajema-7508	69	3	such	such	ADJ
ajema-7508	69	4	as	as	ADP
ajema-7508	69	5	those	those	PRON
ajema-7508	69	6	working	work	VERB
ajema-7508	69	7	with	with	ADP
ajema-7508	69	8	reuter	reuter	PROPN
ajema-7508	69	9	and	and	CCONJ
ajema-7508	69	10	daily	daily	PROPN
ajema-7508	69	11	star	star	PROPN
ajema-7508	69	12	corpora	corpora	PROPN
ajema-7508	69	13	,	,	PUNCT
ajema-7508	69	14	accuracy	accuracy	NOUN
ajema-7508	69	15	was	be	AUX
ajema-7508	69	16	shown	show	VERB
ajema-7508	69	17	to	to	PART
ajema-7508	69	18	exceed	exceed	VERB
ajema-7508	69	19	90	90	NUM
ajema-7508	69	20	%	%	NOUN
ajema-7508	69	21	.	.	PUNCT
ajema-7508	70	1	156	156	NUM
ajema-7508	70	2	journal	journal	NOUN
ajema-7508	70	3	of	of	ADP
ajema-7508	70	4	engineering	engineering	NOUN
ajema-7508	70	5	,	,	PUNCT
ajema-7508	70	6	mechanics	mechanic	NOUN
ajema-7508	70	7	and	and	CCONJ
ajema-7508	70	8	architecture	architecture	NOUN
ajema-7508	70	9	www	www	NOUN
ajema-7508	70	10	.	.	PUNCT
ajema-7508	71	1	grnjournal.us	grnjournal.us	X
ajema-7508	71	2	these	these	DET
ajema-7508	71	3	results	result	NOUN
ajema-7508	71	4	indicate	indicate	VERB
ajema-7508	71	5	that	that	SCONJ
ajema-7508	71	6	the	the	DET
ajema-7508	71	7	bert	bert	PROPN
ajema-7508	71	8	model	model	PROPN
ajema-7508	71	9	,	,	PUNCT
ajema-7508	71	10	with	with	ADP
ajema-7508	71	11	its	its	PRON
ajema-7508	71	12	fundamental	fundamental	ADJ
ajema-7508	71	13	architecture	architecture	NOUN
ajema-7508	71	14	,	,	PUNCT
ajema-7508	71	15	can	can	AUX
ajema-7508	71	16	perform	perform	VERB
ajema-7508	71	17	highlevel	highlevel	ADJ
ajema-7508	71	18	semantic	semantic	ADJ
ajema-7508	71	19	analysis	analysis	NOUN
ajema-7508	71	20	.	.	PUNCT
ajema-7508	72	1	however	however	ADV
ajema-7508	72	2	,	,	PUNCT
ajema-7508	72	3	by	by	ADP
ajema-7508	72	4	enhancing	enhance	VERB
ajema-7508	72	5	it	it	PRON
ajema-7508	72	6	with	with	ADP
ajema-7508	72	7	specialized	specialized	ADJ
ajema-7508	72	8	architectures	architecture	NOUN
ajema-7508	72	9	such	such	ADJ
ajema-7508	72	10	as	as	ADP
ajema-7508	72	11	topic	topic	NOUN
ajema-7508	72	12	modeling	modeling	NOUN
ajema-7508	72	13	,	,	PUNCT
ajema-7508	72	14	sequence	sequence	NOUN
ajema-7508	72	15	models	model	NOUN
ajema-7508	72	16	,	,	PUNCT
ajema-7508	72	17	and	and	CCONJ
ajema-7508	72	18	lexicon	lexicon	NOUN
ajema-7508	72	19	-	-	PUNCT
ajema-7508	72	20	based	base	VERB
ajema-7508	72	21	approaches	approach	NOUN
ajema-7508	72	22	,	,	PUNCT
ajema-7508	72	23	even	even	ADV
ajema-7508	72	24	more	more	ADV
ajema-7508	72	25	flexible	flexible	ADJ
ajema-7508	72	26	,	,	PUNCT
ajema-7508	72	27	consistent	consistent	ADJ
ajema-7508	72	28	,	,	PUNCT
ajema-7508	72	29	and	and	CCONJ
ajema-7508	72	30	highly	highly	ADV
ajema-7508	72	31	accurate	accurate	ADJ
ajema-7508	72	32	solutions	solution	NOUN
ajema-7508	72	33	can	can	AUX
ajema-7508	72	34	be	be	AUX
ajema-7508	72	35	achieved	achieve	VERB
ajema-7508	72	36	.	.	PUNCT
ajema-7508	73	1	topic	topic	NOUN
ajema-7508	73	2	bert	bert	NOUN
ajema-7508	73	3	-	-	PUNCT
ajema-7508	73	4	bigru	bigru	NOUN
ajema-7508	73	5	is	be	AUX
ajema-7508	73	6	considered	consider	VERB
ajema-7508	73	7	the	the	DET
ajema-7508	73	8	most	most	ADV
ajema-7508	73	9	effective	effective	ADJ
ajema-7508	73	10	model	model	NOUN
ajema-7508	73	11	for	for	ADP
ajema-7508	73	12	short	short	ADJ
ajema-7508	73	13	texts	text	NOUN
ajema-7508	73	14	.	.	PUNCT
ajema-7508	74	1	distilbert	distilbert	PROPN
ajema-7508	74	2	,	,	PUNCT
ajema-7508	74	3	on	on	ADP
ajema-7508	74	4	the	the	DET
ajema-7508	74	5	other	other	ADJ
ajema-7508	74	6	hand	hand	NOUN
ajema-7508	74	7	,	,	PUNCT
ajema-7508	74	8	excels	excel	VERB
ajema-7508	74	9	in	in	ADP
ajema-7508	74	10	special	special	ADJ
ajema-7508	74	11	cases	case	NOUN
ajema-7508	74	12	due	due	ADP
ajema-7508	74	13	to	to	ADP
ajema-7508	74	14	its	its	PRON
ajema-7508	74	15	use	use	NOUN
ajema-7508	74	16	of	of	ADP
ajema-7508	74	17	probability	probability	NOUN
ajema-7508	74	18	and	and	CCONJ
ajema-7508	74	19	rulebased	rulebased	ADJ
ajema-7508	74	20	mechanisms	mechanism	NOUN
ajema-7508	74	21	.	.	PUNCT
ajema-7508	75	1	bert	bert	NOUN
ajema-7508	75	2	+	+	CCONJ
ajema-7508	75	3	random	random	ADJ
ajema-7508	75	4	forest	forest	NOUN
ajema-7508	75	5	stands	stand	VERB
ajema-7508	75	6	out	out	ADP
ajema-7508	75	7	as	as	ADP
ajema-7508	75	8	an	an	DET
ajema-7508	75	9	effective	effective	ADJ
ajema-7508	75	10	approach	approach	NOUN
ajema-7508	75	11	for	for	ADP
ajema-7508	75	12	integrating	integrate	VERB
ajema-7508	75	13	the	the	DET
ajema-7508	75	14	model	model	NOUN
ajema-7508	75	15	with	with	ADP
ajema-7508	75	16	classical	classical	ADJ
ajema-7508	75	17	machine	machine	NOUN
ajema-7508	75	18	learning	learning	NOUN
ajema-7508	75	19	methods	method	NOUN
ajema-7508	75	20	.	.	PUNCT
ajema-7508	76	1	conclusion	conclusion	NOUN
ajema-7508	76	2	this	this	DET
ajema-7508	76	3	research	research	NOUN
ajema-7508	76	4	deeply	deeply	ADV
ajema-7508	76	5	examined	examine	VERB
ajema-7508	76	6	the	the	DET
ajema-7508	76	7	process	process	NOUN
ajema-7508	76	8	of	of	ADP
ajema-7508	76	9	text	text	NOUN
ajema-7508	76	10	analysis	analysis	NOUN
ajema-7508	76	11	based	base	VERB
ajema-7508	76	12	on	on	ADP
ajema-7508	76	13	the	the	DET
ajema-7508	76	14	bert	bert	PROPN
ajema-7508	76	15	model	model	PROPN
ajema-7508	76	16	,	,	PUNCT
ajema-7508	76	17	a	a	DET
ajema-7508	76	18	widely	widely	ADV
ajema-7508	76	19	used	use	VERB
ajema-7508	76	20	modern	modern	ADJ
ajema-7508	76	21	approach	approach	NOUN
ajema-7508	76	22	in	in	ADP
ajema-7508	76	23	natural	natural	ADJ
ajema-7508	76	24	language	language	NOUN
ajema-7508	76	25	processing	processing	NOUN
ajema-7508	76	26	.	.	PUNCT
ajema-7508	77	1	the	the	DET
ajema-7508	77	2	primary	primary	ADJ
ajema-7508	77	3	advantage	advantage	NOUN
ajema-7508	77	4	of	of	ADP
ajema-7508	77	5	the	the	DET
ajema-7508	77	6	bert	bert	PROPN
ajema-7508	77	7	model	model	NOUN
ajema-7508	77	8	is	be	AUX
ajema-7508	77	9	its	its	PRON
ajema-7508	77	10	ability	ability	NOUN
ajema-7508	77	11	to	to	PART
ajema-7508	77	12	learn	learn	VERB
ajema-7508	77	13	context	context	NOUN
ajema-7508	77	14	in	in	ADP
ajema-7508	77	15	both	both	DET
ajema-7508	77	16	directions	direction	NOUN
ajema-7508	77	17	simultaneously	simultaneously	ADV
ajema-7508	77	18	,	,	PUNCT
ajema-7508	77	19	which	which	PRON
ajema-7508	77	20	allows	allow	VERB
ajema-7508	77	21	for	for	ADP
ajema-7508	77	22	more	more	ADV
ajema-7508	77	23	precise	precise	ADJ
ajema-7508	77	24	analysis	analysis	NOUN
ajema-7508	77	25	of	of	ADP
ajema-7508	77	26	relationships	relationship	NOUN
ajema-7508	77	27	between	between	ADP
ajema-7508	77	28	linguistic	linguistic	ADJ
ajema-7508	77	29	units	unit	NOUN
ajema-7508	77	30	.	.	PUNCT
ajema-7508	78	1	this	this	DET
ajema-7508	78	2	feature	feature	NOUN
ajema-7508	78	3	enables	enable	VERB
ajema-7508	78	4	the	the	DET
ajema-7508	78	5	model	model	NOUN
ajema-7508	78	6	to	to	PART
ajema-7508	78	7	work	work	VERB
ajema-7508	78	8	effectively	effectively	ADV
ajema-7508	78	9	with	with	ADP
ajema-7508	78	10	ambiguous	ambiguous	ADJ
ajema-7508	78	11	,	,	PUNCT
ajema-7508	78	12	complex	complex	ADJ
ajema-7508	78	13	,	,	PUNCT
ajema-7508	78	14	and	and	CCONJ
ajema-7508	78	15	polysemous	polysemous	ADJ
ajema-7508	78	16	texts	text	NOUN
ajema-7508	78	17	.	.	PUNCT
ajema-7508	79	1	throughout	throughout	ADP
ajema-7508	79	2	the	the	DET
ajema-7508	79	3	study	study	NOUN
ajema-7508	79	4	,	,	PUNCT
ajema-7508	79	5	the	the	DET
ajema-7508	79	6	core	core	NOUN
ajema-7508	79	7	architecture	architecture	NOUN
ajema-7508	79	8	of	of	ADP
ajema-7508	79	9	the	the	DET
ajema-7508	79	10	bert	bert	PROPN
ajema-7508	79	11	model	model	NOUN
ajema-7508	79	12	,	,	PUNCT
ajema-7508	79	13	its	its	PRON
ajema-7508	79	14	training	training	NOUN
ajema-7508	79	15	phases	phase	NOUN
ajema-7508	79	16	,	,	PUNCT
ajema-7508	79	17	the	the	DET
ajema-7508	79	18	role	role	NOUN
ajema-7508	79	19	of	of	ADP
ajema-7508	79	20	the	the	DET
ajema-7508	79	21	[	[	X
ajema-7508	79	22	cls	cls	X
ajema-7508	79	23	]	]	X
ajema-7508	79	24	token	token	VERB
ajema-7508	79	25	in	in	ADP
ajema-7508	79	26	classification	classification	NOUN
ajema-7508	79	27	,	,	PUNCT
ajema-7508	79	28	and	and	CCONJ
ajema-7508	79	29	the	the	DET
ajema-7508	79	30	working	work	VERB
ajema-7508	79	31	mechanism	mechanism	NOUN
ajema-7508	79	32	of	of	ADP
ajema-7508	79	33	the	the	DET
ajema-7508	79	34	softmax	softmax	NOUN
ajema-7508	79	35	layer	layer	NOUN
ajema-7508	79	36	were	be	AUX
ajema-7508	79	37	analyzed	analyze	VERB
ajema-7508	79	38	.	.	PUNCT
ajema-7508	80	1	additionally	additionally	ADV
ajema-7508	80	2	,	,	PUNCT
ajema-7508	80	3	the	the	DET
ajema-7508	80	4	working	work	VERB
ajema-7508	80	5	mechanisms	mechanism	NOUN
ajema-7508	80	6	and	and	CCONJ
ajema-7508	80	7	advantages	advantage	NOUN
ajema-7508	80	8	of	of	ADP
ajema-7508	80	9	improved	improved	ADJ
ajema-7508	80	10	models	model	NOUN
ajema-7508	80	11	based	base	VERB
ajema-7508	80	12	on	on	ADP
ajema-7508	80	13	bert	bert	PROPN
ajema-7508	80	14	,	,	PUNCT
ajema-7508	80	15	including	include	VERB
ajema-7508	80	16	distilbert	distilbert	NOUN
ajema-7508	80	17	,	,	PUNCT
ajema-7508	80	18	topic	topic	NOUN
ajema-7508	80	19	bert	bert	NOUN
ajema-7508	80	20	-	-	PUNCT
ajema-7508	80	21	bigru	bigru	NOUN
ajema-7508	80	22	,	,	PUNCT
ajema-7508	80	23	and	and	CCONJ
ajema-7508	80	24	biterm	biterm	NOUN
ajema-7508	80	25	topic	topic	NOUN
ajema-7508	80	26	model	model	NOUN
ajema-7508	80	27	,	,	PUNCT
ajema-7508	80	28	were	be	AUX
ajema-7508	80	29	thoroughly	thoroughly	ADV
ajema-7508	80	30	reviewed	review	VERB
ajema-7508	80	31	to	to	PART
ajema-7508	80	32	enhance	enhance	VERB
ajema-7508	80	33	practical	practical	ADJ
ajema-7508	80	34	performance	performance	NOUN
ajema-7508	80	35	.	.	PUNCT
ajema-7508	81	1	the	the	DET
ajema-7508	81	2	lightweight	lightweight	ADJ
ajema-7508	81	3	and	and	CCONJ
ajema-7508	81	4	fast	fast	ADJ
ajema-7508	81	5	operation	operation	NOUN
ajema-7508	81	6	of	of	ADP
ajema-7508	81	7	distilbert	distilbert	PROPN
ajema-7508	81	8	was	be	AUX
ajema-7508	81	9	proven	prove	VERB
ajema-7508	81	10	effective	effective	ADJ
ajema-7508	81	11	,	,	PUNCT
ajema-7508	81	12	especially	especially	ADV
ajema-7508	81	13	in	in	ADP
ajema-7508	81	14	resourcelimited	resourcelimited	ADJ
ajema-7508	81	15	environments	environment	NOUN
ajema-7508	81	16	.	.	PUNCT
ajema-7508	82	1	despite	despite	SCONJ
ajema-7508	82	2	having	have	VERB
ajema-7508	82	3	significantly	significantly	ADV
ajema-7508	82	4	fewer	few	ADJ
ajema-7508	82	5	parameters	parameter	NOUN
ajema-7508	82	6	compared	compare	VERB
ajema-7508	82	7	to	to	ADP
ajema-7508	82	8	the	the	DET
ajema-7508	82	9	original	original	ADJ
ajema-7508	82	10	bert	bert	PROPN
ajema-7508	82	11	model	model	NOUN
ajema-7508	82	12	,	,	PUNCT
ajema-7508	82	13	it	it	PRON
ajema-7508	82	14	still	still	ADV
ajema-7508	82	15	produces	produce	VERB
ajema-7508	82	16	results	result	NOUN
ajema-7508	82	17	with	with	ADP
ajema-7508	82	18	high	high	ADJ
ajema-7508	82	19	accuracy	accuracy	NOUN
ajema-7508	82	20	.	.	PUNCT
ajema-7508	83	1	the	the	DET
ajema-7508	83	2	topic	topic	NOUN
ajema-7508	83	3	bert	bert	NOUN
ajema-7508	83	4	-	-	PUNCT
ajema-7508	83	5	bigru	bigru	NOUN
ajema-7508	83	6	model	model	NOUN
ajema-7508	83	7	demonstrated	demonstrate	VERB
ajema-7508	83	8	significantly	significantly	ADV
ajema-7508	83	9	better	well	ADJ
ajema-7508	83	10	results	result	NOUN
ajema-7508	83	11	by	by	ADP
ajema-7508	83	12	enhancing	enhance	VERB
ajema-7508	83	13	topic	topic	NOUN
ajema-7508	83	14	and	and	CCONJ
ajema-7508	83	15	semantic	semantic	ADJ
ajema-7508	83	16	analysis	analysis	NOUN
ajema-7508	83	17	for	for	ADP
ajema-7508	83	18	short	short	ADJ
ajema-7508	83	19	texts	text	NOUN
ajema-7508	83	20	.	.	PUNCT
ajema-7508	84	1	the	the	DET
ajema-7508	84	2	biterm	biterm	NOUN
ajema-7508	84	3	topic	topic	NOUN
ajema-7508	84	4	model	model	NOUN
ajema-7508	84	5	stands	stand	VERB
ajema-7508	84	6	out	out	ADP
ajema-7508	84	7	for	for	ADP
ajema-7508	84	8	its	its	PRON
ajema-7508	84	9	stability	stability	NOUN
ajema-7508	84	10	when	when	SCONJ
ajema-7508	84	11	working	work	VERB
ajema-7508	84	12	with	with	ADP
ajema-7508	84	13	short	short	ADJ
ajema-7508	84	14	and	and	CCONJ
ajema-7508	84	15	abstract	abstract	ADJ
ajema-7508	84	16	texts	text	NOUN
ajema-7508	84	17	.	.	PUNCT
ajema-7508	85	1	in	in	ADP
ajema-7508	85	2	particular	particular	ADJ
ajema-7508	85	3	,	,	PUNCT
ajema-7508	85	4	this	this	DET
ajema-7508	85	5	model	model	NOUN
ajema-7508	85	6	proved	prove	VERB
ajema-7508	85	7	effective	effective	ADJ
ajema-7508	85	8	when	when	SCONJ
ajema-7508	85	9	integrated	integrate	VERB
ajema-7508	85	10	with	with	ADP
ajema-7508	85	11	bert	bert	NOUN
ajema-7508	85	12	,	,	PUNCT
ajema-7508	85	13	especially	especially	ADV
ajema-7508	85	14	for	for	ADP
ajema-7508	85	15	topic	topic	NOUN
ajema-7508	85	16	identification	identification	NOUN
ajema-7508	85	17	and	and	CCONJ
ajema-7508	85	18	sentiment	sentiment	NOUN
ajema-7508	85	19	analysis	analysis	NOUN
ajema-7508	85	20	.	.	PUNCT
ajema-7508	86	1	recommendations	recommendation	NOUN
ajema-7508	86	2	:	:	PUNCT
ajema-7508	86	3	1	1	X
ajema-7508	86	4	.	.	NUM
ajema-7508	86	5	models	model	NOUN
ajema-7508	86	6	based	base	VERB
ajema-7508	86	7	on	on	ADP
ajema-7508	86	8	the	the	DET
ajema-7508	86	9	bert	bert	PROPN
ajema-7508	86	10	model	model	NOUN
ajema-7508	86	11	should	should	AUX
ajema-7508	86	12	be	be	AUX
ajema-7508	86	13	applied	apply	VERB
ajema-7508	86	14	in	in	ADP
ajema-7508	86	15	systems	system	NOUN
ajema-7508	86	16	such	such	ADJ
ajema-7508	86	17	as	as	ADP
ajema-7508	86	18	sentiment	sentiment	NOUN
ajema-7508	86	19	analysis	analysis	NOUN
ajema-7508	86	20	,	,	PUNCT
ajema-7508	86	21	recommendation	recommendation	NOUN
ajema-7508	86	22	systems	system	NOUN
ajema-7508	86	23	,	,	PUNCT
ajema-7508	86	24	and	and	CCONJ
ajema-7508	86	25	automatic	automatic	ADJ
ajema-7508	86	26	label	label	NOUN
ajema-7508	86	27	classification	classification	NOUN
ajema-7508	86	28	to	to	PART
ajema-7508	86	29	enhance	enhance	VERB
ajema-7508	86	30	efficiency	efficiency	NOUN
ajema-7508	86	31	.	.	PUNCT
ajema-7508	87	1	2	2	X
ajema-7508	87	2	.	.	X
ajema-7508	87	3	if	if	SCONJ
ajema-7508	87	4	computer	computer	NOUN
ajema-7508	87	5	resources	resource	NOUN
ajema-7508	87	6	are	be	AUX
ajema-7508	87	7	limited	limited	ADJ
ajema-7508	87	8	,	,	PUNCT
ajema-7508	87	9	it	it	PRON
ajema-7508	87	10	is	be	AUX
ajema-7508	87	11	recommended	recommend	VERB
ajema-7508	87	12	to	to	PART
ajema-7508	87	13	use	use	VERB
ajema-7508	87	14	distilbert	distilbert	NOUN
ajema-7508	87	15	or	or	CCONJ
ajema-7508	87	16	other	other	ADJ
ajema-7508	87	17	lightweight	lightweight	ADJ
ajema-7508	87	18	bert	bert	NOUN
ajema-7508	87	19	variants	variant	NOUN
ajema-7508	87	20	.	.	PUNCT
ajema-7508	88	1	3	3	X
ajema-7508	88	2	.	.	X
ajema-7508	88	3	hybrid	hybrid	ADJ
ajema-7508	88	4	approaches	approach	NOUN
ajema-7508	88	5	can	can	AUX
ajema-7508	88	6	be	be	AUX
ajema-7508	88	7	developed	develop	VERB
ajema-7508	88	8	by	by	ADP
ajema-7508	88	9	combining	combine	VERB
ajema-7508	88	10	the	the	DET
ajema-7508	88	11	bert	bert	PROPN
ajema-7508	88	12	model	model	NOUN
ajema-7508	88	13	with	with	ADP
ajema-7508	88	14	bigru	bigru	NOUN
ajema-7508	88	15	or	or	CCONJ
ajema-7508	88	16	cnn	cnn	PROPN
ajema-7508	88	17	models	model	NOUN
ajema-7508	88	18	for	for	ADP
ajema-7508	88	19	topic	topic	NOUN
ajema-7508	88	20	-	-	PUNCT
ajema-7508	88	21	based	base	VERB
ajema-7508	88	22	analysis	analysis	NOUN
ajema-7508	88	23	or	or	CCONJ
ajema-7508	88	24	multi	multi	ADJ
ajema-7508	88	25	-	-	ADJ
ajema-7508	88	26	class	class	ADJ
ajema-7508	88	27	classification	classification	NOUN
ajema-7508	88	28	.	.	PUNCT
ajema-7508	89	1	4	4	X
ajema-7508	89	2	.	.	X
ajema-7508	89	3	if	if	SCONJ
ajema-7508	89	4	there	there	PRON
ajema-7508	89	5	are	be	VERB
ajema-7508	89	6	small	small	ADJ
ajema-7508	89	7	-	-	PUNCT
ajema-7508	89	8	sized	sized	ADJ
ajema-7508	89	9	training	training	NOUN
ajema-7508	89	10	datasets	dataset	NOUN
ajema-7508	89	11	,	,	PUNCT
ajema-7508	89	12	it	it	PRON
ajema-7508	89	13	is	be	AUX
ajema-7508	89	14	advisable	advisable	ADJ
ajema-7508	89	15	to	to	PART
ajema-7508	89	16	apply	apply	VERB
ajema-7508	89	17	a	a	DET
ajema-7508	89	18	transfer	transfer	NOUN
ajema-7508	89	19	learning	learning	NOUN
ajema-7508	89	20	strategy	strategy	NOUN
ajema-7508	89	21	.	.	PUNCT
ajema-7508	90	1	future	future	ADJ
ajema-7508	90	2	research	research	NOUN
ajema-7508	90	3	directions	direction	NOUN
ajema-7508	90	4	:	:	PUNCT
ajema-7508	90	5			VERB
ajema-7508	90	6	expanding	expand	VERB
ajema-7508	90	7	analysis	analysis	NOUN
ajema-7508	90	8	of	of	ADP
ajema-7508	90	9	uzbek	uzbek	ADJ
ajema-7508	90	10	texts	text	NOUN
ajema-7508	90	11	using	use	VERB
ajema-7508	90	12	multilingual	multilingual	ADJ
ajema-7508	90	13	bert	bert	NOUN
ajema-7508	90	14	.	.	PUNCT
ajema-7508	91	1			VERB
ajema-7508	91	2	improving	improve	VERB
ajema-7508	91	3	efficiency	efficiency	NOUN
ajema-7508	91	4	by	by	ADP
ajema-7508	91	5	fine	fine	ADV
ajema-7508	91	6	-	-	PUNCT
ajema-7508	91	7	tuning	tune	VERB
ajema-7508	91	8	and	and	CCONJ
ajema-7508	91	9	adapting	adapt	VERB
ajema-7508	91	10	bert	bert	NOUN
ajema-7508	91	11	models	model	NOUN
ajema-7508	91	12	for	for	ADP
ajema-7508	91	13	specific	specific	ADJ
ajema-7508	91	14	industry	industry	NOUN
ajema-7508	91	15	domains	domain	NOUN
ajema-7508	91	16	.	.	PUNCT
ajema-7508	92	1			VERB
ajema-7508	92	2	developing	develop	VERB
ajema-7508	92	3	understandable	understandable	ADJ
ajema-7508	92	4	interpretation	interpretation	NOUN
ajema-7508	92	5	methods	method	NOUN
ajema-7508	92	6	for	for	ADP
ajema-7508	92	7	visual	visual	ADJ
ajema-7508	92	8	analysis	analysis	NOUN
ajema-7508	92	9	of	of	ADP
ajema-7508	92	10	model	model	NOUN
ajema-7508	92	11	results	result	NOUN
ajema-7508	92	12	.	.	PUNCT
ajema-7508	93	1	references	reference	NOUN
ajema-7508	93	2	1	1	NUM
ajema-7508	93	3	.	.	PUNCT
ajema-7508	93	4	eshankulov	eshankulov	NOUN
ajema-7508	93	5	,	,	PUNCT
ajema-7508	93	6	h.i	h.i	PROPN
ajema-7508	93	7	.	.	PROPN
ajema-7508	93	8	,	,	PUNCT
ajema-7508	93	9	&	&	CCONJ
ajema-7508	93	10	soliyeva	soliyeva	PROPN
ajema-7508	93	11	,	,	PUNCT
ajema-7508	93	12	d.a	d.a	PROPN
ajema-7508	93	13	.	.	PROPN
ajema-7508	93	14	(	(	PUNCT
ajema-7508	93	15	2025	2025	NUM
ajema-7508	93	16	)	)	PUNCT
ajema-7508	93	17	.	.	PUNCT
ajema-7508	94	1	natural	natural	ADJ
ajema-7508	94	2	language	language	NOUN
ajema-7508	94	3	processing	processing	NOUN
ajema-7508	94	4	models	model	NOUN
ajema-7508	94	5	and	and	CCONJ
ajema-7508	94	6	their	their	PRON
ajema-7508	94	7	applications	application	NOUN
ajema-7508	94	8	.	.	PUNCT
ajema-7508	95	1	scientific	scientific	ADJ
ajema-7508	95	2	bulletin	bulletin	NOUN
ajema-7508	95	3	of	of	ADP
ajema-7508	95	4	bukhara	bukhara	PROPN
ajema-7508	95	5	state	state	PROPN
ajema-7508	95	6	university	university	PROPN
ajema-7508	95	7	,	,	PUNCT
ajema-7508	95	8	2(119	2(119	NUM
ajema-7508	95	9	)	)	PUNCT
ajema-7508	95	10	,	,	PUNCT
ajema-7508	95	11	140–146	140–146	NUM
ajema-7508	95	12	.	.	PUNCT
ajema-7508	96	1	157	157	NUM
ajema-7508	96	2	journal	journal	NOUN
ajema-7508	96	3	of	of	ADP
ajema-7508	96	4	engineering	engineering	NOUN
ajema-7508	96	5	,	,	PUNCT
ajema-7508	96	6	mechanics	mechanic	NOUN
ajema-7508	96	7	and	and	CCONJ
ajema-7508	96	8	architecture	architecture	NOUN
ajema-7508	96	9	www	www	NOUN
ajema-7508	96	10	.	.	PUNCT
ajema-7508	97	1	grnjournal.us	grnjournal.us	X
ajema-7508	97	2	2	2	X
ajema-7508	97	3	.	.	X
ajema-7508	97	4	eshankulov	eshankulov	NOUN
ajema-7508	97	5	,	,	PUNCT
ajema-7508	97	6	k.	k.	PROPN
ajema-7508	97	7	,	,	PUNCT
ajema-7508	97	8	sayidova	sayidova	NOUN
ajema-7508	97	9	,	,	PUNCT
ajema-7508	97	10	n.	n.	NOUN
ajema-7508	97	11	,	,	PUNCT
ajema-7508	97	12	zaripova	zaripova	PROPN
ajema-7508	97	13	,	,	PUNCT
ajema-7508	97	14	g.	g.	PROPN
ajema-7508	97	15	,	,	PUNCT
ajema-7508	97	16	imomova	imomova	PROPN
ajema-7508	97	17	,	,	PUNCT
ajema-7508	97	18	s.	s.	PROPN
ajema-7508	97	19	,	,	PUNCT
ajema-7508	97	20	fayzieva	fayzieva	PROPN
ajema-7508	97	21	,	,	PUNCT
ajema-7508	97	22	d.	d.	PROPN
ajema-7508	97	23	,	,	PUNCT
ajema-7508	97	24	"	"	PUNCT
ajema-7508	97	25	mathematical	mathematical	ADJ
ajema-7508	97	26	model	model	NOUN
ajema-7508	97	27	for	for	ADP
ajema-7508	97	28	information	information	NOUN
ajema-7508	97	29	monitoring	monitoring	NOUN
ajema-7508	97	30	system	system	NOUN
ajema-7508	97	31	of	of	ADP
ajema-7508	97	32	fat	fat	ADJ
ajema-7508	97	33	and	and	CCONJ
ajema-7508	97	34	oil	oil	NOUN
ajema-7508	97	35	enterprises	enterprise	NOUN
ajema-7508	97	36	,	,	PUNCT
ajema-7508	97	37	"	"	PUNCT
ajema-7508	97	38	aip	aip	PROPN
ajema-7508	97	39	conference	conference	NOUN
ajema-7508	97	40	proceedings	proceeding	NOUN
ajema-7508	97	41	,	,	PUNCT
ajema-7508	97	42	3004(1	3004(1	NUM
ajema-7508	97	43	)	)	PUNCT
ajema-7508	97	44	,	,	PUNCT
ajema-7508	97	45	060009	060009	NUM
ajema-7508	97	46	,	,	PUNCT
ajema-7508	97	47	2024	2024	NUM
ajema-7508	97	48	.	.	PUNCT
ajema-7508	98	1	3	3	X
ajema-7508	98	2	.	.	X
ajema-7508	98	3	eshankulov	eshankulov	NOUN
ajema-7508	98	4	,	,	PUNCT
ajema-7508	98	5	k.	k.	PROPN
ajema-7508	98	6	,	,	PUNCT
ajema-7508	98	7	murodova	murodova	PROPN
ajema-7508	98	8	,	,	PUNCT
ajema-7508	98	9	r.	r.	PROPN
ajema-7508	98	10	,	,	PUNCT
ajema-7508	98	11	"	"	PUNCT
ajema-7508	98	12	development	development	NOUN
ajema-7508	98	13	of	of	ADP
ajema-7508	98	14	the	the	DET
ajema-7508	98	15	knowledge	knowledge	NOUN
ajema-7508	98	16	base	base	NOUN
ajema-7508	98	17	of	of	ADP
ajema-7508	98	18	the	the	DET
ajema-7508	98	19	decisionmaking	decisionmake	VERB
ajema-7508	98	20	software	software	NOUN
ajema-7508	98	21	module	module	NOUN
ajema-7508	98	22	based	base	VERB
ajema-7508	98	23	on	on	ADP
ajema-7508	98	24	the	the	DET
ajema-7508	98	25	frame	frame	NOUN
ajema-7508	98	26	model	model	NOUN
ajema-7508	98	27	,	,	PUNCT
ajema-7508	98	28	"	"	PUNCT
ajema-7508	98	29	proceedings	proceeding	NOUN
ajema-7508	98	30	of	of	ADP
ajema-7508	98	31	spie	spie	VERB
ajema-7508	98	32	the	the	DET
ajema-7508	98	33	international	international	ADJ
ajema-7508	98	34	society	society	NOUN
ajema-7508	98	35	for	for	ADP
ajema-7508	98	36	optical	optical	ADJ
ajema-7508	98	37	engineering	engineering	NOUN
ajema-7508	98	38	,	,	PUNCT
ajema-7508	98	39	2024	2024	NUM
ajema-7508	98	40	.	.	PUNCT
ajema-7508	99	1	4	4	NUM
ajema-7508	99	2	.	.	X
ajema-7508	99	3	vaswani	vaswani	NOUN
ajema-7508	99	4	,	,	PUNCT
ajema-7508	99	5	a.	a.	NOUN
ajema-7508	99	6	,	,	PUNCT
ajema-7508	99	7	shazeer	shazeer	NOUN
ajema-7508	99	8	,	,	PUNCT
ajema-7508	99	9	n.	n.	NOUN
ajema-7508	99	10	,	,	PUNCT
ajema-7508	99	11	parmar	parmar	PROPN
ajema-7508	99	12	,	,	PUNCT
ajema-7508	99	13	n.	n.	NOUN
ajema-7508	99	14	,	,	PUNCT
ajema-7508	99	15	uszkoreit	uszkoreit	PROPN
ajema-7508	99	16	,	,	PUNCT
ajema-7508	99	17	j.	j.	PROPN
ajema-7508	99	18	,	,	PUNCT
ajema-7508	99	19	jones	jones	PROPN
ajema-7508	99	20	,	,	PUNCT
ajema-7508	99	21	l.	l.	PROPN
ajema-7508	99	22	,	,	PUNCT
ajema-7508	99	23	gomez	gomez	PROPN
ajema-7508	99	24	,	,	PUNCT
ajema-7508	99	25	a.	a.	NOUN
ajema-7508	99	26	n.	n.	PROPN
ajema-7508	99	27	,	,	PUNCT
ajema-7508	99	28	...	...	PUNCT
ajema-7508	99	29	&	&	CCONJ
ajema-7508	99	30	polosukhin	polosukhin	PROPN
ajema-7508	99	31	,	,	PUNCT
ajema-7508	99	32	i.	i.	NOUN
ajema-7508	99	33	(	(	PUNCT
ajema-7508	99	34	2017	2017	NUM
ajema-7508	99	35	)	)	PUNCT
ajema-7508	99	36	.	.	PUNCT
ajema-7508	100	1	attention	attention	NOUN
ajema-7508	100	2	is	be	AUX
ajema-7508	100	3	all	all	PRON
ajema-7508	100	4	you	you	PRON
ajema-7508	100	5	need	need	VERB
ajema-7508	100	6	.	.	PUNCT
ajema-7508	101	1	advances	advance	NOUN
ajema-7508	101	2	in	in	ADP
ajema-7508	101	3	neural	neural	ADJ
ajema-7508	101	4	information	information	NOUN
ajema-7508	101	5	processing	processing	NOUN
ajema-7508	101	6	systems	system	NOUN
ajema-7508	101	7	(	(	PUNCT
ajema-7508	101	8	neurips	neurip	NOUN
ajema-7508	101	9	)	)	PUNCT
ajema-7508	101	10	.	.	PUNCT
ajema-7508	102	1	https://arxiv.org/abs/1706.03762	https://arxiv.org/abs/1706.03762	VERB
ajema-7508	102	2	5	5	NUM
ajema-7508	102	3	.	.	PUNCT
ajema-7508	103	1	devlin	devlin	PROPN
ajema-7508	103	2	,	,	PUNCT
ajema-7508	103	3	j.	j.	PROPN
ajema-7508	103	4	,	,	PUNCT
ajema-7508	103	5	chang	chang	PROPN
ajema-7508	103	6	,	,	PUNCT
ajema-7508	103	7	m.	m.	PROPN
ajema-7508	103	8	w.	w.	PROPN
ajema-7508	103	9	,	,	PUNCT
ajema-7508	103	10	lee	lee	PROPN
ajema-7508	103	11	,	,	PUNCT
ajema-7508	103	12	k.	k.	PROPN
ajema-7508	103	13	,	,	PUNCT
ajema-7508	103	14	&	&	CCONJ
ajema-7508	103	15	toutanova	toutanova	PROPN
ajema-7508	103	16	,	,	PUNCT
ajema-7508	103	17	k.	k.	PROPN
ajema-7508	103	18	(	(	PUNCT
ajema-7508	103	19	2019	2019	NUM
ajema-7508	103	20	)	)	PUNCT
ajema-7508	103	21	.	.	PUNCT
ajema-7508	104	1	bert	bert	PROPN
ajema-7508	104	2	:	:	PUNCT
ajema-7508	104	3	pre	pre	ADJ
ajema-7508	104	4	-	-	NOUN
ajema-7508	104	5	training	training	NOUN
ajema-7508	104	6	of	of	ADP
ajema-7508	104	7	deep	deep	ADJ
ajema-7508	104	8	bidirectional	bidirectional	ADJ
ajema-7508	104	9	transformers	transformer	NOUN
ajema-7508	104	10	for	for	ADP
ajema-7508	104	11	language	language	NOUN
ajema-7508	104	12	understanding	understanding	NOUN
ajema-7508	104	13	.	.	PUNCT
ajema-7508	105	1	proceedings	proceeding	NOUN
ajema-7508	105	2	of	of	ADP
ajema-7508	105	3	the	the	DET
ajema-7508	105	4	north	north	ADJ
ajema-7508	105	5	american	american	ADJ
ajema-7508	105	6	chapter	chapter	NOUN
ajema-7508	105	7	of	of	ADP
ajema-7508	105	8	the	the	DET
ajema-7508	105	9	association	association	NOUN
ajema-7508	105	10	for	for	ADP
ajema-7508	105	11	computational	computational	ADJ
ajema-7508	105	12	linguistics	linguistic	NOUN
ajema-7508	105	13	(	(	PUNCT
ajema-7508	105	14	naacl	naacl	PROPN
ajema-7508	105	15	)	)	PUNCT
ajema-7508	105	16	.	.	PUNCT
ajema-7508	106	1	https://arxiv.org/abs/1810.04805	https://arxiv.org/abs/1810.04805	PROPN
ajema-7508	106	2	6	6	NUM
ajema-7508	106	3	.	.	PUNCT
ajema-7508	106	4	brown	brown	PROPN
ajema-7508	106	5	,	,	PUNCT
ajema-7508	106	6	t.	t.	PROPN
ajema-7508	106	7	,	,	PUNCT
ajema-7508	106	8	mann	mann	PROPN
ajema-7508	106	9	,	,	PUNCT
ajema-7508	106	10	b.	b.	PROPN
ajema-7508	106	11	,	,	PUNCT
ajema-7508	106	12	ryder	ryder	PROPN
ajema-7508	106	13	,	,	PUNCT
ajema-7508	106	14	n.	n.	NOUN
ajema-7508	106	15	,	,	PUNCT
ajema-7508	106	16	subbiah	subbiah	PROPN
ajema-7508	106	17	,	,	PUNCT
ajema-7508	106	18	m.	m.	NOUN
ajema-7508	106	19	,	,	PUNCT
ajema-7508	106	20	kaplan	kaplan	PROPN
ajema-7508	106	21	,	,	PUNCT
ajema-7508	106	22	j.	j.	PROPN
ajema-7508	106	23	,	,	PUNCT
ajema-7508	106	24	dhariwal	dhariwal	NOUN
ajema-7508	106	25	,	,	PUNCT
ajema-7508	106	26	p.	p.	PROPN
ajema-7508	106	27	,	,	PUNCT
ajema-7508	106	28	...	...	PUNCT
ajema-7508	106	29	&	&	CCONJ
ajema-7508	106	30	amodei	amodei	PROPN
ajema-7508	106	31	,	,	PUNCT
ajema-7508	106	32	d.	d.	PROPN
ajema-7508	106	33	(	(	PUNCT
ajema-7508	106	34	2020	2020	NUM
ajema-7508	106	35	)	)	PUNCT
ajema-7508	106	36	.	.	PUNCT
ajema-7508	107	1	language	language	NOUN
ajema-7508	107	2	models	model	NOUN
ajema-7508	107	3	are	be	AUX
ajema-7508	107	4	few	few	ADJ
ajema-7508	107	5	-	-	PUNCT
ajema-7508	107	6	shot	shot	NOUN
ajema-7508	107	7	learners	learner	NOUN
ajema-7508	107	8	.	.	PUNCT
ajema-7508	108	1	advances	advance	NOUN
ajema-7508	108	2	in	in	ADP
ajema-7508	108	3	neural	neural	ADJ
ajema-7508	108	4	information	information	NOUN
ajema-7508	108	5	processing	processing	NOUN
ajema-7508	108	6	systems	system	NOUN
ajema-7508	108	7	(	(	PUNCT
ajema-7508	108	8	neurips	neurip	NOUN
ajema-7508	108	9	)	)	PUNCT
ajema-7508	108	10	.	.	PUNCT
ajema-7508	109	1	https://arxiv.org/abs/2005.14165	https://arxiv.org/abs/2005.14165	PROPN
ajema-7508	109	2	7	7	NUM
ajema-7508	109	3	.	.	PUNCT
ajema-7508	110	1	radford	radford	PROPN
ajema-7508	110	2	,	,	PUNCT
ajema-7508	110	3	a.	a.	PROPN
ajema-7508	110	4	,	,	PUNCT
ajema-7508	110	5	wu	wu	PROPN
ajema-7508	110	6	,	,	PUNCT
ajema-7508	110	7	j.	j.	PROPN
ajema-7508	110	8	,	,	PUNCT
ajema-7508	110	9	child	child	NOUN
ajema-7508	110	10	,	,	PUNCT
ajema-7508	110	11	r.	r.	PROPN
ajema-7508	110	12	,	,	PUNCT
ajema-7508	110	13	luan	luan	PROPN
ajema-7508	110	14	,	,	PUNCT
ajema-7508	110	15	d.	d.	PROPN
ajema-7508	110	16	,	,	PUNCT
ajema-7508	110	17	amodei	amodei	PROPN
ajema-7508	110	18	,	,	PUNCT
ajema-7508	110	19	d.	d.	PROPN
ajema-7508	110	20	,	,	PUNCT
ajema-7508	110	21	&	&	CCONJ
ajema-7508	110	22	sutskever	sutskever	PROPN
ajema-7508	110	23	,	,	PUNCT
ajema-7508	110	24	i.	i.	PROPN
ajema-7508	110	25	(	(	PUNCT
ajema-7508	110	26	2019	2019	NUM
ajema-7508	110	27	)	)	PUNCT
ajema-7508	110	28	.	.	PUNCT
ajema-7508	111	1	language	language	NOUN
ajema-7508	111	2	models	model	NOUN
ajema-7508	111	3	are	be	AUX
ajema-7508	111	4	unsupervised	unsupervised	ADJ
ajema-7508	111	5	multitask	multitask	ADJ
ajema-7508	111	6	learners	learner	NOUN
ajema-7508	111	7	.	.	PUNCT
ajema-7508	112	1	openai	openai	PROPN
ajema-7508	112	2	.	.	PUNCT
ajema-7508	113	1	8	8	X
ajema-7508	113	2	.	.	X
ajema-7508	114	1	bender	bender	PROPN
ajema-7508	114	2	,	,	PUNCT
ajema-7508	114	3	e.	e.	PROPN
ajema-7508	114	4	m.	m.	PROPN
ajema-7508	114	5	,	,	PUNCT
ajema-7508	114	6	gebru	gebru	NOUN
ajema-7508	114	7	,	,	PUNCT
ajema-7508	114	8	t.	t.	PROPN
ajema-7508	114	9	,	,	PUNCT
ajema-7508	114	10	mcmillan	mcmillan	NOUN
ajema-7508	114	11	-	-	PUNCT
ajema-7508	114	12	major	major	ADJ
ajema-7508	114	13	,	,	PUNCT
ajema-7508	114	14	a.	a.	NOUN
ajema-7508	114	15	,	,	PUNCT
ajema-7508	114	16	&	&	CCONJ
ajema-7508	114	17	shmitchell	shmitchell	PROPN
ajema-7508	114	18	,	,	PUNCT
ajema-7508	114	19	s.	s.	PROPN
ajema-7508	114	20	(	(	PUNCT
ajema-7508	114	21	2021	2021	NUM
ajema-7508	114	22	)	)	PUNCT
ajema-7508	114	23	.	.	PUNCT
ajema-7508	115	1	on	on	ADP
ajema-7508	115	2	the	the	DET
ajema-7508	115	3	dangers	danger	NOUN
ajema-7508	115	4	of	of	ADP
ajema-7508	115	5	stochastic	stochastic	ADJ
ajema-7508	115	6	parrots	parrot	NOUN
ajema-7508	115	7	:	:	PUNCT
ajema-7508	115	8	can	can	AUX
ajema-7508	115	9	language	language	NOUN
ajema-7508	115	10	models	model	NOUN
ajema-7508	115	11	be	be	AUX
ajema-7508	115	12	too	too	ADV
ajema-7508	115	13	big	big	ADJ
ajema-7508	115	14	?	?	PUNCT
ajema-7508	116	1	proceedings	proceeding	NOUN
ajema-7508	116	2	of	of	ADP
ajema-7508	116	3	the	the	DET
ajema-7508	116	4	2021	2021	NUM
ajema-7508	116	5	acm	acm	NOUN
ajema-7508	116	6	conference	conference	NOUN
ajema-7508	116	7	on	on	ADP
ajema-7508	116	8	fairness	fairness	NOUN
ajema-7508	116	9	,	,	PUNCT
ajema-7508	116	10	accountability	accountability	NOUN
ajema-7508	116	11	,	,	PUNCT
ajema-7508	116	12	and	and	CCONJ
ajema-7508	116	13	transparency	transparency	NOUN
ajema-7508	116	14	(	(	PUNCT
ajema-7508	116	15	facct	facct	NOUN
ajema-7508	116	16	)	)	PUNCT
ajema-7508	116	17	.	.	PUNCT
ajema-7508	117	1	https://dl.acm.org/doi/10.1145/3442188.3445922	https://dl.acm.org/doi/10.1145/3442188.3445922	PROPN
ajema-7508	117	2	9	9	NUM
ajema-7508	117	3	.	.	PUNCT
ajema-7508	117	4	bolukbasi	bolukbasi	NOUN
ajema-7508	117	5	,	,	PUNCT
ajema-7508	117	6	t.	t.	PROPN
ajema-7508	117	7	,	,	PUNCT
ajema-7508	117	8	chang	chang	PROPN
ajema-7508	117	9	,	,	PUNCT
ajema-7508	117	10	k.	k.	PROPN
ajema-7508	117	11	w.	w.	PROPN
ajema-7508	117	12	,	,	PUNCT
ajema-7508	117	13	zou	zou	PROPN
ajema-7508	117	14	,	,	PUNCT
ajema-7508	117	15	j.	j.	PROPN
ajema-7508	117	16	y.	y.	PROPN
ajema-7508	117	17	,	,	PUNCT
ajema-7508	117	18	saligrama	saligrama	NOUN
ajema-7508	117	19	,	,	PUNCT
ajema-7508	117	20	v.	v.	PROPN
ajema-7508	117	21	,	,	PUNCT
ajema-7508	117	22	&	&	CCONJ
ajema-7508	117	23	kalai	kalai	PROPN
ajema-7508	117	24	,	,	PUNCT
ajema-7508	117	25	a.	a.	NOUN
ajema-7508	117	26	(	(	PUNCT
ajema-7508	117	27	2016	2016	NUM
ajema-7508	117	28	)	)	PUNCT
ajema-7508	117	29	.	.	PUNCT
ajema-7508	118	1	man	man	NOUN
ajema-7508	118	2	is	be	AUX
ajema-7508	118	3	to	to	PART
ajema-7508	118	4	computer	computer	NOUN
ajema-7508	118	5	programmer	programmer	NOUN
ajema-7508	118	6	as	as	SCONJ
ajema-7508	118	7	woman	woman	NOUN
ajema-7508	118	8	is	be	AUX
ajema-7508	118	9	to	to	ADP
ajema-7508	118	10	homemaker	homemaker	NOUN
ajema-7508	118	11	?	?	PUNCT
ajema-7508	119	1	debiasing	debiase	VERB
ajema-7508	119	2	word	word	NOUN
ajema-7508	119	3	embeddings	embedding	NOUN
ajema-7508	119	4	.	.	PUNCT
ajema-7508	120	1	advances	advance	NOUN
ajema-7508	120	2	in	in	ADP
ajema-7508	120	3	neural	neural	ADJ
ajema-7508	120	4	information	information	NOUN
ajema-7508	120	5	processing	processing	NOUN
ajema-7508	120	6	systems	system	NOUN
ajema-7508	120	7	(	(	PUNCT
ajema-7508	120	8	neurips	neurip	NOUN
ajema-7508	120	9	)	)	PUNCT
ajema-7508	120	10	.	.	PUNCT
ajema-7508	121	1	https://arxiv.org/abs/1607.06520	https://arxiv.org/abs/1607.06520	PROPN
ajema-7508	122	1	10	10	NUM
ajema-7508	122	2	.	.	PUNCT
ajema-7508	122	3	openai	openai	PROPN
ajema-7508	122	4	.	.	PUNCT
ajema-7508	123	1	(	(	PUNCT
ajema-7508	123	2	2023	2023	NUM
ajema-7508	123	3	)	)	PUNCT
ajema-7508	123	4	.	.	PUNCT
ajema-7508	124	1	gpt-4	gpt-4	PART
ajema-7508	124	2	technical	technical	ADJ
ajema-7508	124	3	report	report	NOUN
ajema-7508	124	4	.	.	PUNCT
ajema-7508	125	1	openai	openai	PROPN
ajema-7508	125	2	.	.	PUNCT
ajema-7508	126	1	https://openai.com/research/gpt-4	https://openai.com/research/gpt-4	PROPN