411 lines
12 KiB
Plaintext
411 lines
12 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Conllu Generator\n",
|
|
"\n",
|
|
"tools for creating:\n",
|
|
"* conllu tokens\n",
|
|
"* conllu sentences\n",
|
|
"* conllu documents"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## imports and settings"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import sys\n",
|
|
"sys.path.append(\"../\")\n",
|
|
"\n",
|
|
"import nltk\n",
|
|
"from nltk.tag import pos_tag, map_tag\n",
|
|
"from nltk.stem import PorterStemmer\n",
|
|
"from nltk.corpus import stopwords as nltk_stopwords\n",
|
|
"from Tagging.stemmed_mwe_tokenizer import StemmedMWETokenizer\n",
|
|
"from nltk.stem import WordNetLemmatizer"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"CONLLU_ATTRIBUTES = [\n",
|
|
" \"id\",\n",
|
|
" \"form\",\n",
|
|
" \"lemma\",\n",
|
|
" \"upos\",\n",
|
|
" \"xpos\",\n",
|
|
" \"feats\",\n",
|
|
" \"head\",\n",
|
|
" \"deprel\",\n",
|
|
" \"deps\",\n",
|
|
" \"misc\"\n",
|
|
"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"* default stemming and lemmatization functions"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"porter_stemmer = PorterStemmer()\n",
|
|
"wordnet_lemmatizer = WordNetLemmatizer()\n",
|
|
"\n",
|
|
"def stem(token, stemmer = porter_stemmer):\n",
|
|
" return stemmer.stem(token)\n",
|
|
"\n",
|
|
"def lemmatize(token, lemmatizer = wordnet_lemmatizer, pos = 'n'):\n",
|
|
" return lemmatizer.lemmatize(token, pos)\n",
|
|
"\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# took from: https://stackoverflow.com/a/16053211\n",
|
|
"\n",
|
|
"\n",
|
|
"def replace_tab(s, tabstop=4):\n",
|
|
" result = str()\n",
|
|
" s = s.replace(\"\\t\", \" \\t\")\n",
|
|
" for c in s:\n",
|
|
" if c == '\\t':\n",
|
|
" while (len(result) % (tabstop) != 0):\n",
|
|
" result += ' '\n",
|
|
" else:\n",
|
|
" result += c\n",
|
|
" return result"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Conllu Dict Class"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class ConlluDict(dict):\n",
|
|
"\n",
|
|
" def from_str(self, s: str):\n",
|
|
" entries = s.split(\"|\")\n",
|
|
" for entry in entries:\n",
|
|
" key, val = entry.split(\"=\")\n",
|
|
" self[key.strip()] = val.strip()\n",
|
|
"\n",
|
|
" def __repr__(self):\n",
|
|
" if len(self) == 0:\n",
|
|
" return \"_\"\n",
|
|
"\n",
|
|
" result = \"\"\n",
|
|
" for key, value in self.items():\n",
|
|
" result += key + \"=\" + value + \"|\"\n",
|
|
"\n",
|
|
" return result[:-1]\n",
|
|
"\n",
|
|
" def __str__(self):\n",
|
|
" return self.__repr__()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Conllu Element Class"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class ConlluElement(object):\n",
|
|
" # class uses format described here: https://universaldependencies.org/format.html\n",
|
|
" def __init__(\n",
|
|
" self,\n",
|
|
" id: int,\n",
|
|
" form: str,\n",
|
|
" lemma: str = \"_\",\n",
|
|
" upos: str = \"_\",\n",
|
|
" xpos: str = \"_\",\n",
|
|
" feats: str = \"_\",\n",
|
|
" head: str = \"_\",\n",
|
|
" deprel: str = \"_\",\n",
|
|
" deps: str = \"_\",\n",
|
|
" misc: str = \"_\"):\n",
|
|
" self.id = id\n",
|
|
" self.form = form\n",
|
|
" self.lemma = lemma\n",
|
|
" self.upos = upos\n",
|
|
" self.xpos = xpos\n",
|
|
"\n",
|
|
" self.feats = ConlluDict()\n",
|
|
" if feats != \"_\":\n",
|
|
" self.feats.from_str(feats)\n",
|
|
"\n",
|
|
" self.head = head\n",
|
|
" self.deprel = deprel\n",
|
|
" self.deps = deps\n",
|
|
"\n",
|
|
" self.misc = ConlluDict()\n",
|
|
" if misc != \"_\":\n",
|
|
" self.misc.from_str(misc)\n",
|
|
"\n",
|
|
" def add_feature(self, key: str, value: str):\n",
|
|
" self.feats[key] = value\n",
|
|
"\n",
|
|
" def add_misc(self, key: str, value: str):\n",
|
|
" self.misc[key] = value\n",
|
|
"\n",
|
|
" def __repr__(self):\n",
|
|
" result = \"\"\n",
|
|
" for attr in CONLLU_ATTRIBUTES:\n",
|
|
" result += str(self.__getattribute__(attr)) + \" \\t\"\n",
|
|
" return replace_tab(result, 16)\n",
|
|
" \n",
|
|
" def __getitem__(self, key):\n",
|
|
" \n",
|
|
" # conllu module compability:\n",
|
|
" if key == \"upostag\":\n",
|
|
" key = \"upos\"\n",
|
|
" if key == \"xpostag\":\n",
|
|
" key = \"xpos\"\n",
|
|
" \n",
|
|
" if key not in CONLLU_ATTRIBUTES:\n",
|
|
" return None\n",
|
|
" attr = self.__getattribute__(key)\n",
|
|
" if str(attr) == \"_\":\n",
|
|
" return None\n",
|
|
" return attr"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Conllu Sentence Class"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class ConlluSentence(object):\n",
|
|
" def __init__(self):\n",
|
|
" self.conllu_elements = []\n",
|
|
"\n",
|
|
" def add(self, conllu_element: ConlluElement):\n",
|
|
" self.conllu_elements.append(conllu_element)\n",
|
|
" \n",
|
|
" def get_conllu_elements(self):\n",
|
|
" return self.conllu_elements\n",
|
|
"\n",
|
|
" def __repr__(self):\n",
|
|
" result = \"\"\n",
|
|
" for elem in self.conllu_elements:\n",
|
|
" result += elem.__repr__() + \"\\n\"\n",
|
|
"\n",
|
|
" return result\n",
|
|
"\n",
|
|
" def __str__(self):\n",
|
|
" return self.__repr__()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Conllu Document Class"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class ConlluDocument(object):\n",
|
|
" def __init__(self, id=None):\n",
|
|
" self.conllu_sentences = []\n",
|
|
" self.id = id\n",
|
|
" \n",
|
|
" def add(self, conllu_sentence: ConlluSentence):\n",
|
|
" self.conllu_sentences.append(conllu_sentence)\n",
|
|
" \n",
|
|
" def get_conllu_elements(self):\n",
|
|
" return [c_sent.get_conllu_elements() for c_sent in self.conllu_sentences]\n",
|
|
" \n",
|
|
" def __repr__(self):\n",
|
|
" result = \"# newdoc\\n\"\n",
|
|
" if self.id is not None:\n",
|
|
" result += \"# id: \" + self.id + \"\\n\"\n",
|
|
" for elem in self.conllu_sentences:\n",
|
|
" result += elem.__repr__() + \"\\n\"\n",
|
|
"\n",
|
|
" return result\n",
|
|
"\n",
|
|
" def __str__(self):\n",
|
|
" return self.__repr__()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Conllu Generator Class"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 9,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class ConlluGenerator(object):\n",
|
|
" def __init__(self, documents: list, stemmed_multi_word_tokens=None, stemmer=PorterStemmer(), ids=None):\n",
|
|
" self.documents = documents\n",
|
|
" self.stemmed_multi_word_tokens = stemmed_multi_word_tokens\n",
|
|
" \n",
|
|
" if self.stemmed_multi_word_tokens is not None:\n",
|
|
" self.mwe_tokenizer = StemmedMWETokenizer(\n",
|
|
" [w.split() for w in stemmed_multi_word_tokens])\n",
|
|
" else:\n",
|
|
" self.mwe_tokenizer = None\n",
|
|
" \n",
|
|
" self.stemmer = stemmer\n",
|
|
"\n",
|
|
" self.conllu_documents = []\n",
|
|
"\n",
|
|
" self.ids = ids\n",
|
|
" \n",
|
|
" def tokenize(self):\n",
|
|
" tokenized_documents = []\n",
|
|
"\n",
|
|
" i = 0\n",
|
|
" for doc in self.documents:\n",
|
|
" tokenized_sentences = []\n",
|
|
" sentences = doc.split(\"\\n\")\n",
|
|
" for sent in sentences: \n",
|
|
" if (len(sent) > 0):\n",
|
|
" simple_tokenized = nltk.tokenize.word_tokenize(sent)\n",
|
|
" if self.mwe_tokenizer is None:\n",
|
|
" tokenized_sentences.append(simple_tokenized)\n",
|
|
" else:\n",
|
|
" tokenized_sentences.append(\n",
|
|
" self.mwe_tokenizer.tokenize(simple_tokenized))\n",
|
|
" tokenized_documents.append(tokenized_sentences)\n",
|
|
" \n",
|
|
" # now create initial colln-u elemnts\n",
|
|
" for doc in tokenized_documents:\n",
|
|
" if self.ids:\n",
|
|
" conllu_doc = ConlluDocument(self.ids[i])\n",
|
|
" else:\n",
|
|
" conllu_doc = ConlluDocument()\n",
|
|
" for sent in doc:\n",
|
|
" token_id = 0\n",
|
|
" conllu_sent = ConlluSentence()\n",
|
|
" for token in sent:\n",
|
|
" token_id += 1\n",
|
|
" conllu_sent.add(ConlluElement(\n",
|
|
" id=token_id,\n",
|
|
" form=token,\n",
|
|
" ))\n",
|
|
" conllu_doc.add(conllu_sent)\n",
|
|
" self.conllu_documents.append(conllu_doc)\n",
|
|
" i += 1\n",
|
|
"\n",
|
|
"\n",
|
|
" def pos_tagging_and_lemmatization(self, stem_function = lemmatize):\n",
|
|
" pos_dict = {'ADJ': 'a', 'ADJ_SAT': 's', 'ADV': 'r', 'NOUN': 'n', 'VERB': 'v'}\n",
|
|
" for conllu_document in self.conllu_documents:\n",
|
|
" for conllu_sent in conllu_document.conllu_sentences:\n",
|
|
" tokens = [x.form for x in conllu_sent.conllu_elements]\n",
|
|
" pos_tags = pos_tag(tokens)\n",
|
|
" simplified_tags = [map_tag('en-ptb', 'universal', tag)\n",
|
|
" for word, tag in pos_tags]\n",
|
|
"\n",
|
|
" for i in range(len(tokens)):\n",
|
|
" conllu_elem = conllu_sent.conllu_elements[i]\n",
|
|
" conllu_elem.upos = simplified_tags[i]\n",
|
|
" conllu_elem.xpos = pos_tags[i][1]\n",
|
|
" p = 'n'\n",
|
|
" if conllu_elem.upos in pos_dict:\n",
|
|
" p = pos_dict[conllu_elem.upos]\n",
|
|
" conllu_elem.lemma = stem_function(conllu_elem.form, pos=p).lower()\n",
|
|
"\n",
|
|
" def add_misc_value_by_list(self, key, value, stemmed_keyword_list):\n",
|
|
" for conllu_document in self.conllu_documents:\n",
|
|
" for conllu_sent in conllu_document.conllu_sentences:\n",
|
|
" for elem in conllu_sent.conllu_elements:\n",
|
|
" if elem.lemma in stemmed_keyword_list:\n",
|
|
" elem.add_misc(key, value)\n",
|
|
" \n",
|
|
" def get_conllu_elements(self):\n",
|
|
" return [doc.get_conllu_elements() for doc in self.conllu_documents]\n",
|
|
"\n",
|
|
" def __repr__(self):\n",
|
|
" result = \"\"\n",
|
|
" for document in self.conllu_documents:\n",
|
|
" result += document.__repr__() + \"\\n\"\n",
|
|
" return result\n",
|
|
"\n",
|
|
" def __str__(self):\n",
|
|
" return self.__repr__()"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}
|