master-thesis/Tagging/conllu_generator.ipynb

411 lines
12 KiB
Plaintext

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Conllu Generator\n",
"\n",
"tools for creating:\n",
"* conllu tokens\n",
"* conllu sentences\n",
"* conllu documents"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## imports and settings"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"sys.path.append(\"../\")\n",
"\n",
"import nltk\n",
"from nltk.tag import pos_tag, map_tag\n",
"from nltk.stem import PorterStemmer\n",
"from nltk.corpus import stopwords as nltk_stopwords\n",
"from Tagging.stemmed_mwe_tokenizer import StemmedMWETokenizer\n",
"from nltk.stem import WordNetLemmatizer"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"CONLLU_ATTRIBUTES = [\n",
" \"id\",\n",
" \"form\",\n",
" \"lemma\",\n",
" \"upos\",\n",
" \"xpos\",\n",
" \"feats\",\n",
" \"head\",\n",
" \"deprel\",\n",
" \"deps\",\n",
" \"misc\"\n",
"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* default stemming and lemmatization functions"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"porter_stemmer = PorterStemmer()\n",
"wordnet_lemmatizer = WordNetLemmatizer()\n",
"\n",
"def stem(token, stemmer = porter_stemmer):\n",
" return stemmer.stem(token)\n",
"\n",
"def lemmatize(token, lemmatizer = wordnet_lemmatizer, pos = 'n'):\n",
" return lemmatizer.lemmatize(token, pos)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# took from: https://stackoverflow.com/a/16053211\n",
"\n",
"\n",
"def replace_tab(s, tabstop=4):\n",
" result = str()\n",
" s = s.replace(\"\\t\", \" \\t\")\n",
" for c in s:\n",
" if c == '\\t':\n",
" while (len(result) % (tabstop) != 0):\n",
" result += ' '\n",
" else:\n",
" result += c\n",
" return result"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Conllu Dict Class"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"class ConlluDict(dict):\n",
"\n",
" def from_str(self, s: str):\n",
" entries = s.split(\"|\")\n",
" for entry in entries:\n",
" key, val = entry.split(\"=\")\n",
" self[key.strip()] = val.strip()\n",
"\n",
" def __repr__(self):\n",
" if len(self) == 0:\n",
" return \"_\"\n",
"\n",
" result = \"\"\n",
" for key, value in self.items():\n",
" result += key + \"=\" + value + \"|\"\n",
"\n",
" return result[:-1]\n",
"\n",
" def __str__(self):\n",
" return self.__repr__()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Conllu Element Class"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"class ConlluElement(object):\n",
" # class uses format described here: https://universaldependencies.org/format.html\n",
" def __init__(\n",
" self,\n",
" id: int,\n",
" form: str,\n",
" lemma: str = \"_\",\n",
" upos: str = \"_\",\n",
" xpos: str = \"_\",\n",
" feats: str = \"_\",\n",
" head: str = \"_\",\n",
" deprel: str = \"_\",\n",
" deps: str = \"_\",\n",
" misc: str = \"_\"):\n",
" self.id = id\n",
" self.form = form\n",
" self.lemma = lemma\n",
" self.upos = upos\n",
" self.xpos = xpos\n",
"\n",
" self.feats = ConlluDict()\n",
" if feats != \"_\":\n",
" self.feats.from_str(feats)\n",
"\n",
" self.head = head\n",
" self.deprel = deprel\n",
" self.deps = deps\n",
"\n",
" self.misc = ConlluDict()\n",
" if misc != \"_\":\n",
" self.misc.from_str(misc)\n",
"\n",
" def add_feature(self, key: str, value: str):\n",
" self.feats[key] = value\n",
"\n",
" def add_misc(self, key: str, value: str):\n",
" self.misc[key] = value\n",
"\n",
" def __repr__(self):\n",
" result = \"\"\n",
" for attr in CONLLU_ATTRIBUTES:\n",
" result += str(self.__getattribute__(attr)) + \" \\t\"\n",
" return replace_tab(result, 16)\n",
" \n",
" def __getitem__(self, key):\n",
" \n",
" # conllu module compability:\n",
" if key == \"upostag\":\n",
" key = \"upos\"\n",
" if key == \"xpostag\":\n",
" key = \"xpos\"\n",
" \n",
" if key not in CONLLU_ATTRIBUTES:\n",
" return None\n",
" attr = self.__getattribute__(key)\n",
" if str(attr) == \"_\":\n",
" return None\n",
" return attr"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Conllu Sentence Class"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"class ConlluSentence(object):\n",
" def __init__(self):\n",
" self.conllu_elements = []\n",
"\n",
" def add(self, conllu_element: ConlluElement):\n",
" self.conllu_elements.append(conllu_element)\n",
" \n",
" def get_conllu_elements(self):\n",
" return self.conllu_elements\n",
"\n",
" def __repr__(self):\n",
" result = \"\"\n",
" for elem in self.conllu_elements:\n",
" result += elem.__repr__() + \"\\n\"\n",
"\n",
" return result\n",
"\n",
" def __str__(self):\n",
" return self.__repr__()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Conllu Document Class"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"class ConlluDocument(object):\n",
" def __init__(self, id=None):\n",
" self.conllu_sentences = []\n",
" self.id = id\n",
" \n",
" def add(self, conllu_sentence: ConlluSentence):\n",
" self.conllu_sentences.append(conllu_sentence)\n",
" \n",
" def get_conllu_elements(self):\n",
" return [c_sent.get_conllu_elements() for c_sent in self.conllu_sentences]\n",
" \n",
" def __repr__(self):\n",
" result = \"# newdoc\\n\"\n",
" if self.id is not None:\n",
" result += \"# id: \" + self.id + \"\\n\"\n",
" for elem in self.conllu_sentences:\n",
" result += elem.__repr__() + \"\\n\"\n",
"\n",
" return result\n",
"\n",
" def __str__(self):\n",
" return self.__repr__()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Conllu Generator Class"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"class ConlluGenerator(object):\n",
" def __init__(self, documents: list, stemmed_multi_word_tokens=None, stemmer=PorterStemmer(), ids=None):\n",
" self.documents = documents\n",
" self.stemmed_multi_word_tokens = stemmed_multi_word_tokens\n",
" \n",
" if self.stemmed_multi_word_tokens is not None:\n",
" self.mwe_tokenizer = StemmedMWETokenizer(\n",
" [w.split() for w in stemmed_multi_word_tokens])\n",
" else:\n",
" self.mwe_tokenizer = None\n",
" \n",
" self.stemmer = stemmer\n",
"\n",
" self.conllu_documents = []\n",
"\n",
" self.ids = ids\n",
" \n",
" def tokenize(self):\n",
" tokenized_documents = []\n",
"\n",
" i = 0\n",
" for doc in self.documents:\n",
" tokenized_sentences = []\n",
" sentences = doc.split(\"\\n\")\n",
" for sent in sentences: \n",
" if (len(sent) > 0):\n",
" simple_tokenized = nltk.tokenize.word_tokenize(sent)\n",
" if self.mwe_tokenizer is None:\n",
" tokenized_sentences.append(simple_tokenized)\n",
" else:\n",
" tokenized_sentences.append(\n",
" self.mwe_tokenizer.tokenize(simple_tokenized))\n",
" tokenized_documents.append(tokenized_sentences)\n",
" \n",
" # now create initial colln-u elemnts\n",
" for doc in tokenized_documents:\n",
" if self.ids:\n",
" conllu_doc = ConlluDocument(self.ids[i])\n",
" else:\n",
" conllu_doc = ConlluDocument()\n",
" for sent in doc:\n",
" token_id = 0\n",
" conllu_sent = ConlluSentence()\n",
" for token in sent:\n",
" token_id += 1\n",
" conllu_sent.add(ConlluElement(\n",
" id=token_id,\n",
" form=token,\n",
" ))\n",
" conllu_doc.add(conllu_sent)\n",
" self.conllu_documents.append(conllu_doc)\n",
" i += 1\n",
"\n",
"\n",
" def pos_tagging_and_lemmatization(self, stem_function = lemmatize):\n",
" pos_dict = {'ADJ': 'a', 'ADJ_SAT': 's', 'ADV': 'r', 'NOUN': 'n', 'VERB': 'v'}\n",
" for conllu_document in self.conllu_documents:\n",
" for conllu_sent in conllu_document.conllu_sentences:\n",
" tokens = [x.form for x in conllu_sent.conllu_elements]\n",
" pos_tags = pos_tag(tokens)\n",
" simplified_tags = [map_tag('en-ptb', 'universal', tag)\n",
" for word, tag in pos_tags]\n",
"\n",
" for i in range(len(tokens)):\n",
" conllu_elem = conllu_sent.conllu_elements[i]\n",
" conllu_elem.upos = simplified_tags[i]\n",
" conllu_elem.xpos = pos_tags[i][1]\n",
" p = 'n'\n",
" if conllu_elem.upos in pos_dict:\n",
" p = pos_dict[conllu_elem.upos]\n",
" conllu_elem.lemma = stem_function(conllu_elem.form, pos=p).lower()\n",
"\n",
" def add_misc_value_by_list(self, key, value, stemmed_keyword_list):\n",
" for conllu_document in self.conllu_documents:\n",
" for conllu_sent in conllu_document.conllu_sentences:\n",
" for elem in conllu_sent.conllu_elements:\n",
" if elem.lemma in stemmed_keyword_list:\n",
" elem.add_misc(key, value)\n",
" \n",
" def get_conllu_elements(self):\n",
" return [doc.get_conllu_elements() for doc in self.conllu_documents]\n",
"\n",
" def __repr__(self):\n",
" result = \"\"\n",
" for document in self.conllu_documents:\n",
" result += document.__repr__() + \"\\n\"\n",
" return result\n",
"\n",
" def __str__(self):\n",
" return self.__repr__()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.3"
}
},
"nbformat": 4,
"nbformat_minor": 4
}