{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Conllu Generator\n", "\n", "tools for creating:\n", "* conllu tokens\n", "* conllu sentences\n", "* conllu documents" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## imports and settings" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys\n", "sys.path.append(\"../\")\n", "\n", "import nltk\n", "from nltk.tag import pos_tag, map_tag\n", "from nltk.stem import PorterStemmer\n", "from nltk.corpus import stopwords as nltk_stopwords\n", "from Tagging.stemmed_mwe_tokenizer import StemmedMWETokenizer\n", "from nltk.stem import WordNetLemmatizer" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "CONLLU_ATTRIBUTES = [\n", " \"id\",\n", " \"form\",\n", " \"lemma\",\n", " \"upos\",\n", " \"xpos\",\n", " \"feats\",\n", " \"head\",\n", " \"deprel\",\n", " \"deps\",\n", " \"misc\"\n", "]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "* default stemming and lemmatization functions" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "porter_stemmer = PorterStemmer()\n", "wordnet_lemmatizer = WordNetLemmatizer()\n", "\n", "def stem(token, stemmer = porter_stemmer):\n", " return stemmer.stem(token)\n", "\n", "def lemmatize(token, lemmatizer = wordnet_lemmatizer, pos = 'n'):\n", " return lemmatizer.lemmatize(token, pos)\n", "\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# took from: https://stackoverflow.com/a/16053211\n", "\n", "\n", "def replace_tab(s, tabstop=4):\n", " result = str()\n", " s = s.replace(\"\\t\", \" \\t\")\n", " for c in s:\n", " if c == '\\t':\n", " while (len(result) % (tabstop) != 0):\n", " result += ' '\n", " else:\n", " result += c\n", " return result" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Conllu Dict Class" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "class ConlluDict(dict):\n", "\n", " def from_str(self, s: str):\n", " entries = s.split(\"|\")\n", " for entry in entries:\n", " key, val = entry.split(\"=\")\n", " self[key.strip()] = val.strip()\n", "\n", " def __repr__(self):\n", " if len(self) == 0:\n", " return \"_\"\n", "\n", " result = \"\"\n", " for key, value in self.items():\n", " result += key + \"=\" + value + \"|\"\n", "\n", " return result[:-1]\n", "\n", " def __str__(self):\n", " return self.__repr__()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Conllu Element Class" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "class ConlluElement(object):\n", " # class uses format described here: https://universaldependencies.org/format.html\n", " def __init__(\n", " self,\n", " id: int,\n", " form: str,\n", " lemma: str = \"_\",\n", " upos: str = \"_\",\n", " xpos: str = \"_\",\n", " feats: str = \"_\",\n", " head: str = \"_\",\n", " deprel: str = \"_\",\n", " deps: str = \"_\",\n", " misc: str = \"_\"):\n", " self.id = id\n", " self.form = form\n", " self.lemma = lemma\n", " self.upos = upos\n", " self.xpos = xpos\n", "\n", " self.feats = ConlluDict()\n", " if feats != \"_\":\n", " self.feats.from_str(feats)\n", "\n", " self.head = head\n", " self.deprel = deprel\n", " self.deps = deps\n", "\n", " self.misc = ConlluDict()\n", " if misc != \"_\":\n", " self.misc.from_str(misc)\n", "\n", " def add_feature(self, key: str, value: str):\n", " self.feats[key] = value\n", "\n", " def add_misc(self, key: str, value: str):\n", " self.misc[key] = value\n", "\n", " def __repr__(self):\n", " result = \"\"\n", " for attr in CONLLU_ATTRIBUTES:\n", " result += str(self.__getattribute__(attr)) + \" \\t\"\n", " return replace_tab(result, 16)\n", " \n", " def __getitem__(self, key):\n", " \n", " # conllu module compability:\n", " if key == \"upostag\":\n", " key = \"upos\"\n", " if key == \"xpostag\":\n", " key = \"xpos\"\n", " \n", " if key not in CONLLU_ATTRIBUTES:\n", " return None\n", " attr = self.__getattribute__(key)\n", " if str(attr) == \"_\":\n", " return None\n", " return attr" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Conllu Sentence Class" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "class ConlluSentence(object):\n", " def __init__(self):\n", " self.conllu_elements = []\n", "\n", " def add(self, conllu_element: ConlluElement):\n", " self.conllu_elements.append(conllu_element)\n", " \n", " def get_conllu_elements(self):\n", " return self.conllu_elements\n", "\n", " def __repr__(self):\n", " result = \"\"\n", " for elem in self.conllu_elements:\n", " result += elem.__repr__() + \"\\n\"\n", "\n", " return result\n", "\n", " def __str__(self):\n", " return self.__repr__()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Conllu Document Class" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "class ConlluDocument(object):\n", " def __init__(self, id=None):\n", " self.conllu_sentences = []\n", " self.id = id\n", " \n", " def add(self, conllu_sentence: ConlluSentence):\n", " self.conllu_sentences.append(conllu_sentence)\n", " \n", " def get_conllu_elements(self):\n", " return [c_sent.get_conllu_elements() for c_sent in self.conllu_sentences]\n", " \n", " def __repr__(self):\n", " result = \"# newdoc\\n\"\n", " if self.id is not None:\n", " result += \"# id: \" + self.id + \"\\n\"\n", " for elem in self.conllu_sentences:\n", " result += elem.__repr__() + \"\\n\"\n", "\n", " return result\n", "\n", " def __str__(self):\n", " return self.__repr__()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Conllu Generator Class" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "class ConlluGenerator(object):\n", " def __init__(self, documents: list, stemmed_multi_word_tokens=None, stemmer=PorterStemmer(), ids=None):\n", " self.documents = documents\n", " self.stemmed_multi_word_tokens = stemmed_multi_word_tokens\n", " \n", " if self.stemmed_multi_word_tokens is not None:\n", " self.mwe_tokenizer = StemmedMWETokenizer(\n", " [w.split() for w in stemmed_multi_word_tokens])\n", " else:\n", " self.mwe_tokenizer = None\n", " \n", " self.stemmer = stemmer\n", "\n", " self.conllu_documents = []\n", "\n", " self.ids = ids\n", " \n", " def tokenize(self):\n", " tokenized_documents = []\n", "\n", " i = 0\n", " for doc in self.documents:\n", " tokenized_sentences = []\n", " sentences = doc.split(\"\\n\")\n", " for sent in sentences: \n", " if (len(sent) > 0):\n", " simple_tokenized = nltk.tokenize.word_tokenize(sent)\n", " if self.mwe_tokenizer is None:\n", " tokenized_sentences.append(simple_tokenized)\n", " else:\n", " tokenized_sentences.append(\n", " self.mwe_tokenizer.tokenize(simple_tokenized))\n", " tokenized_documents.append(tokenized_sentences)\n", " \n", " # now create initial colln-u elemnts\n", " for doc in tokenized_documents:\n", " if self.ids:\n", " conllu_doc = ConlluDocument(self.ids[i])\n", " else:\n", " conllu_doc = ConlluDocument()\n", " for sent in doc:\n", " token_id = 0\n", " conllu_sent = ConlluSentence()\n", " for token in sent:\n", " token_id += 1\n", " conllu_sent.add(ConlluElement(\n", " id=token_id,\n", " form=token,\n", " ))\n", " conllu_doc.add(conllu_sent)\n", " self.conllu_documents.append(conllu_doc)\n", " i += 1\n", "\n", "\n", " def pos_tagging_and_lemmatization(self, stem_function = lemmatize):\n", " pos_dict = {'ADJ': 'a', 'ADJ_SAT': 's', 'ADV': 'r', 'NOUN': 'n', 'VERB': 'v'}\n", " for conllu_document in self.conllu_documents:\n", " for conllu_sent in conllu_document.conllu_sentences:\n", " tokens = [x.form for x in conllu_sent.conllu_elements]\n", " pos_tags = pos_tag(tokens)\n", " simplified_tags = [map_tag('en-ptb', 'universal', tag)\n", " for word, tag in pos_tags]\n", "\n", " for i in range(len(tokens)):\n", " conllu_elem = conllu_sent.conllu_elements[i]\n", " conllu_elem.upos = simplified_tags[i]\n", " conllu_elem.xpos = pos_tags[i][1]\n", " p = 'n'\n", " if conllu_elem.upos in pos_dict:\n", " p = pos_dict[conllu_elem.upos]\n", " conllu_elem.lemma = stem_function(conllu_elem.form, pos=p).lower()\n", "\n", " def add_misc_value_by_list(self, key, value, stemmed_keyword_list):\n", " for conllu_document in self.conllu_documents:\n", " for conllu_sent in conllu_document.conllu_sentences:\n", " for elem in conllu_sent.conllu_elements:\n", " if elem.lemma in stemmed_keyword_list:\n", " elem.add_misc(key, value)\n", " \n", " def get_conllu_elements(self):\n", " return [doc.get_conllu_elements() for doc in self.conllu_documents]\n", "\n", " def __repr__(self):\n", " result = \"\"\n", " for document in self.conllu_documents:\n", " result += document.__repr__() + \"\\n\"\n", " return result\n", "\n", " def __str__(self):\n", " return self.__repr__()" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 4 }