{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# crf data Generator" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import sys\n", "sys.path.append(\"../\")" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import Tagging.conllu_batch_generator as cbg" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "def word2features(sent, i):\n", " word = sent[i]['form']\n", " postag = sent[i]['upostag']\n", " features = [\n", " 'bias',\n", " #'word.lower=' + word.lower(),\n", " 'word[-3:]=' + word[-3:],\n", " 'word[-2:]=' + word[-2:],\n", " 'word.isupper=%s' % word.isupper(),\n", " 'word.istitle=%s' % word.istitle(),\n", " 'word.isdigit=%s' % word.isdigit(),\n", " 'postag=' + postag,\n", " 'postag[:2]=' + postag[:2],\n", " ]\n", " if i > 0:\n", " word1 = sent[i-1]['form']\n", " postag1 = sent[i-1]['upostag']\n", " features.extend([\n", " '-1:word.lower=' + word1.lower(),\n", " '-1:word.istitle=%s' % word1.istitle(),\n", " '-1:word.isupper=%s' % word1.isupper(),\n", " '-1:postag=' + postag1,\n", " '-1:postag[:2]=' + postag1[:2],\n", " ])\n", " if i > 1:\n", " word1 = sent[i-2]['form']\n", " postag1 = sent[i-2]['upostag']\n", " features.extend([\n", " '-2:word.lower=' + word1.lower(),\n", " '-2:word.istitle=%s' % word1.istitle(),\n", " '-2:word.isupper=%s' % word1.isupper(),\n", " '-2:postag=' + postag1,\n", " '-2:postag[:2]=' + postag1[:2],\n", " ])\n", " else:\n", " features.append('BOS')\n", "\n", " if i < len(sent)-1:\n", " word1 = sent[i+1]['form']\n", " postag1 = sent[i+1]['upostag']\n", " features.extend([\n", " '+1:word.lower=' + word1.lower(),\n", " '+1:word.istitle=%s' % word1.istitle(),\n", " '+1:word.isupper=%s' % word1.isupper(),\n", " '+1:postag=' + postag1,\n", " '+1:postag[:2]=' + postag1[:2],\n", " ])\n", " if i < len(sent)-2:\n", " word1 = sent[i+1]['form']\n", " postag1 = sent[i+1]['upostag']\n", " features.extend([\n", " '+2:word.lower=' + word1.lower(),\n", " '+2:word.istitle=%s' % word1.istitle(),\n", " '+2:word.isupper=%s' % word1.isupper(),\n", " '+2:postag=' + postag1,\n", " '+2:postag[:2]=' + postag1[:2],\n", " ])\n", " else:\n", " features.append('EOS')\n", "\n", " return features" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "def sent2labels(sent):\n", " labels = []\n", " for token in sent:\n", " if token['misc'] is not None and 'food_type' in token['misc']:\n", " labels.append(token['misc']['food_type'])\n", " else:\n", " labels.append(\"0\")\n", " return labels" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "def sent2features(sent):\n", " return [word2features(sent, i) for i in range(len(sent))]" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "def sent2tokens(sent):\n", " return [token['form'] for token in sent]" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "def feature2tokens(sent):\n", " return [t[1].split(\"=\")[1] for t in sent]" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "class ConlluCRFReaderIterator(object):\n", " def __init__(self, parent):\n", " self._parent = parent\n", " self._iter = self._parent._conllu_reader.__iter__()\n", "\n", " def __next__(self):\n", " features = None\n", " labels = None\n", " tokens = None\n", "\n", " if not self._parent._iter_documents:\n", " next_sent = self._iter.__next__()[0]\n", " features = sent2features(next_sent)\n", " labels = sent2labels(next_sent)\n", " tokens = sent2tokens(next_sent)\n", " else:\n", " next_doc = self._iter.__next__()\n", " features = [sent2features(sentence) for sentence in next_doc]\n", " labels = [sent2labels(sentence) for sentence in next_doc]\n", " tokens = [sent2tokens(sentence) for sentence in next_doc]\n", "\n", " return features, labels, tokens" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "class ConlluCRFReader(object):\n", " def __init__(self, path, iter_documents=False):\n", " self._path = path\n", " self._iter_documents = iter_documents\n", "\n", " self._conllu_reader = cbg.ConlluReader(path, iter_documents)\n", "\n", " def __iter__(self):\n", " return ConlluCRFReaderIterator(self)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.3" } }, "nbformat": 4, "nbformat_minor": 4 }