221 lines
6.2 KiB
Plaintext
221 lines
6.2 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# crf data Generator"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import sys\n",
|
|
"sys.path.append(\"../\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import Tagging.conllu_batch_generator as cbg"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def word2features(sent, i):\n",
|
|
" word = sent[i]['form']\n",
|
|
" postag = sent[i]['upostag']\n",
|
|
" features = [\n",
|
|
" 'bias',\n",
|
|
" #'word.lower=' + word.lower(),\n",
|
|
" 'word[-3:]=' + word[-3:],\n",
|
|
" 'word[-2:]=' + word[-2:],\n",
|
|
" 'word.isupper=%s' % word.isupper(),\n",
|
|
" 'word.istitle=%s' % word.istitle(),\n",
|
|
" 'word.isdigit=%s' % word.isdigit(),\n",
|
|
" 'postag=' + postag,\n",
|
|
" 'postag[:2]=' + postag[:2],\n",
|
|
" ]\n",
|
|
" if i > 0:\n",
|
|
" word1 = sent[i-1]['form']\n",
|
|
" postag1 = sent[i-1]['upostag']\n",
|
|
" features.extend([\n",
|
|
" '-1:word.lower=' + word1.lower(),\n",
|
|
" '-1:word.istitle=%s' % word1.istitle(),\n",
|
|
" '-1:word.isupper=%s' % word1.isupper(),\n",
|
|
" '-1:postag=' + postag1,\n",
|
|
" '-1:postag[:2]=' + postag1[:2],\n",
|
|
" ])\n",
|
|
" if i > 1:\n",
|
|
" word1 = sent[i-2]['form']\n",
|
|
" postag1 = sent[i-2]['upostag']\n",
|
|
" features.extend([\n",
|
|
" '-2:word.lower=' + word1.lower(),\n",
|
|
" '-2:word.istitle=%s' % word1.istitle(),\n",
|
|
" '-2:word.isupper=%s' % word1.isupper(),\n",
|
|
" '-2:postag=' + postag1,\n",
|
|
" '-2:postag[:2]=' + postag1[:2],\n",
|
|
" ])\n",
|
|
" else:\n",
|
|
" features.append('BOS')\n",
|
|
"\n",
|
|
" if i < len(sent)-1:\n",
|
|
" word1 = sent[i+1]['form']\n",
|
|
" postag1 = sent[i+1]['upostag']\n",
|
|
" features.extend([\n",
|
|
" '+1:word.lower=' + word1.lower(),\n",
|
|
" '+1:word.istitle=%s' % word1.istitle(),\n",
|
|
" '+1:word.isupper=%s' % word1.isupper(),\n",
|
|
" '+1:postag=' + postag1,\n",
|
|
" '+1:postag[:2]=' + postag1[:2],\n",
|
|
" ])\n",
|
|
" if i < len(sent)-2:\n",
|
|
" word1 = sent[i+1]['form']\n",
|
|
" postag1 = sent[i+1]['upostag']\n",
|
|
" features.extend([\n",
|
|
" '+2:word.lower=' + word1.lower(),\n",
|
|
" '+2:word.istitle=%s' % word1.istitle(),\n",
|
|
" '+2:word.isupper=%s' % word1.isupper(),\n",
|
|
" '+2:postag=' + postag1,\n",
|
|
" '+2:postag[:2]=' + postag1[:2],\n",
|
|
" ])\n",
|
|
" else:\n",
|
|
" features.append('EOS')\n",
|
|
"\n",
|
|
" return features"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def sent2labels(sent):\n",
|
|
" labels = []\n",
|
|
" for token in sent:\n",
|
|
" if token['misc'] is not None and 'food_type' in token['misc']:\n",
|
|
" labels.append(token['misc']['food_type'])\n",
|
|
" else:\n",
|
|
" labels.append(\"0\")\n",
|
|
" return labels"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def sent2features(sent):\n",
|
|
" return [word2features(sent, i) for i in range(len(sent))]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def sent2tokens(sent):\n",
|
|
" return [token['form'] for token in sent]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def feature2tokens(sent):\n",
|
|
" return [t[1].split(\"=\")[1] for t in sent]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class ConlluCRFReaderIterator(object):\n",
|
|
" def __init__(self, parent):\n",
|
|
" self._parent = parent\n",
|
|
" self._iter = self._parent._conllu_reader.__iter__()\n",
|
|
"\n",
|
|
" def __next__(self):\n",
|
|
" features = None\n",
|
|
" labels = None\n",
|
|
" tokens = None\n",
|
|
"\n",
|
|
" if not self._parent._iter_documents:\n",
|
|
" next_sent = self._iter.__next__()[0]\n",
|
|
" features = sent2features(next_sent)\n",
|
|
" labels = sent2labels(next_sent)\n",
|
|
" tokens = sent2tokens(next_sent)\n",
|
|
" else:\n",
|
|
" next_doc = self._iter.__next__()\n",
|
|
" features = [sent2features(sentence) for sentence in next_doc]\n",
|
|
" labels = [sent2labels(sentence) for sentence in next_doc]\n",
|
|
" tokens = [sent2tokens(sentence) for sentence in next_doc]\n",
|
|
"\n",
|
|
" return features, labels, tokens"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"class ConlluCRFReader(object):\n",
|
|
" def __init__(self, path, iter_documents=False):\n",
|
|
" self._path = path\n",
|
|
" self._iter_documents = iter_documents\n",
|
|
"\n",
|
|
" self._conllu_reader = cbg.ConlluReader(path, iter_documents)\n",
|
|
"\n",
|
|
" def __iter__(self):\n",
|
|
" return ConlluCRFReaderIterator(self)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 4
|
|
}
|