moved 1 million recipes into database, starting coarse tree generation for recipe analysis
This commit is contained in:
		
										
											
												File diff suppressed because it is too large
												Load Diff
											
										
									
								
							| @ -20,7 +20,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 4, | ||||
|    "execution_count": 3, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
| @ -29,7 +29,7 @@ | ||||
|        "TokenList<Dissolve, Jello, in, boiling, water, .>" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 4, | ||||
|      "execution_count": 3, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
| @ -40,7 +40,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 5, | ||||
|    "execution_count": 4, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -109,7 +109,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 6, | ||||
|    "execution_count": 5, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -125,7 +125,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 7, | ||||
|    "execution_count": 6, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -135,7 +135,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 8, | ||||
|    "execution_count": 7, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -145,7 +145,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 9, | ||||
|    "execution_count": 8, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -155,7 +155,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 10, | ||||
|    "execution_count": 9, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -172,7 +172,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 11, | ||||
|    "execution_count": 10, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -181,7 +181,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 14, | ||||
|    "execution_count": 11, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -218,16 +218,16 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 15, | ||||
|    "execution_count": 12, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "45442" | ||||
|        "47538" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 15, | ||||
|      "execution_count": 12, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
| @ -245,7 +245,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 16, | ||||
|    "execution_count": 13, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -254,7 +254,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 17, | ||||
|    "execution_count": 14, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -266,7 +266,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 18, | ||||
|    "execution_count": 15, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -282,7 +282,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 19, | ||||
|    "execution_count": 16, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
| @ -302,7 +302,7 @@ | ||||
|        " 'max_linesearch']" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 19, | ||||
|      "execution_count": 16, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
| @ -313,7 +313,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 20, | ||||
|    "execution_count": 17, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
| @ -322,24 +322,24 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 20, | ||||
|    "execution_count": 18, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "{'num': 455,\n", | ||||
|        "{'num': 830,\n", | ||||
|        " 'scores': {},\n", | ||||
|        " 'loss': 110.581675,\n", | ||||
|        " 'feature_norm': 8.594619,\n", | ||||
|        " 'error_norm': 0.214403,\n", | ||||
|        " 'active_features': 87,\n", | ||||
|        " 'loss': 41171.669638,\n", | ||||
|        " 'feature_norm': 126.341894,\n", | ||||
|        " 'error_norm': 85.690855,\n", | ||||
|        " 'active_features': 6055,\n", | ||||
|        " 'linesearch_trials': 2,\n", | ||||
|        " 'linesearch_step': 0.5,\n", | ||||
|        " 'time': 0.001}" | ||||
|        " 'time': 0.724}" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 20, | ||||
|      "execution_count": 18, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
| @ -357,16 +357,16 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 21, | ||||
|    "execution_count": 19, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
|      "data": { | ||||
|       "text/plain": [ | ||||
|        "<contextlib.closing at 0x7f056bc6d828>" | ||||
|        "<contextlib.closing at 0x7f26d79813c8>" | ||||
|       ] | ||||
|      }, | ||||
|      "execution_count": 21, | ||||
|      "execution_count": 19, | ||||
|      "metadata": {}, | ||||
|      "output_type": "execute_result" | ||||
|     } | ||||
| @ -378,7 +378,7 @@ | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 22, | ||||
|    "execution_count": 20, | ||||
|    "metadata": {}, | ||||
|    "outputs": [ | ||||
|     { | ||||
| @ -388,7 +388,7 @@ | ||||
|      "traceback": [ | ||||
|       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", | ||||
|       "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)", | ||||
|       "\u001b[0;32m<ipython-input-22-a88100b49642>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m130\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0;31m#print(' '.join(feature2tokens(X_test[i])), end='\\n\\n')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Predicted:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtagger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Correct:  \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | ||||
|       "\u001b[0;32m<ipython-input-20-a88100b49642>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m130\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0;31m#print(' '.join(feature2tokens(X_test[i])), end='\\n\\n')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Predicted:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtagger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Correct:  \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mY_test\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", | ||||
|       "\u001b[0;31mIndexError\u001b[0m: list index out of range" | ||||
|      ] | ||||
|     } | ||||
|  | ||||
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							
										
											
												File diff suppressed because one or more lines are too long
											
										
									
								
							| @ -16,14 +16,15 @@ | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import sys\n", | ||||
|     "sys.path.append('../')\n", | ||||
|     "\n", | ||||
|     "from conllu import parse\n", | ||||
|     "from tagging_tools import print_visualized_tags\n", | ||||
|     "from Tagging.tagging_tools import print_visualized_tags\n", | ||||
|     "\n", | ||||
|     "from sklearn import preprocessing\n", | ||||
|     "import numpy as np\n", | ||||
|     "\n", | ||||
|     "sys.path.insert(0, '..')\n", | ||||
|     "\n", | ||||
|     "import settings  # noqa\n", | ||||
|     "\n", | ||||
|     "import gzip" | ||||
|  | ||||
| @ -6,14 +6,15 @@ | ||||
| # read conllu documents in batches | ||||
|  | ||||
| import sys | ||||
| sys.path.append('../') | ||||
|  | ||||
| from conllu import parse | ||||
| from tagging_tools import print_visualized_tags | ||||
| from Tagging.tagging_tools import print_visualized_tags | ||||
|  | ||||
| from sklearn import preprocessing | ||||
| import numpy as np | ||||
|  | ||||
| sys.path.insert(0, '..') | ||||
|  | ||||
| import settings  # noqa | ||||
|  | ||||
| import gzip | ||||
|  | ||||
| @ -25,11 +25,14 @@ | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import sys\n", | ||||
|     "sys.path.append(\"../\")\n", | ||||
|     "\n", | ||||
|     "import nltk\n", | ||||
|     "from nltk.tag import pos_tag, map_tag\n", | ||||
|     "from nltk.stem import PorterStemmer\n", | ||||
|     "from nltk.corpus import stopwords as nltk_stopwords\n", | ||||
|     "from stemmed_mwe_tokenizer import StemmedMWETokenizer\n", | ||||
|     "from Tagging.stemmed_mwe_tokenizer import StemmedMWETokenizer\n", | ||||
|     "from nltk.stem import WordNetLemmatizer" | ||||
|    ] | ||||
|   }, | ||||
| @ -188,7 +191,22 @@ | ||||
|     "        result = \"\"\n", | ||||
|     "        for attr in CONLLU_ATTRIBUTES:\n", | ||||
|     "            result += str(self.__getattribute__(attr)) + \" \\t\"\n", | ||||
|     "        return replace_tab(result, 16)" | ||||
|     "        return replace_tab(result, 16)\n", | ||||
|     "    \n", | ||||
|     "    def __getitem__(self, key):\n", | ||||
|     "        \n", | ||||
|     "        # conllu module compability:\n", | ||||
|     "        if key == \"upostag\":\n", | ||||
|     "            key = \"upos\"\n", | ||||
|     "        if key == \"xpostag\":\n", | ||||
|     "            key = \"xpos\"\n", | ||||
|     "        \n", | ||||
|     "        if key not in CONLLU_ATTRIBUTES:\n", | ||||
|     "            return None\n", | ||||
|     "        attr = self.__getattribute__(key)\n", | ||||
|     "        if str(attr) == \"_\":\n", | ||||
|     "            return None\n", | ||||
|     "        return attr" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
| @ -210,6 +228,9 @@ | ||||
|     "\n", | ||||
|     "    def add(self, conllu_element: ConlluElement):\n", | ||||
|     "        self.conllu_elements.append(conllu_element)\n", | ||||
|     "    \n", | ||||
|     "    def get_conllu_elements(self):\n", | ||||
|     "        return self.conllu_elements\n", | ||||
|     "\n", | ||||
|     "    def __repr__(self):\n", | ||||
|     "        result = \"\"\n", | ||||
| @ -243,6 +264,9 @@ | ||||
|     "    def add(self, conllu_sentence: ConlluSentence):\n", | ||||
|     "        self.conllu_sentences.append(conllu_sentence)\n", | ||||
|     "    \n", | ||||
|     "    def get_conllu_elements(self):\n", | ||||
|     "        return [c_sent.get_conllu_elements() for c_sent in self.conllu_sentences]\n", | ||||
|     "    \n", | ||||
|     "    def __repr__(self):\n", | ||||
|     "        result = \"# newdoc\\n\"\n", | ||||
|     "        if self.id is not None:\n", | ||||
| @ -270,15 +294,18 @@ | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "class ConlluGenerator(object):\n", | ||||
|     "    def __init__(self, documents: list, stemmed_multi_word_tokens, stemmer=PorterStemmer(), ids=None):\n", | ||||
|     "    def __init__(self, documents: list, stemmed_multi_word_tokens=None, stemmer=PorterStemmer(), ids=None):\n", | ||||
|     "        self.documents = documents\n", | ||||
|     "        self.stemmed_multi_word_tokens = stemmed_multi_word_tokens\n", | ||||
|     "        self.mwe_tokenizer = StemmedMWETokenizer(\n", | ||||
|     "            [w.split() for w in stemmed_multi_word_tokens])\n", | ||||
|     "        \n", | ||||
|     "        if self.stemmed_multi_word_tokens is not None:\n", | ||||
|     "            self.mwe_tokenizer = StemmedMWETokenizer(\n", | ||||
|     "                [w.split() for w in stemmed_multi_word_tokens])\n", | ||||
|     "        else:\n", | ||||
|     "            self.mwe_tokenizer = None\n", | ||||
|     "        \n", | ||||
|     "        self.stemmer = stemmer\n", | ||||
|     "\n", | ||||
|     "        self.id_counter = 0\n", | ||||
|     "\n", | ||||
|     "        self.conllu_documents = []\n", | ||||
|     "\n", | ||||
|     "        self.ids = ids\n", | ||||
| @ -293,8 +320,11 @@ | ||||
|     "            for sent in sentences: \n", | ||||
|     "                if (len(sent) > 0):\n", | ||||
|     "                    simple_tokenized = nltk.tokenize.word_tokenize(sent)\n", | ||||
|     "                    tokenized_sentences.append(\n", | ||||
|     "                        self.mwe_tokenizer.tokenize(simple_tokenized))\n", | ||||
|     "                    if self.mwe_tokenizer is None:\n", | ||||
|     "                        tokenized_sentences.append(simple_tokenized)\n", | ||||
|     "                    else:\n", | ||||
|     "                        tokenized_sentences.append(\n", | ||||
|     "                            self.mwe_tokenizer.tokenize(simple_tokenized))\n", | ||||
|     "            tokenized_documents.append(tokenized_sentences)\n", | ||||
|     "        \n", | ||||
|     "        # now create initial colln-u elemnts\n", | ||||
| @ -304,13 +334,14 @@ | ||||
|     "            else:\n", | ||||
|     "                conllu_doc = ConlluDocument()\n", | ||||
|     "            for sent in doc:\n", | ||||
|     "                token_id = 0\n", | ||||
|     "                conllu_sent = ConlluSentence()\n", | ||||
|     "                for token in sent:\n", | ||||
|     "                    token_id += 1\n", | ||||
|     "                    conllu_sent.add(ConlluElement(\n", | ||||
|     "                        id=self.id_counter + 1,\n", | ||||
|     "                        id=token_id,\n", | ||||
|     "                        form=token,\n", | ||||
|     "                    ))\n", | ||||
|     "                    self.id_counter += 1\n", | ||||
|     "                conllu_doc.add(conllu_sent)\n", | ||||
|     "            self.conllu_documents.append(conllu_doc)\n", | ||||
|     "            i += 1\n", | ||||
| @ -340,6 +371,9 @@ | ||||
|     "                for elem in conllu_sent.conllu_elements:\n", | ||||
|     "                    if elem.lemma in stemmed_keyword_list:\n", | ||||
|     "                        elem.add_misc(key, value)\n", | ||||
|     "    \n", | ||||
|     "    def get_conllu_elements(self):\n", | ||||
|     "        return [doc.get_conllu_elements() for doc in self.conllu_documents]\n", | ||||
|     "\n", | ||||
|     "    def __repr__(self):\n", | ||||
|     "        result = \"\"\n", | ||||
|  | ||||
| @ -10,11 +10,14 @@ | ||||
|  | ||||
| # ## imports and settings | ||||
|  | ||||
| import sys | ||||
| sys.path.append("../") | ||||
|  | ||||
| import nltk | ||||
| from nltk.tag import pos_tag, map_tag | ||||
| from nltk.stem import PorterStemmer | ||||
| from nltk.corpus import stopwords as nltk_stopwords | ||||
| from stemmed_mwe_tokenizer import StemmedMWETokenizer | ||||
| from Tagging.stemmed_mwe_tokenizer import StemmedMWETokenizer | ||||
| from nltk.stem import WordNetLemmatizer | ||||
|  | ||||
|  | ||||
| @ -128,6 +131,21 @@ class ConlluElement(object): | ||||
|         for attr in CONLLU_ATTRIBUTES: | ||||
|             result += str(self.__getattribute__(attr)) + " \t" | ||||
|         return replace_tab(result, 16) | ||||
|      | ||||
|     def __getitem__(self, key): | ||||
|          | ||||
|         # conllu module compability: | ||||
|         if key == "upostag": | ||||
|             key = "upos" | ||||
|         if key == "xpostag": | ||||
|             key = "xpos" | ||||
|          | ||||
|         if key not in CONLLU_ATTRIBUTES: | ||||
|             return None | ||||
|         attr = self.__getattribute__(key) | ||||
|         if str(attr) == "_": | ||||
|             return None | ||||
|         return attr | ||||
|  | ||||
|  | ||||
| # ## Conllu Sentence Class | ||||
| @ -138,6 +156,9 @@ class ConlluSentence(object): | ||||
|  | ||||
|     def add(self, conllu_element: ConlluElement): | ||||
|         self.conllu_elements.append(conllu_element) | ||||
|      | ||||
|     def get_conllu_elements(self): | ||||
|         return self.conllu_elements | ||||
|  | ||||
|     def __repr__(self): | ||||
|         result = "" | ||||
| @ -160,6 +181,9 @@ class ConlluDocument(object): | ||||
|     def add(self, conllu_sentence: ConlluSentence): | ||||
|         self.conllu_sentences.append(conllu_sentence) | ||||
|      | ||||
|     def get_conllu_elements(self): | ||||
|         return [c_sent.get_conllu_elements() for c_sent in self.conllu_sentences] | ||||
|      | ||||
|     def __repr__(self): | ||||
|         result = "# newdoc\n" | ||||
|         if self.id is not None: | ||||
| @ -176,15 +200,18 @@ class ConlluDocument(object): | ||||
| # ## Conllu Generator Class | ||||
|  | ||||
| class ConlluGenerator(object): | ||||
|     def __init__(self, documents: list, stemmed_multi_word_tokens, stemmer=PorterStemmer(), ids=None): | ||||
|     def __init__(self, documents: list, stemmed_multi_word_tokens=None, stemmer=PorterStemmer(), ids=None): | ||||
|         self.documents = documents | ||||
|         self.stemmed_multi_word_tokens = stemmed_multi_word_tokens | ||||
|         self.mwe_tokenizer = StemmedMWETokenizer( | ||||
|             [w.split() for w in stemmed_multi_word_tokens]) | ||||
|          | ||||
|         if self.stemmed_multi_word_tokens is not None: | ||||
|             self.mwe_tokenizer = StemmedMWETokenizer( | ||||
|                 [w.split() for w in stemmed_multi_word_tokens]) | ||||
|         else: | ||||
|             self.mwe_tokenizer = None | ||||
|          | ||||
|         self.stemmer = stemmer | ||||
|  | ||||
|         self.id_counter = 0 | ||||
|  | ||||
|         self.conllu_documents = [] | ||||
|  | ||||
|         self.ids = ids | ||||
| @ -199,8 +226,11 @@ class ConlluGenerator(object): | ||||
|             for sent in sentences:  | ||||
|                 if (len(sent) > 0): | ||||
|                     simple_tokenized = nltk.tokenize.word_tokenize(sent) | ||||
|                     tokenized_sentences.append( | ||||
|                         self.mwe_tokenizer.tokenize(simple_tokenized)) | ||||
|                     if self.mwe_tokenizer is None: | ||||
|                         tokenized_sentences.append(simple_tokenized) | ||||
|                     else: | ||||
|                         tokenized_sentences.append( | ||||
|                             self.mwe_tokenizer.tokenize(simple_tokenized)) | ||||
|             tokenized_documents.append(tokenized_sentences) | ||||
|          | ||||
|         # now create initial colln-u elemnts | ||||
| @ -210,13 +240,14 @@ class ConlluGenerator(object): | ||||
|             else: | ||||
|                 conllu_doc = ConlluDocument() | ||||
|             for sent in doc: | ||||
|                 token_id = 0 | ||||
|                 conllu_sent = ConlluSentence() | ||||
|                 for token in sent: | ||||
|                     token_id += 1 | ||||
|                     conllu_sent.add(ConlluElement( | ||||
|                         id=self.id_counter + 1, | ||||
|                         id=token_id, | ||||
|                         form=token, | ||||
|                     )) | ||||
|                     self.id_counter += 1 | ||||
|                 conllu_doc.add(conllu_sent) | ||||
|             self.conllu_documents.append(conllu_doc) | ||||
|             i += 1 | ||||
| @ -246,6 +277,9 @@ class ConlluGenerator(object): | ||||
|                 for elem in conllu_sent.conllu_elements: | ||||
|                     if elem.lemma in stemmed_keyword_list: | ||||
|                         elem.add_misc(key, value) | ||||
|      | ||||
|     def get_conllu_elements(self): | ||||
|         return [doc.get_conllu_elements() for doc in self.conllu_documents] | ||||
|  | ||||
|     def __repr__(self): | ||||
|         result = "" | ||||
|  | ||||
| @ -13,7 +13,17 @@ | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import conllu_batch_generator as cbg" | ||||
|     "import sys\n", | ||||
|     "sys.path.append(\"../\")" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|    "cell_type": "code", | ||||
|    "execution_count": 2, | ||||
|    "metadata": {}, | ||||
|    "outputs": [], | ||||
|    "source": [ | ||||
|     "import Tagging.conllu_batch_generator as cbg" | ||||
|    ] | ||||
|   }, | ||||
|   { | ||||
|  | ||||
| @ -3,7 +3,11 @@ | ||||
|  | ||||
| # # crf data Generator | ||||
|  | ||||
| import conllu_batch_generator as cbg | ||||
| import sys | ||||
| sys.path.append("../") | ||||
|  | ||||
|  | ||||
| import Tagging.conllu_batch_generator as cbg | ||||
|  | ||||
|  | ||||
| def word2features(sent, i): | ||||
|  | ||||
| @ -46,6 +46,12 @@ | ||||
|     "containers = importlib.util.module_from_spec(spec)\n", | ||||
|     "spec.loader.exec_module(containers)\n", | ||||
|     "\n", | ||||
|     "# loading placeholders\n", | ||||
|     "spec = importlib.util.spec_from_file_location(\n", | ||||
|     "    \"placeholders\", \"../\" + settings.placeholder_file)\n", | ||||
|     "placeholders = importlib.util.module_from_spec(spec)\n", | ||||
|     "spec.loader.exec_module(placeholders)\n", | ||||
|     "\n", | ||||
|     "# skipping recipes:\n", | ||||
|     "n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0\n", | ||||
|     "print(\"start reading at recipe \" + str(n_skipped_recipes))\n", | ||||
| @ -80,7 +86,8 @@ | ||||
|     "    \n", | ||||
|     "    cg.add_misc_value_by_list(\"food_type\", \"ingredient\", [w.replace(\" \",\"_\") for w in ingredients.multi_word_ingredients_stemmed] + ingredients.ingredients_stemmed)\n", | ||||
|     "    cg.add_misc_value_by_list(\"food_type\", \"action\", actions.stemmed_cooking_verbs)\n", | ||||
|     "    cg.add_misc_value_by_list(\"food_type\", \"containers\", containers.containers)\n", | ||||
|     "    cg.add_misc_value_by_list(\"food_type\", \"containers\", containers.stemmed_containers)\n", | ||||
|     "    cg.add_misc_value_by_list(\"food_type\", \"placeholders\", placeholders.stemmed_placeholders)\n", | ||||
|     "\n", | ||||
|     "    savefile.write(str(cg))" | ||||
|    ] | ||||
|  | ||||
| @ -30,6 +30,12 @@ spec = importlib.util.spec_from_file_location( | ||||
| containers = importlib.util.module_from_spec(spec) | ||||
| spec.loader.exec_module(containers) | ||||
|  | ||||
| # loading placeholders | ||||
| spec = importlib.util.spec_from_file_location( | ||||
|     "placeholders", "../" + settings.placeholder_file) | ||||
| placeholders = importlib.util.module_from_spec(spec) | ||||
| spec.loader.exec_module(placeholders) | ||||
|  | ||||
| # skipping recipes: | ||||
| n_skipped_recipes = int(sys.argv[1]) if len(sys.argv) > 1 else 0 | ||||
| print("start reading at recipe " + str(n_skipped_recipes)) | ||||
| @ -58,7 +64,8 @@ def process_instructions(instructions: list, document_ids=None): | ||||
|      | ||||
|     cg.add_misc_value_by_list("food_type", "ingredient", [w.replace(" ","_") for w in ingredients.multi_word_ingredients_stemmed] + ingredients.ingredients_stemmed) | ||||
|     cg.add_misc_value_by_list("food_type", "action", actions.stemmed_cooking_verbs) | ||||
|     cg.add_misc_value_by_list("food_type", "containers", containers.containers) | ||||
|     cg.add_misc_value_by_list("food_type", "containers", containers.stemmed_containers) | ||||
|     cg.add_misc_value_by_list("food_type", "placeholders", placeholders.stemmed_placeholders) | ||||
|  | ||||
|     savefile.write(str(cg)) | ||||
|  | ||||
|  | ||||
										
											Binary file not shown.
										
									
								
							
		Reference in New Issue
	
	Block a user