212 lines
7.0 KiB
Python
212 lines
7.0 KiB
Python
|
#!/usr/bin/env python3
|
||
|
# coding: utf-8
|
||
|
|
||
|
# # Recipe class
|
||
|
|
||
|
import sys
|
||
|
sys.path.append("../")
|
||
|
|
||
|
import settings
|
||
|
|
||
|
import pycrfsuite
|
||
|
|
||
|
import json
|
||
|
|
||
|
import db.db_settings as db_settings
|
||
|
from db.database_connection import DatabaseConnection
|
||
|
|
||
|
from Tagging.conllu_generator import ConlluGenerator
|
||
|
from Tagging.crf_data_generator import *
|
||
|
|
||
|
from IPython.display import Markdown, HTML, display
|
||
|
|
||
|
|
||
|
# * get vocabulary
|
||
|
|
||
|
import importlib.util
|
||
|
# loading ingredients:
|
||
|
spec = importlib.util.spec_from_file_location(
|
||
|
"ingredients", "../" + settings.ingredients_file)
|
||
|
ingredients = importlib.util.module_from_spec(spec)
|
||
|
spec.loader.exec_module(ingredients)
|
||
|
|
||
|
# loading actions:
|
||
|
spec = importlib.util.spec_from_file_location(
|
||
|
"actions", "../" + settings.actions_file)
|
||
|
actions = importlib.util.module_from_spec(spec)
|
||
|
spec.loader.exec_module(actions)
|
||
|
|
||
|
# loading containers
|
||
|
spec = importlib.util.spec_from_file_location(
|
||
|
"containers", "../" + settings.container_file)
|
||
|
containers = importlib.util.module_from_spec(spec)
|
||
|
spec.loader.exec_module(containers)
|
||
|
|
||
|
# loading placeholders
|
||
|
spec = importlib.util.spec_from_file_location(
|
||
|
"placeholders", "../" + settings.placeholder_file)
|
||
|
placeholders = importlib.util.module_from_spec(spec)
|
||
|
spec.loader.exec_module(placeholders)
|
||
|
|
||
|
|
||
|
tagger = pycrfsuite.Tagger()
|
||
|
tagger.open('../Tagging/test.crfsuite')
|
||
|
|
||
|
|
||
|
id_query = "select * from recipes where id like %s"
|
||
|
|
||
|
|
||
|
def escape_md_chars(s):
|
||
|
s = s.replace("*", "\*")
|
||
|
s = s.replace("(", "\(")
|
||
|
s = s.replace(")", "\)")
|
||
|
s = s.replace("[", "\[")
|
||
|
s = s.replace("]", "\]")
|
||
|
s = s.replace("_", "\_")
|
||
|
|
||
|
return s
|
||
|
|
||
|
|
||
|
class Recipe(object):
|
||
|
def __init__(self, recipe_db_id = None):
|
||
|
|
||
|
self._sentences = None
|
||
|
self._title = None
|
||
|
self._part = None
|
||
|
self._ingredients = None
|
||
|
self._recipe_id = recipe_db_id
|
||
|
self._get_from_db()
|
||
|
self.annotate_ingredients()
|
||
|
self.annotate_sentences()
|
||
|
|
||
|
def _get_from_db(self):
|
||
|
result = DatabaseConnection.global_single_query(id_query, (self._recipe_id))
|
||
|
assert len(result) > 0
|
||
|
result = result[0]
|
||
|
self._title = result['title']
|
||
|
self._part = result['part']
|
||
|
|
||
|
raw_sentences = json.loads(result['instructions'])
|
||
|
raw_ingredients = json.loads(result['ingredients'])
|
||
|
|
||
|
# throwing the raw data through our connlu generator to annotate them right
|
||
|
cg_sents = ConlluGenerator(["\n".join(raw_sentences)])
|
||
|
cg_ings = ConlluGenerator(["\n".join(raw_ingredients)])
|
||
|
|
||
|
cg_sents.tokenize()
|
||
|
cg_sents.pos_tagging_and_lemmatization()
|
||
|
|
||
|
cg_ings.tokenize()
|
||
|
cg_ings.pos_tagging_and_lemmatization()
|
||
|
|
||
|
# TODO
|
||
|
self._sentences = cg_sents.get_conllu_elements()[0]
|
||
|
self._ingredients = cg_ings.get_conllu_elements()[0]
|
||
|
#self._sentences = json.loads(result['instructions'])
|
||
|
#self._ingredients = json.loads(result['ingredients'])
|
||
|
|
||
|
def avg_sentence_length(self):
|
||
|
return sum([len(s) for s in self._sentences])/len(self._sentences)
|
||
|
|
||
|
def n_instructions(self):
|
||
|
return len(self._sentences)
|
||
|
|
||
|
def max_sentence_length(self):
|
||
|
return max([len(s) for s in self._sentences])
|
||
|
|
||
|
def keyword_ratio(self):
|
||
|
sentence_ratios = []
|
||
|
for sent in self._sentences:
|
||
|
# FIXME: only works if there are no other misc annotations!
|
||
|
sentence_ratios.append(sum([token['misc'] is not None for token in sent]))
|
||
|
return sum(sentence_ratios) / len(sentence_ratios)
|
||
|
|
||
|
def predict_labels(self):
|
||
|
features = [sent2features(sent) for sent in self._sentences]
|
||
|
labels = [tagger.tag(feat) for feat in features]
|
||
|
return labels
|
||
|
|
||
|
def predict_ingredient_labels(self):
|
||
|
features = [sent2features(sent) for sent in self._ingredients]
|
||
|
labels = [tagger.tag(feat) for feat in features]
|
||
|
return labels
|
||
|
|
||
|
def _annotate_sentences(self, sent_token_list, predictions):
|
||
|
# test whether we predicted an label or found it in our label list
|
||
|
for i, ing in enumerate(sent_token_list):
|
||
|
for j, token in enumerate(ing):
|
||
|
lemma = token['lemma']
|
||
|
|
||
|
# check for ingredient
|
||
|
if lemma in ingredients.ingredients_stemmed:
|
||
|
token.add_misc("food_type", "ingredient")
|
||
|
elif predictions[i][j] == 'ingredient':
|
||
|
token.add_misc("food_type", "ingredient")
|
||
|
|
||
|
# check for action
|
||
|
if lemma in actions.stemmed_cooking_verbs:
|
||
|
token.add_misc("food_type", "action")
|
||
|
elif predictions[i][j] == 'action':
|
||
|
token.add_misc("food_type", "action")
|
||
|
|
||
|
# check for container
|
||
|
if lemma in containers.stemmed_containers:
|
||
|
token.add_misc("food_type", "container")
|
||
|
elif predictions[i][j] == 'container':
|
||
|
token.add_misc("food_type", "container")
|
||
|
|
||
|
# check for placeholder
|
||
|
if lemma in placeholders.stemmed_placeholders:
|
||
|
token.add_misc("food_type", "placeholder")
|
||
|
elif predictions[i][j] == 'placeholder':
|
||
|
token.add_misc("food_type", "placeholder")
|
||
|
|
||
|
def annotate_ingredients(self):
|
||
|
self._annotate_sentences(self._ingredients, self.predict_ingredient_labels())
|
||
|
|
||
|
def annotate_sentences(self):
|
||
|
self._annotate_sentences(self._sentences, self.predict_labels())
|
||
|
|
||
|
def recipe_id(self):
|
||
|
return self._recipe_id
|
||
|
|
||
|
def serialize(self):
|
||
|
result = "# newdoc\n"
|
||
|
if self._recipe_id is not None:
|
||
|
result += f"# id: {self._recipe_id}\n"
|
||
|
|
||
|
for sent in self._sentences:
|
||
|
result += f"{sent.serialize()}"
|
||
|
return result + "\n"
|
||
|
|
||
|
def display_recipe(self):
|
||
|
display(Markdown(f"## {self._title}\n({self._recipe_id})"))
|
||
|
display(Markdown(f"### Ingredients"))
|
||
|
display(Markdown("\n".join([f" * '{escape_md_chars(self.tokenlist2str(ing))}'" for ing in self._ingredients])))
|
||
|
display(Markdown(f"### Instructions"))
|
||
|
display(Markdown("\n".join([f" * {escape_md_chars(self.tokenlist2str(ins))}" for ins in self._sentences])))
|
||
|
|
||
|
def tokenlist2str(self, tokenlist):
|
||
|
return " ".join([token['form'] for token in tokenlist])
|
||
|
|
||
|
def tokenarray2str(self, tokenarray):
|
||
|
return "\n".join([self.tokenlist2str(tokenlist) for tokenlist in tokenarray])
|
||
|
|
||
|
|
||
|
def __repr__(self):
|
||
|
s = "recipe: " + (self._recipe_id if self._recipe_id else "") + "\n"
|
||
|
s += "instructions: \n"
|
||
|
for sent in self._sentences:
|
||
|
s += " ".join([token['form'] for token in sent]) + "\n"
|
||
|
|
||
|
s += "\nscores:\n"
|
||
|
s += f"avg_sent_length: {self.avg_sentence_length()}\n"
|
||
|
s += f"n_instructions: {self.n_instructions()}\n"
|
||
|
s += f"keyword_ratio: {self.keyword_ratio()}\n\n\n"
|
||
|
|
||
|
return s
|
||
|
|
||
|
|
||
|
|
||
|
|