master-thesis/RecipeAnalysis/Recipe.py

212 lines
7.0 KiB
Python
Raw Normal View History

#!/usr/bin/env python3
# coding: utf-8
# # Recipe class
import sys
sys.path.append("../")
import settings
import pycrfsuite
import json
import db.db_settings as db_settings
from db.database_connection import DatabaseConnection
from Tagging.conllu_generator import ConlluGenerator
from Tagging.crf_data_generator import *
from IPython.display import Markdown, HTML, display
# * get vocabulary
import importlib.util
# loading ingredients:
spec = importlib.util.spec_from_file_location(
"ingredients", "../" + settings.ingredients_file)
ingredients = importlib.util.module_from_spec(spec)
spec.loader.exec_module(ingredients)
# loading actions:
spec = importlib.util.spec_from_file_location(
"actions", "../" + settings.actions_file)
actions = importlib.util.module_from_spec(spec)
spec.loader.exec_module(actions)
# loading containers
spec = importlib.util.spec_from_file_location(
"containers", "../" + settings.container_file)
containers = importlib.util.module_from_spec(spec)
spec.loader.exec_module(containers)
# loading placeholders
spec = importlib.util.spec_from_file_location(
"placeholders", "../" + settings.placeholder_file)
placeholders = importlib.util.module_from_spec(spec)
spec.loader.exec_module(placeholders)
tagger = pycrfsuite.Tagger()
tagger.open('../Tagging/test.crfsuite')
id_query = "select * from recipes where id like %s"
def escape_md_chars(s):
s = s.replace("*", "\*")
s = s.replace("(", "\(")
s = s.replace(")", "\)")
s = s.replace("[", "\[")
s = s.replace("]", "\]")
s = s.replace("_", "\_")
return s
class Recipe(object):
def __init__(self, recipe_db_id = None):
self._sentences = None
self._title = None
self._part = None
self._ingredients = None
self._recipe_id = recipe_db_id
self._get_from_db()
self.annotate_ingredients()
self.annotate_sentences()
def _get_from_db(self):
result = DatabaseConnection.global_single_query(id_query, (self._recipe_id))
assert len(result) > 0
result = result[0]
self._title = result['title']
self._part = result['part']
raw_sentences = json.loads(result['instructions'])
raw_ingredients = json.loads(result['ingredients'])
# throwing the raw data through our connlu generator to annotate them right
cg_sents = ConlluGenerator(["\n".join(raw_sentences)])
cg_ings = ConlluGenerator(["\n".join(raw_ingredients)])
cg_sents.tokenize()
cg_sents.pos_tagging_and_lemmatization()
cg_ings.tokenize()
cg_ings.pos_tagging_and_lemmatization()
# TODO
self._sentences = cg_sents.get_conllu_elements()[0]
self._ingredients = cg_ings.get_conllu_elements()[0]
#self._sentences = json.loads(result['instructions'])
#self._ingredients = json.loads(result['ingredients'])
def avg_sentence_length(self):
return sum([len(s) for s in self._sentences])/len(self._sentences)
def n_instructions(self):
return len(self._sentences)
def max_sentence_length(self):
return max([len(s) for s in self._sentences])
def keyword_ratio(self):
sentence_ratios = []
for sent in self._sentences:
# FIXME: only works if there are no other misc annotations!
sentence_ratios.append(sum([token['misc'] is not None for token in sent]))
return sum(sentence_ratios) / len(sentence_ratios)
def predict_labels(self):
features = [sent2features(sent) for sent in self._sentences]
labels = [tagger.tag(feat) for feat in features]
return labels
def predict_ingredient_labels(self):
features = [sent2features(sent) for sent in self._ingredients]
labels = [tagger.tag(feat) for feat in features]
return labels
def _annotate_sentences(self, sent_token_list, predictions):
# test whether we predicted an label or found it in our label list
for i, ing in enumerate(sent_token_list):
for j, token in enumerate(ing):
lemma = token['lemma']
# check for ingredient
if lemma in ingredients.ingredients_stemmed:
token.add_misc("food_type", "ingredient")
elif predictions[i][j] == 'ingredient':
token.add_misc("food_type", "ingredient")
# check for action
if lemma in actions.stemmed_cooking_verbs:
token.add_misc("food_type", "action")
elif predictions[i][j] == 'action':
token.add_misc("food_type", "action")
# check for container
if lemma in containers.stemmed_containers:
token.add_misc("food_type", "container")
elif predictions[i][j] == 'container':
token.add_misc("food_type", "container")
# check for placeholder
if lemma in placeholders.stemmed_placeholders:
token.add_misc("food_type", "placeholder")
elif predictions[i][j] == 'placeholder':
token.add_misc("food_type", "placeholder")
def annotate_ingredients(self):
self._annotate_sentences(self._ingredients, self.predict_ingredient_labels())
def annotate_sentences(self):
self._annotate_sentences(self._sentences, self.predict_labels())
def recipe_id(self):
return self._recipe_id
def serialize(self):
result = "# newdoc\n"
if self._recipe_id is not None:
result += f"# id: {self._recipe_id}\n"
for sent in self._sentences:
result += f"{sent.serialize()}"
return result + "\n"
def display_recipe(self):
display(Markdown(f"## {self._title}\n({self._recipe_id})"))
display(Markdown(f"### Ingredients"))
display(Markdown("\n".join([f" * '{escape_md_chars(self.tokenlist2str(ing))}'" for ing in self._ingredients])))
display(Markdown(f"### Instructions"))
display(Markdown("\n".join([f" * {escape_md_chars(self.tokenlist2str(ins))}" for ins in self._sentences])))
def tokenlist2str(self, tokenlist):
return " ".join([token['form'] for token in tokenlist])
def tokenarray2str(self, tokenarray):
return "\n".join([self.tokenlist2str(tokenlist) for tokenlist in tokenarray])
def __repr__(self):
s = "recipe: " + (self._recipe_id if self._recipe_id else "") + "\n"
s += "instructions: \n"
for sent in self._sentences:
s += " ".join([token['form'] for token in sent]) + "\n"
s += "\nscores:\n"
s += f"avg_sent_length: {self.avg_sentence_length()}\n"
s += f"n_instructions: {self.n_instructions()}\n"
s += f"keyword_ratio: {self.keyword_ratio()}\n\n\n"
return s