#!/usr/bin/env python3 # coding: utf-8 # # Recipe class import sys sys.path.append("../") import settings import pycrfsuite import json import db.db_settings as db_settings from db.database_connection import DatabaseConnection from Tagging.conllu_generator import ConlluGenerator from Tagging.crf_data_generator import * from IPython.display import Markdown, HTML, display # * get vocabulary import importlib.util # loading ingredients: spec = importlib.util.spec_from_file_location( "ingredients", "../" + settings.ingredients_file) ingredients = importlib.util.module_from_spec(spec) spec.loader.exec_module(ingredients) # loading actions: spec = importlib.util.spec_from_file_location( "actions", "../" + settings.actions_file) actions = importlib.util.module_from_spec(spec) spec.loader.exec_module(actions) # loading containers spec = importlib.util.spec_from_file_location( "containers", "../" + settings.container_file) containers = importlib.util.module_from_spec(spec) spec.loader.exec_module(containers) # loading placeholders spec = importlib.util.spec_from_file_location( "placeholders", "../" + settings.placeholder_file) placeholders = importlib.util.module_from_spec(spec) spec.loader.exec_module(placeholders) tagger = pycrfsuite.Tagger() tagger.open('../Tagging/test.crfsuite') id_query = "select * from recipes where id like %s" def escape_md_chars(s): s = s.replace("*", "\*") s = s.replace("(", "\(") s = s.replace(")", "\)") s = s.replace("[", "\[") s = s.replace("]", "\]") s = s.replace("_", "\_") return s class Recipe(object): def __init__(self, recipe_db_id = None): self._sentences = None self._title = None self._part = None self._ingredients = None self._recipe_id = recipe_db_id self._get_from_db() self.annotate_ingredients() self.annotate_sentences() def _get_from_db(self): result = DatabaseConnection.global_single_query(id_query, (self._recipe_id)) assert len(result) > 0 result = result[0] self._title = result['title'] self._part = result['part'] raw_sentences = json.loads(result['instructions']) raw_ingredients = json.loads(result['ingredients']) # throwing the raw data through our connlu generator to annotate them right cg_sents = ConlluGenerator(["\n".join(raw_sentences)]) cg_ings = ConlluGenerator(["\n".join(raw_ingredients)]) cg_sents.tokenize() cg_sents.pos_tagging_and_lemmatization() cg_ings.tokenize() cg_ings.pos_tagging_and_lemmatization() # TODO self._sentences = cg_sents.get_conllu_elements()[0] self._ingredients = cg_ings.get_conllu_elements()[0] #self._sentences = json.loads(result['instructions']) #self._ingredients = json.loads(result['ingredients']) def avg_sentence_length(self): return sum([len(s) for s in self._sentences])/len(self._sentences) def n_instructions(self): return len(self._sentences) def max_sentence_length(self): return max([len(s) for s in self._sentences]) def keyword_ratio(self): sentence_ratios = [] for sent in self._sentences: # FIXME: only works if there are no other misc annotations! sentence_ratios.append(sum([token['misc'] is not None for token in sent])) return sum(sentence_ratios) / len(sentence_ratios) def predict_labels(self): features = [sent2features(sent) for sent in self._sentences] labels = [tagger.tag(feat) for feat in features] return labels def predict_ingredient_labels(self): features = [sent2features(sent) for sent in self._ingredients] labels = [tagger.tag(feat) for feat in features] return labels def _annotate_sentences(self, sent_token_list, predictions): # test whether we predicted an label or found it in our label list for i, ing in enumerate(sent_token_list): for j, token in enumerate(ing): lemma = token['lemma'] # check for ingredient if lemma in ingredients.ingredients_stemmed: token.add_misc("food_type", "ingredient") elif predictions[i][j] == 'ingredient': token.add_misc("food_type", "ingredient") # check for action if lemma in actions.stemmed_cooking_verbs: token.add_misc("food_type", "action") elif predictions[i][j] == 'action': token.add_misc("food_type", "action") # check for container if lemma in containers.stemmed_containers: token.add_misc("food_type", "container") elif predictions[i][j] == 'container': token.add_misc("food_type", "container") # check for placeholder if lemma in placeholders.stemmed_placeholders: token.add_misc("food_type", "placeholder") elif predictions[i][j] == 'placeholder': token.add_misc("food_type", "placeholder") def annotate_ingredients(self): self._annotate_sentences(self._ingredients, self.predict_ingredient_labels()) def annotate_sentences(self): self._annotate_sentences(self._sentences, self.predict_labels()) def recipe_id(self): return self._recipe_id def serialize(self): result = "# newdoc\n" if self._recipe_id is not None: result += f"# id: {self._recipe_id}\n" for sent in self._sentences: result += f"{sent.serialize()}" return result + "\n" def display_recipe(self): display(Markdown(f"## {self._title}\n({self._recipe_id})")) display(Markdown(f"### Ingredients")) display(Markdown("\n".join([f" * '{escape_md_chars(self.tokenlist2str(ing))}'" for ing in self._ingredients]))) display(Markdown(f"### Instructions")) display(Markdown("\n".join([f" * {escape_md_chars(self.tokenlist2str(ins))}" for ins in self._sentences]))) def tokenlist2str(self, tokenlist): return " ".join([token['form'] for token in tokenlist]) def tokenarray2str(self, tokenarray): return "\n".join([self.tokenlist2str(tokenlist) for tokenlist in tokenarray]) def __repr__(self): s = "recipe: " + (self._recipe_id if self._recipe_id else "") + "\n" s += "instructions: \n" for sent in self._sentences: s += " ".join([token['form'] for token in sent]) + "\n" s += "\nscores:\n" s += f"avg_sent_length: {self.avg_sentence_length()}\n" s += f"n_instructions: {self.n_instructions()}\n" s += f"keyword_ratio: {self.keyword_ratio()}\n\n\n" return s