jonas/nlp-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of ssh://the-cake-is-a-lie.net:20022/jonas/NLP-LAB

Browse Source
This commit is contained in:
Carsten 2018-05-29 13:20:50 +02:00
commit 05f432fd41
12 changed files with 3849 additions and 250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
Jonas_Solutions/Task03_Instructions.py Normal file
View File

@ -0,0 +1,84 @@
import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D
from sklearn.model_selection import train_test_split
from keras.utils.np_utils import to_categorical
import re
'''
Task 3: playing with NN framwork/keras and basic sentiment analysis
- use the following model as a baseline and improve it!
- export your metadata (just basic hyperparameters and outcomes for test data!)
- test data = 0.3 (not in this example, change it!)
- random_state = 4222
- no need to cross-validation!
'''
# parameters
max_fatures = 500
embed_dim = 128
lstm_out = 196
dropout = 0.1
dropout_1d = 0.4
recurrent_dropout = 0.1
random_state = 1324
validation_size = 1000
batch_size = 16
epochs=2
verbose= 2
df = pd.read_csv('dataset_sentiment.csv')
df = df[['text','sentiment']]
print(df[0:10])
df = df[df.sentiment != "Neutral"]
df['text'] = df['text'].apply(lambda x: x.lower())
df['text'] = df['text'].apply(lambda x: x.replace('rt',' '))
df['text'] = df['text'].apply((lambda x: re.sub('[^a-zA-z0-9\s]','',x)))
tok = Tokenizer(num_words=max_fatures, split=' ')
tok.fit_on_texts(df['text'].values)
X = tok.texts_to_sequences(df['text'].values)
X = pad_sequences(X)
nn = Sequential()
nn.add(Embedding(max_fatures, embed_dim, input_length = X.shape[1]))
nn.add(SpatialDropout1D(dropout_1d))
nn.add(LSTM(lstm_out, dropout=dropout, recurrent_dropout=recurrent_dropout))
nn.add(Dense(2, activation='softmax'))
nn.compile(loss = 'categorical_crossentropy', optimizer='adam', metrics = ['accuracy'])
print(nn.summary())
Y = pd.get_dummies(df['sentiment']).values
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.30, random_state = random_state)
nn.fit(X_train, Y_train, epochs = epochs, batch_size=batch_size, verbose=verbose)
X_validate = X_test[-validation_size:]
Y_validate = Y_test[-validation_size:]
X_test = X_test[:-validation_size]
Y_test = Y_test[:-validation_size]
score, accuracy = nn.evaluate(X_test, Y_test, verbose = 2, batch_size = batch_size)
print("score: %.2f" % (score))
print("acc: %.2f" % (accuracy))
pos_cnt, neg_cnt, pos_ok, neg_ok = 0, 0, 0, 0
for x in range(len(X_validate)):
result = nn.predict(X_validate[x].reshape(1,X_test.shape[1]),batch_size=1,verbose = 2)[0]
if np.argmax(result) == np.argmax(Y_validate[x]):
if np.argmax(Y_validate[x]) == 0: neg_ok += 1
else: pos_ok += 1
if np.argmax(Y_validate[x]) == 0: neg_cnt += 1
else: pos_cnt += 1
print("pos_acc", pos_ok/pos_cnt*100, "%")
print("neg_acc", neg_ok/neg_cnt*100, "%")
X2 = ['what are u going to say about that? the truth, wassock?!']
X2 = tok.texts_to_sequences(X2)
X2 = pad_sequences(X2, maxlen=26, dtype='int32', value=0)
print(X2)
print(nn.predict(X2, batch_size=1, verbose = 2)[0])

586
Jonas_Solutions/Task_02_JonasWeinz.ipynb
View File

File diff suppressed because one or more lines are too long

316
Jonas_Solutions/Task_03.ipynb Normal file
View File

@ -0,0 +1,316 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/jonas/.local/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
" from ._conv import register_converters as _register_converters\n",
"Using TensorFlow backend.\n"
]
}
],
"source": [
"import numpy as np \n",
"import pandas as pd \n",
"from sklearn.feature_extraction.text import CountVectorizer\n",
"from keras.preprocessing.text import Tokenizer\n",
"from keras.preprocessing.sequence import pad_sequences\n",
"from keras.models import Sequential\n",
"from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D\n",
"from sklearn.model_selection import train_test_split\n",
"from keras.utils.np_utils import to_categorical\n",
"import re\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
">**Task 3**: playing with NN framwork/keras and basic sentiment analysis\n",
">- use the following model as a baseline and improve it!\n",
">- export your metadata (just basic hyperparameters and outcomes for test data!)\n",
">- test data = 0.3 (not in this example, change it!)\n",
">- random_state = 4222\n",
">- no need to cross-validation!\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"dataset already downloaded\n"
]
}
],
"source": [
"%%bash\n",
"\n",
"if [ ! -e 'dataset_sentiment.csv' ]\n",
"then\n",
" echo \"downloading dataset\"\n",
" wget https://raw.githubusercontent.com/SmartDataAnalytics/MA-INF-4222-NLP-Lab/master/2018_SoSe/exercises/dataset_sentiment.csv\n",
"else\n",
" echo \"dataset already downloaded\"\n",
"fi"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# parameters\n",
"max_fatures = 500\n",
"embed_dim = 128\n",
"lstm_out = 196\n",
"dropout = 0.1\n",
"dropout_1d = 0.4\n",
"recurrent_dropout = 0.1\n",
"random_state = 1324\n",
"validation_size = 1000\n",
"batch_size = 16\n",
"epochs=2\n",
"verbose= 2"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" text sentiment\n",
"0 RT @NancyLeeGrahn: How did everyone feel about... Neutral\n",
"1 RT @ScottWalker: Didn't catch the full #GOPdeb... Positive\n",
"2 RT @TJMShow: No mention of Tamir Rice and the ... Neutral\n",
"3 RT @RobGeorge: That Carly Fiorina is trending ... Positive\n",
"4 RT @DanScavino: #GOPDebate w/ @realDonaldTrump... Positive\n",
"5 RT @GregAbbott_TX: @TedCruz: \"On my first day ... Positive\n",
"6 RT @warriorwoman91: I liked her and was happy ... Negative\n",
"7 Going on #MSNBC Live with @ThomasARoberts arou... Neutral\n",
"8 Deer in the headlights RT @lizzwinstead: Ben C... Negative\n",
"9 RT @NancyOsborne180: Last night's debate prove... Negative\n"
]
}
],
"source": [
"df = pd.read_csv('dataset_sentiment.csv')\n",
"df = df[['text','sentiment']]\n",
"print(df[0:10])\n",
"\n",
"df = df[df.sentiment != \"Neutral\"]\n",
"df['text'] = df['text'].apply(lambda x: x.lower())\n",
"df['text'] = df['text'].apply(lambda x: x.replace('rt',' '))\n",
"df['text'] = df['text'].apply((lambda x: re.sub('[^a-zA-Z0-9\\s]','',x)))"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"tok = Tokenizer(num_words=max_fatures, split=' ')\n",
"tok.fit_on_texts(df['text'].values)\n",
"X = tok.texts_to_sequences(df['text'].values)\n",
"X = pad_sequences(X)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"embedding_2 (Embedding) (None, 26, 128) 64000 \n",
"_________________________________________________________________\n",
"spatial_dropout1d_2 (Spatial (None, 26, 128) 0 \n",
"_________________________________________________________________\n",
"lstm_2 (LSTM) (None, 196) 254800 \n",
"_________________________________________________________________\n",
"dense_2 (Dense) (None, 2) 394 \n",
"=================================================================\n",
"Total params: 319,194\n",
"Trainable params: 319,194\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n",
"None\n"
]
}
],
"source": [
"nn = Sequential()\n",
"nn.add(Embedding(max_fatures, embed_dim, input_length = X.shape[1]))\n",
"nn.add(SpatialDropout1D(dropout_1d))\n",
"nn.add(LSTM(lstm_out, dropout=dropout, recurrent_dropout=recurrent_dropout))\n",
"nn.add(Dense(2, activation='softmax'))\n",
"nn.compile(loss = 'categorical_crossentropy', optimizer='adam', metrics = ['accuracy'])\n",
"print(nn.summary())"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/2\n",
" - 21s - loss: 0.4322 - acc: 0.8196\n",
"Epoch 2/2\n",
" - 25s - loss: 0.3612 - acc: 0.8509\n"
]
},
{
"data": {
"text/plain": [
"<keras.callbacks.History at 0x7f0646b9e828>"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"Y = pd.get_dummies(df['sentiment']).values\n",
"X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.30, random_state = random_state)\n",
"nn.fit(X_train, Y_train, epochs = epochs, batch_size=batch_size, verbose=verbose)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"score: 0.37\n",
"acc: 0.84\n"
]
}
],
"source": [
"X_validate = X_test[-validation_size:]\n",
"Y_validate = Y_test[-validation_size:]\n",
"X_test = X_test[:-validation_size]\n",
"Y_test = Y_test[:-validation_size]\n",
"score, accuracy = nn.evaluate(X_test, Y_test, verbose = 2, batch_size = batch_size)\n",
"print(\"score: %.2f\" % (score))\n",
"print(\"acc: %.2f\" % (accuracy))"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"pos_cnt, neg_cnt, pos_ok, neg_ok = 0, 0, 0, 0\n",
"for x in range(len(X_validate)):\n",
" result = nn.predict(X_validate[x].reshape(1,X_test.shape[1]),batch_size=1,verbose = 2)[0]\n",
" if np.argmax(result) == np.argmax(Y_validate[x]):\n",
" if np.argmax(Y_validate[x]) == 0: neg_ok += 1\n",
" else: pos_ok += 1\n",
" if np.argmax(Y_validate[x]) == 0: neg_cnt += 1\n",
" else: pos_cnt += 1"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"pos_acc 39.58333333333333 %\n",
"neg_acc 95.29702970297029 %\n"
]
}
],
"source": [
"print(\"pos_acc\", pos_ok/pos_cnt*100, \"%\")\n",
"print(\"neg_acc\", neg_ok/neg_cnt*100, \"%\")"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 48 37\n",
" 311 189 4 144 22 16 1 281]]\n",
"[0.8364928 0.16350722]\n"
]
}
],
"source": [
"X2 = ['what are u going to say about that? the truth, wassock?!']\n",
"X2 = tok.texts_to_sequences(X2)\n",
"X2 = pad_sequences(X2, maxlen=26, dtype='int32', value=0)\n",
"print(X2)\n",
"print(nn.predict(X2, batch_size=1, verbose = 2)[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

94
Jonas_Solutions/output.ttl Normal file
View File

@ -0,0 +1,94 @@
@prefix dct: <http://purl.org/dc/terms/> .
@prefix mexalgo: <http://mex.aksw.org/mex-algo#> .
@prefix mexcore: <http://mex.aksw.org/mex-core#> .
@prefix mexperf: <http://mex.aksw.org/mex-perf#> .
@prefix prov: <http://www.w3.org/ns/prov#> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix this: <http://mex.aksw.org/examples/> .
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
this:measure1 a mexcore:PerformanceMeasure ;
mexperf:accuracy "0.8478535353535354"^^xsd:float ;
mexperf:precision "0.8225446428571429"^^xsd:float ;
mexperf:recall "0.8998778998778999"^^xsd:float ;
prov:wasGeneratedBy this:conf1 .
this:measure2 a mexcore:PerformanceMeasure ;
mexperf:accuracy "0.5776805251641138"^^xsd:float ;
mexperf:precision "0.5369458128078818"^^xsd:float ;
mexperf:recall "0.5240384615384616"^^xsd:float ;
prov:wasGeneratedBy this:conf2 .
this:measure3a a mexcore:PerformanceMeasure ;
mexperf:accuracy "0.5391414141414141"^^xsd:float ;
mexperf:precision "0.5341130604288499"^^xsd:float ;
mexperf:recall "0.3581699346405229"^^xsd:float ;
prov:wasGeneratedBy this:conf3a .
this:measure3b a mexcore:PerformanceMeasure ;
mexperf:accuracy "0.5142231947483589"^^xsd:float ;
mexperf:precision "0.25"^^xsd:float ;
mexperf:recall "0.03365384615384615"^^xsd:float ;
prov:wasGeneratedBy this:conf3b .
this:measure4 a mexcore:PerformanceMeasure ;
mexperf:accuracy "0.7714856762158561"^^xsd:float ;
mexperf:precision "0.7680865449628127"^^xsd:float ;
mexperf:recall "0.7680865449628127"^^xsd:float ;
prov:wasGeneratedBy this:conf4 .
this:Dataset03 a mexcore:dataset ;
rdfs:label "Dataset03" .
this:conf1 a mexcore:ExperimentConfiguration ;
rdfs:label "conf1" ;
prov:used this:Dataset01,
this:model_a ;
prov:wasStartedBy this:jonas_weinz_task_2 .
this:conf2 a mexcore:ExperimentConfiguration ;
rdfs:label "conf2" ;
prov:used this:Dataset02,
this:model_b ;
prov:wasStartedBy this:jonas_weinz_task_2 .
this:conf3a a mexcore:ExperimentConfiguration ;
rdfs:label "conf3a" ;
prov:used this:Dataset01,
this:model_b ;
prov:wasStartedBy this:jonas_weinz_task_2 .
this:conf3b a mexcore:ExperimentConfiguration ;
rdfs:label "conf3b" ;
prov:used this:Dataset02,
this:model_a ;
prov:wasStartedBy this:jonas_weinz_task_2 .
this:conf4 a mexcore:ExperimentConfiguration ;
rdfs:label "conf4" ;
prov:used this:Dataset03,
this:model_c ;
prov:wasStartedBy this:jonas_weinz_task_2 .
this:model_c a mexalgo:Algorithm ;
rdfs:label "model_c" ;
dct:identifier "MLPClassifier" .
this:Dataset01 a mexcore:dataset ;
rdfs:label "Dataset01" .
this:Dataset02 a mexcore:dataset ;
rdfs:label "Dataset02" .
this:model_a a mexalgo:Algorithm ;
rdfs:label "model_a" ;
dct:identifier "RandomForestClassifier" .
this:model_b a mexalgo:Algorithm ;
rdfs:label "model_b" ;
dct:identifier "MLPClassifier" .
this:jonas_weinz_task_2 a mexcore:Experiment .

423
Project/Tools/emoji tester old.ipynb Normal file
View File

@ -0,0 +1,423 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from IPython.display import clear_output, Markdown, Math\n",
"import ipywidgets as widgets\n",
"import os"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"----\n",
"## file input stuff:\n",
"\n",
"* replace `test.txt` with yout whatsapp log file"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%bash\n",
"./whatsapp2csv.sh test.txt"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* read table `test.csv` exported by `whatsapp2csv.sh`"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"messages = pd.read_csv('test.txt.csv', delimiter='\\t')\n",
"messages.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* read emoji-data (can be found here: https://www.unicode.org/Public/emoji/11.0/emoji-data.txt) and generate a table file out of it"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%bash\n",
"if [ ! -e emoji-data.txt ]\n",
"then\n",
" echo \"downloading emoji specification\"\n",
" wget https://www.unicode.org/Public/emoji/11.0/emoji-data.txt\n",
"else\n",
" echo \"found existing emoji specification\"\n",
"fi\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"emoji_blacklist = set([\n",
" 0x1F3FB,\n",
" 0x1F3FC,\n",
" 0x1F3FD,\n",
" 0x1F3FE,\n",
" 0x1F3FF\n",
"])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"emoji_data = pd.read_csv('emoji-data.txt', delimiter=';', comment='#', names=[\"unicode\",\"type\"])\n",
"emoji_data['type'] = emoji_data['type'].str.strip()\n",
"emoji_data = emoji_data[emoji_data['type'] == \"Emoji_Presentation\"]\n",
"emoji_data"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* now build a set out of the unicode types"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"ord(\"😀\") == int('0x1f600',16)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"emoji_codes = emoji_data['unicode']\n",
"emoji_codes.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* we have to iterate over the whole list and extract all given ranges:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"emoji_code_list = []\n",
"for entry in emoji_codes:\n",
" # testing whether we have an entry or a range:\n",
" if '.' in entry:\n",
" # range\n",
" a,b = entry.split(\"..\")\n",
" for i in range(int(a,16),int(b,16) +1):\n",
" if i not in emoji_blacklist:\n",
" emoji_code_list.append(i)\n",
" else:\n",
" # single entry\n",
" if i not in emoji_blacklist:\n",
" emoji_code_list.append(int(entry,16))\n",
"emoji_code_set = set(emoji_code_list)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# simple test:\n",
"print(ord(\"😀\") in emoji_code_set, ord(\"a\") in emoji_code_set)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* expanding column and fill new emojis"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"messages[\"emojis\"] = None"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"for i in messages.index:\n",
" emoji_list = []\n",
" to_remove = []\n",
" m = messages.iloc[i]['message']\n",
" for c in str(m):\n",
" if ord(c) in emoji_code_set:\n",
" emoji_list.append(c)\n",
" elif ord(c) in emoji_blacklist:\n",
" to_remove.append(c)\n",
" \n",
" messages.loc[i,'emojis'] = emoji_list\n",
" #remove emiójis from message\n",
" for e in (emoji_list + to_remove):\n",
" m = m.replace(e,\"\")\n",
" messages.loc[i,'message'] = m\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"messages[:20]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* get a list only containing messaged with emojis"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"emoji_messages = messages[[True if len(e) > 0 else False for e in messages['emojis']]]\n",
"emoji_messages = emoji_messages[emoji_messages['message'] != \"\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"display(emoji_messages)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"----\n",
"## learning part"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import itertools\n",
"import sklearn.utils as sku\n",
"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
"from sklearn.model_selection import train_test_split"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"labels=[e[-1] for e in emoji_messages['emojis']]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"labels[:10]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"X1, Xt1, y1, yt1 = train_test_split(emoji_messages['message'], labels, test_size=0.1, random_state=4222)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"vectorizer = TfidfVectorizer(stop_words='english')\n",
"vec_train = vectorizer.fit_transform(X1)\n",
"vec_test = vectorizer.transform(Xt1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.ensemble import RandomForestClassifier as RFC\n",
"from sklearn.neural_network import MLPClassifier as MLP\n",
"from sklearn.naive_bayes import MultinomialNB as MNB\n",
"#clf_a = RFC(criterion='entropy', random_state=4222)\n",
"clf_a = MLP()\n",
"clf_a.fit(vec_train, y1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pred = clf_a.predict(vectorizer.transform(Xt1))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"testlist = pd.DataFrame({'message': Xt1, 'pred': pred, 'trained': yt1})\n",
"testlist = pd.merge(testlist, emoji_messages['emojis'].to_frame(), left_index=True, right_index=True)\n",
"testlist.to_csv('export.csv')\n",
"testlist"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"display(clf_a.predict(vectorizer.transform([\"Boah Caner\"]))[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(chr(0x1F3F))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"vec_train[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"\n",
"out = widgets.Output()\n",
"\n",
"t = widgets.Text()\n",
"b = widgets.Button(\n",
" description='get smiley',\n",
" disabled=False,\n",
" button_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
" tooltip='Click me',\n",
" icon='check'\n",
")\n",
"\n",
"\n",
"\n",
"def handle_submit(sender):\n",
" with out:\n",
" clear_output()\n",
" with out:\n",
" display(Markdown(\"# \" + str(clf_a.predict(vectorizer.transform([t.value]))[0])))\n",
"\n",
"b.on_click(handle_submit)\n",
" \n",
"display(t)\n",
"display(widgets.VBox([b, out])) "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

676
Project/Tools/emoji tester.ipynb Normal file
View File

@ -0,0 +1,676 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from IPython.display import clear_output, Markdown, Math\n",
"import ipywidgets as widgets\n",
"import os"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def create_widgets(t_text, b_text, out, additional_widgets=[]):\n",
" texts = []\n",
" for t in t_text:\n",
" texts.append(widgets.Text(t))\n",
" \n",
" button = widgets.Button(\n",
" description=b_text,\n",
" disabled=False,\n",
" button_style='', # 'success', 'info', 'warning', 'danger' or ''\n",
" tooltip=b_text,\n",
" icon='check'\n",
" )\n",
" display(widgets.VBox([widgets.HBox(texts + additional_widgets + [button]), out]))\n",
" return texts + [button]\n",
"\n",
"out_convert = widgets.Output()\n",
"out_build = widgets.Output()\n",
"out_train = widgets.Output()\n",
"out_save = widgets.Output()\n",
"out_read = widgets.Output()\n",
"out_test = widgets.Output()\n",
"\n",
"def mp(msg):\n",
" display(Markdown(msg))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Emoji Tester\n",
"\n",
"just run all cells at first. Then select on of the actions below."
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"## converting plain whatsapp export to csv"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "8c94a8d3b3724ad08359817b2086cf85",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Text(value='test.txt'), Button(description='convert whatsapp file to csv', icon='check', style=ButtonStyle(), tooltip='convert whatsapp file to csv'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"## read csv and build database"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ee22e300097e49f1ac24b11662f7dc69",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Text(value='test.txt.csv'), Checkbox(value=False, description='using only last emoji'), Button(description='read', icon='check', style=ButtonStyle(), tooltip='read'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"## Train"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "de06136ee346492d80ef39d304cbc31c",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Dropdown(description='Learning Method', index=1, options=('DecisionTree', 'MLP', 'RandomForest'), value='MLP'), Checkbox(value=False, description='Using one vs all (very slow, only with multi-label!)'), Button(description='train', icon='check', style=ButtonStyle(), tooltip='train'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"## save trained classifier"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b046f3ada7ec4ba7a17965d718552d21",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Text(value='clf.pkl'), Text(value='mlb.pkl'), Text(value='vectorizer.pkl'), Button(description='save classifier', icon='check', style=ButtonStyle(), tooltip='save classifier'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"## import trained classifier"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ac4054e75d8a4214ad040b1b21d6c925",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Text(value='clf.pkl'), Text(value='mlb.pkl'), Text(value='vectorizer.pkl'), Button(description='import classifier', icon='check', style=ButtonStyle(), tooltip='import classifier'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/markdown": [
"## predict emoji on custom text"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "db631acbe9c94cac907efaf501a69c6a",
"version_major": 2,
"version_minor": 0
},
"text/html": [
"<p>Failed to display Jupyter Widget of type <code>VBox</code>.</p>\n",
"<p>\n",
" If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
" that the widgets JavaScript is still loading. If this message persists, it\n",
" likely means that the widgets JavaScript library is either not installed or\n",
" not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
" Widgets Documentation</a> for setup instructions.\n",
"</p>\n",
"<p>\n",
" If you're reading this message in another frontend (for example, a static\n",
" rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
" it may mean that your frontend doesn't currently support widgets.\n",
"</p>\n"
],
"text/plain": [
"VBox(children=(HBox(children=(Text(value=''), Checkbox(value=False, description='Show probabilities (only on trees)'), Button(description='get emoji', icon='check', style=ButtonStyle(), tooltip='get emoji'))), Output()))"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"mp(\"## converting plain whatsapp export to csv\")\n",
"t_convert, b_convert = create_widgets([\"test.txt\"], \"convert whatsapp file to csv\", out_convert)\n",
"mp(\"## read csv and build database\")\n",
"single_label = widgets.Checkbox(value=False, description='using only last emoji', disable=False)\n",
"t_build, b_build = create_widgets([\"test.txt.csv\"], \"read\", out_build, [single_label])\n",
"mp(\"## Train\")\n",
"d = widgets.Dropdown(options=['DecisionTree', 'MLP', 'RandomForest'], value='MLP', description='Learning Method', disabled=False)\n",
"ova = widgets.Checkbox(value=False, description='Using one vs all (very slow, only with multi-label!)', disabled=False)\n",
"b_train = button = widgets.Button(description=\"train\", disabled=False, button_style='', tooltip=\"train\",icon='check')\n",
"display(widgets.VBox([widgets.HBox([d,ova,b_train]), out_train]))\n",
"mp(\"## save trained classifier\")\n",
"t_save_c, t_save_m, t_save_v, b_save = create_widgets([\"clf.pkl\", \"mlb.pkl\", \"vectorizer.pkl\"], \"save classifier\", out_save)\n",
"mp(\"## import trained classifier\")\n",
"t_read_c, t_read_m, t_read_v, b_read = create_widgets([\"clf.pkl\", \"mlb.pkl\", \"vectorizer.pkl\"], \"import classifier\", out_read)\n",
"mp(\"## predict emoji on custom text\")\n",
"b_prop = widgets.Checkbox(value=False, description='Show probabilities (only on trees)', disabled=False)\n",
"t_test, b_test = create_widgets([\"\"], \"get emoji\", out_test,[b_prop])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"----\n",
"## Code Section:"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"def convert(b):\n",
" with out_convert:\n",
" clear_output()\n",
" with out_convert:\n",
" mp(\"**converting \" + t_convert.value + \"…**\")\n",
" import subprocess\n",
" print(str(subprocess.check_output([\"./whatsapp2csv.sh\", t_convert.value])).strip())\n",
" mp(\"**done**\")\n",
"\n",
"b_convert.on_click(convert)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* download emoji specification if not already existing"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"found existing emoji specification\n"
]
}
],
"source": [
"%%bash\n",
"if [ ! -e emoji-data.txt ]\n",
"then\n",
" echo \"downloading emoji specification\"\n",
" wget https://www.unicode.org/Public/emoji/11.0/emoji-data.txt\n",
"else\n",
" echo \"found existing emoji specification\"\n",
"fi\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* stuff for creating emoji database"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"emoji_blacklist = set([\n",
" 0x1F3FB,\n",
" 0x1F3FC,\n",
" 0x1F3FD,\n",
" 0x1F3FE,\n",
" 0x1F3FF,\n",
" 0x2642,\n",
" 0x2640\n",
"])\n",
"\n",
"emoji_code_set = None\n",
"\n",
"def create_emoji_set():\n",
" global emoji_code_set\n",
" \n",
" emoji_data = pd.read_csv('emoji-data.txt', delimiter=';', comment='#', names=[\"unicode\",\"type\"])\n",
" emoji_data['type'] = emoji_data['type'].str.strip()\n",
" emoji_data = emoji_data[emoji_data['type'] == \"Emoji_Presentation\"]\n",
" \n",
" emoji_codes = emoji_data['unicode']\n",
" emoji_codes.head()\n",
" \n",
" emoji_code_list = []\n",
" for entry in emoji_codes:\n",
" # testing whether we have an entry or a range:\n",
" if '.' in entry:\n",
" # range\n",
" a,b = entry.split(\"..\")\n",
" for i in range(int(a,16),int(b,16) +1):\n",
" if i not in emoji_blacklist:\n",
" emoji_code_list.append(i)\n",
" else:\n",
" # single entry\n",
" if i not in emoji_blacklist:\n",
" emoji_code_list.append(int(entry,16))\n",
" emoji_code_set = set(emoji_code_list)\n",
" display(Markdown(\"**imported Emojis** (without modifier):\\n>\" + \"\".join([chr(x) for x in emoji_code_set])))\n",
" display(Markdown(\"**blacklisted Emojis:**\\n>\" + \"\".join([chr(x) for x in emoji_blacklist])))\n",
" f = open('emoji-list.txt', 'w')\n",
" for e in emoji_code_set:\n",
" f.write(chr(e) + \"\\n\")\n",
" f.close()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"* stuff for reading whatsapp messages"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"messages = None\n",
"vectorizer = None\n",
"clf_a = None\n",
"mlb = None\n",
"\n",
"emoji_messages=None"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"def read_message_and_build_db(filename):\n",
" global messages\n",
" global emoji_messages\n",
" global vectorizer\n",
" global clf_a\n",
" global mlb\n",
" \n",
" messages = pd.read_csv(filename, delimiter='\\t')\n",
" mp(\"**filter messages and creating labels. This can take a while...**\")\n",
" messages[\"emojis\"] = None\n",
" \n",
" msg_batchsize = 1000\n",
" msg_counter = 0\n",
" \n",
" for i in messages.index:\n",
" \n",
" msg_counter+=1\n",
" if msg_counter >= msg_batchsize:\n",
" print(str(100 * i / messages.shape[0]) + \"%\")\n",
" msg_counter=0\n",
" \n",
" emoji_list = []\n",
" m = messages.iloc[i]['message']\n",
" m_new = \"\"\n",
" for c in str(m):\n",
" if ord(c) in emoji_code_set:\n",
" emoji_list.append(c)\n",
" elif ord(c) not in emoji_blacklist:\n",
" m_new += c\n",
" # if single label: only use last found emoji\n",
" messages.loc[i,'emojis'] = set(emoji_list) if (not single_label.value) or len(emoji_list)==0 else set(emoji_list[-1])\n",
" #remove emiójis from message\n",
" messages.loc[i,'message'] = m_new\n",
" \n",
" emoji_messages = messages[[True if len(e) > 0 else False for e in messages['emojis']]]\n",
" emoji_messages = emoji_messages[emoji_messages['message'] != \"\"]\n",
" \n",
" mp(\"**Done**\")\n",
" \n",
" display(emoji_messages)\n",
"\n",
"def train(b):\n",
" global messages\n",
" global emoji_messages\n",
" global vectorizer\n",
" global clf_a\n",
" global mlb\n",
" with out_train:\n",
" clear_output()\n",
" # train part:\n",
" import numpy as np\n",
" import itertools\n",
" import sklearn.utils as sku\n",
" from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, HashingVectorizer\n",
" from sklearn.model_selection import train_test_split\n",
" from sklearn.preprocessing import MultiLabelBinarizer\n",
"\n",
" mlb = MultiLabelBinarizer() if not single_label.value else None\n",
" \n",
" if not mlb:\n",
" l = [list(e)[-1] for e in emoji_messages['emojis']]\n",
" \n",
" labels=mlb.fit_transform(emoji_messages['emojis']) if mlb else l\n",
" \n",
" if mlb:\n",
" display(Markdown(\"**emojis contained in Dataset:**\\n >\" + \"\".join(mlb.classes_ )))\n",
" else:\n",
" display(Markdown(\"**emojis contained in Dataset:**\\n >\" + \"\".join(set(l))))\n",
"\n",
" X1, Xt1, y1, yt1 = train_test_split(emoji_messages['message'], labels, test_size=0.1, random_state=4222)\n",
"\n",
" vectorizer = TfidfVectorizer(stop_words='english')\n",
" vec_train = vectorizer.fit_transform(X1)\n",
" vec_test = vectorizer.transform(Xt1)\n",
"\n",
" mp(\"**train classifier. This can take a very long time… Grab a coffe! 😀**\")\n",
"\n",
" from sklearn.ensemble import RandomForestClassifier as RFC\n",
" from sklearn.neural_network import MLPClassifier as MLP\n",
" #from sklearn.naive_bayes import MultinomialNB as MNB\n",
" from sklearn.tree import DecisionTreeClassifier as DTC\n",
" from sklearn.multiclass import OneVsRestClassifier as OVRC\n",
" clf_a = None\n",
" if (d.value == \"DecisionTree\"):\n",
" clf_a = DTC()\n",
" elif d.value == \"MLP\":\n",
" clf_a = MLP(hidden_layer_sizes=(64,))\n",
" elif d.value == \"RandomForest\":\n",
" RFC(criterion='entropy', random_state=4222)\n",
"\n",
" if ova.value:\n",
" clf_a=OVRC(clf_a)\n",
"\n",
" display(clf_a)\n",
" clf_a.fit(vec_train, y1)\n",
"\n",
" mp(\"**training done**\")\n",
"\n",
" pred = clf_a.predict(vectorizer.transform(Xt1))\n",
"\n",
" testlist = pd.DataFrame({'message': Xt1, 'pred': mlb.inverse_transform(pred) if mlb else pred, 'teacher': mlb.inverse_transform(yt1) if mlb else yt1})\n",
" testlist.to_csv('export.csv')\n",
" display(testlist)\n",
" \n",
"def build_db(b):\n",
" with out_build:\n",
" clear_output()\n",
" create_emoji_set()\n",
" read_message_and_build_db(t_build.value)\n",
"b_build.on_click(build_db)\n",
"b_train.on_click(train)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"from sklearn.externals import joblib\n",
"def write_to_file(b):\n",
" global vectorizer\n",
" global clf_a\n",
" global mlb\n",
" \n",
" with out_save:\n",
" clear_output()\n",
" mp(\"**write to file...**\")\n",
" joblib.dump(clf_a, t_save_c.value)\n",
" if mlb:\n",
" joblib.dump(mlb, t_save_m.value) \n",
" joblib.dump(vectorizer, t_save_v.value)\n",
" mp(\"**done**\")\n",
"b_save.on_click(write_to_file)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"def read_from_file(b):\n",
" global vectorizer\n",
" global clf_a\n",
" global mlb\n",
" \n",
" with out_read:\n",
" clear_output()\n",
" mp(\"**read from file…**\")\n",
" clf_a = joblib.load(t_read_c.value)\n",
" if t_read_m.value != \"\":\n",
" mlb = joblib.load(t_read_m.value)\n",
" else:\n",
" mlb = None\n",
" vectorizer = joblib.load(t_read_v.value)\n",
" mp(\"**done**\")\n",
"b_read.on_click(read_from_file)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"def predict(b):\n",
" with out_test:\n",
" clear_output()\n",
" v = mlb.inverse_transform(clf_a.predict(vectorizer.transform([t_test.value])))[0] if mlb else clf_a.predict(vectorizer.transform([t_test.value]))[0]\n",
" mp(\"**prediction:**\\n# \" + (\"\".join(v) if len(v)>0 else \" \"))\n",
" if b_prop.value:\n",
" pred = clf_a.predict_proba(vectorizer.transform([t_test.value]))\n",
" print(mlb.inverse_transform(pred))\n",
"\n",
"b_test.on_click(predict)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

714
Project/Tools/emoji-data.txt Normal file
View File

@ -0,0 +1,714 @@
# emoji-data.txt
# Date: 2018-02-07, 07:55:18 GMT
# © 2018 Unicode®, Inc.
# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
# For terms of use, see http://www.unicode.org/terms_of_use.html
#
# Emoji Data for UTS #51
# Version: 11.0
#
# For documentation and usage, see http://www.unicode.org/reports/tr51
#
# Format:
# <codepoint(s)> ; <property> # <comments>
# Note: there is no guarantee as to the structure of whitespace or comments
#
# Characters and sequences are listed in code point order. Users should be shown a more natural order.
# See the CLDR collation order for Emoji.
# ================================================
# All omitted code points have Emoji=No
# @missing: 0000..10FFFF ; Emoji ; No
0023 ; Emoji # 1.1 [1] (#️) number sign
002A ; Emoji # 1.1 [1] (*️) asterisk
0030..0039 ; Emoji # 1.1 [10] (0️..9️) digit zero..digit nine
00A9 ; Emoji # 1.1 [1] (©️) copyright
00AE ; Emoji # 1.1 [1] (®️) registered
203C ; Emoji # 1.1 [1] (‼️) double exclamation mark
2049 ; Emoji # 3.0 [1] (⁉️) exclamation question mark
2122 ; Emoji # 1.1 [1] (™️) trade mark
2139 ; Emoji # 3.0 [1] (ℹ️) information
2194..2199 ; Emoji # 1.1 [6] (↔️..↙️) left-right arrow..down-left arrow
21A9..21AA ; Emoji # 1.1 [2] (↩️..↪️) right arrow curving left..left arrow curving right
231A..231B ; Emoji # 1.1 [2] (⌚..⌛) watch..hourglass done
2328 ; Emoji # 1.1 [1] (⌨️) keyboard
23CF ; Emoji # 4.0 [1] (⏏️) eject button
23E9..23F3 ; Emoji # 6.0 [11] (⏩..⏳) fast-forward button..hourglass not done
23F8..23FA ; Emoji # 7.0 [3] (⏸️..⏺️) pause button..record button
24C2 ; Emoji # 1.1 [1] (Ⓜ️) circled M
25AA..25AB ; Emoji # 1.1 [2] (▪️..▫️) black small square..white small square
25B6 ; Emoji # 1.1 [1] (▶️) play button
25C0 ; Emoji # 1.1 [1] (◀️) reverse button
25FB..25FE ; Emoji # 3.2 [4] (◻️..◾) white medium square..black medium-small square
2600..2604 ; Emoji # 1.1 [5] (☀️..☄️) sun..comet
260E ; Emoji # 1.1 [1] (☎️) telephone
2611 ; Emoji # 1.1 [1] (☑️) ballot box with check
2614..2615 ; Emoji # 4.0 [2] (☔..☕) umbrella with rain drops..hot beverage
2618 ; Emoji # 4.1 [1] (☘️) shamrock
261D ; Emoji # 1.1 [1] (☝️) index pointing up
2620 ; Emoji # 1.1 [1] (☠️) skull and crossbones
2622..2623 ; Emoji # 1.1 [2] (☢️..☣️) radioactive..biohazard
2626 ; Emoji # 1.1 [1] (☦️) orthodox cross
262A ; Emoji # 1.1 [1] (☪️) star and crescent
262E..262F ; Emoji # 1.1 [2] (☮️..☯️) peace symbol..yin yang
2638..263A ; Emoji # 1.1 [3] (☸️..☺️) wheel of dharma..smiling face
2640 ; Emoji # 1.1 [1] (♀️) female sign
2642 ; Emoji # 1.1 [1] (♂️) male sign
2648..2653 ; Emoji # 1.1 [12] (♈..♓) Aries..Pisces
265F..2660 ; Emoji # 1.1 [2] (♟️..♠️) chess pawn..spade suit
2663 ; Emoji # 1.1 [1] (♣️) club suit
2665..2666 ; Emoji # 1.1 [2] (♥️..♦️) heart suit..diamond suit
2668 ; Emoji # 1.1 [1] (♨️) hot springs
267B ; Emoji # 3.2 [1] (♻️) recycling symbol
267E..267F ; Emoji # 4.1 [2] (♾️..♿) infinity..wheelchair symbol
2692..2697 ; Emoji # 4.1 [6] (⚒️..⚗️) hammer and pick..alembic
2699 ; Emoji # 4.1 [1] (⚙️) gear
269B..269C ; Emoji # 4.1 [2] (⚛️..⚜️) atom symbol..fleur-de-lis
26A0..26A1 ; Emoji # 4.0 [2] (⚠️..⚡) warning..high voltage
26AA..26AB ; Emoji # 4.1 [2] (⚪..⚫) white circle..black circle
26B0..26B1 ; Emoji # 4.1 [2] (⚰️..⚱️) coffin..funeral urn
26BD..26BE ; Emoji # 5.2 [2] (⚽..⚾) soccer ball..baseball
26C4..26C5 ; Emoji # 5.2 [2] (⛄..⛅) snowman without snow..sun behind cloud
26C8 ; Emoji # 5.2 [1] (⛈️) cloud with lightning and rain
26CE ; Emoji # 6.0 [1] (⛎) Ophiuchus
26CF ; Emoji # 5.2 [1] (⛏️) pick
26D1 ; Emoji # 5.2 [1] (⛑️) rescue worker’s helmet
26D3..26D4 ; Emoji # 5.2 [2] (⛓️..⛔) chains..no entry
26E9..26EA ; Emoji # 5.2 [2] (⛩️..⛪) shinto shrine..church
26F0..26F5 ; Emoji # 5.2 [6] (⛰️..⛵) mountain..sailboat
26F7..26FA ; Emoji # 5.2 [4] (⛷️..⛺) skier..tent
26FD ; Emoji # 5.2 [1] (⛽) fuel pump
2702 ; Emoji # 1.1 [1] (✂️) scissors
2705 ; Emoji # 6.0 [1] (✅) white heavy check mark
2708..2709 ; Emoji # 1.1 [2] (✈️..✉️) airplane..envelope
270A..270B ; Emoji # 6.0 [2] (✊..✋) raised fist..raised hand
270C..270D ; Emoji # 1.1 [2] (✌️..✍️) victory hand..writing hand
270F ; Emoji # 1.1 [1] (✏️) pencil
2712 ; Emoji # 1.1 [1] (✒️) black nib
2714 ; Emoji # 1.1 [1] (✔️) heavy check mark
2716 ; Emoji # 1.1 [1] (✖️) heavy multiplication x
271D ; Emoji # 1.1 [1] (✝️) latin cross
2721 ; Emoji # 1.1 [1] (✡️) star of David
2728 ; Emoji # 6.0 [1] (✨) sparkles
2733..2734 ; Emoji # 1.1 [2] (✳️..✴️) eight-spoked asterisk..eight-pointed star
2744 ; Emoji # 1.1 [1] (❄️) snowflake
2747 ; Emoji # 1.1 [1] (❇️) sparkle
274C ; Emoji # 6.0 [1] (❌) cross mark
274E ; Emoji # 6.0 [1] (❎) cross mark button
2753..2755 ; Emoji # 6.0 [3] (❓..❕) question mark..white exclamation mark
2757 ; Emoji # 5.2 [1] (❗) exclamation mark
2763..2764 ; Emoji # 1.1 [2] (❣️..❤️) heavy heart exclamation..red heart
2795..2797 ; Emoji # 6.0 [3] (➕..➗) heavy plus sign..heavy division sign
27A1 ; Emoji # 1.1 [1] (➡️) right arrow
27B0 ; Emoji # 6.0 [1] (➰) curly loop
27BF ; Emoji # 6.0 [1] (➿) double curly loop
2934..2935 ; Emoji # 3.2 [2] (⤴️..⤵️) right arrow curving up..right arrow curving down
2B05..2B07 ; Emoji # 4.0 [3] (⬅️..⬇️) left arrow..down arrow
2B1B..2B1C ; Emoji # 5.1 [2] (⬛..⬜) black large square..white large square
2B50 ; Emoji # 5.1 [1] (⭐) star
2B55 ; Emoji # 5.2 [1] (⭕) heavy large circle
3030 ; Emoji # 1.1 [1] (〰️) wavy dash
303D ; Emoji # 3.2 [1] (〽️) part alternation mark
3297 ; Emoji # 1.1 [1] (㊗️) Japanese “congratulations” button
3299 ; Emoji # 1.1 [1] (㊙️) Japanese “secret” button
1F004 ; Emoji # 5.1 [1] (🀄) mahjong red dragon
1F0CF ; Emoji # 6.0 [1] (🃏) joker
1F170..1F171 ; Emoji # 6.0 [2] (🅰️..🅱️) A button (blood type)..B button (blood type)
1F17E ; Emoji # 6.0 [1] (🅾️) O button (blood type)
1F17F ; Emoji # 5.2 [1] (🅿️) P button
1F18E ; Emoji # 6.0 [1] (🆎) AB button (blood type)
1F191..1F19A ; Emoji # 6.0 [10] (🆑..🆚) CL button..VS button
1F1E6..1F1FF ; Emoji # 6.0 [26] (🇦..🇿) regional indicator symbol letter a..regional indicator symbol letter z
1F201..1F202 ; Emoji # 6.0 [2] (🈁..🈂️) Japanese “here” button..Japanese “service charge” button
1F21A ; Emoji # 5.2 [1] (🈚) Japanese “free of charge” button
1F22F ; Emoji # 5.2 [1] (🈯) Japanese “reserved” button
1F232..1F23A ; Emoji # 6.0 [9] (🈲..🈺) Japanese “prohibited” button..Japanese “open for business” button
1F250..1F251 ; Emoji # 6.0 [2] (🉐..🉑) Japanese “bargain” button..Japanese “acceptable” button
1F300..1F320 ; Emoji # 6.0 [33] (🌀..🌠) cyclone..shooting star
1F321 ; Emoji # 7.0 [1] (🌡️) thermometer
1F324..1F32C ; Emoji # 7.0 [9] (🌤️..🌬️) sun behind small cloud..wind face
1F32D..1F32F ; Emoji # 8.0 [3] (🌭..🌯) hot dog..burrito
1F330..1F335 ; Emoji # 6.0 [6] (🌰..🌵) chestnut..cactus
1F336 ; Emoji # 7.0 [1] (🌶️) hot pepper
1F337..1F37C ; Emoji # 6.0 [70] (🌷..🍼) tulip..baby bottle
1F37D ; Emoji # 7.0 [1] (🍽️) fork and knife with plate
1F37E..1F37F ; Emoji # 8.0 [2] (🍾..🍿) bottle with popping cork..popcorn
1F380..1F393 ; Emoji # 6.0 [20] (🎀..🎓) ribbon..graduation cap
1F396..1F397 ; Emoji # 7.0 [2] (🎖️..🎗️) military medal..reminder ribbon
1F399..1F39B ; Emoji # 7.0 [3] (🎙️..🎛️) studio microphone..control knobs
1F39E..1F39F ; Emoji # 7.0 [2] (🎞️..🎟️) film frames..admission tickets
1F3A0..1F3C4 ; Emoji # 6.0 [37] (🎠..🏄) carousel horse..person surfing
1F3C5 ; Emoji # 7.0 [1] (🏅) sports medal
1F3C6..1F3CA ; Emoji # 6.0 [5] (🏆..🏊) trophy..person swimming
1F3CB..1F3CE ; Emoji # 7.0 [4] (🏋️..🏎️) person lifting weights..racing car
1F3CF..1F3D3 ; Emoji # 8.0 [5] (🏏..🏓) cricket game..ping pong
1F3D4..1F3DF ; Emoji # 7.0 [12] (🏔️..🏟️) snow-capped mountain..stadium
1F3E0..1F3F0 ; Emoji # 6.0 [17] (🏠..🏰) house..castle
1F3F3..1F3F5 ; Emoji # 7.0 [3] (🏳️..🏵️) white flag..rosette
1F3F7 ; Emoji # 7.0 [1] (🏷️) label
1F3F8..1F3FF ; Emoji # 8.0 [8] (🏸..🏿) badminton..dark skin tone
1F400..1F43E ; Emoji # 6.0 [63] (🐀..🐾) rat..paw prints
1F43F ; Emoji # 7.0 [1] (🐿️) chipmunk
1F440 ; Emoji # 6.0 [1] (👀) eyes
1F441 ; Emoji # 7.0 [1] (👁️) eye
1F442..1F4F7 ; Emoji # 6.0[182] (👂..📷) ear..camera
1F4F8 ; Emoji # 7.0 [1] (📸) camera with flash
1F4F9..1F4FC ; Emoji # 6.0 [4] (📹..📼) video camera..videocassette
1F4FD ; Emoji # 7.0 [1] (📽️) film projector
1F4FF ; Emoji # 8.0 [1] (📿) prayer beads
1F500..1F53D ; Emoji # 6.0 [62] (🔀..🔽) shuffle tracks button..downwards button
1F549..1F54A ; Emoji # 7.0 [2] (🕉️..🕊️) om..dove
1F54B..1F54E ; Emoji # 8.0 [4] (🕋..🕎) kaaba..menorah
1F550..1F567 ; Emoji # 6.0 [24] (🕐..🕧) one o’clock..twelve-thirty
1F56F..1F570 ; Emoji # 7.0 [2] (🕯️..🕰️) candle..mantelpiece clock
1F573..1F579 ; Emoji # 7.0 [7] (🕳️..🕹️) hole..joystick
1F57A ; Emoji # 9.0 [1] (🕺) man dancing
1F587 ; Emoji # 7.0 [1] (🖇️) linked paperclips
1F58A..1F58D ; Emoji # 7.0 [4] (🖊️..🖍️) pen..crayon
1F590 ; Emoji # 7.0 [1] (🖐️) hand with fingers splayed
1F595..1F596 ; Emoji # 7.0 [2] (🖕..🖖) middle finger..vulcan salute
1F5A4 ; Emoji # 9.0 [1] (🖤) black heart
1F5A5 ; Emoji # 7.0 [1] (🖥️) desktop computer
1F5A8 ; Emoji # 7.0 [1] (🖨️) printer
1F5B1..1F5B2 ; Emoji # 7.0 [2] (🖱️..🖲️) computer mouse..trackball
1F5BC ; Emoji # 7.0 [1] (🖼️) framed picture
1F5C2..1F5C4 ; Emoji # 7.0 [3] (🗂️..🗄️) card index dividers..file cabinet
1F5D1..1F5D3 ; Emoji # 7.0 [3] (🗑️..🗓️) wastebasket..spiral calendar
1F5DC..1F5DE ; Emoji # 7.0 [3] (🗜️..🗞️) clamp..rolled-up newspaper
1F5E1 ; Emoji # 7.0 [1] (🗡️) dagger
1F5E3 ; Emoji # 7.0 [1] (🗣️) speaking head
1F5E8 ; Emoji # 7.0 [1] (🗨️) left speech bubble
1F5EF ; Emoji # 7.0 [1] (🗯️) right anger bubble
1F5F3 ; Emoji # 7.0 [1] (🗳️) ballot box with ballot
1F5FA ; Emoji # 7.0 [1] (🗺️) world map
1F5FB..1F5FF ; Emoji # 6.0 [5] (🗻..🗿) mount fuji..moai
1F600 ; Emoji # 6.1 [1] (😀) grinning face
1F601..1F610 ; Emoji # 6.0 [16] (😁..😐) beaming face with smiling eyes..neutral face
1F611 ; Emoji # 6.1 [1] (😑) expressionless face
1F612..1F614 ; Emoji # 6.0 [3] (😒..😔) unamused face..pensive face
1F615 ; Emoji # 6.1 [1] (😕) confused face
1F616 ; Emoji # 6.0 [1] (😖) confounded face
1F617 ; Emoji # 6.1 [1] (😗) kissing face
1F618 ; Emoji # 6.0 [1] (😘) face blowing a kiss
1F619 ; Emoji # 6.1 [1] (😙) kissing face with smiling eyes
1F61A ; Emoji # 6.0 [1] (😚) kissing face with closed eyes
1F61B ; Emoji # 6.1 [1] (😛) face with tongue
1F61C..1F61E ; Emoji # 6.0 [3] (😜..😞) winking face with tongue..disappointed face
1F61F ; Emoji # 6.1 [1] (😟) worried face
1F620..1F625 ; Emoji # 6.0 [6] (😠..😥) angry face..sad but relieved face
1F626..1F627 ; Emoji # 6.1 [2] (😦..😧) frowning face with open mouth..anguished face
1F628..1F62B ; Emoji # 6.0 [4] (😨..😫) fearful face..tired face
1F62C ; Emoji # 6.1 [1] (😬) grimacing face
1F62D ; Emoji # 6.0 [1] (😭) loudly crying face
1F62E..1F62F ; Emoji # 6.1 [2] (😮..😯) face with open mouth..hushed face
1F630..1F633 ; Emoji # 6.0 [4] (😰..😳) anxious face with sweat..flushed face
1F634 ; Emoji # 6.1 [1] (😴) sleeping face
1F635..1F640 ; Emoji # 6.0 [12] (😵..🙀) dizzy face..weary cat face
1F641..1F642 ; Emoji # 7.0 [2] (🙁..🙂) slightly frowning face..slightly smiling face
1F643..1F644 ; Emoji # 8.0 [2] (🙃..🙄) upside-down face..face with rolling eyes
1F645..1F64F ; Emoji # 6.0 [11] (🙅..🙏) person gesturing NO..folded hands
1F680..1F6C5 ; Emoji # 6.0 [70] (🚀..🛅) rocket..left luggage
1F6CB..1F6CF ; Emoji # 7.0 [5] (🛋️..🛏️) couch and lamp..bed
1F6D0 ; Emoji # 8.0 [1] (🛐) place of worship
1F6D1..1F6D2 ; Emoji # 9.0 [2] (🛑..🛒) stop sign..shopping cart
1F6E0..1F6E5 ; Emoji # 7.0 [6] (🛠️..🛥️) hammer and wrench..motor boat
1F6E9 ; Emoji # 7.0 [1] (🛩️) small airplane
1F6EB..1F6EC ; Emoji # 7.0 [2] (🛫..🛬) airplane departure..airplane arrival
1F6F0 ; Emoji # 7.0 [1] (🛰️) satellite
1F6F3 ; Emoji # 7.0 [1] (🛳️) passenger ship
1F6F4..1F6F6 ; Emoji # 9.0 [3] (🛴..🛶) kick scooter..canoe
1F6F7..1F6F8 ; Emoji # 10.0 [2] (🛷..🛸) sled..flying saucer
1F6F9 ; Emoji # 11.0 [1] (🛹) skateboard
1F910..1F918 ; Emoji # 8.0 [9] (🤐..🤘) zipper-mouth face..sign of the horns
1F919..1F91E ; Emoji # 9.0 [6] (🤙..🤞) call me hand..crossed fingers
1F91F ; Emoji # 10.0 [1] (🤟) love-you gesture
1F920..1F927 ; Emoji # 9.0 [8] (🤠..🤧) cowboy hat face..sneezing face
1F928..1F92F ; Emoji # 10.0 [8] (🤨..🤯) face with raised eyebrow..exploding head
1F930 ; Emoji # 9.0 [1] (🤰) pregnant woman
1F931..1F932 ; Emoji # 10.0 [2] (🤱..🤲) breast-feeding..palms up together
1F933..1F93A ; Emoji # 9.0 [8] (🤳..🤺) selfie..person fencing
1F93C..1F93E ; Emoji # 9.0 [3] (🤼..🤾) people wrestling..person playing handball
1F940..1F945 ; Emoji # 9.0 [6] (🥀..🥅) wilted flower..goal net
1F947..1F94B ; Emoji # 9.0 [5] (🥇..🥋) 1st place medal..martial arts uniform
1F94C ; Emoji # 10.0 [1] (🥌) curling stone
1F94D..1F94F ; Emoji # 11.0 [3] (🥍..🥏) lacrosse..flying disc
1F950..1F95E ; Emoji # 9.0 [15] (🥐..🥞) croissant..pancakes
1F95F..1F96B ; Emoji # 10.0 [13] (🥟..🥫) dumpling..canned food
1F96C..1F970 ; Emoji # 11.0 [5] (🥬..🥰) leafy green..smiling face with 3 hearts
1F973..1F976 ; Emoji # 11.0 [4] (🥳..🥶) partying face..cold face
1F97A ; Emoji # 11.0 [1] (🥺) pleading face
1F97C..1F97F ; Emoji # 11.0 [4] (🥼..🥿) lab coat..woman’s flat shoe
1F980..1F984 ; Emoji # 8.0 [5] (🦀..🦄) crab..unicorn face
1F985..1F991 ; Emoji # 9.0 [13] (🦅..🦑) eagle..squid
1F992..1F997 ; Emoji # 10.0 [6] (🦒..🦗) giraffe..cricket
1F998..1F9A2 ; Emoji # 11.0 [11] (🦘..🦢) kangaroo..swan
1F9B0..1F9B9 ; Emoji # 11.0 [10] (🦰..🦹) red-haired..supervillain
1F9C0 ; Emoji # 8.0 [1] (🧀) cheese wedge
1F9C1..1F9C2 ; Emoji # 11.0 [2] (🧁..🧂) cupcake..salt
1F9D0..1F9E6 ; Emoji # 10.0 [23] (🧐..🧦) face with monocle..socks
1F9E7..1F9FF ; Emoji # 11.0 [25] (🧧..🧿) red envelope..nazar amulet
# Total elements: 1250
# ================================================
# All omitted code points have Emoji_Presentation=No
# @missing: 0000..10FFFF ; Emoji_Presentation ; No
231A..231B ; Emoji_Presentation # 1.1 [2] (⌚..⌛) watch..hourglass done
23E9..23EC ; Emoji_Presentation # 6.0 [4] (⏩..⏬) fast-forward button..fast down button
23F0 ; Emoji_Presentation # 6.0 [1] (⏰) alarm clock
23F3 ; Emoji_Presentation # 6.0 [1] (⏳) hourglass not done
25FD..25FE ; Emoji_Presentation # 3.2 [2] (◽..◾) white medium-small square..black medium-small square
2614..2615 ; Emoji_Presentation # 4.0 [2] (☔..☕) umbrella with rain drops..hot beverage
2648..2653 ; Emoji_Presentation # 1.1 [12] (♈..♓) Aries..Pisces
267F ; Emoji_Presentation # 4.1 [1] (♿) wheelchair symbol
2693 ; Emoji_Presentation # 4.1 [1] (⚓) anchor
26A1 ; Emoji_Presentation # 4.0 [1] (⚡) high voltage
26AA..26AB ; Emoji_Presentation # 4.1 [2] (⚪..⚫) white circle..black circle
26BD..26BE ; Emoji_Presentation # 5.2 [2] (⚽..⚾) soccer ball..baseball
26C4..26C5 ; Emoji_Presentation # 5.2 [2] (⛄..⛅) snowman without snow..sun behind cloud
26CE ; Emoji_Presentation # 6.0 [1] (⛎) Ophiuchus
26D4 ; Emoji_Presentation # 5.2 [1] (⛔) no entry
26EA ; Emoji_Presentation # 5.2 [1] (⛪) church
26F2..26F3 ; Emoji_Presentation # 5.2 [2] (⛲..⛳) fountain..flag in hole
26F5 ; Emoji_Presentation # 5.2 [1] (⛵) sailboat
26FA ; Emoji_Presentation # 5.2 [1] (⛺) tent
26FD ; Emoji_Presentation # 5.2 [1] (⛽) fuel pump
2705 ; Emoji_Presentation # 6.0 [1] (✅) white heavy check mark
270A..270B ; Emoji_Presentation # 6.0 [2] (✊..✋) raised fist..raised hand
2728 ; Emoji_Presentation # 6.0 [1] (✨) sparkles
274C ; Emoji_Presentation # 6.0 [1] (❌) cross mark
274E ; Emoji_Presentation # 6.0 [1] (❎) cross mark button
2753..2755 ; Emoji_Presentation # 6.0 [3] (❓..❕) question mark..white exclamation mark
2757 ; Emoji_Presentation # 5.2 [1] (❗) exclamation mark
2795..2797 ; Emoji_Presentation # 6.0 [3] (➕..➗) heavy plus sign..heavy division sign
27B0 ; Emoji_Presentation # 6.0 [1] (➰) curly loop
27BF ; Emoji_Presentation # 6.0 [1] (➿) double curly loop
2B1B..2B1C ; Emoji_Presentation # 5.1 [2] (⬛..⬜) black large square..white large square
2B50 ; Emoji_Presentation # 5.1 [1] (⭐) star
2B55 ; Emoji_Presentation # 5.2 [1] (⭕) heavy large circle
1F004 ; Emoji_Presentation # 5.1 [1] (🀄) mahjong red dragon
1F0CF ; Emoji_Presentation # 6.0 [1] (🃏) joker
1F18E ; Emoji_Presentation # 6.0 [1] (🆎) AB button (blood type)
1F191..1F19A ; Emoji_Presentation # 6.0 [10] (🆑..🆚) CL button..VS button
1F1E6..1F1FF ; Emoji_Presentation # 6.0 [26] (🇦..🇿) regional indicator symbol letter a..regional indicator symbol letter z
1F201 ; Emoji_Presentation # 6.0 [1] (🈁) Japanese “here” button
1F21A ; Emoji_Presentation # 5.2 [1] (🈚) Japanese “free of charge” button
1F22F ; Emoji_Presentation # 5.2 [1] (🈯) Japanese “reserved” button
1F232..1F236 ; Emoji_Presentation # 6.0 [5] (🈲..🈶) Japanese “prohibited” button..Japanese “not free of charge” button
1F238..1F23A ; Emoji_Presentation # 6.0 [3] (🈸..🈺) Japanese “application” button..Japanese “open for business” button
1F250..1F251 ; Emoji_Presentation # 6.0 [2] (🉐..🉑) Japanese “bargain” button..Japanese “acceptable” button
1F300..1F320 ; Emoji_Presentation # 6.0 [33] (🌀..🌠) cyclone..shooting star
1F32D..1F32F ; Emoji_Presentation # 8.0 [3] (🌭..🌯) hot dog..burrito
1F330..1F335 ; Emoji_Presentation # 6.0 [6] (🌰..🌵) chestnut..cactus
1F337..1F37C ; Emoji_Presentation # 6.0 [70] (🌷..🍼) tulip..baby bottle
1F37E..1F37F ; Emoji_Presentation # 8.0 [2] (🍾..🍿) bottle with popping cork..popcorn
1F380..1F393 ; Emoji_Presentation # 6.0 [20] (🎀..🎓) ribbon..graduation cap
1F3A0..1F3C4 ; Emoji_Presentation # 6.0 [37] (🎠..🏄) carousel horse..person surfing
1F3C5 ; Emoji_Presentation # 7.0 [1] (🏅) sports medal
1F3C6..1F3CA ; Emoji_Presentation # 6.0 [5] (🏆..🏊) trophy..person swimming
1F3CF..1F3D3 ; Emoji_Presentation # 8.0 [5] (🏏..🏓) cricket game..ping pong
1F3E0..1F3F0 ; Emoji_Presentation # 6.0 [17] (🏠..🏰) house..castle
1F3F4 ; Emoji_Presentation # 7.0 [1] (🏴) black flag
1F3F8..1F3FF ; Emoji_Presentation # 8.0 [8] (🏸..🏿) badminton..dark skin tone
1F400..1F43E ; Emoji_Presentation # 6.0 [63] (🐀..🐾) rat..paw prints
1F440 ; Emoji_Presentation # 6.0 [1] (👀) eyes
1F442..1F4F7 ; Emoji_Presentation # 6.0[182] (👂..📷) ear..camera
1F4F8 ; Emoji_Presentation # 7.0 [1] (📸) camera with flash
1F4F9..1F4FC ; Emoji_Presentation # 6.0 [4] (📹..📼) video camera..videocassette
1F4FF ; Emoji_Presentation # 8.0 [1] (📿) prayer beads
1F500..1F53D ; Emoji_Presentation # 6.0 [62] (🔀..🔽) shuffle tracks button..downwards button
1F54B..1F54E ; Emoji_Presentation # 8.0 [4] (🕋..🕎) kaaba..menorah
1F550..1F567 ; Emoji_Presentation # 6.0 [24] (🕐..🕧) one o’clock..twelve-thirty
1F57A ; Emoji_Presentation # 9.0 [1] (🕺) man dancing
1F595..1F596 ; Emoji_Presentation # 7.0 [2] (🖕..🖖) middle finger..vulcan salute
1F5A4 ; Emoji_Presentation # 9.0 [1] (🖤) black heart
1F5FB..1F5FF ; Emoji_Presentation # 6.0 [5] (🗻..🗿) mount fuji..moai
1F600 ; Emoji_Presentation # 6.1 [1] (😀) grinning face
1F601..1F610 ; Emoji_Presentation # 6.0 [16] (😁..😐) beaming face with smiling eyes..neutral face
1F611 ; Emoji_Presentation # 6.1 [1] (😑) expressionless face
1F612..1F614 ; Emoji_Presentation # 6.0 [3] (😒..😔) unamused face..pensive face
1F615 ; Emoji_Presentation # 6.1 [1] (😕) confused face
1F616 ; Emoji_Presentation # 6.0 [1] (😖) confounded face
1F617 ; Emoji_Presentation # 6.1 [1] (😗) kissing face
1F618 ; Emoji_Presentation # 6.0 [1] (😘) face blowing a kiss
1F619 ; Emoji_Presentation # 6.1 [1] (😙) kissing face with smiling eyes
1F61A ; Emoji_Presentation # 6.0 [1] (😚) kissing face with closed eyes
1F61B ; Emoji_Presentation # 6.1 [1] (😛) face with tongue
1F61C..1F61E ; Emoji_Presentation # 6.0 [3] (😜..😞) winking face with tongue..disappointed face
1F61F ; Emoji_Presentation # 6.1 [1] (😟) worried face
1F620..1F625 ; Emoji_Presentation # 6.0 [6] (😠..😥) angry face..sad but relieved face
1F626..1F627 ; Emoji_Presentation # 6.1 [2] (😦..😧) frowning face with open mouth..anguished face
1F628..1F62B ; Emoji_Presentation # 6.0 [4] (😨..😫) fearful face..tired face
1F62C ; Emoji_Presentation # 6.1 [1] (😬) grimacing face
1F62D ; Emoji_Presentation # 6.0 [1] (😭) loudly crying face
1F62E..1F62F ; Emoji_Presentation # 6.1 [2] (😮..😯) face with open mouth..hushed face
1F630..1F633 ; Emoji_Presentation # 6.0 [4] (😰..😳) anxious face with sweat..flushed face
1F634 ; Emoji_Presentation # 6.1 [1] (😴) sleeping face
1F635..1F640 ; Emoji_Presentation # 6.0 [12] (😵..🙀) dizzy face..weary cat face
1F641..1F642 ; Emoji_Presentation # 7.0 [2] (🙁..🙂) slightly frowning face..slightly smiling face
1F643..1F644 ; Emoji_Presentation # 8.0 [2] (🙃..🙄) upside-down face..face with rolling eyes
1F645..1F64F ; Emoji_Presentation # 6.0 [11] (🙅..🙏) person gesturing NO..folded hands
1F680..1F6C5 ; Emoji_Presentation # 6.0 [70] (🚀..🛅) rocket..left luggage
1F6CC ; Emoji_Presentation # 7.0 [1] (🛌) person in bed
1F6D0 ; Emoji_Presentation # 8.0 [1] (🛐) place of worship
1F6D1..1F6D2 ; Emoji_Presentation # 9.0 [2] (🛑..🛒) stop sign..shopping cart
1F6EB..1F6EC ; Emoji_Presentation # 7.0 [2] (🛫..🛬) airplane departure..airplane arrival
1F6F4..1F6F6 ; Emoji_Presentation # 9.0 [3] (🛴..🛶) kick scooter..canoe
1F6F7..1F6F8 ; Emoji_Presentation # 10.0 [2] (🛷..🛸) sled..flying saucer
1F6F9 ; Emoji_Presentation # 11.0 [1] (🛹) skateboard
1F910..1F918 ; Emoji_Presentation # 8.0 [9] (🤐..🤘) zipper-mouth face..sign of the horns
1F919..1F91E ; Emoji_Presentation # 9.0 [6] (🤙..🤞) call me hand..crossed fingers
1F91F ; Emoji_Presentation # 10.0 [1] (🤟) love-you gesture
1F920..1F927 ; Emoji_Presentation # 9.0 [8] (🤠..🤧) cowboy hat face..sneezing face
1F928..1F92F ; Emoji_Presentation # 10.0 [8] (🤨..🤯) face with raised eyebrow..exploding head
1F930 ; Emoji_Presentation # 9.0 [1] (🤰) pregnant woman
1F931..1F932 ; Emoji_Presentation # 10.0 [2] (🤱..🤲) breast-feeding..palms up together
1F933..1F93A ; Emoji_Presentation # 9.0 [8] (🤳..🤺) selfie..person fencing
1F93C..1F93E ; Emoji_Presentation # 9.0 [3] (🤼..🤾) people wrestling..person playing handball
1F940..1F945 ; Emoji_Presentation # 9.0 [6] (🥀..🥅) wilted flower..goal net
1F947..1F94B ; Emoji_Presentation # 9.0 [5] (🥇..🥋) 1st place medal..martial arts uniform
1F94C ; Emoji_Presentation # 10.0 [1] (🥌) curling stone
1F94D..1F94F ; Emoji_Presentation # 11.0 [3] (🥍..🥏) lacrosse..flying disc
1F950..1F95E ; Emoji_Presentation # 9.0 [15] (🥐..🥞) croissant..pancakes
1F95F..1F96B ; Emoji_Presentation # 10.0 [13] (🥟..🥫) dumpling..canned food
1F96C..1F970 ; Emoji_Presentation # 11.0 [5] (🥬..🥰) leafy green..smiling face with 3 hearts
1F973..1F976 ; Emoji_Presentation # 11.0 [4] (🥳..🥶) partying face..cold face
1F97A ; Emoji_Presentation # 11.0 [1] (🥺) pleading face
1F97C..1F97F ; Emoji_Presentation # 11.0 [4] (🥼..🥿) lab coat..woman’s flat shoe
1F980..1F984 ; Emoji_Presentation # 8.0 [5] (🦀..🦄) crab..unicorn face
1F985..1F991 ; Emoji_Presentation # 9.0 [13] (🦅..🦑) eagle..squid
1F992..1F997 ; Emoji_Presentation # 10.0 [6] (🦒..🦗) giraffe..cricket
1F998..1F9A2 ; Emoji_Presentation # 11.0 [11] (🦘..🦢) kangaroo..swan
1F9B0..1F9B9 ; Emoji_Presentation # 11.0 [10] (🦰..🦹) red-haired..supervillain
1F9C0 ; Emoji_Presentation # 8.0 [1] (🧀) cheese wedge
1F9C1..1F9C2 ; Emoji_Presentation # 11.0 [2] (🧁..🧂) cupcake..salt
1F9D0..1F9E6 ; Emoji_Presentation # 10.0 [23] (🧐..🧦) face with monocle..socks
1F9E7..1F9FF ; Emoji_Presentation # 11.0 [25] (🧧..🧿) red envelope..nazar amulet
# Total elements: 1032
# ================================================
# All omitted code points have Emoji_Modifier=No
# @missing: 0000..10FFFF ; Emoji_Modifier ; No
1F3FB..1F3FF ; Emoji_Modifier # 8.0 [5] (🏻..🏿) light skin tone..dark skin tone
# Total elements: 5
# ================================================
# All omitted code points have Emoji_Modifier_Base=No
# @missing: 0000..10FFFF ; Emoji_Modifier_Base ; No
261D ; Emoji_Modifier_Base # 1.1 [1] (☝️) index pointing up
26F9 ; Emoji_Modifier_Base # 5.2 [1] (⛹️) person bouncing ball
270A..270B ; Emoji_Modifier_Base # 6.0 [2] (✊..✋) raised fist..raised hand
270C..270D ; Emoji_Modifier_Base # 1.1 [2] (✌️..✍️) victory hand..writing hand
1F385 ; Emoji_Modifier_Base # 6.0 [1] (🎅) Santa Claus
1F3C2..1F3C4 ; Emoji_Modifier_Base # 6.0 [3] (🏂..🏄) snowboarder..person surfing
1F3C7 ; Emoji_Modifier_Base # 6.0 [1] (🏇) horse racing
1F3CA ; Emoji_Modifier_Base # 6.0 [1] (🏊) person swimming
1F3CB..1F3CC ; Emoji_Modifier_Base # 7.0 [2] (🏋️..🏌️) person lifting weights..person golfing
1F442..1F443 ; Emoji_Modifier_Base # 6.0 [2] (👂..👃) ear..nose
1F446..1F450 ; Emoji_Modifier_Base # 6.0 [11] (👆..👐) backhand index pointing up..open hands
1F466..1F469 ; Emoji_Modifier_Base # 6.0 [4] (👦..👩) boy..woman
1F46E ; Emoji_Modifier_Base # 6.0 [1] (👮) police officer
1F470..1F478 ; Emoji_Modifier_Base # 6.0 [9] (👰..👸) bride with veil..princess
1F47C ; Emoji_Modifier_Base # 6.0 [1] (👼) baby angel
1F481..1F483 ; Emoji_Modifier_Base # 6.0 [3] (💁..💃) person tipping hand..woman dancing
1F485..1F487 ; Emoji_Modifier_Base # 6.0 [3] (💅..💇) nail polish..person getting haircut
1F4AA ; Emoji_Modifier_Base # 6.0 [1] (💪) flexed biceps
1F574..1F575 ; Emoji_Modifier_Base # 7.0 [2] (🕴️..🕵️) man in suit levitating..detective
1F57A ; Emoji_Modifier_Base # 9.0 [1] (🕺) man dancing
1F590 ; Emoji_Modifier_Base # 7.0 [1] (🖐️) hand with fingers splayed
1F595..1F596 ; Emoji_Modifier_Base # 7.0 [2] (🖕..🖖) middle finger..vulcan salute
1F645..1F647 ; Emoji_Modifier_Base # 6.0 [3] (🙅..🙇) person gesturing NO..person bowing
1F64B..1F64F ; Emoji_Modifier_Base # 6.0 [5] (🙋..🙏) person raising hand..folded hands
1F6A3 ; Emoji_Modifier_Base # 6.0 [1] (🚣) person rowing boat
1F6B4..1F6B6 ; Emoji_Modifier_Base # 6.0 [3] (🚴..🚶) person biking..person walking
1F6C0 ; Emoji_Modifier_Base # 6.0 [1] (🛀) person taking bath
1F6CC ; Emoji_Modifier_Base # 7.0 [1] (🛌) person in bed
1F918 ; Emoji_Modifier_Base # 8.0 [1] (🤘) sign of the horns
1F919..1F91C ; Emoji_Modifier_Base # 9.0 [4] (🤙..🤜) call me hand..right-facing fist
1F91E ; Emoji_Modifier_Base # 9.0 [1] (🤞) crossed fingers
1F91F ; Emoji_Modifier_Base # 10.0 [1] (🤟) love-you gesture
1F926 ; Emoji_Modifier_Base # 9.0 [1] (🤦) person facepalming
1F930 ; Emoji_Modifier_Base # 9.0 [1] (🤰) pregnant woman
1F931..1F932 ; Emoji_Modifier_Base # 10.0 [2] (🤱..🤲) breast-feeding..palms up together
1F933..1F939 ; Emoji_Modifier_Base # 9.0 [7] (🤳..🤹) selfie..person juggling
1F93D..1F93E ; Emoji_Modifier_Base # 9.0 [2] (🤽..🤾) person playing water polo..person playing handball
1F9B5..1F9B6 ; Emoji_Modifier_Base # 11.0 [2] (🦵..🦶) leg..foot
1F9B8..1F9B9 ; Emoji_Modifier_Base # 11.0 [2] (🦸..🦹) superhero..supervillain
1F9D1..1F9DD ; Emoji_Modifier_Base # 10.0 [13] (🧑..🧝) adult..elf
# Total elements: 106
# ================================================
# All omitted code points have Emoji_Component=No
# @missing: 0000..10FFFF ; Emoji_Component ; No
0023 ; Emoji_Component # 1.1 [1] (#️) number sign
002A ; Emoji_Component # 1.1 [1] (*️) asterisk
0030..0039 ; Emoji_Component # 1.1 [10] (0️..9️) digit zero..digit nine
200D ; Emoji_Component # 1.1 [1] (‍) zero width joiner
20E3 ; Emoji_Component # 3.0 [1] (⃣) combining enclosing keycap
FE0F ; Emoji_Component # 3.2 [1] () VARIATION SELECTOR-16
1F1E6..1F1FF ; Emoji_Component # 6.0 [26] (🇦..🇿) regional indicator symbol letter a..regional indicator symbol letter z
1F3FB..1F3FF ; Emoji_Component # 8.0 [5] (🏻..🏿) light skin tone..dark skin tone
1F9B0..1F9B3 ; Emoji_Component # 11.0 [4] (🦰..🦳) red-haired..white-haired
E0020..E007F ; Emoji_Component # 3.1 [96] (󠀠..󠁿) tag space..cancel tag
# Total elements: 146
# ================================================
# All omitted code points have Extended_Pictographic=No
# @missing: 0000..10FFFF ; Extended_Pictographic ; No
00A9 ; Extended_Pictographic# 1.1 [1] (©️) copyright
00AE ; Extended_Pictographic# 1.1 [1] (®️) registered
203C ; Extended_Pictographic# 1.1 [1] (‼️) double exclamation mark
2049 ; Extended_Pictographic# 3.0 [1] (⁉️) exclamation question mark
2122 ; Extended_Pictographic# 1.1 [1] (™️) trade mark
2139 ; Extended_Pictographic# 3.0 [1] (ℹ️) information
2194..2199 ; Extended_Pictographic# 1.1 [6] (↔️..↙️) left-right arrow..down-left arrow
21A9..21AA ; Extended_Pictographic# 1.1 [2] (↩️..↪️) right arrow curving left..left arrow curving right
231A..231B ; Extended_Pictographic# 1.1 [2] (⌚..⌛) watch..hourglass done
2328 ; Extended_Pictographic# 1.1 [1] (⌨️) keyboard
2388 ; Extended_Pictographic# 3.0 [1] (⎈️) HELM SYMBOL
23CF ; Extended_Pictographic# 4.0 [1] (⏏️) eject button
23E9..23F3 ; Extended_Pictographic# 6.0 [11] (⏩..⏳) fast-forward button..hourglass not done
23F8..23FA ; Extended_Pictographic# 7.0 [3] (⏸️..⏺️) pause button..record button
24C2 ; Extended_Pictographic# 1.1 [1] (Ⓜ️) circled M
25AA..25AB ; Extended_Pictographic# 1.1 [2] (▪️..▫️) black small square..white small square
25B6 ; Extended_Pictographic# 1.1 [1] (▶️) play button
25C0 ; Extended_Pictographic# 1.1 [1] (◀️) reverse button
25FB..25FE ; Extended_Pictographic# 3.2 [4] (◻️..◾) white medium square..black medium-small square
2600..2605 ; Extended_Pictographic# 1.1 [6] (☀️..★️) sun..BLACK STAR
2607..2612 ; Extended_Pictographic# 1.1 [12] (☇️..☒️) LIGHTNING..BALLOT BOX WITH X
2614..2615 ; Extended_Pictographic# 4.0 [2] (☔..☕) umbrella with rain drops..hot beverage
2616..2617 ; Extended_Pictographic# 3.2 [2] (☖️..☗️) WHITE SHOGI PIECE..BLACK SHOGI PIECE
2618 ; Extended_Pictographic# 4.1 [1] (☘️) shamrock
2619 ; Extended_Pictographic# 3.0 [1] (☙️) REVERSED ROTATED FLORAL HEART BULLET
261A..266F ; Extended_Pictographic# 1.1 [86] (☚️..♯️) BLACK LEFT POINTING INDEX..MUSIC SHARP SIGN
2670..2671 ; Extended_Pictographic# 3.0 [2] (♰️..♱️) WEST SYRIAC CROSS..EAST SYRIAC CROSS
2672..267D ; Extended_Pictographic# 3.2 [12] (♲️..♽️) UNIVERSAL RECYCLING SYMBOL..PARTIALLY-RECYCLED PAPER SYMBOL
267E..267F ; Extended_Pictographic# 4.1 [2] (♾️..♿) infinity..wheelchair symbol
2680..2685 ; Extended_Pictographic# 3.2 [6] (⚀️..⚅️) DIE FACE-1..DIE FACE-6
2690..2691 ; Extended_Pictographic# 4.0 [2] (⚐️..⚑️) WHITE FLAG..BLACK FLAG
2692..269C ; Extended_Pictographic# 4.1 [11] (⚒️..⚜️) hammer and pick..fleur-de-lis
269D ; Extended_Pictographic# 5.1 [1] (⚝️) OUTLINED WHITE STAR
269E..269F ; Extended_Pictographic# 5.2 [2] (⚞️..⚟️) THREE LINES CONVERGING RIGHT..THREE LINES CONVERGING LEFT
26A0..26A1 ; Extended_Pictographic# 4.0 [2] (⚠️..⚡) warning..high voltage
26A2..26B1 ; Extended_Pictographic# 4.1 [16] (⚢️..⚱️) DOUBLED FEMALE SIGN..funeral urn
26B2 ; Extended_Pictographic# 5.0 [1] (⚲️) NEUTER
26B3..26BC ; Extended_Pictographic# 5.1 [10] (⚳️..⚼️) CERES..SESQUIQUADRATE
26BD..26BF ; Extended_Pictographic# 5.2 [3] (⚽..⚿️) soccer ball..SQUARED KEY
26C0..26C3 ; Extended_Pictographic# 5.1 [4] (⛀️..⛃️) WHITE DRAUGHTS MAN..BLACK DRAUGHTS KING
26C4..26CD ; Extended_Pictographic# 5.2 [10] (⛄..⛍️) snowman without snow..DISABLED CAR
26CE ; Extended_Pictographic# 6.0 [1] (⛎) Ophiuchus
26CF..26E1 ; Extended_Pictographic# 5.2 [19] (⛏️..⛡️) pick..RESTRICTED LEFT ENTRY-2
26E2 ; Extended_Pictographic# 6.0 [1] (⛢️) ASTRONOMICAL SYMBOL FOR URANUS
26E3 ; Extended_Pictographic# 5.2 [1] (⛣️) HEAVY CIRCLE WITH STROKE AND TWO DOTS ABOVE
26E4..26E7 ; Extended_Pictographic# 6.0 [4] (⛤️..⛧️) PENTAGRAM..INVERTED PENTAGRAM
26E8..26FF ; Extended_Pictographic# 5.2 [24] (⛨️..⛿️) BLACK CROSS ON SHIELD..WHITE FLAG WITH HORIZONTAL MIDDLE BLACK STRIPE
2700 ; Extended_Pictographic# 7.0 [1] (✀️) BLACK SAFETY SCISSORS
2701..2704 ; Extended_Pictographic# 1.1 [4] (✁️..✄️) UPPER BLADE SCISSORS..WHITE SCISSORS
2705 ; Extended_Pictographic# 6.0 [1] (✅) white heavy check mark
2708..2709 ; Extended_Pictographic# 1.1 [2] (✈️..✉️) airplane..envelope
270A..270B ; Extended_Pictographic# 6.0 [2] (✊..✋) raised fist..raised hand
270C..2712 ; Extended_Pictographic# 1.1 [7] (✌️..✒️) victory hand..black nib
2714 ; Extended_Pictographic# 1.1 [1] (✔️) heavy check mark
2716 ; Extended_Pictographic# 1.1 [1] (✖️) heavy multiplication x
271D ; Extended_Pictographic# 1.1 [1] (✝️) latin cross
2721 ; Extended_Pictographic# 1.1 [1] (✡️) star of David
2728 ; Extended_Pictographic# 6.0 [1] (✨) sparkles
2733..2734 ; Extended_Pictographic# 1.1 [2] (✳️..✴️) eight-spoked asterisk..eight-pointed star
2744 ; Extended_Pictographic# 1.1 [1] (❄️) snowflake
2747 ; Extended_Pictographic# 1.1 [1] (❇️) sparkle
274C ; Extended_Pictographic# 6.0 [1] (❌) cross mark
274E ; Extended_Pictographic# 6.0 [1] (❎) cross mark button
2753..2755 ; Extended_Pictographic# 6.0 [3] (❓..❕) question mark..white exclamation mark
2757 ; Extended_Pictographic# 5.2 [1] (❗) exclamation mark
2763..2767 ; Extended_Pictographic# 1.1 [5] (❣️..❧️) heavy heart exclamation..ROTATED FLORAL HEART BULLET
2795..2797 ; Extended_Pictographic# 6.0 [3] (➕..➗) heavy plus sign..heavy division sign
27A1 ; Extended_Pictographic# 1.1 [1] (➡️) right arrow
27B0 ; Extended_Pictographic# 6.0 [1] (➰) curly loop
27BF ; Extended_Pictographic# 6.0 [1] (➿) double curly loop
2934..2935 ; Extended_Pictographic# 3.2 [2] (⤴️..⤵️) right arrow curving up..right arrow curving down
2B05..2B07 ; Extended_Pictographic# 4.0 [3] (⬅️..⬇️) left arrow..down arrow
2B1B..2B1C ; Extended_Pictographic# 5.1 [2] (⬛..⬜) black large square..white large square
2B50 ; Extended_Pictographic# 5.1 [1] (⭐) star
2B55 ; Extended_Pictographic# 5.2 [1] (⭕) heavy large circle
3030 ; Extended_Pictographic# 1.1 [1] (〰️) wavy dash
303D ; Extended_Pictographic# 3.2 [1] (〽️) part alternation mark
3297 ; Extended_Pictographic# 1.1 [1] (㊗️) Japanese “congratulations” button
3299 ; Extended_Pictographic# 1.1 [1] (㊙️) Japanese “secret” button
1F000..1F02B ; Extended_Pictographic# 5.1 [44] (🀀️..🀫️) MAHJONG TILE EAST WIND..MAHJONG TILE BACK
1F02C..1F02F ; Extended_Pictographic# NA [4] (🀬️..🀯️) <reserved-1F02C>..<reserved-1F02F>
1F030..1F093 ; Extended_Pictographic# 5.1[100] (🀰️..🂓️) DOMINO TILE HORIZONTAL BACK..DOMINO TILE VERTICAL-06-06
1F094..1F09F ; Extended_Pictographic# NA [12] (🂔️..🂟️) <reserved-1F094>..<reserved-1F09F>
1F0A0..1F0AE ; Extended_Pictographic# 6.0 [15] (🂠️..🂮️) PLAYING CARD BACK..PLAYING CARD KING OF SPADES
1F0AF..1F0B0 ; Extended_Pictographic# NA [2] (🂯️..🂰️) <reserved-1F0AF>..<reserved-1F0B0>
1F0B1..1F0BE ; Extended_Pictographic# 6.0 [14] (🂱️..🂾️) PLAYING CARD ACE OF HEARTS..PLAYING CARD KING OF HEARTS
1F0BF ; Extended_Pictographic# 7.0 [1] (🂿️) PLAYING CARD RED JOKER
1F0C0 ; Extended_Pictographic# NA [1] (🃀️) <reserved-1F0C0>
1F0C1..1F0CF ; Extended_Pictographic# 6.0 [15] (🃁️..🃏) PLAYING CARD ACE OF DIAMONDS..joker
1F0D0 ; Extended_Pictographic# NA [1] (🃐️) <reserved-1F0D0>
1F0D1..1F0DF ; Extended_Pictographic# 6.0 [15] (🃑️..🃟️) PLAYING CARD ACE OF CLUBS..PLAYING CARD WHITE JOKER
1F0E0..1F0F5 ; Extended_Pictographic# 7.0 [22] (🃠️..🃵️) PLAYING CARD FOOL..PLAYING CARD TRUMP-21
1F0F6..1F0FF ; Extended_Pictographic# NA [10] (🃶️..🃿️) <reserved-1F0F6>..<reserved-1F0FF>
1F10D..1F10F ; Extended_Pictographic# NA [3] (🄍️..🄏️) <reserved-1F10D>..<reserved-1F10F>
1F12F ; Extended_Pictographic# 11.0 [1] (🄯️) COPYLEFT SYMBOL
1F16C..1F16F ; Extended_Pictographic# NA [4] (🅬️..🅯️) <reserved-1F16C>..<reserved-1F16F>
1F170..1F171 ; Extended_Pictographic# 6.0 [2] (🅰️..🅱️) A button (blood type)..B button (blood type)
1F17E ; Extended_Pictographic# 6.0 [1] (🅾️) O button (blood type)
1F17F ; Extended_Pictographic# 5.2 [1] (🅿️) P button
1F18E ; Extended_Pictographic# 6.0 [1] (🆎) AB button (blood type)
1F191..1F19A ; Extended_Pictographic# 6.0 [10] (🆑..🆚) CL button..VS button
1F1AD..1F1E5 ; Extended_Pictographic# NA [57] (🆭️..🇥️) <reserved-1F1AD>..<reserved-1F1E5>
1F201..1F202 ; Extended_Pictographic# 6.0 [2] (🈁..🈂️) Japanese “here” button..Japanese “service charge” button
1F203..1F20F ; Extended_Pictographic# NA [13] (🈃️..🈏️) <reserved-1F203>..<reserved-1F20F>
1F21A ; Extended_Pictographic# 5.2 [1] (🈚) Japanese “free of charge” button
1F22F ; Extended_Pictographic# 5.2 [1] (🈯) Japanese “reserved” button
1F232..1F23A ; Extended_Pictographic# 6.0 [9] (🈲..🈺) Japanese “prohibited” button..Japanese “open for business” button
1F23C..1F23F ; Extended_Pictographic# NA [4] (🈼️..🈿️) <reserved-1F23C>..<reserved-1F23F>
1F249..1F24F ; Extended_Pictographic# NA [7] (🉉️..🉏️) <reserved-1F249>..<reserved-1F24F>
1F250..1F251 ; Extended_Pictographic# 6.0 [2] (🉐..🉑) Japanese “bargain” button..Japanese “acceptable” button
1F252..1F25F ; Extended_Pictographic# NA [14] (🉒️..🉟️) <reserved-1F252>..<reserved-1F25F>
1F260..1F265 ; Extended_Pictographic# 10.0 [6] (🉠️..🉥️) ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI
1F266..1F2FF ; Extended_Pictographic# NA[154] (🉦️..🋿️) <reserved-1F266>..<reserved-1F2FF>
1F300..1F320 ; Extended_Pictographic# 6.0 [33] (🌀..🌠) cyclone..shooting star
1F321..1F32C ; Extended_Pictographic# 7.0 [12] (🌡️..🌬️) thermometer..wind face
1F32D..1F32F ; Extended_Pictographic# 8.0 [3] (🌭..🌯) hot dog..burrito
1F330..1F335 ; Extended_Pictographic# 6.0 [6] (🌰..🌵) chestnut..cactus
1F336 ; Extended_Pictographic# 7.0 [1] (🌶️) hot pepper
1F337..1F37C ; Extended_Pictographic# 6.0 [70] (🌷..🍼) tulip..baby bottle
1F37D ; Extended_Pictographic# 7.0 [1] (🍽️) fork and knife with plate
1F37E..1F37F ; Extended_Pictographic# 8.0 [2] (🍾..🍿) bottle with popping cork..popcorn
1F380..1F393 ; Extended_Pictographic# 6.0 [20] (🎀..🎓) ribbon..graduation cap
1F394..1F39F ; Extended_Pictographic# 7.0 [12] (🎔️..🎟️) HEART WITH TIP ON THE LEFT..admission tickets
1F3A0..1F3C4 ; Extended_Pictographic# 6.0 [37] (🎠..🏄) carousel horse..person surfing
1F3C5 ; Extended_Pictographic# 7.0 [1] (🏅) sports medal
1F3C6..1F3CA ; Extended_Pictographic# 6.0 [5] (🏆..🏊) trophy..person swimming
1F3CB..1F3CE ; Extended_Pictographic# 7.0 [4] (🏋️..🏎️) person lifting weights..racing car
1F3CF..1F3D3 ; Extended_Pictographic# 8.0 [5] (🏏..🏓) cricket game..ping pong
1F3D4..1F3DF ; Extended_Pictographic# 7.0 [12] (🏔️..🏟️) snow-capped mountain..stadium
1F3E0..1F3F0 ; Extended_Pictographic# 6.0 [17] (🏠..🏰) house..castle
1F3F1..1F3F7 ; Extended_Pictographic# 7.0 [7] (🏱️..🏷️) WHITE PENNANT..label
1F3F8..1F3FA ; Extended_Pictographic# 8.0 [3] (🏸..🏺) badminton..amphora
1F400..1F43E ; Extended_Pictographic# 6.0 [63] (🐀..🐾) rat..paw prints
1F43F ; Extended_Pictographic# 7.0 [1] (🐿️) chipmunk
1F440 ; Extended_Pictographic# 6.0 [1] (👀) eyes
1F441 ; Extended_Pictographic# 7.0 [1] (👁️) eye
1F442..1F4F7 ; Extended_Pictographic# 6.0[182] (👂..📷) ear..camera
1F4F8 ; Extended_Pictographic# 7.0 [1] (📸) camera with flash
1F4F9..1F4FC ; Extended_Pictographic# 6.0 [4] (📹..📼) video camera..videocassette
1F4FD..1F4FE ; Extended_Pictographic# 7.0 [2] (📽️..📾️) film projector..PORTABLE STEREO
1F4FF ; Extended_Pictographic# 8.0 [1] (📿) prayer beads
1F500..1F53D ; Extended_Pictographic# 6.0 [62] (🔀..🔽) shuffle tracks button..downwards button
1F546..1F54A ; Extended_Pictographic# 7.0 [5] (🕆️..🕊️) WHITE LATIN CROSS..dove
1F54B..1F54F ; Extended_Pictographic# 8.0 [5] (🕋..🕏️) kaaba..BOWL OF HYGIEIA
1F550..1F567 ; Extended_Pictographic# 6.0 [24] (🕐..🕧) one o’clock..twelve-thirty
1F568..1F579 ; Extended_Pictographic# 7.0 [18] (🕨️..🕹️) RIGHT SPEAKER..joystick
1F57A ; Extended_Pictographic# 9.0 [1] (🕺) man dancing
1F57B..1F5A3 ; Extended_Pictographic# 7.0 [41] (🕻️..🖣️) LEFT HAND TELEPHONE RECEIVER..BLACK DOWN POINTING BACKHAND INDEX
1F5A4 ; Extended_Pictographic# 9.0 [1] (🖤) black heart
1F5A5..1F5FA ; Extended_Pictographic# 7.0 [86] (🖥️..🗺️) desktop computer..world map
1F5FB..1F5FF ; Extended_Pictographic# 6.0 [5] (🗻..🗿) mount fuji..moai
1F600 ; Extended_Pictographic# 6.1 [1] (😀) grinning face
1F601..1F610 ; Extended_Pictographic# 6.0 [16] (😁..😐) beaming face with smiling eyes..neutral face
1F611 ; Extended_Pictographic# 6.1 [1] (😑) expressionless face
1F612..1F614 ; Extended_Pictographic# 6.0 [3] (😒..😔) unamused face..pensive face
1F615 ; Extended_Pictographic# 6.1 [1] (😕) confused face
1F616 ; Extended_Pictographic# 6.0 [1] (😖) confounded face
1F617 ; Extended_Pictographic# 6.1 [1] (😗) kissing face
1F618 ; Extended_Pictographic# 6.0 [1] (😘) face blowing a kiss
1F619 ; Extended_Pictographic# 6.1 [1] (😙) kissing face with smiling eyes
1F61A ; Extended_Pictographic# 6.0 [1] (😚) kissing face with closed eyes
1F61B ; Extended_Pictographic# 6.1 [1] (😛) face with tongue
1F61C..1F61E ; Extended_Pictographic# 6.0 [3] (😜..😞) winking face with tongue..disappointed face
1F61F ; Extended_Pictographic# 6.1 [1] (😟) worried face
1F620..1F625 ; Extended_Pictographic# 6.0 [6] (😠..😥) angry face..sad but relieved face
1F626..1F627 ; Extended_Pictographic# 6.1 [2] (😦..😧) frowning face with open mouth..anguished face
1F628..1F62B ; Extended_Pictographic# 6.0 [4] (😨..😫) fearful face..tired face
1F62C ; Extended_Pictographic# 6.1 [1] (😬) grimacing face
1F62D ; Extended_Pictographic# 6.0 [1] (😭) loudly crying face
1F62E..1F62F ; Extended_Pictographic# 6.1 [2] (😮..😯) face with open mouth..hushed face
1F630..1F633 ; Extended_Pictographic# 6.0 [4] (😰..😳) anxious face with sweat..flushed face
1F634 ; Extended_Pictographic# 6.1 [1] (😴) sleeping face
1F635..1F640 ; Extended_Pictographic# 6.0 [12] (😵..🙀) dizzy face..weary cat face
1F641..1F642 ; Extended_Pictographic# 7.0 [2] (🙁..🙂) slightly frowning face..slightly smiling face
1F643..1F644 ; Extended_Pictographic# 8.0 [2] (🙃..🙄) upside-down face..face with rolling eyes
1F645..1F64F ; Extended_Pictographic# 6.0 [11] (🙅..🙏) person gesturing NO..folded hands
1F680..1F6C5 ; Extended_Pictographic# 6.0 [70] (🚀..🛅) rocket..left luggage
1F6C6..1F6CF ; Extended_Pictographic# 7.0 [10] (🛆️..🛏️) TRIANGLE WITH ROUNDED CORNERS..bed
1F6D0 ; Extended_Pictographic# 8.0 [1] (🛐) place of worship
1F6D1..1F6D2 ; Extended_Pictographic# 9.0 [2] (🛑..🛒) stop sign..shopping cart
1F6D3..1F6D4 ; Extended_Pictographic# 10.0 [2] (🛓️..🛔️) STUPA..PAGODA
1F6D5..1F6DF ; Extended_Pictographic# NA [11] (🛕️..🛟️) <reserved-1F6D5>..<reserved-1F6DF>
1F6E0..1F6EC ; Extended_Pictographic# 7.0 [13] (🛠️..🛬) hammer and wrench..airplane arrival
1F6ED..1F6EF ; Extended_Pictographic# NA [3] (🛭️..🛯️) <reserved-1F6ED>..<reserved-1F6EF>
1F6F0..1F6F3 ; Extended_Pictographic# 7.0 [4] (🛰️..🛳️) satellite..passenger ship
1F6F4..1F6F6 ; Extended_Pictographic# 9.0 [3] (🛴..🛶) kick scooter..canoe
1F6F7..1F6F8 ; Extended_Pictographic# 10.0 [2] (🛷..🛸) sled..flying saucer
1F6F9 ; Extended_Pictographic# 11.0 [1] (🛹) skateboard
1F6FA..1F6FF ; Extended_Pictographic# NA [6] (🛺️..🛿️) <reserved-1F6FA>..<reserved-1F6FF>
1F774..1F77F ; Extended_Pictographic# NA [12] (🝴️..🝿️) <reserved-1F774>..<reserved-1F77F>
1F7D5..1F7D8 ; Extended_Pictographic# 11.0 [4] (🟕️..🟘️) CIRCLED TRIANGLE..NEGATIVE CIRCLED SQUARE
1F7D9..1F7FF ; Extended_Pictographic# NA [39] (🟙️..🟿️) <reserved-1F7D9>..<reserved-1F7FF>
1F80C..1F80F ; Extended_Pictographic# NA [4] (🠌️..🠏️) <reserved-1F80C>..<reserved-1F80F>
1F848..1F84F ; Extended_Pictographic# NA [8] (🡈️..🡏️) <reserved-1F848>..<reserved-1F84F>
1F85A..1F85F ; Extended_Pictographic# NA [6] (🡚️..🡟️) <reserved-1F85A>..<reserved-1F85F>
1F888..1F88F ; Extended_Pictographic# NA [8] (🢈️..🢏️) <reserved-1F888>..<reserved-1F88F>
1F8AE..1F8FF ; Extended_Pictographic# NA [82] (🢮️..🣿️) <reserved-1F8AE>..<reserved-1F8FF>
1F90C..1F90F ; Extended_Pictographic# NA [4] (🤌️..🤏️) <reserved-1F90C>..<reserved-1F90F>
1F910..1F918 ; Extended_Pictographic# 8.0 [9] (🤐..🤘) zipper-mouth face..sign of the horns
1F919..1F91E ; Extended_Pictographic# 9.0 [6] (🤙..🤞) call me hand..crossed fingers
1F91F ; Extended_Pictographic# 10.0 [1] (🤟) love-you gesture
1F920..1F927 ; Extended_Pictographic# 9.0 [8] (🤠..🤧) cowboy hat face..sneezing face
1F928..1F92F ; Extended_Pictographic# 10.0 [8] (🤨..🤯) face with raised eyebrow..exploding head
1F930 ; Extended_Pictographic# 9.0 [1] (🤰) pregnant woman
1F931..1F932 ; Extended_Pictographic# 10.0 [2] (🤱..🤲) breast-feeding..palms up together
1F933..1F93A ; Extended_Pictographic# 9.0 [8] (🤳..🤺) selfie..person fencing
1F93C..1F93E ; Extended_Pictographic# 9.0 [3] (🤼..🤾) people wrestling..person playing handball
1F93F ; Extended_Pictographic# NA [1] (🤿️) <reserved-1F93F>
1F940..1F945 ; Extended_Pictographic# 9.0 [6] (🥀..🥅) wilted flower..goal net
1F947..1F94B ; Extended_Pictographic# 9.0 [5] (🥇..🥋) 1st place medal..martial arts uniform
1F94C ; Extended_Pictographic# 10.0 [1] (🥌) curling stone
1F94D..1F94F ; Extended_Pictographic# 11.0 [3] (🥍..🥏) lacrosse..flying disc
1F950..1F95E ; Extended_Pictographic# 9.0 [15] (🥐..🥞) croissant..pancakes
1F95F..1F96B ; Extended_Pictographic# 10.0 [13] (🥟..🥫) dumpling..canned food
1F96C..1F970 ; Extended_Pictographic# 11.0 [5] (🥬..🥰) leafy green..smiling face with 3 hearts
1F971..1F972 ; Extended_Pictographic# NA [2] (🥱️..🥲️) <reserved-1F971>..<reserved-1F972>
1F973..1F976 ; Extended_Pictographic# 11.0 [4] (🥳..🥶) partying face..cold face
1F977..1F979 ; Extended_Pictographic# NA [3] (🥷️..🥹️) <reserved-1F977>..<reserved-1F979>
1F97A ; Extended_Pictographic# 11.0 [1] (🥺) pleading face
1F97B ; Extended_Pictographic# NA [1] (🥻️) <reserved-1F97B>
1F97C..1F97F ; Extended_Pictographic# 11.0 [4] (🥼..🥿) lab coat..woman’s flat shoe
1F980..1F984 ; Extended_Pictographic# 8.0 [5] (🦀..🦄) crab..unicorn face
1F985..1F991 ; Extended_Pictographic# 9.0 [13] (🦅..🦑) eagle..squid
1F992..1F997 ; Extended_Pictographic# 10.0 [6] (🦒..🦗) giraffe..cricket
1F998..1F9A2 ; Extended_Pictographic# 11.0 [11] (🦘..🦢) kangaroo..swan
1F9A3..1F9AF ; Extended_Pictographic# NA [13] (🦣️..🦯️) <reserved-1F9A3>..<reserved-1F9AF>
1F9B0..1F9B9 ; Extended_Pictographic# 11.0 [10] (🦰..🦹) red-haired..supervillain
1F9BA..1F9BF ; Extended_Pictographic# NA [6] (🦺️..🦿️) <reserved-1F9BA>..<reserved-1F9BF>
1F9C0 ; Extended_Pictographic# 8.0 [1] (🧀) cheese wedge
1F9C1..1F9C2 ; Extended_Pictographic# 11.0 [2] (🧁..🧂) cupcake..salt
1F9C3..1F9CF ; Extended_Pictographic# NA [13] (🧃️..🧏️) <reserved-1F9C3>..<reserved-1F9CF>
1F9D0..1F9E6 ; Extended_Pictographic# 10.0 [23] (🧐..🧦) face with monocle..socks
1F9E7..1F9FF ; Extended_Pictographic# 11.0 [25] (🧧..🧿) red envelope..nazar amulet
1FA00..1FA5F ; Extended_Pictographic# NA [96] (🨀️..🩟️) <reserved-1FA00>..<reserved-1FA5F>
1FA60..1FA6D ; Extended_Pictographic# 11.0 [14] (🩠️..🩭️) XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER
1FA6E..1FFFD ; Extended_Pictographic# NA[1424] (🩮️..🿽️) <reserved-1FA6E>..<reserved-1FFFD>
# Total elements: 3793
#EOF

1027
Project/Tools/emoji-list.txt Normal file
View File

File diff suppressed because it is too large Load Diff

18
Project/Tools/stream_language_detector.py Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env python3
from langdetect import detect
import fileinput as fi
import sys
# just a little script to detect languages of lines starting
# with the json keyword 'text' and writing the language as json value to stdout.
# other lines are just passing through so that this script can be used in a shell pipeline
for line in fi.input():
if line.startswith(' "text"'):
try:
sys.stdout.write(' "lang": "' + detect(line[10:]) + '"\n')
except Exception:
sys.stdout.write(' "lang": "NaN"\n')
sys.stdout.write(line)

72
Project/Tools/twitter2messages.sh Executable file
View File

@ -0,0 +1,72 @@
#!/usr/bin/env bash
SCRIPT=`realpath $0`
SCRIPTPATH=`dirname $SCRIPT`
# toolset:---------------------------------------------------------------------
command 2> >(while read line; do echo -e "\e[01;31m$line\e[0m" >&2; done)
function lineprint {
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' =
}
function message {
lineprint
printf "$1\n"
lineprint
}
function error_message {
lineprint
printf "$1\n" >&2
lineprint
}
current_action="IDLE"
function confirm_action {
message "successfully finished action: $current_action"
}
function set_action {
current_action="$1"
message "$1"
}
function perform {
"$@"
local status=$?
if [ $status -ne 0 ]
then
error_message "$current_action failed!"
fi
return $status
}
function perform_and_exit {
perform "$@" || exit 1
}
# -----------------------------------------------------------------------------
INPUT=$1
OUTPUT=$2
if [ $# -ne 2 ]
then
error_message "Error: no input file given. Usage: $0 <Filepath> <outputfile>"
exit 1
fi
set_action "processing all files in $INPUT and write to $OUTPUT"
perform_and_exit export elist=\"`head -c -1 "$SCRIPTPATH/emoji-list.txt" | tr '\n' ',' | sed 's/,/\",\"/g'`\"
perform_and_exit echo "filter by emoji list:"
perform_and_exit echo $elist | tr -d '"' | tr -d ','
#perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.name, text: .text} | select(.text != null) | [select(.text | contains($elist))] | select(any)| unique_by(.id) | .[]" | tee /dev/tty > "$OUTPUT"
perform_and_exit find ./ -type f -name '*.bz2' -exec bzip2 -dc "{}" \; | jq ". | {id: .id, datetime: .created_at, person: .user.id, text: .text, lang: .lang, reply_to: .in_reply_to_status_id} | select(.text != null)" | grep --no-group-separator -Ff "$SCRIPTPATH/emoji-list.txt" -A 3 -B 4 | tee /dev/tty > "$OUTPUT"
# ↑ such obvious, much selfexplaining 💁😈
confirm_action

85
Project/Tools/whatsapp2csv.sh Executable file
View File

@ -0,0 +1,85 @@
#!/usr/bin/env bash
SCRIPT=`realpath $0`
SCRIPTPATH=`dirname $SCRIPT`
# toolset:---------------------------------------------------------------------
command 2> >(while read line; do echo -e "\e[01;31m$line\e[0m" >&2; done)
function lineprint {
printf '%*s\n' "${COLUMNS:-$(tput cols)}" '' | tr ' ' =
}
function message {
lineprint
printf "$1\n"
lineprint
}
function error_message {
lineprint
printf "$1\n" >&2
lineprint
}
current_action="IDLE"
function confirm_action {
message "successfully finished action: $current_action"
}
function set_action {
current_action="$1"
message "$1"
}
function perform {
"$@"
local status=$?
if [ $status -ne 0 ]
then
error_message "$current_action failed!"
fi
return $status
}
function perform_and_exit {
perform "$@" || exit 1
}
# -----------------------------------------------------------------------------
# reading input
INPUT=$1
OUTPUT="$INPUT.csv"
if [ $# -eq 0 ]
then
error_message "Error: no input file given. Usage: $0 <Whatsapp textbackup file>"
exit 1
fi
set_action "processing File: ${INPUT}"
if [ ! -e "$INPUT" ]
then
error_message "Error: file '$INPUT' not found"
exit 1
fi
perform_and_exit echo -e "date\ttime\tperson\tmessage" > "$OUTPUT"
# doing the following things in the pipeline below:
#
# 1. merging multiline messages by replacing newlines with spaces if they are not starting with a number followed by '/' (not stating with a date)
# 2. delete all lines containing the keywords: "added, creates, end-to-end, Media ommited" (which are most probably system messages)
# 3. replace separators of whatsapp's txt-format with '\t' as separator for csv
# 4. delete all double-quotes. Because people are too stupid to type them in pairs and that breaks some csv-interpreters 🤦‍♂
perform_and_exit sed ':a;N;/\n[0-9]\{1,\}\+\//!s/\n/ /;ta;P;D' "$INPUT" | grep -v -E 'left|added|created|end-to-end|Media omitted|import java|<VirtualHost' | sed 's/, /\t/1; s/- /\t/1; s/: /\t/1;' | tr -d '"' >> "$OUTPUT"
confirm_action
message "Wrote output to $OUTPUT"

4
README.md
View File

@ -39,4 +39,8 @@ Repository for the NLP-LAB
* search for predefined Questions in Knowledge Graph * search for predefined Questions in Knowledge Graph
----
## Links
https://github.com/AKSW/NSpM