-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathnetwork_job.py
307 lines (244 loc) · 9.68 KB
/
network_job.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
# Data Analysis
import pandas as pd
import numpy as np
from numpy import asarray
from numpy import savetxt
from numpy import loadtxt
import pickle as pkl
from scipy import sparse
# Data Visualization
import seaborn as sns
import matplotlib.pyplot as plt
import wordcloud
from wordcloud import WordCloud, STOPWORDS
# Text Processing
import re
import itertools
import string
import collections
from collections import Counter
from sklearn.preprocessing import LabelEncoder
import nltk
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import stopwords
from nltk import word_tokenize
from nltk.tokenize import word_tokenize
from nltk.stem import PorterStemmer, WordNetLemmatizer
# Machine Learning packages
import sklearn
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
import sklearn.cluster as cluster
from sklearn.manifold import TSNE
# Model training and evaluation
from sklearn.model_selection import train_test_split
#Models
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import SGDClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from xgboost import XGBClassifier
from xgboost import plot_importance
#Metrics
from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error, accuracy_score, balanced_accuracy_score
from sklearn.metrics import precision_score, recall_score, f1_score, multilabel_confusion_matrix, confusion_matrix
from sklearn.metrics import classification_report
# Ignore noise warning
import warnings
warnings.filterwarnings("ignore")
#loading dataset
data = pd.read_csv("D:/cis_mbti/mbti_1.csv")
data.tail()
def get_types(row):
t = row['type']
I = 0;
N = 0
T = 0;
J = 0
if t[0] == 'I':
I = 1
elif t[0] == 'E':
I = 0
else:
print('I-E not found')
if t[1] == 'N':
N = 1
elif t[1] == 'S':
N = 0
else:
print('N-S not found')
if t[2] == 'T':
T = 1
elif t[2] == 'F':
T = 0
else:
print('T-F not found')
if t[3] == 'J':
J = 1
elif t[3] == 'P':
J = 0
else:
print('J-P not found')
return pd.Series({'IE': I, 'NS': N, 'TF': T, 'JP': J})
data = data.join(data.apply(lambda row: get_types(row), axis=1))
print(data.head(5))
print ("Introversion (I) / Extroversion (E):\t", data['IE'].value_counts()[0], " / ", data['IE'].value_counts()[1])
print ("Intuition (N) / Sensing (S):\t\t", data['NS'].value_counts()[0], " / ", data['NS'].value_counts()[1])
print ("Thinking (T) / Feeling (F):\t\t", data['TF'].value_counts()[0], " / ", data['TF'].value_counts()[1])
print ("Judging (J) / Perceiving (P):\t\t", data['JP'].value_counts()[0], " / ", data['JP'].value_counts()[1])
lemmatiser = WordNetLemmatizer()
# Remove the stop words for speed
useless_words = stopwords.words("english")
# Remove these from the posts
unique_type_list = ['INFJ', 'ENTP', 'INTP', 'INTJ', 'ENTJ', 'ENFJ', 'INFP', 'ENFP',
'ISFP', 'ISTP', 'ISFJ', 'ISTJ', 'ESTP', 'ESFP', 'ESTJ', 'ESFJ']
unique_type_list = [x.lower() for x in unique_type_list]
# Or we can use Label Encoding (as above) of this unique personality type indicator list
# from sklearn.preprocessing import LabelEncoder
# unique_type_list = ['INFJ', 'ENTP', 'INTP', 'INTJ', 'ENTJ', 'ENFJ', 'INFP', 'ENFP',
# 'ISFP', 'ISTP', 'ISFJ', 'ISTJ', 'ESTP', 'ESFP', 'ESTJ', 'ESFJ']
# lab_encoder = LabelEncoder().fit(unique_type_list)
# Splitting the MBTI personality into 4 letters and binarizing it
b_Pers = {'I':0, 'E':1, 'N':0, 'S':1, 'T':0, 'F':1, 'J':0, 'P':1}
b_Pers_list = [{0:'I', 1:'E'}, {0:'N', 1:'S'}, {0:'T', 1:'F'}, {0:'J', 1:'P'}]
def translate_personality(personality):
# transform mbti to binary vector
return [b_Pers[l] for l in personality]
#To show result output for personality prediction
def translate_back(personality):
# transform binary vector to mbti personality
s = ""
for i, l in enumerate(personality):
s += b_Pers_list[i][l]
return s
list_personality_bin = np.array([translate_personality(p) for p in data.type])
print("Binarize MBTI list: \n%s" % list_personality_bin)
def pre_process_text(data, remove_stop_words=True, remove_mbti_profiles=True):
list_personality = []
list_posts = []
len_data = len(data)
i = 0
for row in data.iterrows():
# check code working
# i+=1
# if (i % 500 == 0 or i == 1 or i == len_data):
# print("%s of %s rows" % (i, len_data))
# Remove and clean comments
posts = row[1].posts
# Remove url links
temp = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', ' ', posts)
# Remove Non-words - keep only words
temp = re.sub("[^a-zA-Z]", " ", temp)
# Remove spaces > 1
temp = re.sub(' +', ' ', temp).lower()
# Remove multiple letter repeating words
temp = re.sub(r'([a-z])\1{2,}[\s|\w]*', '', temp)
# Remove stop words
if remove_stop_words:
temp = " ".join([lemmatiser.lemmatize(w) for w in temp.split(' ') if w not in useless_words])
else:
temp = " ".join([lemmatiser.lemmatize(w) for w in temp.split(' ')])
# Remove MBTI personality words from posts
if remove_mbti_profiles:
for t in unique_type_list:
temp = temp.replace(t, "")
# transform mbti to binary vector
type_labelized = translate_personality(row[1].type) # or use lab_encoder.transform([row[1].type])[0]
list_personality.append(type_labelized)
# the cleaned data temp is passed here
list_posts.append(temp)
# returns the result
list_posts = np.array(list_posts)
list_personality = np.array(list_personality)
return list_posts, list_personality
list_posts, list_personality = pre_process_text(data, remove_stop_words=True, remove_mbti_profiles=True)
print("Example :")
print("\nPost before preprocessing:\n\n", data.posts[0])
print("\nPost after preprocessing:\n\n", list_posts[0])
print("\nMBTI before preprocessing:\n\n", data.type[0])
print("\nMBTI after preprocessing:\n\n", list_personality[0])
#datasetmake
def getRandomIndex(n, x):
# 索引范围为[0, n)，随机选x个不重复，注意replace=False才是不重复，replace=True则有可能重复
index = np.random.choice(np.arange(n), size=x, replace=False)
return index
# 先根据上面的函数获取test_index
test_index = np.array(getRandomIndex(list_posts.shape[0], int(list_posts.shape[0]*0.4)))
# 再讲test_index从总的index中减去就得到了train_index
train_index = np.delete(np.arange(list_posts.shape[0]), test_index)
print(list_posts.shape)
trainx=[]
for i in train_index:
trainx.append(list_posts[i])
#print()
trainx=np.array(trainx)
print(trainx.shape)
print(list_personality.shape)
trainy=[]
for i in train_index:
trainy.append(list_personality[i])
#print(trainy.shape)
trainy=np.array(trainy)
print(trainy.shape)
print(list_posts.shape)
valx=[]
for i in test_index:
valx.append(list_posts[i])
#print(valx.shape)
valx=np.array(valx)
print(valx.shape)
print(list_personality.shape)
valy=[]
for i in test_index:
valy.append(list_personality[i])
#print(valy.shape)
valy=np.array(valy)
print(valy.shape)
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
vocab_size = 10000
trunc_type = "post"
pad_type = "post"
oov_tok = "<OOV>"
tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok)
tokenizer.fit_on_texts(list_posts)
maxlen = 1500
train_sequences = tokenizer.texts_to_sequences(trainx)
train_padded = pad_sequences(train_sequences, maxlen = maxlen, truncating = trunc_type, padding = pad_type)
val_sequences = tokenizer.texts_to_sequences(valx)
val_padded = pad_sequences(val_sequences, maxlen = maxlen, truncating = trunc_type, padding = pad_type)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional, Flatten, Dropout, Conv1D, GlobalMaxPooling1D
def create_model():
op = tf.keras.optimizers.Adam(learning_rate=0.00001)
model = Sequential()
model.add(Embedding(vocab_size, 256, input_length=maxlen-1))
model.add(Dropout(0.3))
model.add(Bidirectional(LSTM(200, return_sequences=True)))
model.add(Dropout(0.3))
model.add(Bidirectional(LSTM(20)))
model.add(Dropout(0.3))
model.add(Dense(64, activation='relu'))
model.add(Dense(1, activation='softmax'))
model.compile(loss='binary_crossentropy', optimizer=op, metrics=['accuracy'])
return model
lstmmodel_IE = create_model()
lstmmodel_IE.summary()
lstmmodel_IE.fit(train_padded, trainy[:,0], epochs =20, verbose = 1,
validation_data = (val_padded, valy[:,0]), callbacks = [tf.keras.callbacks.EarlyStopping(patience = 3)])
lstmmodel_NS = create_model()
lstmmodel_NS.summary()
lstmmodel_NS.fit(train_padded, trainy[:,1], epochs =20, verbose = 1,
validation_data = (val_padded, valy[:,1]), callbacks = [tf.keras.callbacks.EarlyStopping(patience = 3)])
lstmmodel_TF = create_model()
lstmmodel_TF.summary()
lstmmodel_TF.fit(train_padded, trainy[:,2], epochs =20, verbose = 1,
validation_data = (val_padded, valy[:,2]), callbacks = [tf.keras.callbacks.EarlyStopping(patience = 3)])
lstmmodel_JP = create_model()
lstmmodel_JP.summary()
lstmmodel_JP.fit(train_padded, trainy[:,3], epochs =20, verbose = 1,
validation_data = (val_padded, valy[:,3]), callbacks = [tf.keras.callbacks.EarlyStopping(patience = 3)])