wea_ondara/master
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

wip

Browse Source
This commit is contained in:
wea_ondara
2019-12-18 13:02:16 +01:00
parent 356eefaf53
commit 2c1524a335
4 changed files with 245 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
its.py
View File

@@ -1,17 +1,17 @@
import os
import os
import sys
from datetime import datetime
from datetime import timedelta
import matplotlib.pyplot as plt
import numpy as np
import os
from datetime import datetime
from datetime import timedelta
from sklearn.linear_model import LinearRegression
from common import calc_intervals, imprt, printnoln, rprint, DAYS_NEW_USER
from common import calc_intervals, printnoln, rprint, DAYS_NEW_USER
from loader import load, dmt, cms
OLD_USER_PERCENTILE = 0.95
from sentiments import readtoxleveltxt
import statsmodels.api as sm
colors = ['red', 'green', 'blue', 'orange', 'deeppink']
@@ -23,33 +23,31 @@ def main(folder, intervl):
start = cms()
printnoln("reading sentiments ...")
cachedsentiments = imprt(folder + "/output/sentiments.py").answers
(_, cachedsentiments) = readtoxleveltxt(folder + "/output/sentiments.txt")
rprint("reading sentiments ... took " + str(cms() - start) + "ms")
outputdir = folder + "/output/its/"
os.system("mkdir -p " + outputdir)
data = []
datasingle = []
count = []
for (option_date_from, option_date_to) in intervals:
if option_date_to <= datetime.fromisoformat("2015-01-01T00:00:00"):
data.append(float("nan"))
continue
print(option_date_from.strftime("%d-%m-%Y") + " to " + option_date_to.strftime("%d-%m-%Y"))
# avg sentiments
# print(dmt(posts).map(lambda p: [cachedsentiments[a['Id']]['compound']
# for a in p['Answers'] if option_date_from <= a['CreationDate'] < option_date_to
# and firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) <= a['CreationDate']])
# .filter(lambda p: p != [])
# .getresults())
# break
filtered = (dmt(posts).map(lambda p: [cachedsentiments[a['Id']]['compound']
for a in p['Answers'] if option_date_from <= a['CreationDate'] < option_date_to
and firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) <= a['CreationDate']])
.filter(lambda p: p != [])
.reduce(lambda a, b: a + b, lambda a, b: a + b, lambda: [])
.getresults())
datasingle.append(filtered)
avg = np.average(filtered) if len(filtered) > 0 else float("nan")
data.append(avg)
count.append(len(filtered))
# filter nan entries
for i in range(len(data)):
@@ -57,37 +55,110 @@ def main(folder, intervl):
del data[i]
del intervals[i]
print("Computing ITS ...")
t = np.reshape(np.array([i for i in range(len(data))]), (-1, 1))
# print("t", t)
x = np.reshape(np.array([(0 if option_date_to <= datetime.fromisoformat("2018-09-01T00:00:00") else 1) for (option_date_from, option_date_to) in intervals]), (-1, 1))
# print("x", x)
X = np.reshape(np.array([data[0] for i in range(len(data))]), (-1, 1))
# print("X", X)
X = np.concatenate((X, t), 1)
# print("Computing ITS ...")
# t = np.reshape(np.array([i for i in range(len(data))]), (-1, 1))
# x = np.reshape(np.array([(0 if option_date_to <= datetime.fromisoformat("2018-09-01T00:00:00") else 1) for (option_date_from, option_date_to) in intervals]), (-1, 1))
# X = np.array(t)
# X = np.concatenate((X, x), 1)
# X = np.concatenate((X, np.multiply(t, x)), 1)
# y = np.reshape(np.array(data), (-1, 1))
# # print("Xfin", X)
# # print("y", y)
# reg = LinearRegression()
# reg.fit(X, y)
# score = reg.score(X, y)
# coef = np.reshape(np.array(reg.coef_), (-1, 1))
# its = X.dot(coef) + reg.intercept_
# print("score: " + str(score))
# print("coef: " + str(coef))
# print("its: " + str(its))
print("Computing full ITS")
t = np.reshape(np.array([i for i in range(len(datasingle)) for j in datasingle[i]]), (-1, 1))
x = np.reshape(np.array([(0 if intervals[i][1] <= datetime.fromisoformat("2018-09-01T00:00:00") else 1) for i in range(len(datasingle)) for j in datasingle[i]]), (-1, 1))
X = np.array(t)
X = np.concatenate((X, x), 1)
X = np.concatenate((X, np.multiply(t, x)), 1)
y = np.reshape(np.array(data), (-1, 1))
y = np.reshape(np.array([d for a in datasingle for d in a]), (-1, 1))
# print("Xfin", X)
# print("y", y)
reg = LinearRegression()
reg.fit(X, y)
score = reg.score(X, y);
coef = np.reshape(np.array(reg.coef_), (-1, 1))
its = X.dot(coef) + data[0]
print("score: " + str(score))
print("coef: " + str(coef))
print("its: " + str(its))
# reg = LinearRegression()
# reg.fit(X, y)
# score2 = reg.score(X, y)
# coef2 = np.reshape(np.array(reg.coef_), (-1, 1))
# its2 = X.dot(coef2) + reg.intercept_
# print("intercept: " + str(reg.intercept_))
# print("score: " + str(score2))
# print("coef: " + str(coef2))
# print("its: " + str(its2))
X = sm.add_constant(X)
res = sm.OLS(y, X).fit()
p2 = res.pvalues
print("coef ols: " + str(res.params))
print("sum ols: " + str(res.summary()))
coef2ols = np.reshape(np.array(res.params), (-1, 1))
its2ols = X.dot(coef2ols)
with open(outputdir + "/summary-i" + str(intervl) + ".txt", "w") as file:
file.write(str(res.summary()))
# print("Computing segmented ITS before")
# X = np.reshape(np.array([i for i in range(len(datasingle)) for j in datasingle[i] if intervals[i][1] <= datetime.fromisoformat("2018-09-01T00:00:00")]), (-1, 1))
# y = np.reshape(np.array([j for i in range(len(datasingle)) for j in datasingle[i] if intervals[i][1] <= datetime.fromisoformat("2018-09-01T00:00:00")]), (-1, 1))
# reg = LinearRegression()
# reg.fit(X, y)
# scoreb = reg.score(X, y)
# coefb = np.reshape(np.array(reg.coef_), (-1, 1))
# itsb = X.dot(coefb) + reg.intercept_
# print("scoreb: " + str(scoreb))
# print("coefb: " + str(coefb))
# print("itsb: " + str(itsb))
# print("Computing segmented ITS after")
# X = np.reshape(np.array([i for i in range(len(datasingle)) for j in datasingle[i] if intervals[i][1] > datetime.fromisoformat("2018-09-01T00:00:00")]), (-1, 1))
# y = np.reshape(np.array([j for i in range(len(datasingle)) for j in datasingle[i] if intervals[i][1] > datetime.fromisoformat("2018-09-01T00:00:00")]), (-1, 1))
# reg = LinearRegression()
# reg.fit(X, y)
# scorea = reg.score(X, y)
# coefa = np.reshape(np.array(reg.coef_), (-1, 1))
# itsa = X.dot(coefa) + reg.intercept_
# print("scorea: " + str(scorea))
# print("coefa: " + str(coefa))
# print("itsa: " + str(itsa))
fig = plt.figure(figsize=(16, 12))
plt.plot([i[0] for i in intervals], data, label="average sentiment")
plt.plot([i[0] for i in intervals], its, label="ITS (score " + str(score) + ")")
plt.grid(True)
for i in range(len(data)):
va = "center"
if 0 < i < len(data) - 1:
if data[i - 1] < data[i] and data[i + 1] < data[i]:
va = "bottom"
elif data[i - 1] > data[i] and data[i + 1] > data[i]:
va = "top"
elif i == 0:
if data[i + 1] < data[i]:
va = "bottom"
else:
va = "top"
elif i == len(data) - 1:
if data[i - 1] < data[i]:
va = "bottom"
else:
va = "top"
plt.text(intervals[i][0], data[i], ("n=" if i == 0 else "") + str(len(datasingle[i])), ha="center", va=va)
# plt.plot([i[0] for i in intervals], its, label="aggregated ITS (score " + str(score) + ")")
# plt.plot([intervals[i][0] for i in range(len(datasingle)) for j in datasingle[i]], its2, label="single ITS (score " + str(score2) + ", p " + str(p2) + ")")
plt.plot([intervals[i][0] for i in range(len(datasingle)) for j in datasingle[i]], its2ols, label="sm single ITS (pvalues " + str(p2) + ")")
# plt.plot([intervals[i][0] for i in range(len(datasingle)) for j in datasingle[i] if intervals[i][1] <= datetime.fromisoformat("2018-09-01T00:00:00")], itsb,
# label="segmented ITS b (score " + str(scoreb) + ")")
# plt.plot([intervals[i][0] for i in range(len(datasingle)) for j in datasingle[i] if intervals[i][1] > datetime.fromisoformat("2018-09-01T00:00:00")], itsa,
# label="segmented ITS a (score " + str(scorea) + ")")
plt.title("Average sentiments for new users")
plt.xticks(rotation=90)
plt.xlabel("months")
plt.ylabel("sentiment")
plt.legend(loc="upper right")
outfile = outputdir + "/average_sentiments.png"
outfile = outputdir + "/average_sentiments-i" + str(intervl) + ".png"
plt.savefig(outfile, bbox_inches='tight')
plt.close(fig)