wip
This commit is contained in:
wea_ondara
@@ -8,6 +8,7 @@ import matplotlib.pyplot as plt
|
||||
from collections import defaultdict
|
||||
from loader import load, dmt, cms
|
||||
import math
|
||||
from common import calc_intervals
|
||||
|
||||
printnoln = lambda text: print(text, end='', flush=True)
|
||||
rprint = lambda text: print('\r' + text)
|
||||
@@ -23,6 +24,7 @@ def main(folder):
|
||||
users, posts, firstcontrib, sumcontrib = load(folder)
|
||||
|
||||
intervals = calc_intervals(posts)
|
||||
cachedsentiments = {}
|
||||
|
||||
postcounts = range(1, 5 + 1)
|
||||
for (option_date_from, option_date_to) in intervals:
|
||||
@@ -79,7 +81,11 @@ def main(folder):
|
||||
printnoln("\rcomputing toxic levels: post #" + str(i + 1) + "/" + str(len(filteredposts)))
|
||||
userid = post['OwnerUserId']
|
||||
for a in post['Answers']:
|
||||
toxlevel = computeToxLevel(a['Body'])
|
||||
if a['Id'] in cachedsentiments.keys():
|
||||
toxlevel = cachedsentiments[a['Id']]
|
||||
else:
|
||||
toxlevel = computeToxLevel(a['Body'])
|
||||
cachedsentiments[a['Id']] = toxlevel
|
||||
toxlevels[userid].append(toxlevel)
|
||||
rprint("computing toxic levels: post #" + str(len(filteredposts)) + "/" + str(len(filteredposts)) + " ... took " + str(cms() - start) + "ms")
|
||||
|
||||
@@ -99,23 +105,17 @@ def main(folder):
|
||||
|
||||
fig, axs = plt.subplots(2, 2, figsize=(16, 12))
|
||||
axs[0, 0].set_title('Neg')
|
||||
axs[0, 0].hist(neglevelsflat, np.linspace(-1, 1, 2 * 100))
|
||||
axs[1, 0].set_title('Neu')
|
||||
axs[1, 0].hist(neulevelsflat, np.linspace(-1, 1, 2 * 100))
|
||||
axs[0, 1].set_title('Pos')
|
||||
axs[0, 1].hist(poslevelsflat, np.linspace(-1, 1, 2 * 100))
|
||||
axs[1, 1].set_title('Compound')
|
||||
axs[0, 0].hist(neglevelsflat, np.linspace(0, 1, 1 * 100))
|
||||
axs[1, 0].hist(neulevelsflat, np.linspace(0, 1, 1 * 100))
|
||||
axs[0, 1].hist(poslevelsflat, np.linspace(0, 1, 1 * 100))
|
||||
axs[1, 1].hist(comlevelsflat, np.linspace(-1, 1, 2 * 100))
|
||||
|
||||
# global
|
||||
# gaxs[0, 0].hist(neglevelsflat, np.linspace(-1, 1, 2 * 100), label=str(option_posts) + " posts")
|
||||
# gaxs[1, 0].hist(neulevelsflat, np.linspace(-1, 1, 2 * 100), label=str(option_posts) + " posts")
|
||||
# gaxs[0, 1].hist(poslevelsflat, np.linspace(-1, 1, 2 * 100), label=str(option_posts) + " posts")
|
||||
# gaxs[1, 1].hist(comlevelsflat, np.linspace(-1, 1, 2 * 100), label=str(option_posts) + " posts")
|
||||
# gaxs[0, 0].hist(neglevelsflat, np.linspace(-1, 1, 2 * 100), alpha=1. / len(postcounts), label=str(option_posts) + " posts")
|
||||
# gaxs[1, 0].hist(neulevelsflat, np.linspace(-1, 1, 2 * 100), alpha=1. / len(postcounts), label=str(option_posts) + " posts")
|
||||
# gaxs[0, 1].hist(poslevelsflat, np.linspace(-1, 1, 2 * 100), alpha=1. / len(postcounts), label=str(option_posts) + " posts")
|
||||
# gaxs[1, 1].hist(comlevelsflat, np.linspace(-1, 1, 2 * 100), alpha=1. / len(postcounts), label=str(option_posts) + " posts")
|
||||
axs[0, 0].set_yscale('log')
|
||||
axs[1, 0].set_yscale('log')
|
||||
axs[0, 1].set_yscale('log')
|
||||
axs[1, 1].set_yscale('log')
|
||||
|
||||
# plt.show()
|
||||
fig.suptitle("Sentiment of answers to the first " + str(option_posts) + " (max) posts\nUsers registered between "
|
||||
@@ -124,22 +124,18 @@ def main(folder):
|
||||
plt.close(fig)
|
||||
|
||||
# global
|
||||
gaxs[0, 0].hist(gneg, np.linspace(-1, 1, 2 * 100), color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
gaxs[1, 0].hist(gneu, np.linspace(-1, 1, 2 * 100), color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
gaxs[0, 1].hist(gpos, np.linspace(-1, 1, 2 * 100), color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
gaxs[0, 0].hist(gneg, np.linspace(0, 1, 1 * 100), color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
gaxs[1, 0].hist(gneu, np.linspace(0, 1, 1 * 100), color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
gaxs[0, 1].hist(gpos, np.linspace(0, 1, 1 * 100), color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
gaxs[1, 1].hist(gcom, np.linspace(-1, 1, 2 * 100), color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
# gaxs[0, 0].hist(gneg, np.linspace(-1, 1, 2 * 100), alpha=1. / len(postcounts), color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
# gaxs[1, 0].hist(gneu, np.linspace(-1, 1, 2 * 100), alpha=1. / len(postcounts), color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
# gaxs[0, 1].hist(gpos, np.linspace(-1, 1, 2 * 100), alpha=1. / len(postcounts), color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
# gaxs[1, 1].hist(gcom, np.linspace(-1, 1, 2 * 100), alpha=1. / len(postcounts), color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
# gaxs[0, 0].hist(gneg, np.linspace(-1, 1, 2 * 100), stacked=True, color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
# gaxs[1, 0].hist(gneu, np.linspace(-1, 1, 2 * 100), stacked=True, color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
# gaxs[0, 1].hist(gpos, np.linspace(-1, 1, 2 * 100), stacked=True, color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
# gaxs[1, 1].hist(gcom, np.linspace(-1, 1, 2 * 100), stacked=True, color=colors[:len(postcounts)], label=[str(option_posts) + " posts" for option_posts in postcounts])
|
||||
gaxs[0, 0].legend(loc="upper right")
|
||||
gaxs[1, 0].legend(loc="upper right")
|
||||
gaxs[0, 1].legend(loc="upper right")
|
||||
gaxs[1, 1].legend(loc="upper right")
|
||||
gaxs[0, 0].set_yscale('log')
|
||||
gaxs[1, 0].set_yscale('log')
|
||||
gaxs[0, 1].set_yscale('log')
|
||||
gaxs[1, 1].set_yscale('log')
|
||||
gfig.suptitle("Sentiment of answers to the first X (max) posts\nUsers registered between " + option_date_from.strftime("%d-%m-%Y") + " to " + option_date_to.strftime("%d-%m-%Y"))
|
||||
gfig.savefig(goutfilename + ".png", bbox_inches='tight')
|
||||
plt.close(gfig)
|
||||
@@ -159,34 +155,6 @@ def dumptoxlevels(lvls, filename):
|
||||
file.write("toxlevels = " + str(lvls).replace("<class 'list'>", "list", 1) + "\n")
|
||||
|
||||
|
||||
def calc_intervals(posts):
|
||||
firstpost = dmt(posts).reduce(lambda acc, e: acc if acc < e['CreationDate'] else e['CreationDate'], lambda acc, e: acc if acc < e else e, lambda: posts[0]['CreationDate'], "firstpost").getresults()
|
||||
lastpost = dmt(posts).reduce(lambda acc, e: acc if acc > e['CreationDate'] else e['CreationDate'], lambda acc, e: acc if acc > e else e, lambda: posts[0]['CreationDate'], "lastpost").getresults()
|
||||
|
||||
# calc quarter beginning
|
||||
firstpost = firstpost.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||
if firstpost.month not in (1, 4, 7, 10):
|
||||
firstpost = firstpost.replace(month={1: 1, 2: 1, 3: 1, 4: 4, 5: 4, 6: 4, 7: 7, 8: 7, 9: 7, 10: 10, 11: 10, 12: 10}[firstpost.month])
|
||||
lastpost = lastpost.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
|
||||
if lastpost.month not in (1, 4, 7, 10):
|
||||
lastpost = lastpost.replace(month={1: 1, 2: 1, 3: 1, 4: 4, 5: 4, 6: 4, 7: 7, 8: 7, 9: 7, 10: 10, 11: 10, 12: 10}[lastpost.month])
|
||||
# add 3 months to last post
|
||||
if lastpost.month == 10:
|
||||
lastpost = lastpost.replace(month=1, year=lastpost.year + 1)
|
||||
else:
|
||||
lastpost = lastpost.replace(month=lastpost.month + 3)
|
||||
|
||||
cdate = firstpost
|
||||
intervals = []
|
||||
while cdate < lastpost:
|
||||
nextquarter = cdate.replace(month=(cdate.month + 3) % 12, year=cdate.year + (0 if cdate.month + 3 < 12 else 1))
|
||||
print("adding interval: " + cdate.strftime("%d-%m-%Y") + " - " + nextquarter.strftime("%d-%m-%Y"))
|
||||
intervals.append((cdate, nextquarter))
|
||||
cdate = nextquarter
|
||||
# sys.exit(0)
|
||||
return intervals
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# execute only if run as a script
|
||||
usage = sys.argv[0] + " <folder>"
|
||||
|
||||
Reference in New Issue
Block a user