160 lines
6.8 KiB
Python
160 lines
6.8 KiB
Python
import os
|
|
import sys
|
|
from collections import defaultdict
|
|
from datetime import timedelta
|
|
|
|
import matplotlib.pyplot as plt
|
|
from matplotlib.ticker import MaxNLocator
|
|
|
|
from common import calc_intervals, IMAGE_MAGICK, DAYS_NEW_USER
|
|
from loader import load, dmt
|
|
from sentiments import readtoxleveltxt
|
|
|
|
colors = ['red', 'green', 'blue', 'orange', 'deeppink']
|
|
|
|
|
|
def main(folder, intervl):
|
|
users, posts, firstcontrib, sumcontrib = load(folder)
|
|
intervals = calc_intervals(posts, intervl)
|
|
(_, cachedsentiments) = readtoxleveltxt(folder + "/output/sentiments.txt")
|
|
|
|
outputdir = folder + "/output/posthist/"
|
|
os.system("mkdir -p " + outputdir)
|
|
|
|
activeusercounts = []
|
|
answerstonewusers = []
|
|
sentimentstonewusers = []
|
|
activitynewusers = []
|
|
imgmagickcmd = IMAGE_MAGICK
|
|
for (option_date_from, option_date_to) in intervals:
|
|
print(option_date_from.strftime("%d-%m-%Y"), option_date_to.strftime("%d-%m-%Y"))
|
|
|
|
# post histograms
|
|
# filter posts by option_date_from <= creation date <= option_date_to
|
|
newposts = dmt(posts).filter(lambda p: option_date_from <= p['CreationDate'] < option_date_to, "filtering posts by date").getresults()
|
|
|
|
postcounts = defaultdict(list)
|
|
i = 0
|
|
for p in newposts:
|
|
postcounts[p['OwnerUserId']].append(p)
|
|
i = i + 1
|
|
# for a in p['Answers']:
|
|
# postcounts[p['OwnerUserId']].append(a)
|
|
postcounts = {id: len(pc) for (id, pc) in postcounts.items()}
|
|
activeusercounts.append(((option_date_from, option_date_to), len(postcounts.keys())))
|
|
|
|
activitynewusersinmonth = defaultdict(int)
|
|
for p in newposts:
|
|
if firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) > p['CreationDate']:
|
|
activitynewusersinmonth[p['OwnerUserId']] += 1
|
|
for a in p['Answers']:
|
|
if firstcontrib[a['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) > a['CreationDate']:
|
|
activitynewusersinmonth[p['OwnerUserId']] += 1
|
|
activitysum = sum(activitynewusersinmonth.values())
|
|
activitynewusers.append(((option_date_from, option_date_to), activitysum / len(activitynewusersinmonth) if len(activitynewusersinmonth) > 0 else float("nan")))
|
|
|
|
histfilename = outputdir + "posthist_" + folder.split("/")[-1] + "_" + option_date_from.strftime("%d-%m-%Y") + "_" + option_date_to.strftime("%d-%m-%Y") + "-i" + str(intervl)
|
|
|
|
histdata = [pc for pc in postcounts.values()]
|
|
fig = plt.figure(figsize=(16, 12))
|
|
plt.hist(histdata, range(max(histdata, default=0) + 1))
|
|
plt.yscale('log')
|
|
plt.ylim(bottom=0)
|
|
plt.xlabel("#posts")
|
|
plt.ylabel("#users with X posts")
|
|
fig.gca().xaxis.set_major_locator(MaxNLocator(integer=True))
|
|
plt.title("Histogram for user post count registered between " + option_date_from.strftime("%d-%m-%Y") + " and " + option_date_to.strftime("%d-%m-%Y"))
|
|
fig.savefig(histfilename + ".png", bbox_inches='tight')
|
|
plt.close(fig)
|
|
imgmagickcmd += " " + histfilename + ".png"
|
|
|
|
# answers to new users
|
|
answers = (dmt(posts).map(lambda q: [a for a in q['Answers'] if option_date_from <= a['CreationDate'] < option_date_to
|
|
and firstcontrib[q['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) <= a['CreationDate']])
|
|
.getresults())
|
|
count = sum([len(a) for a in answers])
|
|
answerstonewusers.append(((option_date_from, option_date_to), count))
|
|
sent = ([cachedsentiments[a['Id']] for al in answers for a in al])
|
|
sentbad = len([1 for a in sent if a['compound'] < -0.05])
|
|
sentneu = len([1 for a in sent if -0.05 <= a['compound'] <= 0.05])
|
|
sentgood = len([1 for a in sent if a['compound'] > 0.05])
|
|
sentimentstonewusers.append(((option_date_from, option_date_to), (sent, sentbad, sentneu, sentgood)))
|
|
|
|
# gen pdf for post histograms
|
|
os.system(imgmagickcmd + " " + outputdir + "/posthist-i" + str(intervl) + ".pdf")
|
|
|
|
# plot posts diagram
|
|
fig = plt.figure(figsize=(16, 12))
|
|
plt.plot([x[0] for (x, y) in activeusercounts], [y for (x, y) in activeusercounts])
|
|
plt.xlabel('time')
|
|
plt.ylabel('#active users')
|
|
plt.yscale('log')
|
|
plt.ylim(bottom=1)
|
|
plt.title("Active users")
|
|
fig.savefig(outputdir + "activeusers-i" + str(intervl) + ".png", bbox_inches='tight')
|
|
plt.close(fig)
|
|
|
|
# plot answers to new users diagram
|
|
fig = plt.figure(figsize=(16, 12))
|
|
plt.plot([x[0] for (x, y) in answerstonewusers], [y for (x, y) in answerstonewusers])
|
|
plt.xlabel('time')
|
|
plt.ylabel('#answers per question of a new user')
|
|
plt.yscale('log')
|
|
plt.ylim(bottom=1)
|
|
plt.title("Answers to new users")
|
|
fig.savefig(outputdir + "answerstonewusers-i" + str(intervl) + ".png", bbox_inches='tight')
|
|
plt.close(fig)
|
|
|
|
# plot sentiments of answers to new users diagram
|
|
fig = plt.figure(figsize=(16, 12))
|
|
plt.plot([x[0] for (x, y) in sentimentstonewusers], [b for (x, [y, b, n, g]) in sentimentstonewusers], label="Neg. answer")
|
|
plt.plot([x[0] for (x, y) in sentimentstonewusers], [n for (x, [y, b, n, g]) in sentimentstonewusers], label="Neu. answer")
|
|
plt.plot([x[0] for (x, y) in sentimentstonewusers], [g for (x, [y, b, n, g]) in sentimentstonewusers], label="Pos. answer")
|
|
plt.xlabel('time')
|
|
plt.ylabel('sentiment')
|
|
plt.yscale('log')
|
|
plt.ylim(bottom=1)
|
|
plt.legend(loc="upper right")
|
|
plt.title("Sentiments of answers to new users")
|
|
fig.savefig(outputdir + "sentimentstonewusers-i" + str(intervl) + ".png", bbox_inches='tight')
|
|
plt.close(fig)
|
|
|
|
# plot activity for new users
|
|
fig = plt.figure(figsize=(16, 12))
|
|
plt.plot([x[0] for (x, y) in activitynewusers], [y for x, y in activitynewusers], label="activity")
|
|
plt.xlabel('time')
|
|
plt.ylabel('#questions or answers created by a new user')
|
|
plt.legend(loc="upper right")
|
|
plt.title("Average activity per new user")
|
|
fig.savefig(outputdir + "activitynewusers-i" + str(intervl) + ".png", bbox_inches='tight')
|
|
plt.close(fig)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# execute only if run as a script
|
|
usage = sys.argv[0] + " <folder>"
|
|
if len(sys.argv) < 2:
|
|
print(usage)
|
|
sys.exit(1)
|
|
folder = sys.argv[1]
|
|
if not os.path.isdir(folder):
|
|
print(folder + " is not a folder")
|
|
sys.exit(1)
|
|
interval = 3
|
|
if len(sys.argv) >= 3:
|
|
if sys.argv[2].startswith("-i"):
|
|
interval = sys.argv[2][2:]
|
|
try:
|
|
interval = int(interval)
|
|
except ValueError:
|
|
print("-i: int required")
|
|
sys.exit(1)
|
|
if interval < 1 or interval > 12:
|
|
print("-i: only 1 - 12")
|
|
sys.exit(1)
|
|
else:
|
|
print("unknown parameter: " + sys.argv[2])
|
|
sys.exit(1)
|
|
|
|
main(folder, interval)
|