wea_ondara/master
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

wip

Browse Source
This commit is contained in:
wea_ondara
2020-01-28 11:33:50 +01:00
parent c2695e0d49
commit 03c86683fb
7 changed files with 47 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
analyze_batch.py
View File

@@ -72,7 +72,7 @@ def main(folder, intervl):
userid = post['OwnerUserId']
# check first contribution
if firstcontrib[userid] + timedelta(days=DAYS_NEW_USER) < post['CreationDate']:
if firstcontrib[userid] + timedelta(days=DAYS_NEW_USER) <= post['CreationDate']:
continue
sortedposts[userid].append(post)
@@ -90,11 +90,11 @@ def main(folder, intervl):
for (i, post) in enumerate(filteredposts):
printnoln("\rcomputing toxic levels: post " + str(i + 1) + "/" + str(len(filteredposts)))
for a in post['Answers']:
if a['Id'] in cachedsentiments.keys():
# if a['Id'] in cachedsentiments.keys():
toxlevel = cachedsentiments[a['Id']]
else:
print("Sentiment not found for " + a['Id'])
continue
# else:
# print("Sentiment not found for " + a['Id'])
# continue
toxlevels.append(toxlevel)
printnoln("\rcomputing toxic levels: post " + str(len(filteredposts)) + "/" + str(len(filteredposts)) + " ... plotting ...")
@@ -178,10 +178,10 @@ def main(folder, intervl):
for (i, post) in enumerate(filteredposts):
printnoln("\rcomputing toxic levels: post " + str(i + 1) + "/" + str(len(filteredposts)))
for a in post['Answers']:
if a['Id'] in cachedsentiments.keys():
# if a['Id'] in cachedsentiments.keys():
toxlevel = cachedsentiments[a['Id']]
else:
print("Sentiment not found for " + a['Id'])
# else:
# print("Sentiment not found for " + a['Id'])
toxlevels.append(toxlevel)
printnoln("\rcomputing toxic levels: post " + str(len(filteredposts)) + "/" + str(len(filteredposts)) + " ... plotting ...")
avgsent[0].append(np.mean([s['compound'] for s in toxlevels]) if len(toxlevels) == 0 else 0)

2
calctoxdiff.py
View File

@@ -59,8 +59,8 @@ def g(srcfile, outputdir, intervals):
plt.ylabel("pvalue")
plt.legend(loc="upper right")
plt.savefig(outputdir + "/ks_averagesentiments_pval.png", bbox_inches='tight')
plt.close(fig)
fig = plt.figure(figsize=(16, 12))
for i in range(len(single)):
plt.plot([iv[0] for iv in intervals], [s if isinstance(s, float) else s.statistic for s in single[i]], label=str(i + 1) + " posts - most posters")

4
its.py
View File

@@ -42,8 +42,8 @@ def main(folder, intervl):
print(option_date_from.strftime("%d-%m-%Y") + " to " + option_date_to.strftime("%d-%m-%Y"))
# avg sentiments
filtered = (dmt(posts).map(lambda p: [cachedsentiments[a['Id']]['compound']
for a in p['Answers'] if option_date_from <= a['CreationDate'] < option_date_to
and firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) <= a['CreationDate']])
for a in p['Answers'] if option_date_from <= p['CreationDate'] < option_date_to
and firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) > a['CreationDate']])
.filter(lambda p: p != [])
.reduce(lambda a, b: a + b, lambda a, b: a + b, lambda: [])
.getresults())

28
loader.py
View File

@@ -72,15 +72,21 @@ def readVotes(folder):
print(prefix + "done")
return votes
VOTE_TAGS = ['PostId', 'VoteTypeId', 'CreationDate']
VOTE_DTAGS = ['CreationDate']
VOTE_ITAGS = ['PostId', 'VoteTypeId']
def mapvote(item):
datetags = ['CreationDate']
vote = {tag: getTag(item, tag) for tag in VOTE_TAGS}
for tag in datetags:
for tag in VOTE_DTAGS:
if vote[tag] is not None:
vote[tag] = datetime.fromisoformat(vote[tag])
else:
print("map vote: tag " + tag + " is None: " + str(vote))
for tag in VOTE_ITAGS:
vote[tag] = int(vote[tag])
return vote
@@ -112,8 +118,12 @@ def computefirstcontrib(posts):
firstcontrib = {id: min(ldate) for (id, ldate) in firstcontrib.items()}
return firstcontrib
USER_TAGS = ['Id', 'CreationDate']
USER_DTAGS = ['CreationDate']
USER_ITAGS = ['Id']
def mapuser(item):
user = {tag: getTag(item, tag) for tag in USER_TAGS}
for tag in USER_DTAGS:
@@ -121,24 +131,38 @@ def mapuser(item):
user[tag] = datetime.fromisoformat(user[tag])
else:
print("map user: tag " + tag + " is None: " + str(user))
for tag in USER_ITAGS:
user[tag] = int(user[tag])
return user
Q_TAGS = ['Id', 'CreationDate', 'Body', 'Title', 'OwnerUserId', 'OwnerDisplayName', 'Score']
Q_DTAGS = ['CreationDate']
Q_ITAGS = ['Id', 'OwnerUserId', 'Score']
Q_BODY = 'Body'
def mapQuestion(item):
question = {tag: getTag(item, tag) for tag in Q_TAGS}
for tag in Q_DTAGS:
question[tag] = datetime.fromisoformat(question[tag])
for tag in Q_ITAGS:
question[tag] = int(question[tag]) if question[tag] is not None else None
question[Q_BODY] = removetags(html.unescape(question[Q_BODY]))
return question
A_TAGS = ['Id', 'ParentId', 'CreationDate', 'Body', 'OwnerUserId', 'Score']
A_DTAGS = ['CreationDate']
A_ITAGS = ['Id', 'ParentId', 'OwnerUserId', 'Score']
def mapAnswer(item):
answer = {tag: getTag(item, tag) for tag in A_TAGS}
for tag in A_DTAGS:
answer[tag] = datetime.fromisoformat(answer[tag])
for tag in A_ITAGS:
answer[tag] = int(answer[tag]) if answer[tag] is not None else None
answer['Body'] = removetags(html.unescape(answer['Body']))
return answer

2
posthist.py
View File

@@ -43,7 +43,7 @@ def main(folder, intervl):
postcounts = {id: len(pc) for (id, pc) in postcounts.items()}
activeusercounts.append(((option_date_from, option_date_to), len(postcounts.keys())))
activitynewusersinmonth = defaultdict(int)
activitynewusersinmonth = defaultdict(int) # TODO match month exactly
for p in newposts:
if firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) > p['CreationDate']:
activitynewusersinmonth[p['OwnerUserId']] += 1

2
sentiments.py
View File

@@ -101,7 +101,7 @@ def readtoxleveltxt(filename):
ra = line.split(";")
ra = [l.split(":") for l in ra]
# print("i1: " + str(ra[0:5]))
ra = {id: {"neg": float(neg), "neu": float(neu), "pos": float(pos), "compound": float(compound)} for [id, neg, neu, pos, compound] in ra}
ra = {int(id): {"neg": float(neg), "neu": float(neu), "pos": float(pos), "compound": float(compound)} for [id, neg, neu, pos, compound] in ra}
# print("i1: " + str(ra)[0:500])
return rq, ra

12
votes.py
View File

@@ -36,12 +36,12 @@ def main(folder, intervl):
print(option_date_from.strftime("%d-%m-%Y") + " to " + option_date_to.strftime("%d-%m-%Y"))
# avg sentiments
scores = (dmt(posts).filter(lambda p: option_date_from <= p['CreationDate'] < option_date_to
and firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) <= p['CreationDate'])
.map(lambda p: int(p['Score']))
and firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) >= p['CreationDate'])
.map(lambda p: p['Score'])
.getresults())
filtered = (dmt(posts).map(lambda p: [cachedsentiments[a['Id']]['compound']
for a in p['Answers'] if option_date_from <= a['CreationDate'] < option_date_to
and firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) <= a['CreationDate']])
for a in p['Answers'] if option_date_from <= p['CreationDate'] < option_date_to
and firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) >= a['CreationDate']])
.filter(lambda p: p != [])
.reduce(lambda a, b: a + b, lambda a, b: a + b, lambda: [])
.getresults())
@@ -102,10 +102,10 @@ def main(folder, intervl):
if option_date_to > interval[1]:
continue
intervalposts = dmt(posts).filter(lambda p: option_date_from <= p['CreationDate'] < option_date_to
and firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) <= p['CreationDate']).getresults()
and firstcontrib[p['OwnerUserId']] + timedelta(days=DAYS_NEW_USER) >= p['CreationDate']).getresults()
intervalpostsids = set(dmt(intervalposts).map(lambda p: p['Id']).getresults())
intervalvotes = dmt(ivvotes).filter(lambda v: v['PostId'] in intervalpostsids).getresults()
intervalscore = sum(dmt(intervalvotes).map(lambda v: 1 if v['VoteTypeId'] == "2" else (-1 if v['VoteTypeId'] == "3" else 0)).getresults())
intervalscore = sum(dmt(intervalvotes).map(lambda v: 1 if v['VoteTypeId'] == 2 else (-1 if v['VoteTypeId'] == 3 else 0)).getresults())
intervalscore = intervalscore / len(intervalpostsids) if len(intervalpostsids) != 0 else float("nan")
scores.append(((option_date_from, option_date_to), intervalscore))
# if all(str(score) == "nan" for iv, score in scores)