wea_ondara/master
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

wip

Browse Source
This commit is contained in:
wea_ondara
2020-01-29 11:28:48 +01:00
parent 28f3476637
commit ed7d1d8d47
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
loader.py
View File

@@ -70,6 +70,8 @@ def readVotes(folder):
votes = dmt(items).map(mapvote, prefix + "mapping votes").getresults() votes = dmt(items).map(mapvote, prefix + "mapping votes").getresults()
print(prefix + "done") print(prefix + "done")
items = None
gc.collect()
return votes return votes
@@ -136,7 +138,7 @@ def mapuser(item):
return user return user
Q_TAGS = ['Id', 'CreationDate', 'Body', 'Title', 'OwnerUserId', 'OwnerDisplayName', 'Score'] Q_TAGS = ['Id', 'CreationDate', 'OwnerUserId', 'OwnerDisplayName', 'Score']
Q_DTAGS = ['CreationDate'] Q_DTAGS = ['CreationDate']
Q_ITAGS = ['Id', 'OwnerUserId', 'Score'] Q_ITAGS = ['Id', 'OwnerUserId', 'Score']
Q_BODY = 'Body' Q_BODY = 'Body'
@@ -155,6 +157,7 @@ def mapQuestion(item):
A_TAGS = ['Id', 'ParentId', 'CreationDate', 'Body', 'OwnerUserId', 'Score'] A_TAGS = ['Id', 'ParentId', 'CreationDate', 'Body', 'OwnerUserId', 'Score']
A_DTAGS = ['CreationDate'] A_DTAGS = ['CreationDate']
A_ITAGS = ['Id', 'ParentId', 'OwnerUserId', 'Score'] A_ITAGS = ['Id', 'ParentId', 'OwnerUserId', 'Score']
A_BODY = 'Body'
def mapAnswer(item): def mapAnswer(item):
@@ -163,7 +166,7 @@ def mapAnswer(item):
answer[tag] = datetime.fromisoformat(answer[tag]) answer[tag] = datetime.fromisoformat(answer[tag])
for tag in A_ITAGS: for tag in A_ITAGS:
answer[tag] = int(answer[tag]) if answer[tag] is not None else None answer[tag] = int(answer[tag]) if answer[tag] is not None else None
answer['Body'] = removetags(html.unescape(answer['Body'])) answer[A_BODY] = removetags(html.unescape(answer[A_BODY]))
return answer return answer