diff --git a/loader.py b/loader.py index c2ad1d3..f6710da 100644 --- a/loader.py +++ b/loader.py @@ -70,6 +70,8 @@ def readVotes(folder): votes = dmt(items).map(mapvote, prefix + "mapping votes").getresults() print(prefix + "done") + items = None + gc.collect() return votes @@ -136,7 +138,7 @@ def mapuser(item): return user -Q_TAGS = ['Id', 'CreationDate', 'Body', 'Title', 'OwnerUserId', 'OwnerDisplayName', 'Score'] +Q_TAGS = ['Id', 'CreationDate', 'OwnerUserId', 'OwnerDisplayName', 'Score'] Q_DTAGS = ['CreationDate'] Q_ITAGS = ['Id', 'OwnerUserId', 'Score'] Q_BODY = 'Body' @@ -155,6 +157,7 @@ def mapQuestion(item): A_TAGS = ['Id', 'ParentId', 'CreationDate', 'Body', 'OwnerUserId', 'Score'] A_DTAGS = ['CreationDate'] A_ITAGS = ['Id', 'ParentId', 'OwnerUserId', 'Score'] +A_BODY = 'Body' def mapAnswer(item): @@ -163,7 +166,7 @@ def mapAnswer(item): answer[tag] = datetime.fromisoformat(answer[tag]) for tag in A_ITAGS: answer[tag] = int(answer[tag]) if answer[tag] is not None else None - answer['Body'] = removetags(html.unescape(answer['Body'])) + answer[A_BODY] = removetags(html.unescape(answer[A_BODY])) return answer