wip
This commit is contained in:
wea_ondara
188
calctoxdiff.py
Normal file
188
calctoxdiff.py
Normal file
@@ -0,0 +1,188 @@
|
||||
import importlib
|
||||
import sys
|
||||
import os
|
||||
from os import listdir
|
||||
from os.path import isfile, join
|
||||
from scipy.stats import ks_2samp
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def main(folder):
|
||||
if folder.endswith("/"):
|
||||
folder = folder[:-1]
|
||||
onlyfiles = [f for f in listdir(folder)]
|
||||
onlyfiles = [f for f in onlyfiles if isfile(join(folder, f))]
|
||||
onlyfiles = [f for f in onlyfiles if f.endswith(".py")]
|
||||
# onlyfiles = [f[:-3] for f in onlyfiles]
|
||||
# onlyfiles = [f.replace(".", "\.") for f in onlyfiles]
|
||||
|
||||
onlyfiles = sorted(onlyfiles)
|
||||
plotbypost(onlyfiles)
|
||||
plotbydate(onlyfiles)
|
||||
|
||||
|
||||
def plotbypost(onlyfiles):
|
||||
files = defaultdict(list)
|
||||
for f in onlyfiles:
|
||||
s = f[:-3].split("_")
|
||||
files[int(s[4])].append(f)
|
||||
files = {p: sorted(l, key=lambda e: datetime.strptime(e.split("_")[2], "%d-%m-%Y")) for (p, l) in files.items()}
|
||||
|
||||
changes_neg = defaultdict(list)
|
||||
changes_neu = defaultdict(list)
|
||||
changes_pos = defaultdict(list)
|
||||
changes_com = defaultdict(list)
|
||||
|
||||
for (p, l) in files.items():
|
||||
if len(l) < 2:
|
||||
continue
|
||||
print(p)
|
||||
for i in range(len(l) - 1):
|
||||
tox1 = imprt(folder + "/" + l[i]).toxlevels
|
||||
tox2 = imprt(folder + "/" + l[i + 1]).toxlevels
|
||||
|
||||
neglevelsflat1 = [item['neg'] for item in flatmap(tox1.values())]
|
||||
neulevelsflat1 = [item['neu'] for item in flatmap(tox1.values())]
|
||||
poslevelsflat1 = [item['pos'] for item in flatmap(tox1.values())]
|
||||
comlevelsflat1 = [item['compound'] for item in flatmap(tox1.values())]
|
||||
|
||||
neglevelsflat2 = [item['neg'] for item in flatmap(tox2.values())]
|
||||
neulevelsflat2 = [item['neu'] for item in flatmap(tox2.values())]
|
||||
poslevelsflat2 = [item['pos'] for item in flatmap(tox2.values())]
|
||||
comlevelsflat2 = [item['compound'] for item in flatmap(tox2.values())]
|
||||
|
||||
ksneg = ks_2samp(neglevelsflat1, neglevelsflat2)
|
||||
ksneu = ks_2samp(neulevelsflat1, neulevelsflat2)
|
||||
kspos = ks_2samp(poslevelsflat1, poslevelsflat2)
|
||||
kscom = ks_2samp(comlevelsflat1, comlevelsflat2)
|
||||
|
||||
changes_neg[p].append(ksneg)
|
||||
changes_neu[p].append(ksneu)
|
||||
changes_pos[p].append(kspos)
|
||||
changes_com[p].append(kscom)
|
||||
|
||||
for (p, l) in files.items():
|
||||
with open(folder + "/ks_" + str(p) + ".log", "w") as f:
|
||||
for i in range(len(l) - 1):
|
||||
f1 = l[i]
|
||||
f2 = l[i + 1]
|
||||
f.write(f1 + " -> " + f2 + ": ks neg = " + str(changes_neg[p][i]) + "; ks neu = " + str(changes_neu[p][i])
|
||||
+ "; ks pos = " + str(changes_pos[p][i]) + "; ks com = " + str(changes_com[p][i]) + "\n")
|
||||
|
||||
for (p, l) in files.items():
|
||||
x = [l[i].split("_")[2] + " -\n" + l[i + 1].split("_")[2] for i in range(len(l) - 1)]
|
||||
fig = plt.figure(figsize=(16, 12))
|
||||
for type, changes in {"neg": changes_neg[p], "neu": changes_neu[p], "pos": changes_pos[p], "com": changes_com[p]}.items():
|
||||
stat = [x.statistic for x in changes]
|
||||
pval = [x.pvalue for x in changes]
|
||||
plt.plot(x, stat, label=type + ".stat")
|
||||
plt.plot(x, pval, label=type + ".pval")
|
||||
plt.title("KS 2-sided test with max " + str(p) + " posts")
|
||||
plt.xticks(rotation=90)
|
||||
plt.legend(loc="upper right")
|
||||
plt.savefig(folder + "/ks_" + str(p) + ".png", bbox_inches='tight')
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def plotbydate(onlyfiles):
|
||||
files = defaultdict(list)
|
||||
for f in onlyfiles:
|
||||
s = f[:-3].split("_")
|
||||
files[(s[2], s[3])].append(f)
|
||||
files = {d: sorted(l, key=lambda e: e.split("_")[4]) for (d, l) in files.items()}
|
||||
|
||||
changes_neg = defaultdict(list)
|
||||
changes_neu = defaultdict(list)
|
||||
changes_pos = defaultdict(list)
|
||||
changes_com = defaultdict(list)
|
||||
|
||||
for (d, l) in files.items():
|
||||
if len(l) < 2:
|
||||
continue
|
||||
print(d)
|
||||
for i in range(len(l) - 1):
|
||||
tox1 = imprt(folder + "/" + l[i]).toxlevels
|
||||
tox2 = imprt(folder + "/" + l[i + 1]).toxlevels
|
||||
|
||||
neglevelsflat1 = [item['neg'] for item in flatmap(tox1.values())]
|
||||
neulevelsflat1 = [item['neu'] for item in flatmap(tox1.values())]
|
||||
poslevelsflat1 = [item['pos'] for item in flatmap(tox1.values())]
|
||||
comlevelsflat1 = [item['compound'] for item in flatmap(tox1.values())]
|
||||
|
||||
neglevelsflat2 = [item['neg'] for item in flatmap(tox2.values())]
|
||||
neulevelsflat2 = [item['neu'] for item in flatmap(tox2.values())]
|
||||
poslevelsflat2 = [item['pos'] for item in flatmap(tox2.values())]
|
||||
comlevelsflat2 = [item['compound'] for item in flatmap(tox2.values())]
|
||||
|
||||
ksneg = ks_2samp(neglevelsflat1, neglevelsflat2)
|
||||
ksneu = ks_2samp(neulevelsflat1, neulevelsflat2)
|
||||
kspos = ks_2samp(poslevelsflat1, poslevelsflat2)
|
||||
kscom = ks_2samp(comlevelsflat1, comlevelsflat2)
|
||||
|
||||
changes_neg[d].append(ksneg)
|
||||
changes_neu[d].append(ksneu)
|
||||
changes_pos[d].append(kspos)
|
||||
changes_com[d].append(kscom)
|
||||
|
||||
for (d, l) in files.items():
|
||||
with open(folder + "/ks_" + d[0] + "_" + d[1] + ".log", "w") as f:
|
||||
for i in range(len(l) - 1):
|
||||
f1 = l[i]
|
||||
f2 = l[i + 1]
|
||||
f.write(f1 + " -> " + f2 + ": ks neg = " + str(changes_neg[d][i]) + "; ks neu = " + str(changes_neu[d][i])
|
||||
+ "; ks pos = " + str(changes_pos[d][i]) + "; ks com = " + str(changes_com[d][i]) + "\n")
|
||||
|
||||
for (d, l) in files.items():
|
||||
x = [l[i].split("_")[4][:-3] + "-" + l[i + 1].split("_")[4][:-3] for i in range(len(l) - 1)]
|
||||
fig = plt.figure(figsize=(16, 12))
|
||||
for type, changes in {"neg": changes_neg[d], "neu": changes_neu[d], "pos": changes_pos[d], "com": changes_com[d]}.items():
|
||||
stat = [x.statistic for x in changes]
|
||||
pval = [x.pvalue for x in changes]
|
||||
plt.plot(x, stat, label=type + ".stat")
|
||||
plt.plot(x, pval, label=type + ".pval")
|
||||
plt.title("KS 2-sided test with between " + d[0] + " and " + d[1])
|
||||
plt.xticks(rotation=90)
|
||||
plt.legend(loc="upper right")
|
||||
plt.savefig(folder + "/ks_" + d[0] + "_" + d[1] + ".png", bbox_inches='tight')
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def imprt(file):
|
||||
spec = importlib.util.spec_from_file_location("module.name", file)
|
||||
foo = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(foo)
|
||||
return foo
|
||||
|
||||
|
||||
def flatmap(arr):
|
||||
return [item for sublist in arr for item in sublist]
|
||||
|
||||
|
||||
def filecmp(file1, file2):
|
||||
if file1 == file2:
|
||||
return 0
|
||||
s1 = file1.split("_")
|
||||
s2 = file2.split("_")
|
||||
d1 = datetime.strptime(s1[2], "%d-%m-%Y")
|
||||
d2 = datetime.strptime(s2[2], "%d-%m-%Y")
|
||||
if d1 < d2:
|
||||
return -1
|
||||
elif d1 > d2:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# execute only if run as a script
|
||||
usage = sys.argv[0] + " <folder>"
|
||||
if len(sys.argv) < 2:
|
||||
print(usage)
|
||||
sys.exit(1)
|
||||
folder = sys.argv[1]
|
||||
if not os.path.isdir(folder):
|
||||
print(folder + " is not a folder")
|
||||
sys.exit(1)
|
||||
|
||||
main(folder)
|
||||
Reference in New Issue
Block a user