wea_ondara/master
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

wip

Browse Source
This commit is contained in:
wea_ondara
2019-12-28 13:14:49 +01:00
parent 88de1e9c7b
commit e54e4f2938
3 changed files with 75 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
mt.py
View File

@@ -1,14 +1,16 @@
import multiprocessing as mp
import threading as th
import time
from math import ceil
from threading import Thread, Lock
class mt():
def __init__(self, threads, data, verbose=False, progressinterval=1000):
def __init__(self, threads, data, verbose=False, progressinterval=1000, useprocesses=False):
self.__running = False
self.__closed = False
self.__data = data
self.__verbose = verbose
self.__useprocesses = useprocesses
# dummy
self.__final = None
self.__comment = None
@@ -18,9 +20,16 @@ class mt():
# thread things
self.__threadcount = threads
self.__threads = []
self.__lock = Lock()
self.__results = []
self.__progress = 0
if useprocesses:
mp.set_start_method('fork', True)
manager = mp.Manager()
self.__results = manager.list()
self.__progress = mp.Value("i", 0)
self.__lock = mp.Lock()
else:
self.__results = []
self.__progress = 0
self.__lock = th.Lock()
self.__progressinterval = progressinterval
for i in range(self.__threadcount):
self.__results.append([])
@@ -32,7 +41,7 @@ class mt():
if self.__running:
self.join()
self.__data = self.getresults()
self.__progress = 0
self.__reset_progress()
self.__running = True
self.__final = self.__getresultsmapfilter
self.__type = "filter"
@@ -43,7 +52,10 @@ class mt():
self.__endtime = None
for i in range(self.__threadcount):
part = self.__data[i * len(self.__data) // self.__threadcount:(i + 1) * len(self.__data) // self.__threadcount]
self.__threads[i] = Thread(target=self.__dofilter, args=(i, part, cond))
if self.__useprocesses:
self.__threads[i] = mp.Process(target=self.__dofilter, args=(i, part, cond))
else:
self.__threads[i] = th.Thread(target=self.__dofilter, args=(i, part, cond))
self.__threads[i].start()
return self
@@ -55,9 +67,9 @@ class mt():
part = list[j * self.__progressinterval: min((j + 1) * self.__progressinterval, len(list))]
results += [l for l in part if cond(l)]
with self.__lock:
self.__progress += len(part)
self.__inc_progress(len(part))
if self.__comment is not None:
print("\r" + self.__comment + ": " + str(self.__progress) + "/" + str(len(self.__data)) + " ...", end='', flush=True)
print("\r" + self.__comment + ": " + str(self.__get_progress()) + "/" + str(len(self.__data)) + " ...", end='', flush=True)
with self.__lock:
self.__results[i] = results
@@ -74,7 +86,7 @@ class mt():
if self.__running:
self.join()
self.__data = self.getresults()
self.__progress = 0
self.__reset_progress()
self.__running = True
self.__final = self.__getresultsmapfilter
self.__type = "map"
@@ -85,7 +97,10 @@ class mt():
self.__endtime = None
for i in range(self.__threadcount):
part = self.__data[i * len(self.__data) // self.__threadcount:(i + 1) * len(self.__data) // self.__threadcount]
self.__threads[i] = Thread(target=self.__domap, args=(i, part, func))
if self.__useprocesses:
self.__threads[i] = mp.Process(target=self.__domap, args=(i, part, func))
else:
self.__threads[i] = th.Thread(target=self.__domap, args=(i, part, func))
self.__threads[i].start()
return self
@@ -96,9 +111,9 @@ class mt():
part = list[j * self.__progressinterval: min((j + 1) * self.__progressinterval, len(list))]
results += [func(l) for l in part]
with self.__lock:
self.__progress += len(part)
self.__inc_progress(len(part))
if self.__comment is not None:
print("\r" + self.__comment + ": " + str(self.__progress) + "/" + str(len(self.__data)) + " ...", end='', flush=True)
print("\r" + self.__comment + ": " + str(self.__get_progress()) + "/" + str(len(self.__data)) + " ...", end='', flush=True)
with self.__lock:
self.__results[i] = results
@@ -115,7 +130,7 @@ class mt():
if self.__running:
self.join()
self.__data = self.getresults()
self.__progress = 0
self.__reset_progress()
self.__running = True
self.__final = lambda: self.__getresultsreduce(aggregator, initval)
self.__type = "reduce"
@@ -126,7 +141,10 @@ class mt():
self.__endtime = None
for i in range(self.__threadcount):
part = self.__data[i * len(self.__data) // self.__threadcount:(i + 1) * len(self.__data) // self.__threadcount]
self.__threads[i] = Thread(target=self.__doreduce, args=(i, part, reducer, initval))
if self.__useprocesses:
self.__threads[i] = mp.Process(target=self.__doreduce, args=(i, part, reducer, initval))
else:
self.__threads[i] = th.Thread(target=self.__doreduce, args=(i, part, reducer, initval))
self.__threads[i].start()
return self
@@ -139,9 +157,9 @@ class mt():
for k in range(len(part)):
val = reducer(val, part[k])
with self.__lock:
self.__progress += len(part)
self.__inc_progress(len(part))
if self.__comment is not None:
print("\r" + self.__comment + ": " + str(self.__progress) + "/" + str(len(self.__data)) + " ...", end='', flush=True)
print("\r" + self.__comment + ": " + str(self.__get_progress()) + "/" + str(len(self.__data)) + " ...", end='', flush=True)
with self.__lock:
self.__results[i] = val
@@ -180,10 +198,10 @@ class mt():
if self.__comment is not None:
dur = self.__endtime - self.__starttime
if self.__verbose:
print(self.__comment + ": " + str(self.__progress) + "/" + str(len(self.__data)) + (
print(self.__comment + ": " + str(self.__get_progress()) + "/" + str(len(self.__data)) + (
" -> #" + str(sum([len(l) for l in self.__results])) if self.__type == "filter" else "") + " ... took " + str(dur) + "ms")
else:
print("\r" + self.__comment + ": " + str(self.__progress) + "/" + str(len(self.__data)) + (
print("\r" + self.__comment + ": " + str(self.__get_progress()) + "/" + str(len(self.__data)) + (
" -> #" + str(sum([len(l) for l in self.__results])) if self.__type == "filter" else "") + " ... took " + str(dur) + "ms")
return self
@@ -195,3 +213,21 @@ class mt():
def __cms(self):
return int(round(time.time() * 1000))
def __reset_progress(self):
if self.__useprocesses:
self.__progress.value = 0
else:
self.__progress = 0
def __inc_progress(self, val):
if self.__useprocesses:
self.__progress.value += val
else:
self.__progress += val
def __get_progress(self):
if self.__useprocesses:
return self.__progress.value
else:
return self.__progress