fixed minor bugs when using different numbers of processes

- mg_motion_mp will now produce _exactly_ the same results with any number of processes (tested from 1 to 12).
- added num_processes parameter
#213

Author: balintlaczko

musicalgestures/_motionvideo_mp_render.py

@@ -125,7 +125,7 @@ def mg_motion_mp(args):
                 motion_frame_rgb = motion_frame
 
             if save_video:
-                # if this is not the first process (rendering the start of the video) then don't save the first frame of the output (it'll always be black).
+                # if this is not the first process (rendering the start of the video) then drop the first frame of the output (it'll always be black).
                 if process_id != 0 and ii > 0:
                     if inverted_motionvideo:
                         out.write(cv2.bitwise_not(
@@ -185,9 +185,7 @@ def mg_motion_mp(args):
 
 def run_pool(func, args, numprocesses):
     pool = multiprocessing.Pool(numprocesses)
-    # results = pool.map(func, args)
     pool.map(func, args)
-    # return results
 
 
 def calc_frame_groups(framecount, num_cores):
@@ -204,18 +202,6 @@ def calc_frame_groups(framecount, num_cores):
     return groups
 
 
-def testhogfunc(process_id):
-    limit = 20
-    count = 0
-    while count < limit:
-        print(process_id, count)
-        futyi = 0
-        for i in range(10_000_000):
-            futyi += 1
-        count += 1
-    return process_id, count
-
-
 def bool_from_str(boolstring):
     return True if boolstring == "True" else False
 
@@ -240,6 +226,7 @@ def bool_from_str(boolstring):
     parser.add_argument('save_data', metavar='save_data', type=str, help='save_data')
     parser.add_argument('save_motiongrams', metavar='save_motiongrams', type=str, help='save_motiongrams')
     parser.add_argument('save_video', metavar='save_video', type=str, help='save_video')
+    parser.add_argument('num_processes', metavar='num_processes', type=str, help='num_processes')
 
     args = parser.parse_args()
 
@@ -257,21 +244,19 @@ def bool_from_str(boolstring):
     fps, width, height, length = int(float(args.fps)), int(float(args.width)), int(float(args.height)), int(float(args.length))
     color, filtertype, thresh, blur, kernel_size = bool_from_str(args.color), args.filtertype, float(args.thresh), args.blur, int(float(args.kernel_size))
     inverted_motionvideo, inverted_motiongram, equalize_motiongram = bool_from_str(args.inverted_motionvideo), bool_from_str(args.inverted_motiongram), bool_from_str(args.equalize_motiongram)
-    save_data, save_motiongrams, save_video = bool_from_str(args.save_data), bool_from_str(args.save_motiongrams), bool_from_str(args.save_video) 
+    save_data, save_motiongrams, save_video = bool_from_str(args.save_data), bool_from_str(args.save_motiongrams), bool_from_str(args.save_video)
+    num_processes = multiprocessing.cpu_count() if int(float(args.num_processes)) < 1 else min(int(float(args.num_processes)), multiprocessing.cpu_count())
 
-    numprocessors = multiprocessing.cpu_count()
-    frame_groups = calc_frame_groups(length, numprocessors)
+    frame_groups = calc_frame_groups(length, num_processes)
 
     feed_args = []
 
-    for i in range(numprocessors):
+    for i in range(num_processes):
         start_frame, num_frames = frame_groups[i]
         initargs = [target_folder, of, fex, fps, width, height, length, color, filtertype, thresh, blur, kernel_size, inverted_motionvideo, inverted_motiongram, equalize_motiongram, save_data, save_motiongrams, save_video, start_frame, num_frames, i, client]
         # client.sendall(bytes(str(initargs), 'utf-8'))
         feed_args.append(initargs)
 
-    # results = run_pool(mg_motion_mp, feed_args, numprocessors)
-    run_pool(mg_motion_mp, feed_args, numprocessors)
+    run_pool(mg_motion_mp, feed_args, num_processes)
 
-    client.close()
-    # print("Closed socket client.")
+    client.close()

musicalgestures/_motionvideo_mp_run.py

@@ -33,7 +33,8 @@ def mg_motion_mp(
         target_name_data=None,
         target_name_mgx=None,
         target_name_mgy=None,
-        overwrite=False):
+        overwrite=False,
+        num_processes=-1):
 
     of, fex = self.of, self.fex
 
@@ -64,7 +65,7 @@ def mg_motion_mp(
 
     save_data_feed = save_data or save_plot
 
-    command = [pythonkw, pyfile, temp_folder, of_feed, fex, self.fps, self.width, self.height, self.length, self.color, filtertype, thresh, blur, kernel_size, inverted_motionvideo, inverted_motiongram, equalize_motiongram, save_data_feed, save_motiongrams, save_video]
+    command = [pythonkw, pyfile, temp_folder, of_feed, fex, self.fps, self.width, self.height, self.length, self.color, filtertype, thresh, blur, kernel_size, inverted_motionvideo, inverted_motiongram, equalize_motiongram, save_data_feed, save_motiongrams, save_video, num_processes]
     command = [str(item) for item in command]
     # print()
     # print(command)
@@ -108,28 +109,54 @@ def mg_motion_mp(
     # print("organizing results...")
     results = os.listdir(temp_folder)
     time_files  = [temp_folder + file for file in results if file.startswith("time")]
+    time_files.sort()
     com_files   = [temp_folder + file for file in results if file.startswith("com")]
+    com_files.sort()
     qom_files   = [temp_folder + file for file in results if file.startswith("qom")]
+    qom_files.sort()
     gramx_files = [temp_folder + file for file in results if file.startswith("gramx")]
+    gramx_files.sort()
     gramy_files = [temp_folder + file for file in results if file.startswith("gramy")]
+    gramy_files.sort()
     video_files = [temp_folder + file for file in results if file.endswith("avi")]
+    video_files.sort()
 
     gramx, gramy, time, com, qom = None, None, None, None, None
 
     if save_motiongrams:
         # load gramx
-        for idx, item in enumerate(gramx_files):
-            if idx == 0:
-                gramx = np.load(item)
-            else:
-                gramx = np.append(gramx, np.load(item)[1:-1], axis=0)
+        # if we only used a single chunk, load everything
+        if len(gramx_files) == 1:
+            gramx = np.load(gramx_files[0])
+        # or in case there were multiple chunks...
+        else:
+            for idx, item in enumerate(gramx_files):
+                if idx == 0:
+                    # do not drop first row in first chunk
+                    gramx = np.load(item)[:-1]
+                elif idx == len(gramy_files) - 1:
+                    # do not drop the last row in last chunk
+                    gramx = np.append(gramx, np.load(item)[1:], axis=0)
+                else:
+                    # else drop first and last rows from chunk
+                    gramx = np.append(gramx, np.load(item)[1:-1], axis=0)
 
         # load gramy
-        for idx, item in enumerate(gramy_files):
-            if idx == 0:
-                gramy = np.load(item)
-            else:
-                gramy = np.append(gramy, np.load(item)[:, 1:], axis=1)
+        # if we only used a single chunk, load everything
+        if len(gramy_files) == 1:
+            gramy = np.load(gramy_files[0])
+        # or in case there were multiple chunks...
+        else:
+            for idx, item in enumerate(gramy_files):
+                if idx == 0:
+                    # do not drop first column in first chunk
+                    gramy = np.load(item)[:, :-1]
+                elif idx == len(gramy_files) - 1:
+                    # do not drop the last column in last chunk
+                    gramy = np.append(gramy, np.load(item)[:, 1:], axis=1)
+                else:
+                    # else drop first and last columns from chunk
+                    gramy = np.append(gramy, np.load(item)[:, 1:-1], axis=1)
 
         if self.color == False:
             # Normalize before converting to uint8 to keep precision