Again bug fix of multiprocessing

yumorishita · Nov 18, 2020 · 5a86452 · 5a86452
1 parent 02b7e68
commit 5a86452
Show file tree

Hide file tree

Showing 10 changed files with 82 additions and 55 deletions.
diff --git a/LiCSBAS_lib/LiCSBAS_inv_lib.py b/LiCSBAS_lib/LiCSBAS_inv_lib.py
@@ -8,7 +8,9 @@
 =========
 Changelog
 =========
-v1.4.1 20201116 Yu Morioshita, GSI
+v1.4.2 20201118 Yu Morishita, GSI
+ - Again Bug fix of multiprocessing
+v1.4.1 20201116 Yu Morishita, GSI
  - Bug fix of multiprocessing in Mac python>=3.8
 v1.4 20200703 Yu Morioshita, GSI
  - Replace problematic terms
@@ -27,7 +29,6 @@
 import numpy as np
 import datetime as dt
 import multiprocessing as multi
-multi.set_start_method('fork') # for python >=3.8 in Mac
 from astropy.stats import bootstrap
 from astropy.utils import NumpyRNGContext
 import LiCSBAS_tools_lib as tools_lib
@@ -137,7 +138,8 @@ def invert_nsbas(unw, G, dt_cum, gamma, n_core):
         print('  {} parallel processing'.format(n_core), flush=True)
 
         args = [i for i in range(n_pt-n_pt_full)]
-        p = multi.Pool(n_core)
+        q = multi.get_context('fork')
+        p = q.Pool(n_core)
         _result = p.map(censored_lstsq_slow_para_wrapper, args) #list[n_pt][length]
         result[:, ~bool_pt_full] = np.array(_result).T
 
@@ -209,7 +211,8 @@ def invert_nsbas_wls(unw, var, G, dt_cum, gamma, n_core):
         print('  {} parallel processing'.format(n_core), flush=True)
 
         args = [i for i in range(n_pt)]
-        p = multi.Pool(n_core)
+        q = multi.get_context('fork')
+        p = q.Pool(n_core)
         _result = p.map(wls_nsbas, args) #list[n_pt][length]
         result = np.array(_result).T
 

diff --git a/bin/LiCSBAS01_get_geotiff.py b/bin/LiCSBAS01_get_geotiff.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-v1.6.1 20201116 Yu Morishita, GSI
+v1.6.2 20201118 Yu Morishita, GSI
 
 ========
 Overview
@@ -38,6 +38,8 @@
 """
 #%% Change log
 '''
+v1.6.2 20201118 Yu Morishita, GSI
+ - Again Bug fix of multiprocessing
 v1.6.1 20201116 Yu Morishita, GSI
  - Bug fix of multiprocessing in Mac python>=3.8
 v1.6 20200911 Yu Morishita, GSI
@@ -71,7 +73,6 @@
 import numpy as np
 import datetime as dt
 import multiprocessing as multi
-multi.set_start_method('fork') # for python >=3.8 in Mac
 import LiCSBAS_tools_lib as tools_lib
 
 class Usage(Exception):
@@ -88,7 +89,7 @@ def main(argv=None):
         argv = sys.argv
 
     start = time.time()
-    ver='1.6.1'; date=20201116; author="Y. Morishita"
+    ver='1.6.2'; date=20201118; author="Y. Morishita"
     print("\n{} ver{} {} {}".format(os.path.basename(argv[0]), ver, date, author), flush=True)
     print("{} {}".format(os.path.basename(argv[0]), ' '.join(argv[1:])), flush=True)
 
@@ -100,6 +101,8 @@ def main(argv=None):
     get_gacos = False
     n_para = 4
 
+    q = multi.get_context('fork')
+
 
     #%% Read options
     try:
@@ -257,7 +260,7 @@ def main(argv=None):
                  os.path.join(gacosdir, imd+'.sltd.geo.tif')
                  ) for i, imd in enumerate(_imdates)]
 
-        p = multi.Pool(n_para)
+        p = q.Pool(n_para)
         rc = p.map(check_gacos_wrapper, args)
         p.close()
 
@@ -289,7 +292,7 @@ def main(argv=None):
                      os.path.join(gacosdir, '{}.sltd.geo.tif'.format(imd))
                      ) for i, imd in enumerate(imdates_dl)]
 
-            p = multi.Pool(n_para)
+            p = q.Pool(n_para)
             p.map(download_wrapper, args)
             p.close()
         else:
@@ -329,7 +332,7 @@ def main(argv=None):
              os.path.join(ifgd, '{}.geo.unw.tif'.format(ifgd))
              ) for i, ifgd in enumerate(ifgdates)]
 
-    p = multi.Pool(n_para)
+    p = q.Pool(n_para)
     rc = p.map(check_exist_wrapper, args)
     p.close()
 
@@ -349,7 +352,7 @@ def main(argv=None):
              os.path.join(ifgd, '{}.geo.cc.tif'.format(ifgd))
              ) for i, ifgd in enumerate(ifgdates)]
 
-    p = multi.Pool(n_para)
+    p = q.Pool(n_para)
     rc = p.map(check_exist_wrapper, args)
     p.close()
 
@@ -378,7 +381,7 @@ def main(argv=None):
                  os.path.join(ifgd, '{}.geo.unw.tif'.format(ifgd))
                  ) for i, ifgd in enumerate(unwdates_dl)]
 
-        p = multi.Pool(n_para)
+        p = q.Pool(n_para)
         p.map(download_wrapper, args)
         p.close()
 
@@ -390,7 +393,7 @@ def main(argv=None):
                  os.path.join(ifgd, '{}.geo.cc.tif'.format(ifgd))
                  ) for i, ifgd in enumerate(ccdates_dl)]
 
-        p = multi.Pool(n_para)
+        p = q.Pool(n_para)
         p.map(download_wrapper, args)
         p.close()
 

diff --git a/bin/LiCSBAS02_ml_prep.py b/bin/LiCSBAS02_ml_prep.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-v1.7.2 202011116 Yu Morishita, GSI
+v1.7.3 202011118 Yu Morishita, GSI
 
 ========
 Overview
@@ -48,6 +48,8 @@
 """
 #%% Change log
 '''
+v1.7.3 20201118 Yu Morishita, GSI
+ - Again Bug fix of multiprocessing
 v1.7.2 20201116 Yu Morishita, GSI
  - Bug fix of multiprocessing in Mac python>=3.8
 v1.7.1 20201028 Yu Morishita, GSI
@@ -89,7 +91,6 @@
 import numpy as np
 import subprocess as subp
 import multiprocessing as multi
-multi.set_start_method('fork') # for python >=3.8 in Mac
 import LiCSBAS_io_lib as io_lib
 import LiCSBAS_tools_lib as tools_lib
 import LiCSBAS_plot_lib as plot_lib
@@ -108,7 +109,7 @@ def main(argv=None):
         argv = sys.argv
 
     start = time.time()
-    ver="1.7.2"; date=202011116; author="Y. Morishita"
+    ver="1.7.3"; date=202011118; author="Y. Morishita"
     print("\n{} ver{} {} {}".format(os.path.basename(argv[0]), ver, date, author), flush=True)
     print("{} {}".format(os.path.basename(argv[0]), ' '.join(argv[1:])), flush=True)
 
@@ -129,6 +130,7 @@ def main(argv=None):
     cmap = 'insar'
     cycle = 3
     n_valid_thre = 0.5
+    q = multi.get_context('fork')
 
 
     #%% Read options
@@ -283,7 +285,7 @@ def main(argv=None):
 
         ### Create float with parallel processing
         print('  {} parallel processing...'.format(n_para), flush=True)
-        p = multi.Pool(n_para)
+        p = q.Pool(n_para)
         rc = p.map(convert_wrapper, range(n_ifg2))
         p.close()
 

diff --git a/bin/LiCSBAS03op_GACOS.py b/bin/LiCSBAS03op_GACOS.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-v1.5.2 20201116 Yu Morishita, GSI
+v1.5.3 20201118 Yu Morishita, GSI
 
 ========
 Overview
@@ -51,6 +51,8 @@
 """
 #%% Change log
 '''
+v1.5.3 20201118 Yu Morishita, GSI
+ - Again Bug fix of multiprocessing
 v1.5.2 20201116 Yu Morishita, GSI
  - Bug fix of multiprocessing in Mac python>=3.8
 v1.5.1 20201028 Yu Morishita, GSI
@@ -83,7 +85,6 @@
 import numpy as np
 import gdal
 import multiprocessing as multi
-multi.set_start_method('fork') # for python >=3.8 in Mac
 import LiCSBAS_io_lib as io_lib
 import LiCSBAS_tools_lib as tools_lib
 import LiCSBAS_plot_lib as plot_lib
@@ -158,7 +159,7 @@ def main(argv=None):
         argv = sys.argv
 
     start = time.time()
-    ver="1.5.2"; date=20201116; author="Y. Morishita"
+    ver="1.5.3"; date=20201118; author="Y. Morishita"
     print("\n{} ver{} {} {}".format(os.path.basename(argv[0]), ver, date, author), flush=True)
     print("{} {}".format(os.path.basename(argv[0]), ' '.join(argv[1:])), flush=True)
 
@@ -179,6 +180,8 @@ def main(argv=None):
     except:
         n_para = multi.cpu_count()
 
+    q = multi.get_context('fork')
+
 
     #%% Read options
     try:
@@ -301,7 +304,7 @@ def main(argv=None):
             _n_para = n_para
 
         print('  {} parallel processing...'.format(_n_para), flush=True)
-        p = multi.Pool(_n_para)
+        p = q.Pool(_n_para)
         no_gacos_imds = p.map(convert_wrapper, range(n_im2))
         p.close()
 
@@ -343,7 +346,7 @@ def main(argv=None):
             _n_para = n_para
 
         print('  {} parallel processing...'.format(_n_para), flush=True)
-        p = multi.Pool(_n_para)
+        p = q.Pool(_n_para)
         _return = p.map(correct_wrapper, range(n_ifg2))
         p.close()
 

diff --git a/bin/LiCSBAS04op_mask_unw.py b/bin/LiCSBAS04op_mask_unw.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-v1.3.2 20201116 Yu Morishita, GSI
+v1.3.3 20201118 Yu Morishita, GSI
 
 ========
 Overview
@@ -42,6 +42,8 @@
 """
 #%% Change log
 '''
+v1.3.3 20201118 Yu Morishita, GSI
+ - Again Bug fix of multiprocessing
 v1.3.2 20201116 Yu Morishita, GSI
  - Bug fix of multiprocessing in Mac python>=3.8
 v1.3.1 20201028 Yu Morishita, GSI
@@ -65,7 +67,6 @@
 import time
 import numpy as np
 import multiprocessing as multi
-multi.set_start_method('fork') # for python >=3.8 in Mac
 import LiCSBAS_io_lib as io_lib
 import LiCSBAS_tools_lib as tools_lib
 import LiCSBAS_plot_lib as plot_lib
@@ -83,7 +84,7 @@ def main(argv=None):
         argv = sys.argv
 
     start = time.time()
-    ver="1.3.2"; date=20201116; author="Y. Morishita"
+    ver="1.3.3"; date=20201118; author="Y. Morishita"
     print("\n{} ver{} {} {}".format(os.path.basename(argv[0]), ver, date, author), flush=True)
     print("{} {}".format(os.path.basename(argv[0]), ' '.join(argv[1:])), flush=True)
 
@@ -103,6 +104,7 @@ def main(argv=None):
         n_para = multi.cpu_count()
 
     cmap_noise = 'viridis'
+    q = multi.get_context('fork')
 
 
     #%% Read options
@@ -258,7 +260,7 @@ def main(argv=None):
             n_para = n_ifg2
 
         print('  {} parallel processing...'.format(n_para), flush=True)
-        p = multi.Pool(n_para)
+        p = q.Pool(n_para)
         p.map(mask_wrapper, range(n_ifg2))
         p.close()
 

diff --git a/bin/LiCSBAS05op_clip_unw.py b/bin/LiCSBAS05op_clip_unw.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-v1.2.2 20201116 Yu Morishita, GSI
+v1.2.3 20201118 Yu Morishita, GSI
 
 ========
 Overview
@@ -42,6 +42,8 @@
 """
 #%% Change log
 '''
+v1.2.3 20201118 Yu Morishita, GSI
+ - Again Bug fix of multiprocessing
 v1.2.2 20201116 Yu Morishita, GSI
  - Bug fix of multiprocessing in Mac python>=3.8
 v1.2.1 20201028 Yu Morishita, GSI
@@ -65,7 +67,6 @@
 import time
 import numpy as np
 import multiprocessing as multi
-multi.set_start_method('fork')
 import LiCSBAS_io_lib as io_lib
 import LiCSBAS_tools_lib as tools_lib
 import LiCSBAS_plot_lib as plot_lib
@@ -84,7 +85,7 @@ def main(argv=None):
         argv = sys.argv
 
     start = time.time()
-    ver="1.2.2"; date=20201116; author="Y. Morishita"
+    ver="1.2.3"; date=20201118; author="Y. Morishita"
     print("\n{} ver{} {} {}".format(os.path.basename(argv[0]), ver, date, author), flush=True)
     print("{} {}".format(os.path.basename(argv[0]), ' '.join(argv[1:])), flush=True)
 
@@ -102,6 +103,8 @@ def main(argv=None):
     except:
         n_para = multi.cpu_count()
 
+    q = multi.get_context('fork')
+
 
     #%% Read options
     try:
@@ -281,7 +284,7 @@ def main(argv=None):
             n_para = n_ifg2
 
         print('  {} parallel processing...'.format(n_para), flush=True)
-        p = multi.Pool(n_para)
+        p = q.Pool(n_para)
         p.map(clip_wrapper, range(n_ifg2))
         p.close()