Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

stackSentinel background task fix #271

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

stackSentinel background task fix #271

Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
0b6e510
wait for background tasks to finish
hardreddata Apr 14, 2021
d4b1f99
tidy based on PR feedback
hardreddata Jul 22, 2022
f8cd5d8
last_line logic in numProcess wrapper
hardreddata Jul 22, 2022
b53e087
Merge pull request #1 from isce-framework/main
hardreddata Jul 27, 2022
4ffbfff
tidy config wrapper calls
hardreddata Jul 27, 2022
8971f52
remove messy line_cnt logic and enumerate loops
hardreddata Jul 27, 2022
2c4f52b
Merge branch 'main' into background-bug
hardreddata Jul 27, 2022
426d033
consider ionospheric phase estimation
hardreddata Jul 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 24 additions & 22 deletions contrib/stack/topsStack/Stack.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,17 +260,19 @@ def denseOffset(self, function):
#self.f.write('ww : 256\n')
#self.f.write('wh : 128\n')

def write_wrapper_config2run_file(self, configName, line_cnt, numProcess = 1):
def write_wrapper_config2run_file(self, configName, line_cnt = 1, last_line = False, numProcess = 1):
# dispassionate list of commands for single process
if numProcess == 1:
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + '\n')
# aggregate background commands between wait blocks for speed gains
elif numProcess > 1:
self.runf.write(self.text_cmd + 'SentinelWrapper.py -c ' + configName + ' &\n')
if line_cnt == numProcess:
if line_cnt % numProcess == 0:
self.runf.write('wait\n\n')
line_cnt = 0
hardreddata marked this conversation as resolved.
Show resolved Hide resolved
return line_cnt
# last line of file, wait for any background processes to complete
if last_line:
self.runf.write('wait\n')
hardreddata marked this conversation as resolved.
Show resolved Hide resolved

def finalize(self):
self.f.close()
Expand Down Expand Up @@ -317,7 +319,7 @@ def unpackSLC(self, acquisitionDates, safe_dict):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj

def unpackStackReferenceSLC(self, safe_dict):
Expand All @@ -338,7 +340,7 @@ def unpackStackReferenceSLC(self, safe_dict):
configObj.finalize()

line_cnt = 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj

def unpackSecondarysSLC(self, stackReferenceDate, secondaryList, safe_dict):
Expand All @@ -358,7 +360,7 @@ def unpackSecondarysSLC(self, stackReferenceDate, secondaryList, safe_dict):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess)
del configObj

def averageBaseline(self, stackReferenceDate, secondaryList):
Expand All @@ -375,7 +377,7 @@ def averageBaseline(self, stackReferenceDate, secondaryList):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess)
del configObj

def gridBaseline(self, stackReferenceDate, secondaryList):
Expand All @@ -392,7 +394,7 @@ def gridBaseline(self, stackReferenceDate, secondaryList):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj
# also add the reference in itself to be consistent with the SLC dir
configName = os.path.join(self.config_path,'config_baselinegrid_reference')
Expand All @@ -405,7 +407,7 @@ def gridBaseline(self, stackReferenceDate, secondaryList):
configObj.finalize()

line_cnt = 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj


Expand Down Expand Up @@ -440,7 +442,7 @@ def geo2rdr_offset(self, secondaryList, fullBurst='False'):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess)
del configObj

def resample_with_carrier(self, secondaryList, fullBurst='False'):
Expand Down Expand Up @@ -470,7 +472,7 @@ def resample_with_carrier(self, secondaryList, fullBurst='False'):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
configObj.write_wrapper_config2run_file(configName, line_cnt, secondary == secondaryList[-1], self.numProcess)
del configObj


Expand Down Expand Up @@ -521,7 +523,7 @@ def pairs_misregistration(self, dateList, safe_dict):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess)
del configObj
########################

Expand Down Expand Up @@ -560,7 +562,7 @@ def generate_burstIgram(self, dateList, safe_dict, pairs):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess)
del configObj


Expand Down Expand Up @@ -594,7 +596,7 @@ def igram_mergeBurst(self, dateList, safe_dict, pairs):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess)
del configObj

def mergeSecondarySLC(self, secondaryList, virtual='True'):
Expand All @@ -619,7 +621,7 @@ def mergeSecondarySLC(self, secondaryList, virtual='True'):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj


Expand All @@ -642,7 +644,7 @@ def mergeReference(self, stackReference, virtual='True'):
configObj.finalize()

line_cnt = 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj

geometryList = ['lat*rdr', 'lon*rdr', 'los*rdr', 'hgt*rdr', 'shadowMask*rdr','incLocal*rdr']
Expand Down Expand Up @@ -670,7 +672,7 @@ def mergeReference(self, stackReference, virtual='True'):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj

def mergeSLC(self, aquisitionDates, virtual='True'):
Expand All @@ -694,7 +696,7 @@ def mergeSLC(self, aquisitionDates, virtual='True'):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
configObj.write_wrapper_config2run_file(configName, line_cnt)


geometryList = ['lat*rdr', 'lon*rdr', 'los*rdr', 'hgt*rdr', 'shadowMask*rdr','incLocal*rdr']
Expand All @@ -719,7 +721,7 @@ def mergeSLC(self, aquisitionDates, virtual='True'):
configObj.finalize()

g_line_cnt += 1
g_line_cnt = configObj.write_wrapper_config2run_file(configName, g_line_cnt)
configObj.write_wrapper_config2run_file(configName, g_line_cnt)
del configObj

def filter_coherence(self, pairs):
Expand All @@ -746,7 +748,7 @@ def filter_coherence(self, pairs):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess)
del configObj


Expand All @@ -772,7 +774,7 @@ def unwrap(self, pairs):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt, self.numProcess)
configObj.write_wrapper_config2run_file(configName, line_cnt, pair == pairs[-1], self.numProcess)
del configObj

def denseOffsets(self, pairs):
Expand All @@ -792,7 +794,7 @@ def denseOffsets(self, pairs):
configObj.finalize()

line_cnt += 1
line_cnt = configObj.write_wrapper_config2run_file(configName, line_cnt)
configObj.write_wrapper_config2run_file(configName, line_cnt)
del configObj

def finalize(self):
Expand Down