added --db-timeout to fix 'db - Connection is not available', this wi…

…ll allow longer time for caper to wait for DB connection, especially for large DB file

DETAILS.md

@@ -51,6 +51,7 @@ We highly recommend to use a default configuration file described in the section
 	--singularity-cachedir|Singularity image URI for a WDL
 	--file-db, -d|DB file for Cromwell's built-in HyperSQL database
 	--no-file-db, -n|Do not use file-db. Call-caching (re-using outputs) will be disabled
+	--db-timeout|Milliseconds to wait for DB connection (default: 10000)
 
 * Choose a default backend. Use `--deepcopy` to recursively auto-copy data files in your input JSON file. All data files will be automatically transferred to a target local/remote storage corresponding to a chosen backend. Make sure that you correctly configure temporary directories for source/target storages (`--tmp-dir`, `--tmp-gcs-bucket` and `--tmp-s3-bucket`).
 

caper/caper.py

@@ -146,6 +146,7 @@ def __init__(self, args):
         # file DB
         self._file_db = args.get('file_db')
         self._no_file_db = args.get('no_file_db')
+        self._db_timeout = args.get('db_timeout')
 
         # MySQL DB
         self._mysql_db_ip = args.get('mysql_db_ip')
@@ -841,7 +842,8 @@ def __get_backend_conf_str(self):
                 mysql_ip=self._mysql_db_ip,
                 mysql_port=self._mysql_db_port,
                 mysql_user=self._mysql_db_user,
-                mysql_password=self._mysql_db_password))
+                mysql_password=self._mysql_db_password,
+                db_timeout=self._db_timeout))
 
         # set header for conf ("include ...")
         assert(Caper.BACKEND_CONF_HEADER.endswith('\n'))

caper/caper_args.py

@@ -12,7 +12,7 @@
 from distutils.util import strtobool
 
 
-__version__ = '0.3.13'
+__version__ = '0.3.14'
 
 DEFAULT_JAVA_HEAP_SERVER = '5G'
 DEFAULT_JAVA_HEAP_RUN = '1G'
@@ -22,6 +22,7 @@
 DEFAULT_CROMWELL_JAR = 'https://github.com/broadinstitute/cromwell/releases/download/42/cromwell-42.jar'
 DEFAULT_MYSQL_DB_IP = 'localhost'
 DEFAULT_MYSQL_DB_PORT = 3306
+DEFAULT_DB_TIMEOUT_MS = 10000
 DEFAULT_MAX_CONCURRENT_WORKFLOWS = 40
 DEFAULT_MAX_CONCURRENT_TASKS = 1000
 DEFAULT_MAX_RETRIES = 1
@@ -42,6 +43,11 @@
 ## You can disable file DB with '--no-file-db' or '-n'
 #file-db=~/.caper/default_file_db
 
+## DB timeout for both file DB and MySQL DB
+## If your DB file is large then you can see "db - Connection is not available" error
+## then try to increase this timeout
+#db-timeout=10000
+
 ## Define to use 'caper server' and all client subcommands like 'caper submit'
 ## This is not required for 'caper run'
 #port=8000
@@ -231,6 +237,13 @@ def parse_caper_arguments():
     # run, server
     parent_host = argparse.ArgumentParser(add_help=False)
 
+    group_db = parent_host.add_argument_group(
+        title='General DB settings (for both file DB and MySQL DB)')
+    group_db.add_argument(
+        '--db-timeout', type=int, default=DEFAULT_DB_TIMEOUT_MS,
+        help='Milliseconds to wait for DB (both for file DB and MySQL DB) '
+             'connection.')
+
     group_file_db = parent_host.add_argument_group(
         title='HyperSQL file DB arguments')
     group_file_db.add_argument(
@@ -595,6 +608,11 @@ def parse_caper_arguments():
             and isinstance(max_concurrent_workflows, str):
         args_d['max_concurrent_workflows'] = int(max_concurrent_workflows)
 
+    db_timeout = args_d.get('db_timeout')
+    if db_timeout is not None \
+            and isinstance(db_timeout, str):
+        args_d['db_timeout'] = int(db_timeout)
+
     max_retries = args_d.get('max_retries')
     if max_retries is not None \
             and isinstance(max_retries, str):

caper/caper_backend.py

@@ -77,7 +77,7 @@ class CaperBackendDatabase(dict):
     }
 
     def __init__(self, file_db=None, mysql_ip=None, mysql_port=None,
-                 mysql_user=None, mysql_password=None):
+                 mysql_user=None, mysql_password=None, db_timeout=None):
         super(CaperBackendDatabase, self).__init__(
             CaperBackendDatabase.TEMPLATE)
         if mysql_user is not None and mysql_password is not None:
@@ -94,6 +94,8 @@ def __init__(self, file_db=None, mysql_ip=None, mysql_port=None,
                     'url': 'jdbc:hsqldb:file:{};shutdown=false;'
                     'hsqldb.tx=mvcc'.format(file_db)
                 }
+        if db_timeout is not None:
+            self['database']['db']['connectionTimeout'] = db_timeout
 
 
 class CaperBackendGCP(dict):