Merge pull request #24 from cedadev/v0.2.2

V0.2.2: Async archive check, logs and recursive check of directories.
cedadev · Nov 21, 2024 · 3a0bac3 · 3a0bac3
2 parents 4b36a5c + cd003e7
commit 3a0bac3
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 8 deletions.
diff --git a/fbi_directory_check/scripts/rescan_directory.py b/fbi_directory_check/scripts/rescan_directory.py
@@ -271,8 +271,9 @@ def _determine_paths(self):
 
         else:
             # Pull files from json
+            logger.info(f'Scanning JSON directory: {self.scan_path}')
             scanpath = f'{os.path.abspath(self.scan_path)}/**/*'
-            jsons = glob.glob(scanpath)
+            jsons = glob.glob(scanpath, recursive=True)
 
             total_json = len(jsons)
             for idx, file in enumerate(jsons):
@@ -285,7 +286,7 @@ def _determine_paths(self):
                     for d in ds:
                         files = glob.glob(f'{d}/**/*.nc', recursive=True)
                         scan_files += files
-                logger.info(f'({idx}/{total_json}) {len(files)} datasets ({file.split("/")[-1]}) ({len(scan_files)} total)')
+                logger.info(f'({idx+1}/{total_json}) {len(files)} datasets ({file.split("/")[-1]}) ({len(scan_files)} total)')
 
         return scan_files
 
@@ -310,7 +311,7 @@ def scan(self) -> list:
                 action = DEPOSIT
 
             if self._dryrun:
-                print(path, action)
+                logger.info(f'{action}: {path}')
                 continue
 
             if self.use_rabbit:
@@ -334,8 +335,10 @@ def save_data(self, outdata):
 
 def main():
 
+    logger.info("Starting rescan check")
     if check_timeout():
         return
+    logger.info("Archive access check: SUCCESS")
 
     r = RescanDirs('')
     if not r.use_rabbit:

diff --git a/fbi_directory_check/utils/utils.py b/fbi_directory_check/utils/utils.py
@@ -134,24 +134,23 @@ async def path_exists(path) -> bool:
         except OSError as e:
             if not pathlib_ignore_error(e):
                 raise
-            return False
+            return ''
         except ValueError:
             # Non-encodable path
-            return False
+            return ''
         return True
 
     async def listfile():
         async with asyncio.timeout(10):
             await path_exists('/neodc/esacci/esacci_terms_and_conditions.txt')
-            return True
-
     try:
         status = asyncio.run(listfile())
     except TimeoutError:
         logger.error('TIMEOUT: ESACCI Directories inaccessible')
         return True
 
-    if not status:
+    # If we didn't get a timeout error, can now perform a standard check.
+    if not os.path.isfile('/neodc/esacci/esacci_terms_and_conditions.txt'):
         logger.error('INACCESSIBLE: ESACCI Directories inaccessible')
         return True
     return False