job fast exit on error (#229)

- **CHANGED API** `Longtail_JobAPI_JobFunc` renamed `is_cancelled` to `detected_error`, now contains first error returned from a job task in the same job group (if any) If `detected_error` is non-zero, try to exit (and cleanup) your task directly and return `0`. - **CHANGED_API** JobAPI `ReadyJobs` now returns first error encountered in a job group for a task as well as any error in the job api itself, removing the need to book keep the error for tasks separately - **NEW API** `Longtail_CompareAndSwap` compare and swap with platform implementations - **FIXED** Fixed memory leaks in command tool - **CHANGED** Refactored all internal usage of JobAPI `ReadyJobs` with new error handling
DanEngelbrecht · Jan 7, 2024 · d4eb67b · d4eb67b
1 parent 8f399e9
commit d4eb67b
Show file tree

Hide file tree

Showing 10 changed files with 290 additions and 376 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,9 +1,15 @@
 ##
+- **CHANGED API** `Longtail_JobAPI_JobFunc` renamed `is_cancelled` to `detected_error`, now contains first error returned from a job task in the same job group (if any) or ECANCELLED if job group was cancelled
+    If `detected_error` is non-zero, try to exit (and cleanup) your task directly and return `0`.
+- **CHANGED_API** JobAPI `ReadyJobs` now returns first error encountered in a job group for a task as well as any error in the job api itself, removing the need to book keep the error for tasks separately
 - **ADDED** Longtail_SetReAllocAndFree
 - **ADDED** Longtail_ReAlloc
 - **ADDED** Longtail_MemTracer_ReAlloc
 - **ADDED** memtracer now tracks allocations in stb_ds
 - **ADDED** memtracer now tracks allocations in zstd
+- **NEW API** `Longtail_CompareAndSwap` compare and swap with platform implementations
+- **FIXED** Fixed memory leaks in command tool
+- **CHANGED** Refactored all internal usage of JobAPI `ReadyJobs` with new error handling
 
 ## 0.4.1
 - **NEW API** `Longtail_ChangeVersion2` added

diff --git a/cmd/main.c b/cmd/main.c
@@ -659,16 +659,14 @@ int DownSync(
         compress_block_store_api = Longtail_CreateCompressBlockStoreAPI(store_block_remotestore_api, compression_registry);
     }
 
-//    struct Longtail_BlockStoreAPI* lru_block_store_api = Longtail_CreateLRUBlockStoreAPI(compress_block_store_api, 32);
-    struct Longtail_BlockStoreAPI* store_block_store_api = Longtail_CreateShareBlockStoreAPI(compress_block_store_api);// lru_block_store_api);
+    struct Longtail_BlockStoreAPI* store_block_store_api = Longtail_CreateShareBlockStoreAPI(compress_block_store_api);
 
     struct Longtail_VersionIndex* source_version_index = 0;
     int err = Longtail_ReadVersionIndex(storage_api, source_path, &source_version_index);
     if (err)
     {
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to read version index from `%s`, %d", source_path, err);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -688,7 +686,6 @@ int DownSync(
     {
         Longtail_Free(source_version_index);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -706,7 +703,6 @@ int DownSync(
     {
         Longtail_Free(source_version_index);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -747,7 +743,6 @@ int DownSync(
             Longtail_Free(source_version_index);
             SAFE_DISPOSE_API(chunker_api);
             SAFE_DISPOSE_API(store_block_store_api);
-//            SAFE_DISPOSE_API(lru_block_store_api);
             SAFE_DISPOSE_API(compress_block_store_api);
             SAFE_DISPOSE_API(store_block_cachestore_api);
             SAFE_DISPOSE_API(store_block_localstore_api);
@@ -797,7 +792,6 @@ int DownSync(
             Longtail_Free(source_version_index);
             SAFE_DISPOSE_API(chunker_api);
             SAFE_DISPOSE_API(store_block_store_api);
-//            SAFE_DISPOSE_API(lru_block_store_api);
             SAFE_DISPOSE_API(compress_block_store_api);
             SAFE_DISPOSE_API(store_block_cachestore_api);
             SAFE_DISPOSE_API(store_block_localstore_api);
@@ -824,7 +818,6 @@ int DownSync(
         Longtail_Free(source_version_index);
         SAFE_DISPOSE_API(chunker_api);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -847,7 +840,6 @@ int DownSync(
         Longtail_Free(source_version_index);
         SAFE_DISPOSE_API(chunker_api);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -876,7 +868,6 @@ int DownSync(
         Longtail_Free(source_version_index);
         SAFE_DISPOSE_API(chunker_api);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -905,7 +896,6 @@ int DownSync(
         Longtail_Free(source_version_index);
         SAFE_DISPOSE_API(chunker_api);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -965,7 +955,6 @@ int DownSync(
     Longtail_Free(source_version_index);
     SAFE_DISPOSE_API(chunker_api);
     SAFE_DISPOSE_API(store_block_store_api);
-//    SAFE_DISPOSE_API(lru_block_store_api);
     SAFE_DISPOSE_API(compress_block_store_api);
     SAFE_DISPOSE_API(store_block_cachestore_api);
     SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1243,16 +1232,14 @@ int VersionIndex_cp(
         compress_block_store_api = Longtail_CreateCompressBlockStoreAPI(store_block_remotestore_api, compression_registry);
     }
 
-//    struct Longtail_BlockStoreAPI* lru_block_store_api = Longtail_CreateLRUBlockStoreAPI(compress_block_store_api, 32);
-    struct Longtail_BlockStoreAPI* store_block_store_api = Longtail_CreateShareBlockStoreAPI(compress_block_store_api);// lru_block_store_api);
+    struct Longtail_BlockStoreAPI* store_block_store_api = Longtail_CreateShareBlockStoreAPI(compress_block_store_api);
 
     struct Longtail_VersionIndex* version_index = 0;
     int err = Longtail_ReadVersionIndex(storage_api, version_index_path, &version_index);
     if (err)
     {
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to read version index from `%s`, %d", version_index_path, err);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1271,7 +1258,6 @@ int VersionIndex_cp(
     {
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Can not create hashing API for version index `%s`, failed with %d", version_index_path, err);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1295,7 +1281,6 @@ int VersionIndex_cp(
     {
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create retarget store index for version index `%s` to `%s`, failed with %d", storage_uri_raw, version_index_path, err);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1314,7 +1299,6 @@ int VersionIndex_cp(
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Store `%s` does not contain all the chunks needed for this version `%s`, Longtail_ValidateStore failed with %d", storage_uri_raw, source_path, err);
         Longtail_Free(block_store_store_index);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1338,7 +1322,6 @@ int VersionIndex_cp(
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create file system for version index `%s`, failed with %d", version_index_path, err);
         Longtail_Free(block_store_store_index);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1359,7 +1342,6 @@ int VersionIndex_cp(
         SAFE_DISPOSE_API(block_store_fs);
         Longtail_Free(block_store_store_index);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1380,7 +1362,6 @@ int VersionIndex_cp(
         SAFE_DISPOSE_API(block_store_fs);
         Longtail_Free(block_store_store_index);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1401,7 +1382,6 @@ int VersionIndex_cp(
         SAFE_DISPOSE_API(block_store_fs);
         Longtail_Free(block_store_store_index);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1437,7 +1417,6 @@ int VersionIndex_cp(
         SAFE_DISPOSE_API(block_store_fs);
         Longtail_Free(block_store_store_index);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1457,7 +1436,6 @@ int VersionIndex_cp(
         SAFE_DISPOSE_API(block_store_fs);
         Longtail_Free(block_store_store_index);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(store_block_cachestore_api);
         SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1473,7 +1451,6 @@ int VersionIndex_cp(
     SAFE_DISPOSE_API(block_store_fs);
     Longtail_Free(block_store_store_index);
     SAFE_DISPOSE_API(store_block_store_api);
-//    SAFE_DISPOSE_API(lru_block_store_api);
     SAFE_DISPOSE_API(compress_block_store_api);
     SAFE_DISPOSE_API(store_block_cachestore_api);
     SAFE_DISPOSE_API(store_block_localstore_api);
@@ -1829,6 +1806,7 @@ int Unpack(
     if (err)
     {
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create hashing api from `%s`, %d", source_path, err);
+        Longtail_Free(archive_index);
         SAFE_DISPOSE_API(storage_api);
         SAFE_DISPOSE_API(compression_registry);
         SAFE_DISPOSE_API(job_api);
@@ -1839,6 +1817,7 @@ int Unpack(
     if (chunker_api == 0)
     {
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create chunking api from `%s`, %d", source_path, err);
+        Longtail_Free(archive_index);
         SAFE_DISPOSE_API(storage_api);
         SAFE_DISPOSE_API(compression_registry);
         SAFE_DISPOSE_API(job_api);
@@ -1860,6 +1839,7 @@ int Unpack(
         {
             LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to scan version content from `%s`, %d", target_path, err);
             SAFE_DISPOSE_API(chunker_api);
+            Longtail_Free(archive_index);
             SAFE_DISPOSE_API(storage_api);
             SAFE_DISPOSE_API(compression_registry);
             SAFE_DISPOSE_API(job_api);
@@ -1902,6 +1882,7 @@ int Unpack(
         {
             LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create version index for `%s`, %d", target_path, err);
             SAFE_DISPOSE_API(chunker_api);
+            Longtail_Free(archive_index);
             SAFE_DISPOSE_API(storage_api);
             SAFE_DISPOSE_API(compression_registry);
             SAFE_DISPOSE_API(job_api);
@@ -1919,6 +1900,9 @@ int Unpack(
     if (archive_block_store_api == 0)
     {
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create archive block store `%s`, %d", source_path, err);
+        Longtail_Free(target_version_index);
+        SAFE_DISPOSE_API(chunker_api);
+        Longtail_Free(archive_index);
         SAFE_DISPOSE_API(storage_api);
         SAFE_DISPOSE_API(compression_registry);
         SAFE_DISPOSE_API(job_api);
@@ -1931,33 +1915,25 @@ int Unpack(
     {
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create compress block store `%s`, %d", source_path, err);
         SAFE_DISPOSE_API(archive_block_store_api);
+        Longtail_Free(target_version_index);
+        SAFE_DISPOSE_API(chunker_api);
+        Longtail_Free(archive_index);
         SAFE_DISPOSE_API(storage_api);
         SAFE_DISPOSE_API(compression_registry);
         SAFE_DISPOSE_API(job_api);
         SAFE_DISPOSE_API(hash_registry);
         return ENOMEM;
     }
 
-//    struct Longtail_BlockStoreAPI* lru_block_store_api = Longtail_CreateLRUBlockStoreAPI(compress_block_store_api, 32);
-//    if (lru_block_store_api == 0)
-//    {
-//        LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create lru block store `%s`, %d", source_path, err);
-//        SAFE_DISPOSE_API(compress_block_store_api);
-//        SAFE_DISPOSE_API(archive_block_store_api);
-//        SAFE_DISPOSE_API(storage_api);
-//        SAFE_DISPOSE_API(compression_registry);
-//        SAFE_DISPOSE_API(job_api);
-//        SAFE_DISPOSE_API(hash_registry);
-//        return ENOMEM;
-//    }
-
-    struct Longtail_BlockStoreAPI* store_block_store_api = Longtail_CreateShareBlockStoreAPI(compress_block_store_api);// lru_block_store_api);
+    struct Longtail_BlockStoreAPI* store_block_store_api = Longtail_CreateShareBlockStoreAPI(compress_block_store_api);
     if (store_block_store_api == 0)
     {
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create share block store `%s`, %d", source_path, err);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(archive_block_store_api);
+        Longtail_Free(target_version_index);
+        SAFE_DISPOSE_API(chunker_api);
+        Longtail_Free(archive_index);
         SAFE_DISPOSE_API(storage_api);
         SAFE_DISPOSE_API(compression_registry);
         SAFE_DISPOSE_API(job_api);
@@ -1975,9 +1951,11 @@ int Unpack(
     {
         LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create diff between `%s` and `%s`, %d", source_path, target_path, err);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(archive_block_store_api);
+        Longtail_Free(target_version_index);
+        SAFE_DISPOSE_API(chunker_api);
+        Longtail_Free(archive_index);
         SAFE_DISPOSE_API(storage_api);
         SAFE_DISPOSE_API(compression_registry);
         SAFE_DISPOSE_API(job_api);
@@ -1992,9 +1970,11 @@ int Unpack(
     {
         Longtail_Free(version_diff);
         SAFE_DISPOSE_API(store_block_store_api);
-//        SAFE_DISPOSE_API(lru_block_store_api);
         SAFE_DISPOSE_API(compress_block_store_api);
         SAFE_DISPOSE_API(archive_block_store_api);
+        Longtail_Free(target_version_index);
+        SAFE_DISPOSE_API(chunker_api);
+        Longtail_Free(archive_index);
         SAFE_DISPOSE_API(storage_api);
         SAFE_DISPOSE_API(compression_registry);
         SAFE_DISPOSE_API(job_api);
@@ -2043,9 +2023,11 @@ int Unpack(
 
     Longtail_Free(version_diff);
     SAFE_DISPOSE_API(store_block_store_api);
-//    SAFE_DISPOSE_API(lru_block_store_api);
     SAFE_DISPOSE_API(compress_block_store_api);
     SAFE_DISPOSE_API(archive_block_store_api);
+    Longtail_Free(target_version_index);
+    SAFE_DISPOSE_API(chunker_api);
+    Longtail_Free(archive_index);
     SAFE_DISPOSE_API(storage_api);
     SAFE_DISPOSE_API(compression_registry);
     SAFE_DISPOSE_API(job_api);