Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use target as chunk source #242

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

use target as chunk source #242

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
caf8665
add Longtail_CreateAssetPartLookup as public function
DanEngelbrecht Jul 26, 2024
634b24a
shell for targetblockstore
DanEngelbrecht Jul 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion all_sources.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ set HASH_REGISTRY_SRC=%BASE_DIR%lib\hashregistry\*.c

set SHAREBLOCKSTORE_SRC=%BASE_DIR%lib\shareblockstore\*.c

set TARGETBLOCKSTORE_SRC=%BASE_DIR%lib\targetblockstore\*.c

set BIKESHED_SRC=%BASE_DIR%lib\bikeshed\*.c

set BLAKE2_SRC=%BASE_DIR%lib\blake2\*.c
Expand All @@ -58,7 +60,7 @@ set ZSTD_SRC=%BASE_DIR%lib\zstd\*.c
set ZSTD_THIRDPARTY_SRC=%BASE_DIR%lib\zstd\ext\common\*.c %BASE_DIR%lib\zstd\ext\compress\*.c %BASE_DIR%lib\zstd\ext\decompress\*.c
set ZSTD_THIRDPARTY_GCC_SRC=%BASE_DIR%lib\zstd\ext\decompress\*.S

set SRC=%BASE_DIR%src\*.c %LIB_SRC% %ARCHIVEBLOCKSTORE_SRC% %ATOMICCANCEL_SRC% %BLOCKSTORESTORAGE_SRC% %COMPRESSBLOCKSTORE_SRC% %CONCURRENTCHUNKWRITE_SRC% %CACHEBLOCKSTORE_SRC% %SHAREBLOCKSTORE_SRC% %FILESTORAGE_SRC% %FSBLOCKSTORE_SRC% %HPCDCCHUNKER_SRC% %LRUBLOCKSTORE_SRC% %MEMSTORAGE_SRC% %MEMTRACER_SRC% %RATELIMITEDPROGRESS_SRC% %COMPRESSION_REGISTRY_SRC% %HASH_REGISTRY_SRC% %BIKESHED_SRC% %BLAKE2_SRC% %BLAKE3_SRC% %MEOWHASH_SRC% %LZ4_SRC% %BROTLI_SRC% %ZSTD_SRC%
set SRC=%BASE_DIR%src\*.c %LIB_SRC% %ARCHIVEBLOCKSTORE_SRC% %ATOMICCANCEL_SRC% %BLOCKSTORESTORAGE_SRC% %COMPRESSBLOCKSTORE_SRC% %CONCURRENTCHUNKWRITE_SRC% %CACHEBLOCKSTORE_SRC% %SHAREBLOCKSTORE_SRC% %FILESTORAGE_SRC% %FSBLOCKSTORE_SRC% %TARGETBLOCKSTORE_SRC% %HPCDCCHUNKER_SRC% %LRUBLOCKSTORE_SRC% %MEMSTORAGE_SRC% %MEMTRACER_SRC% %RATELIMITEDPROGRESS_SRC% %COMPRESSION_REGISTRY_SRC% %HASH_REGISTRY_SRC% %BIKESHED_SRC% %BLAKE2_SRC% %BLAKE3_SRC% %MEOWHASH_SRC% %LZ4_SRC% %BROTLI_SRC% %ZSTD_SRC%
set THIRDPARTY_SRC=%LIB_THIRDPARTY_SRC% %BLAKE3_THIRDPARTY_SRC% %LZ4_THIRDPARTY_SRC% %BROTLI_THIRDPARTY_SRC% %ZSTD_THIRDPARTY_SRC%
set THIRDPARTY_SSE=%BLAKE2_THIRDPARTY_SSE% %BLAKE3_THIRDPARTY_SSE%
set THIRDPARTY_SSE42=%BLAKE3_THIRDPARTY_SSE42%
Expand Down
4 changes: 3 additions & 1 deletion all_sources.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ HASH_REGISTRY_SRC="${BASE_DIR}lib/hashregistry/*.c"

SHAREBLOCKSTORE_SRC="${BASE_DIR}lib/shareblockstore/*.c"

TARGETBLOCKSTORAGE_SRC="${BASE_DIR}lib/targetblockstore/*.c"

BIKESHED_SRC="${BASE_DIR}lib/bikeshed/*.c"

BLAKE2_SRC="${BASE_DIR}lib/blake2/*.c"
Expand All @@ -60,7 +62,7 @@ ZSTD_SRC="${BASE_DIR}lib/zstd/*.c"
ZSTD_THIRDPARTY_SRC="${BASE_DIR}lib/zstd/ext/common/*.c ${BASE_DIR}lib/zstd/ext/compress/*.c ${BASE_DIR}lib/zstd/ext/decompress/*.c"
ZSTD_THIRDPARTY_GCC_SRC="${BASE_DIR}lib/zstd/ext/decompress/*.S"

export SRC="${BASE_DIR}src/*.c $LIB_SRC $ARCHIVEBLOCKSTORE_SRC $ATOMICCANCEL_SRC $BLOCKSTORESTORAGE_SRC $COMPRESSBLOCKSTORE_SRC $CONCURRENTCHUNKWRITE_SRC $CACHEBLOCKSTORE_SRC $SHAREBLOCKSTORE_SRC $FILESTORAGE_SRC $FSBLOCKSTORAGE_SRC $HPCDCCHUNKER_SRC $LRUBLOCKSTORE_SRC $MEMSTORAGE_SRC $MEMTRACER_SRC $RATELIMITEDPROGRESS_SRC $COMPRESSION_REGISTRY_SRC $HASH_REGISTRY_SRC $BIKESHED_SRC $BLAKE2_SRC $BLAKE3_SRC $MEOWHASH_SRC $LZ4_SRC $BROTLI_SRC $ZSTD_SRC"
export SRC="${BASE_DIR}src/*.c $LIB_SRC $ARCHIVEBLOCKSTORE_SRC $ATOMICCANCEL_SRC $BLOCKSTORESTORAGE_SRC $COMPRESSBLOCKSTORE_SRC $CONCURRENTCHUNKWRITE_SRC $CACHEBLOCKSTORE_SRC $SHAREBLOCKSTORE_SRC $FILESTORAGE_SRC $FSBLOCKSTORAGE_SRC $TARGETBLOCKSTORAGE_SRC $HPCDCCHUNKER_SRC $LRUBLOCKSTORE_SRC $MEMSTORAGE_SRC $MEMTRACER_SRC $RATELIMITEDPROGRESS_SRC $COMPRESSION_REGISTRY_SRC $HASH_REGISTRY_SRC $BIKESHED_SRC $BLAKE2_SRC $BLAKE3_SRC $MEOWHASH_SRC $LZ4_SRC $BROTLI_SRC $ZSTD_SRC"
export THIRDPARTY_SRC="$LIB_THIRDPARTY_SRC $BLAKE3_THIRDPARTY_SRC $LZ4_THIRDPARTY_SRC $BROTLI_THIRDPARTY_SRC $ZSTD_THIRDPARTY_SRC"
export THIRDPARTY_SSE="$BLAKE2_THIRDPARTY_SSE $BLAKE3_THIRDPARTY_SSE"
export THIRDPARTY_SSE42="$BLAKE3_THIRDPARTY_SSE42"
Expand Down
27 changes: 26 additions & 1 deletion cmd/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1471,7 +1471,6 @@ int DownSync(
if (err)
{
LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create get required chunks for source `%s`, %d", source_path, err);
Longtail_Free(required_chunk_hashes);
Longtail_Free(version_diff);
Longtail_Free(target_version_index);
Longtail_Free(source_version_index);
Expand All @@ -1488,6 +1487,32 @@ int DownSync(
return err;
}

if (cache_path == 0)
{
struct AssetPartLookup* asset_part_lookup = 0;
err = Longtail_CreateAssetPartLookup(target_version_index, required_chunk_hashes, required_chunk_count, &asset_part_lookup);
if (err)
{
LONGTAIL_LOG(ctx, LONGTAIL_LOG_LEVEL_ERROR, "Failed to create asset part lookup for target `%s`, %d", target_path, err);
Longtail_Free(required_chunk_hashes);
Longtail_Free(version_diff);
Longtail_Free(target_version_index);
Longtail_Free(source_version_index);
SAFE_DISPOSE_API(chunker_api);
SAFE_DISPOSE_API(compress_block_store_api);
SAFE_DISPOSE_API(store_block_cachestore_api);
SAFE_DISPOSE_API(store_block_localstore_api);
SAFE_DISPOSE_API(store_block_remotestore_api);
SAFE_DISPOSE_API(storage_api);
SAFE_DISPOSE_API(compression_registry);
SAFE_DISPOSE_API(hash_registry);
SAFE_DISPOSE_API(job_api);
Longtail_Free((void*)storage_path);
return err;
}
Longtail_Free(asset_part_lookup);
}

struct Longtail_StoreIndex* required_version_store_index;
err = SyncGetExistingContent(
compress_block_store_api,
Expand Down
2 changes: 1 addition & 1 deletion lib/fsblockstore/longtail_fsblockstore.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ struct ScanBlockJob
struct Longtail_BlockIndex* m_BlockIndex;
};

int EndsWith(const char *str, const char *suffix)
static int EndsWith(const char *str, const char *suffix)
{
if (!str || !suffix)
return 0;
Expand Down
Loading
Loading