diff --git a/.gitignore b/.gitignore index f80e5c682..86d5b7799 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ bld/ # Visual Studio 2015/2017 cache/options directory .vs/ +.vscode # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ diff --git a/apps/search_memory_index.cpp b/apps/search_memory_index.cpp index 1a9acc285..2138b9134 100644 --- a/apps/search_memory_index.cpp +++ b/apps/search_memory_index.cpp @@ -145,7 +145,8 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path, double best_recall = 0.0; for (uint32_t test_id = 0; test_id < Lvec.size(); test_id++) - { + { + std::cout<<"L: "<search(query + i * query_aligned_dim, recall_at, L, query_result_ids[test_id].data() + i * recall_at) diff --git a/run_from_python.py b/run_from_python.py new file mode 100644 index 000000000..c2791047d --- /dev/null +++ b/run_from_python.py @@ -0,0 +1,77 @@ +import subprocess +import os + +# Path to the executable and its arguments +exe_path = r"C:\Users\ameyv\MSR\DiskANN\x64\Debug\build_memory_index.exe" +#exe_path = r"C:\Users\ameyv\MSR\DiskANN\x64\Release\build_disk_index.exe" +#exe_path = r"C:\Users\ameyv\MSR\DiskANN\x64\Release\gen_random_slice.exe" + +command_type = 1 + +if command_type == 1: + data_path = r"C:\Users\ameyv\MSR\sift_base.bin" + index_path_prefix = r"C:\Users\ameyv\MSR\sift_memory_index_R64_L100" + arguments = [ + "--data_type", "float", + "--dist_fn", "l2", + "--data_path", data_path, + "--index_path_prefix", index_path_prefix, + "-R", "64", + "-L", "100" + ] +elif command_type == 2: + memory = 1 + data_path = r"C:\Users\ameyv\MSR\sift_base.bin" + index_path_prefix = r"C:\Users\ameyv\MSR\sift_disk_index" + arguments = [ + "--data_type", "float", + "--dist_fn", "l2", + "--data_path", data_path, + "--index_path_prefix", index_path_prefix, + "-R", "16", + "-L", "25", + "-B", "0.003", + "-T","1", + "-M", str(memory) + ] +elif command_type == 3: + base_file_path = r"C:\Users\ameyv\MSR\sift_base.bin" + arguments = ["float", + base_file_path, + "sift_50k", + "0.05" + ] + +# Constructing the complete command +command = [exe_path] + arguments + +# Print the command to debug +print("Executing command:", command) + +# Check if the executable exists and is a file +if not os.path.isfile(exe_path): + print(f"Error: The executable {exe_path} does not exist or is not a file.") +else: + # Execute the executable with arguments and capture the output + try: + # Run the executable with arguments and capture the output and errors + directory = r"C:\Users\ameyv\MSR\DiskANN\results" + "\\" + file_base = r"\log.txt" + file_path = directory + file_base + if os.path.exists(file_path): + # Delete the file + os.remove(file_path) + with open(directory + file_base, 'a') as log_file: # Open the file in append mode + result = subprocess.run(command, stdout=log_file) + + except subprocess.CalledProcessError as e: + print(f"Program failed with return code {e.returncode}") + print(f"Output: {e.output}") + print(f"Errors: {e.stderr}") + + except FileNotFoundError: + print(f"Executable not found: {exe_path}") + + except Exception as e: + print(f"An error occurred: {str(e)}") + diff --git a/src/index.cpp b/src/index.cpp index bf93344fa..61fa86ea3 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -752,6 +752,15 @@ template std::vector Inde } } + // if(_nd > 50000){ + // std::random_device rd; + // std::mt19937 gen(rd()); + // std::uniform_int_distribution dis(0, (uint32_t)_nd-1); + // for(uint32_t i=0;i<1000;i++){ + // init_ids.emplace_back((uint32_t)dis(gen)); + // } + // } + return init_ids; } @@ -879,11 +888,20 @@ std::pair Index::iterate_to_fixed_point( uint32_t hops = 0; uint32_t cmps = 0; - while (best_L_nodes.has_unexpanded_node()) { auto nbr = best_L_nodes.closest_unexpanded(); auto n = nbr.id; + + // if(search_invocation){ + // std::vector id_scratch_temp = {n}; + // std::vector dist_scratch_temp = {0.0}; + // compute_dists(id_scratch_temp, dist_scratch_temp); + // diskann::cout<<"Iteration/Hop: #"< Index::iterate_to_fixed_point( compute_dists(id_scratch, dist_scratch); cmps += (uint32_t)id_scratch.size(); - // Insert pairs into the pool of candidates + // // Insert pairs into the pool of candidates + // if (search_invocation){ + // diskann::cout<<"Comparisons(Neighbors of ID) "<<(uint32_t)id_scratch.size()<get_distance(id_scratch[m], n); + // diskann::cout<<"pt #"<expanded_nodes_vec().size()<expanded_nodes_vec()){ + // diskann::cout< Index::search(const T *query, con } const std::vector unused_filter_label; - const std::vector init_ids = get_init_ids(); + const std::vector init_ids_temp = get_init_ids(); std::shared_lock lock(_update_lock); _data_store->preprocess_query(query, scratch); + std::vector init_ids; + float min_dist = std::numeric_limits::max(); + for(auto id:init_ids_temp){ + + float dist = _data_store->get_distance(query,id); + if(dist < min_dist){ + min_dist = dist; + init_ids.clear(); + init_ids.push_back(id); + } + } + auto retval = iterate_to_fixed_point(scratch, L, init_ids, false, unused_filter_label, true); NeighborPriorityQueue &best_L_nodes = scratch->best_l_nodes();