Showing preview only (677K chars total). Download the full file or copy to clipboard to get everything.
Repository: cmuparlay/ParlayANN
Branch: main
Commit: 573f3cf67350
Files: 157
Total size: 633.9 KB
Directory structure:
gitextract_81v6usbn/
├── .bazelrc
├── .gitmodules
├── CMakeLists.txt
├── LICENSE
├── WORKSPACE
├── algorithms/
│ ├── CMakeLists.txt
│ ├── HCNNG/
│ │ ├── CMakeLists.txt
│ │ ├── Makefile
│ │ ├── clusterEdge.h
│ │ ├── hcnng_index.h
│ │ ├── neighbors.h
│ │ └── scripts/
│ │ ├── fashion
│ │ ├── gist_1
│ │ ├── glove100
│ │ ├── glove25
│ │ ├── nytimes
│ │ └── sift
│ ├── HNSW/
│ │ ├── CMakeLists.txt
│ │ ├── HNSW.hpp
│ │ ├── debug.hpp
│ │ ├── dist.hpp
│ │ ├── h5_ops.hpp
│ │ └── type_point.hpp
│ ├── bench/
│ │ ├── BUILD
│ │ ├── IO.h
│ │ ├── MakeBench
│ │ ├── Makefile
│ │ ├── benchUtils.h
│ │ ├── common/
│ │ │ ├── IO.h
│ │ │ ├── MakeBench
│ │ │ ├── MakeBenchLink
│ │ │ ├── atomics.h
│ │ │ ├── dataGen.h
│ │ │ ├── geometry.h
│ │ │ ├── geometryIO.h
│ │ │ ├── get_time.h
│ │ │ ├── glue.h
│ │ │ ├── graph.h
│ │ │ ├── graphIO.h
│ │ │ ├── graphUtils.h
│ │ │ ├── ligraLight.h
│ │ │ ├── parallelDefs
│ │ │ ├── parallelDefsANN
│ │ │ ├── parallelDefs_OMP
│ │ │ ├── parseCommandLine.h
│ │ │ ├── parse_command_line.h
│ │ │ ├── runTests.py
│ │ │ ├── runTestsANN.py
│ │ │ ├── seqDefs
│ │ │ ├── sequenceIO.h
│ │ │ ├── speculative_for.h
│ │ │ ├── time_loop.h
│ │ │ ├── topology.h
│ │ │ └── topology_from_triangles.h
│ │ ├── get_time.h
│ │ ├── neighborsTime.C
│ │ ├── parallelDefsANN
│ │ ├── parse_command_line.h
│ │ └── time_loop.h
│ ├── pyNNDescent/
│ │ ├── CMakeLists.txt
│ │ ├── Makefile
│ │ ├── clusterPynn.h
│ │ ├── neighbors.h
│ │ ├── pynn_index.h
│ │ └── scripts/
│ │ ├── nytimes
│ │ └── sift
│ ├── tutorial.sh
│ ├── utils/
│ │ ├── BUILD
│ │ ├── NSGDist.h
│ │ ├── beamSearch.h
│ │ ├── check_nn_recall.h
│ │ ├── check_range_recall.h
│ │ ├── csvfile.h
│ │ ├── doublingSearch.h
│ │ ├── earlyStopping.h
│ │ ├── euclidian_point.h
│ │ ├── graph.h
│ │ ├── graph_reorder.h
│ │ ├── hashset.h
│ │ ├── jl_point.h
│ │ ├── mips_point.h
│ │ ├── mmap.h
│ │ ├── parse_results.h
│ │ ├── point_range.h
│ │ ├── rangeSearch.h
│ │ ├── simpleGraph.h
│ │ ├── stats.h
│ │ ├── types.h
│ │ └── union.h
│ ├── vamana/
│ │ ├── BUILD
│ │ ├── CMakeLists.txt
│ │ ├── Makefile
│ │ ├── index.h
│ │ ├── index_test.cc
│ │ ├── neighbors.h
│ │ ├── neighbors.sh
│ │ ├── neighbors_test.cc
│ │ └── scripts/
│ │ ├── OpenAIArXiv
│ │ ├── deep10M
│ │ ├── fashion
│ │ ├── gist
│ │ ├── glove100
│ │ ├── glove25
│ │ ├── msmarco_websearch
│ │ ├── nytimes
│ │ ├── sift
│ │ ├── sift100
│ │ ├── space_1
│ │ ├── space_10
│ │ ├── t2i_1
│ │ ├── t2i_10
│ │ └── wikipedia_cohere
│ └── vamanaRange/
│ ├── CMakeLists.txt
│ ├── Makefile
│ ├── index.h
│ └── neighbors.h
├── build/
│ └── _deps/
│ └── parlaylib-subbuild/
│ └── CMakeLists.txt
├── data_tools/
│ ├── Makefile
│ ├── compute_groundtruth.cpp
│ ├── compute_range_groundtruth.cpp
│ ├── crop.cpp
│ ├── random_sample.cpp
│ └── vec_to_bin.cpp
├── docs/
│ ├── README.md
│ ├── algorithms.md
│ ├── data_tools.md
│ ├── quickstart.md
│ └── rangesearch.md
├── python/
│ ├── __init__.py
│ ├── _builder.py
│ ├── _builder.pyi
│ ├── _common.py
│ ├── _files.py
│ ├── big_env.yml
│ ├── builder.cpp
│ ├── compile.sh
│ ├── defaults.py
│ ├── graph_index.cpp
│ ├── module.cpp
│ ├── scripts/
│ │ ├── fashion_test.py
│ │ ├── gist_test.py
│ │ ├── glove100_test.py
│ │ ├── glove25_test.py
│ │ ├── nyt_test.py
│ │ └── sift_test.py
│ ├── sift_test.py
│ ├── test.py
│ └── wrapper.py
└── rangeSearch/
├── bench/
│ ├── .gitignore
│ ├── IO.h
│ ├── MakeBench
│ ├── Makefile
│ ├── get_time.h
│ ├── parallelDefsANN
│ └── rangeTime.C
└── vamanaRange/
├── Makefile
└── range.h
================================================
FILE CONTENTS
================================================
================================================
FILE: .bazelrc
================================================
# This is from Bazel's former travis setup, to avoid blowing up the RAM usage.
startup --host_jvm_args=-Xmx2500m
startup --host_jvm_args=-Xms2500m
# test --ram_utilization_factor=10 # comment-out for github actions.
# This is so we understand failures better
build --verbose_failures
# This is so we don't use sandboxed execution. Sandboxed execution
# runs stuff in a container, and since Travis already runs its script
# in a container (unless you require sudo in your .travis.yml) this
# fails to run tests.
build --spawn_strategy=standalone --genrule_strategy=standalone
test --test_strategy=standalone
# Below this line, .travis.yml will cat the default bazelrc.
# This is needed so Bazel starts with the base workspace in its
# package path.
# By default build in C++17 mode using the Homegrown scheduler for parallelism.
#build --repo_env=CC=clang++-12
build --repo_env=CC=g++
build --cxxopt=-std=c++17
build --cxxopt=-mcx16 # 16 byte CAS
build --cxxopt=-DHOMEGROWN # use the homegrown scheduler
build --cxxopt=-DLONG # use 8 byte vertex identifiers
build --cxxopt=-DAMORTIZEDPD # use amortized_bytepd encoding scheme for compressed graphs
build --cxxopt=-DUSEMALLOC
build --cxxopt=-DPARLAY_USE_STD_ALLOC
build --cxxopt=-pthread # necessary for homegrown scheduler
build --cxxopt=-march=native
build --cxxopt=-fvisibility=hidden
build --cxxopt=-fvisibility-inlines-hidden
build --cxxopt=-fsized-deallocation # https://github.com/pybind/pybind11/issues/1604 (for clang)
build -c opt
# C++ warning flags.
build --cxxopt=-Wall
build --cxxopt=-Wextra
build --cxxopt=-Wcast-qual
build --cxxopt=-Wno-unused-parameter
build --cxxopt=-Wpointer-arith
# Turning on -Wshadow rather than just -Wshadow=local would be nice, but the
# codebase currently contains lots of instances of global shadowing.
#build --cxxopt=-Wshadow=local
build --cxxopt=-Wvla
# Build without parallelism.
build:serial --cxxopt=-UHOMEGROWN
build:serial --cxxopt=-DPARLAY_SEQUENTIAL
# Build using CilkPlus for parallelism.
build:cilk --cxxopt=-UHOMEGROWN
build:cilk --cxxopt=-DCILK
build:cilk --cxxopt=-fcilkplus
build:cilk --linkopt=-lcilkrts
# Build using OpenMP for parallelism.
build:openmp --cxxopt=-UHOMEGROWN
build:openmp --cxxopt=-DOPENMP
build:openmp --cxxopt=-fopenmp
build:openmp --linkopt=-fopenmp
# Instruments the build with AddressSanitizer
# (https://github.com/google/sanitizers/wiki/AddressSanitizer).
# Invoke by adding the `--config=asan` flag, e.g.,
# bazel run --config=asan <build target>`
build:asan --strip=never
build:asan --cxxopt=-fsanitize=address
build:asan --cxxopt=-O1
build:asan --cxxopt=-g
build:asan --cxxopt=-fno-omit-frame-pointer
build:asan --cxxopt=-Wno-macro-redefined
build:asan --linkopt=-fsanitize=address
================================================
FILE: .gitmodules
================================================
[submodule "parlaylib"]
path = parlaylib
url = https://github.com/cmuparlay/parlaylib.git
================================================
FILE: CMakeLists.txt
================================================
cmake_minimum_required(VERSION 3.15)
project(PARLAYANN VERSION 1
DESCRIPTION "ParlayANN is a library of approximate nearest neighbor search algorithms, along with a set of useful tools for designing such algorithms. It is written in C++ and uses parallel primitives from ParlayLib. Currently it includes implementations of the ANNS algorithms DiskANN, HNSW, HCNNG, and pyNNDescent."
LANGUAGES CXX)
include(CheckCXXCompilerFlag)
include(GNUInstallDirs)
# Set a default build type
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "RELEASE" CACHE STRING "Build type (Release)" FORCE)
message(STATUS "No build type specified. Defaulted to RELEASE.")
message(STATUS "To specify a build type, add -DCMAKE_BUILD_TYPE=<DEBUG/RELEASE/RELWITHDEBINFO/MINSIZEREL>")
endif(NOT CMAKE_BUILD_TYPE)
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -fno-omit-frame-pointer")
set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -march=native")
message(STATUS "PARLAYANN VERSION ${PARLAYANN_VERSION}")
message(STATUS "---------------------------- General configuration -----------------------------")
message(STATUS "CMake Generator: ${CMAKE_GENERATOR}")
message(STATUS "Compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
message(STATUS "CMAKE_CXX_FLAGS: ${CMAKE_CXX_FLAGS}")
message(STATUS "CMAKE_CXX_FLAGS_DEBUG: ${CMAKE_CXX_FLAGS_DEBUG}")
message(STATUS "CMAKE_CXX_FLAGS_RELEASE: ${CMAKE_CXX_FLAGS_RELEASE}")
message(STATUS "CMAKE_CXX_FLAGS_RELWITHDEBINFO: ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
message(STATUS "CMAKE_EXE_LINKER_FLAGS ${CMAKE_EXE_LINKER_FLAGS}")
message(STATUS "CMAKE_INSTALL_PREFIX: ${CMAKE_INSTALL_PREFIX}" )
# Set module path
list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS ON)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
add_library(parlay INTERFACE)
target_include_directories(parlay INTERFACE "${PROJECT_SOURCE_DIR}/parlaylib/include")
# Link against system threads
find_package(Threads REQUIRED)
target_link_libraries(parlay INTERFACE Threads::Threads)
add_subdirectory(algorithms)
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2023 magdalendobson
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: WORKSPACE
================================================
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_configure")
cc_configure()
http_archive(
name = "parlaylib",
sha256 = "68c062ad116fd49d77651d7a24fb985aa66e8ec9ad05176b6af3ab5d29a16b1f",
strip_prefix = "parlaylib-bazel/include/",
urls = ["https://github.com/ParAlg/parlaylib/archive/refs/tags/bazel.tar.gz"],
)
http_archive(
name = "googletest",
sha256 = "b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5",
strip_prefix = "googletest-release-1.11.0",
urls = ["https://github.com/google/googletest/archive/release-1.11.0.tar.gz"],
)
================================================
FILE: algorithms/CMakeLists.txt
================================================
add_subdirectory(HCNNG)
add_subdirectory(HNSW)
add_subdirectory(pyNNDescent)
add_subdirectory(vamana)
add_subdirectory(vamanaRange)
================================================
FILE: algorithms/HCNNG/CMakeLists.txt
================================================
add_executable(neighbors-hcnng ../bench/neighborsTime.C)
target_link_libraries(neighbors-hcnng PRIVATE parlay)
target_precompile_headers(neighbors-hcnng PRIVATE neighbors.h)
================================================
FILE: algorithms/HCNNG/Makefile
================================================
include ../bench/parallelDefsANN
REQUIRE = ../utils/beamSearch.h hcnng_index.h ../utils/graph.h clusterEdge.h
BENCH = neighbors
include ../bench/MakeBench
================================================
FILE: algorithms/HCNNG/clusterEdge.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#pragma once
#include <math.h>
#include <algorithm>
#include <functional>
#include <queue>
#include <random>
#include <set>
#include "../utils/graph.h"
#include "parlay/parallel.h"
#include "parlay/primitives.h"
#include "parlay/random.h"
namespace parlayANN {
std::pair<size_t, size_t>
select_two_random(parlay::sequence<size_t> &active_indices,
parlay::random &rnd) {
size_t first_index = rnd.ith_rand(0) % active_indices.size();
size_t second_index_unshifted = rnd.ith_rand(1) % (active_indices.size() - 1);
size_t second_index = (second_index_unshifted < first_index)
? second_index_unshifted
: (second_index_unshifted + 1);
return {active_indices[first_index], active_indices[second_index]};
}
template <typename Point, typename PointRange, typename indexType>
struct cluster {
using distanceType = typename Point::distanceType;
using edge = std::pair<indexType, indexType>;
using labelled_edge = std::pair<edge, distanceType>;
using GraphI = Graph<indexType>;
using PR = PointRange;
cluster() {}
int generate_index(int N, int i) {
return (N * (N - 1) - (N - i) * (N - i - 1)) / 2;
}
template <typename F>
void recurse(GraphI &G, PR &Points, parlay::sequence<size_t> &active_indices,
parlay::random &rnd, size_t cluster_size, F f, long MSTDeg,
indexType first, indexType second) {
// Split points based on which of the two points are closer.
auto closer_first =
parlay::filter(parlay::make_slice(active_indices), [&](size_t ind) {
distanceType dist_first = Points[ind].distance(Points[first]);
distanceType dist_second = Points[ind].distance(Points[second]);
return dist_first <= dist_second;
});
auto closer_second =
parlay::filter(parlay::make_slice(active_indices), [&](size_t ind) {
distanceType dist_first = Points[ind].distance(Points[first]);
distanceType dist_second = Points[ind].distance(Points[second]);
return dist_second < dist_first;
});
auto left_rnd = rnd.fork(0);
auto right_rnd = rnd.fork(1);
parlay::par_do(
[&]() {
random_clustering(G, Points, closer_first, left_rnd, cluster_size, f,
MSTDeg);
},
[&]() {
random_clustering(G, Points, closer_second, right_rnd, cluster_size,
f, MSTDeg);
});
}
template <typename F>
void random_clustering(GraphI &G, PR &Points,
parlay::sequence<size_t> &active_indices,
parlay::random &rnd, size_t cluster_size, F g,
long MSTDeg) {
if (active_indices.size() <= cluster_size)
g(G, Points, active_indices, MSTDeg);
else {
auto [f, s] = select_two_random(active_indices, rnd);
if (Points[f] == Points[s]) {
parlay::sequence<size_t> closer_first;
parlay::sequence<size_t> closer_second;
for (int i = 0; i < active_indices.size(); i++) {
if (i < active_indices.size() / 2)
closer_first.push_back(active_indices[i]);
else
closer_second.push_back(active_indices[i]);
}
auto left_rnd = rnd.fork(0);
auto right_rnd = rnd.fork(1);
parlay::par_do(
[&]() {
random_clustering(G, Points, closer_first, left_rnd, cluster_size,
g, MSTDeg);
},
[&]() {
random_clustering(G, Points, closer_second, right_rnd,
cluster_size, g, MSTDeg);
});
} else {
recurse(G, Points, active_indices, rnd, cluster_size, g, MSTDeg, f, s);
}
}
}
template <typename F>
void random_clustering_wrapper(GraphI &G, PR &Points, size_t cluster_size,
F f, long MSTDeg) {
std::random_device rd;
std::mt19937 rng(rd());
std::uniform_int_distribution<int> uni(0, Points.size());
parlay::random rnd(uni(rng));
auto active_indices =
parlay::tabulate(Points.size(), [&](size_t i) { return i; });
random_clustering(G, Points, active_indices, rnd, cluster_size, f, MSTDeg);
}
template <typename F>
void multiple_clustertrees(GraphI &G, PR &Points, long cluster_size,
long num_clusters, F f, long MSTDeg) {
for (long i = 0; i < num_clusters; i++) {
random_clustering_wrapper(G, Points, cluster_size, f, MSTDeg);
std::cout << "Built cluster " << i << " of " << num_clusters << std::endl;
}
}
};
} // end namespace
================================================
FILE: algorithms/HCNNG/hcnng_index.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#include <math.h>
#include <algorithm>
#include <queue>
#include <random>
#include <set>
#include "../utils/graph.h"
#include "clusterEdge.h"
#include "parlay/parallel.h"
#include "parlay/primitives.h"
#include "parlay/random.h"
namespace parlayANN {
struct DisjointSet {
parlay::sequence<int> parent;
parlay::sequence<int> rank;
size_t N;
DisjointSet(size_t size) {
N = size;
parent = parlay::sequence<int>(N);
rank = parlay::sequence<int>(N);
parlay::parallel_for(0, N, [&](size_t i) {
parent[i] = i;
rank[i] = 0;
});
}
void _union(int x, int y) {
int xroot = parent[x];
int yroot = parent[y];
int xrank = rank[x];
int yrank = rank[y];
if (xroot == yroot)
return;
else if (xrank < yrank)
parent[xroot] = yroot;
else {
parent[yroot] = xroot;
if (xrank == yrank) rank[xroot] = rank[xroot] + 1;
}
}
int find(int x) {
if (parent[x] == x) return x;
int c = x;
while (parent[c] != c) {
c = parent[c];
}
while (x != c) {
int s = parent[x];
parent[x] = c;
x = s;
}
return c;
}
void flatten() {
for (int i = 0; i < N; i++) find(i);
}
bool is_full() {
flatten();
parlay::sequence<bool> truthvals(N);
parlay::parallel_for(
0, N, [&](size_t i) { truthvals[i] = (parent[i] == parent[0]); });
auto ff = [&](bool a) { return not a; };
auto filtered = parlay::filter(truthvals, ff);
if (filtered.size() == 0) return true;
return false;
}
};
template <typename Point, typename PointRange, typename indexType>
struct hcnng_index {
using distanceType = typename Point::distanceType;
using edge = std::pair<indexType, indexType>;
using labelled_edge = std::pair<edge, distanceType>;
using pid = std::pair<indexType, distanceType>;
using GraphI = Graph<indexType>;
using PR = PointRange;
static constexpr indexType kNullId = std::numeric_limits<indexType>::max();
static constexpr distanceType kNullDist =
std::numeric_limits<distanceType>::max();
static constexpr labelled_edge kNullEdge = {{kNullId, kNullId}, kNullDist};
hcnng_index() {}
static void remove_edge_duplicates(indexType p, GraphI &G) {
parlay::sequence<indexType> points;
for (indexType i = 0; i < G[p].size(); i++) {
points.push_back(G[p][i]);
}
auto np = parlay::remove_duplicates(points);
G[p].update_neighbors(np);
}
void remove_all_duplicates(GraphI &G) {
parlay::parallel_for(0, G.size(),
[&](size_t i) { remove_edge_duplicates(i, G); });
}
// inserts each edge after checking for duplicates
static void process_edges(GraphI &G, parlay::sequence<edge> edges) {
long maxDeg = G.max_degree();
auto grouped = parlay::group_by_key(edges);
parlay::parallel_for(0, grouped.size(), [&](size_t i) {
int32_t index = grouped[i].first;
for (auto c : grouped[i].second) {
if (G[index].size() < maxDeg) {
G[index].append_neighbor(c);
} else {
remove_edge_duplicates(index, G);
G[index].append_neighbor(c);
}
}
});
}
// parameters dim and K are just to interface with the cluster tree code
static void MSTk(GraphI &G, PR &Points,
parlay::sequence<size_t> &active_indices, long MSTDeg) {
// preprocessing for Kruskal's
size_t N = active_indices.size();
long dim = Points.dimension();
DisjointSet disjset(N);
size_t m = 10;
auto less = [&](labelled_edge a, labelled_edge b) {
return a.second < b.second;
};
parlay::sequence<labelled_edge> candidate_edges(N * m, kNullEdge);
parlay::parallel_for(0, N, [&](size_t i) {
std::priority_queue<labelled_edge, std::vector<labelled_edge>,
decltype(less)>
Q(less);
for (indexType j = i + 1; j < N; j++) {
distanceType dist_ij =
Points[active_indices[i]].distance(Points[active_indices[j]]);
if (Q.size() >= m) {
distanceType topdist = Q.top().second;
if (dist_ij < topdist) {
labelled_edge e;
e = std::make_pair(std::make_pair(i, j), dist_ij);
Q.pop();
Q.push(e);
}
} else {
labelled_edge e;
e = std::make_pair(std::make_pair(i, j), dist_ij);
Q.push(e);
}
}
indexType limit = std::min(Q.size(), m);
for (indexType j = 0; j < limit; j++) {
candidate_edges[i * m + j] = Q.top();
Q.pop();
}
});
parlay::sort_inplace(candidate_edges, less);
auto degrees =
parlay::tabulate(active_indices.size(), [&](size_t i) { return 0; });
parlay::sequence<edge> MST_edges = parlay::sequence<edge>();
// modified Kruskal's algorithm
for (indexType i = 0; i < candidate_edges.size(); i++) {
// Since we sorted, any null edges form the suffix.
if (candidate_edges[i].second == kNullDist) break;
labelled_edge e_l = candidate_edges[i];
edge e = e_l.first;
if ((disjset.find(e.first) != disjset.find(e.second)) &&
(degrees[e.first] < MSTDeg) && (degrees[e.second] < MSTDeg)) {
MST_edges.push_back(
std::make_pair(active_indices[e.first], active_indices[e.second]));
MST_edges.push_back(
std::make_pair(active_indices[e.second], active_indices[e.first]));
degrees[e.first] += 1;
degrees[e.second] += 1;
disjset._union(e.first, e.second);
}
if (i % N == 0) {
if (disjset.is_full()) {
break;
}
}
}
process_edges(G, std::move(MST_edges));
}
// robustPrune routine as found in DiskANN paper, with the exception that the
// new candidate set is added to the field new_nbhs instead of directly
// replacing the out_nbh of p
void robustPrune(indexType p, PR &Points, GraphI &G, double alpha) {
// add out neighbors of p to the candidate set.
parlay::sequence<pid> candidates;
for (size_t i = 0; i < G[p].size(); i++) {
candidates.push_back(
std::make_pair(G[p][i], Points[p].distance(Points[G[p][i]])));
}
// Sort the candidate set in reverse order according to distance from p.
auto less = [&](pid a, pid b) { return a.second < b.second; };
parlay::sort_inplace(candidates, less);
parlay::sequence<int> new_nbhs = parlay::sequence<int>();
size_t candidate_idx = 0;
while (new_nbhs.size() < G.max_degree() &&
candidate_idx < candidates.size()) {
// Don't need to do modifications.
indexType p_star = candidates[candidate_idx].first;
candidate_idx++;
if (p_star == p || p_star == kNullId) continue;
new_nbhs.push_back(p_star);
for (size_t i = candidate_idx; i < candidates.size(); i++) {
indexType p_prime = candidates[i].first;
if (p_prime != kNullId) {
distanceType dist_starprime =
Points[p_star].distance(Points[p_prime]);
distanceType dist_pprime = candidates[i].second;
if (alpha * dist_starprime <= dist_pprime)
candidates[i].first = kNullId;
}
}
}
G[p].update_neighbors(new_nbhs);
}
void build_index(GraphI &G, PR &Points, long cluster_rounds,
long cluster_size, long MSTDeg) {
cluster<Point, PointRange, indexType> C;
C.multiple_clustertrees(G, Points, cluster_size, cluster_rounds, MSTk,
MSTDeg);
remove_all_duplicates(G);
// TODO: enable optional pruning (what is below now works, but
// should be connected cleanly)
// parlay::parallel_for(0, G.size(), [&] (size_t i){robustPrune(i, Points,
// G, 1.1);});
}
};
} // namespace parlayANN
================================================
FILE: algorithms/HCNNG/neighbors.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#include <algorithm>
#include <cmath>
#include "parlay/parallel.h"
#include "parlay/primitives.h"
#include "parlay/random.h"
#include "../utils/NSGDist.h"
#include "../utils/types.h"
#include "../utils/beamSearch.h"
#include "../utils/stats.h"
#include "../utils/parse_results.h"
#include "../utils/check_nn_recall.h"
#include "../utils/graph.h"
#include "hcnng_index.h"
namespace parlayANN {
template<typename Point, typename PointRange, typename indexType>
void ANN(Graph<indexType> &G, long k, BuildParams &BP,
PointRange &Query_Points,
groundTruth<indexType> GT, char *res_file,
bool graph_built, PointRange &Points) {
parlay::internal::timer t("ANN");
using findex = hcnng_index<Point, PointRange, indexType>;
double idx_time;
if(!graph_built){
findex I;
I.build_index(G, Points, BP.num_clusters, BP.cluster_size, BP.MST_deg);
idx_time = t.next_time();
} else{idx_time=0;}
std::string name = "HCNNG";
std::string params = "Trees = " + std::to_string(BP.num_clusters);
auto [avg_deg, max_deg] = graph_stats_(G);
Graph_ G_(name, params, G.size(), avg_deg, max_deg, idx_time);
G_.print();
if(Query_Points.size() != 0)
search_and_parse(G_, G, Points, Query_Points, GT, res_file, k, BP.verbose);
}
} // end namespace
================================================
FILE: algorithms/HCNNG/scripts/fashion
================================================
# bash
numactl -i all ./neighbors -R 64 -L 128 -alpha 1.15 -data_type float -file_type bin -dist_func Euclidian -base_path data/fashion-mnist-784-euclidean/fashion-mnist-784-euclidean_base.fbin -query_path data/fashion-mnist-784-euclidean/fashion-mnist-784-euclidean_query.fbin -gt_path data/fashion-mnist-784-euclidean/fashion-mnist-784-euclidean_groundtruth -quantize 8 -two_pass 1 -graph_path data/fashion-mnist-784-euclidean/graph_64 -verbose
================================================
FILE: algorithms/HCNNG/scripts/gist_1
================================================
# bash
numactl -i all ./neighbors -R 100 -L 200 -alpha 1.1 -data_type float -file_type bin -dist_func Euclidian -base_path data/gist/gist_base.fbin -query_path data/gist/gist_query.fbin -gt_path data/gist/gist-1M -quantize 16 -two_pass 1 -quantize_build 1 -graph_path data/gist/graph
================================================
FILE: algorithms/HCNNG/scripts/glove100
================================================
# bash
numactl -i all ./neighbors -R 150 -L 300 -alpha 1 -data_type float -file_type bin -dist_func mips -base_path data/glove-100-angular/glove-100-angular_base.fbin -query_path data/glove-100-angular/glove-100-angular_query.fbin -gt_path data/glove-100-angular/glove-100-angular_groundtruth -num_passes 2 -quantize_build 1 -quantize 16 -normalize -verbose -graph_path data/glove-100-angular/graph
================================================
FILE: algorithms/HCNNG/scripts/glove25
================================================
# bash
numactl -i all ./neighbors -R 150 -L 300 -alpha 1 -data_type float -file_type bin -dist_func mips -base_path data/glove-25-angular/glove-25-angular_base.fbin -query_path data/glove-25-angular/glove-25-angular_query.fbin -gt_path data/glove-25-angular/glove-25-angular_groundtruth -num_passes 2 -quantize_build -normalize -verbose
================================================
FILE: algorithms/HCNNG/scripts/nytimes
================================================
# bash
BUILD_ARGS="-cluster_size 1000 -mst_deg 3 -num_clusters 30 -quantize_bits 8 -verbose"
QUERY_ARGS="-quantize_bits 16 -verbose"
TYPE_ARGS="-data_type float -dist_func mips -normalize -file_type bin"
PATH=data/nytimes-256-angular
DATA_FILE=$PATH/base.fbin
QUERY_FILE=$PATH/query.fbin
GROUNDTRUTH_FILE=$PATH/groundtruth
GRAPH_FILE=$PATH/graphs/graph_hcnng_1000_3_30
# build
echo ./neighbors $BUILD_ARGS $TYPE_ARGS -base_path $DATA_FILE -graph_outfile $GRAPH_FILE
# query
echo ./neighbors $QUERY_ARGS $TYPE_ARGS -base_path $DATA_FILE -query_path $QUERY_FILE -gt_path $GROUNDTRUTH_FILE -graph_path $GRAPH_FILE
================================================
FILE: algorithms/HCNNG/scripts/sift
================================================
# bash
BUILD_ARGS="-cluster_size 1000 -mst_deg 3 -num_clusters 30 -quantize_bits 8 -verbose"
QUERY_ARGS="-quantize_bits 8 -verbose"
TYPE_ARGS="-data_type float -dist_func Euclidian -file_type bin"
PATH=data/sift-128-euclidean
DATA_FILE=$PATH/base.fbin
QUERY_FILE=$PATH/query.fbin
GROUNDTRUTH_FILE=$PATH/groundtruth
GRAPH_FILE=$PATH/graphs/graph_hcnng_1000_3_30
# build
echo ./neighbors $BUILD_ARGS $TYPE_ARGS -base_path $DATA_FILE -graph_outfile $GRAPH_FILE
# query
echo ./neighbors $QUERY_ARGS $TYPE_ARGS -base_path $DATA_FILE -query_path $QUERY_FILE -gt_path $GROUNDTRUTH_FILE -graph_path $GRAPH_FILE
================================================
FILE: algorithms/HNSW/CMakeLists.txt
================================================
# add_executable(neighbors-hnsw ../bench/neighborsTime.C)
# target_link_libraries(neighbors-hnsw PRIVATE parlay)
# target_precompile_headers(neighbors-hnsw PRIVATE HNSW.hpp)
================================================
FILE: algorithms/HNSW/HNSW.hpp
================================================
#ifndef _HNSW_HPP
#define _HNSW_HPP
#include <cstdint>
#include <cstdio>
#include <cstdarg>
#include <cassert>
#include <cmath>
#include <algorithm>
#include <numeric>
#include <random>
#include <memory>
#include <atomic>
#include <fstream>
#include <string>
#include <vector>
#include <unordered_set>
#include <unordered_map>
#include <queue>
#include <set>
#include <iterator>
#include <type_traits>
#include <limits>
#include <thread>
// #include "parallelize.h"
#include <parlay/parallel.h>
#include <parlay/primitives.h>
#include <parlay/delayed_sequence.h>
#include <parlay/random.h>
#include "debug.hpp"
#include "../utils/beamSearch.h"
#define DEBUG_OUTPUT 0
#if DEBUG_OUTPUT
#define debug_output(...) fprintf(stderr, __VA_ARGS__)
#else
#define debug_output(...) do{[](...){}(__VA_ARGS__);}while(0)
#endif // DEBUG_OUTPUT
namespace ANN{
using namespace parlayANN;
enum class type_metric{
L2, ANGULAR, DOT
};
struct point{
float x, y;
};
template<typename U, template<typename> class Allocator=std::allocator>
class HNSW
{
using T = typename U::type_point;
typedef uint32_t node_id;
public:
/*
Construct from the vectors [begin, end).
std::iterator_trait<Iter>::value_type ought to be convertible to T
dim: vector dimension
m_l: control the # of levels (larger m_l leads to more layer)
m: max degree
ef_construction: beam size during the construction
alpha: parameter of the heuristic (similar to the one in vamana)
batch_base: growth rate of the batch size (discarded because of two passes)
*/
template<typename Iter>
HNSW(Iter begin, Iter end, uint32_t dim, float m_l=1, uint32_t m=100, uint32_t ef_construction=50, float alpha=5, float batch_base=2);
/*
Construct from the saved model
getter(i) returns the actual data (convertible to type T) of the vector with id i
*/
template<typename G>
HNSW(const std::string &filename_model, G getter);
parlay::sequence<std::pair<uint32_t,float>> search(
const T &q, uint32_t k, uint32_t ef, const search_control &ctrl={}
);
// parlay::sequence<std::tuple<uint32_t,uint32_t,float>> search_ex(const T &q, uint32_t k, uint32_t ef, uint64_t verbose=0);
// save the current model to a file
void save(const std::string &filename_model) const;
public:
typedef uint32_t type_index;
struct node{
// uint32_t id;
uint32_t level;
parlay::sequence<node_id> *neighbors;
T data;
};
struct dist{
float d;
node_id u;
constexpr bool operator<(const dist &rhs) const{
return d<rhs.d;
}
constexpr bool operator>(const dist &rhs) const{
return d>rhs.d;
}
};
struct dist_ex : dist
{
uint32_t depth;
};
struct nearest{
constexpr bool operator()(const dist &lhs, const dist &rhs) const{
return lhs.d>rhs.d;
}
};
struct farthest{
constexpr bool operator()(const dist &lhs, const dist &rhs) const{
return lhs.d<rhs.d;
}
};
/*
struct cmp_id{
constexpr bool operator()(const dist &lhs, const dist &rhs) const{
return U::get_id(get_node(lhs.u).data)<U::get_id(get_node(rhs.u).data);
}
};
*/
parlay::sequence<node_id> entrance; // To init
// auto m, max_m0, m_L; // To init
uint32_t dim;
float m_l;
uint32_t m;
// uint32_t level_max = 30; // To init
uint32_t ef_construction;
float alpha;
uint32_t n;
Allocator<node> allocator;
parlay::sequence<node> node_pool;
mutable parlay::sequence<size_t> total_visited = parlay::sequence<size_t>(parlay::num_workers());
mutable parlay::sequence<size_t> total_eval = parlay::sequence<size_t>(parlay::num_workers());
mutable parlay::sequence<size_t> total_size_C = parlay::sequence<size_t>(parlay::num_workers());
mutable parlay::sequence<size_t> total_range_candidate = parlay::sequence<size_t>(parlay::num_workers());
static auto neighbourhood(node &u, uint32_t level)
-> parlay::sequence<node_id>&
{
// const constexpr auto level_none = std::numeric_limits<uint32_t>::max();
// return level==level_none? u.final_nbh: u.neighbors[level];
// return level==0? u.final_nbh: u.neighbors[level];
return u.neighbors[level];
}
static auto neighbourhood(const node &u, uint32_t level)
-> const parlay::sequence<node_id>&
{
return neighbourhood(const_cast<node&>(u),level);
}
node& get_node(node_id id)
{
return node_pool[id];
}
const node& get_node(const node_id id) const
{
return node_pool[id];
}
/*
static void add_connection(parlay::sequence<node_id> &neighbors, node &u, uint32_t level)
{
for(auto pv : neighbors)
{
assert(&u!=pv);
pv->neighbors[level].push_back(&u);
u.neighbors[level].push_back(pv);
}
}
*/
class dist_evaluator{
using point_t = T;
using dist_t = float;
std::reference_wrapper<const point_t> p;
uint32_t dim;
public:
dist_evaluator(const point_t &p, uint32_t dim) :
p(p), dim(dim){
}
dist_t operator()(const point_t &pv) const{
return U::distance(p, pv, dim);
}
dist_t operator()(const point_t &pu, const point_t &pv) const{
return U::distance(pu, pv, dim);
}
};
struct graph{
template<class Nbh>
struct edgeRange{
edgeRange(Nbh &nbh) : nbh(nbh){
}
decltype(auto) operator[](node_id pu) const{
return nbh.get()[pu];
}
auto size() const{
return nbh.get().size();
}
void prefetch() const{
int l = (size() * sizeof(node_id))/64;
for (int i=0; i < l; i++)
__builtin_prefetch((char*) nbh.get().data() + i*64);
}
std::reference_wrapper<Nbh> nbh;
};
using nid_t = node_id;
graph(const HNSW<U,Allocator> &hnsw, uint32_t l) :
hnsw(hnsw), l(l){
}
decltype(auto) num_nodes() const{
return hnsw.get().n;
}
decltype(auto) get_node(node_id pu) const{
return hnsw.get().get_node(pu);
}
decltype(auto) get_edges(node_id pu){
return hnsw.get().neighbourhood(hnsw.get().get_node(pu),l);
}
decltype(auto) get_edges(node_id pu) const{
return hnsw.get().neighbourhood(hnsw.get().get_node(pu),l);
}
uint32_t max_degree() const{
return hnsw.get().get_threshold_m(l);
}
auto operator[](node_id pu){
return edgeRange(get_edges(pu));
}
auto operator[](node_id pu) const{
return edgeRange(get_edges(pu));
}
std::reference_wrapper<const HNSW<U,Allocator>> hnsw;
uint32_t l;
};
// node* insert(const T &q, uint32_t id);
template<typename Iter>
void insert(Iter begin, Iter end, bool from_blank);
template<typename Queue>
void select_neighbors_simple_impl(const T &u, Queue &C, uint32_t M)
{
/*
list res;
for(uint32_t i=0; i<M; ++i)
{
res.insert(C.pop_front());
}
return res;
*/
(void)u;
parlay::sequence<typename Queue::value_type> tie;
float dist_tie = 1e20;
while(C.size()>M)
{
const auto &t = C.top();
if(t.d+1e-6<dist_tie) // t.d<dist_tie
{
dist_tie = t.d;
tie.clear();
}
if(fabs(dist_tie-t.d)<1e-6) // t.d==dist_tie
tie.push_back(t);
C.pop();
}
if(fabs(dist_tie-C.top().d)<1e-6) // C.top().d==dist_tie
while(!tie.empty())
{
// C.push({dist_tie,tie.back()});
C.push(tie.back());
tie.pop_back();
}
}
template<typename Queue>
auto select_neighbors_simple(const T &u, const Queue &C, uint32_t M)
{
// The parameter C is intended to be copy constructed
/*
select_neighbors_simple_impl(u, C, M);
return C;
*/
// auto R = parlay::sort(C, farthest());
auto R = C;
if(R.size()>M)
{
std::nth_element(R.begin(), R.begin()+M, R.end(), farthest());
R.resize(M);
}
std::sort(R.begin(), R.end(), farthest());
// if(R.size()>M) R.resize(M);
/*
uint32_t size_R = std::min(C.size(),M);
parlay::sequence<node*> R;
R.reserve(size_R);
for(const auto &e : C)
R.push_back(e.u);
*/
return R;
}
// To optimize
auto select_neighbors_heuristic(const T &u,
/*const std::priority_queue<dist,parlay::sequence<dist>,farthest> &C*/
const parlay::sequence<dist> &C, uint32_t M,
uint32_t level, bool extendCandidate, bool keepPrunedConnections)
{
(void)extendCandidate;
// std::priority_queue<dist,parlay::sequence<dist>,farthest> C_cp=C, W_d;
parlay::sequence<dist> W_d;
std::set<node_id> W_tmp;
// while(!C_cp.empty())
for(auto &e : C) // TODO: add const?
{
// auto &e = C_cp.top();
W_tmp.insert(e.u);
if(extendCandidate)
{
for(node_id e_adj : neighbourhood(get_node(e.u),level))
{
// if(e_adj==nullptr) continue; // TODO: check
if(W_tmp.find(e_adj)==W_tmp.end())
W_tmp.insert(e_adj);
}
}
// C_cp.pop();
}
// std::priority_queue<dist,parlay::sequence<dist>,nearest> W;
parlay::sequence<dist> W;
W.reserve(W_tmp.size());
for(node_id p : W_tmp)
W.push_back({U::distance(get_node(p).data,u,dim), p});
std::sort(W.begin(), W.end(), farthest());
/*
for(auto &e : W_tmp)
W.push(e);
*/
W_tmp.clear();
parlay::sequence<node_id> R;
std::set<node_id> nbh;
// while(W.size()>0 && R.size()<M)
for(const auto &e : W)
{
if(R.size()>=M) break;
// const auto e = W.top();
// W.pop();
const auto d_q = e.d;
bool is_good = true;
for(const auto &r : R)
{
const auto d_r = U::distance(get_node(e.u).data, get_node(r).data, dim);
//if(d_r*(level+1)>d_q*alpha*(entrance->level+1))
if(d_r<d_q*alpha)
{
is_good = false;
break;
}
/*
for(auto *pv : neighbourhood(*e.u,level))
if(pv==e.u)
{
is_good = false;
break;
}
*/
/*
if(nbh.find(e.u)!=nbh.end())
is_good = false;
*/
}
if(is_good)
{
R.push_back(e.u);
/*
for(auto *pv : neighbourhood(*e.u,level))
nbh.insert(pv);
*/
}
else
W_d.push_back(e);
}
// std::sort(W_d.begin(), W_d.end(), nearest());
auto it = W_d.begin();
// std::priority_queue<dist,parlay::sequence<dist>,farthest> res;
auto &res = R;
/*
for(const auto &r : R)
{
res.push({U::distance(u,get_node(r).data,dim), r});
}
*/
if(keepPrunedConnections)
{
// while(W_d.size()>0 && res.size()<M)
// res.push(W_d.top()), W_d.pop();
while(it!=W_d.end() && res.size()<M)
// res.push(*(it++));
res.push_back((it++)->u);
}
return res;
}
template<class Seq_, class D, class G, class Seq=std::remove_cv_t<std::remove_reference_t<Seq_>>>
Seq prune_heuristic(
Seq_ &&cand, uint32_t size, D f_dist, G g) const
{
using nid_t = node_id;
using conn = dist;
Seq workset = std::forward<Seq_>(cand);
/*
if(ctrl.extend_nbh)
{
const auto &g = ctrl.graph;
std::unordered_set<nid_t> cand_ext;
for(const conn &c : workset)
{
cand_ext.insert(c.u);
for(nid_t pv : g.get_edges(c.u))
cand_ext.insert(pv);
}
workset.reserve(workset.size()+cand_ext.size());
for(nid_t pc : cand_ext)
workset.push_back({f_dist(g.get_node(pc).get_coord()), pc});
cand_ext.clear();
}
*/
parlay::sort_inplace(workset);
Seq res, pruned;
std::unordered_set<nid_t> nbh;
for(conn &c : workset)
{
const auto d_cu = c.d*alpha;
bool is_pruned = false;
for(const conn &r : res)
{
const auto d_cr = f_dist(
g.get_node(c.u).data,
g.get_node(r.u).data
);
if(d_cr<d_cu)
{
is_pruned = true;
break;
}
}
if(!is_pruned)
{
res.push_back(std::move(c));
if(res.size()==size) break;
}
else pruned.push_back(std::move(c));
}
return res;
}
auto select_neighbors(const T &u,
/*const std::priority_queue<dist,parlay::sequence<dist>,farthest> &C,*/
const parlay::sequence<dist> &C, uint32_t M,
uint32_t level, bool extendCandidate=false, bool keepPrunedConnections=false)
{
/*
(void)level, (void)extendCandidate, (void)keepPrunedConnections;
return select_neighbors_simple(u,C,M);
*/
// return select_neighbors_heuristic(u, C, M, level, extendCandidate, keepPrunedConnections);
dist_evaluator f_dist(u, dim);
graph g(*this, level);
auto res = prune_heuristic(C, M, f_dist, g);
return parlay::tabulate(res.size(), [&](size_t i){return res[i].u;});
}
uint32_t get_level_random()
{
// static thread_local int32_t anchor;
// uint32_t esp;
// asm volatile("movl %0, %%esp":"=a"(esp));
// static thread_local std::hash<std::thread::id> h;
// static thread_local std::mt19937 gen{h(std::this_thread::get_id())};
static thread_local std::mt19937 gen{parlay::worker_id()};
static thread_local std::uniform_real_distribution<> dis(std::numeric_limits<float>::min(), 1.0);
const uint32_t res = uint32_t(-log(dis(gen))*m_l);
return res;
}
// auto search_layer(const node &u, const parlay::sequence<node_id> &eps, uint32_t ef, uint32_t l_c, uint64_t verbose=0) const; // To static
auto search_layer(const node &u, const parlay::sequence<node_id> &eps, uint32_t ef, uint32_t l_c, search_control ctrl={}) const; // To static
auto search_layer_bak(const node &u, const parlay::sequence<node_id> &eps, uint32_t ef, uint32_t l_c, search_control ctrl={}) const; // To static
auto search_layer_new_ex(const node &u, const parlay::sequence<node_id> &eps, uint32_t ef, uint32_t l_c, search_control ctrl={}) const; // To static
auto beam_search_ex(const node &u, const parlay::sequence<node_id> &eps, uint32_t beamSize, uint32_t l_c, search_control ctrl={}) const;
parlay::sequence<node_id> search_layer_to(
const node &u, uint32_t ef, uint32_t l_stop, const search_control &ctrl={}
);
auto get_threshold_m(uint32_t level) const{
return level==0? m*2: m;
// (void)level;
// return m;
}
public:
auto get_deg(uint32_t level=0)
{
parlay::sequence<uint32_t> res;
res.reserve(node_pool.size());
for(const node &e : node_pool)
{
if(e.level>=level)
res.push_back(e.neighbors[level].size());
}
return res;
}
auto get_indeg(uint32_t level) const
{
static uint32_t *indeg[16] = {nullptr};
auto *&res = indeg[level];
if(!res)
{
res = new uint32_t[n];
for(uint32_t i=0; i<n; ++i)
res[i] = 0;
for(const node_id pu : node_pool)
{
if(get_node(pu).level<level) continue;
for(const node_id pv : get_node(pu).neighbors[level])
res[U::get_id(get_node(pv).data)]++;
}
}
return res;
}
uint32_t get_height() const
{
return get_node(entrance[0]).level;
}
size_t cnt_degree(uint32_t l) const
{
auto cnt_each = parlay::delayed_seq<size_t>(n, [&](size_t i){
node_id pu = i;
return get_node(pu).level<l? 0:
neighbourhood(get_node(pu),l).size();
});
return parlay::reduce(cnt_each, parlay::addm<size_t>());
}
size_t cnt_vertex(uint32_t l) const
{
auto cnt_each = parlay::delayed_seq<size_t>(n, [&](size_t i){
node_id pu = i;
return get_node(pu).level<l? 0: 1;
});
return parlay::reduce(cnt_each, parlay::addm<size_t>());
}
size_t get_degree_max(uint32_t l) const
{
auto cnt_each = parlay::delayed_seq<size_t>(n, [&](size_t i){
node_id pu = i;
return get_node(pu).level<l? 0:
neighbourhood(get_node(pu),l).size();
});
return parlay::reduce(cnt_each, parlay::maxm<size_t>());
}
/*
void debug_output_graph(uint32_t l)
{
// return;
debug_output("Printing the graph at level %u\n", l);
auto node_exist = parlay::pack(
node_pool,
parlay::delayed_seq<bool>(node_pool.size(),[&](size_t i){
return node_pool[i]->level>=l;
})
);
const auto num_vertices = node_exist.size();
const auto num_edges = parlay::reduce(
parlay::delayed_seq<uint64_t>(node_exist.size(),[&](size_t i){
return node_exist[i]->neighbors[l].size();
}),
parlay::addm<uint64_t>{}
);
debug_output("# vertices: %lu, # edges: %llu\n", num_vertices, num_edges);
for(node_id pu : node_exist)
{
debug_output("node_id: %u\n", U::get_id(get_node(pu).data));
// if(!res[i]) continue;
debug_output("\tneighbors:");
for(node_id pv : neighbourhood(get_node(pu),l))
debug_output(" %u", U::get_id(get_node(pv).data));
debug_output("\n");
}
}
*/
};
template<typename U, template<typename> class Allocator>
template<typename G>
HNSW<U,Allocator>::HNSW(const std::string &filename_model, G getter)
{
std::ifstream model(filename_model, std::ios::binary);
if(!model.is_open())
throw std::runtime_error("Failed to open the model");
const auto size_buffer = 1024*1024*1024; // 1G
auto buffer = std::make_unique<char[]>(size_buffer);
model.rdbuf()->pubsetbuf(buffer.get(), size_buffer);
auto read = [&](auto &data, auto ...args){
auto read_impl = [&](auto &f, auto &data, auto ...args){
using T = std::remove_reference_t<decltype(data)>;
if constexpr(std::is_pointer_v<std::decay_t<T>>)
{
auto read_array = [&](auto &data, size_t size, auto ...args){
for(size_t i=0; i<size; ++i)
f(f, data[i], args...);
};
// use the array extent as the size
if constexpr(sizeof...(args)==0 && std::is_array_v<T>)
{
read_array(data, std::extent_v<T>);
}
else
{
static_assert(sizeof...(args), "size was not provided");
read_array(data, args...);
}
}
else
{
static_assert(std::is_standard_layout_v<T>);
model.read((char*)&data, sizeof(data));
}
};
read_impl(read_impl, data, args...);
};
char model_type[5] = {'\000'};
read(model_type, 4);
if(strcmp(model_type,"HNSW"))
throw std::runtime_error("Wrong type of model");
uint32_t version;
read(version);
if(version!=3)
throw std::runtime_error("Unsupported version");
size_t code_U, size_node;
read(code_U);
read(size_node);
fprintf(stderr, "U type loading %s\n", typeid(U).name());
// if((typeid(U).hash_code()^sizeof(U))!=code_U)
// throw std::runtime_error("Inconsistent type `U`");
// if(sizeof(node)!=size_node)
// throw std::runtime_error("Inconsistent type `node`");
// read parameter configuration
read(dim);
read(m_l);
read(m);
read(ef_construction);
read(alpha);
read(n);
puts("Configuration loaded");
printf("dim = %u\n", dim);
printf("m_l = %f\n", m_l);
printf("m = %u\n", m);
printf("efc = %u\n", ef_construction);
printf("alpha = %f\n", alpha);
printf("n = %u\n", n);
// read indices
// std::unordered_map<uint32_t,node*> addr;
node_pool.resize(n);
for(uint32_t i=0; i<n; ++i)
{
// auto *u = new node;
node &u = get_node(i);
read(u.level);
uint32_t id_u; // TODO: use generic type
read(id_u);
u.data = getter(id_u);
// addr[id_u] = u;
}
for(node &u : node_pool)
{
u.neighbors = new parlay::sequence<node_id>[u.level+1];
for(uint32_t l=0; l<=u.level; ++l)
{
size_t size;
read(size);
auto &nbh_u = u.neighbors[l];
nbh_u.reserve(size);
for(size_t i=0; i<size; ++i)
{
uint32_t id_v;
read(id_v);
nbh_u.push_back(id_v);
}
}
}
// read entrances
size_t size;
read(size);
entrance.reserve(size);
for(size_t i=0; i<size; ++i)
{
uint32_t id_u;
read(id_u);
entrance.push_back(id_u);
}
}
template<typename U, template<typename> class Allocator>
template<typename Iter>
HNSW<U,Allocator>::HNSW(Iter begin, Iter end, uint32_t dim_, float m_l_, uint32_t m_, uint32_t ef_construction_, float alpha_, float batch_base)
: dim(dim_), m_l(m_l_), m(m_), ef_construction(ef_construction_), alpha(alpha_), n(std::distance(begin,end))
{
static_assert(std::is_same_v<typename std::iterator_traits<Iter>::value_type, T>);
static_assert(std::is_base_of_v<
std::random_access_iterator_tag, typename std::iterator_traits<Iter>::iterator_category>);
if(n==0) return;
std::random_device rd;
auto perm = parlay::random_permutation<uint32_t>(n, rd());
auto rand_seq = parlay::delayed_seq<T>(n, [&](uint32_t i){
//return *(begin+perm[i]);
return *(begin+i);
});
const auto level_ep = get_level_random();
// node *entrance_init = allocator.allocate(1);
// node_pool.push_back(entrance_init);
node_pool.resize(1);
node_id entrance_init = 0;
new(&get_node(entrance_init)) node{
level_ep,
new parlay::sequence<node_id>[level_ep+1],
*rand_seq.begin()
/*anything else*/
};
entrance.push_back(entrance_init);
uint32_t batch_begin=0, batch_end=1, size_limit=n*0.02;
float progress = 0.0;
while(batch_end<n)
{
batch_begin = batch_end;
batch_end = std::min({n, (uint32_t)std::ceil(batch_begin*batch_base)+1, batch_begin+size_limit});
/*
if(batch_end>batch_begin+100)
batch_end = batch_begin+100;
*/
// batch_end = batch_begin+1;
insert(rand_seq.begin()+batch_begin, rand_seq.begin()+batch_end, true);
// insert(rand_seq.begin()+batch_begin, rand_seq.begin()+batch_end, false);
if(batch_end>n*(progress+0.05))
{
progress = float(batch_end)/n;
fprintf(stderr, "Built: %3.2f%%\n", progress*100);
// fprintf(stderr, "# visited: %lu\n", parlay::reduce(total_visited,parlay::addm<size_t>{}));
// fprintf(stderr, "# eval: %lu\n", parlay::reduce(total_eval,parlay::addm<size_t>{}));
// fprintf(stderr, "size of C: %lu\n", parlay::reduce(total_size_C,parlay::addm<size_t>{}));
}
}
// fprintf(stderr, "# visited: %lu\n", parlay::reduce(total_visited,parlay::addm<size_t>{}));
// fprintf(stderr, "# eval: %lu\n", parlay::reduce(total_eval,parlay::addm<size_t>{}));
// fprintf(stderr, "size of C: %lu\n", parlay::reduce(total_size_C,parlay::addm<size_t>{}));
fprintf(stderr, "Index built\n");
#if 0
for(const auto *pu : node_pool)
{
fprintf(stderr, "[%u] (%.2f,%.2f)\n", U::get_id(get_node(pu).data), get_node(pu).data[0], get_node(pu).data[1]);
for(int32_t l=pu->level; l>=0; --l)
{
fprintf(stderr, "\tlv. %d:", l);
for(const auto *k : pu->neighbors[l])
fprintf(stderr, " %u", U::get_id(get_node(k).data));
fputs("\n", stderr);
}
}
#endif
/*
for(uint32_t l=0; l<entrance[0]->level; ++l)
debug_output_graph(l);
*/
}
template<typename U, template<typename> class Allocator>
template<typename Iter>
void HNSW<U,Allocator>::insert(Iter begin, Iter end, bool from_blank)
{
const auto level_ep = get_node(entrance[0]).level;
const auto size_batch = std::distance(begin,end);
auto node_new = std::make_unique<node_id[]>(size_batch);
auto nbh_new = std::make_unique<parlay::sequence<node_id>[]>(size_batch);
auto eps = std::make_unique<parlay::sequence<node_id>[]>(size_batch);
//const float factor_m = from_blank? 0.5: 1;
const auto factor_m = 1;
debug_output("Insert %lu elements; from blank? [%c]\n", size_batch, "NY"[from_blank]);
// auto *pool = allocator.allocate(size_batch);
// first, query the nearest point as the starting point for each node to insert
if(from_blank)
{
auto offset = node_pool.size();
node_pool.resize(offset+size_batch);
parlay::parallel_for(0, size_batch, [&](uint32_t i){
const T &q = *(begin+i);
const auto level_u = get_level_random();
// auto *const pu = &pool[i]; // TODO: add pointer manager
node_id pu = offset+i;
new(&get_node(pu)) node{
level_u,
new parlay::sequence<node_id>[level_u+1],
q
};
node_new[i] = pu;
});
}
else
{
parlay::parallel_for(0, size_batch, [&](uint32_t i){
node_new[i] = node_pool.size()-size_batch+i;
});
}
debug_output("Nodes are settled\n");
// TODO: merge ops
parlay::parallel_for(0, size_batch, [&](uint32_t i){
auto &u = get_node(node_new[i]);
const auto level_u = u.level;
auto &eps_u = eps[i];
// eps_u.push_back(entrance);
eps_u = entrance;
for(uint32_t l=level_ep; l>level_u; --l)
{
const auto res = search_layer(u, eps_u, 1, l); // TODO: optimize
eps_u.clear();
eps_u.push_back(res[0].u);
}
});
debug_output("Finish searching entrances\n");
// then we process them layer by layer (from high to low)
for(int32_t l_c=level_ep; l_c>=0; --l_c) // TODO: fix the type
{
parlay::sequence<parlay::sequence<std::pair<node_id,node_id>>> edge_add(size_batch);
debug_output("Finding neighbors on lev. %d\n", l_c);
parlay::parallel_for(0, size_batch, [&](uint32_t i){
node_id pu = node_new[i];
auto &u = get_node(pu);
if((uint32_t)l_c>u.level) return;
auto &eps_u = eps[i]; // TODO: check
auto res = search_layer(u, eps_u, ef_construction, l_c);
auto neighbors_vec = select_neighbors(u.data, res, get_threshold_m(l_c)/**factor_m*/, l_c);
// move the content from `neighbors_vec` to `u.neighbors[l_c]`
// auto &nbh_u = nbh_new[i];
auto &edge_u = edge_add[i];
// nbh_u.clear();
edge_u.clear();
// nbh_u.reserve(neighbors_vec.size());
edge_u.reserve(neighbors_vec.size());
/*
for(uint32_t j=0; neighbors_vec.size()>0; ++j)
{
auto *pv = neighbors_vec.top().u;
neighbors_vec.pop();
// nbh_u[j] = pv;
// edge_u[j] = std::make_pair(pv, &u);
nbh_u.push_back(pv);
edge_u.emplace_back(pv, &u);
}
*/
for(node_id pv : neighbors_vec)
edge_u.emplace_back(pv, pu);
nbh_new[i] = std::move(neighbors_vec);
eps_u.clear();
/*
while(res.size()>0)
{
eps_u.push_back(res.top().u); // TODO: optimize
res.pop();
}
*/
eps_u.reserve(res.size());
for(const auto e : res)
eps_u.push_back(e.u);
});
debug_output("Adding forward edges\n");
parlay::parallel_for(0, size_batch, [&](uint32_t i){
auto &u = get_node(node_new[i]);
if((uint32_t)l_c<=u.level)
neighbourhood(u,l_c) = std::move(nbh_new[i]);
});
debug_output("Adding reverse edges\n");
// now we add edges in the other direction
auto edge_add_flatten = parlay::flatten(edge_add);
auto edge_add_grouped = parlay::group_by_key(edge_add_flatten);
parlay::parallel_for(0, edge_add_grouped.size(), [&](size_t j){
node_id pv = edge_add_grouped[j].first;
auto &nbh_v = neighbourhood(get_node(pv),l_c);
auto &nbh_v_add = edge_add_grouped[j].second;
// std::unordered_set<node_id> hash_table(nbh_v.begin(),nbh_v.end());
/*
for(auto it=nbh_v_add.begin(); it!=nbh_v_add.end();)
{
bool is_extant = *it==pv||std::find_if(nbh_v.begin(), nbh_v.end(), [&](const node_id pu_extant){
return *it==pu_extant;
})!=nbh_v.end();
// bool is_extant = hash_table.find(*it)!=hash_table.end();
it = is_extant? nbh_v_add.erase(it): std::next(it);
}
*/
const uint32_t size_nbh_total = nbh_v.size()+nbh_v_add.size();
const auto m_s = get_threshold_m(l_c)*factor_m;
if(size_nbh_total>m_s)
{
auto candidates = parlay::sequence<dist>(size_nbh_total);
for(size_t k=0; k<nbh_v.size(); ++k)
candidates[k] = dist{U::distance(get_node(nbh_v[k]).data,get_node(pv).data,dim), nbh_v[k]};
for(size_t k=0; k<nbh_v_add.size(); ++k)
candidates[k+nbh_v.size()] = dist{U::distance(get_node(nbh_v_add[k]).data,get_node(pv).data,dim), nbh_v_add[k]};
std::sort(candidates.begin(), candidates.end(), farthest());
nbh_v.resize(m_s);
for(size_t k=0; k<m_s; ++k)
nbh_v[k] = candidates[k].u;
/*
auto res = select_neighbors(get_node(pv).data, candidates, m_s, l_c);
nbh_v.clear();
for(auto *pu : res)
nbh_v.push_back(pu);
*/
// nbh_v = select_neighbors(get_node(pv).data, candidates, m_s, l_c);
}
else nbh_v.insert(nbh_v.end(),nbh_v_add.begin(), nbh_v_add.end());
});
}
debug_output("Updating entrance\n");
// finally, update the entrance
node_id node_highest = *std::max_element(
node_new.get(), node_new.get()+size_batch, [&](const node_id u, const node_id v){
return get_node(u).level < get_node(v).level;
});
if(get_node(node_highest).level>level_ep)
{
entrance.clear();
entrance.push_back(node_highest);
debug_output("New entrance [%u] at lev %u\n", U::get_id(get_node(node_highest).data), get_node(node_highest).level);
}
else if(get_node(node_highest).level==level_ep)
{
entrance.push_back(node_highest);
debug_output("New entrance [%u] at lev %u\n", U::get_id(get_node(node_highest).data), get_node(node_highest).level);
}
// and add new nodes to the pool
/*
if(from_blank)
node_pool.insert(node_pool.end(), node_new.get(), node_new.get()+size_batch);
*/
}
template<class Conn, class G, class D, class Seq>
auto beamSearch(
const G &g, D f_dist, const Seq &eps, uint32_t ef, const search_control &ctrl={})
{
using nid_t = typename G::nid_t;
using conn = Conn;
const auto n = g.num_nodes();
const uint32_t bits = ef>2? std::ceil(std::log2(ef))*2-2: 2;
const uint32_t mask = (1u<<bits)-1;
Seq visited(mask+1, n+1);
uint32_t cnt_visited = 0;
parlay::sequence<conn> workset;
std::set<conn> cand; // TODO: test dual heaps
std::unordered_set<nid_t> is_inw; // TODO: test merge instead
// TODO: get statistics about the merged size
// TODO: switch to the alternative if exceeding a threshold
workset.reserve(ef+1);
// debug_output("look at eps\n");
for(nid_t pe : eps)
{
visited[parlay::hash64(pe)&mask] = pe;
const auto d = f_dist(g.get_node(pe).data);
cand.insert({d,pe});
workset.push_back({d,pe});
is_inw.insert(pe);
cnt_visited++;
}
std::make_heap(workset.begin(), workset.end());
uint32_t cnt_eval = 0;
uint32_t limit_eval = ctrl.limit_eval.value_or(n);
while(cand.size()>0)
{
if(cand.begin()->d>workset[0].d*ctrl.beta) break;
if(++cnt_eval>limit_eval) break;
nid_t u = cand.begin()->u;
cand.erase(cand.begin());
for(nid_t pv: g.get_edges(u))
{
const auto h_pv = parlay::hash64_2(pv)&mask;
if(visited[h_pv]==pv) continue;
visited[h_pv] = pv;
cnt_visited++;
const auto d = f_dist(g.get_node(pv).data);
if(!(workset.size()<ef||d<workset[0].d)) continue;
if(!is_inw.insert(pv).second) continue;
cand.insert({d,pv});
workset.push_back({d,pv});
std::push_heap(workset.begin(), workset.end());
if(workset.size()>ef)
{
std::pop_heap(workset.begin(), workset.end());
// is_inw.erase(workset.back().u);
workset.pop_back();
}
if(cand.size()>ef)
cand.erase(std::prev(cand.end()));
}
}
if(ctrl.count_cmps)
*ctrl.count_cmps.value() += cnt_visited;
return workset;
}
template<typename U, template<typename> class Allocator>
auto HNSW<U,Allocator>::search_layer(const node &u, const parlay::sequence<node_id> &eps, uint32_t ef, uint32_t l_c, search_control ctrl) const
{
graph g(*this,l_c);
/*
dist_evaluator f_dist(u.data,dim);
return beamSearch<dist>(g, f_dist, eps, ef, ctrl);
*/
QueryParams QP(ef, ef, 1.35, ctrl.limit_eval.value_or(n), get_threshold_m(l_c));
auto points = parlay::delayed_seq<const T&>(node_pool.size(), [&](size_t i) -> const T&{
return node_pool[i].data;
});
auto res = beam_search_impl<node_id>(u.data, g, points, eps, QP);
const auto &pairElts = std::get<0>(res);
const auto &frontier = std::get<0>(pairElts);
if(ctrl.count_cmps)
*ctrl.count_cmps.value() += std::get<1>(res);
return parlay::tabulate(frontier.size(), [&](size_t i){
const auto &f = frontier[i];
return dist{f.second, f.first};
});
}
template<typename U, template<typename> class Allocator>
auto HNSW<U,Allocator>::search_layer_bak(const node &u, const parlay::sequence<node_id> &eps, uint32_t ef, uint32_t l_c, search_control ctrl) const
{
#define USE_HASHTBL
// #define USE_BOOLARRAY
// #define USE_UNORDERED_SET
#ifdef USE_HASHTBL
const uint32_t bits = ef>2? std::ceil(std::log2(ef*ef))-2: 2;
const uint32_t mask = (1u<<bits)-1;
parlay::sequence<uint32_t> visited(mask+1, n+1);
#endif
#ifdef USE_BOOLARRAY
std::vector<bool> visited(n+1);
#endif
// TODO: Try hash to an array
// TODO: monitor the size of `visited`
uint32_t cnt_visited = 0;
#ifdef USE_UNORDERED_SET
std::unordered_set<uint32_t> visited;
#endif
parlay::sequence<dist> W, discarded;
std::set<dist,farthest> C;
std::set<node_id> w_inserted;
W.reserve(ef+1);
for(node_id ep : eps)
{
#ifdef USE_HASHTBL
const auto id = U::get_id(get_node(ep).data);
visited[parlay::hash64_2(id)&mask] = id;
#endif
#ifdef USE_BOOLARRAY
visited[id] = true;
#endif
#ifdef USE_UNORDERED_SET
visited.insert(U::get_id(get_node(ep).data));
#endif
cnt_visited++;
const auto d = U::distance(u.data,get_node(ep).data,dim);
C.insert({d,ep});
W.push_back({d,ep});
w_inserted.insert(ep);
}
// std::make_heap(C.begin(), C.end(), nearest());
std::make_heap(W.begin(), W.end(), farthest());
uint32_t cnt_eval = 0;
uint32_t limit_eval = ctrl.limit_eval.value_or(n);
while(C.size()>0)
{
if(ctrl.skip_search) break;
if(C.begin()->d>W[0].d*ctrl.beta) break;
if(++cnt_eval>limit_eval) break;
if(ctrl.log_dist)
{
std::array<float,5> t;
if(ctrl.log_size)
{
t[0] = W[0].d;
t[1] = W.size();
t[2] = C.size();
vc_in_search[*ctrl.log_size].push_back(t);
}
auto it = C.begin();
const auto step = C.size()/4;
for(uint32_t i=0; i<4; ++i)
t[i]=it->d, std::advance(it,step);
t[4] = C.rbegin()->d;
dist_in_search[*ctrl.log_dist].push_back(t);
}
const auto &c = get_node(C.begin()->u);
// std::pop_heap(C.begin(), C.end(), nearest());
// C.pop_back();
C.erase(C.begin());
for(node_id pv: neighbourhood(c, l_c))
{
#ifdef USE_HASHTBL
const auto id = U::get_id(get_node(pv).data);
const auto idx = parlay::hash64_2(id)&mask;
if(visited[idx]==id) continue;
visited[idx] = id;
#endif
#ifdef USE_BOOLARRAY
if(visited[id]) continue;
visited[id] = true;
#endif
#ifdef USE_UNORDERED_SET
if(!visited.insert(U::get_id(get_node(pv).data)).second) continue;
#endif
cnt_visited++;
const auto d = U::distance(u.data,get_node(pv).data,dim);
if((W.size()<ef||d<W[0].d) && w_inserted.insert(pv).second)
{
C.insert({d,pv});
// C.push_back({d,pv,dc+1});
// std::push_heap(C.begin(), C.end(), nearest());
W.push_back({d,pv});
std::push_heap(W.begin(), W.end(), farthest());
if(W.size()>ef)
{
std::pop_heap(W.begin(), W.end(), farthest());
// w_inserted.erase(W.back().u);
if(ctrl.radius && W.back().d<=*ctrl.radius)
discarded.push_back(W.back());
W.pop_back();
}
if(C.size()>ef)
C.erase(std::prev(C.end()));
}
}
}
//total_visited += visited.size();
//total_visited += visited.size()-std::count(visited.begin(),visited.end(),n+1);
const auto id = parlay::worker_id();
total_visited[id] += cnt_visited;
total_size_C[id] += C.size()+cnt_eval;
total_eval[id] += cnt_eval;
if(ctrl.count_cmps)
*ctrl.count_cmps.value() += cnt_visited;
if(ctrl.radius)
{
const auto rad = *ctrl.radius;
auto split = std::partition(W.begin(), W.end(), [rad](const dist &e){
return e.d <= rad;
});
W.resize(split-W.begin());
W.append(discarded);
total_range_candidate[parlay::worker_id()] += W.size();
}
return W;
}
template<typename U, template<typename> class Allocator>
auto HNSW<U,Allocator>::search_layer_new_ex(const node &u, const parlay::sequence<node_id> &eps, uint32_t ef, uint32_t l_c, search_control ctrl) const
{
auto verbose_output = [&](const char *fmt, ...){
if(!ctrl.verbose_output) return;
va_list args;
va_start(args, fmt);
vfprintf(stderr, fmt, args);
va_end(args);
};
parlay::sequence<std::array<float,5>> dummy;
auto &dist_range = ctrl.log_dist? dist_in_search[*ctrl.log_dist]: dummy;
uint32_t cnt_eval = 0;
auto *indeg = ctrl.verbose_output? get_indeg(l_c): reinterpret_cast<const uint32_t*>(node_pool.data());
// parlay::sequence<bool> visited(n);
// TODO: Try hash to an array
// TODO: monitor the size of `visited`
std::set<uint32_t> visited;
// std::priority_queue<dist_ex,parlay::sequence<dist_ex>,nearest> C;
// std::priority_queue<dist_ex,parlay::sequence<dist_ex>,farthest> W;
parlay::sequence<dist_ex> /*C, W, */W_;
std::set<dist_ex,farthest> C, C_acc;
uint32_t cnt_used = 0;
for(node_id ep : eps)
{
// visited[U::get_id(get_node(ep).data)] = true;
const auto id = U::get_id(get_node(ep).data);
visited.insert(id);
const auto d = U::distance(u.data,get_node(ep).data,dim);
C.insert({d,ep,1});
C_acc.insert({d,ep,1});
// C.push_back({d,ep,1});
// W.push_back({d,ep,1});
verbose_output("Insert\t[%u](%f) initially\n", id, d);
}
// std::make_heap(C.begin(), C.end(), nearest());
// std::make_heap(W.begin(), W.end(), farthest());
// static thread_local std::mt19937 gen{parlay::worker_id()};
// static thread_local std::exponential_distribution<float> distro{48};
while(C.size()>0)
{
// const auto &f = *(W[0].u);
// if(U::distance(c.data,u.data,dim)>U::distance(f.data,u.data,dim))
// if(C[0].d>W[0].d) break;
if(C_acc.size()==cnt_used) break;
cnt_eval++;
if(ctrl.log_dist)
dist_range.push_back({C.begin()->d,C.rbegin()->d});
/*
const auto dc = C[0].depth;
const auto &c = *(C[0].u);
*/
auto it = C.begin();
/*
float quantile = distro(gen);
if(quantile>C.size())
quantile = C.size();
const auto dis_min = C.begin()->d;
const auto dis_max = C.rbegin()->d;
const auto threshold = quantile/C.size()*(dis_max-dis_min) + dis_min - 1e-6;
auto it = C.lower_bound(dist_ex{threshold,nullptr,0});
*/
const auto dc = it->depth;
const auto &c = *(it->u);
// W_.push_back(C[0]);
W_.push_back(*it);
// std::pop_heap(C.begin(), C.end(), nearest());
// C.pop_back();
C.erase(it);
cnt_used++;
verbose_output("------------------------------------\n");
const uint32_t id_c = U::get_id(c.data);
verbose_output("Eval\t[%u](%f){%u}\t[%u]\n", id_c, it->d, dc, indeg[id_c]);
uint32_t cnt_insert = 0;
for(node_id pv: neighbourhood(c, l_c))
{
// if(visited[U::get_id(get_node(pv).data)]) continue;
// visited[U::get_id(get_node(pv).data)] = true;
if(!visited.insert(U::get_id(get_node(pv).data)).second) continue;
// const auto &f = *(W[0].u);
// if(W.size()<ef||U::distance(get_node(pv).data,u.data,dim)<U::distance(f.data,u.data,dim))
const auto d = U::distance(u.data,get_node(pv).data,dim);
// if(W.size()<ef||d<W[0].d)
// if(C.size()<ef||d<C.rend()->d)
{
// C.push_back({d,pv,dc+1});
// std::push_heap(C.begin(), C.end(), nearest());
/*
W.push_back({d,pv,dc+1});
std::push_heap(W.begin(), W.end(), farthest());
if(W.size()>ef)
{
std::pop_heap(W.begin(), W.end(), farthest());
W.pop_back();
}
*/
if(C.size()<ef || d<C.rbegin()->d)
{
C.insert({d,pv,dc+1});
const uint32_t id_v = U::get_id(get_node(pv).data);
verbose_output("Insert\t[%u](%f){%u}\t[%u](%f)\n",
id_v, d, dc+1,
indeg[id_v], U::distance(c.data,get_node(pv).data,dim)
);
cnt_insert++;
if(C.size()>ef)
{
// std::pop_heap(C.begin(), C.end(), nearest());
// C.pop_back();
C.erase(std::prev(C.end()));
}
}
if(C_acc.size()<ef || d<C_acc.rbegin()->d)
{
C_acc.insert({d,pv,dc+1});
if(C_acc.size()>ef)
{
auto it = std::prev(C_acc.end());
if(std::find_if(W_.begin(), W_.end(), [&](const dist_ex &a){
return a.u==it->u;
})!=W_.end())
cnt_used--;
C_acc.erase(it);
}
}
}
}
verbose_output("%u inserts in this round\n", cnt_insert);
}
if(l_c==0)
{
const auto id = parlay::worker_id();
total_visited[id] += visited.size();
total_size_C[id] += C.size()+cnt_eval;
total_eval[id] += cnt_eval;
}
/*
std::sort(W.begin(), W.end(), farthest());
if(W.size()>ef) W.resize(ef);
*/
return W_;
}
template<typename U, template<typename> class Allocator>
auto HNSW<U,Allocator>::beam_search_ex(const node &u, const parlay::sequence<node_id> &eps, uint32_t beamSize, uint32_t l_c, search_control ctrl) const
// std::pair<parlay::sequence<dist_ex>, parlay::sequence<dist_ex>> beam_search(
// T* p_coords, int beamSize)
{
// beamSize *= 2;
// beamSize = 20000;
// initialize data structures
parlay::sequence<dist_ex> visited;
parlay::sequence<dist_ex> frontier;
auto dist_less = [&](const dist_ex &a, const dist_ex &b) {
return a.d < b.d || (a.d == b.d && a.u < b.u);
// return a.u<b.u;
};
auto dist_eq = [&](const dist_ex &a, const dist_ex &b){
return a.u == b.u;
};
// int bits = std::ceil(std::log2(beamSize * beamSize));
// parlay::sequence<uint32_t> hash_table(1 << bits, std::numeric_limits<uint32_t>::max());
std::set<uint32_t> accessed;
auto make_pid = [&] (node_id ep) {
const auto d = U::distance(u.data,get_node(ep).data,dim);
return dist_ex{d,ep,1};
};
// the frontier starts with the medoid
// frontier.push_back(make_pid(medoid->id));
for(node_id ep : eps)
frontier.push_back(make_pid(ep));
std::sort(frontier.begin(), frontier.end(), dist_less);
// frontier.push_back(make_pid(eps[0]));
parlay::sequence<dist_ex> unvisited_frontier;
// parlay::sequence<dist_ex> unvisited_frontier(beamSize);
parlay::sequence<dist_ex> new_frontier;
// parlay::sequence<dist_ex> new_frontier(2 * beamSize);
bool not_done = true;
for(size_t i=0; i<frontier.size(); ++i)
{
unvisited_frontier.push_back(frontier[i]);
// unvisited_frontier[i] = frontier[i];
accessed.insert(U::get_id(frontier[i].get_node(u).data));
}
// terminate beam search when the entire frontier has been visited
while (not_done) {
// the next node to visit is the unvisited frontier node that is closest
// to p
dist_ex currentPid = unvisited_frontier[0];
node_id current_vtx = currentPid.u;
debug_output("current_vtx ID: %u\n", U::get_id(get_node(current_vtx).data));
auto g = [&](node_id a) {
uint32_t id_a = U::get_id(get_node(a).data);
/*
uint32_t loc = parlay::hash64_2(id_a) & ((1 << bits) - 1);
if (hash_table[loc] == id_a) return false;
hash_table[loc] = id_a;
return true;
*/
return accessed.insert(id_a).second;
};
parlay::sequence<node_id> candidates;
auto f = [&](node_id pu, node_id pv/*, empty_weight wgh*/) {
if (g(pv)) {
candidates.push_back(pv);
}
return true;
};
for(node_id pv : neighbourhood(get_node(current_vtx),l_c))
// current_vtx.out_neighbors().foreach_cond(f);
f(current_vtx, pv);
debug_output("candidates:\n");
for(node_id p : candidates)
debug_output("%u ", U::get_id(get_node(p).data));
debug_output("\n");
auto pairCandidates =
parlay::map(candidates, make_pid);
/*
auto sortedCandidates =
parlay::unique(parlay::sort(pairCandidates, dist_less), dist_eq);
*/
auto &sortedCandidates = pairCandidates;
debug_output("size of sortedCandidates: %lu\n", sortedCandidates.size());
/*
auto f_iter = std::set_union(
frontier.begin(), frontier.end(), sortedCandidates.begin(),
sortedCandidates.end(), new_frontier.begin(), dist_less);\
*/
sortedCandidates.insert(sortedCandidates.end(), frontier);
new_frontier = parlay::unique(parlay::sort(sortedCandidates,dist_less), dist_eq);
// size_t f_size = std::min<size_t>(beamSize, f_iter - new_frontier.begin());
size_t f_size = std::min<size_t>(beamSize, new_frontier.size());
debug_output("f_size: %lu\n", f_size);
debug_output("frontier (size: %lu)\n", frontier.size());
for(const auto &e : frontier)
debug_output("%u ", U::get_id(e.get_node(u).data));
debug_output("\n");
frontier =
parlay::tabulate(f_size, [&](size_t i) { return new_frontier[i]; });
debug_output("size of frontier: %lu\n", frontier.size());
visited.insert(
std::upper_bound(visited.begin(), visited.end(), currentPid, dist_less),
currentPid);
debug_output("size of visited: %lu\n", visited.size());
unvisited_frontier.reserve(frontier.size());
auto uf_iter =
std::set_difference(frontier.begin(), frontier.end(), visited.begin(),
visited.end(), unvisited_frontier.begin(), dist_less);
debug_output("uf_iter - unvisited_frontier.begin(): %lu\n", uf_iter - unvisited_frontier.begin());
not_done = uf_iter > unvisited_frontier.begin();
if(l_c==0)
total_visited[parlay::worker_id()] += candidates.size();
}
parlay::sequence<dist_ex> W;
W.insert(W.end(), visited);
return W;
}
/*
template<typename U, template<typename> class Allocator>
parlay::sequence<std::pair<uint32_t,float>> HNSW<U,Allocator>::search(const T &q, uint32_t k, uint32_t ef, search_control ctrl)
{
auto res_ex = search_ex(q,k,ef,ctrl);
parlay::sequence<std::pair<uint32_t,float>> res;
res.reserve(res_ex.size());
for(const auto &e : res_ex)
res.emplace_back(std::get<0>(e), std::get<2>(e));
return res;
}
*/
template<typename U, template<typename> class Allocator>
parlay::sequence<typename HNSW<U,Allocator>::node_id> HNSW<U,Allocator>::search_layer_to(
const node &u, uint32_t ef, uint32_t l_stop, const search_control &ctrl)
{
auto eps = entrance;
for(uint32_t l_c=get_node(entrance[0]).level; l_c>l_stop; --l_c)
{
search_control c{};
c.log_per_stat = ctrl.log_per_stat; // whether count dist calculations at all layers
// c.limit_eval = ctrl.limit_eval; // whether apply the limit to all layers
c.count_cmps = ctrl.count_cmps;
const auto W = search_layer(u, eps, ef, l_c, c);
eps.clear();
eps.push_back(W[0].u);
/*
while(!W.empty())
{
eps.push_back(W.top().u);
W.pop();
}
*/
}
return eps;
}
template<typename U, template<typename> class Allocator>
parlay::sequence<std::pair<uint32_t,float>> HNSW<U,Allocator>::search(
const T &q, uint32_t k, uint32_t ef, const search_control &ctrl)
{
const auto id = parlay::worker_id();
total_range_candidate[id] = 0;
total_visited[id] = 0;
total_eval[id] = 0;
total_size_C[id] = 0;
node u{n, nullptr, q}; // To optimize
// std::priority_queue<dist,parlay::sequence<dist>,farthest> W;
parlay::sequence<node_id> eps;
if(ctrl.indicate_ep)
eps.push_back(*ctrl.indicate_ep);
else
eps = search_layer_to(u, 1, 0, ctrl);
auto W_ex = search_layer(u, eps, ef, 0, ctrl);
// auto W_ex = search_layer_new_ex(u, eps, ef, 0, ctrl);
// auto W_ex = beam_search_ex(u, eps, ef, 0);
// auto R = select_neighbors_simple(q, W_ex, k);
auto &R = W_ex;
if(!ctrl.radius && R.size()>k) // the range search ignores the given k
{
std::sort(R.begin(), R.end(), farthest());
if(k>0)
k = std::upper_bound(R.begin()+k, R.end(), R[k-1], farthest())-R.begin();
R.resize(k);
}
parlay::sequence<std::pair<uint32_t,float>> res;
res.reserve(R.size());
/*
while(W_ex.size()>0)
{
res.push_back({U::get_id(W_ex.top().get_node(u).data), W_ex.top().depth, W_ex.top().d});
W_ex.pop();
}
*/
for(const auto &e : R)
res.push_back({U::get_id(get_node(e.u).data),/* e.depth,*/ e.d});
return res;
}
template<typename U, template<typename> class Allocator>
void HNSW<U,Allocator>::save(const std::string &filename_model) const
{
std::ofstream model(filename_model, std::ios::binary);
if(!model.is_open())
throw std::runtime_error("Failed to create the model");
const auto size_buffer = 1024*1024*1024; // 1G
auto buffer = std::make_unique<char[]>(size_buffer);
model.rdbuf()->pubsetbuf(buffer.get(), size_buffer);
const auto write = [&](const auto &data, auto ...args){
auto write_impl = [&](auto &f, const auto &data, auto ...args){
using T = std::remove_reference_t<decltype(data)>;
if constexpr(std::is_pointer_v<std::decay_t<T>>)
{
auto write_array = [&](const auto &data, size_t size, auto ...args){
for(size_t i=0; i<size; ++i)
f(f, data[i], args...);
};
// use the array extent as the size
if constexpr(sizeof...(args)==0 && std::is_array_v<T>)
{
write_array(data, std::extent_v<T>);
}
else
{
static_assert(sizeof...(args), "size was not provided");
write_array(data, args...);
}
}
else
{
static_assert(std::is_standard_layout_v<T>);
model.write((const char*)&data, sizeof(data));
}
};
write_impl(write_impl, data, args...);
};
// write header (version number, type info, etc)
write("HNSW", 4);
write(uint32_t(3)); // version
write(typeid(U).hash_code()^sizeof(U));
fprintf(stderr, "U type written %s\n", typeid(U).name());
write(sizeof(node));
// write parameter configuration
write(dim);
write(m_l);
write(m);
write(ef_construction);
write(alpha);
write(n);
// write indices
for(const auto &u : node_pool)
{
write(u.level);
write(uint32_t(U::get_id(u.data)));
}
for(const auto &u : node_pool)
{
for(uint32_t l=0; l<=u.level; ++l)
{
write(u.neighbors[l].size());
for(node_id pv : u.neighbors[l])
write(pv);
}
}
// write entrances
write(entrance.size());
for(node_id pu : entrance)
write(pu);
}
} // namespace HNSW
#endif // _HNSW_HPP
================================================
FILE: algorithms/HNSW/debug.hpp
================================================
#ifndef __DEBUG_HPP__
#define __DEBUG_HPP__
extern parlay::sequence<parlay::sequence<std::array<float,5>>> dist_in_search;
extern parlay::sequence<parlay::sequence<std::array<float,5>>> vc_in_search;
// extern parlay::sequence<uint32_t> round_in_search;
extern parlay::sequence<size_t> per_visited;
extern parlay::sequence<size_t> per_eval;
extern parlay::sequence<size_t> per_size_C;
#include <optional>
struct search_control{
bool verbose_output;
bool skip_search;
float beta = 1;
std::optional<float> radius;
std::optional<uint32_t> log_per_stat;
std::optional<uint32_t> log_dist;
std::optional<uint32_t> log_size;
std::optional<uint32_t> indicate_ep;
std::optional<uint32_t> limit_eval;
std::optional<uint32_t*> count_cmps;
};
#endif // _DEBUG_HPP_
================================================
FILE: algorithms/HNSW/dist.hpp
================================================
#ifndef __DIST_HPP__
#define __DIST_HPP__
#include <type_traits>
#include "type_point.hpp"
#include "../utils/NSGDist.h"
template<typename T>
class descr_ang
{
using promoted_type = std::conditional_t<std::is_integral_v<T>&&sizeof(T)<=4,
std::conditional_t<sizeof(T)==4, int64_t, int32_t>,
float
>;
public:
typedef T type_elem;
typedef point<T> type_point;
static float distance(const type_point &u, const type_point &v, uint32_t dim)
{
const auto *uc=u.coord, *vc=v.coord;
promoted_type dot=0, nu=0, nv=0;
for(uint32_t i=0; i<dim; ++i)
{
nu += promoted_type(uc[i])*uc[i];
nv += promoted_type(vc[i])*vc[i];
dot += promoted_type(uc[i])*vc[i];
}
return 1-dot/(sqrtf(nu)*sqrtf(nv));
}
static auto get_id(const type_point &u)
{
return u.id;
}
};
template<typename T>
class descr_ndot
{
using promoted_type = std::conditional_t<std::is_integral_v<T>&&sizeof(T)<=4,
std::conditional_t<sizeof(T)==4, int64_t, int32_t>,
float
>;
public:
typedef T type_elem;
typedef point<T> type_point;
static float distance(const type_point &u, const type_point &v, uint32_t dim)
{
const auto *uc=u.coord, *vc=v.coord;
promoted_type dot=0;
for(uint32_t i=0; i<dim; ++i)
dot += promoted_type(uc[i])*vc[i];
return -float(dot);
}
static auto get_id(const type_point &u)
{
return u.id;
}
};
template<typename T>
class descr_l2
{
using promoted_type = std::conditional_t<std::is_integral_v<T>&&sizeof(T)<=4,
std::conditional_t<sizeof(T)==4, int64_t, int32_t>,
float
>;
public:
typedef T type_elem;
typedef point<T> type_point;
static float distance(const type_point &u, const type_point &v, uint32_t dim)
{
if constexpr(std::is_integral_v<T>)
{
const auto *uc=u.coord, *vc=v.coord;
promoted_type sum = 0;
for(uint32_t i=0; i<dim; ++i)
{
const auto d = promoted_type(uc[i])-vc[i];
sum += d*d;
}
return sum;
}
else
{
const auto *uc=u.coord, *vc=v.coord;
efanna2e::DistanceL2 distfunc;
return distfunc.compare(uc, vc, dim);
}
}
static auto get_id(const type_point &u)
{
return u.id;
}
};
#endif // __DIST_HPP__
================================================
FILE: algorithms/HNSW/h5_ops.hpp
================================================
#ifndef __H5_OPS_HPP__
#define __H5_OPS_HPP__
#include <cstdio>
#include <array>
#include <tuple>
#include <memory>
#include <type_traits>
#include <H5Cpp.h>
// Return a {reader, dims} tuple
// reader(buffer, index, cnt) reads data with the first dimension
// from `index` to `index+cnt` and writes it into the 1D `buffer`
template<typename T>
auto get_reader(const char *file, const char *dir)
{
H5::H5File file_h5(file, H5F_ACC_RDONLY);
H5::DataSet dset = file_h5.openDataSet(dir);
H5::DataSpace dspace_src = dset.getSpace();
hsize_t dim[2];
dspace_src.getSimpleExtentDims(dim);
fprintf(stderr, "%s: [%llu,%llu]\n", dir, dim[0], dim[1]);
H5::DataType type_dst = dset.getDataType();
if constexpr(std::is_same_v<T,uint32_t>)
type_dst = H5::PredType::NATIVE_UINT32;
else if constexpr(std::is_same_v<T,int32_t>)
type_dst = H5::PredType::NATIVE_INT32;
else if constexpr(std::is_same_v<T,uint8_t>)
type_dst = H5::PredType::NATIVE_UINT8;
else if constexpr(std::is_same_v<T,int8_t>)
type_dst = H5::PredType::NATIVE_INT8;
else if constexpr(std::is_same_v<T,float>)
type_dst = H5::PredType::NATIVE_FLOAT;
else static_assert(std::is_same_v<T,uint32_t>/*always false*/, "Unsupported type");
auto reader = [=,_=std::move(file_h5)](T *buffer, hsize_t index, hsize_t cnt=1){
hsize_t size = dim[1]*cnt;
H5::DataSpace dspace_dst(1,&size,NULL);
hsize_t offset[2] = {index, 0};
hsize_t count[2] = {cnt, dim[1]};
H5::DataSpace dspace_slice;
dspace_slice.copy(dspace_src);
dspace_slice.selectHyperslab(H5S_SELECT_SET, count, offset);
dset.read(buffer, type_dst, dspace_dst, dspace_slice);
};
return std::tuple{reader, std::array{dim[0], dim[1]}};
}
// read a 2D array from H5 file and return a 1D array
template<typename T>
std::pair<std::unique_ptr<T[]>,std::array<hsize_t,2>> read_array_from_HDF5(const char *file, const char *dir)
{
auto [reader,dims] = get_reader<T>(file, dir);
auto buffer = std::make_unique<T[]>(dims[0]*dims[1]);
reader(buffer.get(), 0, dims[0]);
return {std::move(buffer), dims};
}
#endif // __H5_OPS_HPP__
================================================
FILE: algorithms/HNSW/type_point.hpp
================================================
#ifndef __TYPE_POINT_HPP__
#define __TYPE_POINT_HPP__
#include <cstdint>
#include <cstddef>
#include <iterator>
#include <algorithm>
#include <memory>
#include <type_traits>
#include <stdexcept>
#include <any>
#include "benchUtils.h"
#ifdef SUPPORT_HDF5
#include "h5_ops.hpp"
#endif
class internal_termination{
protected:
internal_termination(){}
internal_termination(int){std::terminate();}
};
template<typename T>
class fake_copyable : public internal_termination{
T content;
public:
fake_copyable(const T &c) : content(c){}
fake_copyable(T &&c) : content(std::move(c)){}
fake_copyable(fake_copyable&&) = default;
fake_copyable(const fake_copyable &other [[maybe_unused]])
// The users have to guarantee to hold the points while it is being used in graph.
// Otherwise, uncomment the following guarding code and forbid copy constructions
// or alternatively pass in copy-constructible objects (e.g., `std::shared_ptr`)
// to `point` instead of using this hack
/*
: internal_termination(0),
content(std::move(const_cast<fake_copyable&>(other).content))
*/
: internal_termination()
{
}
};
template<typename T>
fake_copyable(const T&) -> fake_copyable<T>;
template<typename T>
fake_copyable(T&&) -> fake_copyable<T>;
template<typename T>
struct point
{
typedef T type;
uint32_t id;
const T *coord;
point()
: id(~0u), coord(NULL), closure()
{
}
point(uint32_t id_, const T *coord_)
: id(id_), coord(coord_), closure()
{
}
template<class C>
point(uint32_t id_, const T *coord_, C &&closure_)
: id(id_), coord(coord_), closure(std::forward<C>(closure_))
{
}
private:
std::any closure;
};
enum class file_format{
VEC, HDF5, BIN
};
template<typename T>
class point_converter_default
{
public:
using type = point<T>;
template<typename Iter>
type operator()(uint32_t id, Iter begin, [[maybe_unused]] Iter end)
{
using type_src = typename std::iterator_traits<Iter>::value_type;
static_assert(std::is_convertible_v<type_src,T>, "Cannot convert to the target type");
if constexpr(std::is_same_v<Iter,ptr_mapped<T,ptr_mapped_src::PERSISTENT>>||
std::is_same_v<Iter,ptr_mapped<const T,ptr_mapped_src::PERSISTENT>>)
return point<T>(id, &*begin);
else if constexpr(std::is_same_v<Iter,ptr_mapped<T,ptr_mapped_src::TRANSITIVE>>||
std::is_same_v<Iter,ptr_mapped<const T,ptr_mapped_src::TRANSITIVE>>)
{
const T *p = &*begin; // TODO: fix the type to T(*)[]
return point<T>(id, p, fake_copyable(std::unique_ptr<const T>(p)));
}
else
{
const uint32_t dim = std::distance(begin, end);
// T *coord = new T[dim];
auto coord = std::make_unique<T[]>(dim);
for(uint32_t i=0; i<dim; ++i)
coord[i] = *(begin+i);
return point<T>(id, coord.get(), fake_copyable(std::move(coord)));
}
}
};
template<typename Src, class Conv>
inline std::pair<parlay::sequence<typename Conv::type>,uint32_t>
load_from_vec(const char *file, Conv converter, uint32_t max_num)
{
const auto [fileptr, length] = mmapStringFromFile(file);
// Each vector is 4 + sizeof(Src)*dim bytes.
// * first 4 bytes encode the dimension (as an uint32_t)
// * next dim values are Src-type variables representing vector components
// See http://corpus-texmex.irisa.fr/ for more details.
const uint32_t dim = *((const uint32_t*)fileptr);
std::cout << "Dimension = " << dim << std::endl;
const size_t vector_size = sizeof(dim) + sizeof(Src)*dim;
const uint32_t n = std::min<size_t>(length/vector_size, max_num);
// std::cout << "Num vectors = " << n << std::endl;
typedef ptr_mapped<const Src,ptr_mapped_src::PERSISTENT> type_ptr;
parlay::sequence<typename Conv::type> ps(n);
parlay::parallel_for(0, n, [&,fp=fileptr] (size_t i) {
const Src *coord = (const Src*)(fp+sizeof(dim)+i*vector_size);
ps[i] = converter(i, type_ptr(coord), type_ptr(coord+dim));
});
return {std::move(ps), dim};
}
template<class, class=void>
class trait_type{
};
template<class T>
class trait_type<T,std::void_t<typename T::type>>{
public:
using type = typename T::type;
};
template<class T>
class trait_type<T*,void>{
public:
using type = T;
};
template<class T>
class trait_type<parlay::sequence<T>,void>{
public:
using type = T;
};
template<class Conv>
inline std::pair<parlay::sequence<typename Conv::type>,uint32_t>
load_from_HDF5(const char *file, const char *dir, Conv converter, uint32_t max_num)
{
#ifndef SUPPORT_HDF5
(void)file;
(void)dir;
(void)converter;
(void)max_num;
throw std::invalid_argument("HDF5 support is not enabled");
#else
using T = typename trait_type<typename Conv::type>::type;
auto [reader,bound] = get_reader<T>(file, dir);
const size_t n = std::min<size_t>(bound[0], max_num);
const uint32_t dim = bound[1];
parlay::sequence<typename Conv::type> ps(n);
// TODO: parallel for-loop
for(uint32_t i=0; i<n; ++i){
T *coord = new T[dim];
reader(coord, i);
typedef ptr_mapped<T,ptr_mapped_src::TRANSITIVE> type_ptr;
ps[i] = converter(i, type_ptr(coord), type_ptr(coord+dim));
}
return {std::move(ps), dim};
#endif
}
template<typename Src, class Conv>
inline std::pair<parlay::sequence<typename Conv::type>,uint32_t>
load_from_bin(const char *file, Conv converter, uint32_t max_num)
{
auto [fileptr, length] = mmapStringFromFile(file); (void)length;
const uint32_t n = std::min(max_num, *((uint32_t*)fileptr));
const uint32_t dim = *((uint32_t*)(fileptr+sizeof(n)));
const size_t vector_size = sizeof(Src)*dim;
const size_t header_size = sizeof(n)+sizeof(dim);
typedef ptr_mapped<const Src,ptr_mapped_src::PERSISTENT> type_ptr;
parlay::sequence<typename Conv::type> ps(n);
parlay::parallel_for(0, n, [&,fp=fileptr](uint32_t i){
const Src *coord = (const Src*)(fp+header_size+i*vector_size);
ps[i] = converter(i, type_ptr(coord), type_ptr(coord+dim));
});
return {std::move(ps), dim};
}
template<typename Src, class Conv>
inline std::pair<parlay::sequence<typename Conv::type>,uint32_t>
load_from_range(const char *file, Conv converter, uint32_t max_num)
{
auto [fileptr, length] = mmapStringFromFile(file); (void)length;
const int32_t num_points = *(int32_t*)fileptr;
const int32_t num_matches = *(int32_t*)(fileptr+sizeof(num_points));
const size_t header_size = sizeof(num_points)+sizeof(num_matches);
int32_t* begin = (int32_t*)(fileptr+header_size);
int32_t* end = begin + num_points;
auto [offsets, total] = parlay::scan(parlay::make_slice(begin,end));
offsets.push_back(total);
std::cout << "num_matches: " << num_matches << ' ' << total << std::endl;
const size_t index_size = header_size+num_points*sizeof(*begin);
std::cout << "index_size: " << index_size << std::endl;
typedef ptr_mapped<const Src,ptr_mapped_src::PERSISTENT> type_ptr;
const uint32_t n = std::min<uint32_t>(max_num, num_points);
parlay::sequence<typename Conv::type> ps(n);
parlay::parallel_for(0, n, [&,fp=fileptr](uint32_t i){
const Src *begin = (const Src*)(fp+index_size+offsets[i]*sizeof(Src));
const Src *end = (const Src*)(fp+index_size+offsets[i+1]*sizeof(Src));
ps[i] = converter(i, type_ptr(begin), type_ptr(end));
});
return {std::move(ps), 0};
}
/*
template<typename Src=void, class Conv>
inline auto load_point(const char *file, file_format input_format, Conv converter, size_t max_num=0, std::any aux={})
{
if(!max_num)
max_num = std::numeric_limits<decltype(max_num)>::max();
switch(input_format)
{
case file_format::VEC:
return load_from_vec<Src>(file, converter, max_num);
case file_format::HDF5:
return load_from_HDF5(file, std::any_cast<const char*>(aux), converter, max_num);
case file_format::BIN:
return load_from_bin<Src>(file, converter, max_num);
default:
__builtin_unreachable();
}
}
*/
template<class Conv>
inline auto load_point(const char *input_name, Conv converter, size_t max_num=0)
{
auto buffer = std::make_unique<char[]>(strlen(input_name)+1);
strcpy(buffer.get(), input_name);
char *splitter = strchr(buffer.get(), ':');
if(splitter==nullptr)
throw std::invalid_argument("The input spec is not specified");
*(splitter++) = '\0';
const char *file = buffer.get();
const char *input_spec = splitter;
if(!max_num)
max_num = std::numeric_limits<decltype(max_num)>::max();
if(input_spec[0]=='/')
return load_from_HDF5(file, input_spec, converter, max_num);
if(!strcmp(input_spec,"fvecs"))
return load_from_vec<float>(file, converter, max_num);
if(!strcmp(input_spec,"bvecs"))
return load_from_vec<uint8_t>(file, converter, max_num);
if(!strcmp(input_spec,"ivecs"))
return load_from_vec<int32_t>(file, converter, max_num);
if(!strcmp(input_spec,"u8bin"))
return load_from_bin<uint8_t>(file, converter, max_num);
if(!strcmp(input_spec,"i8bin"))
return load_from_bin<int8_t>(file, converter, max_num);
if(!strcmp(input_spec,"ibin"))
return load_from_bin<int32_t>(file, converter, max_num);
if(!strcmp(input_spec,"ubin"))
return load_from_bin<uint32_t>(file, converter, max_num);
if(!strcmp(input_spec,"fbin"))
return load_from_bin<float>(file, converter, max_num);
if(!strcmp(input_spec,"irange"))
return load_from_range<int32_t>(file, converter, max_num);
throw std::invalid_argument("Unsupported input spec");
}
#endif // __TYPE_POINT_HPP_
================================================
FILE: algorithms/bench/BUILD
================================================
package(default_visibility = ["//visibility:public"])
cc_library(
name = "parse_command_line",
hdrs = ["parse_command_line.h"],
)
================================================
FILE: algorithms/bench/IO.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#pragma once
#include <iostream>
#include <fstream>
#include <string>
#include <string>
#include <cstring>
#include "parlay/primitives.h"
#include "parlay/parallel.h"
#include "parlay/io.h"
#include "parlay/internal/get_time.h"
namespace benchIO {
using namespace std;
using parlay::sequence;
using parlay::tabulate;
using parlay::make_slice;
auto is_space = [] (char c) {
switch (c) {
case '\r':
case '\t':
case '\n':
case 0:
case ' ' : return true;
default : return false;
}
};
// parallel code for converting a string to word pointers
// side effects string by setting to null after each word
template <class Seq>
parlay::sequence<char*> stringToWords(Seq &Str) {
size_t n = Str.size();
parlay::parallel_for(0, n, [&] (long i) {
if (is_space(Str[i])) Str[i] = 0;});
// mark start of words
auto FL = parlay::tabulate(n, [&] (long i) -> bool {
return (i==0) ? Str[0] : Str[i] && !Str[i-1];});
// offset for each start of word
auto Offsets = parlay::pack_index<long>(FL);
// pointer to each start of word
auto SA = parlay::tabulate(Offsets.size(), [&] (long j) -> char* {
return Str.begin() + Offsets[j];});
return SA;
}
//using this as a typename so we can replace with parlay::chars easily if desired
using charstring = typename parlay::sequence<char>;
inline int xToStringLen(charstring const &a) { return a.size();}
inline void xToString(char* s, charstring const &a) {
for (int i=0; i < a.size(); i++) s[i] = a[i];}
inline int xToStringLen(long a) { return 21;}
inline void xToString(char* s, long a) { sprintf(s,"%ld",a);}
inline int xToStringLen(unsigned long a) { return 21;}
inline void xToString(char* s, unsigned long a) { sprintf(s,"%lu",a);}
inline uint xToStringLen(uint a) { return 12;}
inline void xToString(char* s, uint a) { sprintf(s,"%u",a);}
inline int xToStringLen(int a) { return 12;}
inline void xToString(char* s, int a) { sprintf(s,"%d",a);}
inline int xToStringLen(double a) { return 18;}
inline void xToString(char* s, double a) { sprintf(s,"%.11le", a);}
inline int xToStringLen(char* a) { return strlen(a)+1;}
inline void xToString(char* s, char* a) { sprintf(s,"%s",a);}
template <class A, class B>
inline int xToStringLen(pair<A,B> a) {
return xToStringLen(a.first) + xToStringLen(a.second) + 1;
}
template <class A, class B>
inline void xToString(char* s, pair<A,B> a) {
int l = xToStringLen(a.first);
xToString(s, a.first);
s[l] = ' ';
xToString(s+l+1, a.second);
}
template <class Seq>
charstring seqToString(Seq const &A) {
size_t n = A.size();
auto L = parlay::tabulate(n, [&] (size_t i) -> long {
typename Seq::value_type x = A[i];
return xToStringLen(x)+1;});
size_t m;
std::tie(L,m) = parlay::scan(std::move(L));
charstring B(m+1, (char) 0);
char* Bs = B.begin();
parlay::parallel_for(0, n-1, [&] (long i) {
xToString(Bs + L[i], A[i]);
Bs[L[i+1] - 1] = '\n';
});
xToString(Bs + L[n-1], A[n-1]);
Bs[m] = Bs[m-1] = '\n';
charstring C = parlay::filter(B, [&] (char c) {return c != 0;});
C[C.size()-1] = 0;
return C;
}
template <class T>
void writeSeqToStream(ofstream& os, parlay::sequence<T> const &A) {
size_t bsize = 10000000;
size_t offset = 0;
size_t n = A.size();
while (offset < n) {
// Generates a string for a sequence of size at most bsize
// and then wrties it to the output stream
charstring S = seqToString(A.cut(offset, min(offset + bsize, n)));
os.write(S.begin(), S.size()-1);
offset += bsize;
}
}
template <class T>
int writeSeqToFile(string header,
parlay::sequence<T> const &A,
char const *fileName) {
auto a = A[0];
//xToStringLena(a);
ofstream file (fileName, ios::out | ios::binary);
if (!file.is_open()) {
std::cout << "Unable to open file: " << fileName << std::endl;
return 1;
}
file << header << endl;
writeSeqToStream(file, A);
file.close();
return 0;
}
template <class T1, class T2>
int write2SeqToFile(string header,
parlay::sequence<T1> const &A,
parlay::sequence<T2> const &B,
char const *fileName) {
ofstream file (fileName, ios::out | ios::binary);
if (!file.is_open()) {
std::cout << "Unable to open file: " << fileName << std::endl;
return 1;
}
file << header << endl;
writeSeqToStream(file, A);
writeSeqToStream(file, B);
file.close();
return 0;
}
charstring readStringFromFile(char const *fileName) {
ifstream file (fileName, ios::in | ios::binary | ios::ate);
if (!file.is_open()) {
std::cout << "Unable to open file: " << fileName << std::endl;
abort();
}
long end = file.tellg();
file.seekg (0, ios::beg);
long n = end - file.tellg();
charstring bytes(n, (char) 0);
file.read (bytes.begin(), n);
file.close();
return bytes;
}
string intHeaderIO = "sequenceInt";
template <class T>
int writeIntSeqToFile(parlay::sequence<T> const &A, char const *fileName) {
return writeSeqToFile(intHeaderIO, A, fileName);
}
sequence<sequence<char>> get_tokens(char const *fileName) {
// parlay::internal::timer t("get_tokens");
// auto S = parlay::chars_from_file(fileName);
auto S = parlay::file_map(fileName);
// t.next("file map");
auto r = parlay::tokens(S, benchIO::is_space);
// t.next("tokens");
return r;
}
template <class T>
parlay::sequence<T> readIntSeqFromFile(char const *fileName) {
auto W = get_tokens(fileName);
string header(W[0].begin(),W[0].end());
if (header != intHeaderIO) {
cout << "readIntSeqFromFile: bad input" << endl;
abort();
}
long n = W.size()-1;
auto A = parlay::tabulate(n, [&] (long i) -> T {
return parlay::chars_to_long(W[i+1]);});
return A;
}
};
================================================
FILE: algorithms/bench/MakeBench
================================================
# ********************
# GENERIC MAKEFILE FOR MOST BENCHMARKS THAT #include <name>.h
# USES FOLLOWING DEFINITIONS
# BENCH : the name of the benchmark
# REQUIRE : dependences
# CC : the compiler
# CFLAGS : compiler flags
# LFLAGS : compiler link flags
# ********************
TIME = ../bench/$(BENCH)Time.C
INCLUDE = -I ../../parlaylib/include/
all : $(BENCH)
$(BENCH) : $(TIME) $(BENCH).h $(REQUIRE)
$(CC) -DSTATS $(CFLAGS) $(INCLUDE) -include $(BENCH).h -o $(BENCH) $(TIME) $(LFLAGS)
clean :
rm -f $(BENCH)
cleanall : clean
rm -f testInputs*; cd ../bench; make -s clean
================================================
FILE: algorithms/bench/Makefile
================================================
include parallelDefsANN
BNCHMRK = neighbors
CHECKFILES = $(BNCHMRK)Check.o
COMMON =
INCLUDE = -Icommon
%.o : %.C $(COMMON)
$(CC) $(CFLAGS) $(INCLUDE) -c $< -o $@
# $(BNCHMRK)Check : $(CHECKFILES)
# $(CC) $(LFLAGS) -o $@ $(CHECKFILES)
clean :
rm -f $(BNCHMRK)Check *.o *.pyc
================================================
FILE: algorithms/bench/benchUtils.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#ifndef __BENCHUTILS_H__
#define __BENCHUTILS_H__
#include <iostream>
#include <algorithm>
#include <iterator>
#include <type_traits>
#include "parlay/parallel.h"
#include "parlay/primitives.h"
#include "common/geometry.h"
#include "common/geometryIO.h"
#include "common/parse_command_line.h"
// #include "../utils/types.h"
// #include "common/time_loop.h"
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
using namespace benchIO;
enum class ptr_mapped_src{
NATIVE, VOLATILE, PERSISTENT, TRANSITIVE
};
namespace detail{
template<typename T, ptr_mapped_src Src>
class ptr_mapped_impl
{
T *ptr_raw;
public:
using difference_type = std::ptrdiff_t;
using value_type = std::remove_cv_t<T>;
using pointer = T*;
using reference = T&;
using iterator_category = std::random_access_iterator_tag;
ptr_mapped_impl(){
}
ptr_mapped_impl(T *p) : ptr_raw(p){
}
template<typename U, ptr_mapped_src SrcOther>
ptr_mapped_impl(const ptr_mapped_impl<U,SrcOther> &ptr) :
ptr_raw(ptr.get()){
static_assert(std::is_convertible_v<U*,T*>);
}
ptr_mapped_impl& operator=(T *p){
ptr_raw = p;
return *this;
}
template<typename U, ptr_mapped_src SrcOther>
ptr_mapped_impl& operator=(const ptr_mapped_impl<U,SrcOther> &ptr){
static_assert(std::is_convertible_v<U*,T*>);
ptr_raw = ptr.get();
}
T* get() const{
return ptr_raw;
}
operator T*() const{
return get();
}
// For simplicity, we only keep the least methods to satisfy the requirements of LegacyIterator
T& operator*() const{
return *get();
}
ptr_mapped_impl& operator++(){
++ptr_raw;
return *this;
}
ptr_mapped_impl& operator+=(size_t n){
ptr_raw += n;
return *this;
}
ptr_mapped_impl operator+(size_t n) const{
return ptr_raw+n;
}
ptr_mapped_impl& operator-=(size_t n){
ptr_raw -= n;
return *this;
}
ptr_mapped_impl operator-(size_t n) const{
return ptr_raw - n;
}
difference_type operator-(const ptr_mapped_impl &other) const{
return ptr_raw - other.ptr_raw;
}
reference operator[](size_t i) const{
return ptr_raw[i];
}
bool operator<(const ptr_mapped_impl &other) const{
return ptr_raw < other.ptr_raw;
}
bool operator>(const ptr_mapped_impl &other) const{
return other<*this;
}
bool operator>=(const ptr_mapped_impl &other) const{
return !(*this<other);
}
bool operator<=(const ptr_mapped_impl &other) const{
return !(*this>other);
}
};
} // namespace detail
template<typename T, ptr_mapped_src Src>
using ptr_mapped = std::conditional_t<Src==ptr_mapped_src::NATIVE, T*, detail::ptr_mapped_impl<T,Src>>;
/*
template<typename T, ptr_mapped_src Src>
struct std::iterator_traits<detail::ptr_mapped_impl<T,Src>>
{
using difference_type = std::ptrdiff_t;
using value_type = std::remove_cv_t<T>;
using pointer = T*;
using reference = T&;
using iterator_category = void;
};
*/
// *************************************************************
// SOME DEFINITIONS
// *************************************************************
// *************************************************************
// Parsing code (should move to common?)
// *************************************************************
// returns a pointer and a length
std::pair<char*, size_t> mmapStringFromFile(const char* filename) {
struct stat sb;
int fd = open(filename, O_RDONLY);
if (fd == -1) {
perror("open");
exit(-1);
}
if (fstat(fd, &sb) == -1) {
perror("fstat");
exit(-1);
}
if (!S_ISREG(sb.st_mode)) {
perror("not a file\n");
exit(-1);
}
char* p =
static_cast<char*>(mmap(0, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0));
if (p == MAP_FAILED) {
perror("mmap");
exit(-1);
}
if (close(fd) == -1) {
perror("close");
exit(-1);
}
size_t n = sb.st_size;
return std::make_pair(p, n);
}
/*
auto parse_fvecs(const char* filename)
{
return parse_vecs<float>(filename, [](size_t id, auto begin, auto end){
typedef typename std::iterator_traits<decltype(begin)>::value_type type_elem;
static_assert(std::is_same_v<decltype(begin),ptr_mapped<type_elem,ptr_mapped_src::DISK>>);
Tvec_point<type_elem> point;
point.id = id;
point.coordinates = parlay::make_slice(begin.get(), end.get());
return point;
}).first;
}
auto parse_ivecs(const char* filename)
{
return parse_vecs<float>(filename, [](size_t id, auto begin, auto end){
typedef typename std::iterator_traits<decltype(begin)>::value_type type_elem;
static_assert(std::is_same_v<decltype(begin),ptr_mapped<type_elem,ptr_mapped_src::DISK>>);
ivec_point point;
point.id = id;
point.coordinates = parlay::make_slice(begin.get(), end.get());
return point;
}).first;
}
*/
// auto parse_bvecs_to_fvecs(const char* filename) {
// using slice_f = typename parlay::slice<float*, float*>;
// auto [fileptr, length] = mmapStringFromFile(filename);
// // std::cout << "Successfully mmap'd" << std::endl;
// // Each vector is 4 + d bytes.
// // * first 4 bytes encode the dimension (as an integer)
// // * next d values are unsigned chars representing vector components
// // See http://corpus-texmex.irisa.fr/ for more details.
// int d = *((int*)fileptr);
// std::cout << "Dimension = " << d << std::endl;
// size_t vector_size = 4 + d;
// size_t num_vectors = length / vector_size;
// std::cout << "Num vectors = " << num_vectors << std::endl;
// parlay::sequence<fvec_point> points(num_vectors);
// // parlay::parallel_for(0, num_vectors, [&] (size_t i) {
// // size_t offset_in_bytes = vector_size * i + 4; // skip dimension
// // float* start = (float*)(fileptr + offset_in_bytes);
// // float* end = start + d;
// // points[i].id = i;
// // points[i].coordinates = parlay::make_slice(start, end);
// // });
// parlay::parallel_for(0, num_vectors, [&] (size_t i) {
// size_t offset_in_bytes = vector_size * i + 4; // skip dimension
// points[i].id = i;
// unsigned char* start = (unsigned char*)(fileptr + offset_in_bytes);
// parlay::sequence<float> coords = *new parlay::sequence<float>(d);
// for(int j=0; j<d; j++){
// float elt = *new float;
// elt = static_cast<float>(*(start+j));
// coords[j] = elt;
// // std::cout << coords[j] << std::endl;
// }
// slice_f slicecoords = *new slice_f(coords.begin(), coords.end());
// // slicecoords = parlay::make_slice(coords.begin(), coords.end());
// points[i].coordinates = slicecoords;
// });
// return points;
// }
#endif // __BENCHUTILS_H__
================================================
FILE: algorithms/bench/common/IO.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#pragma once
#include <iostream>
#include <fstream>
#include <string>
#include <string>
#include <cstring>
#include "../parlay/primitives.h"
#include "../parlay/parallel.h"
#include "../parlay/io.h"
#include "../parlay/internal/get_time.h"
namespace benchIO {
using namespace std;
using parlay::sequence;
using parlay::tabulate;
using parlay::make_slice;
auto is_space = [] (char c) {
switch (c) {
case '\r':
case '\t':
case '\n':
case 0:
case ' ' : return true;
default : return false;
}
};
// parallel code for converting a string to word pointers
// side effects string by setting to null after each word
template <class Seq>
parlay::sequence<char*> stringToWords(Seq &Str) {
size_t n = Str.size();
parlay::parallel_for(0, n, [&] (long i) {
if (is_space(Str[i])) Str[i] = 0;});
// mark start of words
auto FL = parlay::tabulate(n, [&] (long i) -> bool {
return (i==0) ? Str[0] : Str[i] && !Str[i-1];});
// offset for each start of word
auto Offsets = parlay::pack_index<long>(FL);
// pointer to each start of word
auto SA = parlay::tabulate(Offsets.size(), [&] (long j) -> char* {
return Str.begin() + Offsets[j];});
return SA;
}
//using this as a typename so we can replace with parlay::chars easily if desired
using charstring = typename parlay::sequence<char>;
inline int xToStringLen(charstring const &a) { return a.size();}
inline void xToString(char* s, charstring const &a) {
for (int i=0; i < a.size(); i++) s[i] = a[i];}
inline int xToStringLen(long a) { return 21;}
inline void xToString(char* s, long a) { sprintf(s,"%ld",a);}
inline int xToStringLen(unsigned long a) { return 21;}
inline void xToString(char* s, unsigned long a) { sprintf(s,"%lu",a);}
inline uint xToStringLen(uint a) { return 12;}
inline void xToString(char* s, uint a) { sprintf(s,"%u",a);}
inline int xToStringLen(int a) { return 12;}
inline void xToString(char* s, int a) { sprintf(s,"%d",a);}
inline int xToStringLen(double a) { return 18;}
inline void xToString(char* s, double a) { sprintf(s,"%.11le", a);}
inline int xToStringLen(char* a) { return strlen(a)+1;}
inline void xToString(char* s, char* a) { sprintf(s,"%s",a);}
template <class A, class B>
inline int xToStringLen(pair<A,B> a) {
return xToStringLen(a.first) + xToStringLen(a.second) + 1;
}
template <class A, class B>
inline void xToString(char* s, pair<A,B> a) {
int l = xToStringLen(a.first);
xToString(s, a.first);
s[l] = ' ';
xToString(s+l+1, a.second);
}
template <class Seq>
charstring seqToString(Seq const &A) {
size_t n = A.size();
auto L = parlay::tabulate(n, [&] (size_t i) -> long {
typename Seq::value_type x = A[i];
return xToStringLen(x)+1;});
size_t m;
std::tie(L,m) = parlay::scan(std::move(L));
charstring B(m+1, (char) 0);
char* Bs = B.begin();
parlay::parallel_for(0, n-1, [&] (long i) {
xToString(Bs + L[i], A[i]);
Bs[L[i+1] - 1] = '\n';
});
xToString(Bs + L[n-1], A[n-1]);
Bs[m] = Bs[m-1] = '\n';
charstring C = parlay::filter(B, [&] (char c) {return c != 0;});
C[C.size()-1] = 0;
return C;
}
template <class T>
void writeSeqToStream(ofstream& os, parlay::sequence<T> const &A) {
size_t bsize = 10000000;
size_t offset = 0;
size_t n = A.size();
while (offset < n) {
// Generates a string for a sequence of size at most bsize
// and then wrties it to the output stream
charstring S = seqToString(A.cut(offset, min(offset + bsize, n)));
os.write(S.begin(), S.size()-1);
offset += bsize;
}
}
template <class T>
int writeSeqToFile(string header,
parlay::sequence<T> const &A,
char const *fileName) {
auto a = A[0];
//xToStringLena(a);
ofstream file (fileName, ios::out | ios::binary);
if (!file.is_open()) {
std::cout << "Unable to open file: " << fileName << std::endl;
return 1;
}
file << header << endl;
writeSeqToStream(file, A);
file.close();
return 0;
}
template <class T1, class T2>
int write2SeqToFile(string header,
parlay::sequence<T1> const &A,
parlay::sequence<T2> const &B,
char const *fileName) {
ofstream file (fileName, ios::out | ios::binary);
if (!file.is_open()) {
std::cout << "Unable to open file: " << fileName << std::endl;
return 1;
}
file << header << endl;
writeSeqToStream(file, A);
writeSeqToStream(file, B);
file.close();
return 0;
}
charstring readStringFromFile(char const *fileName) {
ifstream file (fileName, ios::in | ios::binary | ios::ate);
if (!file.is_open()) {
std::cout << "Unable to open file: " << fileName << std::endl;
abort();
}
long end = file.tellg();
file.seekg (0, ios::beg);
long n = end - file.tellg();
charstring bytes(n, (char) 0);
file.read (bytes.begin(), n);
file.close();
return bytes;
}
string intHeaderIO = "sequenceInt";
template <class T>
int writeIntSeqToFile(parlay::sequence<T> const &A, char const *fileName) {
return writeSeqToFile(intHeaderIO, A, fileName);
}
sequence<sequence<char>> get_tokens(char const *fileName) {
// parlay::internal::timer t("get_tokens");
// auto S = parlay::chars_from_file(fileName);
auto S = parlay::file_map(fileName);
// t.next("file map");
auto r = parlay::tokens(S, benchIO::is_space);
// t.next("tokens");
return r;
}
template <class T>
parlay::sequence<T> readIntSeqFromFile(char const *fileName) {
auto W = get_tokens(fileName);
string header(W[0].begin(),W[0].end());
if (header != intHeaderIO) {
cout << "readIntSeqFromFile: bad input" << endl;
abort();
}
long n = W.size()-1;
auto A = parlay::tabulate(n, [&] (long i) -> T {
return parlay::chars_to_long(W[i+1]);});
return A;
}
};
================================================
FILE: algorithms/bench/common/MakeBench
================================================
# ********************
# GENERIC MAKEFILE FOR MOST BENCHMARKS THAT #include <name>.h
# USES FOLLOWING DEFINITIONS
# BENCH : the name of the benchmark
# REQUIRE : dependences
# CC : the compiler
# CFLAGS : compiler flags
# LFLAGS : compiler link flags
# ********************
TIME = ../bench/$(BENCH)Time.C
CHECK = $(BENCH)Check
INCLUDE =
all : $(BENCH) testInputs
cd ../bench; make -s $(CHECK)
$(BENCH) : $(TIME) $(BENCH).h $(REQUIRE)
$(CC) $(CFLAGS) $(INCLUDE) -include $(BENCH).h -o $(BENCH) $(TIME) $(LFLAGS)
testInputs : ../bench/testInputs ../bench/testInputs_small
cp ../bench/testInputs ../bench/testInputs_small .
clean :
rm -f $(BENCH)
cleanall : clean
rm -f testInputs*; cd ../bench; make -s clean
================================================
FILE: algorithms/bench/common/MakeBenchLink
================================================
# ********************
# GENERIC MAKEFILE FOR MOST BENCHMARKS THAT LINK
# THE TIMING CODE WITH THE IMPLEMENTATION
# USES FOLLOWING DEFINITIONS
# BENCH : the name of the benchmark
# OBJS : implementation object files
# REQUIRE : dependences for the object files
# CC : the compiler
# CFLAGS : compiler flags
# LFLAGS : compiler link flags
# ********************
TIME = $(BENCH)Time
CHECK = ../bench/$(BENCH)Check
INCLUDE =
# Make benchmark
$(BENCH) : $(TIME).o $(OBJS) $(CHECK) testInputs
$(CC) -o $@ $(TIME).o $(OBJS) $(LFLAGS)
# Timing Code
$(TIME).o : ../bench/$(TIME).C
$(CC) $(CFLAGS) $(INCLUDE) -o $@ -c ../bench/$(TIME).C
# The check code
$(CHECK) : $(CHECK).C
cd ../bench; make -s $(BENCH)Check
# object files
%.o : %.C $(REQUIRE)
$(CC) $(CFLAGS) $(INCLUDE) -c $< -o $@
# copy over the generic test code
testInputs : ../bench/testInputs ../bench/testInputs_small
cp ../bench/testInputs ../bench/testInputs_small .
clean :
rm -f $(BENCH) *.o
cleanall : clean
rm -f testInputs*; cd ../bench; make -s clean
================================================
FILE: algorithms/bench/common/atomics.h
================================================
#pragma once
namespace pbbs {
template <typename ET>
inline bool atomic_compare_and_swap(ET* a, ET oldval, ET newval) {
static_assert(sizeof(ET) <= 8, "Bad CAS length");
if (sizeof(ET) == 1) {
uint8_t r_oval, r_nval;
std::memcpy(&r_oval, &oldval, sizeof(ET));
std::memcpy(&r_nval, &newval, sizeof(ET));
return __sync_bool_compare_and_swap(reinterpret_cast<uint8_t*>(a), r_oval, r_nval);
} else if (sizeof(ET) == 4) {
uint32_t r_oval, r_nval;
std::memcpy(&r_oval, &oldval, sizeof(ET));
std::memcpy(&r_nval, &newval, sizeof(ET));
return __sync_bool_compare_and_swap(reinterpret_cast<uint32_t*>(a), r_oval, r_nval);
} else { // if (sizeof(ET) == 8) {
uint64_t r_oval, r_nval;
std::memcpy(&r_oval, &oldval, sizeof(ET));
std::memcpy(&r_nval, &newval, sizeof(ET));
return __sync_bool_compare_and_swap(reinterpret_cast<uint64_t*>(a), r_oval, r_nval);
}
}
template <typename E, typename EV>
inline E fetch_and_add(E *a, EV b) {
volatile E newV, oldV;
do {oldV = *a; newV = oldV + b;}
while (!atomic_compare_and_swap(a, oldV, newV));
return oldV;
}
template <typename E, typename EV>
inline void write_add(E *a, EV b) {
//volatile E newV, oldV;
E newV, oldV;
do {oldV = *a; newV = oldV + b;}
while (!atomic_compare_and_swap(a, oldV, newV));
}
template <typename E, typename EV>
inline void write_add(std::atomic<E> *a, EV b) {
//volatile E newV, oldV;
E newV, oldV;
do {oldV = a->load(); newV = oldV + b;}
while (!std::atomic_compare_exchange_strong(a, &oldV, newV));
}
template <typename ET, typename F>
inline bool write_min(ET *a, ET b, F less) {
ET c; bool r=0;
do c = *a;
while (less(b,c) && !(r=atomic_compare_and_swap(a,c,b)));
return r;
}
template <typename ET, typename F>
inline bool write_min(std::atomic<ET> *a, ET b, F less) {
ET c; bool r=0;
do c = a->load();
while (less(b,c) && !(r=std::atomic_compare_exchange_strong(a, &c, b)));
return r;
}
template <typename ET, typename F>
inline bool write_max(ET *a, ET b, F less) {
ET c; bool r=0;
do c = *a;
while (less(c,b) && !(r=atomic_compare_and_swap(a,c,b)));
return r;
}
template <typename ET, typename F>
inline bool write_max(std::atomic<ET> *a, ET b, F less) {
ET c; bool r=0;
do c = a->load();
while (less(c,b) && !(r=std::atomic_compare_exchange_strong(a, &c, b)));
return r;
}
}
================================================
FILE: algorithms/bench/common/dataGen.h
================================================
#pragma once
#include "../parlay/utilities.h"
namespace dataGen {
#define HASH_MAX_INT ((unsigned) 1 << 31)
//#define HASH_MAX_LONG ((unsigned long) 1 << 63)
template <class T> T hash(size_t i);
template <>
inline int hash<int>(size_t i) {
return parlay::hash64(i) & ((((size_t) 1) << 31) - 1);}
template <>
inline long hash<long>(size_t i) {
return parlay::hash64(i) & ((((size_t) 1) << 63) - 1);}
template <>
inline unsigned int hash<unsigned int>(size_t i) {
return parlay::hash64(i);}
template <>
inline size_t hash<size_t>(size_t i) {
return parlay::hash64(i);}
template <>
inline double hash<double>(size_t i) {
return ((double) hash<int>(i)/((double) ((((size_t) 1) << 31) - 1)));}
template <>
inline float hash<float>(size_t i) {
return ((double) hash<int>(i)/((double) ((((size_t) 1) << 31) - 1)));}
};
================================================
FILE: algorithms/bench/common/geometry.h
================================================
#pragma once
#include <iostream>
#include <algorithm>
#include <math.h>
#include <iomanip>
#include "../parlay/parallel.h"
#include "../parlay/primitives.h"
using namespace std;
// *************************************************************
// POINTS AND VECTORS (3d), 2d is below
// *************************************************************
template <class Coord>
class point3d;
template <class Coord>
class vector3d {
public:
using coord = Coord;
using vector = vector3d;
using point = point3d<coord>;
coord x;
coord y;
coord z;
vector3d(coord x, coord y, coord z) : x(x), y(y), z(z) {}
vector3d() :x(0), y(0), z(0) {}
vector3d(point p);
vector3d(parlay::slice<coord*,coord*> p) : x(p[0]), y(p[1]), z(p[2]) {};
vector operator+(vector op2) {
return vector(x + op2.x, y + op2.y, z + op2.z);}
vector operator-(vector op2) {
return vector(x - op2.x, y - op2.y, z - op2.z);}
point operator+(point op2);
vector operator*(coord s) {return vector(x * s, y * s, z * s);}
vector operator/(coord s) {return vector(x / s, y / s, z / s);}
coord& operator[] (int i) {return (i==0) ? x : (i==1) ? y : z;}
coord dot(vector v) {return x * v.x + y * v.y + z * v.z;}
vector cross(vector v) {
return vector(y*v.z - z*v.y, z*v.x - x*v.z, x*v.y - y*v.x);
}
coord maxDim() {return max(x,max(y,z));}
void print() {cout << std::setprecision(10) << ":(" << x << "," << y << "," << z << "):";}
coord Length(void) { return sqrt(x*x+y*y+z*z);}
coord sqLength(void) { return x*x+y*y+z*z;}
static const int dim = 3;
};
template <class Coord>
class point3d {
public:
using coord = Coord;
using vector = vector3d<coord>;
using point = point3d;
coord x; coord y; coord z;
int dimension() {return 3;}
point3d(coord x, coord y, coord z) : x(x), y(y), z(z) {}
point3d() : x(0), y(0), z(0) {}
point3d(vector v) : x(v.x), y(v.y), z(v.z) {};
point3d(parlay::slice<coord*,coord*> p) : x(p[0]), y(p[1]), z(p[2]) {};
void print() {cout << ":(" << x << "," << y << "," << z << "):";}
vector operator-(point op2) {
return vector(x - op2.x, y - op2.y, z - op2.z);}
point operator+(vector op2) {
return point(x + op2.x, y + op2.y, z + op2.z);}
point minCoords(point b) {
return point(min(x,b.x),min(y,b.y),min(z,b.z)); }
point maxCoords(point b) {
return point(max(x,b.x),max(y,b.y),max(z,b.z)); }
coord& operator[] (int i) {return (i==0) ? x : (i==1) ? y : z;}
int quadrant(point center) {
int index = 0;
if (x > center.x) index += 1;
if (y > center.y) index += 2;
if (z > center.z) index += 4;
return index;
}
// returns a point offset by offset in one of 8 directions
// depending on dir (an integer from [0..7])
point offsetPoint(int dir, coord offset) {
coord xx = x + ((dir & 1) ? offset : -offset);
coord yy = y + ((dir & 2) ? offset : -offset);
coord zz = z + ((dir & 4) ? offset : -offset);
return point(xx, yy, zz);
}
point changeCoords(std::vector<coord> v){
return point(v[0], v[1], v[2]);
}
// checks if pt is outside of a box centered at this point with
// radius hsize
bool outOfBox(point pt, coord hsize) {
return ((x - hsize > pt.x) || (x + hsize < pt.x) ||
(y - hsize > pt.y) || (y + hsize < pt.y) ||
(z - hsize > pt.z) || (z + hsize < pt.z));
}
static const int dim = 3;
};
template <class coord>
inline point3d<coord> vector3d<coord>::operator+(point3d<coord> op2) {
return point3d<coord>(x + op2.x, y + op2.y, z + op2.z);}
template <class coord>
inline vector3d<coord>::vector3d(point3d<coord> p) { x = p.x; y = p.y; z = p.z;}
// *************************************************************
// POINTS AND VECTORS (2d)
// *************************************************************
template <class Coord>
class point2d;
template <class Coord>
class vector2d {
public:
using coord = Coord;
using point = point2d<coord>;
using vector = vector2d;
coord x; coord y;
vector2d(coord x, coord y) : x(x), y(y) {}
vector2d() : x(0), y(0) {}
vector2d(point p);
vector2d(parlay::slice<coord*,coord*> p) : x(p[0]), y(p[1]) {};
vector operator+(vector op2) {return vector(x + op2.x, y + op2.y);}
vector operator-(vector op2) {return vector(x - op2.x, y - op2.y);}
point operator+(point op2);
vector operator*(coord s) {return vector(x * s, y * s);}
vector operator/(coord s) {return vector(x / s, y / s);}
coord operator[] (int i) {return (i==0) ? x : y;};
coord dot(vector v) {return x * v.x + y * v.y;}
coord cross(vector v) { return x*v.y - y*v.x; }
coord maxDim() {return max(x,y);}
void print() {cout << ":(" << x << "," << y << "):";}
coord Length(void) { return sqrt(x*x+y*y);}
coord sqLength(void) { return x*x+y*y;}
static const int dim = 2;
};
template <class coord>
static std::ostream& operator<<(std::ostream& os, const vector3d<coord> v) {
return os << v.x << " " << v.y << " " << v.z; }
template <class coord>
static std::ostream& operator<<(std::ostream& os, const point3d<coord> v) {
return os << v.x << " " << v.y << " " << v.z;
}
template <class Coord>
class point2d {
public:
using coord = Coord;
using vector = vector2d<coord>;
using point = point2d;
coord x; coord y;
int dimension() {return 2;}
point2d(coord x, coord y) : x(x), y(y) {}
point2d() : x(0), y(0) {}
point2d(vector v) : x(v.x), y(v.y) {};
point2d(parlay::slice<coord*,coord*> p) : x(p[0]), y(p[1]) {};
void print() {cout << ":(" << x << "," << y << "):";}
vector operator-(point op2) {return vector(x - op2.x, y - op2.y);}
point operator+(vector op2) {return point(x + op2.x, y + op2.y);}
coord operator[] (int i) {return (i==0) ? x : y;};
point minCoords(point b) { return point(min(x,b.x),min(y,b.y)); }
point maxCoords(point b) { return point(max(x,b.x),max(y,b.y)); }
int quadrant(point center) {
int index = 0;
if (x > center.x) index += 1;
if (y > center.y) index += 2;
return index;
}
// returns a point offset by offset in one of 4 directions
// depending on dir (an integer from [0..3])
point offsetPoint(int dir, coord offset) {
coord xx = x + ((dir & 1) ? offset : -offset);
coord yy = y + ((dir & 2) ? offset : -offset);
return point(xx,yy);
}
bool outOfBox(point pt, coord hsize) {
return ((x - hsize > pt.x) || (x + hsize < pt.x) ||
(y - hsize > pt.y) || (y + hsize < pt.y));
}
static const int dim = 2;
};
template <class coord>
inline point2d<coord> vector2d<coord>::operator+(point2d<coord> op2) {
return point2d<coord>(x + op2.x, y + op2.y);}
template <class coord>
inline vector2d<coord>::vector2d(point2d<coord> p) { x = p.x; y = p.y;}
template <class coord>
static std::ostream& operator<<(std::ostream& os, const vector2d<coord> v) {
return os << v.x << " " << v.y;}
template <class coord>
static std::ostream& operator<<(std::ostream& os, const point2d<coord> v) {
return os << v.x << " " << v.y; }
// *************************************************************
// GEOMETRY
// *************************************************************
// Returns twice the area of the oriented triangle (a, b, c)
template <class coord>
inline coord triArea(point2d<coord> a, point2d<coord> b, point2d<coord> c) {
return (b-a).cross(c-a);
}
template <class coord>
inline coord triAreaNormalized(point2d<coord> a, point2d<coord> b, point2d<coord> c) {
return triArea(a,b,c)/((b-a).Length()*(c-a).Length());
}
// Returns TRUE if the points a, b, c are in a counterclockise order
template <class coord>
inline bool counterClockwise(point2d<coord> a, point2d<coord> b, point2d<coord> c) {
return (b-a).cross(c-a) > 0.0;
}
template <class coord>
inline vector3d<coord> onParabola(vector2d<coord> v) {
return vector3d<coord>(v.x, v.y, v.x*v.x + v.y*v.y);}
// Returns TRUE if the point d is inside the circle defined by the
// points a, b, c.
// Projects a, b, c onto a parabola centered with d at the origin
// and does a plane side test (tet volume > 0 test)
template <class coord>
inline bool inCircle(point2d<coord> a, point2d<coord> b,
point2d<coord> c, point2d<coord> d) {
vector3d<coord> ad = onParabola(a-d);
vector3d<coord> bd = onParabola(b-d);
vector3d<coord> cd = onParabola(c-d);
return (ad.cross(bd)).dot(cd) > 0.0;
}
// returns a number between -1 and 1, such that -1 is out at infinity,
// positive numbers are on the inside, and 0 is at the boundary
template <class coord>
inline double inCircleNormalized(point2d<coord> a, point2d<coord> b,
point2d<coord> c, point2d<coord> d) {
vector3d<coord> ad = onParabola(a-d);
vector3d<coord> bd = onParabola(b-d);
vector3d<coord> cd = onParabola(c-d);
return (ad.cross(bd)).dot(cd)/(ad.Length()*bd.Length()*cd.Length());
}
// *************************************************************
// TRIANGLES
// *************************************************************
using tri = std::array<int,3>;
template <class point>
struct triangles {
size_t numPoints() {return P.size();};
size_t numTriangles() {return T.size();}
parlay::sequence<point> P;
parlay::sequence<tri> T;
triangles() {}
triangles(parlay::sequence<point> P, parlay::sequence<tri> T)
: P(std::move(P)), T(std::move(T)) {}
};
template <class point>
struct ray {
using vector = typename point::vector;
point o;
vector d;
ray(point _o, vector _d) : o(_o), d(_d) {}
ray() {}
};
template<class coord>
inline coord angle(point2d<coord> a, point2d<coord> b, point2d<coord> c) {
vector2d<coord> ba = (b-a);
vector2d<coord> ca = (c-a);
coord lba = ba.Length();
coord lca = ca.Length();
coord pi = 3.14159;
return 180/pi*acos(ba.dot(ca)/(lba*lca));
}
template<class coord>
inline coord minAngleCheck(point2d<coord> a, point2d<coord> b, point2d<coord> c, coord angle) {
vector2d<coord> ba = (b-a);
vector2d<coord> ca = (c-a);
vector2d<coord> cb = (c-b);
coord lba = ba.Length();
coord lca = ca.Length();
coord lcb = cb.Length();
coord pi = 3.14159;
coord co = cos(angle*pi/180.);
return (ba.dot(ca)/(lba*lca) > co || ca.dot(cb)/(lca*lcb) > co ||
-ba.dot(cb)/(lba*lcb) > co);
}
template<class coord>
inline point2d<coord> triangleCircumcenter(point2d<coord> a, point2d<coord> b, point2d<coord> c) {
vector2d<coord> v1 = b-a;
vector2d<coord> v2 = c-a;
vector2d<coord> v11 = v1 * v2.dot(v2);
vector2d<coord> v22 = v2 * v1.dot(v1);
return a + vector2d<coord>(v22.y - v11.y, v11.x - v22.x)/(2.0 * v1.cross(v2));
}
================================================
FILE: algorithms/bench/common/geometryIO.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#pragma once
#include "../parlay/parallel.h"
#include "../parlay/primitives.h"
#include "geometry.h"
#include "IO.h"
//using namespace geometry;
using namespace benchIO;
template <class coord>
inline int xToStringLen(point2d<coord> a) {
return xToStringLen(a.x) + xToStringLen(a.y) + 1;
}
template <class coord>
inline void xToString(char* s, point2d<coord> a) {
int l = xToStringLen(a.x);
xToString(s, a.x);
s[l] = ' ';
xToString(s+l+1, a.y);
}
template <class coord>
inline int xToStringLen(point3d<coord> a) {
return xToStringLen(a.x) + xToStringLen(a.y) + xToStringLen(a.z) + 2;
}
template <class coord>
inline void xToString(char* s, point3d<coord> a) {
int lx = xToStringLen(a.x);
int ly = xToStringLen(a.y);
xToString(s, a.x);
s[lx] = ' ';
xToString(s+lx+1, a.y);
s[lx+ly+1] = ' ';
xToString(s+lx+ly+2, a.z);
}
// inline int xToStringLen(tri a) {
// return xToStringLen(a[0]) + xToStringLen(a[1]) + xToStringLen(a[2]) + 2;
// }
// inline void xToString(char* s, tri a) {
// int lx = xToStringLen(a[0]);
// int ly = xToStringLen(a[1]);
// xToString(s, a[0]);
// s[lx] = ' ';
// xToString(s+lx+1, a[1]);
// s[lx+ly+1] = ' ';
// xToString(s+lx+ly+2, a[2]);
// }
namespace benchIO {
using namespace std;
string HeaderPoint2d = "pbbs_sequencePoint2d";
string HeaderPoint3d = "pbbs_sequencePoint3d";
string HeaderTriangles = "pbbs_triangles";
template <class Point>
int writePointsToFile(parlay::sequence<Point> const &P, char const *fname) {
string Header = (Point::dim == 2) ? HeaderPoint2d : HeaderPoint3d;
int r = writeSeqToFile(Header, P, fname);
return r;
}
template <class Point, class Seq>
parlay::sequence<Point> parsePoints(Seq W) {
using coord = typename Point::coord;
int d = Point::dim;
size_t n = W.size()/d;
auto a = parlay::tabulate(d * n, [&] (size_t i) -> coord {
return atof(W[i]);});
auto points = parlay::tabulate(n, [&] (size_t i) -> Point {
return Point(a.cut(d*i,d*(i + 1)));});
return points;
}
template <class Point>
parlay::sequence<Point> readPointsFromFile(char const *fname) {
parlay::sequence<char> S = readStringFromFile(fname);
parlay::sequence<char*> W = stringToWords(S);
int d = Point::dim;
if (W.size() == 0 || W[0] != (d == 2 ? HeaderPoint2d : HeaderPoint3d)) {
cout << "readPointsFromFile wrong file type" << endl;
abort();
}
return parsePoints<Point>(W.cut(1,W.size()));
}
// triangles<point2d> readTrianglesFromFileNodeEle(char const *fname) {
// string nfilename(fname);
// _seq<char> S = readStringFromFile((char*)nfilename.append(".node").c_str());
// words W = stringToWords(S.A, S.n);
// triangles<point2d> Tr;
// Tr.numPoints = atol(W.Strings[0]);
// if (W.m < 4*Tr.numPoints + 4) {
// cout << "readStringFromFileNodeEle inconsistent length" << endl;
// abort();
// }
// Tr.P = newA(point2d, Tr.numPoints);
// for(intT i=0; i < Tr.numPoints; i++)
// Tr.P[i] = point2d(atof(W.Strings[4*i+5]), atof(W.Strings[4*i+6]));
// string efilename(fname);
// _seq<char> SN = readStringFromFile((char*)efilename.append(".ele").c_str());
// words WE = stringToWords(SN.A, SN.n);
// Tr.numTriangles = atol(WE.Strings[0]);
// if (WE.m < 4*Tr.numTriangles + 3) {
// cout << "readStringFromFileNodeEle inconsistent length" << endl;
// abort();
// }
// Tr.T = newA(triangle, Tr.numTriangles);
// for (long i=0; i < Tr.numTriangles; i++)
// for (int j=0; j < 3; j++)
// Tr.T[i].C[j] = atol(WE.Strings[4*i + 4 + j]);
// return Tr;
// }
template <class pointT>
triangles<pointT> readTrianglesFromFile(char const *fname, int offset) {
int d = pointT::dim;
parlay::sequence<char> S = readStringFromFile(fname);
parlay::sequence<char*> W = stringToWords(S);
if (W[0] != HeaderTriangles) {
cout << "readTrianglesFromFile wrong file type" << endl;
abort();
}
int headerSize = 3;
size_t n = atol(W[1]);
size_t m = atol(W[2]);
if (W.size() != headerSize + 3 * m + d * n) {
cout << "readTrianglesFromFile inconsistent length" << endl;
abort();
}
auto pts_slice = W.cut(headerSize, headerSize + d * n);
auto tri_slice = W.cut(headerSize + d * n, W.size());
parlay::sequence<pointT> Pts = parsePoints<pointT>(pts_slice);
auto Tri = parlay::tabulate(m, [&] (size_t i ) -> tri {
return {(int) atol(tri_slice[3*i])-offset,
(int) atol(tri_slice[3*i+1])-offset,
(int) atol(tri_slice[3*i+2])-offset};});
return triangles<pointT>(Pts,Tri);
}
template <class pointT>
int writeTrianglesToFile(triangles<pointT> Tr, char* fileName) {
ofstream file (fileName, ios::binary);
if (!file.is_open()) {
std::cout << "Unable to open file: " << fileName << std::endl;
return 1;
}
file << HeaderTriangles << endl;
file << Tr.numPoints() << endl;
file << Tr.numTriangles() << endl;
writeSeqToStream(file, Tr.P);
//writeSeqToStream(file, Tr.T);
auto A = parlay::tabulate(3*Tr.numTriangles(), [&] (size_t i) -> int {
return (Tr.T[i/3])[i%3];});
writeSeqToStream(file, A);
file.close();
return 0;
}
};
================================================
FILE: algorithms/bench/common/get_time.h
================================================
#pragma once
#include <stdlib.h>
#include <sys/time.h>
#include <iomanip>
#include <iostream>
#include <string>
struct timer {
double total_time;
double last_time;
bool on;
std::string name;
struct timezone tzp;
timer(std::string name = "PBBS time", bool _start = true)
: total_time(0.0), on(false), name(name), tzp({0,0}) {
if (_start) start();
}
double get_time() {
timeval now;
gettimeofday(&now, &tzp);
return ((double) now.tv_sec) + ((double) now.tv_usec)/1000000.;
}
void start () {
on = 1;
last_time = get_time();
}
double stop () {
on = 0;
double d = (get_time()-last_time);
total_time += d;
return d;
}
void reset() {
total_time=0.0;
on=0;
}
double get_total() {
if (on) return total_time + get_time() - last_time;
else return total_time;
}
double get_next() {
if (!on) return 0.0;
double t = get_time();
double td = t - last_time;
total_time += td;
last_time = t;
return td;
}
void report(double time, std::string str) {
std::ios::fmtflags cout_settings = std::cout.flags();
std::cout.precision(4);
std::cout << std::fixed;
std::cout << name << ": ";
if (str.length() > 0)
std::cout << str << ": ";
std::cout << time << std::endl;
std::cout.flags(cout_settings);
}
void total() {
report(get_total(),"total");
total_time = 0.0;
}
void reportTotal(std::string str) {
report(get_total(), str);
}
void next(std::string str) {
if (on) report(get_next(), str);
}
};
static timer _tm;
#define startTime() _tm.start();
#define nextTime(_string) _tm.next(_string);
================================================
FILE: algorithms/bench/common/glue.h
================================================
#pragma once
#include "../pbbslib/hash_table.h"
#include "../pbbslib/integer_sort.h"
using intT = int;
using uintT = unsigned int;
#define newA(__E,__n) (__E*) malloc((__n)*sizeof(__E))
namespace utils {
static void myAssert(int cond, std::string s) {
if (!cond) {
std::cout << s << std::endl;
abort();
}
}
inline unsigned int hash(unsigned int a)
{
a = (a+0x7ed55d16) + (a<<12);
a = (a^0xc761c23c) ^ (a>>19);
a = (a+0x165667b1) + (a<<5);
a = (a+0xd3a2646c) ^ (a<<9);
a = (a+0xfd7046c5) + (a<<3);
a = (a^0xb55a4f09) ^ (a>>16);
return a;
}
inline int hashInt(unsigned int a) {
return hash(a) & (((unsigned) 1 << 31) - 1);
}
// template <class E>
// struct identityF { E operator() (const E& x) {return x;}};
// template <class E>
// struct addF { E operator() (const E& a, const E& b) const {return a+b;}};
// template <class E>
// struct absF { E operator() (const E& a) const {return std::abs(a);}};
// template <class E>
// struct zeroF { E operator() (const E& a) const {return 0;}};
// template <class E>
// struct maxF { E operator() (const E& a, const E& b) const {return (a>b) ? a : b;}};
// template <class E>
// struct minF { E operator() (const E& a, const E& b) const {return (a<b) ? a : b;}};
// template <class E1, class E2>
// struct firstF {E1 operator() (std::pair<E1,E2> a) {return a.first;} };
// template <class E1, class E2>
// struct secondF {E2 operator() (std::pair<E1,E2> a) {return a.second;} };
}
// template <class T>
// struct _seq {
// T* A;
// long n;
// _seq() {A = NULL; n=0;}
// _seq(T* _A, long _n) : A(_A), n(_n) {}
// void del() {free(A);}
// };
// namespace osequence {
// template <class ET, class intT, class F>
// ET scan(ET *In, ET* Out, intT n, F f, ET zero) {
// if (In == Out)
// return pbbs::scan_inplace(pbbs::range<ET*>(In,In+n),pbbs::make_monoid(f,zero));
// else {
// std::cout << "NYI in scan" << std::endl;
// return zero;
// }
// }
// template <class ET>
// ET plusScan(ET *In, ET* Out, size_t n) {
// return scan(In, Out, n, [&] (ET a, ET b) {return a + b;}, (ET) 0);
// }
// template <class ET, class PRED>
// size_t filter(ET* In, ET* Out, size_t n, PRED p) {
// pbbs::sequence<ET> r = pbbs::filter(pbbs::range<ET*>(In,In+n), p);
// parallel_for(0, r.size(), [&] (size_t i) {Out[i] = r[i];});
// return r.size();
// }
// };
namespace dataGen {
using namespace std;
#define HASH_MAX_INT ((unsigned) 1 << 31)
//#define HASH_MAX_LONG ((unsigned long) 1 << 63)
template <class T> T hash(intT i);
template <>
inline intT hash<intT>(intT i) {
return utils::hash(i) & (HASH_MAX_INT-1);}
template <>
inline uintT hash<uintT>(intT i) {
return utils::hash(i);}
template <>
inline double hash<double>(intT i) {
return ((double) hash<intT>(i)/((double) HASH_MAX_INT));}
};
// template <class HASH, class ET>
// _seq<ET> removeDuplicates(_seq<ET> S, HASH hashF) {
// return pbbs::remove_duplicates(pbbs::range<ET*>(S.A, S.A+S.n), hashF);
// }
================================================
FILE: algorithms/bench/common/graph.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#pragma once
#include <iostream>
#include <algorithm>
#include "../parlay/parallel.h"
#include "../parlay/primitives.h"
// IntV and IntE should be set depending on the size of the graphs
// intV should have enough range to represent |V|
// intE should have enough range to represent |E|
// intE defaults to intV if not specified
using DefaultIntV = int;
using DefaultWeight = float;
// **************************************************************
// EDGE ARRAY REPRESENTATION
// **************************************************************
template <class intV = DefaultIntV>
struct edge {
intV u;
intV v;
edge() {}
edge(intV f, intV s) : u(f), v(s) {}
};
template <class intV = DefaultIntV>
struct edgeArray {
parlay::sequence<edge<intV>> E;
size_t numRows;
size_t numCols;
size_t nonZeros;
edgeArray(parlay::sequence<edge<intV>> EE, size_t r, size_t c) :
E(std::move(EE)), numRows(r), numCols(c), nonZeros(E.size()) {}
edgeArray() {}
edge<intV> operator[] (const size_t i) const {return E[i];}
};
// **************************************************************
// WEIGHED EDGE ARRAY
// **************************************************************
template <class intV = DefaultIntV, class Weight=DefaultWeight>
struct wghEdge {
intV u, v;
Weight weight;
wghEdge() {}
wghEdge(intV _u, intV _v, Weight w) : u(_u), v(_v), weight(w) {}
};
template <class intV = DefaultIntV, class Weight=DefaultWeight>
struct wghEdgeArray {
using W = Weight;
parlay::sequence<wghEdge<intV,W>> E;
size_t n; size_t m;
wghEdgeArray(parlay::sequence<wghEdge<intV,W>> E_, intV n)
: E(std::move(E_)), n(n), m(E.size()) {}
wghEdgeArray() {}
wghEdge<intV> operator[] (const size_t i) const {return E[i];}
};
// **************************************************************
// ADJACENCY ARRAY REPRESENTATION
// **************************************************************
template <class intV = DefaultIntV>
struct vertex {
const intV* Neighbors;
intV degree;
vertex(const intV* N, const intV d) : Neighbors(N), degree(d) {}
vertex() : Neighbors(NULL), degree(0) {}
};
template <class intV = DefaultIntV>
struct mod_vertex {
intV* Neighbors;
intV degree;
mod_vertex(intV* N, intV d) : Neighbors(N), degree(d) {}
mod_vertex() : Neighbors(NULL), degree(0) {}
};
template <class intV = DefaultIntV, class intE = intV>
struct graph {
using vertexId = intV;
using edgeId = intE;
using MVT = mod_vertex<intV>;
using VT = vertex<intV>;
parlay::sequence<intE> offsets;
parlay::sequence<intV> edges;
parlay::sequence<intV> degrees; // not always used
size_t n;
size_t m;
size_t numVertices() const {return n;}
size_t numEdges() const {
if (degrees.size() == 0) return m;
else {
std::cout << "hello numEdges" << std::endl;
auto dgs = parlay::delayed_seq<intE>(n, [&] (size_t i) {
return degrees[i];});
return parlay::reduce(dgs, parlay::addm<intE>());
}
}
const parlay::sequence<intE>& get_offsets() const {
return offsets;
}
void addDegrees() {
degrees = parlay::tabulate(n, [&] (size_t i) -> intV {
return offsets[i+1] - offsets[i];});
}
MVT operator[] (const size_t i) {
return MVT(edges.data() + offsets[i],
(degrees.size() == 0)
? offsets[i+1] - offsets[i] : degrees[i]);}
const VT operator[] (const size_t i) const {
return VT(edges.data() + offsets[i],
(degrees.size() == 0)
? offsets[i+1] - offsets[i] : degrees[i]);
}
graph(parlay::sequence<intE> offsets_,
parlay::sequence<intV> edges_,
size_t n)
: offsets(std::move(offsets_)), edges(std::move(edges_)), n(n), m(edges.size()) {
if (offsets.size() != n + 1) { std::cout << "error in graph constructor" << std::endl;}
}
};
// **************************************************************
// WEIGHTED ADJACENCY ARRAY REPRESENTATION
// **************************************************************
template <class intV = DefaultIntV, class Weight = DefaultWeight>
struct wghVertex {
intV* Neighbors;
intV degree;
Weight* nghWeights;
wghVertex(intV* N, Weight* W, intV d) : Neighbors(N), nghWeights(W), degree(d) {}
};
template <class intV = DefaultIntV, class Weight=DefaultWeight,
class intE = intV>
struct wghGraph {
using VT = wghVertex<intV,Weight>;
using W = Weight;
parlay::sequence<intE> offsets;
parlay::sequence<intV> edges;
parlay::sequence<Weight> weights;
size_t n;
size_t m;
size_t numVertices() const {return n;}
size_t numEdges() const {return m;}
//const parlay::sequence<intV>& get_offsets() const {
// return offsets;
//}
parlay::sequence<intV> get_offsets() {
return offsets;
}
VT operator[] (const size_t i) {
return VT(edges.begin() + offsets[i],
weights.begin() + offsets[i],
offsets[i+1] - offsets[i]);}
wghGraph(parlay::sequence<intE> offsets_,
parlay::sequence<intV> edges_,
parlay::sequence<Weight> weights_,
size_t n)
: offsets(std::move(offsets_)), edges(std::move(edges_)),
weights(std::move(weights_)), n(n), m(edges.size()) {
if (offsets.size() != n + 1 || weights.size() != edges.size()) {
std::cout << "error in weighted graph constructor" << std::endl;}
}
};
template <typename intV>
struct FlowGraph {
wghGraph<intV> g;
intV source, sink;
FlowGraph(wghGraph<intV> g, intV source, intV sink)
: g(g), source(source), sink(sink) {}
FlowGraph copy() {
return FlowGraph(g.copy(), source, sink);
}
void del() { g.del(); }
};
================================================
FILE: algorithms/bench/common/graphIO.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2010 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#pragma once
#include <iostream>
#include <stdint.h>
#include <cstring>
#include "../parlay/parallel.h"
#include "IO.h"
#include "graphUtils.h"
#include <sys/mman.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
using namespace benchIO;
template <class intV>
int xToStringLen(edge<intV> a) {
return xToStringLen(a.u) + xToStringLen(a.v) + 1;
}
template <class intV>
void xToString(char* s, edge<intV> a) {
int l = xToStringLen(a.u);
xToString(s, a.u);
s[l] = ' ';
xToString(s+l+1, a.v);
}
template <class intV, class Weight>
int xToStringLen(wghEdge<intV,Weight> a) {
return xToStringLen(a.u) + xToStringLen(a.v) + xToStringLen(a.weight) + 2;
}
template <class intV, class Weight>
void xToString(char* s, wghEdge<intV, Weight> a) {
int lu = xToStringLen(a.u);
int lv = xToStringLen(a.v);
xToString(s, a.u);
s[lu] = ' ';
xToString(s+lu+1, a.v);
s[lu+lv+1] = ' ';
xToString(s+lu+lv+2, a.weight);
}
namespace benchIO {
using namespace std;
string AdjGraphHeader = "AdjacencyGraph";
string EdgeArrayHeader = "EdgeArray";
string WghEdgeArrayHeader = "WeightedEdgeArray";
string WghAdjGraphHeader = "WeightedAdjacencyGraph";
template <class intV, class intE>
int writeGraphToFile(graph<intV, intE> const &G, char* fname) {
if (G.degrees.size() > 0) {
graph<intV, intE> GP = packGraph(G);
return writeGraphToFile(GP, fname);
}
size_t m = G.numEdges();
size_t n = G.numVertices();
size_t totalLen = 2 + n + m;
parlay::sequence<size_t> Out(totalLen);
Out[0] = n;
Out[1] = m;
// write offsets to Out[2,..,2+n)
parlay::sequence<intE> const &offsets = G.get_offsets();
parlay::parallel_for (0, n, [&] (size_t i) {
Out[i+2] = offsets[i];});
// write out edges to Out[2+n,..,2+n+m)
parlay::parallel_for(0, n, [&] (size_t i) {
size_t o = offsets[i] + 2 + n;
for (intV j = 0; j < G[i].degree; j++)
Out[o + j] = G[i].Neighbors[j];
});
int r = writeSeqToFile(AdjGraphHeader, Out, fname);
return r;
}
template <class intV, class Weight, class intE>
int writeWghGraphToFile(wghGraph<intV,Weight,intE> G, char* fname) {
size_t m = G.m;
size_t n = G.n;
// weights have to separate since they could be floats
parlay::sequence<size_t> Out1(2 + n + m);
parlay::sequence<Weight> Out2(m);
Out1[0] = n;
Out2[1] = m;
// write offsets to Out[2,..,2+n)
auto offsets = G.get_offsets();
parlay::parallel_for (0, n, [&] (size_t i) {
Out1[i+2] = offsets[i];});
// write out edges to Out1[2+n,..,2+n+m)
// and weights to Out2[0,..,m)
parlay::parallel_for(0, n, [&] (size_t i) {
size_t o = offsets[i];
wghVertex<intV,Weight> v = G[i];
for (intV j = 0; j < v.degree; j++) {
Out1[2 + n + o + j] = v.Neighbors[j];
Out2[o + j] = v.nghWeights[j]; }
});
int r = write2SeqToFile(WghAdjGraphHeader, Out1, Out2, fname);
return r;
}
template <class intV>
int writeEdgeArrayToFile(edgeArray<intV> const &EA, char* fname) {
return writeSeqToFile(EdgeArrayHeader, EA.E, fname);
}
template <class intV, class intE>
int writeWghEdgeArrayToFile(wghEdgeArray<intV,intE>
const &EA, char* fname) {
return writeSeqToFile(WghEdgeArrayHeader, EA.E, fname);
}
template <class intV>
edgeArray<intV> readEdgeArrayFromFile(char* fname) {
parlay::sequence<char> S = readStringFromFile(fname);
parlay::sequence<char*> W = stringToWords(S);
if (W[0] != EdgeArrayHeader) {
cout << "Bad input file" << endl;
abort();
}
long n = (W.size()-1)/2;
auto E = parlay::tabulate(n, [&] (long i) -> edge<intV> {
return edge<intV>(atol(W[2*i + 1]),
atol(W[2*i + 2]));});
auto mon = parlay::make_monoid([&] (edge<intV> a, edge<intV> b) {
return edge<intV>(std::max(a.u, b.u), std::max(a.v, b.v));},
edge<intV>(0,0));
auto r = parlay::reduce(E, mon);
intV maxrc = std::max(r.u, r.v) + 1;
return edgeArray<intV>(std::move(E), maxrc, maxrc);
}
template <class intV, class Weight>
wghEdgeArray<intV,Weight> readWghEdgeArrayFromFile(char* fname) {
using WE = wghEdge<intV,Weight>;
parlay::sequence<char> S = readStringFromFile(fname);
parlay::sequence<char*> W = stringToWords(S);
if (W[0] != WghEdgeArrayHeader) {
cout << "Bad input file" << endl;
abort();
}
long n = (W.size()-1)/3;
auto E = parlay::tabulate(n, [&] (size_t i) -> WE {
return WE(atol(W[3*i + 1]),
atol(W[3*i + 2]),
(Weight) atof(W[3*i + 3]));});
auto mon = parlay::make_monoid([&] (WE a, WE b) {
return WE(std::max(a.u, b.u), std::max(a.v, b.v), 0);},
WE(0,0,0));
auto r = parlay::reduce(E, mon);
return wghEdgeArray<intV,Weight>(std::move(E), max<intV>(r.u, r.v) + 1);
}
template <class intV, class intE=intV>
graph<intV, intE> readGraphFromFile(char* fname) {
auto W = get_tokens(fname);
string header(W[0].begin(), W[0].end());
if (header != AdjGraphHeader) {
cout << "Bad input file: missing header: " << AdjGraphHeader << endl;
abort();
}
// file consists of [type, num_vertices, num_edges, <vertex offsets>, <edges>]
// in compressed sparse row format
long n = parlay::chars_to_long(W[1]);
long m = parlay::chars_to_long(W[2]);
if (W.size() != n + m + 3) {
cout << "Bad input file: length = "<< W.size() << " n+m+3 = " << n+m+3 << endl;
abort(); }
// tags on m at the end (so n+1 total offsets)
auto offsets = parlay::tabulate(n+1, [&] (size_t i) -> intE {
return (i == n) ? m : parlay::chars_to_long(W[i+3]);});
auto edges = parlay::tabulate(m, [&] (size_t i) -> intV {
return parlay::chars_to_long(W[n+i+3]);});
return graph<intV, intE>(std::move(offsets), std::move(edges), n);
}
// parlay::sequence<char> mmapStringFromFile(const char *filename) {
// struct stat sb;
// int fd = open(filename, O_RDONLY);
// if (fd == -1) {
// perror("open");
// exit(-1);
// }
// if (fstat(fd, &sb) == -1) {
// perror("fstat");
// exit(-1);
// }
// if (!S_ISREG (sb.st_mode)) {
// perror("not a file\n");
// exit(-1);
// }
// char *p = static_cast<char*>(mmap(0, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0));
// if (p == MAP_FAILED) {
// perror("mmap");
// exit(-1);
// }
// if (close(fd) == -1) {
// perror("close");
// exit(-1);
// }
// size_t n = sb.st_size;
// return parlay::sequence<char>(p, n); // Yikes!
// }
// template <class intV, class intV>
// graphC<intV, intV> readGraphCFromFile(char* fname, bool mmap=false) {
// parlay::sequence<char*> W;
// if (mmap) {
// cout << "mmapping file" << endl;
// parlay::sequence<char> S = mmapStringFromFile(fname);
// // copy to new sequence
// parlay::sequence<char> bytes = S;
// // and unmap
// if (munmap(S.begin(), S.size()) == -1) {
// perror("munmap");
// exit(-1);
// }
// W = stringToWords(S);
// cout << "mmap'd" << endl;
// } else {
// auto S = readStringFromFile(fname);
// W = stringToWords(S);
// }
// if (W[0] != AdjGraphHeader) {
// cout << "Bad input file: missing header: " << AdjGraphHeader << endl;
// abort();
// }
// // num vertices, num edges, edge offsets, edge pointers
// long len = W.size() -1;
// long n = atol(W[1]);
// long m = atol(W[2]);
// if (len != n + m + 2) {
// cout << "Bad input file: length = "<<len<< " n+m+2 = " << n+m+2 << endl;
// abort();
// }
// sequence<intV> offsets(n+1, [&] (size_t i) {
// return (i == n) ? m : atol(W[i+3]);});
// sequence<intV> edges(m, [&] (size_t i) {
// return atol(W[n+i+3]);});
// return graphC<intV,intV>(offsets,edges,n,m);
// }
template <class intV, class Weight, class intE>
wghGraph<intV, Weight, intE> readWghGraphFromFile(char* fname) {
parlay::sequence<char> S = readStringFromFile(fname);
parlay::sequence<char*> W = stringToWords(S);
if (W[0] != WghAdjGraphHeader) {
cout << "Bad input file" << endl;
abort();
}
long n = atol(W[1]);
long m = atol(W[2]);
if (W.size() != n + 2*m + 3) {
cout << "Bad input file: length = "<< W.size()
<< " n + 2*m + 3 = " << n+2*m+3 << endl;
abort(); }
// tags on m at the end (so n+1 total offsets)
auto offsets = parlay::tabulate(n+1, [&] (size_t i) -> intE {
return (i == n) ? m : atol(W[i+3]);});
auto edges = parlay::tabulate(m, [&] (size_t i) -> intV {
return atol(W[n+i+3]);});
auto weights = parlay::tabulate(m, [&] (size_t i) -> Weight {
return (Weight) atof(W[n+i+3+m]);});
return wghGraph<intV,Weight,intE>(std::move(offsets),
std::move(edges),
std::move(weights), n);
}
// The following two are used by the graph generators to write out in either format
// and either with reordering or not
template <class intV, class intE>
void writeGraphFromAdj(graph<intV,intE> const &G,
char* fname, bool adjArray, bool ordered) {
if (adjArray)
if (ordered) writeGraphToFile(G, fname);
else writeGraphToFile(graphReorder(G), fname);
else {
if (ordered)
writeEdgeArrayToFile(edgesFromGraph(G), fname);
else {
auto B = edgesFromGraph(graphReorder(G));
B = randomShuffle(B);
writeEdgeArrayToFile(B, fname);
}
}
}
template <class intV, class intE=intV>
void writeGraphFromEdges(edgeArray<intV> &EA, char* fname, bool adjArray, bool ordered) {
writeGraphFromAdj(graphFromEdges<intV,intE>(EA, adjArray),
fname, adjArray, ordered);
}
// void errorOut(const char* s) {
// cerr << s << endl;
// throw s;
// }
// void packInt64(int64_t x, uint8_t buf[8]) {
// uint64_t xu = x;
// for (int i = 0; i < 8; ++i)
// buf[i] = (xu >> (8 * i)) & 0xff;
// }
// int64_t unpackInt64(const uint8_t buf[8]) {
// uint64_t xu = 0;
// for (int i = 0; i < 8; ++i)
// xu |= ((uint64_t)buf[i]) << (i * 8);
// return (int64_t)xu;
// }
// void writeInt(ostream& out, char buf[8], int64_t x) {
// packInt64(x, (uint8_t*)buf);
// out.write(buf, 8);
// }
// int64_t readInt(istream& in, char buf[8]) {
// in.read(buf, 8);
// return unpackInt64((uint8_t*)buf);
// }
// template<typename intV>
// void writeFlowGraph(ostream& out, FlowGraph<intV> g) {
// char buf[8];
// out.write("FLOWFLOW", 8);
// writeInt(out, buf, g.g.n);
// writeInt(out, buf, g.g.m);
// writeInt(out, buf, g.source);
// writeInt(out, buf, g.sink);
// intV o = 0;
// for (intV i = 0; i < g.g.n; ++i) {
// writeInt(out, buf, o);
// o += g.g.V[i].degree;
// }
// for (intV i = 0; i < g.g.n; ++i) {
// wghVertex<intV>& v = g.g.V[i];
// for (intV j = 0; j < v.degree; ++j) {
// writeInt(out, buf, v.Neighbors[j]);
// writeInt(out, buf, v.nghWeights[j]);
// }
// }
// }
// template<typename intV>
// FlowGraph<intV> readFlowGraph(istream& in) {
// char buf[10];
// in.read(buf, 8);
// buf[8] = 0;
// if (strcmp(buf, "FLOWFLOW"))
// errorOut("Invalid flow graph input file");
// intV n = readInt(in, buf);
// intV m = readInt(in, buf);
// intV S = readInt(in, buf);
// intV T = readInt(in, buf);
// intV *offset = newA(intV, n);
// intV* adj = newA(intV, m);
// intV* weights = newA(intV, m);
// wghVertex<intV>* v = newA(wghVertex<intV>, n);
// for (intV i = 0; i < n; ++i) {
// offset[i] = readInt(in, buf);
// v[i].Neighbors = adj + offset[i];
// v[i].nghWeights = weights + offset[i];
// if (i > 0)
// v[i - 1].degree = offset[i] - offset[i - 1];
// }
// v[n - 1].degree = m - offset[n - 1];
// free(offset);
// for (intV i = 0; i < m; ++i) {
// adj[i] = readInt(in, buf);
// weights[i] = readInt(in, buf);
// }
// return FlowGraph<intV>(wghGraph<intV>(v, n, m, adj, weights), S, T);
// }
// const char nl = '\n';
// template <typename intV>
// FlowGraph<intV> writeFlowGraphDimacs(ostream& out, FlowGraph<intV> g) {
// out << "c DIMACS flow network description" << nl;
// out << "c (problem-id, nodes, arcs)" << nl;
// out << "p max " << g.g.n << " " << g.g.m << nl;
// out << "c source" << nl;
// out << "n " << g.source + 1 << " s" << nl;
// out << "c sink" << nl;
// out << "n " << g.sink + 1 << " t" << nl;
// out << "c arc description (from, to, capacity)" << nl;
// for (intV i = 0; i < g.g.n; ++i) {
// wghVertex<intV>& v = g.g.V[i];
// for (intV j = 0; j < v.degree; ++j) {
// out << "a " << i + 1 << " " << v.Neighbors[j] + 1 << " "
// << v.nghWeights[j] << nl;
// }
// }
// }
// template<typename intV>
// struct intWghEdge {
// intV from, to, w;
// };
// int readDimacsLinePref(istream& in, const char* expected) {
// char type;
// while (in >> type) {
// if (type == 'c') {
// while (in.peek() != EOF && in.peek() != '\n')
// in.ignore();
// in >> ws;
// continue;
// } else if (!strchr(expected, type)) {
// errorOut((string("Unexpected DIMACS line (expected 'c' or one of '")
// + expected + "')").c_str());
// }
// return type;
// }
// return EOF;
// }
// template <typename intV>
// FlowGraph<intV> readFlowGraphDimacs(istream& in) {
// string tmp;
// intV n, m;
// int type = readDimacsLinePref(in, "p");
// if (type == EOF)
// errorOut("Unexpected EOF while reading DIMACS file");
// in >> tmp >> n >> m;
// intWghEdge<intV>* edges = newA(intWghEdge<intV>, m);
// intV edgei = 0;
// intV* pos = newA(intV, n + 1);
// intV S = -1, T = -1;
// while (EOF != (type = readDimacsLinePref(in, "an"))) {
// if (type == 'n') {
// intV x;
// char st;
// in >> x >> st;
// x--;
// if (st == 's') S = x;
// else T = x;
// } else { // type == 'a'
// intV from, to, cap;
// in >> from >> to >> cap;
// from--; to--;
// edges[edgei] = (intWghEdge<intV>) { from, to, cap };
// edgei++;
// pos[from + 1]++;
// }
// }
// if (S < 0)
// errorOut("No source was specified in DIMACS input file");
// if (T < 0)
// errorOut("No sink was specified in DIMACS input file");
// if (m != edgei)
// errorOut("Inconsistent edge count in DIMACS input file");
// intV* adj = newA(intV, m);
// intV* weights = newA(intV, m);
// wghVertex<intV>* v = newA(wghVertex<intV>, n);
// for (intV i = 0; i < n; ++i) {
// pos[i + 1] += pos[i];
// v[i].Neighbors = adj + pos[i];
// v[i].nghWeights = weights + pos[i];
// v[i].degree = pos[i + 1] - pos[i];
// }
// for (intV i = 0; i < m; ++i) {
// intV& p = pos[edges[i].from];
// adj[p] = edges[i].to;
// weights[p] = edges[i].w;
// p++;
// }
// free(edges);
// free(pos);
// return FlowGraph<intV>(wghGraph<intV>(v, n, m, adj, weights), S, T);
// }
};
================================================
FILE: algorithms/bench/common/graphUtils.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#pragma once
#include <iostream>
#include <fstream>
#include <cstdlib>
#include <math.h>
#include "graph.h"
#include "../parlay/parallel.h"
#include "../parlay/primitives.h"
#include "../parlay/random.h"
#include "dataGen.h"
using namespace std;
template <class intV, class Weight = DefaultWeight>
wghEdgeArray<intV,Weight> addRandWeights(edgeArray<intV> const &G) {
using WE = wghEdge<intV,Weight>;
parlay::random r(257621);
intV m = G.nonZeros;
intV n = G.numRows;
auto E = parlay::tabulate(m, [&] (size_t i) -> WE {
return WE(G.E[i].u, G.E[i].v, (Weight) dataGen::hash<Weight>(i));});
return wghEdgeArray<intV,Weight>(std::move(E), n);
}
template <class intV>
edgeArray<intV> randomShuffle(edgeArray<intV> const &A) {
auto E = parlay::random_shuffle(A.E);
return edgeArray<intV>(std::move(E), A.numRows, A.numCols);
}
template <class intV>
edgeArray<intV> remDuplicates(edgeArray<intV> const &A) {
auto lessE = [&] (edge<intV> a, edge<intV> b) {
return (a.u < b.u) || ((a.u == b.u) && (a.v < b.v));};
parlay::sequence<edge<intV>> E =
parlay::remove_duplicates_ordered(A.E, lessE);
return edgeArray<intV>(std::move(E), A.numRows, A.numCols);
}
template <class intV>
edgeArray<intV> makeSymmetric(edgeArray<intV> const &A) {
parlay::sequence<edge<intV>> EF = parlay::filter(A.E, [&] (edge<intV> e) {
return e.u != e.v;});
auto FE = parlay::delayed_seq<edge<intV>>(EF.size(), [&] (size_t i) {
return edge<intV>(EF[i].v, EF[i].u);});
return remDuplicates(edgeArray<intV>(parlay::append(EF, FE),
A.numRows, A.numCols));
}
template <class intV, class intE = intV>
graph<intV,intE> graphFromEdges(edgeArray<intV> const &EA, bool makeSym) {
edgeArray<intV> SA;
if (makeSym) SA = makeSymmetric<intV>(EA);
edgeArray<intV> const &A = (makeSym) ? SA : EA;
size_t m = A.nonZeros;
size_t n = std::max(A.numCols, A.numRows);
parlay::sequence<size_t> counts;
parlay::sequence<intE> offsets;
parlay::sequence<edge<intV>> E;
size_t nn;
auto getu = [&] (edge<intV> e) {return e.u;};
std::tie(E, counts) = parlay::internal::integer_sort_with_counts(parlay::make_slice(A.E), getu, n);
std::tie(offsets,nn) = parlay::scan(parlay::delayed_seq<intE>(n+1, [&] (size_t i) {
return (i == n) ? 0 : counts[i];}), parlay::addm<intE>());
return graph<intV,intE>(std::move(offsets),
parlay::tabulate(m, [&] (size_t i) -> intV {return E[i].v;}),
n);
}
template <class intV, class Weight, class intE=intV>
wghGraph<intV,Weight,intE>
wghGraphFromEdges(wghEdgeArray<intV,Weight> const &A) {
using WE = wghEdge<intV,Weight>;
size_t n = A.n;
size_t m = A.m;
parlay::sequence<size_t> counts;
parlay::sequence<intE> offsets;
parlay::sequence<WE> E;
size_t nn;
auto getu = [&] (WE e) {return e.u;};
std::tie(E, counts) = parlay::internal::integer_sort_with_counts(parlay::make_slice(A.E), getu, n);
std::tie(offsets,nn) = parlay::scan(parlay::delayed_seq<intE>(n+1, [&] (size_t i) {
return (i == n) ? 0 : counts[i];}), parlay::addm<intE>());
return wghGraph<intV,Weight,intE>(std::move(offsets),
parlay::tabulate(m, [&] (size_t i)->intV {return E[i].v;}),
parlay::tabulate(m, [&] (size_t i) -> Weight {
return E[i].weight;}),
n);
}
template <class intV, class intE>
edgeArray<intV> edgesFromGraph(graph<intV,intE> const &G) {
size_t numRows = G.numVertices();
size_t nonZeros = G.numEdges();
// flatten
parlay::sequence<edge<intV>> E(nonZeros);
parlay::parallel_for(0, numRows, [&] (size_t j) {
size_t off = G.get_offsets()[j];
vertex<intV> v = G[j];
for (size_t i = 0; i < v.degree; i++)
E[off+i] = edge<intV>(j, v.Neighbors[i]);
});
return edgeArray<intV>(std::move(E), numRows, numRows);
}
// offset for start of each vertex if flattening the edge listd
template <class intV, class intE, class Vtx>
parlay::sequence<intE> getOffsets(parlay::sequence<Vtx> const &V) {
size_t n = V.size();
auto degrees = parlay::delayed_seq<intE>(n+1, [&] (size_t i) -> intE {
return (i == n) ? 0 : V[i].degree;});
auto x = parlay::scan(degrees, parlay::addm<intE>());
return x.first;
}
// packs a graph so that there are no gaps in the edge array (i.e. into CSR)
template <class intV, class intE>
graph<intV,intE> packGraph(graph<intV,intE> const &G) {
size_t n = G.numVertices();
auto degrees = parlay::delayed_seq<intE>(n+1, [&] (size_t i) -> intE {
return (i == n) ? 0 : G[i].degree;});
// calculate new offsets
auto sr = parlay::scan(degrees, parlay::addm<intE>());
// allocate new edge array
parlay::sequence<intV> outEdges(sr.second);
// copy edges so they are contiguous
parlay::parallel_for (0, G.n, [&] (size_t i) {
vertex<intV> v = G[i];
size_t offset = sr.first[i];
for (size_t j=0; j < v.degree; j++)
outEdges[offset + j] = v.Neighbors[j];
});
return graph<intV,intE>(std::move(sr.first), std::move(outEdges), n);
}
// if I is NULL then it randomly reorders
template <class intV, class intE>
graph<intV,intE> graphReorder(graph<intV,intE> const &Gr,
parlay::sequence<intV> const &I = parlay::sequence<intV>(0)) {
intV n = Gr.numVertices();
intV m = Gr.numEdges();
bool noI = (I.size()==0);
parlay::sequence<intV> const &II = noI ? parlay::random_permutation<intV>(n) : I;
// now write vertices to new locations
// inverse permutation
parlay::sequence<vertex<intV>> V(n);
parlay::parallel_for (0, n, [&] (size_t i) {
V[II[i]] = Gr[i];});
parlay::sequence<intE> offsets = getOffsets<intV,intE>(V);
parlay::sequence<intV> E(m);
parlay::parallel_for (0, n, [&] (size_t i) {
size_t o = offsets[i];
for (size_t j=0; j < V[i].degree; j++)
E[o + j] = II[V[i].Neighbors[j]];
std::sort(E.begin() + o, E.begin() + o + V[i].degree);
}, 1000);
return graph<intV>(std::move(offsets), std::move(E), n);
}
template <class intV, class intE>
int graphCheckConsistency(graph<intV,intE> const &Gr) {
size_t n = Gr.numVertices();
size_t m = Gr.numEdges();
size_t edgecount = parlay::reduce(parlay::delayed_seq<size_t>(n, [&] (size_t i) {
return Gr[i].degree;}), parlay::addm<size_t>());
if (m != edgecount) {
cout << "bad edge count in graphCheckConsistency: m = "
<< m << " sum of degrees = " << edgecount << endl;
return 1;
}
size_t error_loc = parlay::reduce(parlay::delayed_seq<size_t>(n, [&] (size_t i) {
for (size_t j=0; j < Gr[i].degree; j++)
if (Gr[i].Neighbors[j] >= n) return i;
return n;
}), parlay::minm<size_t>());
if (error_loc < n) {
cout << "edge out of range in graphCheckConsistency: at i = "
<< error_loc << endl;
return 1;
}
}
// template <class intV>
// sparseRowMajor<double,intV> sparseFromCsrFile(const char* fname) {
// FILE *f = fopen(fname,"r");
// if (f == NULL) {
// cout << "Trying to open nonexistant file: " << fname << endl;
// abort();
// }
// intV numRows; intV numCols; intV nonZeros;
// intV nc = fread(&numRows, sizeof(intV), 1, f);
// nc = fread(&numCols, sizeof(intV), 1, f);
// nc = fread(&nonZeros, sizeof(intV), 1, f);
// double *Values = (double *) malloc(sizeof(double)*nonZeros);
// intV *ColIds = (intV *) malloc(sizeof(intV)*nonZeros);
// intV *Starts = (intV *) malloc(sizeof(intV)*(1 + numRows));
// Starts[numRows] = nonZeros;
// size_t r;
// r = fread(Values, sizeof(double), nonZeros, f);
// r = fread(ColIds, sizeof(intV), nonZeros, f);
// r = fread(Starts, sizeof(intV), numRows, f);
// fclose(f);
// return sparseRowMajor<double,intV>(numRows,numCols,nonZeros,Starts,ColIds,Values);
// }
// template <class intV>
// edgeArray<intV> edgesFromMtxFile(const char* fname) {
// ifstream file (fname, ios::in);
// char* line = newA(char,1000);
// intV i,j = 0;
// while (file.peek() == '%') {
// j++;
// file.getline(line,1000);
// }
// intV numRows, numCols, nonZeros;
// file >> numRows >> numCols >> nonZeros;
// //cout << j << "," << numRows << "," << numCols << "," << nonZeros << endl;
// edge<intV> *E = newA(edge<intV>,nonZeros);
// double toss;
// for (i=0, j=0; i < nonZeros; i++) {
// file >> E[j].u >> E[j].v >> toss;
// E[j].u--;
// E[j].v--;
// if (toss != 0.0) j++;
// }
// nonZeros = j;
// //cout << "nonzeros = " << nonZeros << endl;
// file.close();
// return edgeArray<intV>(E,numRows,numCols,nonZeros);
// }
================================================
FILE: algorithms/bench/common/ligraLight.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#include <limits>
#include "parlay/primitives.h"
#include "parlay/parallel.h"
#include "parlay/internal/get_time.h"
#include "parlay/internal/block_delayed.h"
#include "common/graph.h"
namespace delayed = parlay::block_delayed;
namespace ligra {
template<typename vertexId>
struct vertex_subset {
using sparse_t = parlay::sequence<vertexId>;
using dense_t = parlay::sequence<bool>;
bool is_sparse;
size_t n;
size_t size() const {return n;}
sparse_t sparse;
dense_t dense;
vertex_subset(sparse_t x) :
sparse(std::move(x)), is_sparse(true), n(x.size()) {}
vertex_subset(vertexId v) :
sparse(sparse_t(1,v)), is_sparse(true), n(1) {}
vertex_subset(dense_t x) :
dense(std::move(x)), is_sparse(false),
n(parlay::count(x,true)) {}
};
template<typename Graph, typename Fa, typename Cond>
struct edge_map {
using vertexId = typename Graph::vertexId;
using vertex_subset_ = vertex_subset<vertexId>;
using vertex_subset_sparse = parlay::sequence<vertexId>;
using vertex_subset_dense = parlay::sequence<bool>;
Fa fa;
Cond cond;
const Graph& G;
bool dedup;
bool verbose;
parlay::sequence<vertexId> dup_seq;
edge_map(Graph const &G, Fa fa, Cond cond, bool dedup=false,
bool verbose=false) :
G(G), fa(fa), cond(cond), dedup(dedup), verbose(verbose) {
dup_seq = parlay::sequence<vertexId>::uninitialized(G.numVertices());
}
auto edge_map_sparse(vertex_subset_sparse const &vtx_subset) {
if (verbose) std::cout << "edge map sparse: " << vtx_subset.size() << std::endl;
auto nested_edges = parlay::map(vtx_subset, [&] (vertexId v) {
return parlay::delayed_tabulate(G[v].degree, [&, v] (size_t i) {
return std::pair(v, G[v].Neighbors[i]);});});
auto edges = delayed::flatten(nested_edges);
auto r = delayed::filter_map(edges,
[&] (auto x) {return cond(x.second) && fa(x.first, x.second);},
[] (auto x) {return x.second;});
if (dedup) {
parlay::parallel_for(0,r.size(), [&] (size_t i) { dup_seq[r[i]] = i;});
auto flags = parlay::tabulate(r.size(), [&] (size_t i) {return i==dup_seq[r[i]];});
return vertex_subset_(parlay::pack(r, flags));
}
return vertex_subset_(std::move(r));
}
auto edge_map_dense(vertex_subset_dense const &vtx_subset) {
if (verbose) std::cout << "edge map dense: " << vtx_subset.size() << std::endl;
auto r = parlay::tabulate(G.numVertices(), [&] (vertexId v) -> bool {
bool result = false;
if (cond(v)) {
size_t block_size = 5000;
auto vtx = G[v];
auto d = vtx.degree;
auto ngh = vtx.Neighbors;
auto do_block = [&, vsub=vtx_subset.begin()] (size_t i) {
size_t begin = block_size * i;
size_t end = std::min<size_t>(begin + block_size, d);
for (size_t j = begin; j < end; j++) {
if (!cond(v)) return;
vertexId u = ngh[j];
if (vsub[u]) {
bool x = fa(u,v);
if (!result && x) result = true;
}}};
size_t num_blocks = vtx.degree/block_size + 1;
if (num_blocks == 1) do_block(0);
else parlay::parallel_for(0, num_blocks, do_block, 1);
}
return result;});
return vertex_subset_(std::move(r));
}
auto operator() (vertex_subset_ const &vtx_subset) {
parlay::internal::timer t("edge_map", verbose);
auto l = vtx_subset.size();
auto n = G.numVertices();
bool do_dense;
if (vtx_subset.is_sparse) {
auto out_degree = parlay::reduce(parlay::delayed_map(vtx_subset.sparse, [&] (size_t i) {
return G[i].degree;}));
if ((l + out_degree) > G.m/20) {
parlay::sequence<bool> d_vtx_subset(n, false);
parlay::parallel_for(0, l, [&] (size_t i) {
d_vtx_subset[vtx_subset.sparse[i]] = true;});
t.next("convert");
return edge_map_dense(d_vtx_subset);
} else return edge_map_sparse(vtx_subset.sparse);
} else {
if (l > n/20) return edge_map_dense(vtx_subset.dense);
else {
auto s_vtx_subset = parlay::pack_index<vertexId>(vtx_subset.dense);
return edge_map_sparse(s_vtx_subset);
}
}
}
};
}
================================================
FILE: algorithms/bench/common/parallelDefs
================================================
ifeq (, $(shell which jemalloc-config))
JEMALLOC =
else
JEMALLOCLD = $(shell jemalloc-config --libdir)
JEMALLOC = -L$(JEMALLOCLD) -ljemalloc
endif
CCFLAGS = -mcx16 -O3 -std=c++17 -DNDEBUG -I .
CLFLAGS = -ldl $(JEMALLOC)
OMPFLAGS = -DPARLAY_OPENMP -fopenmp
CILKFLAGS = -DPARLAY_CILK -fcilkplus
PBBFLAGS = -DHOMEGROWN -pthread
ifdef OPENMP
CC = g++
CFLAGS = $(OMPFLAGS) $(CCFLAGS)
LFLAGS = $(OMPFLAGS) $(CLFLAGS)
else ifdef CILK
CC = g++
CFLAGS = $(CILKFLAGS) $(CCFLAGS)
LFLAGS = $(CILKFLAGS) $(CLFLAGS)
else
CC = g++
CFLAGS = $(PBBFLAGS) $(CCFLAGS)
LFLAGS = $(PBBFLAGS) $(CLFLAGS)
endif
================================================
FILE: algorithms/bench/common/parallelDefsANN
================================================
ifeq (, $(shell which jemalloc-config))
JEMALLOC =
else
JEMALLOCLD = $(shell jemalloc-config --libdir)
JEMALLOC = -L$(JEMALLOCLD) -ljemalloc
endif
CCFLAGS = -mcx16 -O3 -std=c++17 -march=native -DNDEBUG -I .
CLFLAGS = -ldl $(JEMALLOC)
OMPFLAGS = -DPARLAY_OPENMP -fopenmp
CILKFLAGS = -DPARLAY_CILK -fcilkplus
PBBFLAGS = -DHOMEGROWN -pthread
ifdef OPENMP
CC = g++
CFLAGS = $(OMPFLAGS) $(CCFLAGS)
LFLAGS = $(OMPFLAGS) $(CLFLAGS)
else ifdef CILK
CC = g++
CFLAGS = $(CILKFLAGS) $(CCFLAGS)
LFLAGS = $(CILKFLAGS) $(CLFLAGS)
else
CC = g++
CFLAGS = $(PBBFLAGS) $(CCFLAGS)
LFLAGS = $(PBBFLAGS) $(CLFLAGS)
endif
================================================
FILE: algorithms/bench/common/parallelDefs_OMP
================================================
ifeq (, $(shell which jemalloc-config))
JEMALLOC =
else
JEMALLOCLD = $(shell jemalloc-config --libdir)
JEMALLOC = -L$(JEMALLOCLD) -ljemalloc
endif
CCFLAGS = -mcx16 -O3 -std=c++17
CLFLAGS = -ldl $(JEMALLOC)
OMPFLAGS = -DOPENMP -fopenmp
CILKFLAGS = -DCILK -fcilkplus
PBBFLAGS = -DPARLAY_OPENMP -fopenmp -pthread
ifdef OPENMP
CC = g++
CFLAGS = $(OMPFLAGS) $(CCFLAGS)
LFLAGS = $(OMPFLAGS) $(LCFLAGS)
else ifdef CILK
CC = g++
CFLAGS = $(CILKFLAGS) $(CCFLAGS)
LFLAGS = $(CILKFLAGS) $(CLFLAGS)
else
CC = g++
CFLAGS = $(PBBFLAGS) $(CCFLAGS)
LFLAGS = $(PBBFLAGS) $(CLFLAGS)
endif
================================================
FILE: algorithms/bench/common/parseCommandLine.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#ifndef _PARSE_COMMAND_LINE
#define _PARSE_COMMAND_LINE
#include <iostream>
#include <fstream>
#include <string>
#include <cstring>
using namespace std;
struct commandLine {
int argc;
char** argv;
string comLine;
commandLine(int _c, char** _v, string _cl)
: argc(_c), argv(_v), comLine(_cl) {}
commandLine(int _c, char** _v)
: argc(_c), argv(_v), comLine("bad arguments") {}
void badArgument() {
cout << "usage: " << argv[0] << " " << comLine << endl;
abort();
}
// get an argument
// i is indexed from the last argument = 0, second to last indexed 1, ..
char* getArgument(int i) {
if (argc < 2+i) badArgument();
return argv[argc-1-i];
}
// looks for two filenames
pair<char*,char*> IOFileNames() {
if (argc < 3) badArgument();
return pair<char*,char*>(argv[argc-2],argv[argc-1]);
}
pair<int,char*> sizeAndFileName() {
if (argc < 3) badArgument();
return pair<int,char*>(std::atoi(argv[argc-2]),(char*) argv[argc-1]);
}
bool getOption(string option) {
for (int i = 1; i < argc; i++)
if ((string) argv[i] == option) return true;
return false;
}
char* getOptionValue(string option) {
for (int i = 1; i < argc-1; i++)
if ((string) argv[i] == option) return argv[i+1];
return NULL;
}
string getOptionValue(string option, string defaultValue) {
for (int i = 1; i < argc-1; i++)
if ((string) argv[i] == option) return (string) argv[i+1];
return defaultValue;
}
int getOptionIntValue(string option, int defaultValue) {
for (int i = 1; i < argc-1; i++)
if ((string) argv[i] == option) {
int r = atoi(argv[i+1]);
if (r < 1) badArgument();
return r;
}
return defaultValue;
}
long getOptionLongValue(string option, long defaultValue) {
for (int i = 1; i < argc-1; i++)
if ((string) argv[i] == option) {
long r = atol(argv[i+1]);
if (r < 1) badArgument();
return r;
}
return defaultValue;
}
double getOptionDoubleValue(string option, double defaultValue) {
for (int i = 1; i < argc-1; i++)
if ((string) argv[i] == option) {
double val;
if (sscanf(argv[i+1], "%lf", &val) == EOF) {
badArgument();
}
return val;
}
return defaultValue;
}
};
#endif // _PARSE_COMMAND_LINE
================================================
FILE: algorithms/bench/common/parse_command_line.h
================================================
// This code is part of the Problem Based Benchmark Suite (PBBS)
// Copyright (c) 2011 Guy Blelloch and the PBBS team
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights (to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#pragma once
#include <iostream>
#include <fstream>
#include <string>
#include <cstring>
struct commandLine {
int argc;
char** argv;
std::string comLine;
commandLine(int _c, char** _v, std::string _cl)
: argc(_c), argv(_v), comLine(_cl) {
if (getOption("-h") || getOption("-help"))
badArgument();
}
commandLine(int _c, char** _v)
: argc(_c), argv(_v), comLine("bad arguments") { }
void badArgument() {
std::cout << "usage: " << argv[0] << " " << comLine << std::endl;
exit(0);
}
// get an argument
// i is indexed from the last argument = 0, second to last indexed 1, ..
char* getArgument(int i) {
if (argc < 2+i) badArgument();
return argv[argc-1-i];
}
// looks for two filenames
std::pair<char*,char*> IOFileNames() {
if (argc < 3) badArgument();
return std::pair<char*,char*>(argv[argc-2],argv[argc-1]);
}
std::pair<size_t,char*> sizeAndFileName() {
if (argc < 3) badArgument();
return std::pair<size_t,char*>(std::atoi(argv[argc-2]),(char*) argv[argc-1]);
}
bool getOption(std::string option) {
for (int i = 1; i < argc; i++)
if ((std::string) argv[i] == option) return true;
return false;
}
char* getOptionValue(std::string option) {
for (int i = 1; i < argc-1; i++)
if ((std::string) argv[i] == option) return argv[i+1];
return NULL;
}
std::string getOptionValue(std::string option, std::string defaultValue) {
for (int i = 1; i < argc-1; i++)
if ((std::string) argv[i] == option) return (std::string) argv[i+1];
return defaultValue;
}
long getOptionLongValue(std::string option, long defaultValue) {
for (int i = 1; i < argc-1; i++)
if ((std::string) argv[i] == option) {
long r = atol(argv[i+1]);
if (r < 0) badArgument();
return r;
}
return defaultValue;
}
int getOptionIntValue(std::string option, int defaultValue) {
for (int i = 1; i < argc-1; i++)
if ((std::string) argv[i] == option) {
int r = atoi(argv[i+1]);
if (r < 0) badArgument();
return r;
}
return defaultValue;
}
double getOptionDoubleValue(std::string option, double defaultValue) {
for (int i = 1; i < argc-1; i++)
if ((std::string) argv[i] == option) {
double val;
if (sscanf(argv[i+1], "%lf", &val) == EOF) {
badArgument();
}
return val;
}
return defaultValue;
}
};
================================================
FILE: algorithms/bench/common/runTests.py
================================================
import subprocess
import sys
import random
import os
def onPprocessors(command,p) :
if "OPENMP" in os.environ:
return "OMP_NUM_THREADS="+repr(p)+" " + command
return command
elif "CILK" in os.environ:
return "CILK_NWORKERS="+repr(p)+" " + command
else:
return "PARLAY_NUM_THREADS="+repr(p)+" " + command
def shellGetOutput(str) :
process = subprocess.Popen(str,shell=True,stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, err = process.communicate()
if (len(err) > 0):
raise NameError(str+"\n"+output+err)
return output.decode("utf-8")
def stripFloat(val) :
trunc = float(int(val*1000))/1000
return str(trunc).rstrip('0')
def runSingle(runProgram, options, ifile, procs) :
comString = "./"+runProgram+" "+options+" "+ifile
if (procs > 0) :
comString = onPprocessors(comString,procs)
out = shellGetOutput(comString)
#print(out)
try:
times = [float(str[str.index(':')+2:]) for str in out.split('\n') if str.startswith("Parlay time: ")]
return times
except (ValueError,IndexError):
raise NameError(comString+"\n"+out)
def geomean(a) :
r = 1.0
for x in a :
r = r * x
return r**(1.0/len(a))
def runTest(runProgram, checkProgram, dataDir, test, rounds, procs, noOutput, keepData) :
random.seed()
outFile="/tmp/ofile%d_%d" %(random.randint(0, 1000000), random.randint(0, 1000000))
[weight, inputFileNames, runOptions, checkOptions] = test
if type(inputFileNames) is str :
inputFileNames = [inputFileNames]
shortInputNames = " ".join(inputFileNames)
if len(dataDir)>0:
out = shellGetOutput("cd " + dataDir + "; make " + shortInputNames)
longInputNames = " ".join(dataDir + "/" + name for name in inputFileNames)
runOptions = runOptions + " -r " + repr(rounds)
if (noOutput == 0) :
runOptions = runOptions + " -o " + outFile
times = runSingle(runProgram, runOptions, longInputNames, procs)
if (noOutput == 0) :
checkString = ("./" + checkProgram + " " + checkOptions + " "
+ longInputNames + " " + outFile)
checkOut = shellGetOutput(checkString)
# Allow checker output comments. Comments are lines prefixed by '::'
nonCommentLines = [s for s in checkOut.split('\n') if not s.startswith(':') and len(s)>0]
if (len(nonCommentLines) > 0) :
print("CheckOut:", checkOut)
raise NameError(checkString+"\n"+checkOut)
os.remove(outFile)
if len(dataDir)>0 and not(keepData):
out = shellGetOutput("rm " + longInputNames)
ptimes = str([stripFloat(time)
for time in times])[1:-1]
outputStr = ""
if (len(runOptions) > 0) :
outputStr = " : " + runOptions
print(shortInputNames + outputStr + " : "
+ ptimes + ", geomean = " + stripFloat(geomean(times)))
return [weight,times]
def averageTime(times) :
return sum(times)/len(times)
def timeAll(name, runProgram, checkProgram, dataDir, tests, rounds, procs, noOutput,
addToDatabase, problem, keepData) :
totalTime = 0
totalWeight = 0
try:
results = [runTest(runProgram, checkProgram, dataDir, test, rounds, procs,
noOutput, keepData)
for test in tests]
meanOfMeans = geomean([geomean(times) for (w,times) in results])
meanOfMins = geomean([sorted(times)[0] for (w,times) in results])
print(name + " : " + repr(procs) +" : " +
"geomean of mins = " + stripFloat(meanOfMins) +
", geomean of geomeans = " + stripFloat(meanOfMeans))
if (addToDatabase) :
try:
dbAddResult(problem=problem, program=runProgram, results=results, numProcs=procs, mean=totalTimeMean/totalWeight,
min=totalTimeMin/totalWeight, median=totalTimeMedian/totalWeight, tests=tests)
except:
print("Could not insert result in database. Error:", sys.exc_info()[0])
# if (os.getlogin() == 'akyrola'): raise
return 0
except NameError as x:
print("TEST TERMINATED ABNORMALLY:\n["+str(x) + "]")
return 1
except KeyboardInterrupt:
return 1
def getOption(str) :
a = sys.argv
l = len(a)
for i in range(1, l) :
if (a[i] == str) :
return True
return False
def getArg(str, default) :
a = sys.argv
l = len(a)
for i in range(1, l) :
if (a[i] == str and (i+1 != l)) :
return sys.argv[i+1]
return default
def getArgs() :
noOutput = getOption("-x")
addToDatabase = getOption("-d")
processors = int(getArg("-p", 0))
rounds = int(getArg("-r", 1))
keep = getOption("-k")
return (noOutput, rounds, addToDatabase, processors, keep)
def timeAllArgs(runProgram, problem, checkProgram, dataDir, tests, keepInputData=False) :
keepData = keepInputData
(noOutput, rounds, addToDatabase, procs, keep) = getArgs()
keep = keepInputData or keep
name = os.path.basename(os.getcwd())
timeAll(name, runProgram, checkProgram, dataDir, tests, rounds, procs, noOutput, addToDatabase, problem, keep)
#
# Database insertions
# - akyrola@cs.cmu.edu
import os
def dbInitConnection():
import MySQLdb
global cursor
# TODO: move to a config file
dbconn = MySQLdb.connect (host = "multi6.aladdin.cs.cmu.edu",
user = "pbbs",
passwd = "pbbspasshuuhaa",
db = "pbbsweb")
cursor = dbconn.cursor ()
dbconn.autocommit(1)
def dbAddResult(problem, program, results, numProcs, mean, min, median, tests):
dbInitConnection()
contentHash = computeContentHash(tests)
program = shellGetOutput("pwd").split('/')[-1].replace('\r','').replace('\n', '') + '/' + program
problemId = dbGetProblemId(problem, contentHash)
programId = dbGetProgramId(program, problemId)
hostId = getHostId()
#username = os.getlogin()
# getlogin does not work with some terminals (see various posts on web)
# guyb replaced with the following
username = os.getenv('USER')
if (numProcs == 0): numProcs = detectCPUs()
# Insert run into db
cursor.execute(""" insert into pbbs_runs (problem_id,program_id,numprocs,mean_time,min_time,median_time,username,host_id) values(
%s, %s, %s, %s, %s, %s, %s, %s)
""", (problemId, programId, numProcs, mean, min
gitextract_81v6usbn/
├── .bazelrc
├── .gitmodules
├── CMakeLists.txt
├── LICENSE
├── WORKSPACE
├── algorithms/
│ ├── CMakeLists.txt
│ ├── HCNNG/
│ │ ├── CMakeLists.txt
│ │ ├── Makefile
│ │ ├── clusterEdge.h
│ │ ├── hcnng_index.h
│ │ ├── neighbors.h
│ │ └── scripts/
│ │ ├── fashion
│ │ ├── gist_1
│ │ ├── glove100
│ │ ├── glove25
│ │ ├── nytimes
│ │ └── sift
│ ├── HNSW/
│ │ ├── CMakeLists.txt
│ │ ├── HNSW.hpp
│ │ ├── debug.hpp
│ │ ├── dist.hpp
│ │ ├── h5_ops.hpp
│ │ └── type_point.hpp
│ ├── bench/
│ │ ├── BUILD
│ │ ├── IO.h
│ │ ├── MakeBench
│ │ ├── Makefile
│ │ ├── benchUtils.h
│ │ ├── common/
│ │ │ ├── IO.h
│ │ │ ├── MakeBench
│ │ │ ├── MakeBenchLink
│ │ │ ├── atomics.h
│ │ │ ├── dataGen.h
│ │ │ ├── geometry.h
│ │ │ ├── geometryIO.h
│ │ │ ├── get_time.h
│ │ │ ├── glue.h
│ │ │ ├── graph.h
│ │ │ ├── graphIO.h
│ │ │ ├── graphUtils.h
│ │ │ ├── ligraLight.h
│ │ │ ├── parallelDefs
│ │ │ ├── parallelDefsANN
│ │ │ ├── parallelDefs_OMP
│ │ │ ├── parseCommandLine.h
│ │ │ ├── parse_command_line.h
│ │ │ ├── runTests.py
│ │ │ ├── runTestsANN.py
│ │ │ ├── seqDefs
│ │ │ ├── sequenceIO.h
│ │ │ ├── speculative_for.h
│ │ │ ├── time_loop.h
│ │ │ ├── topology.h
│ │ │ └── topology_from_triangles.h
│ │ ├── get_time.h
│ │ ├── neighborsTime.C
│ │ ├── parallelDefsANN
│ │ ├── parse_command_line.h
│ │ └── time_loop.h
│ ├── pyNNDescent/
│ │ ├── CMakeLists.txt
│ │ ├── Makefile
│ │ ├── clusterPynn.h
│ │ ├── neighbors.h
│ │ ├── pynn_index.h
│ │ └── scripts/
│ │ ├── nytimes
│ │ └── sift
│ ├── tutorial.sh
│ ├── utils/
│ │ ├── BUILD
│ │ ├── NSGDist.h
│ │ ├── beamSearch.h
│ │ ├── check_nn_recall.h
│ │ ├── check_range_recall.h
│ │ ├── csvfile.h
│ │ ├── doublingSearch.h
│ │ ├── earlyStopping.h
│ │ ├── euclidian_point.h
│ │ ├── graph.h
│ │ ├── graph_reorder.h
│ │ ├── hashset.h
│ │ ├── jl_point.h
│ │ ├── mips_point.h
│ │ ├── mmap.h
│ │ ├── parse_results.h
│ │ ├── point_range.h
│ │ ├── rangeSearch.h
│ │ ├── simpleGraph.h
│ │ ├── stats.h
│ │ ├── types.h
│ │ └── union.h
│ ├── vamana/
│ │ ├── BUILD
│ │ ├── CMakeLists.txt
│ │ ├── Makefile
│ │ ├── index.h
│ │ ├── index_test.cc
│ │ ├── neighbors.h
│ │ ├── neighbors.sh
│ │ ├── neighbors_test.cc
│ │ └── scripts/
│ │ ├── OpenAIArXiv
│ │ ├── deep10M
│ │ ├── fashion
│ │ ├── gist
│ │ ├── glove100
│ │ ├── glove25
│ │ ├── msmarco_websearch
│ │ ├── nytimes
│ │ ├── sift
│ │ ├── sift100
│ │ ├── space_1
│ │ ├── space_10
│ │ ├── t2i_1
│ │ ├── t2i_10
│ │ └── wikipedia_cohere
│ └── vamanaRange/
│ ├── CMakeLists.txt
│ ├── Makefile
│ ├── index.h
│ └── neighbors.h
├── build/
│ └── _deps/
│ └── parlaylib-subbuild/
│ └── CMakeLists.txt
├── data_tools/
│ ├── Makefile
│ ├── compute_groundtruth.cpp
│ ├── compute_range_groundtruth.cpp
│ ├── crop.cpp
│ ├── random_sample.cpp
│ └── vec_to_bin.cpp
├── docs/
│ ├── README.md
│ ├── algorithms.md
│ ├── data_tools.md
│ ├── quickstart.md
│ └── rangesearch.md
├── python/
│ ├── __init__.py
│ ├── _builder.py
│ ├── _builder.pyi
│ ├── _common.py
│ ├── _files.py
│ ├── big_env.yml
│ ├── builder.cpp
│ ├── compile.sh
│ ├── defaults.py
│ ├── graph_index.cpp
│ ├── module.cpp
│ ├── scripts/
│ │ ├── fashion_test.py
│ │ ├── gist_test.py
│ │ ├── glove100_test.py
│ │ ├── glove25_test.py
│ │ ├── nyt_test.py
│ │ └── sift_test.py
│ ├── sift_test.py
│ ├── test.py
│ └── wrapper.py
└── rangeSearch/
├── bench/
│ ├── .gitignore
│ ├── IO.h
│ ├── MakeBench
│ ├── Makefile
│ ├── get_time.h
│ ├── parallelDefsANN
│ └── rangeTime.C
└── vamanaRange/
├── Makefile
└── range.h
SYMBOL INDEX (394 symbols across 76 files)
FILE: algorithms/HCNNG/clusterEdge.h
function namespace (line 38) | namespace parlayANN {
function generate_index (line 62) | int generate_index(int N, int i) {
function closer_first (line 71) | auto closer_first =
function closer_second (line 78) | auto closer_second =
function active_indices (line 141) | auto active_indices =
FILE: algorithms/HCNNG/hcnng_index.h
function namespace (line 36) | namespace parlayANN {
function _union (line 53) | void _union(int x, int y) {
function find (line 68) | int find(int x) {
function flatten (line 82) | void flatten() {
function is_full (line 86) | bool is_full() {
function remove_edge_duplicates (line 114) | static void remove_edge_duplicates(indexType p, GraphI &G) {
function remove_all_duplicates (line 123) | void remove_all_duplicates(GraphI &G) {
function process_edges (line 129) | static void process_edges(GraphI &G, parlay::sequence<edge> edges) {
FILE: algorithms/HCNNG/neighbors.h
function namespace (line 37) | namespace parlayANN {
FILE: algorithms/HNSW/HNSW.hpp
type ANN (line 39) | namespace ANN{
type type_metric (line 43) | enum class type_metric{
type point (line 47) | struct point{
class HNSW (line 52) | class HNSW
type node (line 86) | struct node{
type dist (line 93) | struct dist{
type dist_ex (line 106) | struct dist_ex : dist
type nearest (line 111) | struct nearest{
type farthest (line 117) | struct farthest{
method neighbourhood (line 146) | static auto neighbourhood(node &u, uint32_t level)
method neighbourhood (line 155) | static auto neighbourhood(const node &u, uint32_t level)
method node (line 161) | node& get_node(node_id id)
method node (line 166) | const node& get_node(const node_id id) const
class dist_evaluator (line 182) | class dist_evaluator{
method dist_evaluator (line 189) | dist_evaluator(const point_t &p, uint32_t dim) :
method dist_t (line 192) | dist_t operator()(const point_t &pv) const{
method dist_t (line 195) | dist_t operator()(const point_t &pu, const point_t &pv) const{
type graph (line 200) | struct graph{
type edgeRange (line 202) | struct edgeRange{
method edgeRange (line 203) | edgeRange(Nbh &nbh) : nbh(nbh){
method size (line 208) | auto size() const{
method prefetch (line 211) | void prefetch() const{
method graph (line 222) | graph(const HNSW<U,Allocator> &hnsw, uint32_t l) :
method num_nodes (line 226) | decltype(auto) num_nodes() const{
method get_node (line 229) | decltype(auto) get_node(node_id pu) const{
method get_edges (line 232) | decltype(auto) get_edges(node_id pu){
method get_edges (line 235) | decltype(auto) get_edges(node_id pu) const{
method max_degree (line 239) | uint32_t max_degree() const{
method select_neighbors_simple_impl (line 259) | void select_neighbors_simple_impl(const T &u, Queue &C, uint32_t M)
method select_neighbors_simple (line 294) | auto select_neighbors_simple(const T &u, const Queue &C, uint32_t M)
method select_neighbors_heuristic (line 324) | auto select_neighbors_heuristic(const T &u,
method Seq (line 431) | Seq prune_heuristic(
method select_neighbors (line 488) | auto select_neighbors(const T &u,
method get_level_random (line 505) | uint32_t get_level_random()
method get_threshold_m (line 527) | auto get_threshold_m(uint32_t level) const{
method get_deg (line 534) | auto get_deg(uint32_t level=0)
method get_indeg (line 546) | auto get_indeg(uint32_t level) const
method get_height (line 565) | uint32_t get_height() const
method cnt_degree (line 570) | size_t cnt_degree(uint32_t l) const
method cnt_vertex (line 580) | size_t cnt_vertex(uint32_t l) const
method get_degree_max (line 589) | size_t get_degree_max(uint32_t l) const
function beamSearch (line 1017) | auto beamSearch(
FILE: algorithms/HNSW/debug.hpp
type search_control (line 13) | struct search_control{
FILE: algorithms/HNSW/dist.hpp
class descr_ang (line 9) | class descr_ang
method distance (line 18) | static float distance(const type_point &u, const type_point &v, uint32...
method get_id (line 31) | static auto get_id(const type_point &u)
class descr_ndot (line 38) | class descr_ndot
method distance (line 47) | static float distance(const type_point &u, const type_point &v, uint32...
method get_id (line 56) | static auto get_id(const type_point &u)
class descr_l2 (line 63) | class descr_l2
method distance (line 72) | static float distance(const type_point &u, const type_point &v, uint32...
method get_id (line 93) | static auto get_id(const type_point &u)
FILE: algorithms/HNSW/h5_ops.hpp
function get_reader (line 15) | auto get_reader(const char *file, const char *dir)
function read_array_from_HDF5 (line 55) | std::pair<std::unique_ptr<T[]>,std::array<hsize_t,2>> read_array_from_HD...
FILE: algorithms/HNSW/type_point.hpp
class internal_termination (line 18) | class internal_termination{
method internal_termination (line 20) | internal_termination(){}
method internal_termination (line 21) | internal_termination(int){std::terminate();}
class fake_copyable (line 25) | class fake_copyable : public internal_termination{
method fake_copyable (line 28) | fake_copyable(const T &c) : content(c){}
method fake_copyable (line 29) | fake_copyable(T &&c) : content(std::move(c)){}
method fake_copyable (line 31) | fake_copyable(fake_copyable&&) = default;
method fake_copyable (line 32) | fake_copyable(const fake_copyable &other [[maybe_unused]])
type point (line 51) | struct point
method point (line 58) | point()
method point (line 62) | point(uint32_t id_, const T *coord_)
method point (line 67) | point(uint32_t id_, const T *coord_, C &&closure_)
type file_format (line 75) | enum class file_format{
class point_converter_default (line 80) | class point_converter_default
method type (line 86) | type operator()(uint32_t id, Iter begin, [[maybe_unused]] Iter end)
function load_from_vec (line 114) | inline std::pair<parlay::sequence<typename Conv::type>,uint32_t>
class trait_type (line 143) | class trait_type{
class trait_type<T,std::void_t<typename T::type>> (line 147) | class trait_type<T,std::void_t<typename T::type>>{
class trait_type<T*,void> (line 153) | class trait_type<T*,void>{
class trait_type<parlay::sequence<T>,void> (line 159) | class trait_type<parlay::sequence<T>,void>{
function load_from_HDF5 (line 165) | inline std::pair<parlay::sequence<typename Conv::type>,uint32_t>
function load_from_bin (line 193) | inline std::pair<parlay::sequence<typename Conv::type>,uint32_t>
function load_from_range (line 213) | inline std::pair<parlay::sequence<typename Conv::type>,uint32_t>
function load_point (line 262) | inline auto load_point(const char *input_name, Conv converter, size_t ma...
FILE: algorithms/bench/IO.h
function namespace (line 35) | namespace benchIO {
FILE: algorithms/bench/benchUtils.h
type class (line 45) | enum class
function ptr_raw (line 65) | ptr_raw(p){
function T (line 85) | T* get() const{
function operator (line 89) | operator T*() const{
type stat (line 173) | struct stat
FILE: algorithms/bench/common/IO.h
function namespace (line 35) | namespace benchIO {
FILE: algorithms/bench/common/atomics.h
function namespace (line 3) | namespace pbbs {
FILE: algorithms/bench/common/dataGen.h
function namespace (line 4) | namespace dataGen {
FILE: algorithms/bench/common/geometry.h
function vector (line 36) | vector operator*(coord s) {return vector(x * s, y * s, z * s);}
function vector (line 37) | vector operator/(coord s) {return vector(x / s, y / s, z / s);}
function print (line 44) | void print() {cout << std::setprecision(10) << ":(" << x << "," << y << ...
function coord (line 45) | coord Length(void) { return sqrt(x*x+y*y+z*z);}
function coord (line 46) | coord sqLength(void) { return x*x+y*y+z*z;}
function dimension (line 57) | int dimension() {return 3;}
function print (line 62) | void print() {cout << ":(" << x << "," << y << "," << z << "):";}
function point (line 67) | point minCoords(point b) {
function point (line 69) | point maxCoords(point b) {
function point (line 87) | point changeCoords(std::vector<coord> v){
function outOfBox (line 92) | bool outOfBox(point pt, coord hsize) {
function vector (line 128) | vector operator*(coord s) {return vector(x * s, y * s);}
function vector (line 129) | vector operator/(coord s) {return vector(x / s, y / s);}
function coord (line 130) | coord operator[] (int i) {return (i==0) ? x : y;}
function coord (line 131) | coord dot(vector v) {return x * v.x + y * v.y;}
function coord (line 132) | coord cross(vector v) { return x*v.y - y*v.x; }
function coord (line 133) | coord maxDim() {return max(x,y);}
function print (line 134) | void print() {cout << ":(" << x << "," << y << "):";}
function coord (line 135) | coord Length(void) { return sqrt(x*x+y*y);}
function coord (line 136) | coord sqLength(void) { return x*x+y*y;}
function dimension (line 156) | int dimension() {return 2;}
function print (line 161) | void print() {cout << ":(" << x << "," << y << "):";}
function coord (line 164) | coord operator[] (int i) {return (i==0) ? x : y;}
function point (line 165) | point minCoords(point b) { return point(min(x,b.x),min(y,b.y)); }
function point (line 166) | point maxCoords(point b) { return point(max(x,b.x),max(y,b.y)); }
function quadrant (line 167) | int quadrant(point center) {
function point (line 175) | point offsetPoint(int dir, coord offset) {
function outOfBox (line 180) | bool outOfBox(point pt, coord hsize) {
function numPoints (line 259) | size_t numPoints() {return P.size();}
function numTriangles (line 260) | size_t numTriangles() {return T.size();}
function coord (line 278) | inline coord angle(point2d<coord> a, point2d<coord> b, point2d<coord> c) {
FILE: algorithms/bench/common/geometryIO.h
function namespace (line 75) | namespace benchIO {
FILE: algorithms/bench/common/get_time.h
function get_time (line 9) | struct timer {
function start (line 27) | void start () {
function stop (line 32) | double stop () {
function reset (line 39) | void reset() {
function get_total (line 44) | double get_total() {
function get_next (line 49) | double get_next() {
function report (line 58) | void report(double time, std::string str) {
function total (line 69) | void total() {
function reportTotal (line 74) | void reportTotal(std::string str) {
function next (line 78) | void next(std::string str) {
FILE: algorithms/bench/common/glue.h
function namespace (line 11) | namespace utils {
function namespace (line 96) | namespace dataGen {
FILE: algorithms/bench/common/graph.h
function addDegrees (line 130) | void addDegrees() {
function MVT (line 135) | MVT operator[] (const size_t i) {
function VT (line 140) | const VT operator[] (const size_t i) const {
function VT (line 184) | VT operator[] (const size_t i) {
function FlowGraph (line 206) | FlowGraph copy() {
function del (line 209) | void del() { g.del(); }
FILE: algorithms/bench/common/graphIO.h
function namespace (line 69) | namespace benchIO {
FILE: algorithms/bench/common/graphUtils.h
function E (line 43) | auto E = parlay::tabulate(m, [&] (size_t i) -> WE {
FILE: algorithms/bench/common/ligraLight.h
function namespace (line 32) | namespace ligra {
FILE: algorithms/bench/common/parse_command_line.h
type commandLine (line 30) | struct commandLine {
FILE: algorithms/bench/common/runTests.py
function onPprocessors (line 6) | def onPprocessors(command,p) :
function shellGetOutput (line 15) | def shellGetOutput(str) :
function stripFloat (line 24) | def stripFloat(val) :
function runSingle (line 28) | def runSingle(runProgram, options, ifile, procs) :
function geomean (line 40) | def geomean(a) :
function runTest (line 46) | def runTest(runProgram, checkProgram, dataDir, test, rounds, procs, noOu...
function averageTime (line 81) | def averageTime(times) :
function timeAll (line 84) | def timeAll(name, runProgram, checkProgram, dataDir, tests, rounds, proc...
function getOption (line 112) | def getOption(str) :
function getArg (line 120) | def getArg(str, default) :
function getArgs (line 128) | def getArgs() :
function timeAllArgs (line 136) | def timeAllArgs(runProgram, problem, checkProgram, dataDir, tests, keepI...
function dbInitConnection (line 149) | def dbInitConnection():
function dbAddResult (line 163) | def dbAddResult(problem, program, results, numProcs, mean, min, median, ...
function computeContentHash (line 196) | def computeContentHash(tests):
function dbGetProblemId (line 206) | def dbGetProblemId(probname, contentHash):
function dbGetProgramId (line 216) | def dbGetProgramId(progname, problemId):
function getHostId (line 227) | def getHostId():
function detectCPUModel (line 246) | def detectCPUModel():
function detectCPUs (line 263) | def detectCPUs():
FILE: algorithms/bench/common/runTestsANN.py
function addLineToFile (line 6) | def addLineToFile(oFile, line):
function onPprocessors (line 17) | def onPprocessors(command,p) :
function shellGetOutput (line 26) | def shellGetOutput(str) :
function stripFloat (line 35) | def stripFloat(val) :
function runSingle (line 39) | def runSingle(runProgram, options, ifile, procs, oFile) :
function runTest (line 53) | def runTest(runProgram, checkProgram, dataDir, test, rounds, procs, noOu...
function averageTime (line 99) | def averageTime(times) :
function timeAll (line 103) | def timeAll(name, runProgram, checkProgram, dataDir, tests, rounds, proc...
function getOption (line 139) | def getOption(str) :
function getArg (line 147) | def getArg(str, default) :
function getArgs (line 155) | def getArgs() :
function timeAllArgs (line 161) | def timeAllArgs(runProgram, problem, checkProgram, dataDir, tests, oFile) :
FILE: algorithms/bench/common/sequenceIO.h
function namespace (line 33) | namespace parlay {
function namespace (line 37) | namespace benchIO {
FILE: algorithms/bench/common/speculative_for.h
function namespace (line 28) | namespace pbbs {
FILE: algorithms/bench/common/topology.h
function setT (line 55) | void setT(tri_t *t1, tri_t *t2, tri_t* t3) {
function setV (line 57) | void setV(vtx_t *v1, vtx_t *v2, vtx_t *v3) {
function locate (line 59) | int locate(tri_t *t) {
function update (line 67) | void update(tri_t *t, tri_t *tn) {
function print (line 86) | void print() {
function mod3 (line 95) | inline int mod3(int i) {return (i>2) ? i-3 : i;}
function print (line 121) | void print() {
function simplex (line 135) | simplex across() {
function simplex (line 142) | simplex rotClockwise() { return simplex(t,mod3(o+1));}
function valid (line 144) | bool valid() {return (!boundary);}
function isTriangle (line 145) | bool isTriangle() {return (!boundary);}
function isBoundary (line 146) | bool isBoundary() {return boundary;}
function vtx_t (line 148) | vtx_t *firstVertex() {return t->vtx[o];}
function inCirc (line 150) | bool inCirc(vtx_t *v) {
function farAngle (line 157) | double farAngle() {
function outside (line 163) | bool outside(vtx_t *v) {
function flip (line 169) | void flip() {
function split (line 195) | void split(vtx_t* v, tri_t* ta0, tri_t* ta1) {
function splitBoundary (line 211) | void splitBoundary(vtx_t* v, tri_t* ta) {
function simplex (line 228) | simplex extend(vtx_t* v, tri_t* ta) {
FILE: algorithms/bench/common/topology_from_triangles.h
function getKey (line 51) | struct hashEdges {
function hash (line 56) | size_t hash(kType s) { return hash64(s.first)+3*(hash64(s.second)); }
function cmp (line 57) | int cmp(kType s1, kType s2) {
function cas (line 63) | bool cas(eType* p, eType o, eType n) {
function replaceQ (line 66) | bool replaceQ(eType s, eType s2) {return 0;}
function EdgeTable (line 71) | EdgeTable makeEdgeTable(size_t m) {
FILE: algorithms/bench/get_time.h
function namespace (line 9) | namespace cpam {
FILE: algorithms/bench/parse_command_line.h
type commandLine (line 30) | struct commandLine {
FILE: algorithms/pyNNDescent/clusterPynn.h
function namespace (line 39) | namespace parlayANN {
FILE: algorithms/pyNNDescent/neighbors.h
function namespace (line 35) | namespace parlayANN {
FILE: algorithms/pyNNDescent/pynn_index.h
function namespace (line 34) | namespace parlayANN {
function undirect_and_prune (line 198) | void undirect_and_prune(GraphI &G, PR &Points, double alpha){
FILE: algorithms/utils/NSGDist.h
type Metric (line 23) | enum Metric { L2 = 0, INNER_PRODUCT = 1, FAST_L2 = 2, PQ = 3 }
function class (line 24) | class Distance {
function compare (line 335) | float compare(const float *a, const float *b, float norm,
FILE: algorithms/utils/beamSearch.h
function namespace (line 21) | namespace parlayANN {
type EStop (line 411) | struct EStop {
function indices (line 468) | auto indices = parlay::tabulate(Query_Points.size(), [&](size_t i) {
FILE: algorithms/utils/check_nn_recall.h
function namespace (line 15) | namespace parlayANN {
FILE: algorithms/utils/check_range_recall.h
function namespace (line 14) | namespace parlayANN {
FILE: algorithms/utils/csvfile.h
function namespace (line 31) | namespace parlayANN {
FILE: algorithms/utils/doublingSearch.h
function namespace (line 19) | namespace parlayANN{
FILE: algorithms/utils/earlyStopping.h
function namespace (line 18) | namespace parlayANN{
FILE: algorithms/utils/euclidian_point.h
function namespace (line 43) | namespace parlayANN {
function const (line 272) | int8_t operator [] (long j) const {
function distance (line 276) | float distance(const Euclidean_JL_Sparse_Point &q) const {
function prefetch (line 282) | void prefetch() const {
function normalize (line 302) | void normalize() {
type Euclidean_Bit_Point (line 338) | struct Euclidean_Bit_Point {
function const (line 356) | int8_t operator [] (long j) const {
function distance (line 361) | float distance(const Euclidean_Bit_Point &q) const {
function prefetch (line 371) | void prefetch() const {
function normalize (line 395) | void normalize() {
FILE: algorithms/utils/graph.h
function namespace (line 43) | namespace parlayANN {
FILE: algorithms/utils/graph_reorder.h
type tagged_w_type (line 21) | struct tagged_w_type {w_type w; edge_id i;}
function greater (line 22) | bool greater(tagged_w_type a, tagged_w_type b) {
type vertex_info (line 24) | struct vertex_info {
function recursive_reorder (line 41) | void recursive_reorder(parlay::sequence<w_edge>& E,
FILE: algorithms/utils/hashset.h
function hash (line 19) | size_t hash(K const& k) const noexcept {
FILE: algorithms/utils/jl_point.h
function namespace (line 19) | namespace parlayANN {
type parameters (line 113) | struct parameters {
function const (line 128) | int8_t operator [] (long j) const {
function distance (line 132) | float distance(const Mips_JL_Bit_Point &q) const {
function prefetch (line 138) | void prefetch() const {
function normalize (line 158) | void normalize() {
type parameters (line 199) | struct parameters {
function const (line 217) | int8_t operator [] (long j) const {
function distance (line 221) | float distance(const Mips_JL_Sparse_Point &q) const {
function prefetch (line 227) | void prefetch() const {
function normalize (line 247) | void normalize() {
type parameters (line 292) | struct parameters {
function const (line 310) | int8_t operator [] (long j) const {
function distance (line 314) | float distance(const Mips_JL_Sparse_Point_Normalized &q) const {
function prefetch (line 322) | void prefetch() const {
function normalize (line 342) | void normalize() {
FILE: algorithms/utils/mips_point.h
function namespace (line 42) | namespace parlayANN {
type Mips_2Bit_Point (line 510) | struct Mips_2Bit_Point {
type parameters (line 527) | struct parameters {
function const (line 541) | int operator [] (long i) const {
function distance_8 (line 545) | float distance_8(byte* p_, byte* q_) const {
function distance (line 560) | float distance(const Mips_2Bit_Point &x) const {
function prefetch (line 564) | void prefetch() const {
function normalize (line 589) | void normalize() {
type Mips_Bit_Point (line 640) | struct Mips_Bit_Point {
function const (line 657) | int8_t operator [] (long j) const {
function distance (line 662) | float distance(const Mips_Bit_Point &q) const {
function prefetch (line 672) | void prefetch() const {
function normalize (line 696) | void normalize() {
type Mips_4Bit_Point (line 721) | struct Mips_4Bit_Point {
type parameters (line 736) | struct parameters {
function const (line 750) | int operator [] (long i) const {
function triple (line 754) | static int16_t triple(word a, word b, word plus, word minus) {
function distance (line 759) | float distance(byte* p_, byte* q_) const {
function distance (line 784) | float distance(const Mips_4Bit_Point &x) const {
function prefetch (line 788) | void prefetch() const {
function normalize (line 813) | void normalize() {
FILE: algorithms/utils/mmap.h
function namespace (line 11) | namespace parlayANN {
FILE: algorithms/utils/parse_results.h
function namespace (line 10) | namespace parlayANN {
function print (line 48) | struct range_result {
function print (line 102) | struct nn_result {
function print_verbose (line 146) | void print_verbose() {
function print (line 158) | struct lsh_result {
function pred (line 195) | auto pred = [&](res R) { return R.recall >= b; }
function pred2 (line 202) | auto pred2 = [&](res R) { return R.recall <= c; }
FILE: algorithms/utils/point_range.h
function namespace (line 40) | namespace parlayANN {
FILE: algorithms/utils/simpleGraph.h
function indexType (line 55) | indexType operator [] (indexType j) const {
function append_neighbor (line 62) | void append_neighbor(indexType nbh){
function clear_neighbors (line 79) | void clear_neighbors(){
function prefetch (line 83) | void prefetch() const {
function indexType (line 93) | indexType* begin() const {return (*edges).data();}
function indexType (line 95) | indexType* end() const {return (*edges).data() + size();}
function save (line 122) | void save(char* oFile) {
FILE: algorithms/utils/stats.h
function namespace (line 39) | namespace parlayANN {
function GG (line 56) | auto GG = parlay::tabulate(n, [&] (indexType i) {
function increment_dist (line 133) | void increment_dist(indexType i, indexType j){
function increment_visited (line 135) | void increment_visited(indexType i, indexType j){
function clear (line 141) | void clear(){
function parlay (line 147) | static parlay::sequence<indexType> statistics(parlay::sequence<indexType...
FILE: algorithms/utils/types.h
function namespace (line 36) | namespace parlayANN {
function max_degree (line 225) | long max_degree(){
type QueryParams (line 233) | struct QueryParams{
type Point (line 274) | typedef Point type_point;
FILE: algorithms/utils/union.h
function namespace (line 8) | namespace parlayANN {
FILE: algorithms/vamana/index.h
function namespace (line 40) | namespace parlayANN {
FILE: algorithms/vamana/index_test.cc
function TEST (line 5) | TEST(PlaceHolderTest, BuildPlaceHolder) { EXPECT_EQ(7 * 6, 42); }
FILE: algorithms/vamana/neighbors.h
function namespace (line 39) | namespace parlayANN {
FILE: algorithms/vamana/neighbors_test.cc
function TEST (line 5) | TEST(PlaceHolderTest, BuildPlaceHolder) { EXPECT_EQ(7 * 6, 42); }
FILE: algorithms/vamanaRange/index.h
function BP (line 54) | BP(BP) {}
function indexType (line 56) | indexType get_start() { return start_point; }
function new_end (line 82) | auto new_end =std::unique(candidates.begin(), candidates.end(),
function set_start (line 144) | void set_start(){start_point = 0;}
FILE: data_tools/compute_groundtruth.cpp
function compute_groundtruth (line 23) | parlay::sequence<parlay::sequence<pid>> compute_groundtruth(PointRange &B,
function write_ibin (line 64) | void write_ibin(parlay::sequence<parlay::sequence<pid>> &result, const s...
function main (line 102) | int main(int argc, char* argv[]) {
FILE: data_tools/compute_range_groundtruth.cpp
function compute_range_groundtruth (line 15) | parlay::sequence<parlay::sequence<int>> compute_range_groundtruth(PointR...
function write_nonzero_elts (line 33) | void write_nonzero_elts(parlay::sequence<parlay::sequence<int>> &result,...
function write_rangeres (line 65) | void write_rangeres(parlay::sequence<parlay::sequence<int>> &result, con...
function main (line 92) | int main(int argc, char* argv[]) {
FILE: data_tools/crop.cpp
function mmapStringFromFile (line 18) | std::pair<char*, size_t> mmapStringFromFile(const char* filename) {
function crop_file (line 48) | void crop_file(char* iFile, int n, char* oFile){
function main (line 68) | int main(int argc, char* argv[]) {
FILE: data_tools/random_sample.cpp
function random_sample (line 14) | void random_sample(char* iFile, int n, char* oFile){
function main (line 48) | int main(int argc, char* argv[]) {
FILE: data_tools/vec_to_bin.cpp
function convert_onebyte (line 10) | auto convert_onebyte(const char* infile, const char* outfile) {
function convert_fourbyte (line 24) | auto convert_fourbyte(const char* infile, const char* outfile) {
function main (line 38) | int main(int argc, char* argv[]) {
FILE: python/__init__.py
class QueryResponse (line 82) | class QueryResponse(NamedTuple):
class QueryResponseBatch (line 96) | class QueryResponseBatch(NamedTuple):
FILE: python/_builder.py
function _valid_path_and_dtype (line 45) | def _valid_path_and_dtype(
function build_memory_index (line 71) | def build_memory_index(
FILE: python/_builder.pyi
function numpy_to_diskann_file (line 29) | def numpy_to_diskann_file(vectors: np.ndarray, file_handler: BinaryIO): ...
function build_vamana_index (line 30) | def build_vamana_index(
function build_hcnng_index (line 41) | def build_hcnng_index(
function build_pynndescent_index (line 51) | def build_pynndescent_index(
FILE: python/_common.py
function valid_dtype (line 45) | def valid_dtype(dtype: Type) -> VectorDType:
function _assert (line 59) | def _assert(statement_eval: bool, message: str):
function _valid_metric (line 64) | def _valid_metric(metric: str) -> str:
function _assert_dtype (line 75) | def _assert_dtype(dtype: Type):
function _castable_dtype_or_raise (line 82) | def _castable_dtype_or_raise(
function _assert_2d (line 93) | def _assert_2d(vectors: np.ndarray, name: str):
function _assert_is_positive_uint32 (line 100) | def _assert_is_positive_uint32(test_value: int, parameter: str):
function _assert_is_nonnegative_uint32 (line 107) | def _assert_is_nonnegative_uint32(test_value: int, parameter: str):
function _assert_is_nonnegative_uint64 (line 114) | def _assert_is_nonnegative_uint64(test_value: int, parameter: str):
function _assert_existing_directory (line 121) | def _assert_existing_directory(path: str, parameter: str):
function _assert_existing_file (line 128) | def _assert_existing_file(path: str, parameter: str):
class _DataType (line 133) | class _DataType(Enum):
method from_type (line 139) | def from_type(cls, vector_dtype: VectorDType) -> "DataType":
method to_type (line 147) | def to_type(self) -> VectorDType:
function _build_metadata_path (line 159) | def _build_metadata_path(index_path_and_prefix: str) -> str:
function _write_index_metadata (line 163) | def _write_index_metadata(
function _read_index_metadata (line 181) | def _read_index_metadata(
function _ensure_index_metadata (line 197) | def _ensure_index_metadata(
function _valid_index_prefix (line 231) | def _valid_index_prefix(index_directory: str, index_prefix: str) -> str:
FILE: python/_files.py
class Metadata (line 33) | class Metadata(NamedTuple):
function vectors_metadata_from_file (line 42) | def vectors_metadata_from_file(vector_file: str) -> Metadata:
function _write_bin (line 56) | def _write_bin(data: np.ndarray, file_handler: BinaryIO):
function vectors_to_file (line 64) | def vectors_to_file(vector_file: str, vectors: VectorLikeBatch) -> None:
function vectors_from_file (line 78) | def vectors_from_file(vector_file: str, dtype: VectorDType) -> npt.NDArr...
FILE: python/builder.cpp
function build_vamana_index (line 37) | void build_vamana_index(std::string metric, std::string &vector_bin_path,
function build_hcnng_index (line 119) | void build_hcnng_index(std::string metric, std::string &vector_bin_path,
function build_pynndescent_index (line 163) | void build_pynndescent_index(std::string metric, std::string &vector_bin...
function build_hnsw_index (line 207) | void build_hnsw_index(std::string metric, std::string &vector_bin_path,
FILE: python/graph_index.cpp
type GraphIndex (line 49) | struct GraphIndex{
method GraphIndex (line 82) | GraphIndex(std::string &data_path, std::string &index_path, bool is_hn...
method search_dispatch (line 120) | auto search_dispatch(Point &q, QueryParams &QP, bool quant)
method NeighborsAndDistances (line 192) | NeighborsAndDistances batch_search(py::array_t<T, py::array::c_style |...
method single_search (line 218) | py::array_t<unsigned int>
method NeighborsAndDistances (line 237) | NeighborsAndDistances batch_search_from_string(std::string &queries,
method check_recall (line 259) | void check_recall(std::string &queries_file,
FILE: python/module.cpp
type Variant (line 44) | struct Variant
function add_variant (line 59) | inline void add_variant(py::module_ &m, const Variant &variant)
function add_hcnng_variant (line 87) | inline void add_hcnng_variant(py::module_ &m, const Variant &variant)
function add_pynndescent_variant (line 105) | inline void add_pynndescent_variant(py::module_ &m, const Variant &variant)
function add_hnsw_variant (line 124) | inline void add_hnsw_variant(py::module_ &m, const Variant &variant)
function PYBIND11_MODULE (line 132) | PYBIND11_MODULE(_ParlayANNpy, m)
FILE: python/wrapper.py
function build_vamana_index (line 3) | def build_vamana_index(metric, dtype, data_dir, index_dir, R, L, alpha, ...
function build_hcnng_index (line 27) | def build_hcnng_index(metric, dtype, data_dir, index_dir, mst_deg, num_c...
function build_pynndescent_index (line 50) | def build_pynndescent_index(metric, dtype, data_dir, index_dir, max_deg,...
function build_hnsw_index (line 73) | def build_hnsw_index(metric, dtype, data_dir, index_dir, R, efc, m_l, al...
function load_index (line 96) | def load_index(metric, dtype, data_dir, index_dir, hnsw=False):
FILE: rangeSearch/bench/IO.h
function namespace (line 35) | namespace benchIO {
FILE: rangeSearch/bench/get_time.h
function namespace (line 9) | namespace cpam {
FILE: rangeSearch/bench/rangeTime.C
function main (line 61) | int main(int argc, char* argv[]) {
FILE: rangeSearch/vamanaRange/range.h
function namespace (line 40) | namespace parlayANN{
Condensed preview — 157 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (686K chars).
[
{
"path": ".bazelrc",
"chars": 2754,
"preview": "# This is from Bazel's former travis setup, to avoid blowing up the RAM usage.\nstartup --host_jvm_args=-Xmx2500m\nstartup"
},
{
"path": ".gitmodules",
"chars": 92,
"preview": "[submodule \"parlaylib\"]\n\tpath = parlaylib\n\turl = https://github.com/cmuparlay/parlaylib.git\n"
},
{
"path": "CMakeLists.txt",
"chars": 2298,
"preview": "cmake_minimum_required(VERSION 3.15)\nproject(PARLAYANN VERSION 1\n DESCRIPTION \"ParlayANN is a library of approxim"
},
{
"path": "LICENSE",
"chars": 1071,
"preview": "MIT License\n\nCopyright (c) 2023 magdalendobson\n\nPermission is hereby granted, free of charge, to any person obtaining a "
},
{
"path": "WORKSPACE",
"chars": 656,
"preview": "load(\"@bazel_tools//tools/build_defs/repo:http.bzl\", \"http_archive\")\nload(\"@bazel_tools//tools/cpp:cc_configure.bzl\", \"c"
},
{
"path": "algorithms/CMakeLists.txt",
"chars": 132,
"preview": "add_subdirectory(HCNNG)\nadd_subdirectory(HNSW)\nadd_subdirectory(pyNNDescent)\nadd_subdirectory(vamana)\nadd_subdirectory(v"
},
{
"path": "algorithms/HCNNG/CMakeLists.txt",
"chars": 179,
"preview": "add_executable(neighbors-hcnng ../bench/neighborsTime.C)\n target_link_libraries(neighbors-hcnng PRIVATE parlay)\n targe"
},
{
"path": "algorithms/HCNNG/Makefile",
"chars": 164,
"preview": "include ../bench/parallelDefsANN \n\nREQUIRE = ../utils/beamSearch.h hcnng_index.h ../utils/graph.h clusterEdge.h\nBENCH"
},
{
"path": "algorithms/HCNNG/clusterEdge.h",
"chars": 5890,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/HCNNG/hcnng_index.h",
"chars": 8989,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/HCNNG/neighbors.h",
"chars": 2488,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/HCNNG/scripts/fashion",
"chars": 450,
"preview": "# bash\n\nnumactl -i all ./neighbors -R 64 -L 128 -alpha 1.15 -data_type float -file_type bin -dist_func Euclidian -base_p"
},
{
"path": "algorithms/HCNNG/scripts/gist_1",
"chars": 285,
"preview": "# bash\n\nnumactl -i all ./neighbors -R 100 -L 200 -alpha 1.1 -data_type float -file_type bin -dist_func Euclidian -base_p"
},
{
"path": "algorithms/HCNNG/scripts/glove100",
"chars": 400,
"preview": "# bash\n\nnumactl -i all ./neighbors -R 150 -L 300 -alpha 1 -data_type float -file_type bin -dist_func mips -base_path dat"
},
{
"path": "algorithms/HCNNG/scripts/glove25",
"chars": 338,
"preview": "# bash\n\nnumactl -i all ./neighbors -R 150 -L 300 -alpha 1 -data_type float -file_type bin -dist_func mips -base_path dat"
},
{
"path": "algorithms/HCNNG/scripts/nytimes",
"chars": 616,
"preview": "# bash\nBUILD_ARGS=\"-cluster_size 1000 -mst_deg 3 -num_clusters 30 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-quantize_bits "
},
{
"path": "algorithms/HCNNG/scripts/sift",
"chars": 607,
"preview": "# bash\nBUILD_ARGS=\"-cluster_size 1000 -mst_deg 3 -num_clusters 30 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-quantize_bits "
},
{
"path": "algorithms/HNSW/CMakeLists.txt",
"chars": 179,
"preview": "# add_executable(neighbors-hnsw ../bench/neighborsTime.C)\n# target_link_libraries(neighbors-hnsw PRIVATE parlay)\n# t"
},
{
"path": "algorithms/HNSW/HNSW.hpp",
"chars": 47637,
"preview": "#ifndef _HNSW_HPP\n#define _HNSW_HPP\n\n#include <cstdint>\n#include <cstdio>\n#include <cstdarg>\n#include <cassert>\n#include"
},
{
"path": "algorithms/HNSW/debug.hpp",
"chars": 767,
"preview": "#ifndef __DEBUG_HPP__\n#define __DEBUG_HPP__\n\nextern parlay::sequence<parlay::sequence<std::array<float,5>>> dist_in_sear"
},
{
"path": "algorithms/HNSW/dist.hpp",
"chars": 2115,
"preview": "#ifndef __DIST_HPP__\n#define __DIST_HPP__\n\n#include <type_traits>\n#include \"type_point.hpp\"\n#include \"../utils/NSGDist.h"
},
{
"path": "algorithms/HNSW/h5_ops.hpp",
"chars": 2070,
"preview": "#ifndef __H5_OPS_HPP__\n#define __H5_OPS_HPP__\n\n#include <cstdio>\n#include <array>\n#include <tuple>\n#include <memory>\n#in"
},
{
"path": "algorithms/HNSW/type_point.hpp",
"chars": 9159,
"preview": "#ifndef __TYPE_POINT_HPP__\n#define __TYPE_POINT_HPP__\n\n#include <cstdint>\n#include <cstddef>\n#include <iterator>\n#includ"
},
{
"path": "algorithms/bench/BUILD",
"chars": 134,
"preview": "package(default_visibility = [\"//visibility:public\"])\n\ncc_library(\n name = \"parse_command_line\",\n hdrs = [\"parse_comma"
},
{
"path": "algorithms/bench/IO.h",
"chars": 7181,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/MakeBench",
"chars": 594,
"preview": "# ********************\n# GENERIC MAKEFILE FOR MOST BENCHMARKS THAT #include <name>.h\n# USES FOLLOWING DEFINITIONS\n# B"
},
{
"path": "algorithms/bench/Makefile",
"chars": 300,
"preview": "include parallelDefsANN\r\nBNCHMRK = neighbors\r\n\r\nCHECKFILES = $(BNCHMRK)Check.o\r\n\r\nCOMMON =\r\n\r\nINCLUDE = -Icommon\r\n\r\n%.o "
},
{
"path": "algorithms/bench/benchUtils.h",
"chars": 7876,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/IO.h",
"chars": 7193,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/MakeBench",
"chars": 734,
"preview": "# ********************\n# GENERIC MAKEFILE FOR MOST BENCHMARKS THAT #include <name>.h\n# USES FOLLOWING DEFINITIONS\n# B"
},
{
"path": "algorithms/bench/common/MakeBenchLink",
"chars": 1046,
"preview": "# ********************\n# GENERIC MAKEFILE FOR MOST BENCHMARKS THAT LINK\n# THE TIMING CODE WITH THE IMPLEMENTATION\n# USES"
},
{
"path": "algorithms/bench/common/atomics.h",
"chars": 2503,
"preview": "#pragma once\n\nnamespace pbbs {\n\n template <typename ET>\n inline bool atomic_compare_and_swap(ET* a, ET oldval, ET newv"
},
{
"path": "algorithms/bench/common/dataGen.h",
"chars": 877,
"preview": "#pragma once\n#include \"../parlay/utilities.h\"\n\nnamespace dataGen {\n\n#define HASH_MAX_INT ((unsigned) 1 << 31)\n\n //#defi"
},
{
"path": "algorithms/bench/common/geometry.h",
"chars": 11008,
"preview": "#pragma once\n#include <iostream>\n#include <algorithm>\n#include <math.h>\n#include <iomanip>\n#include \"../parlay/parallel."
},
{
"path": "algorithms/bench/common/geometryIO.h",
"chars": 6560,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/get_time.h",
"chars": 1668,
"preview": "#pragma once\n\n#include <stdlib.h>\n#include <sys/time.h>\n#include <iomanip>\n#include <iostream>\n#include <string>\n\nstruct"
},
{
"path": "algorithms/bench/common/glue.h",
"chars": 3128,
"preview": "#pragma once\n\n#include \"../pbbslib/hash_table.h\"\n#include \"../pbbslib/integer_sort.h\"\n\nusing intT = int;\nusing uintT = u"
},
{
"path": "algorithms/bench/common/graph.h",
"chars": 6747,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/graphIO.h",
"chars": 16515,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2010 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/graphUtils.h",
"chars": 9613,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/ligraLight.h",
"chars": 5222,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/parallelDefs",
"chars": 593,
"preview": "ifeq (, $(shell which jemalloc-config))\nJEMALLOC =\nelse\nJEMALLOCLD = $(shell jemalloc-config --libdir)\nJEMALLOC = -L$(JE"
},
{
"path": "algorithms/bench/common/parallelDefsANN",
"chars": 635,
"preview": "ifeq (, $(shell which jemalloc-config))\r\nJEMALLOC =\r\nelse\r\nJEMALLOCLD = $(shell jemalloc-config --libdir)\r\nJEMALLOC = -L"
},
{
"path": "algorithms/bench/common/parallelDefs_OMP",
"chars": 577,
"preview": "ifeq (, $(shell which jemalloc-config))\nJEMALLOC =\nelse\nJEMALLOCLD = $(shell jemalloc-config --libdir)\nJEMALLOC = -L$(JE"
},
{
"path": "algorithms/bench/common/parseCommandLine.h",
"chars": 3484,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/parse_command_line.h",
"chars": 3579,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/runTests.py",
"chars": 10842,
"preview": "import subprocess\nimport sys\nimport random\nimport os\n\ndef onPprocessors(command,p) :\n if \"OPENMP\" in os.environ:\n re"
},
{
"path": "algorithms/bench/common/runTestsANN.py",
"chars": 5623,
"preview": "import subprocess\nimport sys\nimport random\nimport os\n\ndef addLineToFile(oFile, line):\n with open(\"oFile\", \"a+\") as fi"
},
{
"path": "algorithms/bench/common/seqDefs",
"chars": 282,
"preview": "ifeq (, $(shell which jemalloc-config))\nJEMALLOC =\nelse\nJEMALLOCLD = $(shell jemalloc-config --libdir)\nJEMALLOC = -L$(JE"
},
{
"path": "algorithms/bench/common/sequenceIO.h",
"chars": 7138,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/speculative_for.h",
"chars": 4593,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/time_loop.h",
"chars": 432,
"preview": "#include \"../parlay/internal/get_time.h\"\n\ntemplate<class F, class G, class H>\nvoid time_loop(int rounds, double delay, F"
},
{
"path": "algorithms/bench/common/topology.h",
"chars": 7448,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/common/topology_from_triangles.h",
"chars": 5363,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/get_time.h",
"chars": 1611,
"preview": "#pragma once\n\n#include <stdlib.h>\n#include <sys/time.h>\n#include <iomanip>\n#include <iostream>\n#include <string>\n\nnamesp"
},
{
"path": "algorithms/bench/neighborsTime.C",
"chars": 11403,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/parallelDefsANN",
"chars": 635,
"preview": "ifeq (, $(shell which jemalloc-config))\r\nJEMALLOC =\r\nelse\r\nJEMALLOCLD = $(shell jemalloc-config --libdir)\r\nJEMALLOC = -L"
},
{
"path": "algorithms/bench/parse_command_line.h",
"chars": 3579,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/bench/time_loop.h",
"chars": 456,
"preview": "#include \"get_time.h\"\n\n#ifndef TIMELOOP\n#define TIMELOOP\n\ntemplate<class F, class G, class H>\nvoid time_loop(int rounds,"
},
{
"path": "algorithms/pyNNDescent/CMakeLists.txt",
"chars": 197,
"preview": "add_executable(neighbors-pynndescent ../bench/neighborsTime.C)\n target_link_libraries(neighbors-pynndescent PRIVATE par"
},
{
"path": "algorithms/pyNNDescent/Makefile",
"chars": 156,
"preview": "include ../bench/parallelDefsANN\n\nREQUIRE = ../utils/beamSearch.h pynn_index.h ../utils/graph.h clusterPynn.h\nBENCH = n"
},
{
"path": "algorithms/pyNNDescent/clusterPynn.h",
"chars": 6521,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/pyNNDescent/neighbors.h",
"chars": 2497,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/pyNNDescent/pynn_index.h",
"chars": 10876,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/pyNNDescent/scripts/nytimes",
"chars": 629,
"preview": "# bash\nBUILD_ARGS=\"-R 40 -cluster_size 100 -num_clusters 10 -alpha 1.2 -delta 0.05 -quantize_bits 8 -verbose\"\nQUERY_ARGS"
},
{
"path": "algorithms/pyNNDescent/scripts/sift",
"chars": 620,
"preview": "# bash\nBUILD_ARGS=\"-R 40 -cluster_size 100 -num_clusters 10 -alpha 1.2 -delta 0.05 -quantize_bits 8 -verbose\"\nQUERY_ARGS"
},
{
"path": "algorithms/tutorial.sh",
"chars": 1009,
"preview": "#!/bin/bash\n\ncd vamana\nmake\necho \"Vamana:\"\n./neighbors -R 32 -L 64 -a 1.2 -graph_outfile ../../data/sift/sift_learn_32_6"
},
{
"path": "algorithms/utils/BUILD",
"chars": 3152,
"preview": "# ANNS utilility.\n\npackage(default_visibility = [\"//algorithms:__subpackages__\"])\n\n\ncc_library(\n name = \"csvfile\",\n "
},
{
"path": "algorithms/utils/NSGDist.h",
"chars": 9259,
"preview": "//\n// Created by 付聪 on 2017/6/21.\n//\n\n#ifndef EFANNA2E_DISTANCE_H\n#define EFANNA2E_DISTANCE_H\n\n#include <math.h>\n#includ"
},
{
"path": "algorithms/utils/beamSearch.h",
"chars": 25848,
"preview": "#ifndef ALGORITHMS_ANN_BEAM_SEARCH_H_\n#define ALGORITHMS_ANN_BEAM_SEARCH_H_\n\n#include <algorithm>\n#include <functional>\n"
},
{
"path": "algorithms/utils/check_nn_recall.h",
"chars": 11027,
"preview": "#ifndef ALGORITHMS_CHECK_NN_RECALL_H_\n#define ALGORITHMS_CHECK_NN_RECALL_H_\n\n#include <algorithm>\n#include <set>\n\n#inclu"
},
{
"path": "algorithms/utils/check_range_recall.h",
"chars": 6108,
"preview": "#include <algorithm>\n#include <set>\n\n#include \"beamSearch.h\"\n#include \"doublingSearch.h\"\n#include \"rangeSearch.h\"\n#inclu"
},
{
"path": "algorithms/utils/csvfile.h",
"chars": 3264,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/utils/doublingSearch.h",
"chars": 4628,
"preview": "#include <algorithm>\n#include <functional>\n#include <random>\n#include <set>\n#include <unordered_set>\n#include <queue>\n\n#"
},
{
"path": "algorithms/utils/earlyStopping.h",
"chars": 970,
"preview": "#pragma once\n#include <algorithm>\n#include <functional>\n#include <random>\n#include <set>\n#include <unordered_set>\n#inclu"
},
{
"path": "algorithms/utils/euclidian_point.h",
"chars": 13213,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/utils/graph.h",
"chars": 8390,
"preview": "// This code is part of the Parlay Project\n// Copyright (c) 2024 Guy Blelloch, Magdalen Dobson and the Parlay team\n//\n//"
},
{
"path": "algorithms/utils/graph_reorder.h",
"chars": 4698,
"preview": "#include <atomic>\n#include <limits>\n#include <optional>\n#include <tuple>\n#include <utility>\n\n#include <parlay/delayed.h>"
},
{
"path": "algorithms/utils/hashset.h",
"chars": 1686,
"preview": "#ifndef ALGORITHMS_ANN_HASHSET_H_\n#define ALGORITHMS_ANN_HASHSET_H_\n\n#include <vector>\n#include <cmath>\nnamespace parlay"
},
{
"path": "algorithms/utils/jl_point.h",
"chars": 11297,
"preview": "#pragma once\n\n#include <algorithm>\n#include <iostream>\n#include <bitset>\n\n#include \"parlay/parallel.h\"\n#include \"parlay/"
},
{
"path": "algorithms/utils/mips_point.h",
"chars": 25190,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/utils/mmap.h",
"chars": 910,
"preview": "#ifndef ALGORITHMS_ANN_MMAP_H_\n#define ALGORITHMS_ANN_MMAP_H_\n\n#include <algorithm>\n#include <iostream>\n\n#include \"parla"
},
{
"path": "algorithms/utils/parse_results.h",
"chars": 6042,
"preview": "#ifndef ALGORITHMS_UTILS_PARSE_RESULTS_H_\n#define ALGORITHMS_UTILS_PARSE_RESULTS_H_\n\n#include <algorithm>\n#include <set>"
},
{
"path": "algorithms/utils/point_range.h",
"chars": 4959,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/utils/rangeSearch.h",
"chars": 9897,
"preview": "#include <algorithm>\n#include <functional>\n#include <random>\n#include <set>\n#include <unordered_set> \n#include <queue>\n"
},
{
"path": "algorithms/utils/simpleGraph.h",
"chars": 4255,
"preview": "// This code is part of the Parlay Project\n// Copyright (c) 2024 Guy Blelloch, Magdalen Dobson and the Parlay team\n//\n//"
},
{
"path": "algorithms/utils/stats.h",
"chars": 10865,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/utils/types.h",
"chars": 9565,
"preview": "#ifndef ALGORITHMS_ANN_TYPES_H_\n#define ALGORITHMS_ANN_TYPES_H_\n\n// This code is part of the Problem Based Benchmark Sui"
},
{
"path": "algorithms/utils/union.h",
"chars": 3000,
"preview": "#ifndef ALGORITHMS_ANN_UNION_\n#define ALGORITHMS_ANN_UNION_\n\n#include <set>\n#include \"parlay/parallel.h\"\n#include \"parla"
},
{
"path": "algorithms/vamana/BUILD",
"chars": 1791,
"preview": "# Vamana algorithm.\n\ncc_library(\n name = \"index\",\n hdrs = [\"index.h\"],\n deps = [\n \"@parlaylib//parlay:de"
},
{
"path": "algorithms/vamana/CMakeLists.txt",
"chars": 357,
"preview": "function(add_neighbors NAME DEFS)\n add_executable(${NAME} ../bench/neighborsTime.C)\n target_link_libraries(${NAME} PRI"
},
{
"path": "algorithms/vamana/Makefile",
"chars": 337,
"preview": "include ../bench/parallelDefsANN\n\nREQUIRE = ../utils/beamSearch.h index.h ../utils/check_nn_recall.h ../utils/NSGDist.h"
},
{
"path": "algorithms/vamana/index.h",
"chars": 12310,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/vamana/index_test.cc",
"chars": 130,
"preview": "#include \"algorithms/vamana/index.h\"\n\n#include <gtest/gtest.h>\n\nTEST(PlaceHolderTest, BuildPlaceHolder) { EXPECT_EQ(7 * "
},
{
"path": "algorithms/vamana/neighbors.h",
"chars": 7852,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/vamana/neighbors.sh",
"chars": 1843,
"preview": "#!/bin/bash\ncd ~/ParlayANN/algorithms/vamana\nmake \n\nP=/ssd1/data/bigann\n# ./neighbors -R 64 -L 128 -alpha 1.2 -data_type"
},
{
"path": "algorithms/vamana/neighbors_test.cc",
"chars": 134,
"preview": "#include \"algorithms/vamana/neighbors.h\"\n\n#include <gtest/gtest.h>\n\nTEST(PlaceHolderTest, BuildPlaceHolder) { EXPECT_EQ("
},
{
"path": "algorithms/vamana/scripts/OpenAIArXiv",
"chars": 633,
"preview": "# bash\nNAME=OpenAIArXiv\nBUILD_ARGS=\"-R 100 -L 200 -alpha 1.05 -num_passes 2 -quantize_mode 3 -verbose\"\nQUERY_ARGS=\"-quan"
},
{
"path": "algorithms/vamana/scripts/deep10M",
"chars": 626,
"preview": "# bash\nNAME=deep10M\nBUILD_ARGS=\"-R 64 -L 128 -alpha 1.05 -num_passes 2 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-quantize_"
},
{
"path": "algorithms/vamana/scripts/fashion",
"chars": 610,
"preview": "# bash\nNAME=fashion-mnist-784-euclidean\nBUILD_ARGS=\"-R 40 -L 80 -alpha 1.1 -num_passes 2 -quantize_bits 8 -verbose\"\nQUER"
},
{
"path": "algorithms/vamana/scripts/gist",
"chars": 623,
"preview": "# bash\nNAME=gist\nBUILD_ARGS=\"-R 100 -L 200 -alpha 1.1 -num_passes 2 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-quantize_bit"
},
{
"path": "algorithms/vamana/scripts/glove100",
"chars": 639,
"preview": "# bash\nNAME=glove-100-angular\nBUILD_ARGS=\"-R 100 -L 200 -alpha 1 -num_passes 2 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-q"
},
{
"path": "algorithms/vamana/scripts/glove25",
"chars": 638,
"preview": "# bash\nNAME=glove-25-angular\nBUILD_ARGS=\"-R 100 -L 200 -alpha 1 -num_passes 2 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-qu"
},
{
"path": "algorithms/vamana/scripts/msmarco_websearch",
"chars": 614,
"preview": "# bash\nBUILD_ARGS=\"-R 64 -L 128 -alpha 1 -num_passes 1 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-quantize_bits 16 -quantiz"
},
{
"path": "algorithms/vamana/scripts/nytimes",
"chars": 633,
"preview": "# bash\nBUILD_ARGS=\"-R 130 -L 260 -alpha .85 -num_passes 2 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-quantize_bits 16 -quan"
},
{
"path": "algorithms/vamana/scripts/sift",
"chars": 593,
"preview": "# bash\nBUILD_ARGS=\"-R 64 -L 128 -alpha 1.15 -num_passes 2 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-quantize_bits 8 -verbo"
},
{
"path": "algorithms/vamana/scripts/sift100",
"chars": 584,
"preview": "# bash\nBUILD_ARGS=\"-R 64 -L 128 -alpha 1.15 -num_passes 2 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-quantize_bits 8 -verbo"
},
{
"path": "algorithms/vamana/scripts/space_1",
"chars": 556,
"preview": "# bash\nNAME=space_1\nBUILD_ARGS=\"-R 64 -L 128 -alpha 1.1 -num_passes 2 -verbose\"\nQUERY_ARGS=\"-verbose\"\nTYPE_ARGS=\"-data_t"
},
{
"path": "algorithms/vamana/scripts/space_10",
"chars": 557,
"preview": "# bash\nNAME=space_10\nBUILD_ARGS=\"-R 64 -L 128 -alpha 1.1 -num_passes 2 -verbose\"\nQUERY_ARGS=\"-verbose\"\nTYPE_ARGS=\"-data_"
},
{
"path": "algorithms/vamana/scripts/t2i_1",
"chars": 615,
"preview": "# bash\nNAME=t2i_1\nBUILD_ARGS=\"-R 100 -L 200 -alpha 1 -num_passes 2 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-quantize_bits"
},
{
"path": "algorithms/vamana/scripts/t2i_10",
"chars": 616,
"preview": "# bash\nNAME=t2i_10\nBUILD_ARGS=\"-R 100 -L 200 -alpha 1 -num_passes 2 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-quantize_bit"
},
{
"path": "algorithms/vamana/scripts/wikipedia_cohere",
"chars": 615,
"preview": "# bash\nBUILD_ARGS=\"-R 64 -L 128 -alpha .98 -num_passes 1 -quantize_bits 8 -verbose\"\nQUERY_ARGS=\"-quantize_bits 16 -quant"
},
{
"path": "algorithms/vamanaRange/CMakeLists.txt",
"chars": 218,
"preview": "# TODO: fix build\n#add_executable(neighbors-vamanaRange ../bench/neighborsTime.C)\n# target_link_libraries(neighbors-vam"
},
{
"path": "algorithms/vamanaRange/Makefile",
"chars": 232,
"preview": "include ../bench/parallelDefsANN\n\nREQUIRE = ../utils/beamSearch.h index.h ../utils/check_nn_recall.h ../utils/NSGDist.h"
},
{
"path": "algorithms/vamanaRange/index.h",
"chars": 10674,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "algorithms/vamanaRange/neighbors.h",
"chars": 5324,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "build/_deps/parlaylib-subbuild/CMakeLists.txt",
"chars": 1796,
"preview": "# Distributed under the OSI-approved BSD 3-Clause License. See accompanying\n# file Copyright.txt or https://cmake.org/l"
},
{
"path": "data_tools/Makefile",
"chars": 557,
"preview": "include ../algorithms/bench/parallelDefsANN\n\nvec_to_bin : vec_to_bin.cpp\n\t$(CC) $(CFLAGS) -o vec_to_bin vec_to_bin.cpp $"
},
{
"path": "data_tools/compute_groundtruth.cpp",
"chars": 6493,
"preview": "/*\n Example usage:\n ./compute_groundtruth -base_path ~/data/sift/sift-1M \\\n -query_path ~/data/sift/query-10K -da"
},
{
"path": "data_tools/compute_range_groundtruth.cpp",
"chars": 6290,
"preview": "#include <iostream>\n#include <algorithm>\n#include \"parlay/parallel.h\"\n#include \"parlay/primitives.h\"\n#include \"parlay/io"
},
{
"path": "data_tools/crop.cpp",
"chars": 2194,
"preview": "#include <iostream>\n#include <algorithm>\n#include \"parlay/parallel.h\"\n#include \"parlay/primitives.h\"\n#include \"parlay/io"
},
{
"path": "data_tools/random_sample.cpp",
"chars": 1835,
"preview": "#include <iostream>\n#include <algorithm>\n#include \"parlay/parallel.h\"\n#include \"parlay/primitives.h\"\n#include \"parlay/io"
},
{
"path": "data_tools/vec_to_bin.cpp",
"chars": 1781,
"preview": "#include <iostream>\n#include <algorithm>\n#include \"parlay/parallel.h\"\n#include \"parlay/primitives.h\"\n#include \"parlay/io"
},
{
"path": "docs/README.md",
"chars": 2520,
"preview": "# ParlayANN\n\nParlayANN is a library of approximate nearest neighbor search algorithms, along with a set of useful tools "
},
{
"path": "docs/algorithms.md",
"chars": 11714,
"preview": "# Algorithms\n\nThe algorithms in this folder share a common main file and thus a common commandline interface. The comman"
},
{
"path": "docs/data_tools.md",
"chars": 3751,
"preview": "# Data Tools\n\nParlayANN provides various useful tools for manipulating and reading datasets in common formats. For all o"
},
{
"path": "docs/quickstart.md",
"chars": 5666,
"preview": "# Quickstart\n\nThe following is a crash course in quickly building and querying an index using ParlayANN.\n\nFirst, downloa"
},
{
"path": "docs/rangesearch.md",
"chars": 2134,
"preview": "# Range Search \n\nRange search is defined as finding every point within a specified radius of a query point with respect "
},
{
"path": "python/__init__.py",
"chars": 6504,
"preview": "# // This code is part of the Problem Based Benchmark Suite (PBBS)\n# // Copyright (c) 2011 Guy Blelloch and the PBBS tea"
},
{
"path": "python/_builder.py",
"chars": 4770,
"preview": "# // This code is part of the Problem Based Benchmark Suite (PBBS)\n# // Copyright (c) 2011 Guy Blelloch and the PBBS tea"
},
{
"path": "python/_builder.pyi",
"chars": 2186,
"preview": "# // This code is part of the Problem Based Benchmark Suite (PBBS)\n# // Copyright (c) 2011 Guy Blelloch and the PBBS tea"
},
{
"path": "python/_common.py",
"chars": 7888,
"preview": "# // This code is part of the Problem Based Benchmark Suite (PBBS)\n# // Copyright (c) 2011 Guy Blelloch and the PBBS tea"
},
{
"path": "python/_files.py",
"chars": 3587,
"preview": "# // This code is part of the Problem Based Benchmark Suite (PBBS)\n# // Copyright (c) 2011 Guy Blelloch and the PBBS tea"
},
{
"path": "python/big_env.yml",
"chars": 3078,
"preview": "name: bigann\nchannels:\n - conda-forge\n - defaults\ndependencies:\n - _libgcc_mutex=0.1=conda_forge\n - _openmp_mutex=4."
},
{
"path": "python/builder.cpp",
"chars": 12142,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "python/compile.sh",
"chars": 555,
"preview": "# g++ -DSTATS -DHOMEGROWN -pthread -mcx16 -O3 -Wall -shared -std=c++17 -march=native -DNDEBUG -I . -fPIC $(python3 -m py"
},
{
"path": "python/defaults.py",
"chars": 2862,
"preview": "# // This code is part of the Problem Based Benchmark Suite (PBBS)\n# // Copyright (c) 2011 Guy Blelloch and the PBBS tea"
},
{
"path": "python/graph_index.cpp",
"chars": 13357,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "python/module.cpp",
"chars": 8637,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "python/scripts/fashion_test.py",
"chars": 788,
"preview": "import _ParlayANNpy as pann\nimport wrapper as wp\nimport time\n\nNAME = \"fashion-mnist-784-euclidean\"\nDATA_DIR = \"data/\" + "
},
{
"path": "python/scripts/gist_test.py",
"chars": 735,
"preview": "import _ParlayANNpy as pann\nimport wrapper as wp\nimport time\n\nNAME = \"gist\"\nDATA_DIR = \"data/\" + NAME + \"/\"\nmetric = \"Eu"
},
{
"path": "python/scripts/glove100_test.py",
"chars": 751,
"preview": "import _ParlayANNpy as pann\nimport wrapper as wp\nimport time\n\nNAME = \"glove-100-angular\"\nDATA_DIR = \"data/\" + NAME + \"/\""
},
{
"path": "python/scripts/glove25_test.py",
"chars": 749,
"preview": "import _ParlayANNpy as pann\nimport wrapper as wp\nimport time\n\nNAME = \"glove-25-angular\"\nDATA_DIR = \"data/\" + NAME + \"/\"\n"
},
{
"path": "python/scripts/nyt_test.py",
"chars": 750,
"preview": "import _ParlayANNpy as pann\nimport wrapper as wp\nimport time\n\nNAME = \"nytimes-256-angular\"\nDATA_DIR = \"data/\" + NAME + \""
},
{
"path": "python/scripts/sift_test.py",
"chars": 1070,
"preview": "import _ParlayANNpy as pann\nimport wrapper as wp\nimport time\n\nNAME = \"sift-128-euclidean\"\nDATA_DIR = \"data/\" + NAME + \"/"
},
{
"path": "python/sift_test.py",
"chars": 733,
"preview": "import _ParlayANNpy as pann\nimport wrapper as wp\nimport time\n\nNAME = \"sift-128-euclidean\"\nDATA_DIR = \"data/\" + NAME + \"/"
},
{
"path": "python/test.py",
"chars": 1656,
"preview": "import wrapper as wp\n\n\nFERN_DATA_DIR = \"/ssd1/anndata/bigann/\"\nAWARE_DATA_DIR = \"/ssd1/data/bigann/\"\n\nDATA_DIR = FERN_DA"
},
{
"path": "python/wrapper.py",
"chars": 5648,
"preview": "from _ParlayANNpy import *\n\ndef build_vamana_index(metric, dtype, data_dir, index_dir, R, L, alpha, two_pass):\n if me"
},
{
"path": "rangeSearch/bench/.gitignore",
"chars": 32,
"preview": "neighborsCheck\nneighborsCheck.o\n"
},
{
"path": "rangeSearch/bench/IO.h",
"chars": 7181,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "rangeSearch/bench/MakeBench",
"chars": 567,
"preview": "# ********************\n# GENERIC MAKEFILE FOR MOST BENCHMARKS THAT #include <name>.h\n# USES FOLLOWING DEFINITIONS\n# B"
},
{
"path": "rangeSearch/bench/Makefile",
"chars": 296,
"preview": "include parallelDefsANN\r\nBNCHMRK = range\r\n\r\nCHECKFILES = $(BNCHMRK)Check.o\r\n\r\nCOMMON =\r\n\r\nINCLUDE = -Icommon\r\n\r\n%.o : %."
},
{
"path": "rangeSearch/bench/get_time.h",
"chars": 1611,
"preview": "#pragma once\n\n#include <stdlib.h>\n#include <sys/time.h>\n#include <iomanip>\n#include <iostream>\n#include <string>\n\nnamesp"
},
{
"path": "rangeSearch/bench/parallelDefsANN",
"chars": 636,
"preview": "ifeq (, $(shell which jemalloc-config))\r\nJEMALLOC =\r\nelse\r\nJEMALLOCLD = $(shell jemalloc-config --libdir)\r\nJEMALLOC = -L"
},
{
"path": "rangeSearch/bench/rangeTime.C",
"chars": 11328,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
},
{
"path": "rangeSearch/vamanaRange/Makefile",
"chars": 390,
"preview": "include ../bench/parallelDefsANN\n\nREQUIRE = ../utils/beamSearch.h ../utils/doublingSearch.h ../../algorithms/vamana/inde"
},
{
"path": "rangeSearch/vamanaRange/range.h",
"chars": 4346,
"preview": "// This code is part of the Problem Based Benchmark Suite (PBBS)\n// Copyright (c) 2011 Guy Blelloch and the PBBS team\n//"
}
]
About this extraction
This page contains the full source code of the cmuparlay/ParlayANN GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 157 files (633.9 KB), approximately 187.1k tokens, and a symbol index with 394 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.