Showing preview only (432K chars total). Download the full file or copy to clipboard to get everything.
Repository: ease-lab/Hermes
Branch: master
Commit: 949229c23881
Files: 56
Total size: 412.5 KB
Directory structure:
gitextract_es11wv71/
├── .clang-format
├── .gitignore
├── AUTHORS
├── CMakeLists.txt
├── LICENSE
├── README.md
├── bin/
│ ├── copy-exec-files.sh
│ ├── copy-n-exec-hermesKV.sh
│ ├── copy-n-exec-rCRAQ.sh
│ ├── copy-traces.sh
│ ├── csv_latency_parser.py
│ ├── exec-derecho.sh
│ ├── format.sh
│ ├── get-system-xput-files.sh
│ ├── setup.sh
│ └── trace-spliter.sh
├── exec/
│ ├── Makefile
│ ├── hosts.sh
│ ├── results/
│ │ ├── latency/
│ │ │ └── .gitinclude
│ │ └── xput/
│ │ ├── all-nodes/
│ │ │ └── .gitkeep
│ │ └── per-node/
│ │ └── .gitkeep
│ ├── run-hades.sh
│ ├── run-hermesKV.sh
│ ├── run-rCRAQ.sh
│ └── run.sh
├── include/
│ ├── hades/
│ │ └── hades.h
│ ├── hermes/
│ │ ├── config.h
│ │ ├── inline-util.h
│ │ ├── spacetime.h
│ │ └── util.h
│ ├── mica-herd/
│ │ ├── city.h
│ │ ├── hrd.h
│ │ ├── mica.h
│ │ └── sizes.h
│ ├── utils/
│ │ ├── bit_vector.h
│ │ ├── concur_ctrl.h
│ │ └── time_rdtsc.h
│ └── wings/
│ ├── wings.h
│ └── wings_api.h
├── src/
│ ├── CR/
│ │ ├── crKV.c
│ │ └── cr_worker.c
│ ├── hades/
│ │ ├── hades.c
│ │ └── test.c
│ ├── hermes/
│ │ ├── hermesKV.c
│ │ ├── hermes_worker.c
│ │ ├── main.c
│ │ ├── spacetime.c
│ │ ├── stats.c
│ │ └── util.c
│ ├── mica-herd/
│ │ ├── city.c
│ │ ├── herd.c
│ │ └── mica.c
│ └── wings/
│ └── wings.c
└── tla/
├── Hermes.tla
├── HermesRMWs.tla
└── README.md
================================================
FILE CONTENTS
================================================
================================================
FILE: .clang-format
================================================
---
BasedOnStyle: Chromium
AlignAfterOpenBracket: Align
AlignConsecutiveDeclarations: 'false'
AlignEscapedNewlines: Left
AlignOperands: 'true'
AllowShortFunctionsOnASingleLine: All
AllowShortIfStatementsOnASingleLine: WithoutElse
AlwaysBreakAfterDefinitionReturnType: TopLevel
AlwaysBreakTemplateDeclarations: 'Yes'
BinPackArguments: 'true'
BinPackParameters: 'true'
BreakBeforeBraces: WebKit
CompactNamespaces: 'false'
Cpp11BracedListStyle: 'true'
IndentWrappedFunctionNames: 'false'
Language: Cpp
NamespaceIndentation: None
SpaceAfterTemplateKeyword: 'true'
SpaceBeforeAssignmentOperators: 'true'
SpaceBeforeCpp11BracedList: 'true'
SpaceBeforeParens: ControlStatements
SpaceInEmptyParentheses: 'false'
SpacesInAngles: 'false'
SpacesInParentheses: 'false'
SpacesInSquareBrackets: 'false'
UseTab: Never
...
================================================
FILE: .gitignore
================================================
# ignore temporary files
.*.swp
\#*#
*.pyc
*.o
*.hi
*.dump
*.log
*.rej
*.orig
*.patch
*.diff
.tags*
# ignore executables
/src/mica/test
/src/libhrd/main
/src/herd-hybrid/main
/src/herd-UD/main
src/Armonia/main
/src/CR/cr
/src/hermes/hermes
/src/hades/hades
/src/hermes/hermes-wings
# ignore debug files
/debug/*.txt
# ignore traces
/traces/*.txt
# ignore ide files
/.idea/
/cmake-build-debug/
/src/cmake-build-debug/
/src/.idea/
/src/cache/cmake-build-debug/
/src/cache/.idea/
/src/Armonia/armonia-ec
/src/Armonia/armonia-sc
/src/Armonia/throughput.txt
/src/herd-UD/throughput.txt
/bin/traces
#/results/*
/exec/results/*.txt
/exec/results/xput/*.txt
/exec/results/xput/*.csv
/exec/results/xput/per-node/*.csv
/exec/results/xput/per-node/*.txt
/exec/results/xput/all-nodes/*.txt
/exec/results/latency/*.txt
/exec/results/latency/*.csv
/results/*.txt
/results/xput/*.txt
/results/xput/*.csv
/results/xput/per-node/*.csv
/results/xput/per-node/*.txt
/results/xput/all-nodes/*.txt
/results/latency/*.txt
/results/latency/*.csv
traces/trace-parts/*
/results/scattered-results/*
/results/aggregated-system-results/*.csv
/traces/system-traces/*.txt
/traces/current-splited-traces/*.txt
/traces/*.txt
traces/
./exec/hermesKV
./exec/rCRAQ
./exec/hades
================================================
FILE: AUTHORS
================================================
Run `git shortlog -se` for an up-to-date list of contributors.
---
Principal authors: Antonios Katsarakis <antonios.katsarakis AT ed.ac.uk>
Vasilis Gavrielatos <vasilis.gavrielatos AT ed.ac.uk>
================================================
FILE: CMakeLists.txt
================================================
######################################################################################
# WARNING: DO NOT MAKE through cmake use the Makefile in /exec/ to compile instead!!!!
######################################################################################
cmake_minimum_required(VERSION 2.8.12)
project(hermes)
set(Hermes_VERSION_MAJOR 1)
set(Hermes_VERSION_MINOR 0)
include_directories(include/hermes
include/libhrd
/usr/include/
include/optik
include/mica-herd)
set(CMAKE_C_STANDARD 11)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
set(SOURCE_FILES_cr
#Source files
src/CR/cr_worker.c
src/wings/wings.c
src/hermes/main.c
src/hermes/stats.c
src/hermes/spacetime.c
src/mica-herd/mica.c
src/mica-herd/city.c
src/mica-herd/herd.c
##### header files ####
include/wings/wings.h
include/wings/wings_api.h
include/mica-herd/city.h
include/mica-herd/hrd.h
include/mica-herd/sizes.h
include/hermes/util.h
include/hermes/config.h
include/utils/bit_vector.h
include/utils/concur_ctrl.h
src/CR/crKV.c)
set(SOURCE_FILES_hades
#Source files
src/wings/wings.c
src/hades/hades.c
##### header files ####
include/wings/wings_api.h
include/wings/wings.h
include/hades/hades.h
src/hades/test.c)
set(SOURCE_FILES_hermes
#Source files
src/hermes/main.c
src/hermes/util.c
# src/hermes/worker.c
src/hermes/hermes_worker.c
src/hermes/stats.c
src/hermes/spacetime.c
src/mica-herd/herd.c
src/mica-herd/mica.c
src/mica-herd/city.c
src/wings/wings.c
##### header files ####
include/mica-herd/hrd.h
include/mica-herd/city.h
include/mica-herd/sizes.h
include/hermes/util.h
include/hermes/config.h
include/utils/concur_ctrl.h
include/utils/bit_vector.h
include/hades/hades.h
include/wings/wings.h
include/wings/wings_api.h src/hermes/hermesKV.c)
add_executable(cr ${SOURCE_FILES_cr})
add_executable(hades ${SOURCE_FILES_hades})
add_executable(hermes ${SOURCE_FILES_hermes})
target_link_libraries(cr pthread ibverbs rt memcached numa rdmacm)
target_link_libraries(hades pthread ibverbs rt memcached numa rdmacm)
target_link_libraries(hermes pthread ibverbs rt memcached numa rdmacm)
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
# Hermes Reliable Replication Protocol
<img align="left" height="160" src="https://github.com/akatsarakis/Hermes/blob/master/hermes-logo.png">
This is the publicly available artifact repository supporting the ASPLOS'20 paper [_"Hermes: A Fast, Fault-Tolerant and Linearizable Replication Protocol"_](http://hermes-protocol.com "Hermes Arxiv version"). The repository contains both code to experimentally evaluate Hermes(KV) and complete Hermes TLA+ specifications which can be used to verify Hermes correctness via model-checking.
[](https://www.sigarch.org/call-contributions/ieee-micro-top-picks/)
[](https://www.acm.org/publications/policies/artifact-review-badging#available)
[](https://www.acm.org/publications/policies/artifact-review-badging#functional)
[]()
[](http://hermes-protocol.com/)
[](https://github.com/ease-lab/Hermes/blob/master/LICENSE)
[]()
<a href="https://twitter.com/intent/follow?screen_name=ease_lab" target="_blank">
<img src="https://img.shields.io/twitter/follow/ease_lab?style=social&logo=twitter" alt="follow on Twitter"></a>
## Citation
```
@inbook{Katsarakis:20,
author = {Katsarakis, Antonios and Gavrielatos, Vasilis and Katebzadeh, M.R. Siavash and Joshi, Arpit and Dragojevic, Aleksandar and Grot, Boris and Nagarajan, Vijay},
title = {Hermes: A Fast, Fault-Tolerant and Linearizable Replication Protocol},
year = {2020},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
booktitle = {Proceedings of the Twenty-Fifth International Conference on Architectural Support for Programming Languages and Operating Systems},
pages = {201–217},
numpages = {17}
}
```
----
## High Perfomance Features
- _Reads_: i) Local ii) Load-balanced (served by any replica)
- _Updates (Writes and RMWs)_: i) Inter-key concurrent ii) Decentralized iii) Fast (1rtt commit -- any replica)
- _Writes_: iv) Non-conflicting (i.e., never abort)
## Consistency and Properties
Linearizable reads, writes and RMWs with the following properties:
1. _Writes_: from a live replica _always commit_ after Invalidating (and getting acknowledgments from) the rest live replicas.
1. _RMWs_: at most one of possible concurrent RMWs to a key can commit, and this only once all acknowledgments from live replicas are gathered.
1. _Reads_: return the local value if the targeted keys are found in the Valid state and the coordinator was considered live at the time of reading. The later can be ensured locally if the coordinator has a lease for (and is part of) the membership.
## Fault Tolerance
Coupling Invalidations with per-key logical timestamps (i.e., Lamport clocks) and propagating the value to be updated with the invalidation message (_early value propagation_), Hermes allows any replica blocked by an update (write or RMW) to safely replay the update and unblock it self and the rest of followers.
----
## Hardware dependencies
A homogeneous cluster of x86_64 nodes interconnected via RDMA network cards and switched
(tested on "Mellanox ConnectX-4" Infiniband infrastructure).
## Software requirements
Linux OS (tested on Ubuntu 18.04 4.15.0-55-generic) with root access.
The software is tested using the following version of Mellanox OFED RDMA drivers
`MLNX_OFED_LINUX-4.4-2.0.7.0`.
Third-party libraries that you will require to run the experiments include:
1. _parallel_ (Cluster management scripts only)
1. _libmemcached-dev_ (used to exchange QP informations for the setup of RDMA connections)
1. _libnuma-dev_ (for mbind)
## Setup
On every node:
1. Install Mellanox OFED ibverbs drivers
1. `./hermes/bin/setup.sh`
On manager (just pick on node in the cluster):
1. Fill variables in `/hermes/exec/hosts.sh`
1. Configure setup and default parameters in `/hermes/include/hermes/config.h`
1. From `/hermes/exec/` compile _hermesKV_ through make
1. scp _hermesKV_ and the configured hosts.sh in the `/hermes/exec/` directory of all other nodes in the cluster.
## Compilation
`cd hermes/exec; make`
_Warning_: Do not compile through cmake; instead use the Makefile in exec/ directory.
## Run
Run first on manager:
`./run-hermesKV.sh <experiment_parameters>`
Then run on all other member nodes
`./run-hermesKV.sh <experiment_parameters>`
> Note that some members will eagerly terminate if experiment
uses smaller number of nodes than specified in hosts.sh
An experiment example for three nodes 12 worker threads and 35% write ratio would be as follows:
`./run-hermesKV.sh -W 12 -w 350 -M 3`
Supported command-line arguments for the experiments are detailed in the run-hermesKV.sh script.
---
## Acknowledgments
Hermes is based on [HERD/MICA](https://github.com/efficient/HERD "Apache 2.0") design as an underlying KVS, the code of which we have adapted to implement HermesKV.
## Other Implementations of Hermes
- [Odyssey](https://github.com/vasigavr1/Odyssey) - Hermes is also implemed in the Odyssey framework by [Vasilis Gavrielatos](https://github.com/vasigavr1)
- [Olympus](https://github.com/sadraskol/olympus) - in Rust by [Thomas Bracher](https://twitter.com/sadraskol)
## Contact
Antonios Katsarakis: <a href="http://antonis.io/" title="Personal webpage" target="_blank">`antonis.io`</a> | [`antoniskatsarakis@yahoo.com`](mailto:antoniskatsarakis@yahoo.com?subject=[GitHub]%20Zeus%20Specification "Email")
================================================
FILE: bin/copy-exec-files.sh
================================================
#!/usr/bin/env bash
FILES_TO_CPY=(
"hosts.sh"
"run.sh"
"run-hermesKV.sh"
"hermesKV"
"run-rCRAQ.sh"
"rCRAQ"
# "hades"
# "run-hades.sh"
)
EXEC_FOLDER="${HOME}/hermes/exec"
cd $EXEC_FOLDER
# get Hosts
source ../exec/hosts.sh
make clean; make
cd -
for FILE in "${FILES_TO_CPY[@]}"
do
parallel scp ${EXEC_FOLDER}/${FILE} {}:${EXEC_FOLDER}/${FILE} ::: $(echo ${REMOTE_HOSTS[@]})
echo "${FILE} copied to {${REMOTE_HOSTS[@]}}"
done
================================================
FILE: bin/copy-n-exec-hermesKV.sh
================================================
#!/usr/bin/env bash
### Runs to make
#declare -a write_ratios=(0 10 50 200 500 1000)
declare -a write_ratios=(1000)
declare -a rmw_ratios=(0)
#declare -a num_workers=(5 10 15 20 25 30 36)
declare -a num_workers=(1)
#declare -a batch_sizes=(25 50 75 100 125 150 200 250)
declare -a batch_sizes=(50)
declare -a credits=(50)
#declare -a coalesce=(1 5 10 15)
declare -a coalesce=(15)
#declare -a num_machines=(2 3 5 7)
declare -a num_machines=(5)
# Set LAT_WORKER to -1 to disable latency measurement or to worker id (i.e., from 0 up to [num-worker - 1])
LAT_WORKER="-1"
#LAT_WORKER="0"
EXEC_FOLDER="${HOME}/hermes/exec"
REMOTE_COMMAND="cd ${EXEC_FOLDER}; bash run-hermesKV.sh"
PASS="${1}"
if [ -z "$PASS" ]
then
echo "\$PASS is empty! --> sudo pass for remotes is expected to be the first arg"
exit;
fi
echo "\$PASS is OK!"
cd ${EXEC_FOLDER}
# get Hosts
source ./hosts.sh
../bin/copy-exec-files.sh
# Execute locally and remotely
for M in "${num_machines[@]}"; do
for RMW in "${rmw_ratios[@]}"; do
for WR in "${write_ratios[@]}"; do
for W in "${num_workers[@]}"; do
for BA in "${batch_sizes[@]}"; do
for CRD in "${credits[@]}"; do
for COAL in "${coalesce[@]}"; do
args=" -M ${M} -R ${RMW} -w ${WR} -W ${W} -b ${BA} -c ${CRD} -C ${COAL} -l ${LAT_WORKER}"
echo ${PASS} | ./run-hermesKV.sh ${args} &
sleep 2 # give some leeway so that manager starts before executing the members
parallel "echo ${PASS} | ssh -tt {} $'${REMOTE_COMMAND} ${args}'" ::: $(echo ${REMOTE_HOSTS[@]}) >/dev/null
done
done
done
done
done
done
done
cd - >/dev/null
../bin/get-system-xput-files.sh
================================================
FILE: bin/copy-n-exec-rCRAQ.sh
================================================
#!/usr/bin/env bash
USE_SAME_BATCH_N_CREDITS=0
### Runs to make
declare -a write_ratios=(1000)
#declare -a num_workers=(5 10 15 20 25 30 36)
declare -a num_workers=(1)
#declare -a batch_sizes=(25 50 75 100 125 150 200 250)
declare -a batch_sizes=(50)
declare -a credits=(15) # WARNING credits for CR must be divided by the num_machines (i.e., credits % num_machines == 0)
#declare -a coalesce=(1 5 10 15)
declare -a coalesce=(10)
#declare -a num_machines=(2 3 5 7)
declare -a num_machines=(3)
# Set LAT_WORKER to -1 to disable latency measurement or to worker id (i.e., from 0 up to [num-worker - 1])
LAT_WORKER="-1"
#LAT_WORKER="0"
#LOCAL_HOST=`hostname`
EXEC_FOLDER="${HOME}/hermes/exec"
REMOTE_COMMAND="cd ${EXEC_FOLDER}; bash run-rCRAQ.sh"
PASS="${1}"
if [ -z "$PASS" ]
then
echo "\$PASS is empty! --> sudo pass for remotes is expected to be the first arg"
exit;
fi
echo "\$PASS is OK!"
cd ${EXEC_FOLDER}
# get Hosts
source ./hosts.sh
../bin/copy-exec-files.sh
if [ ${USE_SAME_BATCH_N_CREDITS} -eq 0 ]
then
for M in "${num_machines[@]}"; do
# Execute locally and remotely
for WR in "${write_ratios[@]}"; do
for W in "${num_workers[@]}"; do
for BA in "${batch_sizes[@]}"; do
for CRD in "${credits[@]}"; do
for COAL in "${coalesce[@]}"; do
args=" -M ${M} -w ${WR} -W ${W} -b ${BA} -c ${CRD} -C ${COAL} -l ${LAT_WORKER}"
echo ${PASS} | ./run-rCRAQ.sh ${args} &
sleep 2
parallel "echo ${PASS} | ssh -tt {} $'${REMOTE_COMMAND} ${args}'" ::: $(echo ${REMOTE_HOSTS[@]}) >/dev/null
done
done
done
done
done
done
else
# Execute locally and remotely
for M in "${num_machines[@]}"; do
for WR in "${write_ratios[@]}"; do
for W in "${num_workers[@]}"; do
for BA in "${batch_sizes[@]}"; do
for COAL in "${coalesce[@]}"; do
args=" -M ${M} -w ${WR} -W ${W} -b ${BA} -c ${BA} -C ${COAL} -l ${LAT_WORKER}"
echo ${PASS} | ./run-rCRAQ.sh ${args} &
sleep 2
parallel "echo ${PASS} | ssh -tt {} $'${REMOTE_COMMAND} ${args}'" ::: $(echo ${REMOTE_HOSTS[@]}) >/dev/null
done
done
done
done
done
fi
cd - >/dev/null
../bin/get-system-xput-files.sh
================================================
FILE: bin/copy-traces.sh
================================================
#!/usr/bin/env bash
# Copy (per-thread splitted) trace folder
FOLDERS_TO_CPY=( "traces/current-splitted-traces" )
HOME_FOLDER="${HOME}/hermes"
cd ${HOME_FOLDER} >/dev/null
# get Hosts
source ./exec/hosts.sh
cd - >/dev/null
for FOLDER in "${FOLDERS_TO_CPY[@]}"
do
parallel scp -r ${HOME_FOLDER}/${FOLDER} {}:${HOME_FOLDER}/${FOLDER} ::: $(echo ${REMOTE_HOSTS[@]})
echo "${FOLDER} copied to {${REMOTE_HOSTS[@]}}"
done
================================================
FILE: bin/csv_latency_parser.py
================================================
#!/usr/bin/python
import sys, os, ntpath, getopt
"""
========
Parser for aggregated over time results
========
"""
class LatencyParser:
def __init__(self):
self.latency_values = []
self.reads = []
self.max_read_latency = 0
self.max_write_latency = 0
self.writes = []
self.all_reqs = []
self.parseInputStats()
self.printAllStats()
# self.printStats(all_reqs)
def printStats(self, array, max_latency):
self.avgLatency(array)
#self.percentileLatency(array, 20)
self.percentileLatency(array, 50)
self.percentileLatency(array, 90)
self.percentileLatency(array, 95)
self.percentileLatency(array, 99)
#self.percentileLatency(array, 99.9)
#self.percentileLatency(array, 99.99)
#self.percentileLatency(array, 99.999)
#self.percentileLatency(array, 99.9999)
#self.percentileLatency(array, 100)
print "Max Latency: ", max_latency, "us"
def printAllStats(self):
print "~~~~~~ Write Stats ~~~~~~~"
self.printStats(self.writes, self.max_write_latency)
print "\n~~~~~~ Read Stats ~~~~~~~~"
self.printStats(self.reads, self.max_read_latency)
print "\n~~~~~~ Overall Stats ~~~~~~~~~"
self.printStats(self.all_reqs, max(self.max_read_latency, self.max_write_latency))
def avgLatency(self, array):
cummulative = 0
total_reqs = 0
for x in xrange(len(self.latency_values)):
cummulative = self.latency_values[x] * array[x] + cummulative
total_reqs += array[x]
if total_reqs > 0:
print "Reqs measured: ", total_reqs, "| Avg Latency: ", cummulative / total_reqs
else:
print "No reqs measured"
def percentileLatency(self, array, percentage):
total_reqs = 0
sum_reqs = 0
for x in xrange(len(self.latency_values)):
#cummulative = self.latency_values[x] * array[x] + cummulative
total_reqs += array[x]
if total_reqs > 0:
if percentage == 100:
for x in reversed(xrange(len(self.latency_values))):
if array[x] > 0:
if self.latency_values[x] == -1:
print percentage, "%: >", self.latency_values[x-1], "us"
else:
print percentage, "%: ", self.latency_values[x], "us"
return
else:
for x in xrange(len(self.latency_values)):
sum_reqs += array[x]
if ((100.0 * sum_reqs) / total_reqs) >= percentage:
if self.latency_values[x] == -1:
print percentage, "%: >", self.latency_values[x-1], "us"
else:
print percentage, "% : ", self.latency_values[x], "us"
return
else:
print "No reqs measured"
def parseInputStats(self):
lr_lines = 0
for line in sys.stdin: # input from standard input
if line[0] == '#':
continue
(command, words) = line.strip().split(":",1)
command = command.strip()
if command == 'reads':
words = words.strip().split(",")
#if int(words[0].strip()) != -1:
self.latency_values.append(int(words[0].strip()))
self.reads.append(int(words[1].strip()))
self.all_reqs.append(int(words[1].strip()))
elif command == 'writes':
words = words.strip().split(",")
self.writes.append(int(words[1].strip()))
self.all_reqs[lr_lines] = self.all_reqs[lr_lines] + self.writes[-1]
lr_lines = lr_lines + 1
elif command == 'reads-hl':
words = words.strip().split(",")
self.max_read_latency = int(words[0].strip())
elif command == 'writes-hl':
words = words.strip().split(",")
self.max_write_latency = int(words[0].strip())
if __name__ == '__main__':
LatencyParser()
================================================
FILE: bin/exec-derecho.sh
================================================
#!/usr/bin/env bash
HOSTS=( ##### network cluster #####
"houston"
"sanantonio"
"austin"
"indianapolis"
"philly"
# "atlanta"
##### compute cluster #####
# "baltimore"
# "chicago"
# "detroit"
)
NUM_NODES=5
NUM_SENDERS=0 #0 - all senders, 1 - half senders, 2 - one sender
REQS_PER_SENDER=10000000
### Runs to make
#declare -a delivery_mode=(0 1) #0 - ordered mode, 1 - unordered mode
#declare -a object_size=(40 1024)
#declare -a window_size=(128 256)
declare -a delivery_mode=(0) #0 - ordered mode, 1 - unordered mode
declare -a object_size=(256 1024)
declare -a window_size=(128 256)
declare -a iterations=(1 2 3 4) #(1 2 3) for 3 iterations
if [[ $NUM_NODES -ne ${#HOSTS[@]} ]] ; then
echo "Num_nodes($NUM_NODES) != #Hosts(${#HOSTS[@]})"
exit 1
fi
LOCAL_HOST=`hostname`
HOME_FOLDER="${HOME}/derecho-unified/Release/applications/tests/performance_tests/"
#pin derecho threads to cores (w/o using hyperthreads) of numa node 0
COMMAND_NO_ARGS="taskset -c 0,2,4,6,8,10,12,14,16,18 ./bandwidth_test "
total_iters=0
cd ${HOME_FOLDER} >/dev/null
# Execute locally and remotely
for del_mode in "${delivery_mode[@]}"; do
for obj_size in "${object_size[@]}"; do
for win_size in "${window_size[@]}"; do
for iter in "${iterations[@]}"; do
total_iters=$((total_iters + 1))
args="--DERECHO/max_payload_size=${obj_size} --DERECHO/window_size=${win_size} -- ${NUM_NODES} ${NUM_SENDERS} ${REQS_PER_SENDER} ${del_mode}"
COMMAND=" ${COMMAND_NO_ARGS} ${args}"
echo "Running Derecho with: delivery_mode:${del_mode} obj size: $obj_size, window_size: $win_size nodes: $NUM_NODES "
${COMMAND} >/dev/null &
sleep 1
parallel "ssh -tt {} $'cd ${HOME_FOLDER}; ${COMMAND}'" ::: $(echo ${HOSTS[@]/$LOCAL_HOST}) >/dev/null
sleep 9 # give local node some leeway to log the results into a file
done
done
done
done
tail -${total_iters} data_derecho_bw
cd - >/dev/null
================================================
FILE: bin/format.sh
================================================
#!/bin/bash
SCRIPT_DIR="$(dirname "$0")"
cd "${SCRIPT_DIR}"
FORMAT_FILES_IN_DIRECTORIES="../src/ ../include/"
clang-format --version > /dev/null || exit 1
if [ "$1" = "check" ]; then # Check clang-format has been applied!
find ${FORMAT_FILES_IN_DIRECTORIES} \
-regex '.*\.\(cpp\|hpp\|cc\|cxx\)' \
-exec clang-format -style=file -output-replacements-xml -i {} \; |
grep -c "<replacement " >/dev/null
if [ $? -ne 1 ]; then
echo "Format check: Failed!"
echo " -- Files do not match clang-format. Run bin/format.sh before adding files to git!"
exit 1
else
echo "Format check: Passed!"
fi
else # Apply clang-format to all files
find ${FORMAT_FILES_IN_DIRECTORIES} \
-regex '.*\.\(c\|h\|cpp\|hpp\|cc\|cxx\)' \
-exec clang-format -style=file -i {} \;
fi
================================================
FILE: bin/get-system-xput-files.sh
================================================
#!/usr/bin/env bash
EXEC_FOLDER="${HOME}/hermes/exec"
RESULTS_FOLDER="${HOME}/hermes/exec/results"
RESULT_FOLDER="${RESULTS_FOLDER}/xput/per-node/"
RESULT_OUT_FOLDER="${RESULTS_FOLDER}/xput/per-node/"
RESULT_OUT_FOLDER_MERGE="${RESULTS_FOLDER}/xput/all-nodes/"
cd ${EXEC_FOLDER} >/dev/null
# get Hosts
source ./hosts.sh
cd - >/dev/null
# Gather remote files
parallel "scp {}:${RESULT_FOLDER}* ${RESULT_OUT_FOLDER} " ::: $(echo ${REMOTE_HOSTS[@]})
echo "xPut result files copied from: {${REMOTE_HOSTS}}"
# group all files
ls ${RESULT_OUT_FOLDER} | awk -F '-' '!x[$2]++{print $1}' | while read -r line; do
# Create an intermediate file print the 3rd line for all files with the same prefix to the same file
awk 'FNR==3 {print $0}' ${RESULT_OUT_FOLDER}/$line* > ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt
# Sum up the xPut of the (3rd iteration) from every node to create the final file
awk -F ':' '{sum += $2} END {print sum}' ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt > ${RESULT_OUT_FOLDER_MERGE}/$line.txt
rm -rf ${RESULT_OUT_FOLDER_MERGE}/$line-inter.txt
done
echo "System-wide xPut results produced in ${RESULT_OUT_FOLDER_MERGE} directory!"
================================================
FILE: bin/setup.sh
================================================
#!/usr/bin/env bash
# Exec this script in every cluster node after you have
# installed the (Infiniband) Verbs drivers through Mellanox OFED:
# 1. Download the MLNX_OFED (tested on --> MLNX_OFED_LINUX-4.4-2.0.7.0-ubuntu18.04-x86_64)
# https://www.mellanox.com/page/products_dyn?product_family=26
# 2. tar -xvf the tar file
# 3. install through --> sudo ./mlnxofedinstall
if ! [ -x "$(command -v ofed_info)" ]; then
echo "Error: mellanox ofed is not installed." >&2
echo " Please install the (Infiniband) Verbs drivers through Mellanox OFED by:"
echo " 1. Download the MLNX_OFED (tested on --> MLNX_OFED_LINUX-4.4-2.0.7.0-ubuntu18.04-x86_64)"
echo " https://www.mellanox.com/page/products_dyn?product_family=26"
echo " 2. tar -xvf the tar file"
echo " 3. install through --> sudo ./mlnxofedinstall"
exit 1
else
MLNX_OFED_VERSION=`ofed_info | head -1`
echo "Running OFED driver version: ${MLNX_OFED_VERSION}" >&2
fi
# Install required Libraries (memcached is used to setup RDMA connection and numa for mbind)
sudo apt --yes install libmemcached-dev libnuma-dev memcached
# start a subnet manager
sudo /etc/init.d/opensmd start # there must be at least one subnet-manager in an infiniband subnet cluster
# start the driver
sudo /etc/init.d/openibd start
# Configure (2MB) huge-pages for the KVS
# Note that such a huge page allocation is not permanent and must be re-applied after a node reboot.
#echo 8192 | sudo tee /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages
echo 4096 | sudo tee /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages
echo 10000000001 | sudo tee /proc/sys/kernel/shmmax
echo 10000000001 | sudo tee /proc/sys/kernel/shmall
================================================
FILE: bin/trace-spliter.sh
================================================
#!/usr/bin/env bash
INPUT_DIR="${HOME}/hermes/traces/system-traces/"
INPUT_FILENAME="simple_trace_w_100000000_k_1000000_a_0.99.txt"
OUTPUT_DIR="${HOME}/hermes/traces/current-splited-traces/"
OUTPUT_PREFIX="t_"
OUTPUT_SUFFIX="_a_0.99.txt"
MAX_NUM_NODES=10
MAX_THREADS_PER_NODE=40
CHUNKS=$(expr ${MAX_NUM_NODES} \* ${MAX_THREADS_PER_NODE})
LINES=$(wc -l ${INPUT_DIR}/${INPUT_FILENAME} | cut -d ' ' -f1)
echo "Splitting trace with $LINES lines into $CHUNKS (per-thread) chunks ..."
split -l $(expr ${LINES} / ${CHUNKS}) \
-a 4 -d \
--additional-suffix=${OUTPUT_SUFFIX} \
${INPUT_DIR}/${INPUT_FILENAME} \
${OUTPUT_DIR}/${OUTPUT_PREFIX}
================================================
FILE: exec/Makefile
================================================
CPPFLAGS := -O3 #-Wno-unused-result -Wall -Werror
LD := gcc -O3 -flto
LDFLAGS := ${LDFLAGS} -libverbs -lrt -lpthread -lmemcached -lnuma # -lrdmacm --> TODO we do not use hw multicast because it helps only on master-based patterns
CFLAGS = -I../include/mica-herd -I../include/hermes -I../include/wings -I../include/hades
APPS := hermesKV rCRAQ
PROF := -g -fno-omit-frame-pointer
all: ${APPS} clean-o
hermesKV: ../src/wings/wings.o ../src/hades/hades.o \
../src/mica-herd/herd.o ../src/mica-herd/mica.o ../src/mica-herd/city.o \
../src/hermes/main.o ../src/hermes/hermes_worker.o ../src/hermes/util.o \
../src/hermes/stats.o ../src/hermes/spacetime.o ../src/hermes/hermesKV.o
${LD} -o $@ $^ ${LDFLAGS}
rCRAQ: ../src/mica-herd/herd.o ../src/mica-herd/mica.o \
../src/mica-herd/city.o ../src/hermes/main.o ../src/CR/cr_worker.o ../src/CR/crKV.o \
../src/hermes/spacetime.o ../src/hermes/util.o ../src/hermes/stats.o ../src/wings/wings.o
${LD} -o $@ $^ ${LDFLAGS}
hades-exec: ../src/hades/hades.o ../src/hades/test.o ../src/wings/wings.o ../src/mica-herd/herd.o
${LD} -o hades $^ ${LDFLAGS}
hades: hades-exec clean-o
PHONY: clean
clean:
@rm -f ../src/hermes/*.o ../src/mica-herd/*.o ../src/wings/*.o \
../src/CR/*.o ../src/hades/*.o ${APPS} hades
clean-o:
@rm -f ../src/hermes/*.o ../src/mica-herd/*.o ../src/wings/*.o \
../src/CR/*.o ../src/hades/*.o
================================================
FILE: exec/hosts.sh
================================================
#!/usr/bin/env bash
ALL_IPS=(
### TO BE FILLED: Please provide all cluster IPs
# Node w/ first IP (i.e., "manager") must run script before the rest of the nodes
# (instantiates a memcached to setup RDMA connections)
#
10.0.3.1
10.0.3.2
10.0.3.3
10.0.3.4
10.0.3.5
)
### TO BE FILLED: Modify to get the local IP of the node running the script (must be one of the cluster nodes)
LOCAL_IP=$(ip addr | grep 'state UP' -A2 | grep 'inet 10.0.3'| awk '{print $2}' | cut -f1 -d'/')
#LOCAL_IP="129.215.164.2"
### Fill the RDMA device name (the "hca_id" of the device when executing ibv_devinfo)
#NET_DEVICE_NAME="mlx5_0"
NET_DEVICE_NAME="mlx4_0"
##########################################
### NO NEED TO CHANGE BELOW THIS POINT ###
##########################################
REMOTE_IPS=${ALL_IPS[@]/$LOCAL_IP}
REMOTE_HOSTS=${ALL_IPS[@]/$LOCAL_IP}
NODE_ID=-1
for i in "${!ALL_IPS[@]}"; do
if [ "${ALL_IPS[i]}" == "$LOCAL_IP" ]; then
NODE_ID=$i
fi
done
if [[ ${NODE_ID} == -1 ]]; then
echo "Error Local IP: ${LOCAL_IP} n is not in ALL_IPS:"
echo " {${ALL_IPS[@]}}"
exit
fi
echo "Local node id:" ${NODE_ID}
================================================
FILE: exec/results/latency/.gitinclude
================================================
================================================
FILE: exec/results/xput/all-nodes/.gitkeep
================================================
================================================
FILE: exec/results/xput/per-node/.gitkeep
================================================
================================================
FILE: exec/run-hades.sh
================================================
#!/usr/bin/env bash
source run.sh
blue "Running hades"
sudo LD_LIBRARY_PATH=/usr/local/lib/ -E \
./hades \
--machine-id ${NODE_ID} \
--dev-name ${NET_DEVICE_NAME} \
2>&1
================================================
FILE: exec/run-hermesKV.sh
================================================
#!/usr/bin/env bash
source run.sh
#### Get CLI arguments
# Use -1 for the default (#define in config.h) values if not argument is passed
CREDITS="-1"
NUM_WORKERS="-1"
WRITE_RATIO="-1"
MAX_COALESCE="-1"
MAX_BATCH_SIZE="-1"
RMW_RATIO="-1"
NUM_MACHINES="-1"
LAT_WORKER="-1"
# Each letter is an option argument, if it's followed by a collum
# it requires an argument. The first colum indicates the '\?'
# help/error command when no arguments are given
while getopts ":W:w:l:R:C:c:b:M:h" opt; do
case $opt in
W)
NUM_WORKERS=$OPTARG # Number of threads: this must be smaller than MAX_WORKERS_PER_MACHINE of config.h
;;
w)
WRITE_RATIO=$OPTARG # given number is divided by 10 to give write rate % (i.e., 55 means 5.5 % writes)
;;
R)
RMW_RATIO=$OPTARG # percentage of writes to be rmws (i.e., -w 500 -R 500 means 25 % of RMWs and 25% of writes)
# RMW is disabled by default (no usage through the artifact) can be enabled through config.h)
;;
C)
MAX_COALESCE=$OPTARG # maximum number of readily-available messages to be "batched" in a network packet
# must be smaller than MTU and it is capped by MAX_REQ_COALESCE in config.h
;;
c)
CREDITS=$OPTARG # maximum number of credits per node per thread; credits correspond to messages and not packets
# it is capped by MAX_CREDITS_PER_REMOTE_WORKER in config.h
;;
b)
MAX_BATCH_SIZE=$OPTARG # amount of requests and protocol messages that can be batched to the KVS
# it is capped by MAX_BATCH_KVS_OPS_SIZE in config.h
;;
M)
NUM_MACHINES=$OPTARG # it is capped by MAX_MACHINE_NUM in config.h and the number of IPS as indicated in hosts.sh
;;
l)
LAT_WORKER=$OPTARG # An id of the worker who is measuring the latency
# if -1 Latency is disabled
# otherwise it is capped by running worker threads (NUM_WORKERS-1)
;;
h)
echo "Usage: -W <# workers> -w <write ratio> (x1000 --> 10 for 1%)"
echo " -c <# credits> -b <max batch size> -C <max coalescing>"
echo " -M <# nodes> -l <latency worker> -R <rmw ratio>"
exit 1
;;
\?)
echo "Invalid option: -$OPTARG use -h to get info for arguments" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
blue "Running hermes threads"
sudo LD_LIBRARY_PATH=/usr/local/lib/ -E \
./hermesKV \
--machine-id ${NODE_ID} \
--is-roce 0 \
--dev-name ${NET_DEVICE_NAME} \
--num-machines ${NUM_MACHINES} \
--num-workers ${NUM_WORKERS} \
--lat-worker ${LAT_WORKER} \
--rmw-ratio ${RMW_RATIO} \
--write-ratio ${WRITE_RATIO} \
--credits ${CREDITS} \
--max-coalesce ${MAX_COALESCE} \
--max-batch-size ${MAX_BATCH_SIZE} \
--hermes \
2>&1
================================================
FILE: exec/run-rCRAQ.sh
================================================
#!/usr/bin/env bash
source run.sh
#### Get CLI arguments
# Use -1 for the default (#define in config.h) values if not argument is passed
CREDITS="-1"
NUM_WORKERS="-1"
WRITE_RATIO="-1"
MAX_COALESCE="-1"
MAX_BATCH_SIZE="-1"
RMW_RATIO="-1"
NUM_MACHINES="-1"
LAT_WORKER="-1"
# Each letter is an option argument, if it's followed by a collum
# it requires an argument. The first colum indicates the '\?'
# help/error command when no arguments are given
while getopts ":W:w:C:c:b:M:l:h" opt; do
case $opt in
W)
NUM_WORKERS=$OPTARG
;;
w)
WRITE_RATIO=$OPTARG
;;
C)
MAX_COALESCE=$OPTARG
;;
c)
CREDITS=$OPTARG
;;
b)
MAX_BATCH_SIZE=$OPTARG
;;
M)
NUM_MACHINES=$OPTARG
;;
l)
LAT_WORKER=$OPTARG
;;
h)
echo "Usage: -W <# workers> -w <write ratio> (x1000 --> 10 for 1%)"
echo " -c <# credits> -b <max batch size> -C <max coalescing>"
echo " -M <# nodes> -l <latency worker> "
exit 1
;;
\?)
echo "Invalid option: -$OPTARG use -h to get info for arguments" >&2
exit 1
;;
:)
echo "Option -$OPTARG requires an argument." >&2
exit 1
;;
esac
done
blue "Running hermes threads"
sudo LD_LIBRARY_PATH=/usr/local/lib/ -E \
./rCRAQ \
--machine-id ${NODE_ID} \
--is-roce 0 \
--dev-name ${NET_DEVICE_NAME} \
--num-machines ${NUM_MACHINES} \
--num-workers ${NUM_WORKERS} \
--lat-worker ${LAT_WORKER} \
--rmw-ratio ${RMW_RATIO} \
--write-ratio ${WRITE_RATIO} \
--credits ${CREDITS} \
--max-coalesce ${MAX_COALESCE} \
--max-batch-size ${MAX_BATCH_SIZE} \
2>&1
================================================
FILE: exec/run.sh
================================================
#!/usr/bin/env bash
source ./hosts.sh
export HRD_REGISTRY_IP="${ALL_IPS[0]}" # I.E. first IP node (HOUSTON) has a memcached server (used to initialize RDMA QPs)
export MLX5_SINGLE_THREADED=1
export MLX5_SCATTER_TO_CQE=1
sudo killall memcached
sudo killall hades
sudo killall rCRAQ
sudo killall hermesKV
# A function to echo in blue color
function blue() {
es=`tput setaf 4`
ee=`tput sgr0`
echo "${es}$1${ee}"
}
#### free the pages workers use
blue "Removing SHM keys used by HermesKV/rCRAQ"
for i in `seq 0 28`; do
key=`expr 3185 + $i`
sudo ipcrm -M $key 2>/dev/null
key=`expr 4185 + $i`
sudo ipcrm -M $key 2>/dev/null
done
: ${HRD_REGISTRY_IP:?"Need to set HRD_REGISTRY_IP non-empty"}
blue "Reset server QP registry"
memcached -l ${HRD_REGISTRY_IP} 1>/dev/null 2>/dev/null &
sleep 1
================================================
FILE: include/hades/hades.h
================================================
//
// Created by akatsarakis on 17/01/19.
//
#ifndef HADES_H
#define HADES_H
#include "../../include/wings/wings.h"
#include "../utils/bit_vector.h"
#include "../utils/time_rdtsc.h"
// Send heartbeats
// Recv heartbeats
// Change View
// Update local membership
// (Ostracism)
// arbitration --> a node provides an obolus
// all nodes are able to communicate w/ each other
// fd provides a view as a membership change
// only as long as it differs with the current view
// and agrees with a majority of other node views.
// The update granularity of local view works as a lease
// to membership changes which prevents sequentially
// consistent reads in the presence of network partitions
// I.E. a node in a minority partition is able to detect
// that cannot reach the majority of nodes and stops serving
// local reads, maintaining linearizability (instead of sequential
// consistency) For this
// Epochs
// Guarantees Nodes in the same EPOCH id have the same group view
#define ENABLE_ARBITRATION 1
// Hades debug Tests
#define FAKE_LINK_FAILURE 0
#define FAKE_LINK_FAILURE_AFTER_SEC 15
#define STOP_FAKE_LINK_FAILURE_AFTER_SEC 20
#define FAKE_ONE_WAY_LINK_FAILURE 0
#define FAKE_LINK_FAILURE_NODE_A 2
#define FAKE_LINK_FAILURE_NODE_B 1
static_assert(FAKE_LINK_FAILURE_NODE_A != FAKE_LINK_FAILURE_NODE_B, "");
typedef struct {
uint8_t node_id : 8;
uint8_t epoch_id : 8;
uint8_t same_w_local_membership : 1;
uint8_t have_ostracised_for_dst_node : 7;
bit_vector_t view;
} __attribute__((packed)) hades_view_t;
static_assert(sizeof(hades_view_t) <= 4,
"Currently send using a 4B header only field (RDMA immediate)");
typedef struct {
hades_view_t last_local_view;
hades_view_t intermediate_local_view;
bit_vector_t curr_g_membership;
uint8_t nodes_in_membership;
uint8_t max_num_nodes;
uint8_t* recved_views_flag;
hades_view_t* remote_recved_views;
// Polling
uint16_t max_views_to_poll;
hades_view_t* poll_buff; // used for polling remote views
// Timing
uint32_t send_view_every_us;
uint32_t update_local_view_every_ms;
struct timespec* ts_last_send; // issues views to remotes iff have not send a
// view within the predefined timeout
struct timespec
ts_last_view_change; // update views and possible changes membership iff
// pre-defined timeout is exceed
// Ostracism
uint8_t*
have_ostracized_for; // an array storing info whether or not in a view
// the sender ostracized someone for this node
} hades_ctx_t;
typedef struct {
hades_ctx_t ctx;
ud_channel_t* hviews_c;
ud_channel_t* hviews_crd_c;
} hades_wings_ctx_t;
void* hades_full_thread(void* node_id);
uint16_t poll_for_remote_views(hades_wings_ctx_t* hw_ctx);
void update_view_and_issue_hbs(hades_wings_ctx_t* hw_ctx);
inline static void
hades_ctx_init(hades_ctx_t* ctx, uint8_t node_id, uint8_t max_nodes,
uint16_t max_views_to_poll, uint32_t send_view_us,
uint32_t update_local_view_ms)
{
assert(max_views_to_poll > 0);
ctx->intermediate_local_view.epoch_id = 0;
ctx->intermediate_local_view.node_id = node_id;
ctx->nodes_in_membership = 1;
bv_init(&ctx->curr_g_membership);
bv_bit_set(&ctx->curr_g_membership, node_id);
bv_init(&ctx->intermediate_local_view.view);
bv_bit_set(&ctx->intermediate_local_view.view, node_id);
ctx->last_local_view = ctx->intermediate_local_view;
ctx->max_num_nodes = max_nodes;
ctx->recved_views_flag = malloc(sizeof(uint8_t) * max_nodes);
ctx->remote_recved_views = malloc(sizeof(hades_view_t) * max_nodes);
for (int i = 0; i < max_nodes; ++i) {
ctx->recved_views_flag[i] = 0;
bv_init(&ctx->remote_recved_views[i].view);
}
ctx->max_views_to_poll = max_views_to_poll;
ctx->poll_buff = malloc(sizeof(hades_view_t) * max_views_to_poll);
// Setup timers
init_rdtsc(1, 0); /// WARNING: this is not thread safe!!
get_rdtsc_timespec(&ctx->ts_last_view_change);
ctx->ts_last_send = malloc(sizeof(struct timespec) * max_nodes);
for (int i = 0; i < max_nodes; ++i)
get_rdtsc_timespec(&ctx->ts_last_send[i]);
ctx->send_view_every_us = send_view_us;
ctx->update_local_view_every_ms = update_local_view_ms;
assert(2 * 1000 * update_local_view_ms > send_view_us);
// Ostracism
ctx->have_ostracized_for = malloc(sizeof(uint8_t) * max_nodes);
for (int i = 0; i < max_nodes; ++i)
ctx->have_ostracized_for[i] = 0;
}
// WARNING: hades wings_ctx_init initializes only the first part of the
// required channels wings_setup_channel_qps_and_recvs must be called by
// the application afterwards to finish the initialization of wings.
inline static void
hades_wings_ctx_init(hades_wings_ctx_t* wctx, uint8_t node_id,
uint8_t max_nodes, uint16_t max_views_to_poll,
uint32_t send_view_us, uint32_t update_local_view_ms,
ud_channel_t* hviews_c, ud_channel_t* hviews_crd_c,
uint16_t worker_lid)
{
hades_ctx_init(&wctx->ctx, node_id, max_nodes, max_views_to_poll,
send_view_us, update_local_view_ms);
wctx->hviews_c = hviews_c;
wctx->hviews_crd_c = hviews_crd_c;
const uint8_t is_bcast = 0;
const uint8_t stats_on = 1;
const uint8_t prints_on = 1;
const uint8_t is_hdr_only = 1;
const uint8_t expl_crd_ctrl = 1;
const uint8_t enable_inlining = 1;
const uint8_t disable_crd_ctrl = 0;
const uint8_t credits =
(const uint8_t)(2 * update_local_view_ms * 1000 / send_view_us);
char qp_name[200];
sprintf(qp_name, "%s%d", "\033[1m\033[32mHades\033[0m", worker_lid);
wings_ud_channel_init(
wctx->hviews_c, qp_name, REQ, 1, sizeof(hades_view_t) - sizeof(uint8_t),
0, enable_inlining, is_hdr_only, is_bcast, disable_crd_ctrl,
expl_crd_ctrl, wctx->hviews_crd_c, credits, max_nodes,
(uint8_t)machine_id, stats_on, prints_on);
}
// How does somebody joins?
// epoch id 0
// must see at least a majority of views with same epoch id > 0
// || majority of views with epoch id 0
#endif // HADES_H
================================================
FILE: include/hermes/config.h
================================================
//
// Created by akatsarakis on 15/03/18.
//
#ifndef SPACETIME_CONFIG_H
#define SPACETIME_CONFIG_H
#include <assert.h>
#include <stdint.h>
#include "sizes.h"
// MAX_ defines are treated as DEFAULT_ as well (i.e., if not altered by CLI
// args)
/*-------------------------------------------------
------------ SETUP & DEFAULT SETTINGS -------------
--------------------------------------------------*/
#define MAX_MACHINE_NUM 5 // maximum nodes
#define MAX_WORKERS_PER_MACHINE 15 // maximum number of threads per node
#define DEFAULT_WORKERS_PER_MACHINE 2
#define DEFAULT_THREAD_OF_STAT_THREAD \
(15) // WARNING make sure this is not co-located with a worker thread
// Number of sockets (numa nodes), cores and h/w threads per core on each node
#define TOTAL_THREADS_PER_CORE 2
#define TOTAL_CORES_PER_SOCKET 10
#define TOTAL_NUMBER_OF_SOCKETS 2
/*-------------------------------------------------
-------------------------------------------------
-------------------------------------------------
-------- No need to change beyond this point ----
-------------------------------------------------
-------------------------------------------------
--------------------------------------------------*/
// Default workload writes / updates accesses (the rest are reads)
#define DEFAULT_UPDATE_RATIO 1000 // is divided by 10 (i.e., 25 --> 2.5 %)
// both writes and RMWs (RMW_RATIO inderectly provides WRITE_RATIO)
#define ENABLE_RMWs \
0 // if RMWs is not enabled then all UPDATE_RATIO == WRITE_RATIO
#define DEFAULT_RMW_RATIO 0 // is divided by 10 (i.e., 25 --> 2.5 %)
// percentage of UPDATE_RATIO to be RMWs
// Max operations per-thread to batches to the KVS (either received packets or
// read/write/RMW requests)
#define MAX_BATCH_KVS_OPS_SIZE 250
static_assert(MAX_WORKERS_PER_MACHINE <= 254, "");
static_assert(MAX_WORKERS_PER_MACHINE <= TOTAL_NUMBER_OF_SOCKETS *
TOTAL_THREADS_PER_CORE *
TOTAL_CORES_PER_SOCKET,
"");
static_assert(DEFAULT_UPDATE_RATIO <= 1000 && DEFAULT_RMW_RATIO >= 0, "");
/*-------------------------------------------------
----------------- RDMA SETTINGS -------------------
--------------------------------------------------*/
// Request coalescing (max --readily available-- messages to batch in a single
// RDMA packet)
#define MAX_REQ_COALESCE 15
// Flow control
#define MAX_CREDITS_PER_REMOTE_WORKER (MAX_REQ_COALESCE)
// Request inlining
#define DISABLE_INLINING 0
/*-------------------------------------------------
----------------- SECONDARY SETTINGS --------------
--------------------------------------------------*/
// LATENCY
#define DEFAULT_MEASURE_LATENCY 0
#define DEFAULT_WORKER_MEASURING_LATENCY 0
#define MAX_LATENCY 1000 // in us
#define LATENCY_BUCKETS 1000
#define LATENCY_PRECISION \
(MAX_LATENCY / LATENCY_BUCKETS) // latency granularity in us
// FAIRNESS
#define ENABLE_VIRTUAL_NODE_IDS 0 // 0
#define VIRTUAL_NODE_IDS_PER_NODE 20
// SKEW
#define ENABLE_COALESCE_OF_HOT_REQS \
0 // 0 //WARNING!!! this must be disabled for cr
#define COALESCE_N_HOTTEST_KEYS 100
#define ENABLE_READ_COMPLETE_AFTER_VAL_RECV_OF_HOT_REQS 0 // 1
#define ENABLE_WRITE_COALESCE_TO_THE_SAME_KEY_IN_SAME_NODE 0
// DEBUG
#define ENABLE_ASSERTIONS 0
#define DISABLE_VALS_FOR_DEBUGGING 0
#define KEY_NUM 0 // use 0 to disable
// REQUESTS
#define FEED_FROM_TRACE 0
#define ZIPF_EXPONENT_OF_TRACE \
99 // if FEED_FROM_TRACE == 1 | this is divided by 100 (e.g. use 99 for a =
// 0.99)
#define NUM_OF_REP_REQS K_256 // if FEED_FROM_TRACE == 0
#define USE_A_SINGLE_KEY 0 // if FEED_FROM_TRACE == 0
#define ST_KEY_ID_255_OR_HIGHER 255
/*-------------------------------------------------
---------------- Debug and others -----------------
--------------------------------------------------*/
// DBG Prints
/// Warning some prints assume that there are no faults (multiplications with
/// REMOTE_MACHINES)
#define MAX_THREADS_TO_PRINT 1
#define ENABLE_REQ_PRINTS 0
#define ENABLE_BATCH_OP_PRINTS 0
#define ENABLE_INV_PRINTS 0
#define ENABLE_ACK_PRINTS 0
#define ENABLE_VAL_PRINTS 0
// Stats prints
#define PRINT_STATS_EVERY_MSECS 4000 // 5000 //10000 //10
#define PRINT_WORKER_STATS 0
// Stats
#define EXIT_ON_STATS_PRINT 1
#define PRINT_NUM_STATS_BEFORE_EXITING 5
#define DUMP_XPUT_STATS_TO_FILE 1
// FAILURE DETECTION (RM)
#define ENABLE_HADES_FAILURE_DETECTION 0
#define WORKER_WITH_FAILURE_DETECTOR 0
static_assert(ENABLE_HADES_FAILURE_DETECTION == 0,
"WARNING HADES is currently not working");
// FAKE NODE FAILURE
#define FAKE_FAILURE 0
#define NODE_TO_FAIL 2
#define ROUNDS_BEFORE_FAILURE 2
// Rarely (or never) change
#define BASE_SHM_KEY 24
#define WORKER_SL 0 // service level for the workers
#define MAX_REMOTE_MACHINES (MAX_MACHINE_NUM - 1)
#define HERMES_CEILING(x, y) (((x) + (y)-1) / (y))
#define GROUP_MEMBERSHIP_ARRAY_SIZE \
HERMES_CEILING(MAX_MACHINE_NUM, 8) // assuming uint8_t
#define TOTAL_HW_CORES \
(TOTAL_THREADS_PER_CORE * TOTAL_CORES_PER_SOCKET * TOTAL_NUMBER_OF_SOCKETS)
static_assert(MAX_WORKERS_PER_MACHINE < TOTAL_HW_CORES - 1,
"Leave at least a hw thread free for OS etc..");
#define KV_SOCKET 0 // socket to allocate KVS (huge-)pages
#define USE_ALL_SOCKETS 1
#define ENABLE_HYPERTHREADING 1
#define SOCKET_TO_START_SPAWNING_THREADS 0
// Debug
//#define SPACETIME DEBUG 2
#ifndef SPACETIME_DEBUG
#define SPACETIME_DEBUG 0
#endif
////////////////////////////////
/// Hermes NOT TUNABLE
////////////////////////////////
/*-------------------------------------------------
----------------- MAX HERMES OPS SIZE -------------
--------------------------------------------------*/
#define MAX_MSG_RECV_OPS_SIZE \
(MAX_CREDITS_PER_REMOTE_WORKER * MAX_REMOTE_MACHINES * MAX_REQ_COALESCE)
#define HERMES_MAX_BATCH_SIZE MAX(MAX_MSG_RECV_OPS_SIZE, MAX_BATCH_KVS_OPS_SIZE)
/*-------------------------------------------------
---------------- QPs Numbers ----------------------
--------------------------------------------------*/
typedef enum {
INV_UD_QP_ID = 0,
ACK_UD_QP_ID,
VAL_UD_QP_ID,
CRD_UD_QP_ID,
END_HERMES_QPS_ENUM
} hermes_qps_enum;
// QPs
#define TOTAL_WORKER_UD_QPs END_HERMES_QPS_ENUM
#define TOTAL_WORKER_N_FAILURE_DETECTION_UD_QPs \
(TOTAL_WORKER_UD_QPs + (ENABLE_HADES_FAILURE_DETECTION ? 2 : 0))
/*-------------------------------------------------
----------------- CR CONFIGURATION ----------------
--------------------------------------------------*/
#define CR_ENABLE_REMOTE_READS 0
#define CR_REMOTE_READS_CREDITS 20
#define MAX_CREDITS_PER_REMOTE_WORKER_CR 250 //(MAX_BATCH_KVS_OPS_SIZE) // CR
#define CR_ACK_CREDITS (255) // //(MAX_MACHINE_NUM * 255)
#define CR_ENABLE_EARLY_INV_CRDS \
1 // optimization to increase request pipelining
typedef enum {
CR_INV_UD_QP_ID = 0,
#ifdef CR_ENABLE_EARLY_INV_CRDS
CR_INV_CRD_UD_QP_ID,
#endif
CR_ACK_UD_QP_ID,
CR_REMOTE_WRITES_UD_QP_ID,
CR_REMOTE_WRITE_CRD_UD_QP_ID,
CR_REMOTE_READS_UD_QP_ID,
CR_REMOTE_READS_RESP_UD_QP_ID
} cr_qps_enum;
#define CR_TOTAL_WORKER_UD_QPs \
(TOTAL_WORKER_UD_QPs + (CR_ENABLE_REMOTE_READS ? 2 : 0) + \
(CR_ENABLE_EARLY_INV_CRDS ? 1 : 0))
// Max CR batch op size
#define MAX_MSG_RECV_OPS_SIZE_CR \
(MAX_REQ_COALESCE * MAX_CREDITS_PER_REMOTE_WORKER_CR * MAX_REMOTE_MACHINES)
#define CR_MAX_BATCH_SIZE MAX(MAX_MSG_RECV_OPS_SIZE_CR, MAX_BATCH_KVS_OPS_SIZE)
// CR DEBUG
#define CR_ENABLE_ONLY_HEAD_REQS 0
#define CR_ENABLE_ALL_NODES_GETS_EXCEPT_HEAD 0
#define CR_ENABLE_BLOCKING_INVALID_WRITES_ON_HEAD 0
/*-------------------------------------------------
----------------- Global Vars ---------------------
--------------------------------------------------*/
struct thread_params {
int id;
};
struct latency_counters {
uint32_t read_reqs[LATENCY_BUCKETS + 1];
uint32_t write_reqs[LATENCY_BUCKETS + 1];
int max_read_latency;
int max_write_latency;
long long total_measurements;
};
extern struct latency_counters latency_count;
// global config (CLI) configurable vars
extern uint8_t is_CR;
extern int update_ratio;
extern int rmw_ratio;
extern int num_workers;
extern int credits_num;
extern int max_coalesce;
extern int max_batch_size; // for batches to KVS
extern int machine_num; // must be smaller or equal to MAX_MACHINE_NUM
extern int remote_machine_num; // must be smaller or equal to MAX_MACHINE_NUM
extern int worker_measuring_latency;
// extern int value_size; // must be smaller or equal to MAX_MACHINE_NUM
#endif // SPACETIME_CONFIG_H
================================================
FILE: include/hermes/inline-util.h
================================================
//
// Created by akatsarakis on 23/05/18.
//
#ifndef HERMES_INLINE_UTIL_H
#define HERMES_INLINE_UTIL_H
#include <infiniband/verbs.h>
#include "../hades/hades.h"
#include "../utils/concur_ctrl.h"
#include "config.h"
#include "spacetime.h"
#include "util.h"
/* ---------------------------------------------------------------------------
----------------------------------- MEMBERSHIP -------------------------------
---------------------------------------------------------------------------*/
static inline uint8_t
node_is_in_membership(spacetime_group_membership last_group_membership,
int node_id)
{
return (uint8_t)(bv_bit_get(last_group_membership.g_membership,
(uint8_t)node_id) == 1
? 1
: 0);
}
static inline void
group_membership_update(hades_ctx_t hades_ctx)
{
seqlock_lock(&group_membership.lock);
bv_copy((bit_vector_t*)&group_membership.g_membership,
hades_ctx.curr_g_membership);
bv_copy((bit_vector_t*)&group_membership.w_ack_init,
group_membership.g_membership);
bv_reverse((bit_vector_t*)&group_membership.w_ack_init);
bv_bit_set((bit_vector_t*)&group_membership.w_ack_init, (uint8_t)machine_id);
group_membership.num_of_alive_remotes =
bv_no_setted_bits(group_membership.g_membership);
seqlock_unlock(&group_membership.lock);
if (group_membership.num_of_alive_remotes < (machine_num / 2)) {
colored_printf(RED, "Majority is down!\n");
exit(-1);
}
}
static inline uint8_t
group_membership_has_changed(spacetime_group_membership* last_group_membership,
uint16_t worker_lid)
{
uint32_t debug_lock_free_membership_read_cntr = 0;
spacetime_group_membership lock_free_read_group_membership;
do { // Lock free read of group membership
if (ENABLE_ASSERTIONS) {
debug_lock_free_membership_read_cntr++;
if (debug_lock_free_membership_read_cntr == M_4) {
printf("Worker %u stuck on a lock-free read (for group membership)\n",
worker_lid);
debug_lock_free_membership_read_cntr = 0;
}
}
lock_free_read_group_membership =
*((spacetime_group_membership*)&group_membership);
} while (!(seqlock_version_is_same_and_valid(
&group_membership.lock, &lock_free_read_group_membership.lock)));
for (int i = 0; i < GROUP_MEMBERSHIP_ARRAY_SIZE; i++)
if (!bv_are_equal(lock_free_read_group_membership.g_membership,
last_group_membership->g_membership)) {
*last_group_membership = lock_free_read_group_membership;
return 1;
}
return 0;
}
/* ---------------------------------------------------------------------------
----------------------------------- LATENCY -------------------------------
---------------------------------------------------------------------------*/
// Add latency to histogram (in microseconds)
static inline void
bookkeep_latency(int useconds, uint8_t op)
{
uint32_t* latency_array;
int* max_latency_ptr;
switch (op) {
case ST_OP_PUT:
latency_array = latency_count.write_reqs;
max_latency_ptr = &latency_count.max_write_latency;
break;
case ST_OP_GET:
latency_array = latency_count.read_reqs;
max_latency_ptr = &latency_count.max_read_latency;
break;
default:
assert(0);
}
latency_count.total_measurements++;
if (useconds > MAX_LATENCY)
latency_array[LATENCY_BUCKETS]++;
else
latency_array[useconds / LATENCY_PRECISION]++;
if (*max_latency_ptr < useconds) *max_latency_ptr = useconds;
}
// Necessary bookkeeping to initiate the latency measurement
static inline void
start_latency_measurement(struct timespec* start)
{
clock_gettime(CLOCK_MONOTONIC, start);
}
static inline void
stop_latency_measurment(uint8_t req_opcode, struct timespec* start)
{
struct timespec end;
clock_gettime(CLOCK_MONOTONIC, &end);
int useconds = (int)(((end.tv_sec - start->tv_sec) * 1000000) +
((end.tv_nsec - start->tv_nsec) / 1000));
if (ENABLE_ASSERTIONS) assert(useconds >= 0);
// printf("Latency of %s %u us\n", code_to_str(req_opcode), useconds);
bookkeep_latency(useconds, req_opcode);
}
static inline void
stop_latency_of_completed_writes(spacetime_op_t* ops, uint16_t worker_lid,
struct timespec* stopwatch)
{
if (machine_id == 0 && worker_lid == worker_measuring_latency)
if (ops[0].op_meta.opcode == ST_OP_PUT &&
(ops[0].op_meta.state == ST_MISS ||
ops[0].op_meta.state == ST_PUT_COMPLETE))
stop_latency_measurment(ops[0].op_meta.opcode, stopwatch);
}
static inline void
stop_latency_of_completed_reads(spacetime_op_t* ops, uint16_t worker_lid,
struct timespec* stopwatch)
{
if (machine_id == 0 && worker_lid == worker_measuring_latency)
if (ops[0].op_meta.opcode == ST_OP_GET &&
(ops[0].op_meta.state == ST_MISS ||
ops[0].op_meta.state == ST_GET_COMPLETE))
stop_latency_measurment(ops[0].op_meta.opcode, stopwatch);
}
/* ---------------------------------------------------------------------------
---------------------------------- Refill Requests ---------------------------
---------------------------------------------------------------------------*/
static inline int
refill_ops(uint32_t* trace_iter, uint16_t worker_lid,
struct spacetime_trace_command* trace, spacetime_op_t* ops,
uint32_t* refilled_per_ops_debug_cnt, struct timespec* start,
spacetime_op_t** n_hottest_keys_in_ops_get,
spacetime_op_t** n_hottest_keys_in_ops_put)
{
static uint8_t first_iter_has_passed[MAX_WORKERS_PER_MACHINE] = {0};
int refilled_ops = 0, node_suspected = -1;
for (int i = 0; i < max_batch_size; i++) {
if (ENABLE_ASSERTIONS && first_iter_has_passed[worker_lid] == 1) {
assert(ops[i].op_meta.opcode == ST_OP_PUT ||
ops[i].op_meta.opcode == ST_OP_GET ||
(is_CR == 0 && ops[i].op_meta.opcode == ST_OP_RMW));
assert(ops[i].op_meta.state == ST_PUT_COMPLETE ||
ops[i].op_meta.state == ST_GET_COMPLETE ||
ops[i].op_meta.state == ST_PUT_SUCCESS ||
ops[i].op_meta.state == ST_REPLAY_SUCCESS ||
ops[i].op_meta.state == ST_NEW ||
ops[i].op_meta.state == ST_MISS ||
ops[i].op_meta.state == ST_PUT_STALL ||
ops[i].op_meta.state == ST_REPLAY_COMPLETE ||
ops[i].op_meta.state == ST_IN_PROGRESS_PUT ||
//<RMW>
ops[i].op_meta.state == ST_RMW_STALL ||
ops[i].op_meta.state == ST_RMW_ABORT ||
ops[i].op_meta.state == ST_RMW_SUCCESS ||
ops[i].op_meta.state == ST_RMW_COMPLETE ||
ops[i].op_meta.state == ST_IN_PROGRESS_RMW ||
// ops[i].op_meta.state ==
// ST_IN_PROGRESS_PUT
//|| <RMW>
ops[i].op_meta.state == ST_IN_PROGRESS_GET ||
ops[i].op_meta.state == ST_IN_PROGRESS_REPLAY ||
ops[i].op_meta.state ==
ST_OP_MEMBERSHIP_CHANGE || /// TODO check this
ops[i].op_meta.state ==
ST_OP_MEMBERSHIP_COMPLETE || /// TODO check this
ops[i].op_meta.state == ST_PUT_COMPLETE_SEND_VALS ||
ops[i].op_meta.state == ST_GET_STALL);
}
if (first_iter_has_passed[worker_lid] == 0 ||
ops[i].op_meta.state == ST_MISS ||
ops[i].op_meta.state == ST_PUT_COMPLETE ||
ops[i].op_meta.state == ST_RMW_ABORT ||
ops[i].op_meta.state == ST_RMW_COMPLETE ||
ops[i].op_meta.state == ST_OP_MEMBERSHIP_COMPLETE ||
ops[i].op_meta.state == ST_GET_COMPLETE) {
if (first_iter_has_passed[worker_lid] != 0) {
if (ENABLE_REQ_PRINTS && worker_lid < MAX_THREADS_TO_PRINT)
colored_printf(
GREEN,
"W%d--> Key Hash:%" PRIu64
"\n\t\tType: %s, version %d, tie-b: %d, value(len-%d): %c\n",
worker_lid, ((uint64_t*)&ops[i].op_meta.key)[0],
code_to_str(ops[i].op_meta.state), ops[i].op_meta.ts.version,
ops[i].op_meta.ts.tie_breaker_id, ops[i].op_meta.val_len,
ops[i].value[0]);
/// Stats
if (ops[i].op_meta.state != ST_MISS) {
if (ops[i].op_meta.state != ST_RMW_ABORT)
w_stats[worker_lid].completed_ops_per_worker +=
ENABLE_COALESCE_OF_HOT_REQS ? ops[i].no_coales : 1;
} else
w_stats[worker_lid].reqs_missed_in_kvs++;
if (ops[i].op_meta.state == ST_PUT_COMPLETE)
w_stats[worker_lid].completed_wrs_per_worker++;
else if (ops[i].op_meta.state == ST_RMW_COMPLETE)
w_stats[worker_lid].completed_rmws_per_worker++;
else if (ops[i].op_meta.state == ST_RMW_ABORT)
w_stats[worker_lid].aborted_rmws_per_worker++;
// reset op bucket
ops[i].no_coales = 1;
ops[i].op_meta.state = ST_EMPTY;
ops[i].op_meta.opcode = ST_EMPTY;
refilled_per_ops_debug_cnt[i] = 0;
refilled_ops++;
}
if (ENABLE_ASSERTIONS)
assert(trace[*trace_iter].opcode == ST_OP_PUT ||
trace[*trace_iter].opcode == ST_OP_RMW ||
trace[*trace_iter].opcode == ST_OP_GET);
if (machine_id == 0 && worker_lid == worker_measuring_latency && i == 0)
start_latency_measurement(start);
/// INSERT new req(s) to ops
uint8_t key_id;
if (ENABLE_COALESCE_OF_HOT_REQS &&
trace[*trace_iter].opcode != ST_OP_RMW) {
// see if you could coalesce any requests
spacetime_op_t** n_hottest_keys_in_ops;
do {
key_id = trace[*trace_iter].key_id;
n_hottest_keys_in_ops = trace[*trace_iter].opcode == ST_OP_GET
? n_hottest_keys_in_ops_get
: n_hottest_keys_in_ops_put;
// if we can coalesce (a hot) req
if (key_id < COALESCE_N_HOTTEST_KEYS && // is a hot key
n_hottest_keys_in_ops[key_id] !=
NULL && // exists in the ops array
n_hottest_keys_in_ops[key_id]->op_meta.opcode ==
trace[*trace_iter]
.opcode) // has the same code with the last inserted
{
n_hottest_keys_in_ops[key_id]->no_coales++;
*trace_iter =
trace[*trace_iter + 1].opcode != NOP ? *trace_iter + 1 : 0;
} else
break;
} while (1);
if (key_id < COALESCE_N_HOTTEST_KEYS)
n_hottest_keys_in_ops[key_id] = &ops[i];
}
ops[i].op_meta.state = ST_NEW;
ops[i].op_meta.opcode =
(uint8_t)(CR_ENABLE_ALL_NODES_GETS_EXCEPT_HEAD && machine_id != 0
? ST_OP_GET
: trace[*trace_iter].opcode);
memcpy(&ops[i].op_meta.key, &trace[*trace_iter].key_hash,
sizeof(spacetime_key_t));
if (ops[i].op_meta.opcode == ST_OP_PUT ||
ops[i].op_meta.opcode == ST_OP_RMW)
memset(ops[i].value, ((uint8_t)'a' + machine_id), ST_VALUE_SIZE);
else if (ENABLE_READ_COMPLETE_AFTER_VAL_RECV_OF_HOT_REQS) {
// if its a read reset the timestamp
ops[i].op_meta.ts.version = 0;
ops[i].op_meta.ts.tie_breaker_id = 0;
}
ops[i].RMW_flag = ops[i].op_meta.opcode == ST_OP_RMW ? 1 : 0;
ops[i].op_meta.val_len = (uint8)(ops[i].op_meta.opcode == ST_OP_GET
? 0
: ST_VALUE_SIZE >> SHIFT_BITS);
// instead of MOD add
*trace_iter = trace[*trace_iter + 1].opcode != NOP ? *trace_iter + 1 : 0;
if (ENABLE_REQ_PRINTS && worker_lid < MAX_THREADS_TO_PRINT)
colored_printf(RED, "W%d--> Op: %s, hash(1st 8B):%" PRIu64 "\n",
worker_lid, code_to_str(ops[i].op_meta.opcode),
((uint64_t*)&ops[i].op_meta.key)[0]);
} else
refilled_per_ops_debug_cnt[i]++;
}
if (refilled_ops == 0) w_stats[worker_lid].wasted_loops++;
if (first_iter_has_passed[worker_lid] == 0)
first_iter_has_passed[worker_lid] = 1;
if (ENABLE_ASSERTIONS)
for (int i = 0; i < max_batch_size; i++)
assert(ops[i].op_meta.opcode == ST_OP_PUT ||
ops[i].op_meta.opcode == ST_OP_GET ||
(ops[i].op_meta.opcode == ST_OP_RMW && is_CR == 0));
return node_suspected;
}
#endif // HERMES_INLINE_UTIL_H
================================================
FILE: include/hermes/spacetime.h
================================================
//
// Created by akatsarakis on 04/05/18.
//
#ifndef HERMES_SPACETIME_H
#define HERMES_SPACETIME_H
// Optik Options
#ifndef CORE_NUM
#define DEFAULT
#define CORE_NUM 8
#endif
#include "../utils/bit_vector.h"
#include "../utils/concur_ctrl.h"
#include "config.h"
#include "hrd.h"
#include "mica.h"
#define SPACETIME_NUM_KEYS (1000 * 1000)
#define SPACETIME_NUM_BKTS (2 * 1024 * 1024)
#define SPACETIME_LOG_CAP (1024 * 1024 * 1024)
//#define SPACETIME_NUM_KEYS (60 * 1000 * 1000)
//#define SPACETIME_NUM_BKTS (64 * 1024 * 1024)
//#define SPACETIME_LOG_CAP (4 * ((unsigned long long) M_1024)) //(1024 * 1024
//* 1024)
#define ST_VALUE_SIZE (KVS_VALUE_SIZE - sizeof(spacetime_object_meta))
// Special EMPTY opcodes
#define NOP 150 // trace
#define LAST_WRITER_ID_EMPTY 127 // 255
#define ST_OP_BUFFER_INDEX_EMPTY 255
/////////////////////////////////////////////
//// ENUMS
/////////////////////////////////////////////
/// WARNING the monotonically increasing assigned numbers to States are used for
/// comparisons (do not reorder / change numbers)
// States
typedef enum {
VALID_STATE = 1,
INVALID_STATE,
INVALID_WRITE_STATE,
WRITE_STATE,
REPLAY_STATE,
} __attribute__((packed)) hermes_states_t;
// Input Opcodes
typedef enum {
ST_OP_GET = 111,
ST_OP_PUT,
ST_OP_RMW,
ST_OP_INV,
ST_OP_ACK,
ST_OP_VAL,
ST_OP_CRD,
ST_OP_MEMBERSHIP_CHANGE,
ST_OP_MEMBERSHIP_COMPLETE // 119
} __attribute__((packed)) input_opcodes_t;
// Response Opcodes
typedef enum {
ST_GET_COMPLETE = 121,
ST_PUT_SUCCESS, // broadcast invs
ST_REPLAY_SUCCESS, // broadcast invs
ST_INV_SUCCESS, // send ack
ST_ACK_SUCCESS,
ST_LAST_ACK_SUCCESS, // complete local write
ST_LAST_ACK_NO_BCAST_SUCCESS, // complete local write
ST_PUT_COMPLETE, // broadcast invs
ST_VAL_SUCCESS, // 129
ST_MISS, // 130
ST_GET_STALL,
ST_PUT_STALL,
ST_PUT_COMPLETE_SEND_VALS,
ST_SEND_CRD, // 134
// RMW opcodes
ST_RMW_SUCCESS, // 135
ST_RMW_STALL,
ST_RMW_COMPLETE,
ST_RMW_ABORT,
ST_OP_INV_ABORT, // 139 //send inv instead of ACK
} __attribute__((packed)) response_opcodes_t;
// ops bucket states
typedef enum {
ST_EMPTY = 140,
ST_NEW,
ST_COMPLETE,
ST_IN_PROGRESS_PUT,
ST_IN_PROGRESS_REPLAY,
ST_REPLAY_COMPLETE,
ST_IN_PROGRESS_GET, // Used only in Chain Replication
ST_REPLAY_COMPLETE_SEND_VALS,
ST_IN_PROGRESS_RMW,
ST_RMW_COMPLETE_SEND_VALS // 149
} __attribute__((packed)) op_bucket_states_t;
// failure detection (deprecated)
typedef enum {
ST_OP_HEARTBEAT = 151, // WARNING: 150 opcode is used (see NOP define)!!
ST_OP_SUSPICION,
ST_INV_OUT_OF_GROUP
} __attribute__((packed)) fs_ops_t;
// receive_buff_types
typedef enum {
ST_INV_BUFF = 161,
ST_ACK_BUFF,
ST_VAL_BUFF,
ST_CRD_BUFF
} __attribute__((packed)) rcv_buff_types_t;
/////////////////////////////////////////////
//// Hermes(msg and KV -- spacetime) structs
/////////////////////////////////////////////
// Fixed-size 8 (or 16) byte keys
typedef struct {
// uint64 __unused; // This should be 8B ////// Uncomment this for
// fixed-size 16 byte keys instead
uint64_t bkt : 48;
unsigned int tag : 16;
} spacetime_key_t;
typedef volatile struct {
hermes_states_t state;
bit_vector_t ack_bv;
uint8_t RMW_flag : 1;
uint8_t last_writer_id : 7;
uint8_t op_buffer_index; // TODO change to uint16_t for a buffer >= 256
conc_ctrl_t cctrl;
timestamp_t last_local_write_ts;
} spacetime_object_meta;
typedef struct {
spacetime_key_t key; /* This must be the 1st field and 8B or 16B aligned */
uint8_t opcode; // both recv / resp //TODO create a union
union {
uint8_t state; // HERMES: used by spacetime_op_t
uint8_t sender; // HERMES: used by spacetime_inv/ack/val_t
uint8_t initiator; // CR: used by spacetime_inv/ack
};
union {
uint8_t val_len; // HERMES: unused for spacetime_ack_t and spacetime_val_t
// (align for using a single memcpy)
uint8_t buff_idx; // CR: used for spacetime_ack_t buffer index of
// write initiated this req
};
timestamp_t ts;
} spacetime_op_meta_t, spacetime_ack_t, spacetime_val_t;
typedef struct {
spacetime_op_meta_t op_meta; // op_t/inv_t: uses the state/sender part of the
// op_meta union (not sender/state)
union {
struct { // Hermes struct
uint8_t RMW_flag : 1; // 1 indicates RMWs while 0 normal writes
uint16_t no_coales : 15; // used only for skew optimizations
};
struct { // CR struct
uint8_t buff_idx; // for spacetime_inv_t buffer index of write
// initiated this req
uint8_t initiator; // for spacetime_inv_t buffer index of write
// initiated this req
};
};
uint8_t value[ST_VALUE_SIZE];
} spacetime_op_t, spacetime_inv_t;
typedef struct {
volatile uint8_t num_of_alive_remotes;
volatile bit_vector_t g_membership;
volatile bit_vector_t w_ack_init;
seqlock_t lock;
} spacetime_group_membership;
struct spacetime_kv {
// TODO may add kvs stats
struct mica_kv hash_table;
};
struct spacetime_trace_command {
spacetime_key_t key_hash;
uint8_t opcode;
uint8_t key_id; // stores key ids 0-254 otherwise it is set to 255 to
// indicate other key ids
};
void spacetime_init(int spacetime_id);
void spacetime_populate_fixed_len(struct spacetime_kv* kv, int n, int val_len);
///////////////////////////////////////
//////////////////// Hermes
///////////////////////////////////////
enum hermes_batch_type_t {
local_ops,
local_ops_after_membership_change,
invs,
acks,
vals
};
void hermes_batch_ops_to_KVS(enum hermes_batch_type_t type, uint8_t* op_array,
int op_num, uint16_t sizeof_op_elem,
spacetime_group_membership curr_membership,
int* node_suspected,
spacetime_op_t* read_write_ops, uint8_t thread_id);
///////////////////////////////////////
//////////////////// CR(AQ)
///////////////////////////////////////
enum cr_type_t {
Local_ops, // All nodes
Remote_writes, // Head
Remote_reads, // Tail
Invs, // All except Head
Acks // All except Tail
};
void cr_batch_ops_to_KVS(enum cr_type_t cr_type, uint8_t* op_array, int op_num,
uint16_t sizeof_op_elem,
spacetime_op_t* read_write_op);
///////////////////////////////////////
//////////////////// Helpers
///////////////////////////////////////
static inline uint8_t
is_last_ack(bit_vector_t gathered_acks,
spacetime_group_membership curr_g_membership)
{
bv_and(&gathered_acks, curr_g_membership.g_membership);
return bv_are_equal(gathered_acks, curr_g_membership.g_membership);
}
// TODO: adapt and use the following functions to re-enable variable length
// object support
static inline uint8_t
get_val_len(struct mica_op* op_t)
{
return (op_t->val_len >> SHIFT_BITS) - sizeof(spacetime_op_meta_t);
}
static inline uint8_t
set_val_len(spacetime_op_meta_t* op_t)
{
return (op_t->val_len >> SHIFT_BITS) + sizeof(spacetime_op_meta_t);
}
extern struct spacetime_kv kv;
extern spacetime_group_membership group_membership;
#endif // HERMES_SPACETIME_H
================================================
FILE: include/hermes/util.h
================================================
//
// Created by akatsarakis on 15/03/18.
//
#ifndef HERMES_UTIL_H
#define HERMES_UTIL_H
#include <stdint.h>
#include <stdio.h>
#include <time.h>
#include "config.h"
#include "hrd.h"
#include "spacetime.h"
struct worker_stats {
long long completed_ops_per_worker;
long long completed_wrs_per_worker;
long long completed_rmws_per_worker;
long long aborted_rmws_per_worker;
long long reqs_missed_in_kvs;
long long issued_invs_per_worker;
long long issued_acks_per_worker;
long long issued_vals_per_worker;
long long issued_crds_per_worker;
long long issued_packet_invs_per_worker;
long long issued_packet_acks_per_worker;
long long issued_packet_vals_per_worker;
long long issued_packet_crds_per_worker;
long long inv_ss_completions_per_worker;
long long ack_ss_completions_per_worker;
long long val_ss_completions_per_worker;
long long crd_ss_completions_per_worker;
long long received_invs_per_worker;
long long received_acks_per_worker;
long long received_vals_per_worker;
long long received_crds_per_worker;
long long received_packet_invs_per_worker;
long long received_packet_acks_per_worker;
long long received_packet_vals_per_worker;
long long received_packet_crds_per_worker;
long long received_acks_stalled; // for faking tail-latency
long long stalled_time_per_worker;
long long wasted_loops;
long long total_loops;
double empty_reqs_per_trace;
long long cold_keys_per_trace;
double tot_empty_reqs_per_trace;
};
struct stats {
double xput_per_worker[MAX_WORKERS_PER_MACHINE];
double rmw_xput_per_worker[MAX_WORKERS_PER_MACHINE];
double rmw_abort_rate_per_worker[MAX_WORKERS_PER_MACHINE];
double issued_invs_avg_coalesing[MAX_WORKERS_PER_MACHINE];
double issued_acks_avg_coalesing[MAX_WORKERS_PER_MACHINE];
double issued_vals_avg_coalesing[MAX_WORKERS_PER_MACHINE];
double issued_crds_avg_coalesing[MAX_WORKERS_PER_MACHINE];
double received_invs_avg_coalesing[MAX_WORKERS_PER_MACHINE];
double received_acks_avg_coalesing[MAX_WORKERS_PER_MACHINE];
double received_vals_avg_coalesing[MAX_WORKERS_PER_MACHINE];
double received_crds_avg_coalesing[MAX_WORKERS_PER_MACHINE];
double percentage_of_wasted_loops[MAX_WORKERS_PER_MACHINE];
double completed_reqs_per_loop[MAX_WORKERS_PER_MACHINE];
// long long issued_packet_acks_per_worker;
double batch_size_per_worker[MAX_WORKERS_PER_MACHINE];
double empty_reqs_per_worker[MAX_WORKERS_PER_MACHINE];
double stalled_time_per_worker[MAX_WORKERS_PER_MACHINE];
double average_coalescing_per_worker[MAX_WORKERS_PER_MACHINE];
double acks_per_worker[MAX_WORKERS_PER_MACHINE];
double invs_per_worker[MAX_WORKERS_PER_MACHINE];
double updates_per_worker[MAX_WORKERS_PER_MACHINE];
double write_ratio_per_worker[MAX_WORKERS_PER_MACHINE];
};
// init all stats to 0
static inline void
init_stats(struct worker_stats* w_stats)
{
memset(w_stats, 0, sizeof(struct worker_stats) * MAX_WORKERS_PER_MACHINE);
}
void trace_init(struct spacetime_trace_command** trace, uint16_t worker_lid);
void* run_worker(void* arg);
void* print_stats_thread(void* no_arg);
void dump_latency_stats(void);
// Maybe inline these
uint8_t is_state_code(uint8_t code);
uint8_t is_input_code(uint8_t code);
uint8_t is_response_code(uint8_t code);
uint8_t is_bucket_state_code(uint8_t code);
int spawn_stats_thread(void);
char* code_to_str(uint8_t code);
void setup_kvs_buffs(spacetime_op_t** ops, spacetime_inv_t** inv_recv_ops,
spacetime_ack_t** ack_recv_ops,
spacetime_val_t** val_recv_ops);
extern dbit_vector_t* g_share_qs_barrier;
extern volatile struct worker_stats w_stats[MAX_WORKERS_PER_MACHINE];
#endif // HERMES_UTIL_H
================================================
FILE: include/mica-herd/city.h
================================================
// city.h - cityhash-c
// CityHash on C
// Copyright (c) 2011-2012, Alexander Nusov
//
// - original copyright notice -
// Copyright (c) 2011 Google, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//
// CityHash, by Geoff Pike and Jyrki Alakuijala
//
// This file provides a few functions for hashing strings. On x86-64
// hardware in 2011, CityHash64() is faster than other high-quality
// hash functions, such as Murmur. This is largely due to higher
// instruction-level parallelism. CityHash64() and CityHash128() also perform
// well on hash-quality tests.
//
// CityHash128() is optimized for relatively long strings and returns
// a 128-bit hash. For strings more than about 2000 bytes it can be
// faster than CityHash64().
//
// Functions in the CityHash family are not suitable for cryptography.
//
// WARNING: This code has not been tested on big-endian platforms!
// It is known to work well on little-endian platforms that have a small penalty
// for unaligned reads, such as current Intel and AMD moderate-to-high-end CPUs.
//
// By the way, for some hash functions, given strings a and b, the hash
// of a+b is easily derived from the hashes of a and b. This property
// doesn't hold for any hash functions in this file.
#ifndef CITY_HASH_H_
#define CITY_HASH_H_
#include <stdint.h>
#include <stdlib.h>
typedef uint8_t uint8;
typedef uint32_t uint32;
typedef uint64_t uint64;
typedef struct _uint128 uint128;
struct _uint128 {
uint64 first;
uint64 second;
};
#define Uint128Low64(x) (x).first
#define Uint128High64(x) (x).second
// Hash function for a byte array.
uint64 CityHash64(const char* buf, size_t len);
// Hash function for a byte array. For convenience, a 64-bit seed is also
// hashed into the result.
uint64 CityHash64WithSeed(const char* buf, size_t len, uint64 seed);
// Hash function for a byte array. For convenience, two seeds are also
// hashed into the result.
uint64 CityHash64WithSeeds(const char* buf, size_t len, uint64 seed0,
uint64 seed1);
// Hash function for a byte array.
uint128 CityHash128(const char* s, size_t len);
// Hash function for a byte array. For convenience, a 128-bit seed is also
// hashed into the result.
uint128 CityHash128WithSeed(const char* s, size_t len, uint128 seed);
#endif // CITY_HASH_H_
================================================
FILE: include/mica-herd/hrd.h
================================================
#ifndef HRD_H
#define HRD_H
#include <assert.h>
#include <errno.h>
#include <numaif.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#include <infiniband/verbs.h>
#include <libmemcached/memcached.h>
#include <malloc.h>
#include <time.h>
#include "sizes.h"
//<vasilis> Multicast
// TODO we do not use hw multicast because it helps only on master-based
// patterns
//#include <rdma/rdma_cma.h>
#include <arpa/inet.h>
#include <byteswap.h>
#include <netdb.h>
#include <netinet/in.h>
#include <sys/socket.h>
// <vasilis>
#define USE_BIG_OBJECTS 0
#define EXTRA_CACHE_LINES 0
#define BASE_VALUE_SIZE 46 // max is --> 46
#define SHIFT_BITS \
(USE_BIG_OBJECTS == 1 ? 3 : 0) // number of bits to shift left or right to
// calculate the value length
#define HRD_DEFAULT_PSN \
3185 /* PSN for all queues */ // starting Packet Sequence Number
#define HRD_DEFAULT_QKEY 0x11111111
#define HRD_QP_NAME_SIZE 200 /* Size (in bytes) of a queue pair name */
#define HRD_RESERVED_NAME_PREFIX "__HRD_RESERVED_NAME_PREFIX"
#define KVS_VALUE_SIZE \
(USE_BIG_OBJECTS == 1 \
? ((EXTRA_CACHE_LINES * 64) + BASE_VALUE_SIZE) \
: BASE_VALUE_SIZE) //(169 + 64)// 46 + 64 + 64//32 //(46 + 64)
#define HUGE_PAGE_SIZE 2097152
#define LEVERAGE_TLB_COALESCING 1
/*
* Small max_inline_data reduces the QP's max WQE size, which reduces the
* DMA size in doorbell method of WQE fetch.
*/
#define HRD_MAX_INLINE \
188 //(USE_BIG_OBJECTS == 1 ? ((EXTRA_CACHE_LINES * 64) + 60) : 60) //60 is
// what kalia had here//
// This is required for ROCE not sure yet why
// <vasilis>
#define IB_PHYS_PORT 1
// </vasilis>
// <akatsarakis>
#define USE_HUGE_PAGES 1
// </akatsarakis>
#ifndef likely
#define likely(x) __builtin_expect(!!(x), 1)
#endif
#ifndef unlikely
#define unlikely(x) __builtin_expect(!!(x), 0)
#endif
/* Compare, print, and exit */
#define CPE(val, msg, err_code) \
if (unlikely(val)) { \
fprintf(stderr, msg); \
fprintf(stderr, " Error %d \n", err_code); \
exit(err_code); \
}
/* vasilis added a ceiling and a MAX*/
#define CEILING(x, y) (((x) + (y)-1) / (y))
#define MAX(x, y) (x > y ? x : y)
int is_roce;
int machine_id;
char *remote_IP, *local_IP;
/* Registry info about a QP */
struct hrd_qp_attr {
char name[HRD_QP_NAME_SIZE];
// ROCE
uint64_t
gid_global_interface_id; // Store the gid fields separately because I
uint64_t gid_global_subnet_prefix; // don't like unions. Needed for RoCE only
/* Info about the RDMA buffer associated with this QP */
uintptr_t buf_addr;
uint32_t buf_size;
uint32_t rkey;
int lid;
int qpn;
uint8_t sl;
};
struct hrd_ud_ctrl_blk {
int local_hid; /* Local ID on the machine this process runs on */
/* Info about the device/port to use for this control block */
struct ibv_context* ctx;
int device_id; /* Resovled by libhrd from @port_index */
int dev_port_id; /* 1-based within dev @device_id. Resolved by libhrd */
int numa_node_id; /* NUMA node id */
struct ibv_pd* pd; /* A protection domain for this control block */
/* Datagram QPs */
int num_dgram_qps;
struct ibv_qp** dgram_qp;
struct ibv_cq **dgram_send_cq, **dgram_recv_cq;
volatile uint8_t* dgram_buf; /* A buffer for RECVs on dgram QPs */
int* recv_q_depth;
int* send_q_depth;
int dgram_buf_shm_key;
struct ibv_mr* dgram_buf_mr;
};
/* Major initialzation functions */
struct hrd_ud_ctrl_blk* hrd_ud_ctrl_blk_init(
int local_hid, int port_index,
int numa_node_id, /* -1 means don't use hugepages */
int num_dgram_qps, int dgram_buf_size, int dgram_buf_shm_key,
int* recv_q_depth, int* send_q_depth);
int hrd_ud_ctrl_blk_destroy(struct hrd_ud_ctrl_blk* cb);
/* RDMA resolution functions */
struct ibv_device* hrd_resolve_port_index(struct hrd_ud_ctrl_blk* cb,
int port_index);
uint16_t hrd_get_local_lid(struct ibv_context* ctx, int port_id);
void hrd_create_dgram_qps(struct hrd_ud_ctrl_blk* cb);
/* Fill @wc with @num_comps comps from this @cq. Exit on error. */
static inline uint32_t
hrd_poll_cq(struct ibv_cq* cq, int num_comps, struct ibv_wc* wc)
{
int comps = 0;
uint32_t debug_cnt = 0;
while (comps < num_comps) {
if (debug_cnt > M_256) {
printf("Someone is stuck waiting for a completion %d / %d \n", comps,
num_comps);
debug_cnt = 0;
}
int new_comps = ibv_poll_cq(cq, num_comps - comps, &wc[comps]);
if (new_comps != 0) {
// printf("I see completions %d\n", new_comps);
/* Ideally, we should check from comps -> new_comps - 1 */
if (wc[comps].status != 0) {
fprintf(stderr, "Bad wc status %d\n", wc[comps].status);
exit(0);
}
comps += new_comps;
}
debug_cnt++;
}
return debug_cnt;
}
static inline struct ibv_mr*
register_buffer(struct ibv_pd* pd, void* buf, uint32_t size)
{
int ib_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ |
IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_ATOMIC;
struct ibv_mr* mr = ibv_reg_mr(pd, (char*)buf, size, ib_flags);
assert(mr != NULL);
return mr;
}
/* Registry functions */
void hrd_publish(const char* key, void* value, int len);
int hrd_get_published(const char* key, void** value);
///* Publish the nth connected queue pair from this cb with this name */
// void hrd_publish_conn_qp(struct hrd_ud_ctrl_blk *cb, int n, const char
// *qp_name);
/* Publish the nth datagram queue pair from this cb with this name */
void hrd_publish_dgram_qp(struct hrd_ud_ctrl_blk* cb, int n,
const char* qp_name, uint8_t sl);
struct hrd_qp_attr* hrd_get_published_qp(const char* qp_name);
/* Utility functions */
static inline uint32_t
hrd_fastrand(uint64_t* seed)
{
*seed = *seed * 1103515245 + 12345;
return (uint32_t)(*seed >> 32);
}
void* hrd_malloc_socket(int shm_key, uint64_t size, int socket_id);
int hrd_free(int shm_key, void* shm_buf);
char* hrd_getenv(const char* name);
// Like printf, but colorfur. Limited to 1000 characters.
typedef enum { YELLOW = 0, RED, GREEN, CYAN } color_print_t;
void colored_printf(color_print_t color, const char* format, ...);
extern char dev_name[50];
#endif /* HRD_H */
================================================
FILE: include/mica-herd/mica.h
================================================
#ifndef MICA_H
#define MICA_H
#include <stdint.h>
#include "city.h"
#include "hrd.h"
/*
* The polling logic in HERD requires the following:
* 1. 0 < MICA_OP_GET < MICA_OP_PUT < HERD_OP_GET < HERD_OP_PUT
* 2. HERD_OP_GET = MICA_OP_GET + HERD_MICA_OFFSET
* 3. HERD_OP_PUT = MICA_OP_PUT + HERD_MICA_OFFSET
*
* This allows us to detect HERD requests by checking if the request region
* opcode is more than MICA_OP_PUT. And then we can convert a HERD opcode to
* a MICA opcode by subtracting HERD_MICA_OFFSET from it.
*/
#define MICA_OP_PUT 112
/* Ensure that a mica_op is cacheline aligned */
#define MICA_OP_METADATA \
(sizeof(struct mica_key) + sizeof(uint8_t) + sizeof(uint8_t))
#define MICA_MIN_VALUE (64 - MICA_OP_METADATA)
#define MICA_MAX_VALUE \
(USE_BIG_OBJECTS == 1 ? (MICA_MIN_VALUE + (EXTRA_CACHE_LINES * 64)) \
: MICA_MIN_VALUE)
#define MICA_LOG_BITS 40
#define MICA_INDEX_SHM_KEY 3185
#define MICA_LOG_SHM_KEY 4185
/*
* Debug values:
* 0: No safety checks on fast path
* 1: Sanity checks for arguments
* 2: Pretty print GET/PUT operations
*/
#define MICA_DEBUG 0
struct mica_resp {
uint8_t type;
uint8_t val_len;
uint16_t unused[3]; /* Make val_ptr 8-byte aligned */
uint8_t* val_ptr;
};
/* Fixed-size 16 byte keys */
struct mica_key {
unsigned long long __unused : 64;
unsigned int bkt : 32;
unsigned int server : 16;
unsigned int tag : 16;
};
struct mica_op {
struct mica_key key; /* This must be the 1st field and 16B aligned */
uint8_t opcode;
uint8_t val_len;
uint8_t value[MICA_MAX_VALUE];
};
struct mica_slot {
uint32_t in_use : 1;
uint32_t tag : (64 - MICA_LOG_BITS - 1);
uint64_t offset : MICA_LOG_BITS;
};
struct mica_bkt {
struct mica_slot slots[8];
};
struct mica_kv {
struct mica_bkt* ht_index;
uint8_t* ht_log;
/* Metadata */
int instance_id; /* ID of this MICA instance. Used for shm keys */
uint64_t num_bkts; /* Number of buckets requested by user */
uint64_t bkt_mask; /* Mask down from a mica_key's @bkt to a bucket */
uint64_t log_cap; /* Capacity of circular log in bytes */
uint64_t log_mask; /* Mask down from a slot's @offset to a log offset */
/* State */
uint64_t log_head;
/* Stats */
long long num_insert_op; /* Number of PUT requests executed */
long long num_index_evictions; /* Number of entries evicted from index */
};
void mica_init(struct mica_kv* kv, int instance_id, int node_id, int num_bkts,
uint64_t log_cap);
/* Single-key INSERT */
void mica_insert_one(struct mica_kv* kv, struct mica_op* op,
struct mica_resp* res);
/* Helpers */
uint128* mica_gen_keys(int n);
///* Debug functions */
void mica_print_op(struct mica_op* op);
#endif
================================================
FILE: include/mica-herd/sizes.h
================================================
#define K_32 32768
#define K_64 65536
#define K_128 131072
#define K_128_ 131071
#define K_256 262144
#define K_256_ 262143
#define K_512 524288
#define K_512_ 524287
#define M_1 1048576
#define M_1_ 1048575
#define M_2 2097152
#define M_2_ 2097151
#define M_4 4194304
#define M_4_ 4194303
#define M_8 8388608
#define M_8_ 8388607
#define M_16 16777216
#define M_16_ 16777215
#define M_32 33554432
#define M_32_ 33554431
#define M_128 134217728
#define M_128_ 134217727
#define M_256 268435456
#define M_256_ 268435455
#define M_512 536870912
#define M_512_ 536870911
#define M_1024 1073741824
#define M_1024_ 1073741823
#define MILLION 1000000
================================================
FILE: include/utils/bit_vector.h
================================================
//
// Created by akatsarakis on 11/12/18.
//
#ifndef HERMES_BIT_VECTOR_H
#define HERMES_BIT_VECTOR_H
#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
// Change accordingly
#define BV_BIT_VECTOR_SIZE \
8 // Set if you use statical bit vector (bit_vector_t)
#define BV_ENABLE_BIT_VECTOR_ASSERTS 1
// Do not change the following defines
#define BV_CEILING(x, y) (((x) + (y)-1) / (y))
#define BV_BITS_IN_A_BYTE 8
#define BV_BIT_VECTOR_SIZE_IN_BYTES \
BV_CEILING(BV_BIT_VECTOR_SIZE, BV_BITS_IN_A_BYTE)
#define BV_BIT_SLOT(bit) (bit / BV_BITS_IN_A_BYTE)
#define BV_BIT_MOD(bit) ((uint8_t)1 << bit % BV_BITS_IN_A_BYTE)
// print binary numbers
#define BYTE_TO_BINARY_PATTERN "%c%c%c%c%c%c%c%c"
#define BYTE_TO_BINARY(byte) \
(byte & 0x80 ? '1' : '0'), (byte & 0x40 ? '1' : '0'), \
(byte & 0x20 ? '1' : '0'), (byte & 0x10 ? '1' : '0'), \
(byte & 0x08 ? '1' : '0'), (byte & 0x04 ? '1' : '0'), \
(byte & 0x02 ? '1' : '0'), (byte & 0x01 ? '1' : '0')
typedef struct {
uint8_t bit_array[BV_BIT_VECTOR_SIZE_IN_BYTES];
} bit_vector_t;
typedef struct {
uint8_t bv_size; // in bits
uint8_t* bit_array; // bit_array len == ceil(bv_size / 8)
} dbit_vector_t;
// returns the least amount of bytes that required to store x bits
static inline uint16_t
bv_bits_to_bytes(uint16_t bits)
{
return (uint16_t)BV_CEILING(bits, BV_BITS_IN_A_BYTE);
}
/////////////////////////////////////////
/// Internal Bitvector API functions (should not be called directly)
/////////////////////////////////////////
static inline void
bv_init_internal(uint8_t* bit_array, uint16_t size_in_bits)
{
for (int i = 0; i < bv_bits_to_bytes(size_in_bits); ++i)
bit_array[i] = 0;
}
static inline uint8_t
bv_bit_get_internal(const uint8_t* bit_array, uint16_t size_in_bits,
uint8_t bit)
{
if (BV_ENABLE_BIT_VECTOR_ASSERTS) assert(bit < size_in_bits);
return (uint8_t)((bit_array[BV_BIT_SLOT(bit)] & BV_BIT_MOD(bit)) == 0 ? 0
: 1);
}
static inline void
bv_bit_set_internal(uint8_t* bit_array, uint16_t size_in_bits, uint8_t bit)
{
if (BV_ENABLE_BIT_VECTOR_ASSERTS) assert(bit < size_in_bits);
bit_array[BV_BIT_SLOT(bit)] |= BV_BIT_MOD(bit);
}
static inline void
bv_bit_reset_internal(uint8_t* bit_array, uint16_t size_in_bits, uint8_t bit)
{
if (BV_ENABLE_BIT_VECTOR_ASSERTS) assert(bit < size_in_bits);
bit_array[BV_BIT_SLOT(bit)] &= ~(BV_BIT_MOD(bit));
}
static inline void
bv_set_all_internal(uint8_t* bit_array, uint16_t size_in_bits)
{
uint8_t bytes = (uint8_t)bv_bits_to_bytes(size_in_bits);
uint8_t unused = (uint8_t)(bytes * 8 - size_in_bits);
uint8_t last_byte = (uint8_t)(255 >> unused);
for (int i = 0; i < bytes - 1; ++i)
bit_array[i] = 255;
bit_array[bytes - 1] = last_byte;
}
static inline void
bv_reset_all_internal(uint8_t* bit_array, uint16_t size_in_bits)
{
for (int i = 0; i < bv_bits_to_bytes(size_in_bits); ++i)
bit_array[i] = 0;
}
static inline uint8_t
bv_are_equal_internal(uint8_t* ba1, uint16_t size_in_bits1, uint8_t* ba2,
uint16_t size_in_bits2)
{
if (size_in_bits1 != size_in_bits2) return 0;
uint16_t size_in_bytes = bv_bits_to_bytes(size_in_bits1);
// shift the unused bits to avoid failing due to them
// (difference only in the unused bits)
uint8_t unused_ms_bits =
(uint8_t)(BV_BITS_IN_A_BYTE * size_in_bytes - size_in_bits1);
uint8_t last_byte1 = ba1[size_in_bytes - 1] << unused_ms_bits;
uint8_t last_byte2 = ba2[size_in_bytes - 1] << unused_ms_bits;
return (uint8_t)(memcmp(ba1, ba2, (size_t)(size_in_bytes - 1)) == 0 &&
last_byte1 == last_byte2
? 1
: 0);
}
static inline void
bv_copy_internal(uint8_t* ba_dst, uint16_t size_in_bits_dst, uint8_t* ba_src,
uint16_t size_in_bits_src)
{
// allow copy only if sizes match
if (size_in_bits_dst != size_in_bits_src) assert(0);
memcpy(ba_dst, ba_src, bv_bits_to_bytes(size_in_bits_src));
}
static inline uint8_t
bv_no_setted_bits_internal(uint8_t* bit_array, uint16_t size_in_bits)
{
uint8_t cnt = 0;
for (uint8_t i = 0; i < size_in_bits; ++i)
cnt += bv_bit_get_internal(bit_array, size_in_bits, i);
return cnt;
}
/// Bitvector Bitwise ops internal
static inline void
bv_reverse_internal(uint8_t* bit_array, uint16_t size_in_bits)
{
for (int i = 0; i < bv_bits_to_bytes(size_in_bits); ++i)
bit_array[i] = ~bit_array[i];
}
static inline void
bv_and_internal(uint8_t* ba_dst, uint16_t size_in_bits_dst,
const uint8_t* ba_src, uint16_t size_in_bits_src)
{
// allow and only if sizes match
if (size_in_bits_dst != size_in_bits_src) assert(0);
for (int i = 0; i < bv_bits_to_bytes(size_in_bits_dst); ++i)
ba_dst[i] &= ba_src[i];
}
static inline void
bv_or_internal(uint8_t* ba_dst, uint16_t size_in_bits_dst,
const uint8_t* ba_src, uint16_t size_in_bits_src)
{
// allow or only if sizes match
if (size_in_bits_dst != size_in_bits_src) assert(0);
for (int i = 0; i < bv_bits_to_bytes(size_in_bits_dst); ++i)
ba_dst[i] |= ba_src[i];
}
/// Bitvector Print functions
static inline void
bv_print_internal(const uint8_t* bit_array, uint16_t size_in_bits)
{
for (int i = bv_bits_to_bytes(size_in_bits) - 1; i >= 0; --i)
printf(BYTE_TO_BINARY_PATTERN, BYTE_TO_BINARY(bit_array[i]));
}
static inline void
bv_print_enhanced_internal(const uint8_t* bit_array, uint16_t size_in_bits)
{
printf("Bit vector: ");
bv_print_internal(bit_array, size_in_bits);
printf("\n");
}
/////////////////////////////////////////
/// Dynamic Bitvector API functions
/////////////////////////////////////////
static inline void
dbv_init(dbit_vector_t** bv, uint8_t size)
{
uint16_t bv_size_in_bytes = bv_bits_to_bytes(size);
*bv = malloc(sizeof(dbit_vector_t));
(*bv)->bit_array = malloc(bv_size_in_bytes * sizeof(uint8_t));
(*bv)->bv_size = size;
bv_init_internal((*bv)->bit_array, size);
}
static inline void
dbv_destroy(dbit_vector_t* bv)
{
free(bv->bit_array);
free(bv);
}
static inline uint8_t
dbv_bit_get(dbit_vector_t bv, int bit)
{
return bv_bit_get_internal(bv.bit_array, bv.bv_size, bit);
}
static inline void
dbv_bit_set(dbit_vector_t* bv, uint8_t bit)
{
bv_bit_set_internal(bv->bit_array, bv->bv_size, bit);
}
static inline void
dbv_bit_reset(dbit_vector_t* bv, uint8_t bit)
{
bv_bit_reset_internal(bv->bit_array, bv->bv_size, bit);
}
static inline void
dbv_set_all(dbit_vector_t* bv)
{
bv_set_all_internal(bv->bit_array, bv->bv_size);
}
static inline void
dbv_reset_all(dbit_vector_t* bv)
{
bv_reset_all_internal(bv->bit_array, bv->bv_size);
}
static inline uint8_t
dbv_no_setted_bits(dbit_vector_t bv)
{
return bv_no_setted_bits_internal(bv.bit_array, bv.bv_size);
}
static inline uint8_t
dbv_are_equal(dbit_vector_t bv1, dbit_vector_t bv2)
{
return bv_are_equal_internal(bv1.bit_array, bv1.bv_size, bv2.bit_array,
bv2.bv_size);
}
static inline void
dbv_copy(dbit_vector_t* bv_dst, dbit_vector_t bv_src)
{
bv_copy_internal(bv_dst->bit_array, bv_dst->bv_size, bv_src.bit_array,
bv_src.bv_size);
}
static inline uint8_t
dbv_is_all_set(dbit_vector_t bv)
{
dbit_vector_t* bv_tmp;
dbv_init(&bv_tmp, bv.bv_size);
dbv_set_all(bv_tmp);
return dbv_are_equal(bv, *bv_tmp);
}
/// Bitvector bitwise ops
static inline void
dbv_reverse(dbit_vector_t* bv)
{
bv_reverse_internal(bv->bit_array, bv->bv_size);
}
static inline void
dbv_and(dbit_vector_t* bv_dst, dbit_vector_t bv_src)
{
bv_and_internal(bv_dst->bit_array, bv_dst->bv_size, bv_src.bit_array,
bv_src.bv_size);
}
static inline void
dbv_or(dbit_vector_t* bv_dst, dbit_vector_t bv_src)
{
bv_or_internal(bv_dst->bit_array, bv_dst->bv_size, bv_src.bit_array,
bv_src.bv_size);
}
/// Bitvector Print functions
static inline void
dbv_print(dbit_vector_t bv)
{
bv_print_internal(bv.bit_array, bv.bv_size);
}
static inline void
dbv_print_enhanced(dbit_vector_t bv)
{
bv_print_enhanced_internal(bv.bit_array, bv.bv_size);
}
static inline void
dbv_unit_test(void)
{
dbit_vector_t* bv;
dbit_vector_t* bv_set__all;
dbv_init(&bv, 22);
dbv_init(&bv_set__all, 22);
dbv_set_all(bv_set__all);
for (uint8_t i = 0; i < bv->bv_size; ++i)
dbv_bit_set(bv, i);
assert(dbv_are_equal(*bv, *bv_set__all) == 1);
for (uint8_t i = 0; i < bv->bv_size; ++i)
dbv_bit_reset(bv, i);
dbv_reverse(bv);
assert(dbv_are_equal(*bv, *bv_set__all) == 1);
for (uint8_t i = 0; i < bv->bv_size; ++i)
if (i % 2 == 0) {
dbv_bit_reset(bv, i);
assert(dbv_bit_get(*bv, i) == 0);
} else {
dbv_bit_set(bv, i);
assert(dbv_bit_get(*bv, i) == 1);
}
dbv_reset_all(bv);
assert(dbv_are_equal(*bv, *bv_set__all) == 0);
dbv_set_all(bv);
dbv_and(bv, *bv_set__all);
assert(dbv_are_equal(*bv, *bv_set__all) == 1);
dbv_copy(bv, *bv_set__all);
assert(dbv_are_equal(*bv, *bv_set__all) == 1);
dbv_reset_all(bv);
dbv_or(bv, *bv_set__all);
assert(dbv_are_equal(*bv, *bv_set__all) == 1);
printf("Dynamic Bit Vector Unit Test was Successful!\n");
}
/////////////////////////////////////////
/// Static Bitvector API functions
/////////////////////////////////////////
static inline void
bv_init(bit_vector_t* bv)
{
bv_init_internal(bv->bit_array, BV_BIT_VECTOR_SIZE);
}
static inline uint8_t
bv_bit_get(bit_vector_t bv, int bit)
{
return bv_bit_get_internal(bv.bit_array, BV_BIT_VECTOR_SIZE, bit);
}
static inline void
bv_bit_set(bit_vector_t* bv, uint8_t bit)
{
bv_bit_set_internal(bv->bit_array, BV_BIT_VECTOR_SIZE, bit);
}
static inline void
bv_bit_reset(bit_vector_t* bv, uint8_t bit)
{
bv_bit_reset_internal(bv->bit_array, BV_BIT_VECTOR_SIZE, bit);
}
static inline void
bv_set_all(bit_vector_t* bv)
{
bv_set_all_internal(bv->bit_array, BV_BIT_VECTOR_SIZE);
}
static inline void
bv_reset_all(bit_vector_t* bv)
{
bv_reset_all_internal(bv->bit_array, BV_BIT_VECTOR_SIZE);
}
static inline uint8_t
bv_no_setted_bits(bit_vector_t bv)
{
return bv_no_setted_bits_internal(bv.bit_array, BV_BIT_VECTOR_SIZE);
}
static inline uint8_t
bv_are_equal(bit_vector_t bv1, bit_vector_t bv2)
{
return bv_are_equal_internal(bv1.bit_array, BV_BIT_VECTOR_SIZE, bv2.bit_array,
BV_BIT_VECTOR_SIZE);
}
static inline void
bv_copy(bit_vector_t* bv_dst, bit_vector_t bv_src)
{
bv_copy_internal(bv_dst->bit_array, BV_BIT_VECTOR_SIZE, bv_src.bit_array,
BV_BIT_VECTOR_SIZE);
}
/// Bitvector bitwise ops
static inline void
bv_reverse(bit_vector_t* bv)
{
bv_reverse_internal(bv->bit_array, BV_BIT_VECTOR_SIZE);
}
static inline void
bv_and(bit_vector_t* bv_dst, bit_vector_t bv_src)
{
bv_and_internal(bv_dst->bit_array, BV_BIT_VECTOR_SIZE, bv_src.bit_array,
BV_BIT_VECTOR_SIZE);
}
static inline void
bv_or(bit_vector_t* bv_dst, bit_vector_t bv_src)
{
bv_or_internal(bv_dst->bit_array, BV_BIT_VECTOR_SIZE, bv_src.bit_array,
BV_BIT_VECTOR_SIZE);
}
/// Bitvector Print functions
static inline void
bv_print(bit_vector_t bv)
{
bv_print_internal(bv.bit_array, BV_BIT_VECTOR_SIZE);
}
static inline void
bv_print_enhanced(bit_vector_t bv)
{
bv_print_enhanced_internal(bv.bit_array, BV_BIT_VECTOR_SIZE);
}
/////////////////////////////////////////
/// Bitvector unit test functions
/////////////////////////////////////////
static inline void
bv_unit_test(void)
{
bit_vector_t bv;
bit_vector_t bv_set__all;
bv_init(&bv);
bv_set_all(&bv_set__all);
dbv_unit_test();
for (uint8_t i = 0; i < BV_BIT_VECTOR_SIZE; ++i)
bv_bit_set(&bv, i);
assert(bv_are_equal(bv, bv_set__all) == 1);
for (uint8_t i = 0; i < BV_BIT_VECTOR_SIZE; ++i)
bv_bit_reset(&bv, i);
bv_reverse(&bv);
assert(bv_are_equal(bv, bv_set__all) == 1);
for (uint8_t i = 0; i < BV_BIT_VECTOR_SIZE; ++i)
if (i % 2 == 0) {
bv_bit_reset(&bv, i);
assert(bv_bit_get(bv, i) == 0);
} else {
bv_bit_set(&bv, i);
assert(bv_bit_get(bv, i) == 1);
}
bv_reset_all(&bv);
assert(bv_are_equal(bv, bv_set__all) == 0);
bv_set_all(&bv);
bv_and(&bv, bv_set__all);
assert(bv_are_equal(bv, bv_set__all) == 1);
bv_copy(&bv, bv_set__all);
assert(bv_are_equal(bv, bv_set__all) == 1);
bv_reset_all(&bv);
bv_or(&bv, bv_set__all);
assert(bv_are_equal(bv, bv_set__all) == 1);
printf("Static Bit Vector Unit Test was Successful!\n");
}
#endif // HERMES_BIT_VECTOR_H
================================================
FILE: include/utils/concur_ctrl.h
================================================
//
// Created by akatsarakis on 11/12/18.
//
#ifndef HERMES_SEQLOCK_H
#define HERMES_SEQLOCK_H
#include <assert.h>
#include <stdint.h>
#define ENABLE_LOCK_ASSERTS 1
#define TIE_BREAKER_ID_EMPTY 255
#define SEQLOCK_LOCKED 0x1
#define SEQLOCK_FREE 0x0
#define LOCK_PAUSE() asm volatile("mfence");
#define COMPILER_BARRIER() asm volatile("" ::: "memory")
#if !defined(COMPILER_NO_REORDER)
#define COMPILER_NO_REORDER(exec) \
COMPILER_BARRIER(); \
exec; \
COMPILER_BARRIER()
#endif
typedef volatile struct {
uint8_t tie_breaker_id;
uint32_t version;
} __attribute__((packed)) timestamp_t;
typedef struct {
uint8_t lock;
uint32_t version; /// for lock-free reads
} __attribute__((packed)) seqlock_t;
typedef volatile struct {
uint8_t lock;
timestamp_t
ts; /// ts.version used for both lock-free reads & as part of timestamp
} __attribute__((packed)) conc_ctrl_t;
/////////////////////////////////////////
/// Timestamp comparison functions
/////////////////////////////////////////
static inline void
timestamp_init(timestamp_t* ts)
{
ts->version = 0;
ts->tie_breaker_id = TIE_BREAKER_ID_EMPTY;
}
static inline int
timestamp_is_equal(uint32_t v1, uint8_t tie_breaker1, uint32_t v2,
uint8_t tie_breaker2)
{
return (v1 == v2 && tie_breaker1 == tie_breaker2);
}
static inline int
timestamp_is_smaller(uint32_t v1, uint8_t tie_breaker1, uint32_t v2,
uint8_t tie_breaker2)
{
return (v1 < v2 || (v1 == v2 && tie_breaker1 < tie_breaker2));
}
/////////////////////////////////////////
/// seqlock locking / unlocking functions
/////////////////////////////////////////
static inline void
seqlock_init(seqlock_t* seqlock)
{
seqlock->version = 0;
seqlock->lock = SEQLOCK_FREE;
}
static inline int
seqlock_lock(seqlock_t* seqlock)
{
do {
// Spin until the seqlock is unlocked
while (seqlock->lock == SEQLOCK_LOCKED) {
LOCK_PAUSE();
}
// try to atomically get the lock via a CAS
if (__sync_val_compare_and_swap(&seqlock->lock, 0, 1) == 0) {
seqlock->version++;
break;
}
} while (1); // retry if CAS failed
return 1;
}
static inline void
seqlock_unlock(seqlock_t* seqlock)
{
if (ENABLE_LOCK_ASSERTS) {
assert(seqlock->lock == SEQLOCK_LOCKED);
assert(seqlock->version % 2 == 1);
}
COMPILER_NO_REORDER(seqlock->version++);
COMPILER_NO_REORDER(seqlock->lock = SEQLOCK_FREE);
}
// This is used to validate a lock-free read
// i.e. --> do { <Lock free read> } while
// (!(seqlock_version_is_same_and_valid(...));
static inline int
seqlock_version_is_same_and_valid(seqlock_t* seqlock1, seqlock_t* seqlock2)
{
return (seqlock1->version == seqlock2->version && seqlock1->version % 2 == 0);
}
/////////////////////////////////////////
/// ccctrl locking / unlocking functions
/////////////////////////////////////////
static inline void
cctrl_init(conc_ctrl_t* cctrl)
{
timestamp_init(&cctrl->ts);
cctrl->lock = SEQLOCK_FREE;
}
static inline int
cctrl_lock(conc_ctrl_t* cctrl)
{
do {
// Spin until the seqlock is unlocked
while (cctrl->lock == SEQLOCK_LOCKED) {
LOCK_PAUSE();
}
// try to atomically get the lock via a CAS
if (__sync_val_compare_and_swap(&cctrl->lock, 0, 1) == 0) {
cctrl->ts.version++;
break;
}
} while (1); // retry if CAS failed
return 1;
}
static inline void
cctrl_unlock_custom_version(conc_ctrl_t* cctrl, uint8_t cid, uint32_t version)
{
if (ENABLE_LOCK_ASSERTS) {
assert(cctrl->lock == SEQLOCK_LOCKED);
assert(cctrl->ts.version % 2 == 1);
}
cctrl->ts.tie_breaker_id = cid;
COMPILER_NO_REORDER(cctrl->ts.version = version);
COMPILER_NO_REORDER(cctrl->lock = SEQLOCK_FREE);
}
static inline void
cctrl_unlock_inc_version_by_three(conc_ctrl_t* cctrl, uint8_t cid,
uint32_t* resp_version)
{
if (ENABLE_LOCK_ASSERTS) {
assert(cctrl->lock == SEQLOCK_LOCKED);
assert(cctrl->ts.version % 2 == 1);
}
cctrl->ts.tie_breaker_id = cid;
COMPILER_NO_REORDER(cctrl->ts.version += 3);
COMPILER_NO_REORDER(*resp_version = cctrl->ts.version);
COMPILER_NO_REORDER(cctrl->lock = SEQLOCK_FREE);
}
static inline void
cctrl_unlock_inc_version(conc_ctrl_t* cctrl, uint8_t cid,
uint32_t* resp_version)
{
if (ENABLE_LOCK_ASSERTS) {
assert(cctrl->lock == SEQLOCK_LOCKED);
assert(cctrl->ts.version % 2 == 1);
}
cctrl->ts.tie_breaker_id = cid;
COMPILER_NO_REORDER(*resp_version = ++cctrl->ts.version);
COMPILER_NO_REORDER(cctrl->lock = SEQLOCK_FREE);
}
static inline void
cctrl_unlock_dec_version(conc_ctrl_t* cctrl)
{
if (ENABLE_LOCK_ASSERTS) {
assert(cctrl->lock == SEQLOCK_LOCKED);
assert(cctrl->ts.version % 2 == 1);
}
// keep same ts.tie_breaker_id
COMPILER_NO_REORDER(cctrl->ts.version--);
COMPILER_NO_REORDER(cctrl->lock = SEQLOCK_FREE);
}
// This is used to validate a lock-free read
// i.e. --> do { <Lock free read> } while
// (!(cctrl_timestamp_is_same_and_valid(...));
static inline int
cctrl_timestamp_is_same_and_valid(volatile conc_ctrl_t* cctrl1,
volatile conc_ctrl_t* cctrl2)
{
return cctrl1->ts.version % 2 == 0 &&
timestamp_is_equal(cctrl1->ts.version, cctrl1->ts.tie_breaker_id,
cctrl2->ts.version, cctrl2->ts.tie_breaker_id);
}
#endif // HERMES_SEQLOCK_H
================================================
FILE: include/utils/time_rdtsc.h
================================================
#ifndef HERMES_TIME_H
#define HERMES_TIME_H
#include <assert.h>
#include <stdint.h> /* for uint64_t */
#include <stdio.h>
#include <time.h> /* for struct timespec */
#define ENABLE_STATIC_TICKS_PER_NS 1
#define RDTSC_TYPICAL_TICKS_PER_NS 2.2
double g_ticks_per_ns;
// assembly code to read the TSC
static inline uint64_t
RDTSC()
{
unsigned int hi, lo;
__asm__ volatile("rdtsc" : "=a"(lo), "=d"(hi));
return ((uint64_t)hi << 32) | lo;
}
static const int NANO_SECONDS_IN_SEC = 1000000000;
// returns a static buffer of struct timespec with the time difference of
// ts1 and ts2 ts1 is assumed to be greater than ts2
static struct timespec*
timespec_diff(struct timespec* ts1, struct timespec* ts2)
{
static struct timespec ts;
ts.tv_sec = ts1->tv_sec - ts2->tv_sec;
ts.tv_nsec = ts1->tv_nsec - ts2->tv_nsec;
if (ts.tv_nsec < 0) {
ts.tv_sec--;
ts.tv_nsec += NANO_SECONDS_IN_SEC;
}
return &ts;
}
static void
calibrate_ticks()
{
struct timespec begin_ts, end_ts;
printf("Start RDTSC calibration: patience is a virtue\n");
clock_gettime(CLOCK_MONOTONIC, &begin_ts);
uint64_t begin = RDTSC();
// do something CPU intensive
for (volatile unsigned long long i = 0; i < 1000000000ULL; ++i)
;
uint64_t end = RDTSC();
clock_gettime(CLOCK_MONOTONIC, &end_ts);
struct timespec* tmp_ts = timespec_diff(&end_ts, &begin_ts);
uint64_t ns_elapsed =
(uint64_t)(tmp_ts->tv_sec * 1000000000LL + tmp_ts->tv_nsec);
g_ticks_per_ns = (double)(end - begin) / (double)ns_elapsed;
printf("RDTSC calibration is done (ticks_per_ns: %.2f)\n", g_ticks_per_ns);
}
// Call once (it is not thread safe) before using RDTSC, has side effect of
// binding process to CPU1
static inline void
init_rdtsc(uint8_t auto_calibration, double ticks_per_ns)
{
if (auto_calibration > 0)
calibrate_ticks();
else {
assert(ticks_per_ns > 0);
g_ticks_per_ns = ticks_per_ns;
}
}
static inline void
get_timespec(struct timespec* ts, uint64_t nsecs)
{
ts->tv_sec = nsecs / NANO_SECONDS_IN_SEC;
ts->tv_nsec = nsecs % NANO_SECONDS_IN_SEC;
}
// ts will be filled with time converted from TSC reading
static inline void
get_rdtsc_timespec(struct timespec* ts)
{
get_timespec(ts, (uint64_t)(RDTSC() / g_ticks_per_ns));
}
static inline double
time_elapsed_in_us(struct timespec start)
{
struct timespec now, *diff;
get_rdtsc_timespec(&now);
diff = timespec_diff(&now, &start);
return diff->tv_sec * 1000000 + diff->tv_nsec / 1000;
}
static inline double
time_elapsed_in_ms(struct timespec start)
{
struct timespec now, *diff;
get_rdtsc_timespec(&now);
diff = timespec_diff(&now, &start);
return diff->tv_sec * 1000 + diff->tv_nsec / 1000000;
}
static inline double
time_elapsed_in_sec(struct timespec start)
{
struct timespec now, *diff;
get_rdtsc_timespec(&now);
diff = timespec_diff(&now, &start);
return diff->tv_sec + diff->tv_nsec / NANO_SECONDS_IN_SEC;
}
#endif // HERMES_TIME_H
================================================
FILE: include/wings/wings.h
================================================
//
// Created by akatsarakis on 06/02/19.
//
#ifndef WINGS_INTERNAL_INLINES_H
#define WINGS_INTERNAL_INLINES_H
#include "wings_api.h"
/// WARNING!!
/// Functions starting with underscore (i.e. "_wings_*")
/// are internal and should not be called directly
void wings_reconfigure_wrs_ah(ud_channel_t* ud_c, uint8_t endpoint_id);
/* --------------------------------------------------------------------------
--------------------------------- Helper Functions --------------------------
---------------------------------------------------------------------------*/
static inline void
_wings_assert_binary(uint8_t var)
{
assert(var == 0 || var == 1);
}
static inline uint16_t
_wings_ud_recv_max_pkt_size(ud_channel_t* ud_c)
{
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->type != CRD && ud_c->is_header_only == 0);
// TODO add assertion that this must be smaller than max_MTU
assert(ud_c->max_msg_size > 0 && ud_c->max_coalescing > 0);
return sizeof(wings_ud_recv_pkt_t) +
ud_c->max_msg_size * ud_c->max_coalescing;
}
static inline uint16_t
_wings_ud_send_max_pkt_size(ud_channel_t* ud_c)
{
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->type != CRD && ud_c->is_header_only == 0);
// TODO add assertion that this must be smaller than max_MTU
assert(ud_c->max_msg_size > 0 && ud_c->max_coalescing > 0);
return sizeof(wings_ud_send_pkt_t) +
ud_c->max_msg_size * ud_c->max_coalescing;
}
static inline void
_wings_assertions(ud_channel_t* ud_channel)
{
_wings_assert_binary(ud_channel->expl_crd_ctrl);
_wings_assert_binary(ud_channel->is_bcast_channel);
_wings_assert_binary(ud_channel->is_inlining_enabled);
assert(ud_channel->num_channels > 1);
assert(ud_channel->max_msg_size > 0);
assert(ud_channel->max_coalescing > 0);
assert(_wings_ud_send_max_pkt_size(ud_channel) < MAX_MTU_SIZE);
assert(ud_channel->send_q_depth > 0 || ud_channel->recv_q_depth > 0);
assert(ud_channel->channel_providing_crds != NULL ||
ud_channel->disable_crd_ctrl);
}
static inline uint8_t*
_wings_get_n_msg_ptr_from_send_pkt(ud_channel_t* ud_c, wings_ud_send_pkt_t* pkt,
uint8_t n)
{
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->type != CRD && ud_c->is_header_only == 0);
assert(ud_c->max_coalescing > n && pkt->req_num >= n);
// return &pkt->reqs[n * ud_c->max_msg_size];
return &pkt->reqs[n * ud_c->small_msg_size];
}
static inline uint8_t*
_wings_get_n_msg_ptr_from_recv_pkt(ud_channel_t* ud_c,
wings_ud_recv_pkt_t* recv_pkt, uint8_t n)
{
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->type != CRD && ud_c->is_header_only == 0);
return _wings_get_n_msg_ptr_from_send_pkt(ud_c, &recv_pkt->pkt, n);
}
static inline wings_ud_send_pkt_t*
_wings_get_nth_pkt_ptr_from_send_buff(ud_channel_t* ud_c, uint16_t n)
{
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->type != CRD && ud_c->is_header_only == 0);
return (wings_ud_send_pkt_t*)&(
(uint8_t*)ud_c->send_pkt_buff)[n * _wings_ud_send_max_pkt_size(ud_c)];
}
static inline wings_ud_recv_pkt_t*
_wings_get_nth_pkt_ptr_from_recv_buff(ud_channel_t* ud_c, uint16_t n)
{
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->type != CRD && ud_c->is_header_only == 0);
return (wings_ud_recv_pkt_t*)&ud_c
->recv_pkt_buff[n * _wings_ud_recv_max_pkt_size(ud_c)];
}
static inline wings_ud_send_pkt_t*
_wings_curr_send_pkt_ptr(ud_channel_t* ud_c)
{
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->type != CRD && ud_c->is_header_only == 0);
return _wings_get_nth_pkt_ptr_from_send_buff(ud_c,
(uint16_t)ud_c->send_push_ptr);
}
static inline void
_wings_inc_send_push_ptr(ud_channel_t* ud_c)
{
if (ud_c->is_header_only) return;
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->type != CRD && ud_c->is_header_only == 0);
if (ud_c->is_bcast_channel)
WINGS_MOD_ADD(ud_c->send_push_ptr,
ud_c->send_pkt_buff_len); // TODO change this to deal with
// failures see comment below
// WINGS_MOD_ADD(*inv_push_ptr, INV_SEND_OPS_SIZE / MAX_REMOTE_MACHINES *
// last_g_membership.num_of_alive_remotes);
// //got to the next "packet" + dealing with
// failutes
else
WINGS_MOD_ADD(ud_c->send_push_ptr, ud_c->send_pkt_buff_len);
_wings_curr_send_pkt_ptr(ud_c)->req_num =
0; // Reset data left from previous unicasts / bcasts
}
static inline void
_wings_inc_recv_push_ptr(ud_channel_t* ud_c)
{
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->type != CRD && ud_c->is_header_only == 0);
WINGS_MOD_ADD(ud_c->recv_push_ptr, ud_c->recv_q_depth);
}
static inline void
_wings_inc_recv_pull_ptr(ud_channel_t* ud_c)
{
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->type != CRD && ud_c->is_header_only == 0);
WINGS_MOD_ADD(ud_c->recv_pull_ptr, ud_c->recv_pkt_buff_len);
}
/* ---------------------------------------------------------------------------
----------------------------------- RECVs ------------------------------------
---------------------------------------------------------------------------*/
static inline void
_wings_post_hdr_only_recvs(ud_channel_t* ud_c, uint16_t num_recvs)
{
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->is_header_only || ud_c->type == CRD);
struct ibv_recv_wr* bad_recv_wr;
for (uint16_t i = 0; i < num_recvs; ++i)
ud_c->recv_wr[i].next = (i == num_recvs - 1) ? NULL : &ud_c->recv_wr[i + 1];
int ret = ibv_post_recv(ud_c->qp, ud_c->recv_wr, &bad_recv_wr);
CPE(ret, "ibv_post_recv error: posting recvs for credits", ret);
}
static inline void
_wings_post_recvs(ud_channel_t* ud_c, uint16_t num_of_receives)
{
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->type != CRD && ud_c->is_header_only == 0);
void* next_buff_addr;
if (WINGS_ENABLE_ASSERTIONS) assert(num_of_receives <= ud_c->max_recv_wrs);
int req_size = _wings_ud_recv_max_pkt_size(ud_c);
for (int i = 0; i < num_of_receives; ++i) {
next_buff_addr =
(void*)(ud_c->recv_pkt_buff) + (ud_c->recv_push_ptr * req_size);
// TODO optimize by reseting only the req_num of wings_recv_pkt
memset(next_buff_addr, 0,
(size_t)req_size); // reset the buffer before posting the receive
if (WINGS_ENABLE_BATCH_POST_RECVS_TO_NIC)
ud_c->recv_wr[i].sg_list->addr = (uintptr_t)next_buff_addr;
else
assert(0);
// hrd_post_dgram_recv(ud_c->qp, next_buff_addr, req_size,
// cb->dgram_buf_mr->lkey);
_wings_inc_recv_push_ptr(ud_c);
}
if (WINGS_ENABLE_BATCH_POST_RECVS_TO_NIC) {
ud_c->recv_wr[num_of_receives - 1].next = NULL;
if (WINGS_ENABLE_ASSERTIONS) {
for (int i = 0; i < num_of_receives; i++) {
assert(ud_c->recv_wr[i].num_sge == 1);
assert(ud_c->recv_wr[i].sg_list->length == req_size);
// TODO add
// assert(ud_c->recv_wr[i].sg_list->lkey ==
// cb->dgram_buf_mr->lkey);
assert(i == num_of_receives - 1 ||
ud_c->recv_wr[i].next == &ud_c->recv_wr[i + 1]);
}
assert(ud_c->recv_wr[num_of_receives - 1].next == NULL);
}
struct ibv_recv_wr* bad_recv_wr;
int ret = ibv_post_recv(ud_c->qp, ud_c->recv_wr, &bad_recv_wr);
CPE(ret, "ibv_post_recv error: while posting recvs", ret);
// recover next ptr of last wr to NULL
ud_c->recv_wr[num_of_receives - 1].next =
(ud_c->max_recv_wrs == num_of_receives - 1)
? NULL
: &ud_c->recv_wr[num_of_receives];
}
}
static inline void
_wings_poll_crds_and_post_recvs(ud_channel_t* ud_c)
{
if (WINGS_ENABLE_ASSERTIONS) assert(ud_c->type == CRD);
int crd_pkts_found =
ibv_poll_cq(ud_c->recv_cq, ud_c->max_recv_wrs, ud_c->recv_wc);
if (crd_pkts_found > 0) {
if (unlikely(ud_c->recv_wc[crd_pkts_found - 1].status != 0)) {
fprintf(stderr,
"Bad wc status when polling for credits to send a broadcast %d\n",
ud_c->recv_wc[crd_pkts_found - 1].status);
exit(0);
}
if (ud_c->enable_stats) ud_c->stats.recv_total_pkts += crd_pkts_found;
if (WINGS_ENABLE_RECV_PRINTS && ud_c->enable_prints)
colored_printf(GREEN, "^^^ Polled reqs: %s %d, (total: %d)!\n",
ud_c->qp_name, crd_pkts_found,
ud_c->stats.recv_total_pkts);
for (int i = 0; i < crd_pkts_found; i++) {
wings_crd_t* crd_ptr = (wings_crd_t*)&ud_c->recv_wc[i].imm_data;
if (ud_c->enable_stats) ud_c->stats.recv_total_msgs += crd_ptr->crd_num;
ud_c->channel_providing_crds->credits_per_channels[crd_ptr->sender_id] +=
crd_ptr->crd_num;
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->channel_providing_crds->num_crds_per_channel >=
ud_c->channel_providing_crds
->credits_per_channels[crd_ptr->sender_id]);
if (WINGS_ENABLE_CREDIT_PRINTS && ud_c->enable_prints)
printf(
"$$$ Credits: %s \033[1m\033[32mincremented\033[0m to %d (for "
"endpoint %d)\n",
ud_c->channel_providing_crds->qp_name,
ud_c->channel_providing_crds
->credits_per_channels[crd_ptr->sender_id],
crd_ptr->sender_id);
}
if (WINGS_ENABLE_POST_RECV_PRINTS && ud_c->enable_prints)
colored_printf(YELLOW, "vvv Post Receives: %s %d\n", ud_c->qp_name,
crd_pkts_found);
_wings_post_hdr_only_recvs(ud_c, (uint16_t)crd_pkts_found);
} else if (unlikely(crd_pkts_found < 0)) {
printf("ERROR In the credit CQ\n");
exit(0);
}
}
static inline void
_wings_enque_to_overflown_msgs(ud_channel_t* ud_c, uint8_t* msg_ptr)
{
if (WINGS_ENABLE_ASSERTIONS) {
assert(ud_c->is_header_only == 0);
assert(ud_c->enable_overflow_msgs);
assert(ud_c->num_overflow_msgs < ud_c->max_coalescing);
}
uint8_t* dst_ptr =
&ud_c->overflow_msg_buff[ud_c->num_overflow_msgs * ud_c->max_msg_size];
memcpy(dst_ptr, msg_ptr, ud_c->max_msg_size);
ud_c->num_overflow_msgs++;
}
static inline uint16_t
_wings_deque_from_overflown_msgs(ud_channel_t* ud_c, uint16_t max_msgs_to_poll,
uint8_t* recv_ops)
{
if (WINGS_ENABLE_ASSERTIONS) assert(ud_c->is_header_only == 0);
uint8_t msgs_to_copy = (uint8_t)(ud_c->num_overflow_msgs <= max_msgs_to_poll
? ud_c->num_overflow_msgs
: max_msgs_to_poll);
if (ud_c->num_overflow_msgs > 0) {
ud_c->num_overflow_msgs -= msgs_to_copy;
// Copy msgs from overflow_buff to recv_ops
memcpy(recv_ops, ud_c->overflow_msg_buff,
msgs_to_copy * ud_c->max_msg_size);
if (msgs_to_copy == max_msgs_to_poll)
// Move rest of overflown msgs to the top of the (FIFO) buffer
for (int i = 0; i < ud_c->num_overflow_msgs; ++i) {
uint8_t* dst_ptr = &ud_c->overflow_msg_buff[ud_c->max_msg_size * i];
uint8_t* src_ptr =
&ud_c->overflow_msg_buff[ud_c->max_msg_size * (i + msgs_to_copy)];
memcpy(dst_ptr, src_ptr, ud_c->max_msg_size);
}
}
return msgs_to_copy;
}
static inline uint16_t
wings_poll_buff_and_post_recvs(ud_channel_t* ud_c, uint16_t max_msgs_to_poll,
uint8_t* recv_ops)
{
if (WINGS_ENABLE_ASSERTIONS) assert(ud_c->type != CRD);
int index = 0;
uint8_t sender = 0;
uint16_t msgs_polled = 0;
uint8_t *next_packet_reqs, *recv_op_ptr, *next_req, *next_packet_req_num_ptr;
uint16_t dequed_msgs = 0;
uint16_t remaining_msgs_to_poll = max_msgs_to_poll;
if (max_msgs_to_poll < 1) return 0;
if (ud_c->enable_overflow_msgs) {
dequed_msgs =
_wings_deque_from_overflown_msgs(ud_c, max_msgs_to_poll, recv_ops);
if (max_msgs_to_poll == dequed_msgs) return max_msgs_to_poll;
recv_ops = &recv_ops[dequed_msgs * ud_c->max_msg_size];
remaining_msgs_to_poll -= dequed_msgs;
}
uint16_t max_pkts_to_poll =
(uint16_t)((remaining_msgs_to_poll / ud_c->max_coalescing) +
(ud_c->enable_overflow_msgs ? 1 : 0));
// poll completion q
uint16_t pkts_polled =
(uint16_t)ibv_poll_cq(ud_c->recv_cq, max_pkts_to_poll, ud_c->recv_wc);
for (int i = 0; i < pkts_polled; ++i) {
if (ud_c->is_header_only) {
recv_op_ptr = &recv_ops[i * ud_c->max_msg_size];
memcpy(recv_op_ptr, &ud_c->recv_wc[i].imm_data, ud_c->max_msg_size);
msgs_polled++;
sender = ((wings_hdr_only_t*)&ud_c->recv_wc[i].imm_data)->sender_id;
if (!ud_c->disable_crd_ctrl)
ud_c->channel_providing_crds
->credits_per_channels[sender]++; // increment packet credits
} else {
uint16_t max_req_size = _wings_ud_recv_max_pkt_size(ud_c);
index = (ud_c->recv_pull_ptr + 1) % ud_c->recv_q_depth;
wings_ud_recv_pkt_t* next_packet =
(wings_ud_recv_pkt_t*)&ud_c->recv_pkt_buff[index * max_req_size];
sender = next_packet->pkt.sender_id;
next_packet_reqs = next_packet->pkt.reqs;
next_packet_req_num_ptr = &next_packet->pkt.req_num;
if (WINGS_ENABLE_ASSERTIONS)
assert(next_packet->pkt.req_num > 0 &&
next_packet->pkt.req_num <= ud_c->max_coalescing);
// TODO add membership and functionality
// if(node_is_in_membership(last_group_membership, sender))
uint16_t msg_size = next_packet->pkt.only_small_msgs == 1
? ud_c->small_msg_size
: ud_c->max_msg_size;
for (int j = 0; j < next_packet->pkt.req_num; ++j) {
next_req = &next_packet_reqs[j * msg_size];
if (msgs_polled >= remaining_msgs_to_poll)
_wings_enque_to_overflown_msgs(ud_c, next_req);
else {
recv_op_ptr = &recv_ops[msgs_polled * ud_c->max_msg_size];
memcpy(recv_op_ptr, next_req, msg_size);
}
msgs_polled++;
if (!ud_c->disable_crd_ctrl)
ud_c->channel_providing_crds
->credits_per_channels[sender]++; // increment packet credits
}
*next_packet_req_num_ptr =
0; // TODO can be removed since we already reset on posting receives
_wings_inc_recv_pull_ptr(ud_c);
}
if (WINGS_ENABLE_ASSERTIONS)
if (!ud_c->disable_crd_ctrl)
assert(ud_c->channel_providing_crds->credits_per_channels[sender] <=
ud_c->channel_providing_crds->num_crds_per_channel);
}
if (pkts_polled > 0) {
// Refill recvs
if (ud_c->is_header_only)
_wings_post_hdr_only_recvs(ud_c, pkts_polled);
else
_wings_post_recvs(ud_c, pkts_polled);
if (WINGS_ENABLE_STAT_COUNTING) {
ud_c->stats.recv_total_msgs += msgs_polled;
ud_c->stats.recv_total_pkts += pkts_polled;
}
if (WINGS_ENABLE_RECV_PRINTS && ud_c->enable_prints)
colored_printf(
GREEN,
"^^^ Polled msgs: %d packets %s %d, (total pkts: %d, msgs %d)!\n",
pkts_polled, ud_c->qp_name, msgs_polled, ud_c->stats.recv_total_pkts,
ud_c->stats.recv_total_msgs);
if (WINGS_ENABLE_CREDIT_PRINTS && ud_c->enable_prints &&
!ud_c->disable_crd_ctrl)
printf(
"$$$ Credits: %s \033[1m\033[32mincremented\033[0m to %d (for "
"machine %d)\n",
ud_c->channel_providing_crds->qp_name,
ud_c->channel_providing_crds->credits_per_channels[sender], sender);
if (WINGS_ENABLE_POST_RECV_PRINTS && ud_c->enable_prints)
colored_printf(YELLOW, "vvv Post Receives: %s %d\n", ud_c->qp_name,
pkts_polled);
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->max_coalescing != 1 || pkts_polled == msgs_polled);
}
return msgs_polled + dequed_msgs >= max_msgs_to_poll
? max_msgs_to_poll
: msgs_polled + dequed_msgs;
}
/* ---------------------------------------------------------------------------
----------------------------------- CREDITS ----------------------------------
---------------------------------------------------------------------------*/
static inline uint8_t
_wings_node_is_in_membership(uint8_t node_id, bit_vector_t membership)
{
if (WINGS_ENABLE_ASSERTIONS) assert(node_id < 8);
return bv_bit_get(membership, node_id) == 1 ? 1 : 0;
}
// For all the CREDIT functions --> if its a bcast channel endpoint_id is
// ignored
static inline uint8_t
_wings_has_sufficient_crds_no_polling_membership(ud_channel_t* ud_c,
uint8_t endpoint_id,
bit_vector_t* membership)
{
uint8_t check_membership = membership == NULL ? 0 : 1;
if (ud_c->disable_crd_ctrl)
return 1;
else if (!ud_c->is_bcast_channel)
return (uint8_t)(ud_c->credits_per_channels[endpoint_id] > 0);
else
for (int i = 0; i < ud_c->num_channels; ++i) {
if (i == ud_c->channel_id) continue;
if (check_membership == 1 &&
!_wings_node_is_in_membership(i, *membership))
continue; // skip machine if not in membership
if (ud_c->credits_per_channels[i] <= 0) return 0;
}
return 1;
}
// For all the CREDIT functions --> if its a bcast channel endpoint_id is
// ignored
static inline uint8_t
_wings_has_sufficient_crds_no_polling(ud_channel_t* ud_c, uint8_t endpoint_id)
{
return _wings_has_sufficient_crds_no_polling_membership(ud_c, endpoint_id,
NULL);
}
static inline uint8_t
_wings_has_sufficient_crds_membership(ud_channel_t* ud_c, uint8_t endpoint_id,
bit_vector_t* membership)
{
if (_wings_has_sufficient_crds_no_polling_membership(ud_c, endpoint_id,
membership))
return 1;
if (ud_c->expl_crd_ctrl) {
_wings_poll_crds_and_post_recvs(ud_c->channel_providing_crds);
if (_wings_has_sufficient_crds_no_polling_membership(ud_c, endpoint_id,
membership))
return 1;
}
return 0;
}
static inline uint8_t
_wings_has_sufficient_crds(ud_channel_t* ud_c, uint8_t endpoint_id)
{
if (_wings_has_sufficient_crds_no_polling(ud_c, endpoint_id)) return 1;
if (ud_c->expl_crd_ctrl) {
_wings_poll_crds_and_post_recvs(ud_c->channel_providing_crds);
if (_wings_has_sufficient_crds_no_polling(ud_c, endpoint_id)) return 1;
}
return 0;
}
static inline void
_wings_dec_crds_membership(ud_channel_t* ud_c, uint8_t endpoint_id,
bit_vector_t* membership)
{
uint8_t check_membership = membership == NULL ? 0 : 1;
if (ud_c->disable_crd_ctrl) return;
if (WINGS_ENABLE_ASSERTIONS)
assert(_wings_has_sufficient_crds_no_polling_membership(ud_c, endpoint_id,
membership));
if (!ud_c->is_bcast_channel)
ud_c->credits_per_channels[endpoint_id]--;
else
for (int i = 0; i < ud_c->num_channels; ++i) {
if (i == ud_c->channel_id) continue;
if (check_membership == 1 &&
!_wings_node_is_in_membership(i, *membership))
continue; // skip machine if not in membership
ud_c->credits_per_channels[i]--;
}
if (WINGS_ENABLE_CREDIT_PRINTS && ud_c->enable_prints) {
if (ud_c->is_bcast_channel)
endpoint_id = (uint8_t)(ud_c->channel_id == 0 ? 1 : 0);
printf("$$$ Credits: %s \033[31mdecremented\033[0m to %d", ud_c->qp_name,
ud_c->credits_per_channels[endpoint_id]);
if (ud_c->is_bcast_channel)
printf(" (all endpoints)\n");
else
printf(" (for endpoint %d)\n", endpoint_id);
}
}
static inline void
_wings_dec_crds(ud_channel_t* ud_c, uint8_t endpoint_id)
{
_wings_dec_crds_membership(ud_c, endpoint_id, NULL);
}
static inline void
wings_reset_credits(ud_channel_t* ud_c, uint8_t endpoint_id)
{
ud_c->credits_per_channels[endpoint_id] =
(uint16_t)ud_c->channel_providing_crds->num_crds_per_channel;
}
/* ---------------------------------------------------------------------------
----------------------------------- SENDs ------------------------------------
---------------------------------------------------------------------------*/
static inline void
_wings_forge_crd_wr(ud_channel_t* ud_c, uint16_t dst_qp_id,
uint16_t crd_pkts_to_send, uint16_t crd_to_send)
{
if (WINGS_ENABLE_ASSERTIONS) assert(ud_c->type == CRD);
ud_c->send_wr[crd_pkts_to_send].send_flags = IBV_SEND_INLINE;
ud_c->send_wr[crd_pkts_to_send].wr.ud.ah = ud_c->remote_qps[dst_qp_id].ah;
ud_c->send_wr[crd_pkts_to_send].wr.ud.remote_qpn =
ud_c->remote_qps[dst_qp_id].qpn;
((wings_crd_t*)&ud_c->send_wr[crd_pkts_to_send].imm_data)->crd_num =
crd_to_send;
if (ud_c->enable_stats) ud_c->stats.send_total_msgs += crd_to_send;
if (crd_pkts_to_send > 0)
ud_c->send_wr[crd_pkts_to_send - 1].next = &ud_c->send_wr[crd_pkts_to_send];
// Selective Signaling --> Do a Signaled Send every ss_granularity pkts
if (ud_c->total_pkts_send % ud_c->ss_granularity == 0) {
// if not the first SS --> poll the previous SS completion
if (ud_c->total_pkts_send > 0) {
struct ibv_wc signal_send_wc;
hrd_poll_cq(ud_c->send_cq, 1, &signal_send_wc);
if (ud_c->enable_stats) ud_c->stats.ss_completions++;
if (WINGS_ENABLE_SS_PRINTS && ud_c->enable_prints)
colored_printf(RED, "^^^ Polled SS completion: %s %d (total %d)\n",
ud_c->qp_name, 1, ud_c->stats.ss_completions);
}
ud_c->send_wr[crd_pkts_to_send].send_flags |= IBV_SEND_SIGNALED;
if (WINGS_ENABLE_SS_PRINTS && ud_c->enable_prints)
colored_printf(RED, "vvv Send SS: %s\n", ud_c->qp_name);
}
ud_c->total_pkts_send++;
}
static inline void
_wings_forge_wr(ud_channel_t* ud_c, uint8_t dst_qp_id, uint8_t* req_to_copy,
uint16_t pkts_in_batch, uint16_t* msgs_in_batch,
copy_and_modify_input_elem_t copy_and_modify_elem,
uint8_t is_small_msg)
// dst_qp_id is ignored if its a bcast channel
{
struct ibv_wc signal_send_wc;
uint8_t curr_req_num = 1;
uint8_t* next_req_ptr;
if (ud_c->is_header_only)
next_req_ptr = ((wings_hdr_only_t*)&ud_c->send_wr[pkts_in_batch].imm_data)
->inlined_payload;
else {
wings_ud_send_pkt_t* curr_pkt_ptr = _wings_curr_send_pkt_ptr(ud_c);
next_req_ptr = _wings_get_n_msg_ptr_from_send_pkt(ud_c, curr_pkt_ptr,
curr_pkt_ptr->req_num);
curr_req_num = ++curr_pkt_ptr->req_num;
curr_pkt_ptr->sender_id = ud_c->channel_id;
uint16_t msg_size =
is_small_msg == 1 ? ud_c->small_msg_size : ud_c->max_msg_size;
ud_c->send_sgl[pkts_in_batch].length =
sizeof(wings_ud_send_pkt_t) +
// ud_c->max_msg_size *
// curr_pkt_ptr->req_num;
msg_size * curr_pkt_ptr->req_num;
if (WINGS_ENABLE_ASSERTIONS)
assert(is_small_msg == 1 ||
curr_req_num == 1); // we only do coalescing for small msgs
if (curr_req_num == 1) {
ud_c->send_sgl[pkts_in_batch].addr = (uint64_t)curr_pkt_ptr;
#if WINGS_ENABLE_TWO_MSG_SIZES == 1
curr_pkt_ptr->only_small_msgs = is_small_msg == 1 ? 1 : 0;
#endif
}
}
//<Copy & modify elem!> --> callback func that copies and manipulated data
// from req_to_copy buff
copy_and_modify_elem(next_req_ptr, req_to_copy);
if (WINGS_ENABLE_ASSERTIONS) {
assert(dst_qp_id != machine_id || ud_c->is_bcast_channel);
assert(curr_req_num <= ud_c->max_coalescing);
}
if (ud_c->enable_stats) ud_c->stats.send_total_msgs++;
if (curr_req_num == 1) {
if (!ud_c->is_bcast_channel) { // set the dst qp
ud_c->send_wr[pkts_in_batch].wr.ud.ah = ud_c->remote_qps[dst_qp_id].ah;
ud_c->send_wr[pkts_in_batch].wr.ud.remote_qpn =
ud_c->remote_qps[dst_qp_id].qpn;
}
uint16_t wr_idx =
(uint16_t)(pkts_in_batch *
(ud_c->is_bcast_channel ? ud_c->num_channels - 1 : 1));
ud_c->send_wr[wr_idx].send_flags =
ud_c->is_inlining_enabled ? IBV_SEND_INLINE : 0;
if (wr_idx > 0) // set previous send_wr to point to curr
ud_c->send_wr[wr_idx - 1].next = &ud_c->send_wr[wr_idx];
// Selective Signaling --> Do a Signaled Send every ss_granularity pkts
if (ud_c->total_pkts_send % ud_c->ss_granularity == 0) {
// if not the first SS --> poll the previous SS completion
if (ud_c->total_pkts_send > 0) {
hrd_poll_cq(ud_c->send_cq, 1, &signal_send_wc);
if (ud_c->enable_stats) ud_c->stats.ss_completions++;
if (WINGS_ENABLE_SS_PRINTS && ud_c->enable_prints)
colored_printf(RED, "^^^ Polled SS completion: %s %d (total %d)\n",
ud_c->qp_name, 1, ud_c->stats.ss_completions);
}
ud_c->send_wr[wr_idx].send_flags |= IBV_SEND_SIGNALED;
if (WINGS_ENABLE_SS_PRINTS && ud_c->enable_prints)
colored_printf(RED, "vvv Send SS: %s\n", ud_c->qp_name);
}
ud_c->total_pkts_send++;
}
(*msgs_in_batch)++;
}
static inline void
_wings_batch_pkts_2_NIC(ud_channel_t* ud_c, uint16_t pkts_in_batch,
uint16_t msgs_in_batch)
{
int ret;
struct ibv_send_wr* bad_send_wr;
if (ud_c->enable_stats) ud_c->stats.send_total_pkts += pkts_in_batch;
uint16_t remote_channels = (uint16_t)(ud_c->num_channels - 1);
uint16_t wr_idx = (uint16_t)(pkts_in_batch *
(ud_c->is_bcast_channel ? remote_channels : 1));
ud_c->send_wr[wr_idx - 1].next = NULL;
if (WINGS_ENABLE_ASSERTIONS) {
assert(pkts_in_batch <= ud_c->max_send_wrs);
assert(pkts_in_batch <= ud_c->send_pkt_buff_len);
assert(ud_c->type == CRD || ud_c->max_coalescing > 1 ||
msgs_in_batch == pkts_in_batch);
assert(ud_c->type == CRD || ud_c->max_coalescing > 1 ||
ud_c->stats.send_total_msgs == ud_c->stats.send_total_pkts);
assert(ud_c->send_wr[wr_idx - 1].next == NULL);
for (int i = 0; i < wr_idx; ++i) {
uint16_t sgl_idx =
(uint16_t)(i / (ud_c->is_bcast_channel ? remote_channels : 1));
if (ud_c->type != CRD && !ud_c->is_header_only) {
assert(ud_c->send_wr[i].num_sge == 1);
assert(ud_c->send_wr[i].opcode == IBV_WR_SEND);
assert(ud_c->send_wr[i].sg_list == &ud_c->send_sgl[sgl_idx]);
wings_ud_send_pkt_t* curr_send_pkt =
(wings_ud_send_pkt_t*)ud_c->send_sgl[sgl_idx].addr;
assert(curr_send_pkt->req_num > 0);
} else {
assert(ud_c->send_wr[i].num_sge == 0);
assert(ud_c->send_wr[i].sg_list->length == 0);
assert(ud_c->send_wr[i].opcode == IBV_WR_SEND_WITH_IMM);
if (ud_c->type == CRD) {
assert(((wings_crd_t*)&(ud_c->send_wr[i].imm_data))->crd_num > 0);
assert(((wings_crd_t*)&(ud_c->send_wr[i].imm_data))->sender_id ==
ud_c->channel_id);
} else
assert(((wings_hdr_only_t*)&(ud_c->send_wr[i].imm_data))->sender_id ==
ud_c->channel_id);
}
assert(ud_c->send_wr[i].wr.ud.remote_qkey == HRD_DEFAULT_QKEY);
assert(i == wr_idx - 1 || ud_c->send_wr[i].next == &ud_c->send_wr[i + 1]);
assert(!ud_c->is_inlining_enabled ||
ud_c->send_wr[i].send_flags == IBV_SEND_INLINE ||
ud_c->send_wr[i].send_flags ==
(IBV_SEND_INLINE | IBV_SEND_SIGNALED));
}
}
if (WINGS_ENABLE_SEND_PRINTS &&
ud_c->enable_prints) // TODO make this work w/ bcasts
colored_printf(CYAN,
">>> Send: %d packets %s %d (Total packets: %d, msgs: %d)\n",
pkts_in_batch, ud_c->qp_name, msgs_in_batch,
ud_c->stats.send_total_pkts, ud_c->stats.send_total_msgs);
ret = ibv_post_send(ud_c->qp, ud_c->send_wr, &bad_send_wr);
CPE(ret, "ibv_post_send error while sending msgs to the NIC", ret);
}
static inline void
_wings_check_if_batch_n_inc_pkt_ptr(ud_channel_t* ud_c,
uint16_t* pkts_in_batch_ptr,
uint16_t* msgs_in_batch_ptr)
{
(*pkts_in_batch_ptr)++;
uint16_t send_pkts = *pkts_in_batch_ptr;
uint16_t total_msgs_in_batch = *msgs_in_batch_ptr;
uint16_t max_pkt_batch =
ud_c->is_bcast_channel ? ud_c->max_pcie_bcast_batch : ud_c->max_send_wrs;
if (send_pkts == max_pkt_batch) {
_wings_batch_pkts_2_NIC(ud_c, send_pkts, total_msgs_in_batch);
*pkts_in_batch_ptr = 0;
*msgs_in_batch_ptr = 0;
}
_wings_inc_send_push_ptr(ud_c); // go to the next pkt
}
static inline uint8_t
wings_set_sender_id_n_msg_type(uint8_t sender_id, uint8_t is_small_msg)
{
if (WINGS_ENABLE_ASSERTIONS) {
assert(sender_id < 128);
assert(is_small_msg == 0 || is_small_msg == 1);
}
return (is_small_msg == 0) ? sender_id + 128 : sender_id;
}
static inline uint8_t
_wings_get_sender_id_n_msg_type(uint8_t skip_or_sender_id,
uint8_t* is_small_msg)
{
if (WINGS_ENABLE_ASSERTIONS) assert(skip_or_sender_id < 258);
*is_small_msg = (skip_or_sender_id >= 128) ? 0 : 1;
return (skip_or_sender_id >= 128) ? skip_or_sender_id - 128
: skip_or_sender_id;
}
static inline uint8_t
wings_issue_pkts(ud_channel_t* ud_c, bit_vector_t* membership,
uint8_t* input_array_of_elems, uint16_t input_array_len,
uint16_t size_of_input_elems,
uint16_t* input_array_rolling_idx,
skip_input_elem_or_get_dst_id_t skip_or_get_sender_id_func_ptr,
modify_input_elem_after_send_t modify_elem_after_send,
copy_and_modify_input_elem_t copy_and_modify_elem)
{
uint8_t curr_msg_dst;
uint8_t is_small_msg = 0;
uint8_t last_msg_dst = 255;
uint8_t has_outstanding_msgs = 0;
uint16_t msgs_in_batch = 0, pkts_in_batch = 0, idx = 0;
if (WINGS_ENABLE_ASSERTIONS)
assert(ud_c->is_header_only ||
_wings_curr_send_pkt_ptr(ud_c)->req_num == 0);
for (int i = 0; i < input_array_len; i++) {
idx = (uint16_t)(input_array_rolling_idx == NULL
? i
: (i + *input_array_rolling_idx) % input_array_len);
// Skip or Respond (copy and send ?)
uint8_t* curr_elem = &input_array_of_elems[idx * size_of_input_elems];
int skip_or_sender_id = skip_or_get_sender_id_func_ptr(curr_elem);
if (skip_or_sender_id < 0) continue;
if (WINGS_ENABLE_ASSERTIONS) assert(skip_or_sender_id < 258);
curr_msg_dst =
_wings_get_sender_id_n_msg_type(skip_or_sender_id, &is_small_msg);
if (ud_c->is_header_only) is_small_msg = 1;
// Break if we do not have sufficient credits
if (!_wings_has_sufficient_crds_membership(ud_c, curr_msg_dst,
membership)) {
has_outstanding_msgs = 1;
if (ud_c->enable_stats) ud_c->stats.no_stalls_due_to_credits++;
if (input_array_rolling_idx != NULL) *input_array_rolling_idx = idx;
break; // we need to break for broadcast (lets assume it is ok to break
// for unicasts as well since it may only harm perf)
}
_wings_dec_crds_membership(ud_c, curr_msg_dst, membership);
if ((!ud_c->is_bcast_channel && !ud_c->is_header_only) ||
is_small_msg == 0) {
// Send unicasts because if we cannot coalesce pkts, due to different
// endpoints
if (_wings_curr_send_pkt_ptr(ud_c)->req_num > 0 &&
(is_small_msg == 0 || curr_msg_dst != last_msg_dst))
_wings_check_if_batch_n_inc_pkt_ptr(ud_c, &pkts_in_batch,
&msgs_in_batch);
}
last_msg_dst = curr_msg_dst;
// Create the messages
_wings_forge_wr(ud_c, curr_msg_dst, curr_elem, pkts_in_batch,
&msgs_in_batch, copy_and_modify_elem, is_small_msg);
modify_elem_after_send(curr_elem); // E.g. Change the state of the element
// which triggered a send
// Check if we should send a batch since we might have reached the max batch
// size
if (is_small_msg == 0 || ud_c->is_header_only ||
_wings_curr_send_pkt_ptr(ud_c)->req_num == ud_c->max_coalescing) {
_wings_check_if_batch_n_inc_pkt_ptr(ud_c, &pkts_in_batch, &msgs_in_batch);
}
}
// Even if the last pkt is not full do the appropriate actions and incl to NIC
// batch
wings_ud_send_pkt_t* curr_pkt_ptr = NULL;
if (!ud_c->is_header_only && is_small_msg == 1) {
curr_pkt_ptr = _wings_curr_send_pkt_ptr(ud_c);
if (curr_pkt_ptr->req_num > 0 &&
curr_pkt_ptr->req_num < ud_c->max_coalescing)
pkts_in_batch++;
}
// Force a batch to send the last set of requests (even < max batch size)
if (pkts_in_batch > 0)
_wings_batch_pkts_2_NIC(ud_c, pkts_in_batch, msgs_in_batch);
if (!ud_c->is_header_only && is_small_msg == 1)
// Move to next packet and reset data left from previous bcasts/unicasts
if (curr_pkt_ptr->req_num > 0 &&
curr_pkt_ptr->req_num < ud_c->max_coalescing)
_wings_inc_send_push_ptr(ud_c);
return has_outstanding_msgs;
}
static inline void
wings_issue_credits(
ud_channel_t* ud_c, bit_vector_t* membership, uint8_t* input_array_of_elems,
uint16_t input_array_len, uint16_t size_of_input_elems,
skip_input_elem_or_get_dst_id_t skip_or_get_sender_id_func_ptr,
modify_input_elem_after_send_t modify_elem_after_send)
{
if (WINGS_ENABLE_ASSERTIONS) assert(ud_c->type == CRD);
for (int i = 0; i < ud_c->num_channels; ++i)
ud_c->no_crds_to_send_per_endpoint[i] = 0;
for (int i = 0; i < input_array_len; ++i) {
// Skip or Respond (copy and send ?)
uint8_t* curr_elem = &input_array_of_elems[i * size_of_input_elems];
int skip_or_sender_id = skip_or_get_sender_id_func_ptr(curr_elem);
if (WINGS_ENABLE_ASSERTIONS) assert(skip_or_sender_id < 255);
if (skip_or_sender_id < 0) continue;
uint8_t curr_msg_dst = (uint8_t)skip_or_sender_id;
// Check if we have sufficient credits --> (we should always have enough
// credits for CRDs)
if (!_wings_has_sufficient_crds_membership(ud_c, curr_msg_dst, membership))
assert(0);
if (ud_c->no_crds_to_send_per_endpoint[curr_msg_dst] == 0 &&
ud_c->credits_per_channels[curr_msg_dst] == 0)
assert(0);
_wings_dec_crds_membership(ud_c, curr_msg_dst, membership);
ud_c->no_crds_to_send_per_endpoint[curr_msg_dst]++;
modify_elem_after_send(curr_elem); // E.g. Change the state of the element
// which triggered a send
}
uint16_t send_crd_packets = 0, total_credits_to_send = 0;
for (uint16_t i = 0; i < ud_c->num_channels; ++i) {
if (i == ud_c->channel_id) continue;
if (ud_c->no_crds_to_send_per_endpoint[i] > 0) {
_wings_forge_crd_wr(ud_c, i, send_crd_packets,
ud_c->no_crds_to_send_per_endpoint[i]);
send_crd_packets++;
total_credits_to_send += ud_c->no_crds_to_send_per_endpoint[i];
if (send_crd_packets == ud_c->max_send_wrs) {
_wings_batch_pkts_2_NIC(ud_c, send_crd_packets, total_credits_to_send);
send_crd_packets = 0;
total_credits_to_send = 0;
}
}
}
if (send_crd_packets > 0)
_wings_batch_pkts_2_NIC(ud_c, send_crd_packets, total_credits_to_send);
}
#endif // WINGS_INTERNAL_INLINES_H
================================================
FILE: include/wings/wings_api.h
================================================
//
// Created by akatsarakis on 06/02/19.
//
#ifndef WINGS_API_H
#define WINGS_API_H
#include "../utils/bit_vector.h"
#include "hrd.h"
/// WARNING!!
/// Accessible functions not defined below (in wings_api.h but exist only in
/// wings.h) and starting with underscore
/// (i.e. "_wings_*") are internal and should not be called directly
/// by the application
#define WINGS_ENABLE_ASSERTIONS 0
#define WINGS_MAX_SUPPORTED_INLINING 187
#define WINGS_ENABLE_BATCH_POST_RECVS_TO_NIC 1
#define WINGS_ENABLE_STAT_COUNTING 1
#define WINGS_MIN_PCIE_BCAST_BATCH 1
#define WINGS_MIN(x, y) (x < y ? x : y)
#define WINGS_ENABLE_PRINTS 0
#define WINGS_ENABLE_SS_PRINTS (1 && WINGS_ENABLE_PRINTS)
#define WINGS_ENABLE_SEND_PRINTS (1 && WINGS_ENABLE_PRINTS)
#define WINGS_ENABLE_RECV_PRINTS (1 && WINGS_ENABLE_PRINTS)
#define WINGS_ENABLE_CREDIT_PRINTS (1 && WINGS_ENABLE_PRINTS)
#define WINGS_ENABLE_POST_RECV_PRINTS (1 && WINGS_ENABLE_PRINTS)
#define WINGS_IS_ROCE 0
#define MAX_MTU_SIZE 4096
/* Useful when `x = (x + 1) % N` is done in a loop */
#define WINGS_MOD_ADD(x, N) \
do { \
x = x + 1; \
if (x == N) x = 0; \
} while (0)
/* ah pointer and qpn are accessed together in the critical path
so we are putting them in the same cache line */
typedef struct {
struct ibv_ah* ah;
uint32_t qpn;
// no padding needed- false sharing is not an issue, only fragmentation
} qp_info_t;
typedef struct {
uint8_t only_small_msgs : 1; // support for up to 256 unique senders per
// instance (e.g. thread)
uint8_t sender_id : 7; // support for up to 128 unique senders per instance
// (e.g. thread)
uint8_t req_num; // <= max_coalescing of a channel
uint8_t reqs[]; // sizeof(req_num * req_size)
} wings_pkt_t, wings_ud_send_pkt_t;
// Packets with GRH
typedef struct {
struct ibv_grh grh;
wings_pkt_t pkt;
} __attribute__((packed))
wings_ud_recv_pkt_t; // rcved rdma ud pkts come with a grh padding
typedef struct {
uint8_t sender_id; // support for up to 256 unique senders per instance (e.g.
// thread)
uint16_t crd_num; // credit num
} __attribute__((packed)) wings_crd_t; // always send as inlined_payload
typedef struct {
uint8_t sender_id; // support for up to 256 unique senders per instance (e.g.
// thread)
uint8_t inlined_payload[3]; // available space to be used by the application
} __attribute__((packed)) wings_hdr_only_t; // always send as inlined_payload
static_assert(sizeof(wings_hdr_only_t) == 4 * sizeof(uint8_t), "");
typedef struct {
uint64_t send_total_msgs;
uint64_t send_total_pkts;
uint64_t send_total_pcie_batches;
uint64_t ss_completions;
uint64_t recv_total_msgs;
uint64_t recv_total_pkts;
uint64_t
no_stalls_due_to_credits; // number of stalls due to not enough credits
} ud_channel_stats_t;
enum channel_type { REQ, RESP, CRD };
typedef struct _ud_channel_t {
struct ibv_qp* qp;
enum channel_type type;
uint8_t max_coalescing;
uint8_t expl_crd_ctrl;
uint8_t disable_crd_ctrl;
uint8_t is_header_only;
uint8_t is_bcast_channel;
uint8_t is_inlining_enabled;
struct _ud_channel_t* channel_providing_crds;
char* qp_name;
uint16_t qp_id; // id of qp in cb
uint16_t max_msg_size;
uint16_t small_msg_size;
uint8_t channel_id; // id of the curr channel (e.g. local node id)
uint16_t num_channels; // e.g. remote nodes + local node
uint16_t num_crds_per_channel;
uint16_t* credits_per_channels; // array size of num_channels denoting
// available space on remote sides
/// Credits refer to msgs irrespective if coalesed or not --> a remote buffer
/// must be able to handle max_number_of_msgs * max_coalescing
volatile uint8_t* recv_pkt_buff; /// Intermediate buffs where reqs are copied
/// when pkts are received
wings_ud_send_pkt_t* send_pkt_buff; /// Intermediate buffs where reqs are
/// copied when pkts are send
uint16_t send_pkt_buff_len;
uint16_t recv_pkt_buff_len;
uint16_t max_send_wrs;
uint16_t max_recv_wrs;
uint16_t send_q_depth;
uint16_t recv_q_depth;
uint16_t ss_granularity; // selective signaling granularity
uint16_t max_pcie_bcast_batch;
uint64_t total_pkts_send; // used for selective signaling
int send_push_ptr;
int recv_push_ptr;
int recv_pull_ptr;
struct ibv_send_wr* send_wr;
struct ibv_recv_wr* recv_wr; // Used only to batch post recvs to the NIC
struct ibv_sge* send_sgl;
struct ibv_sge* recv_sgl; // Used only to batch post recvs to the NIC
struct ibv_cq* send_cq;
struct ibv_cq* recv_cq;
struct ibv_wc* recv_wc; // (size of max_recv_wrs) Used on polling recv req cq
// (only for immediates)
/// Send wcs are omitted since they are only used for selective signaling
/// (within send function calls)
struct ibv_mr* send_mem_region; // NULL if inlining is enabled
struct ibv_pd* pd; // A protection domain for this ud channel
// Remote QPs
qp_info_t* remote_qps;
// Used only for type == CRD
uint16_t* no_crds_to_send_per_endpoint;
// Stats
ud_channel_stats_t stats;
uint8_t enable_overflow_msgs;
uint8_t num_overflow_msgs; // msgs in overflow_msg_buff always <=
// max_coalescing - 1
uint8_t* overflow_msg_buff; // use to keep message in case of polling
// a pkt and it doesn't fit in the recv array we
// Toggles
uint8_t enable_stats;
uint8_t enable_prints;
} ud_channel_t;
// Define some function pointers used when issuing pkts
typedef void (*modify_input_elem_after_send_t)(uint8_t*);
typedef int (*skip_input_elem_or_get_dst_id_t)(
uint8_t*); // Should return -1 to skip otherwise returns the sender id
typedef void (*copy_and_modify_input_elem_t)(uint8_t* msg_to_send,
uint8_t* triggering_req);
static inline void
wings_NOP_modify_elem_after_send(uint8_t* req)
{ /*Do not change anything*/
}
/// Init and Util functions
void wings_print_ud_c_overview(ud_channel_t* ud_c);
void wings_ud_channel_destroy(
ud_channel_t* ud_c); // This must be used to destroy all ud_c (both CRD and
// typical ud_c)
// This is used to int only non-CRDs channels (CRDs are initialized internally)
void wings_ud_channel_init(ud_channel_t* ud_c, char* qp_name,
enum channel_type type, uint8_t max_coalescing,
uint16_t max_req_size, uint16_t small_req_size,
uint8_t enable_inlining, uint8_t is_header_only,
uint8_t is_bcast,
// Credits
uint8_t disable_crd_ctrl, uint8_t expl_crd_ctrl,
ud_channel_t* linked_channel,
uint16_t crds_per_channel, uint16_t num_channels,
uint8_t channel_id,
// Toggles
uint8_t stats_on, uint8_t prints_on);
void wings_setup_channel_qps_and_recvs(ud_channel_t** ud_c_array,
uint16_t ud_c_num,
dbit_vector_t* shared_rdy_var,
uint16_t worker_lid);
/// Main functions
static inline uint16_t wings_poll_buff_and_post_recvs(ud_channel_t* ud_c,
uint16_t max_pkts_to_poll,
uint8_t* recv_buff_space);
static inline uint8_t wings_issue_pkts(
ud_channel_t* ud_c, bit_vector_t* membership, uint8_t* input_array_of_elems,
uint16_t input_array_len, uint16_t size_of_input_elems,
uint16_t* input_array_rolling_idx,
skip_input_elem_or_get_dst_id_t skip_or_get_sender_id_func_ptr,
modify_input_elem_after_send_t modify_elem_after_send,
copy_and_modify_input_elem_t copy_and_modify_elem);
static inline void wings_issue_credits(
ud_channel_t* ud_c, bit_vector_t* membership, uint8_t* input_array_of_elems,
uint16_t input_array_len, uint16_t size_of_input_elems,
skip_input_elem_or_get_dst_id_t skip_or_get_sender_id_func_ptr,
modify_input_elem_after_send_t modify_elem_after_send);
#endif // WINGS_API_H
================================================
FILE: src/CR/crKV.c
================================================
//
// Created by akatsarakis on 07/03/19.
//
#include <spacetime.h>
#include <util.h>
//////////////////////////////////////////////////
//////////////////// Chain Replication / CRAQ KVS
//////////////////////////////////////////////////
//////////////////////////////////////////////////
//////////// Helper functions ////////////////////
static inline uint8_t
head_id()
{
return 0;
}
static inline uint8_t
tail_id()
{
return machine_num - 1;
}
//////////// Assertion functions
static inline void
cr_assertions_inv(spacetime_inv_t* inv_ptr)
{
assert(inv_ptr->op_meta.ts.version % 2 == 0);
assert(inv_ptr->op_meta.opcode == ST_OP_INV ||
inv_ptr->op_meta.opcode == ST_OP_MEMBERSHIP_CHANGE);
assert(inv_ptr->op_meta.val_len == ST_VALUE_SIZE);
}
//////////// Skip functions
static inline uint8_t
cr_skip_op(spacetime_op_t* op_ptr)
{
return (uint8_t)((op_ptr->op_meta.state == ST_PUT_SUCCESS ||
op_ptr->op_meta.state == ST_IN_PROGRESS_GET ||
op_ptr->op_meta.state == ST_IN_PROGRESS_PUT)
? 1
: 0);
}
static inline uint8_t
cr_skip_inv(spacetime_inv_t* inv_ptr)
{
return (uint8_t)(inv_ptr->op_meta.opcode == ST_OP_MEMBERSHIP_CHANGE ? 1 : 0);
}
static inline uint8_t
cr_skip_ack(spacetime_ack_t* ack_ptr)
{
return (uint8_t)(ack_ptr->opcode == ST_OP_MEMBERSHIP_CHANGE ? 1 : 0);
}
static inline uint8_t
cr_skip_remote_reads(spacetime_op_t* op_ptr)
{
return (uint8_t)((op_ptr->op_meta.state == ST_EMPTY) ? 1 : 0);
}
static inline uint8_t
cr_skip_remote_writes(spacetime_op_t* op_ptr)
{
return (uint8_t)((op_ptr->op_meta.state == ST_EMPTY ||
op_ptr->op_meta.state == ST_PUT_SUCCESS ||
op_ptr->op_meta.state == ST_IN_PROGRESS_PUT)
? 1
: 0);
}
//////////// Exec functions
static inline void
cr_exec_write(spacetime_op_t* op_ptr, struct mica_op* kv_ptr)
{
spacetime_object_meta* curr_meta = (spacetime_object_meta*)kv_ptr->value;
uint8_t* kv_value_ptr = (uint8_t*)&curr_meta[1];
if (ENABLE_ASSERTIONS) {
assert(machine_id == head_id()); // Only head must exec writes
assert(op_ptr->op_meta.opcode == ST_OP_PUT);
assert(op_ptr->op_meta.val_len == ST_VALUE_SIZE);
}
op_ptr->op_meta.state = ST_EMPTY;
cctrl_lock(&curr_meta->cctrl);
switch (curr_meta->state) {
case INVALID_STATE:
// Do not initiate a new write until you get to valid state
if (CR_ENABLE_BLOCKING_INVALID_WRITES_ON_HEAD) {
cctrl_unlock_dec_version(&curr_meta->cctrl);
op_ptr->op_meta.state = ST_PUT_STALL;
break;
}
case VALID_STATE:
curr_meta->state = INVALID_STATE;
memcpy(kv_value_ptr, op_ptr->value, ST_VALUE_SIZE);
kv_ptr->val_len = op_ptr->op_meta.val_len + sizeof(spacetime_object_meta);
cctrl_unlock_inc_version(&curr_meta->cctrl, (uint8_t)machine_id,
(uint32_t*)&(op_ptr->op_meta.ts.version));
op_ptr->op_meta.state = ST_PUT_SUCCESS;
op_ptr->op_meta.ts.tie_breaker_id = (uint8_t)machine_id;
break;
default:
assert(0);
}
}
static inline void
cr_exec_remote_reads(spacetime_op_t* op_ptr, struct mica_op* kv_ptr)
{
if (ENABLE_ASSERTIONS) {
assert(machine_id == tail_id());
assert(op_ptr->op_meta.opcode == ST_OP_GET);
}
// the following variables used to validate atomicity between a lock-free read
// of an object
spacetime_object_meta prev_meta;
spacetime_object_meta* curr_meta = (spacetime_object_meta*)kv_ptr->value;
uint8_t* kv_value_ptr = (uint8_t*)&curr_meta[1];
do {
prev_meta = *curr_meta;
// switch template with all states
switch (curr_meta->state) {
case VALID_STATE:
memcpy(op_ptr->value, kv_value_ptr, ST_VALUE_SIZE);
op_ptr->op_meta.state = ST_GET_COMPLETE;
op_ptr->op_meta.val_len =
kv_ptr->val_len - sizeof(spacetime_object_meta);
break;
case INVALID_STATE:
default:
assert(0);
}
} while (
!cctrl_timestamp_is_same_and_valid(&prev_meta.cctrl, &curr_meta->cctrl));
}
static inline void
cr_exec_op(spacetime_op_t* op_ptr, struct mica_op* kv_ptr, uint8_t idx)
{
if (ENABLE_ASSERTIONS) assert(idx < max_batch_size);
// the following variables used to validate atomicity between a lock-free read
// of an object
spacetime_object_meta prev_meta;
spacetime_object_meta* curr_meta = (spacetime_object_meta*)kv_ptr->value;
uint8_t* kv_value_ptr = (uint8_t*)&curr_meta[1];
if (op_ptr->op_meta.opcode == ST_OP_GET) {
// Lock free reads through versioning (successful when version is even)
op_ptr->op_meta.state = ST_EMPTY;
do {
prev_meta = *curr_meta;
// switch template with all states
switch (curr_meta->state) {
case VALID_STATE:
memcpy(op_ptr->value, kv_value_ptr, ST_VALUE_SIZE);
op_ptr->op_meta.state = ST_GET_COMPLETE;
op_ptr->op_meta.val_len =
kv_ptr->val_len - sizeof(spacetime_object_meta);
break;
case INVALID_STATE:
if (ENABLE_ASSERTIONS)
assert(machine_id != tail_id()); // tail should always be valid
op_ptr->op_meta.state = ST_GET_STALL;
break;
default:
assert(0);
}
} while (!cctrl_timestamp_is_same_and_valid(&prev_meta.cctrl,
&curr_meta->cctrl));
if (op_ptr->op_meta.state == ST_GET_STALL) op_ptr->buff_idx = idx;
}
else if (op_ptr->op_meta.opcode == ST_OP_PUT) {
if (machine_id == head_id()) // if it is head
cr_exec_write(op_ptr, kv_ptr);
else
op_ptr->op_meta.state = ST_PUT_SUCCESS;
if (op_ptr->op_meta.state == ST_PUT_SUCCESS)
// Set idx that we cannot set while dispatching the req
op_ptr->buff_idx = idx;
}
}
static inline void
cr_complete_local_write(spacetime_op_t* read_write_op, uint8_t idx,
const uint64_t* key)
{
/// completed read / write --> remove it from the ops buffer
if (ENABLE_ASSERTIONS) {
assert(read_write_op[idx].op_meta.state == ST_IN_PROGRESS_PUT);
assert(((uint64_t*)&read_write_op[idx].op_meta.key)[0] == key[0]);
}
if (read_write_op[idx].op_meta.opcode == ST_OP_PUT)
read_write_op[idx].op_meta.state = ST_PUT_COMPLETE;
else
assert(0);
}
static inline void
cr_exec_inv(spacetime_inv_t* inv_ptr, struct mica_op* kv_ptr,
spacetime_op_t* read_write_op)
{
// the following variables used to validate atomicity between a lock-free read
// of an object
spacetime_object_meta lock_free_meta;
spacetime_object_meta* curr_meta = (spacetime_object_meta*)kv_ptr->value;
uint8_t* kv_value_ptr = (uint8_t*)&curr_meta[1];
if (ENABLE_ASSERTIONS) assert(inv_ptr->op_meta.opcode == ST_OP_INV);
uint32_t debug_cntr = 0;
do { // Lock free read of keys meta
if (ENABLE_ASSERTIONS) {
debug_cntr++;
if (debug_cntr == M_4) {
printf("Worker stuck on a lock-free read (for INV)\n");
debug_cntr = 0;
}
}
lock_free_meta = *curr_meta;
} while (!cctrl_timestamp_is_same_and_valid(&lock_free_meta.cctrl,
&curr_meta->cctrl));
// lock and proceed iff remote.TS >= local.TS
// inv TS >= local timestamp
if (!timestamp_is_smaller(inv_ptr->op_meta.ts.version,
inv_ptr->op_meta.ts.tie_breaker_id,
lock_free_meta.cctrl.ts.version,
lock_free_meta.cctrl.ts.tie_breaker_id)) {
// Lock and check again if inv TS > local timestamp
cctrl_lock(&curr_meta->cctrl);
/// Warning: use op.version + 1 bellow since optik_lock() increases
/// curr_meta->version by 1
if (timestamp_is_smaller(
curr_meta->cctrl.ts.version - 1, curr_meta->cctrl.ts.tie_breaker_id,
inv_ptr->op_meta.ts.version, inv_ptr->op_meta.ts.tie_breaker_id)) {
// printf("Received
// an invalidation with >= timestamp\n");
/// Update Value, TS and last_writer_id
// curr_meta->last_writer_id =
// inv_ptr->op_meta.sender;
kv_ptr->val_len =
inv_ptr->op_meta.val_len + sizeof(spacetime_object_meta);
if (ENABLE_ASSERTIONS) {
// assert(kv_ptr->val_len ==
// KVS_VALUE_SIZE
//>> SHIFT_BITS);
assert(inv_ptr->op_meta.val_len == ST_VALUE_SIZE >> SHIFT_BITS);
}
memcpy(kv_value_ptr, inv_ptr->value, ST_VALUE_SIZE);
/// Update state
switch (curr_meta->state) {
case VALID_STATE:
if (machine_id != tail_id()) // Tail never gets invalid
curr_meta->state = INVALID_STATE;
break;
case INVALID_STATE:
break;
default:
assert(0);
}
cctrl_unlock_custom_version(&curr_meta->cctrl,
inv_ptr->op_meta.ts.tie_breaker_id,
inv_ptr->op_meta.ts.version);
} else if (timestamp_is_equal(curr_meta->cctrl.ts.version - 1,
curr_meta->cctrl.ts.tie_breaker_id,
inv_ptr->op_meta.ts.version,
inv_ptr->op_meta.ts.tie_breaker_id))
assert(0);
else
cctrl_unlock_dec_version(&curr_meta->cctrl);
}
inv_ptr->op_meta.opcode = ST_INV_SUCCESS;
if (inv_ptr->op_meta.initiator == machine_id && machine_id == tail_id())
cr_complete_local_write(read_write_op, inv_ptr->buff_idx,
(uint64_t*)&inv_ptr->op_meta.key);
if (ENABLE_ASSERTIONS) assert(inv_ptr->op_meta.opcode == ST_INV_SUCCESS);
}
static inline void
cr_exec_ack(spacetime_ack_t* ack_ptr, struct mica_op* kv_ptr,
spacetime_op_t* read_write_op)
{
if (ENABLE_ASSERTIONS) assert(machine_id != tail_id());
// the following variables used to validate atomicity between a lock-free read
// of an object
spacetime_object_meta lock_free_read_meta;
spacetime_object_meta* curr_meta = (spacetime_object_meta*)kv_ptr->value;
if (ack_ptr->opcode != ST_OP_ACK) assert(0);
uint32_t debug_cntr = 0;
do { // Lock free read of keys meta
if (ENABLE_ASSERTIONS) {
debug_cntr++;
if (debug_cntr == M_4) {
printf("Worker stuck on a lock-free read (for ACK)\n");
debug_cntr = 0;
}
}
lock_free_read_meta = *curr_meta;
} while (!cctrl_timestamp_is_same_and_valid(&lock_free_read_meta.cctrl,
&curr_meta->cctrl));
if (ENABLE_ASSERTIONS)
assert(!timestamp_is_smaller(lock_free_read_meta.cctrl.ts.version,
lock_free_read_meta.cctrl.ts.tie_breaker_id,
ack_ptr->ts.version,
ack_ptr->ts.tie_breaker_id));
if (timestamp_is_equal(ack_ptr->ts.version, ack_ptr->ts.tie_breaker_id,
lock_free_read_meta.cctrl.ts.version,
lock_free_read_meta.cctrl.ts.tie_breaker_id)) {
/// Lock and check again if ack TS == last local write
cctrl_lock(&curr_meta->cctrl);
if (timestamp_is_equal(ack_ptr->ts.version, ack_ptr->ts.tie_breaker_id,
curr_meta->cctrl.ts.version - 1,
curr_meta->cctrl.ts.tie_breaker_id)) {
switch (curr_meta->state) {
case INVALID_STATE:
curr_meta->state = VALID_STATE;
ack_ptr->opcode = ST_LAST_ACK_SUCCESS;
break;
case VALID_STATE:
default:
assert(0);
}
}
cctrl_unlock_dec_version(&curr_meta->cctrl);
}
if (machine_id == ack_ptr->initiator)
cr_complete_local_write(read_write_op, ack_ptr->buff_idx,
(uint64_t*)&ack_ptr->key);
ack_ptr->opcode = ST_LAST_ACK_SUCCESS;
}
//////////// Dispatcher functions
static inline uint8_t
cr_skip_dispatcher(enum cr_type_t cr_type, void* ptr)
{
switch (cr_type) {
case Local_ops:
return cr_skip_op(ptr);
case Invs:
return cr_skip_inv(ptr);
case Acks:
return cr_skip_ack(ptr);
case Remote_reads:
return cr_skip_remote_reads(ptr);
case Remote_writes:
return cr_skip_remote_writes(ptr);
default:
assert(0);
}
}
static inline void
cr_assertions_dispatcher(enum cr_type_t cr_type, void* ptr)
{
if (ENABLE_ASSERTIONS) switch (cr_type) {
case Invs:
cr_assertions_inv(ptr);
case Acks:
case Remote_writes:
case Local_ops:
case Remote_reads:
break;
default:
assert(0);
}
}
static inline void
cr_exec_dispatcher(enum cr_type_t cr_type, void* op_ptr, struct mica_op* kv_ptr,
uint8_t idx, spacetime_op_t* read_write_op)
{
switch (cr_type) {
case Invs:
cr_exec_inv(op_ptr, kv_ptr, read_write_op);
break;
case Acks:
cr_exec_ack(op_ptr, kv_ptr, read_write_op);
break;
case Remote_writes:
cr_exec_write(op_ptr, kv_ptr);
break;
case Local_ops:
cr_exec_op(op_ptr, kv_ptr, idx);
break;
case Remote_reads:
cr_exec_remote_reads(op_ptr, kv_ptr);
break;
default:
assert(0);
}
}
//////////////////////////////////////////////////
//////////// Batch function //////////////////////
void
cr_batch_ops_to_KVS(enum cr_type_t cr_type, uint8_t* op_array, int op_num,
uint16_t sizeof_op_elem, spacetime_op_t* read_write_op)
{
#if SPACETIME_DEBUG == 1
// assert(kv.hash_table != NULL);
assert(op_array != NULL);
assert(op_num > 0 && op_num <= CACHE_BATCH_SIZE);
assert(resp != NULL);
#endif
#if SPACETIME_DEBUG == 2
for (I = 0; I < op_num; I++)
mica_print_op(&(*op_array)[I]);
#endif
int key_in_store[CR_MAX_BATCH_SIZE]; // Is this key in the datastore?
unsigned int tag[CR_MAX_BATCH_SIZE];
// unsigned int bkt[CR_MAX_BATCH_SIZE];
uint64_t bkt[CR_MAX_BATCH_SIZE];
struct mica_bkt* bkt_ptr[CR_MAX_BATCH_SIZE];
struct mica_op* kv_ptr[CR_MAX_BATCH_SIZE]; // Ptr to KV item in log
if (ENABLE_ASSERTIONS) assert(read_write_op != NULL || cr_type != Acks);
// We first lookup the key in the datastore.
// The first two @I loops work for both GETs and PUTs.
for (int I = 0; I < op_num; I++) {
spacetime_op_meta_t* op_ptr =
(spacetime_op_meta_t*)&op_array[sizeof_op_elem * I];
cr_assertions_dispatcher(cr_type, op_ptr);
if (cr_skip_dispatcher(cr_type, op_ptr)) continue;
bkt[I] = op_ptr->key.bkt & kv.hash_table.bkt_mask;
bkt_ptr[I] = &kv.hash_table.ht_index[bkt[I]];
__builtin_prefetch(bkt_ptr[I], 0, 0);
tag[I] = op_ptr->key.tag;
key_in_store[I] = 0;
kv_ptr[I] = NULL;
}
for (int I = 0; I < op_num; I++) {
spacetime_op_meta_t* op_ptr =
(spacetime_op_meta_t*)&op_array[sizeof_op_elem * I];
if (cr_skip_dispatcher(cr_type, op_ptr)) continue;
for (int j = 0; j < 8; j++) {
if (bkt_ptr[I]->slots[j].in_use == 1 &&
bkt_ptr[I]->slots[j].tag == tag[I]) {
uint64_t log_offset =
bkt_ptr[I]->slots[j].offset & kv.hash_table.log_mask;
// We can interpret the log entry as mica_op, even though it
// may not contain the full MICA_MAX_VALUE value.
kv_ptr[I] = (struct mica_op*)&kv.hash_table.ht_log[log_offset];
// Small values (1--64 bytes) can span 2 cache lines
__builtin_prefetch(kv_ptr[I], 0, 0);
__builtin_prefetch((uint8_t*)kv_ptr[I] + 64, 0, 0);
// Detect if the head has wrapped around for this index entry
if (kv.hash_table.log_head - bkt_ptr[I]->slots[j].offset >=
kv.hash_table.log_cap) {
kv_ptr[I] = NULL; // If so, we mark it "not found"
}
break;
}
}
}
for (int I = 0; I < op_num; I++) {
spacetime_op_meta_t* op_ptr =
(spacetime_op_meta_t*)&op_array[sizeof_op_elem * I];
if (cr_skip_dispatcher(cr_type, op_ptr)) continue;
if (kv_ptr[I] != NULL) {
// We had a tag match earlier. Now compare log entry.
long long* key_ptr_log = (long long*)kv_ptr[I];
long long* key_ptr_req = (long long*)&op_ptr->key;
if (key_ptr_log[1] == key_ptr_req[0]) { // Key Found 8 Byte keys
key_in_store[I] = 1;
cr_exec_dispatcher(cr_type, op_ptr, kv_ptr[I], (uint8_t)I,
read_write_op);
}
}
if (key_in_store[I] ==
0) // KVS miss --> We get here if either tag or log key match failed
op_ptr->state = ST_MISS;
}
if (ENABLE_ASSERTIONS)
if (cr_type == Acks)
for (int I = 0; I < max_batch_size; I++)
assert(read_write_op[I].op_meta.opcode == ST_OP_GET ||
read_write_op[I].op_meta.state == ST_MISS ||
read_write_op[I].op_meta.state == ST_EMPTY ||
read_write_op[I].op_meta.state == ST_PUT_STALL ||
read_write_op[I].op_meta.state == ST_PUT_SUCCESS ||
read_write_op[I].op_meta.state == ST_PUT_COMPLETE ||
read_write_op[I].op_meta.state == ST_IN_PROGRESS_PUT ||
read_write_op[I].op_meta.state ==
ST_OP_MEMBERSHIP_CHANGE || /// TODO check this
read_write_op[I].op_meta.state == ST_IN_PROGRESS_REPLAY);
}
================================================
FILE: src/CR/cr_worker.c
================================================
#include <spacetime.h>
#include <time.h>
#include "../../include/utils/concur_ctrl.h"
#include "inline-util.h"
#include "util.h"
///
#include "../../include/utils/time_rdtsc.h"
#include "../../include/wings/wings.h"
///
static inline uint8_t
head_id(void)
{
return (uint8_t)0;
}
static inline uint8_t
tail_id(void)
{
return machine_num - 1;
}
static inline uint8_t
next_node_in_chain(void)
{
return (uint8_t)((machine_id + 1) % machine_num);
}
static inline uint8_t
prev_node_in_chain(void)
{
return (uint8_t)(machine_id == 0 ? tail_id() : machine_id - 1);
}
int
inv_skip_or_fwd_to_next_node(uint8_t* req)
{
spacetime_inv_t* inv_req = (spacetime_inv_t*)req;
return inv_req->op_meta.opcode == ST_INV_SUCCESS
? next_node_in_chain()
: -1; // invs should only be fwded to next node
}
void
inv_fwd_modify_elem_after_send(uint8_t* req)
{
spacetime_inv_t* inv_req = (spacetime_inv_t*)req;
// empty inv buffer
if (inv_req->op_meta.opcode == ST_INV_SUCCESS ||
inv_req->op_meta.opcode == ST_OP_MEMBERSHIP_CHANGE)
inv_req->op_meta.opcode = ST_EMPTY;
else
assert(0);
}
void
inv_fwd_copy_and_modify_elem(uint8_t* msg_to_send, uint8_t* triggering_req)
{
spacetime_inv_t* inv_recv = (spacetime_inv_t*)triggering_req;
spacetime_inv_t* inv_to_send = (spacetime_inv_t*)msg_to_send;
// Copy op to inv and set opcode
memcpy(inv_to_send, inv_recv, sizeof(spacetime_inv_t));
inv_to_send->op_meta.opcode = ST_OP_INV;
}
int
inv_skip_or_get_sender_id(uint8_t* req)
{
spacetime_op_t* op_req = (spacetime_op_t*)req;
if (ENABLE_ASSERTIONS) {
assert(is_input_code(op_req->op_meta.opcode));
assert(is_response_code(op_req->op_meta.state) ||
is_bucket_state_code(op_req->op_meta.state));
}
return op_req->op_meta.state == ST_PUT_SUCCESS
? next_node_in_chain()
: -1; // since invs should only be fwded to next node
}
void
inv_modify_elem_after_send(uint8_t* req)
{
spacetime_op_t* op_req = (spacetime_op_t*)req;
if (op_req->op_meta.state == ST_PUT_SUCCESS)
op_req->op_meta.state = ST_IN_PROGRESS_PUT;
else
assert(0);
}
void
inv_copy_and_modify_elem(uint8_t* msg_to_send, uint8_t* triggering_req)
{
if (ENABLE_ASSERTIONS) assert(machine_id == head_id());
spacetime_op_t* op = (spacetime_op_t*)triggering_req;
spacetime_inv_t* inv_to_send = (spacetime_inv_t*)msg_to_send;
// Copy op to inv, set sender and opcode
memcpy(inv_to_send, op, sizeof(spacetime_inv_t));
inv_to_send->op_meta.opcode = ST_OP_INV;
inv_to_send->op_meta.initiator = (uint8_t)machine_id;
}
int
remote_write_skip_or_get_sender_id(uint8_t* req)
{
spacetime_op_t* op_req = (spacetime_op_t*)req;
if (ENABLE_ASSERTIONS) {
assert(is_input_code(op_req->op_meta.opcode));
assert(is_response_code(op_req->op_meta.state) ||
is_bucket_state_code(op_req->op_meta.state));
}
return op_req->op_meta.state == ST_PUT_SUCCESS
? head_id()
: -1; // send remote writes to head
}
void
remote_write_copy_and_modify_elem(uint8_t* msg_to_send, uint8_t* triggering_req)
{
if (ENABLE_ASSERTIONS) assert(machine_id != head_id());
spacetime_op_t* op = (spacetime_op_t*)triggering_req;
spacetime_inv_t* inv_to_send = (spacetime_inv_t*)msg_to_send;
// Copy op to inv, set sender and opcode
memcpy(inv_to_send, op, sizeof(spacetime_inv_t));
inv_to_send->op_meta.state = ST_NEW;
inv_to_send->op_meta.opcode = ST_OP_PUT;
inv_to_send->initiator = (uint8_t)machine_id;
inv_to_send->op_meta.initiator = (uint8_t)machine_id;
}
int
remote_write_head_skip_or_get_sender_id(uint8_t* req)
{
spacetime_op_t* op_req = (spacetime_op_t*)req;
if (ENABLE_ASSERTIONS) {
assert(machine_id == head_id());
assert(is_input_code(op_req->op_meta.opcode) ||
op_req->op_meta.opcode == ST_EMPTY);
assert(is_response_code(op_req->op_meta.state) ||
is_bucket_state_code(op_req->op_meta.state));
}
return op_req->op_meta.state == ST_PUT_SUCCESS
? next_node_in_chain()
: -1; // remote writes must always be fwded to head
}
void
remote_write_head_copy_and_modify_elem(uint8_t* msg_to_send,
uint8_t* triggering_req)
{
spacetime_op_t* op = (spacetime_op_t*)triggering_req;
spacetime_inv_t* inv_to_send = (spacetime_inv_t*)msg_to_send;
// Copy op to inv, set sender and opcode
memcpy(inv_to_send, op, sizeof(spacetime_inv_t));
inv_to_send->op_meta.opcode = ST_OP_INV;
inv_to_send->op_meta.initiator = op->initiator;
}
void
remote_write_head_modify_elem_after_send(uint8_t* req)
{
spacetime_op_t* op_req = (spacetime_op_t*)req;
if (op_req->op_meta.state == ST_PUT_SUCCESS)
op_req->op_meta.state = ST_SEND_CRD;
else
assert(0);
}
void
ack_fwd_modify_elem_after_send(uint8_t* req)
{
spacetime_ack_t* ack_req = (spacetime_ack_t*)req;
if (ENABLE_ASSERTIONS) assert(ack_req->opcode == ST_LAST_ACK_SUCCESS);
ack_req->opcode = ST_EMPTY;
}
int
ack_fwd_skip_or_get_sender_id(uint8_t* req)
{
spacetime_ack_t* ack_req = (spacetime_ack_t*)req;
if (ack_req->opcode == ST_ACK_SUCCESS) {
ack_req->opcode = ST_EMPTY;
return -1;
} else if (ack_req->opcode == ST_EMPTY)
return -1;
if (ENABLE_ASSERTIONS) assert(ack_req->opcode == ST_LAST_ACK_SUCCESS);
return prev_node_in_chain();
}
void
ack_fwd_copy_and_modify_elem(uint8_t* msg_to_send, uint8_t* triggering_req)
{
spacetime_ack_t* ack_to_send = (spacetime_ack_t*)msg_to_send;
memcpy(ack_to_send, triggering_req,
sizeof(spacetime_ack_t)); // copy req to next_req_ptr
ack_to_send->opcode = ST_OP_ACK;
}
int
ack_skip_or_get_sender_id(uint8_t* req)
{
spacetime_inv_t* inv_req = (spacetime_inv_t*)req;
if (ENABLE_ASSERTIONS)
assert(inv_req->op_meta.opcode == ST_INV_SUCCESS ||
inv_req->op_meta.opcode == ST_EMPTY);
return prev_node_in_chain();
}
void
ack_modify_elem_after_send(uint8_t* req)
{
spacetime_inv_t* inv_req = (spacetime_inv_t*)req;
// empty inv buffer
if (inv_req->op_meta.opcode == ST_INV_SUCCESS ||
inv_req->op_meta.opcode == ST_OP_MEMBERSHIP_CHANGE)
inv_req->op_meta.opcode = ST_EMPTY;
else
assert(0);
}
void
ack_copy_and_modify_elem(uint8_t* msg_to_send, uint8_t* triggering_req)
{
spacetime_ack_t* ack_to_send = (spacetime_ack_t*)msg_to_send;
spacetime_inv_t* inv_ptr = (spacetime_inv_t*)triggering_req;
memcpy(ack_to_send, inv_ptr,
sizeof(spacetime_ack_t)); // copy req to next_req_ptr
ack_to_send->opcode = ST_OP_ACK;
ack_to_send->buff_idx = inv_ptr->buff_idx;
}
int
rem_write_crd_skip_or_get_sender_id(uint8_t* req)
{
spacetime_op_t* op_ptr = (spacetime_op_t*)req;
if (ENABLE_ASSERTIONS)
assert(op_ptr->op_meta.state == ST_EMPTY ||
op_ptr->op_meta.state == ST_SEND_CRD ||
op_ptr->op_meta.state == ST_PUT_STALL ||
op_ptr->op_meta.state == ST_PUT_SUCCESS);
return op_ptr->op_meta.state == ST_SEND_CRD ? op_ptr->initiator : -1;
}
void
rem_write_crd_modify_elem_after_send(uint8_t* req)
{
spacetime_op_t* op = (spacetime_op_t*)req;
// empty inv buffer
if (op->op_meta.state == ST_SEND_CRD)
op->op_meta.state = ST_EMPTY;
else
assert(0);
}
int
inv_crd_skip_or_get_sender_id(uint8_t* req)
{
spacetime_inv_t* op_ptr = (spacetime_inv_t*)req;
if (ENABLE_ASSERTIONS)
assert(op_ptr->op_meta.opcode == ST_EMPTY ||
op_ptr->op_meta.opcode == ST_INV_SUCCESS);
return op_ptr->op_meta.opcode == ST_INV_SUCCESS ? prev_node_in_chain() : -1;
}
void
inv_crd_modify_elem_after_send(uint8_t* req)
{
if (ENABLE_ASSERTIONS) {
spacetime_inv_t* op = (spacetime_inv_t*)req;
assert(op->op_meta.opcode == ST_INV_SUCCESS);
}
}
int
remote_read_skip_or_get_sender_id(uint8_t* req)
{
spacetime_op_t* op_req = (spacetime_op_t*)req;
if (ENABLE_ASSERTIONS) {
assert(is_input_code(op_req->op_meta.opcode));
assert(is_response_code(op_req->op_meta.state) ||
is_bucket_state_code(op_req->op_meta.state));
}
return op_req->op_meta.state == ST_GET_STALL
? tail_id()
: -1; // send remote writes to head
}
void
remote_read_modify_elem_after_send(uint8_t* req)
{
spacetime_op_t* op_req = (spacetime_op_t*)req;
if (op_req->op_meta.state == ST_GET_STALL)
op_req->op_meta.state = ST_IN_PROGRESS_GET;
else
assert(0);
}
void
remote_read_copy_and_modify_elem(uint8_t* msg_to_send, uint8_t* triggering_req)
{
if (ENABLE_ASSERTIONS) assert(machine_id != tail_id());
spacetime_op_t* op = (spacetime_op_t*)triggering_req;
spacetime_op_t* op_to_send = (spacetime_op_t*)msg_to_send;
// Copy op to inv, set sender and opcode
memcpy(op_to_send, op, sizeof(spacetime_op_t));
op_to_send->op_meta.state = ST_NEW;
op_to_send->op_meta.opcode = ST_OP_GET;
op_to_send->initiator = (uint8_t)machine_id;
op_to_send->op_meta.initiator = (uint8_t)machine_id;
}
int
remote_read_resp_skip_or_get_sender_id(uint8_t* req)
{
spacetime_op_t* op_req = (spacetime_op_t*)req;
if (ENABLE_ASSERTIONS) {
if (op_req->op_meta.opcode != ST_OP_GET) {
printf("Opcode: %d, state: %d\n", op_req->op_meta.opcode,
op_req->op_meta.state);
printf("Opcode: %s, state: %s\n", code_to_str(op_req->op_meta.opcode),
code_to_str(op_req->op_meta.state));
}
assert(op_req->op_meta.opcode == ST_OP_GET);
assert(op_req->op_meta.state == ST_GET_COMPLETE);
}
return op_req->initiator; // send remote writes to head
}
void
remote_read_resp_modify_elem_after_send(uint8_t* req)
{
spacetime_op_t* op_req = (spacetime_op_t*)req;
if (op_req->op_meta.state == ST_GET_COMPLETE)
op_req->op_meta.state = ST_EMPTY;
else {
printf("St_opcode: %s\n", code_to_str(op_req->op_meta.state));
assert(0);
}
}
void
remote_read_resp_copy_and_modify_elem(uint8_t* msg_to_send,
uint8_t* triggering_req)
{
if (ENABLE_ASSERTIONS) assert(machine_id == tail_id());
spacetime_op_t* op = (spacetime_op_t*)triggering_req;
spacetime_op_t* op_to_send = (spacetime_op_t*)msg_to_send;
// Copy op to inv, set sender and opcode
memcpy(op_to_send, op, sizeof(spacetime_op_t));
}
void
print_ops_and_remote_write_ops(spacetime_op_t* ops,
spacetime_op_t* remote_writes)
{
// for(int i = 0; i < MAX_BATCH_KVS_OPS_SIZE; ++i)
for (int i = 0; i < max_batch_size; ++i)
printf("ops[%d]: state-> %s, key-> %lu \n", i,
code_to_str(ops[i].op_meta.state),
*((uint64_t*)&ops[i].op_meta.key));
if (machine_id == head_id())
// for(int i = 0; i < MAX_BATCH_KVS_OPS_SIZE; ++i)
for (int i = 0; i < max_batch_size; ++i)
printf("remote_writes[%d]: state-> %s, key-> %lu \n", i,
code_to_str(remote_writes[i].op_meta.state),
*((uint64_t*)&remote_writes[i].op_meta.key));
}
void
print_total_stalls_due_to_credits(ud_channel_t* inv_ud_c,
ud_channel_t* ack_ud_c,
ud_channel_t* rem_writes_ud_c,
ud_channel_t* rem_reads_ud_c)
{
// Stalls
colored_printf(GREEN, "$$$ CRD STALLs : %s %d, %s %d, %s %d,",
inv_ud_c->qp_name, inv_ud_c->stats.send_total_msgs,
ack_ud_c->qp_name, ack_ud_c->stats.send_total_msgs,
rem_writes_ud_c->qp_name,
rem_writes_ud_c->stats.send_total_msgs);
if (CR_ENABLE_REMOTE_READS)
colored_printf(GREEN, ", %s %d\n", rem_reads_ud_c->qp_name,
rem_reads_ud_c->stats.send_total_msgs);
else
printf("\n");
}
void
print_total_send_recv_msgs_n_credits(
ud_channel_t* inv_ud_c, ud_channel_t* inv_crd_ud_c, ud_channel_t* ack_ud_c,
ud_channel_t* rem_writes_ud_c, ud_channel_t* crd_ud_c,
ud_channel_t* rem_reads_ud_c, ud_channel_t* rem_read_resp_ud_c)
{
// Sends
colored_printf(GREEN, "--> Total Send: %s %d", inv_ud_c->qp_name,
inv_ud_c->stats.send_total_msgs);
if (CR_ENABLE_EARLY_INV_CRDS)
colored_printf(GREEN, ", %s %d", inv_crd_ud_c->qp_name,
inv_crd_ud_c->stats.send_total_msgs);
colored_printf(GREEN, ", %s %d, %s %d, %s %d", ack_ud_c->qp_name,
ack_ud_c->stats.send_total_msgs, rem_writes_ud_c->qp_name,
rem_writes_ud_c->stats.send_total_msgs, crd_ud_c->qp_name,
crd_ud_c->stats.send_total_msgs);
if (CR_ENABLE_REMOTE_READS)
colored_printf(GREEN, ", %s %d, %s %d\n", rem_reads_ud_c->qp_name,
rem_reads_ud_c->stats.send_total_msgs,
rem_read_resp_ud_c->qp_name,
rem_read_resp_ud_c->stats.send_total_msgs);
else
printf("\n");
// Receives
colored_printf(GREEN, "vvv Total Recv: %s %d", inv_ud_c->qp_name,
inv_ud_c->stats.recv_total_msgs);
if (CR_ENABLE_EARLY_INV_CRDS)
colored_printf(GREEN, ", %s %d", inv_crd_ud_c->qp_name,
inv_crd_ud_c->stats.recv_total_msgs);
colored_printf(GREEN, ", %s %d, %s %d, %s %d", ack_ud_c->qp_name,
ack_ud_c->stats.recv_total_msgs, rem_writes_ud_c->qp_name,
rem_writes_ud_c->stats.recv_total_msgs, crd_ud_c->qp_name,
crd_ud_c->stats.recv_total_msgs);
if (CR_ENABLE_REMOTE_READS)
colored_printf(GREEN, ", %s %d, %s %d\n", rem_reads_ud_c->qp_name,
rem_reads_ud_c->stats.recv_total_msgs,
rem_read_resp_ud_c->qp_name,
rem_read_resp_ud_c->stats.recv_total_msgs);
else
printf("\n");
// Credits
uint8_t remote_node =
(uint8_t)(machine_id == head_id() ? next_node_in_chain() : head_id());
printf("Inv credits: %d, ack credits: %d, remote_write_crds: %d\n",
inv_ud_c->credits_per_channels[remote_node],
ack_ud_c->credits_per_channels[remote_node],
rem_writes_ud_c->credits_per_channels[head_id()]);
}
static inline void
cr_complete_local_reads(spacetime_op_t* remote_reads_resps,
uint16_t remote_read_resps_polled, spacetime_op_t* ops)
{
for (int i = 0; i < remote_read_resps_polled; ++i) {
uint16_t idx = remote_reads_resps[i].buff_idx;
/// completed read / write --> remove it from the ops buffer
if (ENABLE_ASSERTIONS) {
assert(ops[idx].op_meta.state == ST_IN_PROGRESS_GET);
assert(((uint64_t*)&ops[idx].op_meta.key)[0] ==
((uint64_t*)&remote_reads_resps[i].op_meta.key)[0]);
}
if (ops[idx].op_meta.opcode == ST_OP_GET)
ops[idx].op_meta.state = ST_GET_COMPLETE;
gitextract_es11wv71/
├── .clang-format
├── .gitignore
├── AUTHORS
├── CMakeLists.txt
├── LICENSE
├── README.md
├── bin/
│ ├── copy-exec-files.sh
│ ├── copy-n-exec-hermesKV.sh
│ ├── copy-n-exec-rCRAQ.sh
│ ├── copy-traces.sh
│ ├── csv_latency_parser.py
│ ├── exec-derecho.sh
│ ├── format.sh
│ ├── get-system-xput-files.sh
│ ├── setup.sh
│ └── trace-spliter.sh
├── exec/
│ ├── Makefile
│ ├── hosts.sh
│ ├── results/
│ │ ├── latency/
│ │ │ └── .gitinclude
│ │ └── xput/
│ │ ├── all-nodes/
│ │ │ └── .gitkeep
│ │ └── per-node/
│ │ └── .gitkeep
│ ├── run-hades.sh
│ ├── run-hermesKV.sh
│ ├── run-rCRAQ.sh
│ └── run.sh
├── include/
│ ├── hades/
│ │ └── hades.h
│ ├── hermes/
│ │ ├── config.h
│ │ ├── inline-util.h
│ │ ├── spacetime.h
│ │ └── util.h
│ ├── mica-herd/
│ │ ├── city.h
│ │ ├── hrd.h
│ │ ├── mica.h
│ │ └── sizes.h
│ ├── utils/
│ │ ├── bit_vector.h
│ │ ├── concur_ctrl.h
│ │ └── time_rdtsc.h
│ └── wings/
│ ├── wings.h
│ └── wings_api.h
├── src/
│ ├── CR/
│ │ ├── crKV.c
│ │ └── cr_worker.c
│ ├── hades/
│ │ ├── hades.c
│ │ └── test.c
│ ├── hermes/
│ │ ├── hermesKV.c
│ │ ├── hermes_worker.c
│ │ ├── main.c
│ │ ├── spacetime.c
│ │ ├── stats.c
│ │ └── util.c
│ ├── mica-herd/
│ │ ├── city.c
│ │ ├── herd.c
│ │ └── mica.c
│ └── wings/
│ └── wings.c
└── tla/
├── Hermes.tla
├── HermesRMWs.tla
└── README.md
SYMBOL INDEX (442 symbols across 28 files)
FILE: bin/csv_latency_parser.py
class LatencyParser (line 10) | class LatencyParser:
method __init__ (line 11) | def __init__(self):
method printStats (line 22) | def printStats(self, array, max_latency):
method printAllStats (line 36) | def printAllStats(self):
method avgLatency (line 45) | def avgLatency(self, array):
method percentileLatency (line 56) | def percentileLatency(self, array, percentage):
method parseInputStats (line 83) | def parseInputStats(self):
FILE: include/hades/hades.h
type hades_view_t (line 48) | typedef struct {
type hades_ctx_t (line 58) | typedef struct {
type hades_wings_ctx_t (line 88) | typedef struct {
function hades_ctx_init (line 98) | inline static void
function hades_wings_ctx_init (line 145) | inline static void
FILE: include/hermes/config.h
type hermes_qps_enum (line 173) | typedef enum {
type cr_qps_enum (line 198) | typedef enum {
type thread_params (line 228) | struct thread_params {
type latency_counters (line 232) | struct latency_counters {
type latency_counters (line 240) | struct latency_counters
FILE: include/hermes/inline-util.h
function node_is_in_membership (line 19) | static inline uint8_t
function group_membership_update (line 29) | static inline void
function group_membership_has_changed (line 51) | static inline uint8_t
function bookkeep_latency (line 84) | static inline void
function start_latency_measurement (line 111) | static inline void
function stop_latency_measurment (line 117) | static inline void
function stop_latency_of_completed_writes (line 129) | static inline void
function stop_latency_of_completed_reads (line 140) | static inline void
function refill_ops (line 154) | static inline int
FILE: include/hermes/spacetime.h
type hermes_states_t (line 42) | typedef enum {
type input_opcodes_t (line 51) | typedef enum {
type response_opcodes_t (line 65) | typedef enum {
type op_bucket_states_t (line 92) | typedef enum {
type fs_ops_t (line 106) | typedef enum {
type rcv_buff_types_t (line 113) | typedef enum {
type spacetime_key_t (line 125) | typedef struct {
type spacetime_object_meta (line 132) | typedef volatile struct {
type spacetime_op_meta_t (line 142) | typedef struct {
type spacetime_op_t (line 159) | typedef struct {
type spacetime_group_membership (line 177) | typedef struct {
type spacetime_kv (line 184) | struct spacetime_kv {
type spacetime_trace_command (line 189) | struct spacetime_trace_command {
type spacetime_kv (line 197) | struct spacetime_kv
type hermes_batch_type_t (line 203) | enum hermes_batch_type_t {
type hermes_batch_type_t (line 211) | enum hermes_batch_type_t
type cr_type_t (line 220) | enum cr_type_t {
type cr_type_t (line 228) | enum cr_type_t
function is_last_ack (line 235) | static inline uint8_t
function get_val_len (line 245) | static inline uint8_t
function set_val_len (line 251) | static inline uint8_t
type spacetime_kv (line 257) | struct spacetime_kv
FILE: include/hermes/util.h
type worker_stats (line 15) | struct worker_stats {
type stats (line 58) | struct stats {
function init_stats (line 90) | static inline void
type spacetime_trace_command (line 96) | struct spacetime_trace_command
type worker_stats (line 115) | struct worker_stats
FILE: include/mica-herd/city.h
type uint8 (line 54) | typedef uint8_t uint8;
type uint32 (line 55) | typedef uint32_t uint32;
type uint64 (line 56) | typedef uint64_t uint64;
type uint128 (line 58) | typedef struct _uint128 uint128;
type _uint128 (line 59) | struct _uint128 {
FILE: include/mica-herd/hrd.h
type hrd_qp_attr (line 96) | struct hrd_qp_attr {
type hrd_ud_ctrl_blk (line 114) | struct hrd_ud_ctrl_blk {
type hrd_ud_ctrl_blk (line 138) | struct hrd_ud_ctrl_blk
type hrd_ud_ctrl_blk (line 144) | struct hrd_ud_ctrl_blk
type ibv_device (line 147) | struct ibv_device
type hrd_ud_ctrl_blk (line 147) | struct hrd_ud_ctrl_blk
type ibv_context (line 150) | struct ibv_context
type hrd_ud_ctrl_blk (line 152) | struct hrd_ud_ctrl_blk
function hrd_poll_cq (line 155) | static inline uint32_t
type ibv_mr (line 181) | struct ibv_mr
type ibv_pd (line 182) | struct ibv_pd
type ibv_mr (line 186) | struct ibv_mr
type hrd_ud_ctrl_blk (line 200) | struct hrd_ud_ctrl_blk
type hrd_qp_attr (line 203) | struct hrd_qp_attr
function hrd_fastrand (line 206) | static inline uint32_t
type color_print_t (line 218) | typedef enum { YELLOW = 0, RED, GREEN, CYAN } color_print_t;
FILE: include/mica-herd/mica.h
type mica_resp (line 42) | struct mica_resp {
type mica_key (line 50) | struct mica_key {
type mica_op (line 57) | struct mica_op {
type mica_slot (line 64) | struct mica_slot {
type mica_bkt (line 70) | struct mica_bkt {
type mica_kv (line 74) | struct mica_kv {
type mica_kv (line 95) | struct mica_kv
type mica_kv (line 99) | struct mica_kv
type mica_op (line 99) | struct mica_op
type mica_resp (line 100) | struct mica_resp
type mica_op (line 106) | struct mica_op
FILE: include/utils/bit_vector.h
type bit_vector_t (line 37) | typedef struct {
type dbit_vector_t (line 41) | typedef struct {
function bv_bits_to_bytes (line 47) | static inline uint16_t
function bv_init_internal (line 57) | static inline void
function bv_bit_get_internal (line 64) | static inline uint8_t
function bv_bit_set_internal (line 74) | static inline void
function bv_bit_reset_internal (line 82) | static inline void
function bv_set_all_internal (line 90) | static inline void
function bv_reset_all_internal (line 103) | static inline void
function bv_are_equal_internal (line 110) | static inline uint8_t
function bv_copy_internal (line 131) | static inline void
function bv_no_setted_bits_internal (line 141) | static inline uint8_t
function bv_reverse_internal (line 152) | static inline void
function bv_and_internal (line 159) | static inline void
function bv_or_internal (line 170) | static inline void
function bv_print_internal (line 183) | static inline void
function bv_print_enhanced_internal (line 190) | static inline void
function dbv_init (line 201) | static inline void
function dbv_destroy (line 211) | static inline void
function dbv_bit_get (line 218) | static inline uint8_t
function dbv_bit_set (line 224) | static inline void
function dbv_bit_reset (line 230) | static inline void
function dbv_set_all (line 236) | static inline void
function dbv_reset_all (line 242) | static inline void
function dbv_no_setted_bits (line 248) | static inline uint8_t
function dbv_are_equal (line 254) | static inline uint8_t
function dbv_copy (line 261) | static inline void
function dbv_is_all_set (line 268) | static inline uint8_t
function dbv_reverse (line 278) | static inline void
function dbv_and (line 284) | static inline void
function dbv_or (line 291) | static inline void
function dbv_print (line 300) | static inline void
function dbv_print_enhanced (line 306) | static inline void
function dbv_unit_test (line 312) | static inline void
function bv_init (line 359) | static inline void
function bv_bit_get (line 365) | static inline uint8_t
function bv_bit_set (line 371) | static inline void
function bv_bit_reset (line 377) | static inline void
function bv_set_all (line 383) | static inline void
function bv_reset_all (line 389) | static inline void
function bv_no_setted_bits (line 395) | static inline uint8_t
function bv_are_equal (line 401) | static inline uint8_t
function bv_copy (line 408) | static inline void
function bv_reverse (line 416) | static inline void
function bv_and (line 422) | static inline void
function bv_or (line 429) | static inline void
function bv_print (line 438) | static inline void
function bv_print_enhanced (line 444) | static inline void
function bv_unit_test (line 453) | static inline void
FILE: include/utils/concur_ctrl.h
type timestamp_t (line 28) | typedef volatile struct {
type seqlock_t (line 33) | typedef struct {
type conc_ctrl_t (line 38) | typedef volatile struct {
function timestamp_init (line 47) | static inline void
function timestamp_is_equal (line 54) | static inline int
function timestamp_is_smaller (line 61) | static inline int
function seqlock_init (line 72) | static inline void
function seqlock_lock (line 79) | static inline int
function seqlock_unlock (line 99) | static inline void
function seqlock_version_is_same_and_valid (line 114) | static inline int
function cctrl_init (line 124) | static inline void
function cctrl_lock (line 131) | static inline int
function cctrl_unlock_custom_version (line 151) | static inline void
function cctrl_unlock_inc_version_by_three (line 164) | static inline void
function cctrl_unlock_inc_version (line 179) | static inline void
function cctrl_unlock_dec_version (line 193) | static inline void
function cctrl_timestamp_is_same_and_valid (line 209) | static inline int
FILE: include/utils/time_rdtsc.h
function RDTSC (line 15) | static inline uint64_t
type timespec (line 26) | struct timespec
type timespec (line 27) | struct timespec
type timespec (line 27) | struct timespec
type timespec (line 29) | struct timespec
function calibrate_ticks (line 39) | static void
function init_rdtsc (line 60) | static inline void
function get_timespec (line 71) | static inline void
function get_rdtsc_timespec (line 79) | static inline void
function time_elapsed_in_us (line 85) | static inline double
function time_elapsed_in_ms (line 94) | static inline double
function time_elapsed_in_sec (line 103) | static inline double
FILE: include/wings/wings.h
function _wings_assert_binary (line 18) | static inline void
function _wings_ud_recv_max_pkt_size (line 24) | static inline uint16_t
function _wings_ud_send_max_pkt_size (line 35) | static inline uint16_t
function _wings_assertions (line 46) | static inline void
function wings_ud_send_pkt_t (line 82) | static inline wings_ud_send_pkt_t*
function wings_ud_recv_pkt_t (line 91) | static inline wings_ud_recv_pkt_t*
function wings_ud_send_pkt_t (line 100) | static inline wings_ud_send_pkt_t*
function _wings_inc_send_push_ptr (line 109) | static inline void
function _wings_inc_recv_push_ptr (line 131) | static inline void
function _wings_inc_recv_pull_ptr (line 139) | static inline void
function _wings_post_hdr_only_recvs (line 150) | static inline void
function _wings_post_recvs (line 164) | static inline void
function _wings_poll_crds_and_post_recvs (line 219) | static inline void
function _wings_enque_to_overflown_msgs (line 276) | static inline void
function _wings_deque_from_overflown_msgs (line 292) | static inline uint16_t
function wings_poll_buff_and_post_recvs (line 322) | static inline uint16_t
function _wings_node_is_in_membership (line 456) | static inline uint8_t
function _wings_has_sufficient_crds_no_polling_membership (line 466) | static inline uint8_t
function _wings_has_sufficient_crds_no_polling (line 493) | static inline uint8_t
function _wings_has_sufficient_crds_membership (line 500) | static inline uint8_t
function _wings_has_sufficient_crds (line 518) | static inline uint8_t
function _wings_dec_crds_membership (line 531) | static inline void
function _wings_dec_crds (line 568) | static inline void
function wings_reset_credits (line 574) | static inline void
function _wings_forge_crd_wr (line 584) | static inline void
function _wings_forge_wr (line 624) | static inline void
function _wings_batch_pkts_2_NIC (line 714) | static inline void
function _wings_check_if_batch_n_inc_pkt_ptr (line 782) | static inline void
function wings_set_sender_id_n_msg_type (line 802) | static inline uint8_t
function _wings_get_sender_id_n_msg_type (line 812) | static inline uint8_t
function wings_issue_pkts (line 822) | static inline uint8_t
function wings_issue_credits (line 921) | static inline void
FILE: include/wings/wings_api.h
type qp_info_t (line 44) | typedef struct {
type wings_pkt_t (line 50) | typedef struct {
type wings_ud_recv_pkt_t (line 60) | typedef struct {
type wings_crd_t (line 66) | typedef struct {
type wings_hdr_only_t (line 72) | typedef struct {
type ud_channel_stats_t (line 80) | typedef struct {
type channel_type (line 93) | enum channel_type { REQ, RESP, CRD }
type ud_channel_t (line 95) | typedef struct _ud_channel_t {
function wings_NOP_modify_elem_after_send (line 188) | static inline void
type channel_type (line 202) | enum channel_type
FILE: src/CR/crKV.c
function head_id (line 14) | static inline uint8_t
function tail_id (line 20) | static inline uint8_t
function cr_assertions_inv (line 27) | static inline void
function cr_skip_op (line 38) | static inline uint8_t
function cr_skip_inv (line 48) | static inline uint8_t
function cr_skip_ack (line 54) | static inline uint8_t
function cr_skip_remote_reads (line 60) | static inline uint8_t
function cr_skip_remote_writes (line 66) | static inline uint8_t
function cr_exec_write (line 77) | static inline void
function cr_exec_remote_reads (line 116) | static inline void
function cr_exec_op (line 148) | static inline void
function cr_complete_local_write (line 200) | static inline void
function cr_exec_inv (line 216) | static inline void
function cr_exec_ack (line 299) | static inline void
function cr_skip_dispatcher (line 360) | static inline uint8_t
function cr_assertions_dispatcher (line 379) | static inline void
function cr_exec_dispatcher (line 395) | static inline void
function cr_batch_ops_to_KVS (line 422) | void
FILE: src/CR/cr_worker.c
function head_id (line 12) | static inline uint8_t
function tail_id (line 18) | static inline uint8_t
function next_node_in_chain (line 24) | static inline uint8_t
function prev_node_in_chain (line 30) | static inline uint8_t
function inv_skip_or_fwd_to_next_node (line 36) | int
function inv_fwd_modify_elem_after_send (line 45) | void
function inv_fwd_copy_and_modify_elem (line 59) | void
function inv_skip_or_get_sender_id (line 70) | int
function inv_modify_elem_after_send (line 86) | void
function inv_copy_and_modify_elem (line 97) | void
function remote_write_skip_or_get_sender_id (line 112) | int
function remote_write_copy_and_modify_elem (line 128) | void
function remote_write_head_skip_or_get_sender_id (line 145) | int
function remote_write_head_copy_and_modify_elem (line 163) | void
function remote_write_head_modify_elem_after_send (line 177) | void
function ack_fwd_modify_elem_after_send (line 188) | void
function ack_fwd_skip_or_get_sender_id (line 198) | int
function ack_fwd_copy_and_modify_elem (line 213) | void
function ack_skip_or_get_sender_id (line 223) | int
function ack_modify_elem_after_send (line 235) | void
function ack_copy_and_modify_elem (line 248) | void
function rem_write_crd_skip_or_get_sender_id (line 261) | int
function rem_write_crd_modify_elem_after_send (line 275) | void
function inv_crd_skip_or_get_sender_id (line 287) | int
function inv_crd_modify_elem_after_send (line 299) | void
function remote_read_skip_or_get_sender_id (line 308) | int
function remote_read_modify_elem_after_send (line 324) | void
function remote_read_copy_and_modify_elem (line 335) | void
function remote_read_resp_skip_or_get_sender_id (line 352) | int
function remote_read_resp_modify_elem_after_send (line 371) | void
function remote_read_resp_copy_and_modify_elem (line 384) | void
function print_ops_and_remote_write_ops (line 397) | void
function print_total_stalls_due_to_credits (line 415) | void
function print_total_send_recv_msgs_n_credits (line 434) | void
function cr_complete_local_reads (line 485) | static inline void
function get_first_free_slot (line 507) | static inline int
function cr_move_stalled_writes_to_top_n_return_free_space (line 518) | static inline uint16_t
function debugg (line 568) | static inline void
type thread_params (line 599) | struct thread_params
type thread_params (line 599) | struct thread_params
type spacetime_trace_command (line 751) | struct spacetime_trace_command
type timespec (line 776) | struct timespec
FILE: src/hades/hades.c
type hades_view_wrapper_w_dst_id_t (line 8) | typedef struct {
function hades_skip_or_get_dst_id (line 13) | int
function hades_copy_and_modify_elem (line 19) | void
function hades_crd_skip_or_get_sender_id (line 29) | int
function print_send_hbt (line 36) | static inline void
function print_recved_hbts (line 47) | static inline void
function majority_of_nodes (line 60) | static inline uint8_t
function check_if_majority_is_rechable (line 67) | static inline void
function skip_to_apply_fake_link_failure (line 86) | static inline uint8_t
function is_in_membership (line 118) | static inline uint8_t
function skip_arbitration (line 125) | static inline uint8_t
function view_arbitration_via_ostracism (line 151) | static inline void
function get_max_received_epoch_id (line 188) | static inline uint8_t
function update_view_n_membership (line 199) | static inline void
function issue_heartbeats (line 262) | static inline void
function update_view_and_issue_hbs (line 293) | void
function poll_for_remote_views (line 302) | uint16_t
FILE: src/hades/test.c
function main (line 8) | int
FILE: src/hermes/hermesKV.c
function hermes_assertions_begin_inv (line 14) | static inline void
function hermes_assertions_begin_ack (line 32) | static inline void
function hermes_assertions_begin_val (line 55) | static inline void
function hermes_assertions_end_read_write_ops (line 71) | static inline void
function hermes_lock_free_read_obj_meta (line 95) | static inline __attribute__((always_inline)) void
function hermes_update_actions_n_unlock (line 114) | static inline void
function hermes_local_state_to_op (line 159) | static inline void
function hermes_write_replay_actions (line 172) | static inline void
function hermes_check_membership_n_write_replay_actions (line 194) | static inline void
function hermes_marshal_write_coalesce_optimization (line 212) | static inline void
function hermes_complete_coalesced_write (line 226) | static inline void
function hermes_complete_hot_read_optimization (line 241) | static inline void
function hermes_read_actions (line 261) | static inline void
function hermes_exec_read (line 271) | static inline void
function hermes_exec_write (line 335) | static inline void
function hermes_exec_rmw (line 380) | static inline void
function hermes_exec_check_update_completion (line 451) | static inline void
function hermes_exec_inv (line 517) | static inline void
function hermes_exec_ack (line 630) | static inline void
function hermes_exec_val (line 721) | static inline void
function hermes_skip_op (line 751) | static inline uint8_t
function hermes_skip_op_after_membship_change (line 767) | static inline uint8_t
function hermes_skip_inv (line 777) | static inline uint8_t
function hermes_skip_ack (line 789) | static inline uint8_t
function hermes_skip_dispatcher (line 797) | static inline uint8_t
function hermes_assertions_begin_dispatcher (line 817) | static inline void
function hermes_print_dispatcher (line 851) | static inline void
function hermes_assertions_end_dispatcher (line 879) | static inline void
function hermes_exec_dispatcher (line 898) | static inline void
function hermes_batch_ops_to_KVS (line 959) | void
FILE: src/hermes/hermes_worker.c
function inv_skip_or_get_sender_id (line 12) | int
function inv_modify_elem_after_send (line 31) | void
function inv_copy_and_modify_elem (line 53) | void
function ack_skip_or_get_sender_id (line 69) | int
function ack_modify_elem_after_send (line 87) | void
function ack_copy_and_modify_elem (line 101) | void
function val_skip_or_get_sender_id (line 124) | int
function val_modify_elem_after_send (line 140) | void
function val_copy_and_modify_elem (line 150) | void
function memb_change_skip_or_get_sender_id (line 161) | int
function memb_change_modify_elem_after_send (line 173) | void
function memb_change_copy_and_modify_elem (line 192) | void
function rem_write_crd_skip_or_get_sender_id (line 203) | int
function rem_write_crd_modify_elem_after_send (line 214) | void
function print_total_send_recv_msgs (line 226) | void
function spin_until_all_nodes_are_in_membership (line 240) | void
function failure_detection_n_membership (line 259) | static inline void
type thread_params (line 298) | struct thread_params
type thread_params (line 298) | struct thread_params
type spacetime_trace_command (line 409) | struct spacetime_trace_command
type timespec (line 437) | struct timespec
type timespec (line 454) | struct timespec
FILE: src/hermes/main.c
type latency_counters (line 16) | struct latency_counters
type worker_stats (line 17) | struct worker_stats
function group_membership_init (line 36) | void
function main (line 52) | int
FILE: src/hermes/spacetime.c
type spacetime_kv (line 15) | struct spacetime_kv
function spacetime_object_meta_init (line 17) | void
function spacetime_init (line 26) | void
function spacetime_populate_fixed_len (line 35) | void
FILE: src/hermes/stats.c
function xput_file_name (line 3) | static inline void
function dump_xput_stats (line 15) | void
function dump_latency_stats (line 39) | void
function safe_division (line 75) | static inline double
type worker_stats (line 96) | struct worker_stats
type stats (line 98) | struct stats
type worker_stats (line 101) | struct worker_stats
type timespec (line 102) | struct timespec
type worker_stats (line 111) | struct worker_stats
type worker_stats (line 164) | struct worker_stats
FILE: src/hermes/util.c
function spawn_stats_thread (line 11) | int
function is_state_code (line 33) | uint8_t
function is_input_code (line 49) | uint8_t
function is_response_code (line 69) | uint8_t
function is_bucket_state_code (line 99) | uint8_t
function create_uni_trace (line 234) | void
function parse_trace (line 288) | int
function trace_init (line 410) | void
function setup_kvs_buffs (line 442) | void
FILE: src/mica-herd/city.c
function uint64 (line 38) | static uint64
function uint32 (line 46) | static uint32
function uint64 (line 89) | static uint64
function uint32 (line 95) | static uint32
function uint64 (line 109) | static inline uint64
function uint64 (line 124) | static uint64
function uint64 (line 134) | static uint64
function uint64 (line 140) | static uint64
function uint64 (line 146) | static uint64
function uint64 (line 155) | static uint64
function uint64 (line 180) | static uint64
function uint128 (line 194) | uint128
function uint128 (line 213) | uint128
function uint64 (line 221) | static uint64
function uint64 (line 246) | uint64
function uint64 (line 289) | uint64
function uint64 (line 295) | uint64
function uint128 (line 303) | static uint128
function uint128 (line 339) | uint128
function uint128 (line 409) | uint128
function CityHashCrc256Long (line 436) | static void
function CityHashCrc256Short (line 509) | static void
function CityHashCrc256 (line 518) | void
function uint128 (line 528) | uint128
function uint128 (line 545) | uint128
FILE: src/mica-herd/herd.c
type ibv_device (line 13) | struct ibv_device
type hrd_ud_ctrl_blk (line 14) | struct hrd_ud_ctrl_blk
type ibv_device (line 16) | struct ibv_device
type ibv_context (line 33) | struct ibv_context
type ibv_device_attr (line 36) | struct ibv_device_attr
type ibv_port_attr (line 46) | struct ibv_port_attr
function hrd_free (line 195) | int
function hrd_get_local_lid (line 235) | uint16_t
function memcached_st (line 263) | memcached_st*
function hrd_publish (line 286) | void
function hrd_get_published (line 318) | int
type hrd_ud_ctrl_blk (line 356) | struct hrd_ud_ctrl_blk
type hrd_ud_ctrl_blk (line 387) | struct hrd_ud_ctrl_blk
type hrd_ud_ctrl_blk (line 388) | struct hrd_ud_ctrl_blk
type hrd_ud_ctrl_blk (line 388) | struct hrd_ud_ctrl_blk
type hrd_ud_ctrl_blk (line 389) | struct hrd_ud_ctrl_blk
type ibv_device (line 402) | struct ibv_device
type ibv_qp (line 421) | struct ibv_qp
type ibv_qp (line 421) | struct ibv_qp
type ibv_cq (line 423) | struct ibv_cq
type ibv_cq (line 423) | struct ibv_cq
type ibv_cq (line 425) | struct ibv_cq
type ibv_cq (line 425) | struct ibv_cq
function hrd_ud_ctrl_blk_destroy (line 462) | int
function hrd_create_dgram_qps (line 516) | void
function hrd_publish_dgram_qp (line 595) | void
type hrd_qp_attr (line 634) | struct hrd_qp_attr
type hrd_qp_attr (line 637) | struct hrd_qp_attr
type hrd_qp_attr (line 656) | struct hrd_qp_attr
function colored_printf (line 666) | void
FILE: src/mica-herd/mica.c
function is_power_of_2 (line 4) | int
function mica_init (line 16) | void
function mica_insert_one (line 81) | void
function uint128 (line 153) | uint128*
FILE: src/wings/wings.c
type hrd_ud_ctrl_blk (line 29) | struct hrd_ud_ctrl_blk
type hrd_ud_ctrl_blk (line 31) | struct hrd_ud_ctrl_blk
type hrd_ud_ctrl_blk (line 33) | struct hrd_ud_ctrl_blk
type hrd_ud_ctrl_blk (line 49) | struct hrd_ud_ctrl_blk
function wings_ud_channel_destroy (line 51) | void
function wings_ud_channel_init (line 67) | void
function wings_setup_channel_qps_and_recvs_w_shm_key (line 242) | void
function wings_setup_channel_qps_and_recvs (line 300) | void
function wings_print_ud_c_overview (line 309) | void
function _wings_print_on_off_toggle (line 356) | void
function _wings_ud_channel_crd_init (line 366) | void
function _wings_ud_channel_init_recv (line 457) | void
function _wings_setup_crd_wr_and_sgl (line 486) | void
function _wings_setup_send_wr_and_sgl (line 524) | void
function _wings_setup_recv_wr_and_sgl (line 626) | void
function _wings_setup_incoming_buff_and_post_initial_recvs (line 655) | void
function _wings_simple_hash (line 681) | unsigned long
function _wings_get_remote_qp (line 692) | void
function _wings_get_remote_qps (line 737) | void
function _wings_share_qp_info_via_memcached (line 753) | void
function wings_reconfigure_wrs_ah (line 785) | void
Condensed preview — 56 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (440K chars).
[
{
"path": ".clang-format",
"chars": 809,
"preview": "---\nBasedOnStyle: Chromium\nAlignAfterOpenBracket: Align\nAlignConsecutiveDeclarations: 'false'\nAlignEscapedNewlines: Lef"
},
{
"path": ".gitignore",
"chars": 1245,
"preview": "# ignore temporary files\n.*.swp\n\\#*#\n*.pyc\n*.o\n*.hi\n*.dump\n*.log\n*.rej\n*.orig\n*.patch\n*.diff\n.tags*\n\n# ignore executable"
},
{
"path": "AUTHORS",
"chars": 202,
"preview": "Run `git shortlog -se` for an up-to-date list of contributors.\n---\n\nPrincipal authors: Antonios Katsarakis <antonios.ka"
},
{
"path": "CMakeLists.txt",
"chars": 2545,
"preview": "######################################################################################\n# WARNING: DO NOT MAKE through cm"
},
{
"path": "LICENSE",
"chars": 11358,
"preview": "\n Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README.md",
"chars": 5767,
"preview": "# Hermes Reliable Replication Protocol\n\n<img align=\"left\" height=\"160\" src=\"https://github.com/akatsarakis/Hermes/blob/m"
},
{
"path": "bin/copy-exec-files.sh",
"chars": 499,
"preview": "#!/usr/bin/env bash\n\nFILES_TO_CPY=(\n \"hosts.sh\"\n \"run.sh\"\n \"run-hermesKV.sh\"\n \"hermesKV\"\n "
},
{
"path": "bin/copy-n-exec-hermesKV.sh",
"chars": 1747,
"preview": "#!/usr/bin/env bash\n\n### Runs to make\n#declare -a write_ratios=(0 10 50 200 500 1000)\ndeclare -a write_ratios=(1000)\ndec"
},
{
"path": "bin/copy-n-exec-rCRAQ.sh",
"chars": 2362,
"preview": "#!/usr/bin/env bash\n\nUSE_SAME_BATCH_N_CREDITS=0\n\n### Runs to make\ndeclare -a write_ratios=(1000)\n#declare -a num_workers"
},
{
"path": "bin/copy-traces.sh",
"chars": 421,
"preview": "#!/usr/bin/env bash\n\n# Copy (per-thread splitted) trace folder\nFOLDERS_TO_CPY=( \"traces/current-splitted-traces\" )\nHOME_"
},
{
"path": "bin/csv_latency_parser.py",
"chars": 4248,
"preview": "#!/usr/bin/python\n\nimport sys, os, ntpath, getopt\n\n\"\"\"\n========\nParser for aggregated over time results\n========\n\"\"\"\ncla"
},
{
"path": "bin/exec-derecho.sh",
"chars": 2061,
"preview": "#!/usr/bin/env bash\nHOSTS=( ##### network cluster #####\n \"houston\"\n \"sanantonio\"\n \"austin\"\n "
},
{
"path": "bin/format.sh",
"chars": 800,
"preview": "#!/bin/bash\n\nSCRIPT_DIR=\"$(dirname \"$0\")\"\ncd \"${SCRIPT_DIR}\"\n\nFORMAT_FILES_IN_DIRECTORIES=\"../src/ ../include/\"\n\nclang-f"
},
{
"path": "bin/get-system-xput-files.sh",
"chars": 1179,
"preview": "#!/usr/bin/env bash\n\nEXEC_FOLDER=\"${HOME}/hermes/exec\"\nRESULTS_FOLDER=\"${HOME}/hermes/exec/results\"\n\nRESULT_FOLDER=\"${RE"
},
{
"path": "bin/setup.sh",
"chars": 1732,
"preview": "#!/usr/bin/env bash\n# Exec this script in every cluster node after you have\n# installed the (Infiniband) Verbs drivers t"
},
{
"path": "bin/trace-spliter.sh",
"chars": 663,
"preview": "#!/usr/bin/env bash\n\nINPUT_DIR=\"${HOME}/hermes/traces/system-traces/\"\nINPUT_FILENAME=\"simple_trace_w_100000000_k_1000000"
},
{
"path": "exec/Makefile",
"chars": 1434,
"preview": "CPPFLAGS := -O3 #-Wno-unused-result -Wall -Werror\nLD := gcc -O3 -flto\nLDFLAGS := ${LDFLAGS} -libverbs -lrt -lpthre"
},
{
"path": "exec/hosts.sh",
"chars": 1187,
"preview": "#!/usr/bin/env bash\n\n\nALL_IPS=(\n### TO BE FILLED: Please provide all cluster IPs\n # Node w/ first IP (i.e., \"manager\""
},
{
"path": "exec/results/latency/.gitinclude",
"chars": 0,
"preview": ""
},
{
"path": "exec/results/xput/all-nodes/.gitkeep",
"chars": 0,
"preview": ""
},
{
"path": "exec/results/xput/per-node/.gitkeep",
"chars": 0,
"preview": ""
},
{
"path": "exec/run-hades.sh",
"chars": 223,
"preview": "#!/usr/bin/env bash\n\nsource run.sh\n\nblue \"Running hades\"\n\nsudo LD_LIBRARY_PATH=/usr/local/lib/ -E \\\n\t./hades "
},
{
"path": "exec/run-hermesKV.sh",
"chars": 3114,
"preview": "#!/usr/bin/env bash\n\nsource run.sh\n\n#### Get CLI arguments\n# Use -1 for the default (#define in config.h) values if not "
},
{
"path": "exec/run-rCRAQ.sh",
"chars": 1808,
"preview": "#!/usr/bin/env bash\n\nsource run.sh\n\n\n#### Get CLI arguments\n# Use -1 for the default (#define in config.h) values if not"
},
{
"path": "exec/run.sh",
"chars": 799,
"preview": "#!/usr/bin/env bash\n\nsource ./hosts.sh\n\nexport HRD_REGISTRY_IP=\"${ALL_IPS[0]}\" # I.E. first IP node (HOUSTON) has a memc"
},
{
"path": "include/hades/hades.h",
"chars": 6136,
"preview": "//\n// Created by akatsarakis on 17/01/19.\n//\n\n#ifndef HADES_H\n#define HADES_H\n\n#include \"../../include/wings/wings.h\"\n#i"
},
{
"path": "include/hermes/config.h",
"chars": 8587,
"preview": "//\n// Created by akatsarakis on 15/03/18.\n//\n\n#ifndef SPACETIME_CONFIG_H\n#define SPACETIME_CONFIG_H\n#include <assert.h>\n"
},
{
"path": "include/hermes/inline-util.h",
"chars": 12569,
"preview": "//\n// Created by akatsarakis on 23/05/18.\n//\n\n#ifndef HERMES_INLINE_UTIL_H\n#define HERMES_INLINE_UTIL_H\n\n#include <infin"
},
{
"path": "include/hermes/spacetime.h",
"chars": 7436,
"preview": "//\n// Created by akatsarakis on 04/05/18.\n//\n\n#ifndef HERMES_SPACETIME_H\n#define HERMES_SPACETIME_H\n\n// Optik Options\n#i"
},
{
"path": "include/hermes/util.h",
"chars": 3713,
"preview": "//\n// Created by akatsarakis on 15/03/18.\n//\n\n#ifndef HERMES_UTIL_H\n#define HERMES_UTIL_H\n\n#include <stdint.h>\n#include "
},
{
"path": "include/mica-herd/city.h",
"chars": 3341,
"preview": "// city.h - cityhash-c\n// CityHash on C\n// Copyright (c) 2011-2012, Alexander Nusov\n//\n// - original copyright notice -\n"
},
{
"path": "include/mica-herd/hrd.h",
"chars": 6511,
"preview": "#ifndef HRD_H\n#define HRD_H\n\n#include <assert.h>\n#include <errno.h>\n#include <numaif.h>\n#include <stdarg.h>\n#include <st"
},
{
"path": "include/mica-herd/mica.h",
"chars": 2798,
"preview": "#ifndef MICA_H\n#define MICA_H\n\n#include <stdint.h>\n#include \"city.h\"\n#include \"hrd.h\"\n\n/*\n * The polling logic in HERD r"
},
{
"path": "include/mica-herd/sizes.h",
"chars": 660,
"preview": "#define K_32 32768\n\n#define K_64 65536\n\n#define K_128 131072\n#define K_128_ 131071\n\n#define K_256 262144\n#define K_256_ "
},
{
"path": "include/utils/bit_vector.h",
"chars": 12613,
"preview": "//\n// Created by akatsarakis on 11/12/18.\n//\n\n#ifndef HERMES_BIT_VECTOR_H\n#define HERMES_BIT_VECTOR_H\n\n#include <assert."
},
{
"path": "include/utils/concur_ctrl.h",
"chars": 5427,
"preview": "//\n// Created by akatsarakis on 11/12/18.\n//\n\n#ifndef HERMES_SEQLOCK_H\n#define HERMES_SEQLOCK_H\n\n#include <assert.h>\n#in"
},
{
"path": "include/utils/time_rdtsc.h",
"chars": 2945,
"preview": "\n#ifndef HERMES_TIME_H\n#define HERMES_TIME_H\n#include <assert.h>\n#include <stdint.h> /* for uint64_t */\n#include <stdio."
},
{
"path": "include/wings/wings.h",
"chars": 35485,
"preview": "//\n// Created by akatsarakis on 06/02/19.\n//\n\n#ifndef WINGS_INTERNAL_INLINES_H\n#define WINGS_INTERNAL_INLINES_H\n\n#includ"
},
{
"path": "include/wings/wings_api.h",
"chars": 8476,
"preview": "//\n// Created by akatsarakis on 06/02/19.\n//\n\n#ifndef WINGS_API_H\n#define WINGS_API_H\n#include \"../utils/bit_vector.h\"\n#"
},
{
"path": "src/CR/crKV.c",
"chars": 17306,
"preview": "//\n// Created by akatsarakis on 07/03/19.\n//\n\n#include <spacetime.h>\n#include <util.h>\n\n////////////////////////////////"
},
{
"path": "src/CR/cr_worker.c",
"chars": 35609,
"preview": "#include <spacetime.h>\n#include <time.h>\n#include \"../../include/utils/concur_ctrl.h\"\n#include \"inline-util.h\"\n#include "
},
{
"path": "src/hades/hades.c",
"chars": 14066,
"preview": "//\n// Created by akatsarakis on 12/02/19.\n//\n\n#include \"../../include/hades/hades.h\"\n#include <getopt.h>\n\ntypedef struct"
},
{
"path": "src/hades/test.c",
"chars": 754,
"preview": "//\n// Created by akatsarakis on 21/05/19.\n//\n\n#include <getopt.h>\n#include \"../../include/hades/hades.h\"\n\nint\nmain(int a"
},
{
"path": "src/hermes/hermesKV.c",
"chars": 39399,
"preview": "//\n// Created by akatsarakis on 07/03/19.\n//\n\n#include <inline-util.h>\n#include <spacetime.h>\n\n/////////////////////////"
},
{
"path": "src/hermes/hermes_worker.c",
"chars": 21447,
"preview": "#include <spacetime.h>\n#include <time.h>\n#include \"../../include/utils/concur_ctrl.h\"\n#include \"inline-util.h\"\n#include "
},
{
"path": "src/hermes/main.c",
"chars": 9248,
"preview": "#define _GNU_SOURCE\n#include <getopt.h>\n#include <infiniband/verbs.h>\n#include <malloc.h>\n#include <pthread.h>\n#include "
},
{
"path": "src/hermes/spacetime.c",
"chars": 2416,
"preview": "//\n// Created by akatsarakis on 04/05/18.\n//\n#include <config.h>\n#include <inline-util.h>\n#include <spacetime.h>\n#includ"
},
{
"path": "src/hermes/stats.c",
"chars": 10141,
"preview": "#include \"util.h\"\n\nstatic inline void\nxput_file_name(char* filename)\n{\n char* path = \"./results/xput/per-node\";\n\n spri"
},
{
"path": "src/hermes/util.c",
"chars": 14237,
"preview": "//\n// Created by akatsarakis on 15/03/18.\n//\n#define _GNU_SOURCE\n\n#include \"util.h\"\n#include \"hrd.h\"\n#include \"inline-ut"
},
{
"path": "src/mica-herd/city.c",
"chars": 15795,
"preview": "// city.c - cityhash-c\n// CityHash on C\n// Copyright (c) 2011-2012, Alexander Nusov\n//\n// - original copyright notice -\n"
},
{
"path": "src/mica-herd/herd.c",
"chars": 21629,
"preview": "#include \"hrd.h\"\n\n/* Every thread creates a TCP connection to the registry only once. */\n__thread memcached_st* memc = N"
},
{
"path": "src/mica-herd/mica.c",
"chars": 5320,
"preview": "#include \"mica.h\"\n#include \"hrd.h\"\n\nint\nis_power_of_2(int x)\n{\n return (x == 1 || x == 2 || x == 4 || x == 8 || x == 16"
},
{
"path": "src/wings/wings.c",
"chars": 29359,
"preview": "//\n// Created by akatsarakis on 22/01/19.\n//\n\n#include \"../../include/wings/wings.h\"\n#include <config.h>\n#include <infin"
},
{
"path": "tla/Hermes.tla",
"chars": 11726,
"preview": "------------------------------- MODULE Hermes -------------------------------\nEXTENDS Integers,\n FiniteSe"
},
{
"path": "tla/HermesRMWs.tla",
"chars": 10786,
"preview": "------------------------------- MODULE HermesRMWs -------------------------------\nEXTENDS Hermes\n \nVARIAB"
},
{
"path": "tla/README.md",
"chars": 283,
"preview": "# Hermes-Protocol\nTLA spec - Hermes: fault-tolerant replication protocol with strong consistency and high performance\n\n-"
}
]
About this extraction
This page contains the full source code of the ease-lab/Hermes GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 56 files (412.5 KB), approximately 118.0k tokens, and a symbol index with 442 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.