Showing preview only (587K chars total). Download the full file or copy to clipboard to get everything.
Repository: SyntheticSemantics/ems
Branch: master
Commit: 62e999decd40
Files: 69
Total size: 561.9 KB
Directory structure:
gitextract__ryzapkl/
├── .gitignore
├── .travis.yml
├── Docs/
│ ├── docs.css
│ ├── index.html
│ └── reference.html
├── Examples/
│ ├── Interlanguage/
│ │ ├── README.md
│ │ ├── interlanguage.js
│ │ └── interlanguage.py
│ ├── KeyValueStore/
│ │ ├── README.md
│ │ └── kv_store.js
│ ├── README.md
│ ├── STREAMS/
│ │ ├── README.md
│ │ ├── streams_bulk_sync_parallel.js
│ │ └── streams_fork_join.js
│ ├── WebServer/
│ │ ├── README.md
│ │ └── webServer.js
│ ├── concurrent_Q_and_TM.js
│ ├── wordCount.js
│ └── workQ_and_TM.js
├── LICENSE
├── Makefile
├── Python/
│ ├── README.md
│ ├── __init__.py
│ ├── ems.py
│ └── setup.py
├── README.md
├── Tests/
│ ├── 3dSpace.js
│ ├── CASdataFlow.js
│ ├── ES6/
│ │ └── harmony_proxies.js
│ ├── accum_omp.c
│ ├── barrier.js
│ ├── check_image_files.sh
│ ├── fj_args.js
│ ├── fj_args.py
│ ├── fork_join.js
│ ├── fullArrays.js
│ ├── issue11_bsp.js
│ ├── issue11_fj.js
│ ├── loopScheduling.js
│ ├── mapped_test.js
│ ├── py_api.py
│ ├── readers-writer.js
│ ├── refactoring.js
│ ├── stack_and_queue.js
│ ├── strcpy.js
│ ├── stringTags.js
│ ├── testUtils.js
│ ├── test_alloc.c
│ ├── tm.js
│ ├── tm_noq.js
│ ├── tm_noq_strings.js
│ └── v8Types.js
├── Vagrantfile
├── binding.gyp
├── index.html
├── nodejs/
│ ├── ems.js
│ ├── nodejs.cc
│ └── nodejs.h
├── package.json
└── src/
├── collectives.cc
├── ems.cc
├── ems.h
├── ems_alloc.cc
├── ems_alloc.h
├── ems_proto.h
├── ems_types.h
├── loops.cc
├── primitives.cc
└── rmw.cc
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
**/#*#
**~
build/
node_modules/
.vagrant
.local
*.o
package-lock.json
================================================
FILE: .travis.yml
================================================
language: node_js
os:
- linux
dist: trusty
branches:
only:
- gh-pages
- /.*/
node_js:
- "14.7"
- "12.18"
- "10.13" # -- NAPI Introduced
# - "8.1"
# - "7.1"
# - "6.1"
# - "5.1"
# - "4.1" -- NAN based
# - "0.10" -- Last native interface before NAN
cache:
npm: false
before_script:
- sudo -H apt-get install libffi-dev
- curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
- npm install bindings file-uri-to-path node-addon-api
- npm link ../ems
- sudo apt-get install python-dev python3-dev python3-pip
- sudo apt-get install -y build-essential
- sudo python2 -m pip install --upgrade pip
- sudo pip install cffi
- sudo python3 -m pip install cffi
script:
- make test
env:
- CXX=g++-4.8
sudo: true
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- g++-4.8
================================================
FILE: Docs/docs.css
================================================
/*-----------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.6.1 |
| http://mogill.com/ jace@mogill.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2011-2014, Synthetic Semantics LLC. All rights reserved. |
| Copyright (c) 2015-2020, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-----------------------------------------------------------------------------*/
@font-face {
font-family: 'Gotthard';
font-style: normal;
src: local('Gotthard'), url(./Fonts/gotthard.ttf) format('truetype');
}
@font-face {
font-family: 'Alegreya SC';
font-style: normal;
font-weight: 400;
src: local('Alegreya SC'), local('AlegreyaSC-Regular'), url(./Fonts/AlegreyaSC-Regular.woff) format('woff');
}
@font-face {
font-family: 'Gafata';
font-style: normal;
font-weight: 400;
src: local('Gafata'), local('Gafata-Regular'), url(./Fonts/Gafata-Regular.woff) format('woff');
}
body {
/*
background: #1c1e20;
background: -moz-radial-gradient(center, circle cover, #555a5f 0%, #1c1e20 100%);
background: -webkit-gradient(radial, center center, 0px, center center, 100%, color-stop(0%, #555a5f), color-stop(100%, #1c1e20));
background: -webkit-radial-gradient(center, circle cover, #555a5f 0%, #1c1e20 100%);
background: -o-radial-gradient(center, circle cover, #555a5f 0%, #1c1e20 100%);
background: -ms-radial-gradient(center, circle cover, #555a5f 0%, #1c1e20 100%);
background: radial-gradient(center, circle cover, #555a5f 0%, #1c1e20 100%);
background-color: #2b2b2b;
font-size: 0.15em;
*/
background-image:url('fabric_of_squares_gray_@2X.png');
font-family: "Gafata";
}
h1 {
font-size: 2.5em;
line-height: 1.0em;
color: #404080;
font-family: "Alegreya SC";
text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.3);
background-color:rgba(0,0,0,.2);
box-shadow: 6px 6px 6px rgba(0,0,0,.7);
margin-right: 1%; margin-left: 1%;
margin-top: 5px; margin-bottom:5px;
/*
border-left-width:10px; border-left-style:solid;
border-right-width:10px; border-right-style:solid;
*/
padding-left:2%; padding-right:2%;
padding-top:0px; padding-bottom:0px;
border-top-width:2px; border-top-style:solid;
border-bottom-width:2px; border-bottom-style:solid;
}
h2 {
font-size: 2.0em;
line-height: 1.0em;
color: #404080;
font-family: "Alegreya SC";
text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.3);
background-color:rgba(0,0,0,.25);
box-shadow: 4px 4px 4px rgba(0,0,0,.5);
margin-right: 1.1%; margin-left: 1.1%;
margin-top: 5px; margin-bottom:5px;
/*
border-left-width:7px; border-left-style:solid;
border-right-width:7px; border-right-style:solid;
*/
border-top-width:2px; border-top-style:solid;
border-bottom-width:2px; border-bottom-style:solid;
padding-left:2%; padding-right:2%;
padding-top:0px; padding-bottom:0px;
}
h3 {
font-size: 1.75em;
line-height: 1.0em;
color: #404080;
font-family: "Alegreya SC";
text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.2);
background-color:rgba(0,0,0,.2);
box-shadow: 2px 2px 2px rgba(0,0,0,.3);
margin-right: 1.2%; margin-left: 1.2%;
margin-top: 2px; margin-bottom:2px;
/*
border-left-width:4px; border-left-style:solid;
border-right-width:4px; border-right-style:solid;
*/
border-top-width:2px; border-top-style:solid;
border-bottom-width:2px; border-bottom-style:solid;
padding-left:2%; padding-right:2%;
padding-top:0px; padding-bottom:0px;
}
h4 {
font-size: 1.5em;
line-height: 1.0em;
color: #404080;
font-family: "Alegreya SC";
text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.3);
background-color:rgba(0,0,0,.1);
box-shadow: 2px 2px 2px rgba(0,0,0,.3);
margin-right: 1.3%; margin-left: 1.3%;
margin-top: 2px; margin-bottom:2px;
border-left-width:2px; border-left-style:solid;
border-right-width:2px; border-right-style:solid;
padding-left:2%; padding-right:2%;
padding-top:0px; padding-bottom:0px;
}
h5 {
font-size: 1.5em;
line-height: 1.0em;
color: #404080;
text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.3);
background-color:rgba(0,0,0,.1);
margin-left: 5.0%;
margin-right: 5%;
margin-top: 20px; margin-bottom:2px;
border-top-width:2px; border-top-style:solid;
border-bottom-width:2px; border-bottom-style:solid;
/* padding-left:2%; padding-right:2%;*/
padding-top:0px; padding-bottom:2px;
}
h6 {
font-size: 1.25em;
line-height: 1.0em;
color: #404080;
text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.3);
margin-right: 5%;
margin-left: 5%;
margin-top: 2px;
margin-bottom:2px;
/* border-top-width:2px; border-top-style:solid; */
border-bottom-width:2px; border-bottom-style:solid;
/* padding-left:2%; padding-right:2%;*/
padding-top:0px; padding-bottom:2px;
}
a:link {color:#404040; text-decoration:none;} /* unvisited link */
a:visited {color:#404040; text-decoration:none;} /* visited link */
a:hover {color:#c02020; background-color: rgba(200,0,0,0.1); text-decoration:none;} /* mouse over link */
a:active {color:#0000FF; text-decoration:none;} /* selected link */
p {
/*
margin-top: 5%;
margin-bottom: 5%;
text-indent: 1.5em;
*/
margin-right: 5%;
margin-left: 5%;
}
/*
p + p {
text-indent: 1.5em;
margin-right: 5%;
margin-left: 5%;
}
*/
.pmulti {
text-indent: 1.5em;
text-align:justify;
text-justify:inter-word;
-webkit-column-count:2; /* Chrome, Safari, Opera */
-moz-column-count:2; /* Firefox */
column-count:2;
-webkit-column-gap:40px; /* Chrome, Safari, Opera */
-moz-column-gap:40px; /* Firefox */
column-gap:40px;
-webkit-column-rule:3px outset rgba(0, 0, 0, 0.1); /* Chrome, Safari, Opera */
-moz-column-rule:3px outset rgba(0, 0, 0, 0.1); /* Firefox */
column-rule:3px outset rgba(0, 0, 0, 0.1);
}
.figcaption {
font-size: 0.85em;
font-style:italic;
text-align:left;
text-justify:none;
/* color: rgba(0,0,0,0.7); */
}
TODO {
color: #d03030;
background-color: rgba(200,0,0,0.1);
}
/* --------- Code Inline -------------------- */
*#xfoo {
font-family:"Monaco";
font-size: 1.95em;
background-color: rgba(200,0,0,0.5);
/* color: rgba(200,0,0,0.5); */
color: #ff0000 ! important;
}
.apiBlock {
margin-left: 6%;
margin-right: 6%;
/* border-left-width:2px; border-left-style:solid;*/
/* border-right-width:2px; border-right-style:solid;*/
}
.apiBlock .Label {
/*
margin-left: 200px;
padding-left: 2.5%;
padding-right: 2.5%;
margin-right: 20px;
padding-bottom: 20px;
*/
font-family:"Monaco";
font-size: 1.0em;
width: 120px;
color: #7070a0;
vertical-align:text-top;
}
.apiBlock .apiFunc .Proto {
/*
margin-left: 0%;
margin-right: 0%;
padding-bottom: 20px;
*/
font-family:"Monaco";
font-size: 0.85em;
vertical-align:text-top;
}
/* --------- Synopsis -------------------- */
.apiBlock .apiSynopsis {
margin-right: 5%; margin-left: 5%;
}
/* --------- Arguments -------------------- */
.apiBlock .apiArgs {
/*
margin-left: 5%; margin-right: 5%;
color: #00ff00;
margin-top: 0px; margin-bottom:0px;
*/
}
.apiBlock .apiArgs .argName {
/* background-color:rgba(100, 0, 0, 0.3); */
font-family:"Monaco";
font-size: 0.85em;
min-width:100px;
}
.apiBlock .apiArgs .argType {
/* background-color:rgba(0, 100, 0, 0.3); */
font-family:"Monaco";
font-style:italic;
font-size: 0.93em;
color: rgba(0, 0, 0, 0.6);
width:100px;
}
.apiBlock .apiArgs .argDesc {
/* background-color:rgba(0, 100, 0, 0.3); */
/*
font-family:"Monaco";
font-style:italic;
font-size: 0.93em;
max-width:350px;
color: rgba(0, 0, 0, 0.6);
*/
padding-top:8px;
}
/* --------- Return Value -------------------- */
.apiBlock .apiRetVal {
/* margin-right: 3%; margin-left: 3%;
*/
}
.apiBlock .apiRetVal .Label {
width: 120px;
}
.apiBlock .apiRetVal .Type {
/* background-color:rgba(100, 0, 0, 0.3); */
font-family:"Monaco";
/* font-style:italic;*/
font-size: 0.80em;
white-space: pre;
padding-bottom: 20px;
}
/*
.apiBlock .apiRetVal .Desc {
min-width: 500px;
}
*/
/* --------- Examples -------------------- */
.apiBlock .Examples {
/* margin-right: 5%; margin-left: 5%;
*/
}
.apiBlock .Examples .Label {
width: 120px;
}
.apiBlock .Examples .Example {
/*
display:block;
margin-left: 0%;
margin-right: 20px;
float: left;
padding-left:10px;
padding-top:7px;
background-color: rgba(0, 0, 0, 0.1);
*/
font-family:"Monaco";
font-size: 0.80em;
/* width:450px;*/
/*
*/
max-width:500px;
/* min-width:300px; */
white-space: pre;
padding-bottom:15px;
padding-right:20px;
}
.apiBlock .Examples .Desc {
/*
display:block;
font-family:'Gafata';
background-color:#303030;
padding-bottom: 20px;
color: #707070;
width: 500px;
*/
padding-top:8px;
}
/*
function {
margin-top: 5%;
margin-right: 10%;
margin-left: 10%;
line-height: 1;
}
*/
dd {
padding-bottom: 10px;
}
/*
dt {
float:left;
width:100px;
clear: left;
float:left;
}
dl.inline dt, dl.inline dd {
display: inline;
float: left;
margin: 0 0.5em 0 0;
}
dl.inline dd + dt, dl.inline dd + dd {
clear: left;
}
*/
/*
dd,dt {
padding-top:5px;
padding-bottom:5px;
}
*/
/*
dt {
float:left;
padding-right: 5px;
min-width:120px;
font-weight: bolder;
}
*/
/* dd {padding-left: 5px;} Does not work */
/*
dt {
clear: left;
}
*/
/*
dt, dd {
float:left;
min-height:1.5em;
}
*/
================================================
FILE: Docs/index.html
================================================
<html>
<!-- --------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.6.1 |
| http://mogill.com/ jace@mogill.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2011-2014, Synthetic Semantics LLC. All rights reserved. |
| Copyright (c) 2015-2020, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-------------------------------------------------------------------------- -->
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<link rel="stylesheet" type="text/css" href="./docs.css">
<link rel="icon" href="favicon.ico" type="image/x-icon">
<link rel="shortcut icon" href="favicon.ico" type="image/x-icon">
<title>Extended Memory Semantics -- Overview</title>
</head>
<body>
<div style="font-family:Gotthard; font-size: 40px; vertical-align:middle; margin-left: 1%">
<a href="http://mogill.com">
<span style="vertical-align:middle;">Extended Memory Semantics</span>
</a>
</div>
<div style="padding-left:3%; font-size:1.2em;">
<a href="index.html"> Overview of EMS </a>
|
<a href="reference.html"> API Reference </a>
|
<a href="https://www.npmjs.org/package/ems"> Node.js NPM </a>
|
<a href="https://github.com/mogill/ems"> Download at GitHub </a>
</div>
<h1>
Extended Memory Semantics
</h1>
<p class="pmulti" >
<span style="font-size: 1.1em; font-weight:bold; font-style:italic;">
Extended Memory Semantics (EMS) complements serial programming
models with
transactional and other fine-grained synchronization capabilities
to support parallel programming.
</span>
<br><br>
Much of the challenge in implementing distributed and parallel
programs derives from finding, marshaling, and synchronizing
data. Extended Memory Semantics (EMS) unifies these tasks into a
single programming and execution model. EMS implements a shared
address space with a rich set of primitives for parallel access of data
structures. It is not a source of parallelism itself, instead it
complements other parallel programming models and integrates shared
memory data access and synchronization.
<br><br>
EMS leverages existing tool chains instead of replacing them and is
compatible with legacy applications, libraries, frameworks,
operating systems, and hardware.
Because EMS represents
memory and not processes, it may persist independently of any
application, and it's state may be replicated, archived, or forked.
Applications may attach and detach from the memory in much the
same way applications use a shared database or filesystem.
</p>
<P>
<h6>Synchronization As a Property of the Data, Not a Duty for Tasks
</h6>
<p class="pmulti">
EMS internally stores tags that are used for synchronization of
user data, allowing synchronization to happen independently of
the number or kind of processes accessing the data. The tags
can be thought of as being in one of three states, <em>Empty,
Full,</em> or <em>Read-Only</em>, and the EMS primitives enforce
atomic access through automatic state transitions.
<br>
<img style="clear:both; height:200px; margin-left: 30px;"
src="./fsmSimple.svg" type="image/svg+xml" />
<span class="figcaption">
<br><br> EMS Data Tag Transitions & Atomic operations:
F=Full, E=Empty, X=Don't Care, RW=Readers-Writer lock (# of current readers)
CAS=Compare-and-Swap, FAA=Fetch-and-Add
</span>
<br><br>
The function name <code>readFE</code> means "Read when full and mark empty",
<code>writeEF</code> means "Write when empty and mark full",
<code>writeXF</code> means "Write unconditionally and mark
full", etc. In the most simple case, full-empty tags are used
to block readers until memory is marked full by a writer thread
that itself blocks until the memory is marked empty. This is
effectively a dataflow or producer-consumer execution model that
may involve any number of producers and consumers.
<br><br>
<img style="height:120px; margin-left: 30px;"
src="./memLayoutLogical.svg" type="image/svg+xml" />
<span class="figcaption">
<br><br>EMS memory is an array of JSON primitive values
(Number, Boolean, String, or Undefined) accessed using atomic
operators and/or transactional memory. Safe parallel access
is managed by passing through multiple gates: First mapping a
key to an index, then accessing user data protected by EMS
tags, and completing the whole operation atomically.
</span>
<br><br>
The EMS array may be indexed directly using an integer, or using a key
mapping of any primitive type. When a map is used, the key and data
itself are updated atomically.
<br><br>
The full-empty primitives are
used construct other thread-safe data types such as
atomic read-modify-write operations and Transactional Memory (TM).
</p>
<h3>
Principles of Operation
</h3>
<p class="pmulti" >
When the <code>require('ems')(...)</code> statement is executed by a program,
EMS first creates a shared memory region to rendezvous and
communicate with other EMS threads, then,
using the built-in
<img src="./nodejs.svg" type="image/svg+xml" height="16px" style="vertical-align:text-top;" />
fork primitives,
creates the additional threads executing
using one of two execution models: fork-join or Bulk Synchronous Parallel (BSP).
BSP invokes the same script as the master thread (found in <code>process.argv[2]</code>),
whereas fork-join execution invokes parallel region around a function.
<br><br>
Under BSP, all threads execute the entire program unless statements are explicitly skipped.
Fork-join parallelism has a single entry point and executes sequentially
until a parallel region is started with <code>ems.parallel( func )</code>.
<br>
<img style="width:200px; margin-left: 80px;" src="./BSPvsForkJoin.svg" type="image/svg+xml" />
<span class="figcaption"><br>
Fork-Join parallelism follows the traditional single-threaded execution
model until a parallel region where threads are dynamically added to
perform iterations of a loop. Under BSP parallelism every thread
enters the program at the main entry point.
</span>
<BR><BR>
Fork-join creates parallel regions
much like OpenMP's <code>#pragma omp parallel</code> directive.
Under BSP, all threads enter the main program and execute all
statements, synchronizing at barriers.
<BR><BR>
In addition to ordinary sequential loops,
within a parallel region <code>ems.parForEach( func )</code>
loops distribute iterations among the threads
using several load balancing scheduling options.
<br><br>
The master thread preserves all the characteristics and capabilities of
an ordinary
<img src="./nodejs.svg" type="image/svg+xml" height="16px" style="vertical-align:text-top;"/>
job, and all legacy applications, modules, packages, frameworks, and test apparatus
will work normally. Software that does not use EMS is not affected by it's presence.
<br><br>
Atomic operations like compare-and-swap (CAS) and fetch-and-add (FAA)
that are typically non-blocking will block if the full/empty
tag is set to empty.
Stack/queue operators are deadlock free,
blocking operations and should be thought of as thread-safe but not concurrent.
EMS transactions are also deadlock free and support element-level locking
for the highest possible currency.
<br><br>
Dataflow programs directly manipulating the full/empty tags
may deadlock if a program attempts to re-acquire a lock
already held, or acquire locks in a different order than other threads.
<br><br>
EMS programs may be run with any number of threads, including
single threaded and over-subscribed.
</P>
<br>
<center>
<img src="./blockDiagram.svg" type="image/svg+xml" width="90%" style="vertical-align:text-top;"/>
<span class="figcaption">
<br><br>
A logical overview of what program statements cause threads to be created
and how shared data is referenced.
</span>
</center>
<h3>Performance</h3>
<p>
These experiments were run on an Amazon EC2 instance:<br>
<code>cr1.8xlarge: 244 GiB memory, 88 EC2 Compute Units, 240 GB of local instance storage, 64-bit platform, 10 Gigabit Ethernet</code>
</p>
<p class="pmulti" >
<img src="tm_from_q.svg" type="image/svg+xml" height="260px" style="vertical-align:text-top;"/>
<span class="figcaption"><br>
Using the general transactional memory capabilities to process
randomly generated operations stored in a shared work queue.
</span>
<br><br><br><br>
<img src="./TMfromLoop.png" type="image/svg+xml" height="260px" style="vertical-align:text-top;"/>
<span class="figcaption"><br>
Transaction processing but generating the work from a loop instead of reading it from a shared queue.
</span>
<br><br><br><br>
<img src="./ccab.png" type="image/svg+xml" height="260px" style="vertical-align:text-top;"/>
<span class="figcaption"><br>
Perform the operation <code>c[i] = c[i] + a[i] * b[i]</code> atomically
</span>
<br><br><br><br>
<img src="./wordcount.svg" type="image/svg+xml" height="260px" style="vertical-align:text-top;"/>
<span class="figcaption"><br>
Word Count of documents from Project Gutenberg in a variety of languages. Average document was
about 250kb in length.
</span>
</p>
<h3>Built-In Composed Operations and Parallel Data Structures</h3>
<p>
High-level data abstractions can be constructed from the EMS primitives,
and EMS itself
composes the primitives to implement transactional memory (TM), stacks, and queues.
User defined composed operations can be added to EMS classes just as new methods
are added to other JavaScript objects.
</p>
<h6>Transactional Memory</h6>
<P>
Transactional Memory (TM) provides atomic access to multiple shared objects
in a manner similar to transactional databases. EMS implements mutual exclusion
on specific data elements using the Full/Empty tags,
and shared read-only access with a multiple readers-single writer tag.
</p>
<h6>Stacks and Queues</h6>
<P>
Parallel-safe stacks and queues are built-in intrinsics based on Full/Empty tags.
Stacks and queues are by definition serial data structures and do
not support any concurrency. Although highly efficient, a shared
resource like these can become a hot-spot when dozens of threads compete
for access.
</p>
<h5>
Types of Parallelism
</h5>
<P>
<center>
<figure>
<img src="./typesOfParallelism.svg" type="image/svg+xml" width="500px" />
<figcaption><br>
EMS Data Tag Transitions - The four data element states
and the intrinsic EMS atomic operations to transition
between them.
</figcaption>
</figure>
</center>
</p>
<h5>
Why Shared Memory Parallelism?
</h5>
<P>
<center>
<figure>
<img src="./timelines.svg" type="image/svg+xml" width="100%" style="max-width:900px;"/>
<figcaption><br>
Multithreading complements other forms of parallelism and can be
combined with other forms of concurrency for multiplicative benefits.
</figcaption>
</figure>
</center>
</p>
<h5>
Contrary Notions of Strong & Weak Scaling
</h5>
<P>
<center>
<table width="90%" style="border:0px solid #303030; border-collapse:collapse; padding-bottom:24%;">
<tr>
<td style="border:0px; vertical-align:top;" align="top" width="20%">
</TD>
<td style="border:0px solid #303030; vertical-align:top;" align="top" width="40%">
<center style="float:top; font-size:1.3em;">
<b>Strong Scaling</b> <br>
<img src="./strong_scaling.svg" type="image/svg+xml" width="300px"
style="border:0; box-shadow: 0px 0px 0px rgba(0,0,0,0);" />
</center>
</TD>
<td style="border:0px solid #303030; vertical-align:top;" align="top" width="40%">
<center style="float:top; font-size:1.3em;">
<b>Weak Scaling</b> <br>
<img src="./weak_scaling.svg" type="image/svg+xml" width="300px"
style="border:0; box-shadow: 0px 0px 0px rgba(0,0,0,0);" />
</center>
</td>
</tr>
<tr style="background-color:rgba(0,0,150,.2);">
<td style="vertical-align:middle; text-align:right; font-size:1.3em; padding-right:25px;">
<em>Scaling Goal</em>
</TD>
<td style="vertical-align:middle; padding-top:1%; padding-bottom:1%; padding-left:2%;">
Solve the same problem, only faster
</TD>
<td style="vertical-align:middle; padding-top:1%; padding-bottom:1%; padding-left:2%;">
Solve a bigger problem in the same amount of time
</TD>
</tr>
<tr style="background-color:rgba(0,100,0,.2);">
<td style="vertical-align:middle; text-align:right; font-size:1.3em; padding-right:25px;">
<em>Problem Size</em>
</TD>
<td style="vertical-align:middle; padding-top:1%; padding-bottom:1%; padding-left:2%;">
Stays constant while number of processors increases
</TD>
<td style="vertical-align:middle; padding-top:1%; padding-bottom:1%; padding-left:2%;">
Grows with the number of processors
</TD>
</tr>
<tr style="background-color:rgba(0,0,150,.2);">
<td style="vertical-align:middle; text-align:right; font-size:1.3em; padding-right:25px;">
<em>Scaling is limited by</em>
</TD>
<td style="vertical-align:middle; padding-top:1%; padding-bottom:1%; padding-left:2%;">
Inter-process communication
</TD>
<td style="vertical-align:middle; padding-top:1%; padding-bottom:1%; padding-left:2%;">
Problem size
</TD>
</tr>
<tr style="background-color:rgba(0,100,0,.2);">
<td style="vertical-align:middle; text-align:right; font-size:1.3em; padding-right:25px;">
<em>Resiliency</em>
</TD>
<td style="vertical-align:middle; padding-top:1%; padding-bottom:1%; padding-left:2%;">
Single failure causes entire job to fail,
SLAs achieved through efficient checkpoint-restart.
</TD>
<td style="vertical-align:middle; padding-top:1%; padding-bottom:1%; padding-left:2%;">
Failed sub-tasks are detected and retried. SLAs achieved through fault resiliency.
</TD>
</tr>
<tr style="background-color:rgba(0,0,150,.2);">
<td style="vertical-align:middle; text-align:right; font-size:1.3em; padding-right:25px;">
<em>Programming Models</em>
</TD>
<td style="vertical-align:middle; padding-top:1%; padding-bottom:1%; padding-left:2%;">
MPI, GASNet, Chapel, X10, Co-Array Fortran, UPC
</TD>
<td style="vertical-align:middle; padding-top:1%; padding-bottom:1%; padding-left:2%;">
Batch jobs, Map-Reduce
</TD>
</tr>
</table>
</center>
</p>
<h5>
Historical Precedents for Data-Centric Multithreading
</h5>
<p class="pmulti">
EMS builds on three experimental computer architectures from the
1980's: the NYU Ultracomputer, the MIT J-Machine, and the Tera Multi-Threaded Architecture (MTA).
Specifically, the Ultra introduced combining networks as a basic
architectural feature,
the J-Machine made moving a task to data as easy as moving data to the processor,
and the MTA used massive multithreading to mask latency and
had fine-grained synchronization associated with the data, not tasks.
</P>
<h3>
Bugs
</h3>
<P>
The old data argument for CAS requires a deep compare -- this is possible but
entirely compatible with JS semantics which compares type, not contents.
<br><br>
Similarly, FAA does not implement add on object type data, but extending the
semantics of JS would make this sensible
</P>
<h3>
Roadmap
</h3>
<h6>
Languages and APIs
</h6>
<P>
In addition to JavaScript, Python, and C/C++, EMS can be added to other languages.
Languages that can share the EMS API:
<br> - Scala
<br> - PHP
<br> - Java
<br> - Perl
<br> - Your name here
</P>
<h6>
Examples, Benchmarks, Tests
</h6>
<P>
Other programs included with the distribution for demonstration, test, and benchmarking purposes.
<br> - Matrix multiply
<br> - Graph500
<br> - Sobel Filter
<br> - MongoDB Replica Server
</P>
<h6>
RiverTrail Extensions
</h6>
<P>
A few years ago a proposed set of language extensions called RiverTrail
was being supported by Intel and was implemented as a Firefox Plug-In.
The hooks used by the extension are no longer supported in Firefox, and
the future of RiverTrail is unclear. To the extent possible, EMS should
implement RiverTrail extensions.
</P>
<hr>
<div style="font-size:.85em;">
This browsing experience is Copyright ©2014-2020,
<span style="font-family:Gotthard;">
<a href="http://mogill.com">
Jace Mogill</a>
</span>.
Proudly designed and built in Cascadia.
<img src="./Flag_of_Cascadia.svg" type="image/svg+xml" height="20px" style="vertical-align:middle;">
</div>
</body>
</html>
================================================
FILE: Docs/reference.html
================================================
<html>
<!-- --------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.6.1 |
| http://mogill.com/ jace@mogill.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2011-2014, Synthetic Semantics LLC. All rights reserved. |
| Copyright (c) 2015-2020, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-------------------------------------------------------------------------- -->
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<link rel="stylesheet" type="text/css" href="./docs.css">
<link rel="icon" href="favicon.ico" type="image/x-icon">
<link rel="shortcut icon" href="favicon.ico" type="image/x-icon">
<title>Extended Memory Semantics -- API Reference</title>
</head>
<body>
<div style="font-family:Gotthard; font-size: 40px; vertical-align:middle; margin-left: 1%">
<a href="http://mogill.com">
<img src="./synsem_logo_black.svg" type="image/svg+xml" height="50px" style="vertical-align:middle;" />
<span style="vertical-align:middle;"> Extended Memory Semantics</span>
</a>
</div>
<div style="padding-left:3%; font-size:1.7em;">
<a href="index.html"> Overview of EMS </a>
|
<a href="reference.html"> API Documentation </a>
|
<a href="https://www.npmjs.org/package/ems"> Node.js NPM </a>
|
<a href="https://github.com/mogill/ems"> Download at GitHub </a>
</div>
<h1>
Extended Memory Semantics
</h1>
<p>
EMS internally stores tags that are used for synchronization of user data,
allowing synchronization to happen independently of the number or kind of processes accessing
the data.
The EMS primitives enforce atomic access using automatic state transitions,
and higher level intrinsics like stacks, queues, and transactional
memory are built on the EMS primitives.
<center>
<table width="90%" >
<tr>
<td>
<center>
<figure>
<img src="./memLayoutLogical.svg" type="image/svg+xml" height="160px" />
<span class="figcaption">
<br>EMS memory is an array of JSON values
accessed using atomic operators and/or transactional memory.
Safe parallel access is managed by passing through multiple gates:
First mapping a key to an index, then
accessing user data protected by EMS tags, and completing the whole
operation atomically.
</span>
</figure>
</center>
</td>
<td width="50%">
<center>
<figure>
<img src="./fsmSimple.svg" type="image/svg+xml" height="200px" />
<span class="figcaption">
<br>
EMS Data Tag Transitions & Atomic operations:
F=Full, E=Empty, X=Don't Care, RW=Readers-Writer lock (# of current readers)
CAS=Compare-and-Swap, FAA=Fetch-and-Add
</figcaption>
</span>
</center>
</td>
</tr>
</table>
</center>
</p>
<h3>
EMS Class & Array Methods
</h3>
<P>
Operations which inspect or affect the global EMS state are performed
using class methods of
the EMS object returned by the <code>require</code> statement.
Operations that modify the data and tags are performed using methods
belonging to EMS array to be operated on.
</P>
<h5>
Module Require
</h5>
<table class="apiBlock" >
<tr class="apiFunc" >
<td class="Label"> REQUIRE </td>
<td colspan=3 class="Proto" style="padding-bottom: 20px;"> require('ems') ( nThreads [, threadAffinity [, parallelType [, contextName ] ] ] )</td>
</tr>
<tr class="apiSynopsis">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Initialize the EMS module, starting
all the other threads. Thread identity and processor affinity
is assigned when the thread is created.
<BR><BR> </td>
</tr>
<tr class="apiArgs">
<td class="Label"> ARGUMENTS </td>
<td class="argName"> nThreads</td>
<td class="argType"> <Number></td>
<td class="argDesc" > Total number of threads the job should use.
</td>
</tr>
<tr class="apiArgs" >
<td class="Label"> </td>
<td class="argName"> threadAffinity </td>
<td class="argType"> <Boolean> </td>
<td class="argDesc"> (Optional, Default = <code>false</code>, Affects only Linux.)
Set the scheduling affinity of each thread to it's own
core, assigning over-subscribed threads in a round-robin
fashion.
</td>
</tr>
<tr class="apiArgs" >
<td class="Label"> </td>
<td class="argName"> parallelType </td>
<td class="argType"> <String> </td>
<td class="argDesc"> (Optional, Default=<code>bsp</code>)
One of <code>bsp</code>, <code>fj</code>, or <code>user</code>. Execution model:
<code>bsp</code> will use EMS' built-in Bulk Synchronous Parallel execution,
<code>fj</code> uses EMS' built-in Fork-join execution
<code>user</code> creates no parallelism
</td>
</tr>
<tr class="apiArgs" >
<td class="Label"> </td>
<td class="argName"> contextName </td>
<td class="argType"> <String> </td>
<td class="argDesc"> (Optional, Default=<code>anonymous</code>)
Unique name of parallel context being initialized, required to distinguish
between multiple EMS parallel programs running simultaneously on the
same system.
</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type" colspan="2" >ems = {
nThreads : Number, // Number of threads executing
myID : Number // This thread's ID 0..nThreads-1
}</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">ems = require('ems')(process.argv[2]) </td>
<td class="Desc"> Use the first command line argument as the number of nodes:
<br> <code>node foo.js 4</code> executes using 4 threads.</td>
</tr>
<tr class="Examples">
<td class="Label"> </td>
<td class="Example">ems = require('ems')()</td>
<td class="Desc" > Run on one node </td>
</tr>
<tr class="Examples">
<td class="Label"> </td>
<td class="Example">ems = require('ems')(process.argv[2],
false, true) </td>
<td class="Desc" > Use first command line argument as number of nodes, do not set
affinity of the threads to a specific CPU, execute using fork-join parallelism. </td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<!-- ----------------------------------------------------------------------------- -->
<h5> Create a new EMS Array </h5>
<table class="apiBlock" >
<tr class="apiFunc">
<td class="Label"> CLASS METHOD </td>
<td colspan=3 class="Proto"> ems.new( [ nElements [, heapSize [, fileName] ] ] ) </td>
</tr>
<tr class="apiFunc">
<td class="Label" style="padding-bottom: 20px;"> </td>
<td colspan=3 class="Proto"> ems.new( emsArrayDescriptor ) </td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Attach to an existing or create a new EMS array.
Creation of new (do not use existing) EMS memory regions implies a barrier,
whereas using an existing EMS file will block until the file exists.
<BR><BR> </td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName"> nElements </td>
<td class="argType"> <Number></td>
<td class="argDesc" > (Optional, Default is 1) Maximum number of elements in the EMS array or map. </td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName"> </td>
<td class="argType"> <Array></td>
<td class="argDesc" > An array of dimensions of a multi-dimensional array.
A 100×30×50 cube is described by <code>[100, 30, 50]</code>. </td>
</tr>
<tr class="apiArgs" >
<td class="Label"> </td>
<td class="argName"> heapSize </td>
<td class="argType"> <Number> </td>
<td class="argDesc">
(Optional, Default is 0)
Maximum number of bytes reserved for strings,
arrays, maps, and object elements in this array.
The actual amount of memory allocated for use is
rounded up to the nearest power of 2
<TODO>(until a better memory allocator is implemented)</TODO>.
Additional memory is allocated for bookkeeping.
Memory block size is defined in <code>ems_alloc.h: EMS_MEM_BLOCKSZ</code>
<TODO>but should be configurable at create time</TODO>.
</td>
</tr>
<tr class="apiArgs" >
<td class="Label"> </td>
<td class="argName"> fileName </td>
<td class="argType"> <String> </td>
<td class="argDesc">
(Optional, Default is anonymous) Fully qualified file name
of the file to use as a persistent backing store for the EMS array,
tags, and bookkeeping information.
</td>
</tr>
<tr class="apiArgs" >
<td class="Label"> </td>
<td class="argName">emsArrayDescriptor</td>
<td class="argType"> <Object> </td>
<td class="argDesc">
(Alternative to two scalar argument) A complete EMS Array
descriptor may be passed as the only argument instead of
scalar arguments.<code style="white-space: pre;">
emsArrayDescriptor = {
dimensions : 100, // Required: Max # elements in EMS array
// Integer for 1-D, or array of dims [x, y, z]
heapSize : 100000, // Optional, default=0: Space, in bytes, for
// strings, maps, and objects. Rounded up to nearest power of two
mlock : 0, // Optional, default=0%: % of EMS memory to lock into RAM
useMap : true, // Optional, default=false: Map keys to indexes
useExisting : true, // Optional, default=false:
// Preserve data if an file already exists
persist : true, // Optional, default=true:
// Preserve the file after threads exit
doDataFill : false, // Optional, default=false: Initialize memory
dataFill : undefined, // Optional, If this property is defined,
// the EMS memory is filled with this value
setFEtags : 'full', // Optional, If defined, set 'full' or 'empty'
filename : '/path/to/file' // Optional, default=anonymous:
// Path to the persistent file of this array
}</code>
</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type" colspan="2" ><EMS Array></td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">var foo = ems.new(nItems) </td>
<td class="Desc"> Create a new non-persistent shared memory EMS
array with no heap space. Scalar data (Number, Boolean, Undefined)
may be stored in this array, but not strings, arrays, or objects.
</td>
</tr>
<tr class="Examples">
<td class="Label"> </td>
<td class="Example">var foo = ems.new(nItems, size,
'/tmp/EMS_foo') </td>
<td class="Desc" > Create a file-backed EMS shared memory
space with the filename <code>/tmp/EMS_foo</code>. In addition to the
scalar storage,
space for strings totaling
<code>size</code> bytes is also reserved for strings, arrays, objects, and maps.</td>
</tr>
<tr class="Examples">
<td class="Label"> </td>
<td class="Example">var x = ems.new(nItems, size) </td>
<td class="Desc" > Create a new non-persistent shared memory EMS
array that has space for strings totaling <code>size</code> bytes </td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Parallel Region (Fork-Join execution only) </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label" style="padding-bottom: 20px; vertical-align:text-top;"> CLASS METHOD </td>
<td colspan=3 class="Proto"> ems.parallel( [args, ...] func ) </td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> When using fork-join execution, the master
thread enters a parallel region by executing <code>func</code>
once from each process. The master process first starts all the
other processes running the function asynchronously, then executes the function
itself synchronously. All processes join (perform a <code>barrier</code>)
after completion of the work function.
The results of the function are discarded. Global variables on each node
are persistent between parallel regions.
</td>
</tr>
<tr class="apiArgs">
<td class="Label"> ARGUMENTS </td>
<td class="argName"> args </td>
<td class="argType"> <Any></td>
<td class="argDesc"> Zero or more arguments to be passed to the function.
</td>
</tr>
<tr class="apiArgs">
<td class="Label"> </td>
<td class="argName"> func </td>
<td class="argType"> <Function></td>
<td class="argDesc" > Function to be executed once on every thread. The
optional arguments are used when calling the function. Return value is ignored.
</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">ems.parallel( doWork )</td>
<td class="Desc"> The function <code>doWork</code> is executed by every thread. </td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">ems.parallel( foo, "Smith", 123, doWork )</td>
<td class="Desc"> The function call <code>doWork(foo, "Smith", 123)</code>
is performed once by each process.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Parallel Loops </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label" style="padding-bottom: 20px;"> CLASS METHOD </td>
<td colspan=3 class="Proto"> ems.parForEach( first, last, function [, scheduling [, minChunk] ] ) </td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Parallel loop execution, distributing
the iterations among all the threads. The function is invoked
with one argument, the current iteration number. Iterations are
divided among the threads according to the
<code>scheduling</code> method specified.
Parallel for loops <em>must not</em> be nested.
<TODO>The system should check for this and fall back on serial execution.</todo>
A barrier is implied at
the end. <BR><BR>
</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName"> first </td>
<td class="argType"> <Number></td>
<td class="argDesc" > Index to start iterating </td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName"> last </td>
<td class="argType"> <Number></td>
<td class="argDesc" > Index to stop iterating (non-inclusive).</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName"> func </td>
<td class="argType"> <Function></td>
<td class="argDesc" > Loop body, only input argument is current loop index:
<br><code>function foo(idx) {...}</code></td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName"> scheduling </td>
<td class="argType"> <String></td>
<td class="argDesc" >
<code>guided [minChunk]</code>: Decreasing amounts of work are assigned to each task until <code>minChunk</code> iterations per thread is reached. Load balancing occurs when new chunks are assigned to threads.<br>
<code>static</code>: Equal number of iterations are given to each thread, no dynamic load balancing is performed.<br>
<code>dynamic</code>: All threads share one index which is atomically
incremented by 1 after each iteration. Provides ideal load balancing at the
cost of high per-iteration overhead.<br>
</td>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName"> minChunk </td>
<td class="argType"> <Number></td>
<td class="argDesc" >(Optional, only used when <code>scheduling='guided'</code>, default=<code>1</code>) Minimum number of iterations assigned to a single thread.
</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">ems.parForEach(0, nItems-1, func)</td>
<td class="Desc">Execute the <code>func</code> function <code>nItems-1</code> times with indexes
<code>0..nItems-1</code>, inclusive. </td>
</tr>
<tr class="Examples">
<td class="Label"> </td>
<td class="Example">ems.parForEach(10000, 20000, func,
'guided', 200)</td>
<td class="Desc" > Distribute iterations numbered 10,000-20,000 (inclusive) using the <code>guided</code>
method with a minimum chunk size of 200 iterations </td>
</tr>
<tr class="Examples">
<td class="Label"> </td>
<td class="Example">ems.parForEach(0, 999,
func, 'static')</td>
<td class="Desc" > Execute <code>func()</code> 1,000 times with indexes
0..999 inclusive. Distribute the iterations evenly across
threads in contiguous blocks. </td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Barrier Synchronization </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label" style="padding-bottom: 20px; vertical-align:text-top;"> CLASS METHOD </td>
<td colspan=3 class="Proto"> ems.barrier() </td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> All the threads must reach the same
barrier before proceeding. Failure to call a barrier from every
process will result in deadlock.
If called outside a parallel region, a barrier has no effect.
</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">ems.barrier()</td>
<td class="Desc"> All threads reach the barrier before proceeding. </td>
</tr>
<!--
<tr class="Examples">
<td class="Label"> </td>
<td class="Example">if(ems.myID == 0) {
// One thread does global initialization
ems.barrier()
} else {
ems.barrier()
// Other threads wait for initialization to complete
}</td>
<td class="Desc" > Initialize some shared data and then allow all threads to access it </td>
</tr>
-->
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Critical Region </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label" style="padding-bottom: 20px;"> CLASS METHOD </td>
<td colspan=3 class="Proto">ems.critical( func )</td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Perform function <code>func()</code>
mutually exclusive of other threads. Serializes execution through
all critical regions.
<TODO> Named regions would be more like OpenMP </todo>
<br><br></td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName"> func</td>
<td class="argType"> <Function></td>
<td class="argDesc" > Function to perform sequentially. </td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">ems.critical( function() {
// Use a shared resources
} )</td>
<td class="Desc"> A shared resource is accessed sequentially, but in no particular order.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Execute on Master Thread Only </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label" style="padding-bottom: 20px;"> CLASS METHOD </td>
<td colspan=3 class="Proto">ems.master( func )</td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Perform function <code>func()</code>
only on thread 0, implies a barrier. <br><br> </td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName"> func</td>
<td class="argType"> <Function></td>
<td class="argDesc" > Function to perform only by task 0. </td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">ems.master( function() {
console.log("Only task 0")
} )</td>
<td class="Desc">Console logging performed only by task 0</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Execute once on any thread </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label" style="padding-bottom: 20px;"> CLASS METHOD </td>
<td colspan=3 class="Proto">ems.single( func )</td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Perform function <code>func()</code>
only once by the first thread to reach the statement.
Implies a barrier. <br><br> </td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName"> func</td>
<td class="argType"> <Function></td>
<td class="argDesc" > Function to be performed once. </td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">ems.single( function() {
console.log("Only first task")
} )</td>
<td class="Desc">Console logging is performed only once.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Print Diagnostic Message </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label" style="padding-bottom: 20px;"> CLASS METHOD </td>
<td colspan=3 class="Proto">ems.diag( message )</td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Print a diagnostic message to the console with a prefix indicating the task ID. <br><br> </td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">message</td>
<td class="argType"> <String></td>
<td class="argDesc" > Text of message to print to the console </td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">ems.diag( "Hello, world!" )
</td>
<td class="Desc"><code>EMS 3: Hello, world!</code> appears on console</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<!-- ----------------------------------------------------------------------------- -->
<!-- ----------------------------------------------------------------------------- -->
<!-- ----------------------------------------------------------------------------- -->
<!-- ----------------------------------------------------------------------------- -->
<h3>
EMS Array Methods
</h3>
<h5> Intrinsic Atomic Operations (AMOs) </h5>
<h6> Read EMS Memory </h6>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> ARRAY METHOD </td>
<td colspan=3 class="Proto">emsArray.read( index )</td>
</tr>
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> </td>
<td colspan=3 class="Proto">emsArray.readFE( index )</td>
</tr>
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> </td>
<td colspan=3 class="Proto">emsArray.readFF( index ) <BR></td>
</tr>
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> </td>
<td colspan=3 class="Proto">emsArray.readRW( index ), emsArray.releaseRW( index ) <BR><BR></td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> The <code>read</code> family of EMS
memory operations return the data stored in an EMS array element.
The value may be any JSON type.
<br>
<dl>
<dt> <code>read</code> </dt>
<dd> Immediately and unconditionally returns the
stored value, ignoring the tag state.
<TODO> Reading uninitialized mapped indexes
will return <code>undefined</code> regardless of the default value
that would be returned with <code>read__, cas, faa</code>.
</TODO>
</dd>
<dt> <code>readFE</code> </dt>
<dd> Blocks until the data element is full, then
atomically reads the value and marks it empty.
</dd>
<dt> <code>readFF</code> </dt>
<dd>
Blocks until the data element is full, then
atomically reads leaving it full. This allows safe read access
of data which may be updated
simultaneously. <code>readFF</code> ensures mutual exclusion,
and will block if the data is already under a Readers-Writer
lock.
</dd>
<dt> <code>readRW, releaseRW</code> </dt>
<dd>
Blocks until the data element is full or
already under a Readers-Writer lock, then increments the
Readers-Writer reference count. The
function <code>emsArray.releaseRW()</code> decrements the reference
count, restoring the state to <code>Full</code> if no readers remain.
</dd>
</dl>
</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">index</td>
<td class="argType"> <Number | String></td>
<td class="argDesc" > Index in the EMS array of data to read</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type" colspan="2" > read__ : < Number | Boolean | String | Undefined | Array | Object ></td>
</tr>
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> </td>
<td class="Type" > releaseRW : <Number></td>
<td class="Desc"> Number of pending readers sharing the lock.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">var n = histogram.read(3) </td>
<td class="Desc">Read the value in bin 3 of the histogram.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">var word = dictionary.read(idx) </td>
<td class="Desc">Read dictionary word at index/key <code>idx</code> in the dictionary array.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">var x = arr.readFE(idx) </td>
<td class="Desc">Block until the element at index <code>i</code>
is full, atomically read the value and mark it empty. </td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">var x = arr.readFF(idx) </td>
<td class="Desc"> Block until the element at index <code>i</code> is full,
atomically read the value and leave it full.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">var x = arr.readRW(idx) </td>
<td class="Desc"> Acquire a shared-exclusive readers-writer lock.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Write EMS Memory </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> ARRAY METHOD </td>
<td colspan=3 class="Proto">emsArray.write( index, value )
</td>
</tr>
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> </td>
<td colspan=3 class="Proto">emsArray.writeXE( index, value )</td>
</tr>
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> </td>
<td colspan=3 class="Proto">emsArray.writeXF( index, value )</td>
</tr>
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> </td>
<td colspan=3 class="Proto">emsArray.writeEF( index, value ) <BR><br></td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Write a value to an element of an EMS array.
<br>
<dl>
<dt> <code>write</code> </dt>
<dd> Immediately and unconditionally
writes the value to memory. This operation does not honor or
modify the full/empty tag status.
</dd>
<dt> <code>writeXE</code> </dt>
<dd> Unconditionally and atomically writes the value to the data element and marks the element empty.</dd>
<dt> <code>writeXF</code> </dt>
<dd> Unconditionally and atomically writes the value to the data element and marks the element full.</dd>
<dt> <code>writeEF</code> </dt>
<dd> Blocks until the element is empty, and then atomically writes the value and marks the element full.</dd>
</dl>
</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">index</td>
<td class="argType"> <Number | String></td>
<td class="argDesc" > Index in the EMS array of data to read</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName">value</td>
<td class="argType"> <Any></td>
<td class="argDesc" > Primitive value to store in the array at element numbered index.</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">histogram.write(idx, 0) </td>
<td class="Desc" >Initialize the value of <code>histogram[idx]</code> to 0.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">dictionary.write(idx, "Hello") </td>
<td class="Desc">Add the string <code>"Hello"</code> to the EMS
array <code>dictionary</code> at index <code>idx</code>.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">arr.writeXE(i, undefined) </td>
<td class="Desc">Purge the memory at index <code>i</code> of the EMS array <code>arr</code>.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">arr.writeXF(j, 'Mr. Jones') </td>
<td class="Desc">Unconditionally write the
string <code>'Mr. Jones'</code>to the EMS array <code>arr</code>
at index <code>j</code> and atomically mark the element full.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">arr.writeEF(2, v) </td>
<td class="Desc">Block until the element at index 2
of <code>arr</code> is empty, atomically write the
value <code>v</code> and mark the memory full.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Atomic Fetch and Add </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> ARRAY METHOD </td>
<td colspan=3 class="Proto">emsArray.faa( index, value )<BR><BR></td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3>
Atomically read the array's JSON primitive element (scalar or string, not array or object),
add the value, and write the new value
back to memory. Return the original contents of the memory.
<BR><BR></td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">index</td>
<td class="argType"> <Integer | String></td>
<td class="argDesc" >Index of the element in the EMS array <code>emsArray</code>
to atomically add to.</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName">value</td>
<td class="argType"> < Number | Boolean | String | Undefined ></td>
<td class="argDesc" >Value to add to the EMS memory.</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type" >< Number | Boolean | String |<br> Undefined ></td>
<td class="Desc"> The results are the same type as if
<code>a + b</code> were performed. </td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">oldVal =
statistics.faa( timerIdx, elapsed )
</td>
<td class="Desc">Return the value in memory before the add operation.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">currentSum =
arr.faa( index, value ) + value
</td>
<td class="Desc">Return the current value after the atomic operation has occurred.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Atomic Compare and Swap </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> ARRAY METHOD </td>
<td colspan=3 class="Proto">emsArray.cas( index, oldValue, newValue )<BR><BR></td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3>
Atomically read the JSON primitive element (scalar or string, not object or array)
stored at the array's index,
compare the original value to <code>oldValue</code>, and if they are equivalent
store the new value.
The CAS operation succeeded
if the value returned is equivalent
to <code>oldValue</code>.
CAS will block until the EMS memory is marked full. CAS is the
equivalent of atomically performing:<br>
<code>if( arr[idx] == oldValue ) then arr[idx] = newValue</code>
<BR><BR></td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName"> index </td>
<td class="argType"> <Integer | String></td>
<td class="argDesc" > Index into the EMS array to update.
</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName"> oldValue </td>
<td class="argType"> <Number | Boolean | String | Undefined></td>
<td class="argDesc" > Value to compare to the value stored in memory.
</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName"> newValue </td>
<td class="argType"> <Any ></td>
<td class="argDesc" > Value to store if the value in memory is <code>oldValue</code>
</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type">< Number | Boolean | String | Undefined ></td>
<td class="Desc"> The value in memory when the compare was performed. </td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">acquiredLock = arr.cas(index,
UNLOCKED, LOCKED) == UNLOCKED </td>
<td class="Desc">Evaluates as <code>true</code> if the lock stored
at <code>arr[index]</code> was acquired.
</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">oldWord = users.cas(1234,
'Cooking', 'Eating')</td>
<td class="Desc">Attempt to atomically update user 1234's record from the string
<code>'Cooking'</code> to the string <code>'Eating'</code></td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<!-- ----------------------------------------------------------------------------- -->
<!-- ----------------------------------------------------------------------------- -->
<h3> Composed Array Operations </h3>
<P>
Composed operations use EMS intrinsics to perform
deadlock free atomic operations
involving multiple EMS elements. The composed operations use the tags
and require data to be full or empty as appropriate for the semantics of the operation.
</P>
<h5> Transactional Processing of Multiple Elements </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> CLASS METHOD </td>
<td colspan=3 class="Proto">ems.tmStart( tmElements )</td>
</tr>
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> </td>
<td colspan=3 class="Proto">ems.tmEnd( tmHandle, doCommit )<BR><BR></td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3>
Lock (by transitioning tags from Full to Empty)
one or more EMS elements in a deadlock
free way. When multiple locks must be acquired, this function
guarantees at least one thread will always make progress. The
optional third element indicates the element is read-only and will
not be modified by the task while the lock is held. Read-only
data is locked using a Readers-Writer lock, permitting additional concurrency.
<BR>
Performing transactions within transactions can result in deadlock
if the thread tries to recursively lock an element.
<BR><BR></td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName"> tmElements </td>
<td class="argType"> <Array></td>
<td class="argDesc" > Array identifying which EMS array
elements should be locked. Each array element is itself an array
naming the EMS array and index/key of the data
and an optional Read-Only hint:
<code>[ emsArray, index <em>(, isReadOnly)</em> ]</code>
</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName"> tmHandle </td>
<td class="argType"> <Object></td>
<td class="argDesc" > Returned from <code>tmStart()</code>,
contains state information needed to abort or commit the transaction.
</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName"> doCommit </td>
<td class="argType"> <Boolean></td>
<td class="argDesc" > Commits the transaction if <code>true</code>,
or aborts and rolls back the transaction if <code>false</code> or <code>undefined</code>.
</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type" >ems.tmStart() : < tmHandle ></td>
<td class="Desc">Transaction Handle used later to commit or abort.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">tm = ems.tmStart( [ [users, 293, true],
[comments, 8922] ] ) </td>
<td class="Desc">Lock element 293 in the <code>users</code>
EMS array with a read-only intent, and also lock record 8922 in
the <code>comments</code> EMS array.
</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">tm = ems.tmStart( [ [arrA, idxA0],
[arrA, idxA1] ] ) </td>
<td class="Desc">Lock indexes <code>idxA0</code> and <code>idxA1</code> in array <code>arrA</code>
for update to both values.
</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">tm = ems.tmStart([ [arrA, idxA0],
[arrA, idxA1, true],
[arrB, idxB0, true] ]) </td>
<td class="Desc">Acquire and free locks on the elements in <code>lockList</code>.
Element <code>arrA[idxA0]</code> may be modified, but elements
<code>arrA[idxA1]</code> and <code>arrB[idxB0]</code> are read-only.
</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">ems.tmEnd( tm, true )
</td>
<td class="Desc">Commit the transaction
</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Stacks & Queues </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> ARRAY METHOD </td>
<td colspan=3 class="Proto">emsArray.push( value )<BR></td>
</tr>
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> </td>
<td colspan=3 class="Proto">emsArray.pop( )<BR></td>
</tr>
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> </td>
<td colspan=3 class="Proto">emsArray.enqueue( value )<BR></td>
</tr>
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> </td>
<td colspan=3 class="Proto">emsArray.dequeue( )<BR><BR></td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Append or remove data from a LIFO or
FIFO. If the queue or stack is empty, the <code>pop</code>
or <code>dequeue</code> operation returns
<code>Undefined</code>, which is indistinguishable from an <code>Undefined</code>
that was explicitly pushed onto the stack.
<BR><BR></td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">value</td>
<td class="argType"> <Any></td>
<td class="argDesc" >
Value to add to the queue or stack.
</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type" colspan=2>emsArray.pop(), emsArray.dequeue() : < Any ></td>
</tr>
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> </td>
<td class="Type">emsArray.push() : < Number >
emsArray.enqueue() : < Number ></td>
<td class="Desc">The number of elements presently on the stack or queue</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">comments.push( "Hello, world" ) </td>
<td class="Desc">Append the string to the EMS array <code>comments</code></td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">mostRecent = comments.pop() </td>
<td class="Desc">Atomically return the value
at the top of the stack.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Look up a key used to map a value </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label" style="padding-bottom: 20px;"> CLASS METHOD </td>
<td colspan=3 class="Proto">emsArray.index2key( index )</td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3>
Convert an index into an EMS array to the key used to map
a value to that hashed index. This function can be used
to iterate over all the elements of a mapped array.
<br><br> </td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">index</td>
<td class="argType"> <Number></td>
<td class="argDesc" >
Index of the element in the EMS array to get the key for </td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type">< Any ></td>
<td class="Desc">The key used to map the value which hashed to this index.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Freeing EMS Arrays </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label" style="padding-bottom: 20px;"> CLASS METHOD </td>
<td colspan=3 class="Proto">emsArray.destroy( remove_file )</td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3>
Release the persistent and non-persistent resources associated with an EMS array,
alternatively persisting the data if <code>remove_file</code> is <code>false</code>.
Implies a barrier.
<br><br> </td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">remove_file</td>
<td class="argType"> <Boolean></td>
<td class="argDesc" >
If true, remove the file, else allow the EMS file to persist.</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type"></td>
<td class="Desc">None.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5> Synchronize EMS Memory to Persistent Storage </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label" style="padding-bottom: 20px;"> CLASS METHOD </td>
<td colspan=3 class="Proto">emsArray.sync( [ index [, nElements] ] )</td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3>
Synchronize the EMS memory with persistent storage.
<br><br> </td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">index</td>
<td class="argType"> <String | Number></td>
<td class="argDesc" >
(Optional, default = entire EMS array)
Index of the element in the EMS array to synchronize to disk </td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"></td>
<td class="argName">nElements</td>
<td class="argType"> <Number></td>
<td class="argDesc" >
(Optional, only defined if <code>index</code> is also defined,
default = 1)
Number of sequential indexes, starting with <code>index</code>,
that should be synchronized to disk </td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type">< Boolean ></td>
<td class="Desc">True if memory was successfully synchronized to disk,
otherwise false.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">users.sync( userID )</td>
<td class="Desc">The user's record is committed to disk before the function returns.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5 style='background-color:rgba(100, 0, 0, 0.3);'> TODO Reduce </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> ARRAY METHOD </td>
<td colspan=3 class="Proto">emsArray.reduce( func )<BR><BR></td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Perform a parallel reduction on the elements of an EMS array. <BR><BR></td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">func</td>
<td class="argType"> <Function></td>
<td class="argDesc" > Function to combine this element with the
partial reduction. Function arguments are <code>func(element,
sum)</code>, and the function returns a new <code>sum</code>.
The arguments and results may be of any type.</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type" colspan="2" >< Any ></td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">sum = distances.reduce(
function( val, sum ) {
return(sum+val) } ) </td>
<td class="Desc">Perform the arithmetic sum on the values in the array.</td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="Example">max = distances.reduce(
function( val, currMax ) {
return(val > currMax ?
tmp : currMax) } ) </td>
<td class="Desc">Find the maximum value in the distance array.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5 style='background-color:rgba(100, 0, 0, 0.3);'> TODO: Permute </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> ARRAY METHOD </td>
<td colspan=3 class="Proto">emsArray.permute( order )<BR><BR></td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Reorder the array indexes to the
ordering specified in the
array <code>order</code>. If <code>order</code> contains the same
index more than once, the output for those indexes is
undefined. <BR><BR></td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">order</td>
<td class="argType"> <Array></td>
<td class="argDesc" > Integer permutation array.</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">arr.permute( order ) </td>
<td class="Desc">Reorder the array <code>arr</code>
to the new element ordering.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5 style='background-color:rgba(100, 0, 0, 0.3);'> TODO: Scan </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> ARRAY METHOD </td>
<td colspan=3 class="Proto">emsArray.scan( func )<BR><BR></td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Perform a parallel prefix operation on
the EMS array elements:<br>
<code>a[i] = func(a[i], a[i-1])</code> <BR><BR></td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">func</td>
<td class="argType"> <Function></td>
<td class="argDesc" >Function to combine this element and the
previous element. Function arguments are<br>
<code>func(thisElement, previousElement)</code>,
and returns a new partial result.</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type" colspan="2" >< Array ></td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">partialSums = subtotals.scan(
function(val, sum ) { return(sum+val) } ) </td>
<td class="Desc">Return a EMS array with the partial sums in each element.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5 style='background-color:rgba(100, 0, 0, 0.3);'> TODO: Map </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> ARRAY METHOD </td>
<td colspan=3 class="Proto">emsArray.map( func [, name] )<BR><BR></td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Produce a new EMS array of the same length
by invoking <code>func()</code>
for each element in the array to produce the corresponding element
in the new EMS array.
<BR><BR></td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">func</td>
<td class="argType"> <Function></td>
<td class="argDesc" >Function to perform on each array element. The function is invoked with one argument, the EMS primitive element, and returns a new primitive element.</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName">Persistent filename</td>
<td class="argType"> <String></td>
<td class="argDesc" >(Optional, default is non-persistent)
If the new mapping should have a persistent backing on a filesystem,
this assigns the fully qualified filename.</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type" colspan="2" >< Number | Boolean | String | Undefined ></td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">sizescm = sizesInches.map(
function(inches) {
return(inches * 2.54) } )</td>
<td class="Desc">Scales each element in the array <code>values</code> from inches to cm, producing a new EMS array with scaled values.</td>
</tr>
</table>
<!-- ----------------------------------------------------------------------------- -->
<h5 style='background-color:rgba(100, 0, 0, 0.3);'> TODO: Filter </h5>
<table class="apiBlock" >
<tr class="apiFunc" style="vertical-align:text-top;">
<td class="Label"> ARRAY METHOD </td>
<td colspan=3 class="Proto">emsArray.filter( func [, name] )<BR><BR></td>
</tr>
<tr class="apiSynopsis" style="vertical-align:text-top;">
<td class="Label"> SYNOPSIS </td>
<td class="Desc" colspan=3> Produce a new EMS array containing only the elements
which evaluate as true when passed as the argument to <code>func(element)</code>.
The elements of the new array may appear in any order.
<BR><BR></td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> ARGUMENTS </td>
<td class="argName">func</td>
<td class="argType"> <Function></td>
<td class="argDesc" >Function to perform on each array element. The function is invoked with one argument, the EMS primitive element, and returns a boolean indicating if the element should be included in the new array.</td>
</tr>
<tr class="apiArgs" style="vertical-align:text-top;">
<td class="Label"> </td>
<td class="argName">name</td>
<td class="argType"> <String></td>
<td class="argDesc" >(Optional, default is non-persistent)
If the new mapping requires persistent backing on a filesystem,
the fully qualified filename is defined.</td>
</tr>
</table>
<br>
<table class="apiBlock" >
<tr class="apiRetVal" style="vertical-align:text-top;">
<td class="Label" style="vertical-align:text-top"> RETURNS </td>
<td class="Type" colspan="2" >< EMS Array ></td>
</tr>
<tr class="Examples" style="vertical-align:text-top;">
<td class="Label"> EXAMPLES </td>
<td class="Example">bigParts = allParts.filter(
function(part) {
return(part.size > 100) } ) </td>
<td class="Desc">The new EMS array returned contains only parts whose size is greater than 100.</td>
</tr>
</table>
<hr>
<div style="font-size:.85em;">
This browsing experience is Copyright ©2014-2020,
<span style="font-family:Gotthard;">
<a href="http://mogill.com">
Jace Mogill</a>
</span>.
Proudly designed and built in Cascadia.
<img src="./Flag_of_Cascadia.svg" type="image/svg+xml" height="20px" style="vertical-align:middle;">
</div>
</body>
</html>
================================================
FILE: Examples/Interlanguage/README.md
================================================
# Inter-Language Persistent Shared Memory
In *The Future™*, memory will have new capabilities that
programmers will make decisions about how best to use:
- __Non-Volatility__ - No more "loading" files into variables in memory,
just use the same variables as last time.
- __Sharing__ - Directly sharing objects between programs instead of
communicating them through network messages.
- __Selectable Latency__ - Expensive fast memory for frequently accessed variables,
cheaper slower memory for less frequently used data.
- __Correctness__ - Very inexpensive bulk memory might give approximate values instead
of the exact value stored.
- __Location__ - Allocate memory in another another rack or data center
- __Mirroring__ - Automatic fail-over to backup memory
EMS presently supports the first two and is ready
to incorporate new memory capabilities as they become available.
## Sharing Objects Between Programs
Sharing EMS data between programs written in different languages
is no different than sharing data between programs in the same language,
or sharing data between same program run multiple times.
### Javascript Semantics in Other Languages
Every language has it's own semantics and storage sequences for it's data structures,
EMS makes the various representations interoperable using Javascript's semantics.
That is, if your Python program relies on horrifying semantics like:
```python
>>> a = [1,2,3]
>>> a + "abc"
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: can only concatenate list (not "str") to list
>>> a += "abc"
>>> a
[1, 2, 3, 'a', 'b', 'c']
>>>
```
You will be horrified to learn EMS will instead follow Javascript's horrifying semantics:
```javascript
> a = [1,2,3]
[ 1, 2, 3 ]
> a + 'abc'
'1,2,3abc'
> a += 'abc'
'1,2,3abc'
>
```
You may be nonplussed to learn this is *The Future™*.
I, too, was horrified and baffled at this discovery.
Alas, this is an inevitable truth of *The Future™*.
### Possible Orders for Starting Processes
Two processes sharing memory may start in any order, meaning programs
must take care when initializing to not overwrite another program which
may also be trying to initialize, or detect when it is necessary to
recover from a previous error.
| Persistent EMS Array File | (Re-)Initialize | Use |
| :------------- |:-------------:| :-----:|
| Already exists | A one-time initialization process truncates and creates the EMS array. The first program to start must create the EMS file with the `useExisting : false` attribute | Any number of processes may attach to the EMS array in any order using the attribute `useExisting : true` |
| Does not yet exist | Subsequent programs attach to the new EMS file with the `useExisting : true` attribute | N/A |
# Running the Example
The programs `interlanguage.py` and `interlanguage.js` are complementary
to each other, and must be started in the correct order:
__First start the Javascript program, then start the Python.__
```bash
dunlin> node interlanguage.js
JS Started, waiting for Python to start...
[ EXECUTION STALLS UNTIL PYTHON PROGRAM STARTS ]
Hello from Python
...
```
Second, start the Python program
```bash
dunlin> python interlanguage.py
Start this program after the JS program.
If this Python appears hung and does not respond to ^C, also try ^\ and ^Z
----------------------------------------------------------------------------
Hello from Javascript
...
```
Due to a well-intentioned signal mask in the Python runtime meant to
provide resiliency in the face of a failing native plug-in module,
sometimes Python will appear hung when it is blocked on an EMS operation.
In these situations an alternative way to kill the Python interpreter
is with SIGQUIT (Control-\ from most shells), or back-grounding the
job with Control-Z and sending the job a kill signal.
### What the Example is Doing
- Each process writes a hello message to an EMS variable, marking it full.
- Each process reads the variable written by the other process, blocking until it has been written
- A simple point-to-point barrier is demonstrated by handshaking
- Illustrate some of the differences between Python and Javascript nested objects
- Both JS and Py enter a loop that increments a shared counter
- JS checks the results to make sure every iteration occurred once
- Illustrate some quirks of language design in JS and Py
- JS leaves a shared counter running in the background via `setInterval`
- Python enters a loop that occasionally reads the shared counter, showing it being updated
- Python over-writes the counter with `stop`
- JS notices the counter has been replaced with `stop` and exits.
### Interactive Example
Illustration of sharing data between the Node.js and Python
done interactively from the REPLs.
<img src="../../Docs/ems_js_py.gif" />
================================================
FILE: Examples/Interlanguage/interlanguage.js
================================================
/*-----------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.4.5 |
| Synthetic Semantics http://www.synsem.com/ mogill@synsem.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2017, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-----------------------------------------------------------------------------*/
/*
Start this JS program first, then start the Python program
Possible Orders for Starting Processes
---------------------------------------------
A persistent EMS file already exists
1. A one-time initialization process creates the EMS array
2. Any number of processes my attach to the EMS array in
any order.
A new EMS array will be created
1. The first program to start must create the EMS file with
the "useExisting : false" attribute
2. Subsequent programs attach to the new EMS file with the
"useExisting : true" attribute
*/
"use strict";
// Initialize EMS with 1 process, no thread-CPU affinity,
// "user" mode parallelism, and a unique namespace for EMS runtime
// ("jsExample") to keep the JS program distinct from Python EMS
// program also running.
let ems = require("ems")(1, false, "user", "jsExample");
if (Number(process.version[1]) < 6) {
console.log('Requires Node.js version 6.x or higher -- will not work with polyfill Harmony');
process.exit(1);
}
const maxNKeys = 100;
const bytesPerEntry = 100; // Bytes of storage per key, used for key (dictionary word) itself
let shared = ems.new({
"ES6proxies": true, // Enable native JS object-like syntax
"useExisting": false, // Create a new EMS memory area, do not use existing one if present
"filename": "/tmp/interlanguage.ems", // Persistent EMS array's filename
"dimensions": maxNKeys, // Maximum # of different keys the array can store
"heapSize": maxNKeys * bytesPerEntry,
"setFEtags": "empty", // Set default full/empty state of EMS memory to empty
"doSetFEtags": true,
"useMap": true // Use a key-index mapping, not integer indexes
});
//------------------------------------------------------------------------------------------
// Begin Main Program
console.log("JS Started, waiting for Python to start...");
// One-time initialization should be performed before syncing to Py
shared.writeXF("nestedObj", undefined);
// Initial synchronization with Python
// Write a value to empty memory and mark it full
shared.writeEF("JS hello", "from Javascript");
// Wait for Python to start
console.log("Hello " + shared.readFE("Py hello"));
/*
This synchronous exchange was a "barrier" between the two processes.
The idiom of exchanging messages by writeEF and readFE constitutes
a synchronization point between two processes. A barrier synchronizing
N tasks would require N² variables, which is a reasonable way to
implement a barrier when N is small. For larger numbers of processes
barriers can be implemented using shared counters and the Fetch-And-Add
(FAA) EMS instruction.
The initialization of EMS values as "empty" occurs when the EMS array
was created with:
"setFEtags": "empty",
"doSetFEtags": true,
If it becomes necessary to reset a barrier, writeXE() will
unconditionally and immediately write the value and mark it empty.
*/
// Convenience function to synchronize with another process
// The Python side reverses the barrier names
function barrier(message) {
console.log("Entering Barrier:", message);
shared.writeEF("py side barrier", undefined);
shared.readFE("js side barrier");
console.log("Completed Barrier.");
console.log("---------------------------------------------------");
}
// --------------------------------------------------------------------
barrier("Trying out the barrier utility function");
// --------------------------------------------------------------------
// JS and Python EMS can read sub-objects with the same syntax as native objects,
// but writes are limited to only top level attributes. This corresponds to the
// implied "emsArray.read(key).subobj" performed
shared.top = {};
console.log("Assignment to top-level attributes works normally:", shared.top);
shared.top.subobj = 1; // Does not take effect like top-level attributes
console.log("But sub-object attributes do not take effect. No foo?", shared.top.subobj);
// A nested object can be written at the top level.
shared.nestedObj = {"one":1, "subobj":{"left":"l", "right":"r"}};
console.log("Nested object read references work normally", shared.nestedObj.subobj.left);
// A workaround to operating on nested objects
let nestedObjTemp = shared.nestedObj;
nestedObjTemp.subobj.middle = "m";
shared.nestedObj = nestedObjTemp;
// The explicitly parallel-safe way to modify objects is similar to that "workaround"
nestedObjTemp = shared.readFE("nestedObj");
nestedObjTemp.subobj.front = "f";
shared.writeEF("nestedObj", nestedObjTemp);
// --------------------------------------------------------------------
barrier("This barrier matches where Python is waiting to use nestedObj");
// --------------------------------------------------------------------
// Python does some work now
// Initialize the counter for the next section
shared.writeXF("counter", 0);
// --------------------------------------------------------------------
barrier("JS and Py are synchronized at the end of working with nestedObj");
// --------------------------------------------------------------------
// ---------------------------------------------------------------------------
// Use the EMS atomic operation intrinsics to coordinate access to data
// This is 10x as many iterations as Python in the same amount of time
// because V8's TurboFan compiler kicks in.
for (let count = 0; count < 1000000; count += 1) {
let value = shared.faa("counter", 1);
if (count % 100000 == 0) {
console.log("JS iteration", count, " Shared Counter=", value)
}
}
// --------------------------------------------------------------------
barrier("Waiting for Py to finish it's counter loop");
// --------------------------------------------------------------------
console.log("The shared counter should be 11000000 ==", shared.counter);
// ---------------------------------------------------------------------------
// Ready to earn your Pro Card?
// Wait for Py to initialize the array+string
barrier("Wait until it's time to glimpse into the future of Javascript");
console.log("The array+string from Py:", shared.arrayPlusString);
barrier("Waiting for shared.arrayPlusString to be initialized");
console.log("JS can do array+string, it produces a string:", shared.arrayPlusString + " world!");
shared.arrayPlusString += " RMW world!";
console.log("JS performing \"array + string\" produces a self-consistent string:", shared.arrayPlusString);
barrier("Arrive at the future.");
// ---------------------------------------------------------------------------
// Leave a counter running on a timer
// First re-initialize the counter to 0
shared.writeXF("counter", 0);
function incrementCounter() {
// Atomically increment the shared counter
let oldValue = shared.faa("counter", 1);
// If the counter has been set to "Stop", clear the timer interval
if (oldValue == "stop") {
console.log("Timer counter was signaled to stop");
clearInterval(timerCounter);
}
// Show counter periodically
if (oldValue % 100 == 0) {
console.log("Counter =", shared.counter);
}
}
var timerCounter = setInterval(incrementCounter, 10); // Increment the counter every 1ms
barrier("Welcome to The Future™!");
// Fall into Node.js event loop with Interval Timer running
================================================
FILE: Examples/Interlanguage/interlanguage.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
+-----------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.4.5 |
| Synthetic Semantics http://www.synsem.com/ mogill@synsem.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2017, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-----------------------------------------------------------------------------+
"""
"""
tl;dr Start the Javascript first, then the Python
--------------------------------------------------
See interlanguage.js for more detailed description of starting processes
sharing EMS memory
NOTE ABOUT "STUCK" JOBS":
Python runtime can get "stuck" and may not respond to Control-C
but can still be interrupted with Control-Z or Control-\
This "stuck" behavior is due to how the Python/C interface protects
the Python runtime.
"""
import time
import sys
import random
sys.path.append("../../Python/")
import ems
# Initialize EMS: 1 process, no thread-core affinity, user provided parallelism
# Initialize EMS with 1 process, no thread-CPU affinity,
# "user" mode parallelism, and a unique namespace for EMS runtime
# ("pyExample") to keep the JS program distinct from Javascript EMS
# program also running.
ems.initialize(1, False, "user", "pyExample")
# The EMS array attributes must be the same by all programs sharing the array,
# with the exception of the the initial file creation which uses "useExisting:False"
# instead of "useExisting:True"
maxNKeys = 100
bytesPerEntry = 100 # Bytes of storage per key, used for key (dictionary word) itself
shared = ems.new({
'useExisting': True, # Connect the EMS memory created by JS, do not create a new memory
'filename': "/tmp/interlanguage.ems", # Persistent EMS array's filename
'dimensions': maxNKeys, # Maximum # of different keys the array can store
'heapSize': maxNKeys * 100, # 100 bytes of storage per key, used for key (dictionary word) itself
'useMap': True # Use a key-index mapping, not integer indexes
})
# ------------------------------------------------------------------------------------------
# Begin Main Program
print """Start this program after the JS program.
If this Python appears hung and does not respond to ^C, also try ^\ and ^Z
----------------------------------------------------------------------------
"""
# Initial synchronization with JS
# Write a value to empty memory and mark it full
shared.writeEF("Py hello", "from Python")
# Wait for JS to start
print "Hello ", shared.readFE("JS hello")
def barrier(message):
"""Define a convenience function to synchronize with another process
The JS side reverses the barrier names"""
global shared
naptime = random.random() + 0.5
time.sleep(naptime) # Delay 0.5-1.5sec to make synchronization apparent
print "Entering Barrier:", message
shared.writeEF("js side barrier", None)
shared.readFE("py side barrier")
print "Completed Barrier after delaying for", naptime, "seconds"
print "------------------------------------------------------------------"
print "Trying out the new barrier by first napping for 1 second..."
time.sleep(1)
# ---------------------------------------------------------------------
barrier("Trying out the barrier utility function")
# ---------------------------------------------------------------------
# --------------------------------------------------------------------
# This barrier matches where JS has set up "shared.top", and is now waiting
# for Python to finish at the next barrier
barrier("Finished waiting for JS to finish initializing nestedObj")
print "Value of shared.nestedObj left by JS:", shared.nestedObj
try:
shared.nestedObj.number = 987 # Like JS, setting attributes of sub-objects will not work
print "ERROR -- This should not work"
exit()
except:
print "Setting attributes of sub-objects via native syntax will not work"
print "These two expressions are the same:", \
shared["nestedObj"].read()["subobj"]["middle"], "or", \
shared.read("nestedObj")["subobj"]["middle"]
# --------------------------------------------------------------------
barrier("JS and Py are synchronized at the end of working with nestedObj")
# --------------------------------------------------------------------
# Enter the shared counter loop
for count in xrange(100000):
value = shared.faa("counter", 1)
if count % 10000 == 0:
print "Py iteration", count, " Shared counter=", value
barrier("Waiting for Js to finish it's counter loop")
# --------------------------------------------------------------------
# Ready to earn your Pro Card?
# Make JS wait while Python demonstrates a "create approach to language design"
shared.arrayPlusString = [1, 2, 3] # Initialize arrayPlusString before starting next phase
barrier("Wait until it's time to glimpse into the future of Python")
try:
dummy = shared.arrayPlusString + "hello"
print "ERROR -- Should result in: TypeError: can only concatenate list (not \"str\") to list"
exit()
except:
print "Adding an array and string is an error in Python"
shared.arrayPlusString += "hello"
print "However 'array += string' produces an array", shared.arrayPlusString
# Let JS know 'shared.arrayPlusString' was initialized
barrier("shared.arrayPlusString is now initialized")
# JS does it's work now, then wait for JS to do it's work
barrier("Arrive at the future.")
# JS does all the work in this section
barrier("Welcome to The Future™!")
# --------------------------------------------------------------------
# JS is now in it's event loop with the counter running
for sampleN in xrange(10):
time.sleep(random.random())
print "Shared counter is now", shared.counter
# Restart the counter
shared.counter = 0
for sampleN in xrange(5):
time.sleep(random.random() + 0.5)
print "Shared reset counter is now", shared.counter
# Wait 1 second and stop the JS timer counter
time.sleep(1)
shared.counter = "stop"
# Show the counter has stopped. Last value = "stop" + 1
for sampleN in xrange(5):
time.sleep(random.random() + 0.5)
print "Shared reset counter should have stopped changing:", shared.counter
print "Exiting normally."
================================================
FILE: Examples/KeyValueStore/README.md
================================================
# Key-Value Store Example
This example implements a simple REST server, the API is a 1-to-1
mapping of REST commands to EMS commands,
effectively making the EMS shared memory model
an object store service over the network.
The RESTful API inherits all the atomicity
and persistence properties of the underlying EMS array.
## Parallelism & Deadlocks
The HTTP web interface used is Node's built-in HTTP module which
implements a serial event loop to process HTTP requests one at
time in a synchronous (blocking) fashion.
The execution time of EMS operations is similar to any other variable reference,
making EMS most efficient as a synchronous programming model,
however, because a `readFE` request may block the Node event loop
clients susceptible to deadlock.
A `writeEF` request that would unblock execution cannot handled
by the single-threaded Node HTTP server that is blocked on a
`readFE`, effectively deadlocking the system.
The two potential solutions, asynchronous execution and multi-processing,
are complementary forms of parallelism that may be used
to address the deadlock problem:
- __Asynchronous Execution__:<br>
Blocking calls are turned into continuations that will be scheduled
for execution again later.
The number of in-flight operations before potential deadlock is equal
to the number of continuations the runtime implements.
<br>
This is not presently implemented, and would require the addition
of new non-blocking variants of EMS functions, such as
`nbReadFE()` and `nbWriteEF()`.
- __Multi-Processing__:<br>
Events are distributed across multiple single-threaded event loops.
Idle threads are assigned new events as they arrive, and each event
loop completes an event before beginning a new one. The number of
in-flight operations before potential deadlock is equal to the number
of processes.
<br>
This is implemented using Node's built-in [Cluster](https://nodejs.org/api/cluster.html) module.
## Starting the EMS Server process
Using Node's built-in `cluster` module that forks multiple Node.js
processes, we implement the multiple event loops needed for multi-processing.
Node's built-in `http` module provides the serial HTTP event loop which is
then parallelized using the `cluster` module.
`cluster` manages all child process creation, destruction, and communication.
Starting the web server from the command line:
```bash
node kv_store.js [# processes]
```
If the number of processes is not specified, 8 are started.
## Manually Sending Commands using CURL
An EMS array is presented as a Key-Value store with EMS memory operations
presented as a REST interface.
An ordinary Node.js Cluster is started to demonstrate parallelism outside of EMS.
```bash
node kv_store.js 8 # Start 8 servers, logs will appear on this terminal
```
```bash
dunlin> curl localhost:11080/writeXF?foo=Hello # Unconditionally write "Hello" to the key "foo" and mark Full
dunlin> curl localhost:11080/faa?foo=+World! # Fetch And Add returns original value: "Hello"
"Hello"
dunlin> curl localhost:11080/readFF?foo # Read Full and leave Full, final value is "Hello World!"
"Hello World!"
dunlin>
```
## Producer-Consumer
<table>
<tr>
<td width="50%">
```bash
dunlin> curl localhost:11080/readFE?bar # EMS memory initialized as empty, this blocks
```
</td>
<td>
```bash
dunlin> curl localhost:11080/readFF?foo # Read Full and leave Full, final value is "Hello World!"
```
</td>
</tr>
</table>
================================================
FILE: Examples/KeyValueStore/kv_store.js
================================================
/*-----------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.4.5 |
| Synthetic Semantics http://www.synsem.com/ mogill@synsem.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2011-2014, Synthetic Semantics LLC. All rights reserved. |
| Copyright (c) 2015-2017, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-----------------------------------------------------------------------------*/
'use strict';
var ems = require('ems')(1, false, 'user');
var http = require('http');
var url_module = require("url");
var cluster = require('cluster');
var port = 8080;
var shared_counters;
var persistent_data;
var myID;
var persistent_data_options = {
dimensions: [100000],
heapSize: [10000000],
useExisting: false,
useMap: true,
filename: '/tmp/persistent_shared_web_data.ems'
};
var shared_counters_options = {
dimensions: [100],
heapSize: [10000],
useExisting: false,
useMap: true,
filename: '/tmp/counters.ems'
};
function handleRequest(request, response) {
var requestN = shared_counters.faa('nRequests', 1); // Increment the count of requests
var parsed_request = url_module.parse(request.url, true);
var key = Object.keys(parsed_request.query)[0];
var value = parsed_request.query[key];
if (value === '') { value = undefined; }
var data;
var opname;
if (!key) {
response.end("ERROR: No key specified in request");
return;
}
if (parsed_request.pathname.indexOf("/readFE") >= 0) {
data = persistent_data.readFE(key);
opname = 'readFE';
} else if (parsed_request.pathname.indexOf("/readFF") >= 0) {
data = persistent_data.readFF(key);
opname = 'readFF';
} else if (parsed_request.pathname.indexOf("/read") >= 0) {
data = persistent_data.read(key);
opname = 'read';
} else if (parsed_request.pathname.indexOf("/writeXE") >= 0) {
data = persistent_data.writeXE(key, value);
opname = 'writeXE';
} else if (parsed_request.pathname.indexOf("/writeXF") >= 0) {
data = persistent_data.writeXF(key, value);
opname = 'writeXF';
} else if (parsed_request.pathname.indexOf("/writeEF") >= 0) {
data = persistent_data.writeEF(key, value);
opname = 'writeEF';
} else if (parsed_request.pathname.indexOf("/write") >= 0) {
data = persistent_data.write(key, value);
opname = 'write';
} else if (parsed_request.pathname.indexOf("/faa") >= 0) {
data = persistent_data.faa(key, value);
opname = 'faa';
} else if (parsed_request.pathname.indexOf("/cas") >= 0) {
opname = 'cas';
var old_new_vals = value.split(',');
if (old_new_vals[0] === '') { old_new_vals[0] = undefined; }
if (old_new_vals[1] === '') { old_new_vals[1] = undefined; }
data = persistent_data.cas(key, old_new_vals[0], old_new_vals[1]);
} else {
data = "ERROR: No EMS command specified.";
}
var datastr = JSON.stringify(data);
console.log("Request #" + requestN + ", slave process " + myID + ": Op(" +
opname + ") key(" + key + ") val(" + value + ") data=" + datastr);
response.end(datastr);
}
if (cluster.isMaster) {
/* The Master Process creates the EMS arrays that the slave processes
* will attach to before creating the slave processes. This prevents
* parallel hazards if all slaves tried creating the EMS arrays.
*/
persistent_data_options.setFEtags = 'empty';
persistent_data_options.doSetFEtags = true;
persistent_data = ems.new(persistent_data_options);
shared_counters_options.dataFill = 0;
shared_counters_options.doDataFill = true;
shared_counters_options.setFEtags = 'full';
shared_counters_options.doSetFEtags = true;
shared_counters = ems.new(shared_counters_options);
// Seed the GUID generator with a unique starting value
shared_counters.writeXF('GUID', Math.floor(Math.random() * 10000000));
// All the one-time initialization is complete, now start slave processes.
// The number of processes is the limit of the number of pending requests
// before deadlock occurs.
var nProcesses = Number(process.argv[2]) || 8; // Number of "cluster" processes to start
for (var procnum = 0; procnum < nProcesses; procnum++) {
cluster.fork();
}
console.log(
"REST API: <EMSCommand>?key[=value]\n" +
" EMSCommand: read, readFE, readFF, writeXE, writeXF, writeEF, faa, cas\n" +
" key: The index into the EMS array, must be a valid JSON element\n" +
" value: For write commands, the value stored in EMS.\n" +
" For CAS, the value is the old and new values separated by a comma:\n" +
" /cas?key=oldval,newval\n");
cluster.on('exit', function (worker, code, signal) {
console.log("worker " + worker.process.pid + "died");
});
} else {
/* Each Slave Cluster processes must connect to the EMS array created
* by the master process.
*/
persistent_data_options.useExisting = true;
persistent_data = ems.new(persistent_data_options);
shared_counters_options.useExisting = true;
shared_counters = ems.new(shared_counters_options);
myID = shared_counters.faa('myID', 1);
http.createServer(handleRequest).listen(port, function () {
console.log("Server " + myID + " listening on: http://localhost:" + port);
});
}
================================================
FILE: Examples/README.md
================================================
# Extended Memory Semantics (EMS) Examples
#### Table of Contents
* [Simple Loop Benchmarks](#Simple-Loop-Benchmarks)
* [Key-Value Store](#kv_store.js)
* [Work Queues](#Work-Queues)
* [Parallel Web Server](#web_server.js)
* [Word Counting](#Word-Count)
* [Implied EMS Operations](#harmony_proxies.js)
* [Inter-Language Programming](#Inter-language-Programming)
## Simple Loop Benchmarks
Click here for a __[Detailed Description of the EMS<nolink>.js STREAMS implementation](https://github.com/SyntheticSemantics/ems/tree/master/Examples/STREAMS)__.
The original [STREAMS](https://www.cs.virginia.edu/stream/)
benchmark attempts to measure the _Speed Not To Exceed_
bandwidth of a CPU to it's attached RAM by performing large
numbers of simple operations on an array of integers or floats.
This version of STREAMS for EMS<nolink>.js measures the rate of different atomic operations on
shared integer data, it is implemented twice using two different parallelization schemes:
Bulk Synchronous Parallel (BSP), and Fork-Join (FJ).
Experimental results are from the BSP version
on an AWS `c4.8xlarge (132 ECUs, 36 vCPUs, 2.9 GHz, Intel Xeon E5-2666v3, 60 GiB memory`.
<center><img src="../Docs/streams.svg" type="image/svg+xml" height="360px">
</center>
## Parallelism Within a Single HTTP Request
Click here for the [Web Server Parallelism Example](https://github.com/SyntheticSemantics/ems/tree/master/Examples/WebServer)
<img src="../Docs/parWebServer.svg" type="image/svg+xml">
EMS is a memory model and has no daemon process to monitor a network interface
for operations to execute.
This example builds on a standard HTTP server to implement
shared memory parallelism within a single web request.
## EMS as a Key Value Store
Click here for the [EMS Key-Value Store Example](https://github.com/SyntheticSemantics/ems/tree/master/Examples/KeyValueStore)
An EMS array is used as a Key-Value store with a 1-to-1 mapping of REST operations
to EMS memory operations.
The KV server is implemented using the Node.js built-in `Cluster` module to
demonstrate parallelism sources outside of EMS.
## Work Queues
The parameters of the experiment are defined in the `initializeSharedData()` function.
An experiment creates several new EMS arrays and a work queue of
randomly generated transactions that update one or more of the arrays.
Parallel processes dequeue the work and perform atomic updates on
the new EMS arrays. Finally, the experiment performs checksums to ensure
all the transactions were performed properly.
```javascript
arrLen = 1000000; // Maximum number of elements the EMS array can contain
heapSize = 100000; // Amount of persistent memory to reserve for transactions
nTransactions = 1000000; // Number of transactions to perform in the experiment
nTables = 6; // Number of EMS arrays to perform transactions across
maxNops = 5; // Maximum number of EMS arrays to update during a transaction
```
### `concurrent_Q_and_TM.js`
```bash
node concurrent_Q_and_TM.js 4 # 1 process enqueues then processes work, 3 processes perform work
````
All processes are started, one enqueues transactions while the others dequeue
and perform the work. When all the work has been enqueued, that process also
begins dequeing and performing work.
<img style="vertical-align:text-top;" src="../Docs/tm_from_q.svg" />
### `workQ_and_TM.js`
```bash
node workQ_and_TM.js 4 # Enqueue all the work, all 4 processes deqeue and perform work
````
All the transactions are enqueued, then all the processes begin
to dequeue and perform work until no work remains in the queue.
## [Word Counting](#Word-Count)
Map-Reduce is often demonstrated using word counting because each document can
be processed in parallel, and the results of each document's dictionary reduced
into a single dictionary.
In contrast with Map-Reduce implementations using a
dictionary per document, the EMS implementation
maintains a single shared dictionary, eliminating the need for a reduce phase.
The word counts are sorted in parallel and the most frequently appearing words
are printed with their counts.
### Word Counting Using Atomic Operations
A parallel word counting program is implemented in Python and Javascript.
A shared dictionary is used by all the tasks, updates are made using
atomic operations. Specifically, `Compare And Swap (CAS)` is used to
initialize a new word from undefined to a count of 1. If this CAS
fails, the word already exists in the dictionary and the operation
is retried using `Fetch and Add`, atomically incrementing the count.
### Forming the "Bag of Words" with Word Counts
```javascript
var file_list = fs.readdirSync(doc_path);
var splitPattern = new RegExp(/[ \n,\.\\/_\-\<\>:\;\!\@\#\$\%\&\*\(\)=\[\]|\"\'\{\}\?\—]/);
// Iterate over all the files in parallel
ems.parForEach(0, file_list.length, function (fileNum) {
var text = fs.readFileSync(doc_path + file_list[fileNum], 'utf8', "r");
var words = text.replace(/[\n\r]/g, ' ').toLowerCase().split(splitPattern);
// Sequentially iterate over all the words in one document
words.forEach(function (word) {
wordCounts.faa(word, 1); // Atomically increment the count of times this word was seen
});
});
```
### Sorting the Word Count list by Frequency
Parallel sorting of the word count is implemented as a multi-step process:
1. The bag of words is processed by all the processess, each process
building an ordered list of the most common words it encounters
2. The partial lists from all the processes are concatenated into a single list
3. The list of the most common words seen is sorted and truncated
```javascript
var biggest_counts = new Array(local_sort_len).fill({"key": 0, "count": 0});
ems.parForEach(0, maxNKeys, function (keyN) {
var key = wordCounts.index2key(keyN);
if (key) {
// Perform an insertion sort of the new key into the biggest_counts
// list, deleting the last (smallest) element to preserve length.
var keyCount = wordCounts.read(key);
var idx = local_sort_len - 1;
while (idx >= 0 && biggest_counts[idx].count < keyCount) { idx -= 1; }
var newBiggest = {"key": key, "count": keyCount};
if (idx < 0) {
biggest_counts = [newBiggest].concat(biggest_counts.slice(0, biggest_counts.length - 1));
} else if (idx < local_sort_len) {
var left = biggest_counts.slice(0, idx + 1);
var right = biggest_counts.slice(idx + 1);
biggest_counts = left.concat([newBiggest].concat(right)).slice(0, -1);
}
}
});
// Concatenate all the partial (one per process) lists into one list
stats.writeXF('most_frequent', []); // Initialize the list
ems.barrier(); // Wait for all procs to have finished initialization
stats.writeEF('most_frequent', stats.readFE('most_frequent').concat(biggest_counts));
ems.barrier(); // Wait for all procs to have finished merge
ems.master(function() { // Sort & print the list of words, only one process is needed
biggest_counts = stats.readFF('most_frequent');
biggest_counts.sort(function (a, b) { return b.count - a.count; });
// Print only the first "local_sort_len" items -- assume the worst case
// of all the largest counts are discovered by a single process
console.log("Most frequently appearing terms:");
for (var index = 0; index < local_sort_len; index += 1) {
console.log(index + ': ' + biggest_counts[index].key + " " + biggest_counts[index].count);
}
});
```
### Word Count Performance
This section reports the results of running the Word Count example on
documents from Project Gutenberg.
2,981,712,952 words in several languages were parsed, totaling 12,664,852,220 bytes of text.
<img height="300px" src="../Docs/wordcount.svg" />
## Inter-Language Programming
[Inter-Language Shared Memory Programming](https://github.com/SyntheticSemantics/ems/tree/master/Examples/Interlanguage)
The programs `interlanguage.js` and `interlanguage.py` demonstrate sharing
objects between Javascript and Python.
A variety of synchronization and execution models are shown.
### Example
<img src="../Docs/ems_js_py.gif" />
###### Copyright (C)2011-2019 Jace A Mogill
================================================
FILE: Examples/STREAMS/README.md
================================================
# EMS.js STREAMS Bandwidth Benchmark
The original [STREAMS](https://www.cs.virginia.edu/stream/)
benchmark measures the _Speed Not To Exceed_
bandwidth of a CPU to it's attached RAM by performing large
numbers of simple operations. This version for
EMS<nolink>.js measures the rate of different atomic operations on
shared integer data and is implemented using two different parallelization schemes:
Bulk Synchronous Parallel (BSP), and Fork-Join.
## Parallel Execution Models
The EMS.js STREAMS benchmark is implemented twice, once using Bulk Synchronous Parallelism
and again using Fork-Join parallelism.
BSP goes parallel at program startup,
decomposes loop iterations across processes, and performs a barrier at the end
of each loop. FJ starts serial and goes parallel for each loop, joining back
to serial execution before the next loop.
<table>
<tr>
<td width="50%">
<center>
<img src="../../Docs/ParallelContextsBSP.svg" type="image/svg+xml" height="500px">
</center>
</td>
<td width="50%">
<center>
<img src="../../Docs/ParallelContextsFJ.svg" type="image/svg+xml" height="500px">
</td>
</tr>
</table>
## Running the Benchmark
To run either version of STREAMS benchmark, specify the number of processes
as an argument to the program:
```bash
node streams_bulk_sync_parallel.js 8
```
Running the BSP version of the STREAMS benchmark reports the rate at
which each loop is performed, each loop exercising a single
EMS read and write primitive.
A second set of "Native" loops performs similar operations on ordinary
volatile private memory, taking advantage of all possible native
optimizations and just-in-time compilation.
These results show a performance difference of 20-50x
accessing EMS memory versus native memory,
which is still very favorable compared to the 1000-5000x difference accessing
a remote key-value store, or worse accessing magnetic media.
```bash
dunlin> time node streams_bulk_sync_parallel.js 3
EMStask 0: 1,000,000 write 6,896,551 ops/sec
EMStask 0: 1,000,000 writeXE 7,246,376 ops/sec
EMStask 0: 1,000,000 writeXF 7,246,376 ops/sec
EMStask 0: 1,000,000 read 7,042,253 ops/sec
EMStask 0: 1,000,000 reread 6,896,551 ops/sec
EMStask 0: 1,000,000 readFF 7,299,270 ops/sec
EMStask 0: 1,000,000 readFE 7,194,244 ops/sec
EMStask 0: 1,000,000 writeEF 7,299,270 ops/sec
EMStask 0: 1,000,000 copy 3,521,126 ops/sec
EMStask 0: 1,000,000 recopy 3,484,320 ops/sec
EMStask 0: 1,000,000 c=a*b 2,364,066 ops/sec
EMStask 0: 1,000,000 c+=a*b 1,715,265 ops/sec
EMStask 0: 1,000,000 checksum Infinity ops/sec
------------------------ START NATIVE ARRAY
EMStask 0: 1,000,000 write 333,333,333 ops/sec
EMStask 0: 1,000,000 rd/write 250,000,000 ops/sec
EMStask 0: 1,000,000 read 125,000,000 ops/sec
EMStask 0: 1,000,000 reread 250,000,000 ops/sec
EMStask 0: 1,000,000 copy 200,000,000 ops/sec
EMStask 0: 1,000,000 rmwcopy 250,000,000 ops/sec
EMStask 0: 1,000,000 c=a*b 100,000,000 ops/sec
EMStask 0: 1,000,000 c+=a*b 142,857,142 ops/sec
EMStask 0: 1,000,000 sum Infinity ops/sec
real 0m2.938s
user 0m8.190s
sys 0m0.172s
```
__NOTE:__ Adding timing instrumentation to the Fork-Join version is
left as an exercise to the reader.
## Benchmark Results
Benchmark results for the BSP implementation on an AWS
`c4.8xlarge (132 ECUs, 36 vCPUs, 2.9 GHz, Intel Xeon E5-2666v3, 60 GiB memory`.
<center><img src="../../Docs/streams.svg" type="image/svg+xml" height="300px">
</center>
================================================
FILE: Examples/STREAMS/streams_bulk_sync_parallel.js
================================================
/*-----------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.0.0 |
| Synthetic Semantics http://www.synsem.com/ mogill@synsem.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2011-2014, Synthetic Semantics LLC. All rights reserved. |
| Copyright (c) 2015-2016, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-----------------------------------------------------------------------------*/
//
// Based on John D. McCalpin's (Dr. Bandwdith) STREAMS benchmark:
// https://www.cs.virginia.edu/stream/
//
//
// Also see the Fork-Join version in the Examples directory
// ===============================================================================
var ems = require('ems')(parseInt(process.argv[2]), false);
var arrLen = 1000000;
var a = ems.new(arrLen);
var b = ems.new(arrLen);
var c = ems.new(arrLen);
var x = new Array(arrLen);
var y = new Array(arrLen);
var z = new Array(arrLen);
//-------------------------------------------------------------------
// Timer function
function stopTimer(timer, nOps, label) {
function fmtNumber(n) {
var s = ' ' + n.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
if(n < 1) return n;
else { return s.substr(s.length - 15, s.length); }
}
ems.master( function() {
var now = new Date().getTime();
var x = (nOps*1000000) / ((now - timer) *1000);
ems.diag(fmtNumber(nOps) + label + fmtNumber(Math.floor(x).toString()) + " ops/sec");
} );
}
var timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { a.write(idx, idx) } );
stopTimer(timeStart, arrLen, " write ");
timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { a.writeXE(idx, idx) } );
stopTimer(timeStart, arrLen, " writeXE ");
timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { a.writeXF(idx, idx) } );
stopTimer(timeStart, arrLen, " writeXF ");
timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { a.read(idx, idx) } );
stopTimer(timeStart, arrLen, " read ");
timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { a.read(idx, idx) } );
stopTimer(timeStart, arrLen, " reread ");
timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { a.readFF(idx, idx) } );
stopTimer(timeStart, arrLen, " readFF ");
timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { a.readFE(idx, idx) } );
stopTimer(timeStart, arrLen, " readFE ");
timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { a.writeEF(idx, idx) } );
stopTimer(timeStart, arrLen, " writeEF ");
timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { b.writeXF(idx, a.readFF(idx)) } );
stopTimer(timeStart, arrLen, " copy ");
timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { b.writeXF(idx, a.readFF(idx)) } );
stopTimer(timeStart, arrLen, " recopy ");
timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { c.writeXF(idx, a.readFF(idx) * b.readFF(idx)) } );
stopTimer(timeStart, arrLen, " c=a*b ");
timeStart = new Date().getTime();
ems.parForEach(0, arrLen, function(idx) { c.writeXF(idx, c.readFF(idx) + (a.readFF(idx) * b.readFF(idx))) } );
stopTimer(timeStart, arrLen, " c+=a*b ");
timeStart = new Date().getTime();
stopTimer(timeStart, arrLen, " checksum ");
//===========================================================================================
var idx;
if(ems.myID == 0) {
console.log('------------------------ START NATIVE ARRAY');
timeStart = new Date().getTime();
for(idx = 0; idx < arrLen; idx++ ) { x[idx] = idx }
stopTimer(timeStart, arrLen, " write ");
timeStart = new Date().getTime();
for(idx = 0; idx < arrLen; idx++ ) { x[idx] += idx }
stopTimer(timeStart, arrLen, " rd/write");
timeStart = new Date().getTime();
var dummy = 0;
for(idx = 0; idx < arrLen; idx++ ) { dummy += x[idx] }
stopTimer(timeStart, arrLen, " read ");
timeStart = new Date().getTime();
for(idx = 0; idx < arrLen; idx++ ) { dummy += x[idx] }
stopTimer(timeStart, arrLen, " reread ");
timeStart = new Date().getTime();
for(idx = 0; idx < arrLen; idx++ ) { y[idx] = x[idx] }
stopTimer(timeStart, arrLen, " copy ");
timeStart = new Date().getTime();
for(idx = 0; idx < arrLen; idx++ ) { y[idx] += x[idx] }
stopTimer(timeStart, arrLen, " rmwcopy ");
timeStart = new Date().getTime();
for(idx = 0; idx < arrLen; idx++ ) { z[idx] = x[idx] * y[idx] }
stopTimer(timeStart, arrLen, " c=a*b ");
timeStart = new Date().getTime();
for(idx = 0; idx < arrLen; idx++ ) { z[idx] += x[idx] * y[idx] }
stopTimer(timeStart, arrLen, " c+=a*b ");
timeStart = new Date().getTime();
stopTimer(timeStart, arrLen, " sum ");
}
================================================
FILE: Examples/STREAMS/streams_fork_join.js
================================================
/*-----------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.0.0 |
| Synthetic Semantics http://www.synsem.com/ mogill@synsem.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2011-2014, Synthetic Semantics LLC. All rights reserved. |
| Copyright (c) 2015-2016, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-----------------------------------------------------------------------------*/
//
// Based on John D. McCalpin's (Dr. Bandwdith) STREAMS benchmark:
// https://www.cs.virginia.edu/stream/
//
// Also see the Bulk Synchronous Parallel implementation in the Examples directory
// ===============================================================================
'use strict';
var ems = require('ems')(parseInt(process.argv[2]), true, 'fj');
ems.parallel( function() {
var arrLen = 1000000;
var array_a = ems.new(arrLen);
var array_b = ems.new(arrLen);
var array_c = ems.new(arrLen);
ems.parForEach(0, arrLen, function(idx) { array_a.write(idx, idx)});
ems.parForEach(0, arrLen, function(idx) { array_a.writeXE(idx, idx)});
ems.parForEach(0, arrLen, function(idx) { array_a.writeXF(idx, idx)});
ems.parForEach(0, arrLen, function(idx) { array_a.read(idx, idx)});
ems.parForEach(0, arrLen, function(idx) { array_a.readFF(idx, idx)});
ems.parForEach(0, arrLen, function(idx) { array_a.readFE(idx, idx)});
ems.parForEach(0, arrLen, function(idx) { array_a.writeEF(idx, idx)});
ems.parForEach(0, arrLen, function(idx) { array_b.writeXF(idx, array_a.readFF(idx))});
ems.parForEach(0, arrLen, function(idx) { array_c.writeXF(idx, array_a.readFF(idx) * array_b.readFF(idx))});
ems.parForEach(0, arrLen, function(idx) { array_c.writeXF(idx, array_c.readFF(idx) + (array_a.readFF(idx) * array_b.readFF(idx)))});
} );
// Fork-Join tasks must exit themselves, otherwise they will wait indefinitely for more work
process.exit(0);
================================================
FILE: Examples/WebServer/README.md
================================================
# Parallelism Generating a Single HTTP Response
Using Fork-Join (FJ) parallelism a conventional
HTTP server can use multiple cores to generate
a single a response by forking threads
when a request is received, generating the response in
parallel, joining all the threads when the work is complete,
and then returning the response from the original single thread.
HTTP applications frequently require state to persist between requests,
this example uses the URL path as a session ID that can be used to
persist state between requests. Each request itself may include
one or more fork-join parallel regions.
## Parallel Web Server Architecture
This example combines Node.js' built-in HTTP server module
with EMS for shared memory and task level parallelism.
When the server is started it allocates a globally shared
EMS shared memory (`serverState.ems`) that exists until the server exits.
The global shared memory is used for diagnostic counters like the
number of queries as well as key-value pairs that can be read
or written from a parallel or serial region of any request or session.
<img src="../../Docs/parWebServer.svg" type="image/svg+xml">
## REST API
A valid request URL consists of path naming the session ID
and a query that specifies if the session is new/restarting, or
is a continuation of a previous session.
__REST API:__ `/sessionID?<old|new>&<keep|free>`
- `sessionID` - An unique session identifier. The shared memory
persistent file will be named `session.sessionID.ems` and stored
in the current working directory.
- `?<old|new>` - A `new` session will remove an existing
session-private shared memory if present, an `old` session
will attach to a previously created shared memory that was
persisted to disk.
- `?<keep|free>` - Release or preserve, respectively, the
session's shared memory at the end of the request.
## Fork-Join Parallelism
A process using Fork-Join parallelism starts single-threaded,
creating parallel regions when needed, then returning to
single-threaded execution when all the threads in the
parallel region have completed execution.
EMS amortizes the cost of starting new processes by using a thread pool
where idle threads can quickly be restarted when they are needed again.
By way of comparison, all the processes in Bulk Synchronous Parallelism
are always executing, barriers can be used to synchronize execution.
<table>
<tr>
<td>
<center>
Fork-Join Parallelism<br>
<img src="../../Docs/tasksAndLoopsFJ.svg" type="image/svg+xml" height="360px">
</center>
</td>
<td>
<center>
Bulk Synchronous Parallelism<br>
<img src="../../Docs/tasksAndLoopsBSP.svg" type="image/svg+xml" height="360px">
</center>
</td>
</tr>
</table>
## Web Server Process
The web server is a conventional single-threaded based on Node.js'
built-in HTTP module. As part of the single-threaded initialization
EMS is initialized for Fork-Join parallelism and a shared memory
is created for the processes to share diagnostic counters and other
common state. This shared memory lasts for the life of the web
server process and does not persist between invocations of the server.
When a request is received, the URL is parsed, then the server enters
a parallel region. From the parallel region each process creates or
attaches to shared memory created
## Example
Start the server:
`node webServer.js [# of processes, default=8]`
From another terminal or browser, issue requests to the server.
A parallel region is created for each request received by the web server,
handling the single request with multiple processes.
Different variables in the response are used to illustrate
different types of data updates.
- `requestN` - Every request is given an unique ID
- `sessionData` - Each process records it's "work" in the private EMS
memory allocated for the session (`foo` or `bar`).
The results in the form
`(Req:<Request Number>/<EMS ProcessID>)`, the processes may complete
in any order, but each request is handled sequentially.
- `bignum` - The sum of a loop incrementing a counter in session
private EMS memory
- `randomSum` - The sum of random numbers generated by all the processes,
stored in the shared global EMS memory
Following the join at the end of the parallel region,
single-threaded execution continues, copying the different
variables into a response object
```
dunlin> curl 'http://localhost:8080/foo?new&keep' # Create a new persistent memory for session "foo"
{"requestN":0,"sessionData":"0(Req:0/2) (Req:0/7) (Req:0/1) (Req:0/4) (Req:0/5) (Req:0/3) (Req:0/0) (Req:0/6) ","bignum":100000,"randomSum":6103}
dunlin> curl 'http://localhost:8080/foo?old&keep' # Modify the existing "foo" memory
{"requestN":1,"sessionData":"0(Req:0/2) (Req:0/7) (Req:0/1) (Req:0/4) (Req:0/5) (Req:0/3) (Req:0/0) (Req:0/6) (Req:1/0) (Req:1/4) (Req:1/1) (Req:1/3) (Req:1/5) (Req:1/7) (Req:1/6) (Req:1/2) ","bignum":200000,"randomSum":16670}
dunlin> curl 'http://localhost:8080/bar?old' # Error because "bar" does not yet exist, 'keep' implied
ERROR: Requested "old" shared memory bar does not already exist.
dunlin> curl 'http://localhost:8080/bar?new&free' # Create and free session memory "bar"
{"requestN":3,"sessionData":"0(Req:3/4) (Req:3/7) (Req:3/5) (Req:3/3) (Req:3/0) (Req:3/6) (Req:3/2) (Req:3/1) ","bignum":100000,"randomSum":25530}
dunlin> curl 'http://localhost:8080/foo?old&keep' # Modify the existing "foo" memory again
{"requestN":4,"sessionData":"0(Req:0/2) (Req:0/7) (Req:0/1) (Req:0/4) (Req:0/5) (Req:0/3) (Req:0/0) (Req:0/6) (Req:1/0) (Req:1/4) (Req:1/1) (Req:1/3) (Req:1/5) (Req:1/7) (Req:1/6) (Req:1/2) (Req:4/0) (Req:4/5) (Req:4/2) (Req:4/7) (Req:4/6) (Req:4/1) (Req:4/3) (Req:4/4) ","bignum":300000,"randomSum":30114}
dunlin> curl 'http://localhost:8080/foo?new&free' # Modify the existing "foo" memory again, this time freeing it
{"requestN":5,"sessionData":"0(Req:5/4) (Req:5/7) (Req:5/5) (Req:5/0) (Req:5/2) (Req:5/6) (Req:5/1) (Req:5/3) ","bignum":100000,"randomSum":36887}
dunlin> curl 'http://localhost:8080/foo?old&free' # The memory "foo" has been released
ERROR: Requested "old" shared memory foo does not already exist.
```
###### Copyright (C)2017 Jace A Mogill - All Rights Reserved.
================================================
FILE: Examples/WebServer/webServer.js
================================================
/*-----------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.4.5 |
| Synthetic Semantics http://www.synsem.com/ mogill@synsem.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2011-2014, Synthetic Semantics LLC. All rights reserved. |
| Copyright (c) 2015-2017, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-----------------------------------------------------------------------------*/
'use strict';
// Number of processes to use in parallel regions
var nProcs = process.argv[2] || 8;
// Initialize EMS for FJ parallelism
var ems = require('ems')(parseInt(nProcs), true, 'fj');
// Load modules for dealing with HTTP
var http = require('http');
// Load modules for parsing a request URL
var url = require('url');
// Port number to which the server listens
var port = 8080;
// Global, persistent, shared memory that is visible from any process during any HTTP request callback
var globallySharedData;
/**
* Fork into a parallel region to initialize each parallel processes
* by attaching the shared server status and loading 'require'd modules
*/
ems.parallel(function () {
globallySharedData = ems.new({
dimensions: [1000],
heapSize: [10000],
useExisting: false,
useMap: true,
doDataFill: true,
dataFill: 0,
filename: 'serverState.ems'
});
});
// Shared memory that last for a session, global so it is scoped to last more more than one request
var sessionSharedData;
/**
* When an HTTP request arrives this callback is executed.
* From within that callback, parallel processes are forked
* to work on the response, then join together back to the original
* callback thread from where the response is issued.
* @param request - Request object with URL
* @param response - Object used respond to the request
*/
function handleRequest(request, response) {
// Increment (Fetch And Add) the number of requests counter
var requestN = globallySharedData.faa('nRequests', 1);
// Request URL parsed into UTF8
var parsedRequest = url.parse(request.url, true);
// Session ID part of the request path
var key = parsedRequest.pathname.split(/\//g)[1];
if(!key) {
response.end('ERROR: Incorrectly formatted or missing session unique ID (' + parsedRequest.pathname + ')');
} else {
// Fork a parallel region
ems.parallel(parsedRequest, requestN, key, (parsedRequest, requestN, key) => {
/// EMS options for the session's shared memory
var sessionData = {
dimensions: [100], // Up to 100 keys in this memory
heapSize: [100000], // At least 100KB of memory
useMap: true, // Index using any type of key, not linear integer indexing
filename: 'session.' + key + '.ems', // Filename of persistent memory image
};
// Set EMS memory options to create a new or attach to an existing persistent EMS memory
if (parsedRequest.query.new === "") {
sessionData.dataFill = 0;
sessionData.doDataFill = true;
sessionData.useExisting = false;
sessionData.doSetFEtags = true;
sessionData.setFEtags = 'full';
} else if (parsedRequest.query.old === "") {
sessionData.useExisting = true;
} else {
// Send only one response, but all tasks exit the parallel region
if (ems.myID === 0) {
response.end('ERROR: Invalid new/old state (' + JSON.stringify(parsedRequest.query) + ') for private shared memory');
}
return;
}
// Create or attach to existing EMS memory
sessionSharedData = ems.new(sessionData);
if (!sessionSharedData) {
// Send only one response, but all tasks exit the parallel region
if (ems.myID === 0) {
response.end('ERROR: Requested "old" shared memory ' + key + ' does not already exist.');
}
return;
}
// Each task generates it's own random number and performs all the iterations.
var sum = 0;
for(var idx = 0; idx < Math.floor(Math.random() * 1000); idx += 1) {
sum += idx;
}
// All processes accumulate (Fetch And Add) directly into globally shared counter
globallySharedData.faa('sum', sum);
// Distribute (ie: map) the iterations of a loop across all the processes
ems.parForEach(0, 100000, function(idx) {
sessionSharedData.faa('bignum', 1); // Accumulate into a per-session counter
});
// Each process appends it's results to the session's EMS memory using Fetch And Add.
// Note that the memory was initialized with 0 to which this text is appended
sessionSharedData.faa('results', "(Req:" + requestN + "/" + ems.myID + ") ");
});
// All parallel processes have returned before the serial region resumes here
/// Construct a return object containing data from both the globally shared memory
/// and session-private data.
var resp = { 'requestN' : requestN };
// Parallel region may have exited due to error, in which case there are no results to return
if (sessionSharedData) {
resp.sessionData = sessionSharedData.readFF('results');
resp.bignum = sessionSharedData.readFF('bignum');
resp.randomSum = globallySharedData.readFF('sum');
response.end(JSON.stringify(resp));
if (parsedRequest.query.free === "") {
sessionSharedData.destroy(true);
}
} else {
response.end('ERROR: Parallel region exited without creating/accessing session shared memory');
}
}
}
// Create the Web server
http.createServer(handleRequest).listen(port, function () {
console.log(
"REST API: /UID?[old|new]&[keep|free]\n" +
" /UID Unique session identifier\n" +
" ?[old|new] A 'new' session will remove and create a new named private shared memory\n" +
" ?[keep|free] Release or preserve, respectively, the named private shared memory\n" +
"\nExamples:\n" +
" curl http://localhost:8080/foo?new&keep # Create a new persistent memory for session \"foo\"\n" +
" curl http://localhost:8080/foo?old&keep # Modify the existing \"foo\" memory\n" +
" curl http://localhost:8080/bar?old # Error because \"bar\" does not yet exist\n" +
" curl http://localhost:8080/bar?new&free # Create and free session memory \"bar\"\n" +
"\nServer listening on: http://localhost:" + port + "\n"
);
});
================================================
FILE: Examples/concurrent_Q_and_TM.js
================================================
/*-----------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.0.0 |
| Synthetic Semantics http://www.synsem.com/ mogill@synsem.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2011-2014, Synthetic Semantics LLC. All rights reserved. |
| Copyright (c) 2015-2016, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-----------------------------------------------------------------------------+
| Program Description: |
| |
| Sequentially enqueue transactions from thread 0 while all other |
| threads consume them. When all the operations have been queued |
| thread 0 also begins dequeing work and processing transactions. |
| |
| Unlike the workQ_and_TM.js example, randomInRange() is only called |
| from thread 0 so it is defined only in the main thread's context. |
| |
+-----------------------------------------------------------------------------*/
'use strict';
// Initialize EMS to use the fork-join execution model
var ems = require('ems')(parseInt(process.argv[2]), true, 'fj');
var assert = require('assert');
var workQ;
var totalNops;
var checkNops;
var arrLen;
var heapSize;
var nTransactions;
var nTables;
var maxNops;
var tables;
//---------------------------------------------------------------------------
// Generate a random integer within a range (inclusive) from 'low' to 'high'
//
function randomInRange(low, high) {
return (Math.floor((Math.random() * (high - low)) + low));
}
//-------------------------------------------------------------------
// Timer functions
function timerStart() {
return new Date().getTime();
}
function timerStop(timer, nOps, label, myID) {
function fmtNumber(n) {
var s = ' ' + n.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
if (n < 1) return n;
else {
return s.substr(s.length - 15, s.length);
}
}
var now = new Date().getTime();
var opsPerSec = (nOps * 1000000) / ((now - timer) * 1000);
if (typeof myID === undefined || myID === 0) {
console.log(fmtNumber(nOps) + label + fmtNumber(Math.floor(opsPerSec).toString()) + " ops/sec");
}
}
//---------------------------------------------------------------------------
// Initialize shared data: global scalars, EMS buffers for statistics
// and checksums, an EMS array to be used as a work queue, and many
// tables to perform transactions on.
//
function initializeSharedData() {
arrLen = 1000000; // Maximum number of elements the EMS array can contain
heapSize = 100000; // Amount of persistent memory to reserve for transactions
nTransactions = 1000000; // Number of transactions to perform in the experiment
nTables = 6; // Number of EMS arrays to perform transactions across
maxNops = 5; // Maximum number of EMS arrays to update during a transaction
tables = [];
totalNops = ems.new(2);
checkNops = ems.new(1);
//---------------------------------------------------------------------------
// The queue of transactions to perform
// [ table#, index, read-only ]
workQ = ems.new({
dimensions: [nTransactions + ems.nThreads],
heapSize: nTransactions * 200,
useExisting: false,
setFEtags: 'empty'
});
//---------------------------------------------------------------------------
// Create all the tables
for (var tableN = 0; tableN < nTables; tableN++) {
tables[tableN] = ems.new({
dimensions: [arrLen],
heapSize: 0,
useExisting: false,
filename: '/tmp/EMS_tm' + tableN,
doDataFill: true,
dataFill: 0,
setFEtags: 'full'
});
}
}
//---------------------------------------------------------------------------
// Create 'nTransactions' many transactions, each having a random number (up
// to maxNops) randomly chosen values, some of which are read-only.
// Because tables and elements are chosen at random, it is necessary to
// remove duplicate elements in a single transaction, to prevent deadlocks.
//
// Transactions are pushed onto a shared queue
function generateTransactions() {
//---------------------------------------------------------------------------
// Generate operations involving random elements in random EMS arrays
// and enqueue them on the work queue
for (var transN = 0; transN < nTransactions; transN++) {
var ops = [];
var nOps = randomInRange(1, maxNops);
// var indexes = [];
for (var opN = 0; opN < nOps; opN++) {
var tableN = randomInRange(0, nTables);
var idx = randomInRange(0, arrLen);
if (transN % 2 == 0 || opN % 3 > 0) {
ops.push([tableN, idx, true]);
} else {
ops.push([tableN, idx]);
}
}
// De-duplicate operands in a single transaction as that would deadlock
var indicies = [];
var uids = [];
for (var i = 0; i < ops.length; ++i) {
indicies[i] = i;
uids[i] = (ops[i][0] * 1000000000000) + ops[i][1];
}
var uniq = [];
for (opN = 0; opN < ops.length; opN++) {
var isDupl = false;
for (var checkN = 0; checkN < ops.length; checkN++) {
if (opN != checkN && uids[opN] == uids[checkN]) {
isDupl = true;
break;
}
}
if (!isDupl) {
uniq.push(ops[opN]);
}
}
workQ.enqueue(JSON.stringify(uniq));
}
}
//------------------------------------------------------------------
// Simultaneously generate and consume transactions
// Thread 0 enqueues them while the rest dequeue transactions.
// When thread 0 finishes creating new work, it also begins to
// dequeue transactions from the workQ.
//
// If there is nothing on the queue, keep checking until the "DONE"
// message is dequeued.
//
function performTransactions() {
//------------------------------------------------------------------
// Generate the transactions concurrently with their consumption
if (ems.myID == 0) {
var startTime = timerStart();
generateTransactions();
// After all the work has been enqueued, add DONE semaphores to the
// end of the queue so they are processed only after all the work
// has been issued. Each thread enqueues one event and can only
// consume one before exiting.
for (var taskN = 0; taskN < ems.nThreads; taskN++) {
workQ.enqueue("DONE");
}
timerStop(startTime, nTransactions, " transactions enqueued ", ems.myID);
}
//------------------------------------------------------------------
// Process the transactions
var rwNops = 0;
var readNops = 0;
while (true) {
var str = workQ.dequeue();
if (str !== undefined) {
if (str === "DONE") {
break
} else {
var ops = JSON.parse(str);
for (var opN = 0; opN < ops.length; opN++) {
ops[opN][0] = tables[ops[opN][0]];
}
var transaction = ems.tmStart(ops);
ops.forEach(function (op) {
var tmp = op[0].read(op[1]);
if (op[2] != true) {
rwNops++;
op[0].write(op[1], tmp + 1);
} else {
readNops++;
}
});
ems.tmEnd(transaction, true);
}
} else {
// Queue was empty, but have not yet seen DONE event, so
// keep trying to dequeue work
}
}
totalNops.faa(0, rwNops);
totalNops.faa(1, readNops);
}
//=======================================================================
// B E G I N P R O G R A M M A I N
//-------------------------------------------------------
// Initialize the shared data
var startTime = timerStart();
ems.parallel(initializeSharedData);
timerStop(startTime, nTables, " tables initialized ", ems.myID);
// Data to be initialized only once
totalNops.writeXF(0, 0);
totalNops.writeXF(1, 0);
checkNops.writeXF(0, 0);
// Perform all the transactions
startTime = timerStart();
ems.parallel(performTransactions);
timerStop(startTime, nTransactions, " transactions performed", ems.myID);
timerStop(startTime, totalNops.readFF(0), " table updates ", ems.myID);
timerStop(startTime, totalNops.readFF(0) + totalNops.readFF(1), " elements referenced ", ems.myID);
//------------------------------------------------------------------
// Sum all the values in the tables to account for all the transactions
startTime = timerStart();
ems.parallel(function () {
ems.parForEach(0, nTables, function (tableN) {
var localSum = 0;
for (var idx = 0; idx < arrLen; idx++) {
localSum += tables[tableN].read(idx);
}
checkNops.faa(0, localSum);
});
});
timerStop(startTime, nTables * arrLen, " elements checked ", ems.myID);
assert(checkNops.readFF(0) == totalNops.readFF(0),
"Error in final sum = " + checkNops.readFF(0) + " should be=" + totalNops.readFF(0));
console.log("Results are correct");
// One liner to get all processes to exit
ems.parallel(function () { process.exit(0) });
================================================
FILE: Examples/wordCount.js
================================================
/*-----------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.4.0 |
| Synthetic Semantics http://www.synsem.com/ mogill@synsem.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2011-2014, Synthetic Semantics LLC. All rights reserved. |
| Copyright (c) 2015-2016, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-----------------------------------------------------------------------------*/
'use strict';
var ems = require('ems')(parseInt(process.argv[2]));
var fs = require('fs');
//-------------------------------------------------------------------
// Allocate the word count dictionary
var maxNKeys = 100000000;
var wordCounts = ems.new({
dimensions: [maxNKeys], // Maximum # of different keys the array can store
heapSize: maxNKeys * 10, // 10 bytes of storage per key, used for key (dictionary word) itself
useMap: true, // Use a key-index mapping, not integer indexes
setFEtags: 'full', // Initial full/empty state of array elements
doDataFill: true, // Initialize data values
dataFill: 0 // Initial value of new keys
});
//-------------------------------------------------------------------
// Use a key-value mapped array to store execution statistics
var nStats = 200;
var stats = ems.new({
dimensions: [nStats], // Maximum number of stats that can be stored
heapSize: nStats * 200,// Space for keys, data values, and sorting
useMap: true, // Use a key-index mapping, not integer indexes
setFEtags: 'full', // Initial full/empty state of array elements
doDataFill: true, // Initialize data values
dataFill: 0 // Initial value of new keys
});
// ===============================================================================
// Perform Word Count
//
// Use the word as a key to the EMS array, the value is the count of the
// number of times the word has been encountered. This is incremented
// atomically.
//
// Bookkeeping for statistics is performed by keeping track of the total number
// of words and bytes processed in the "stats" EMS array.
//
var doc_path = '/path/to/your/document/collection/';
if (!process.argv[3]) {
console.log("usage: wordCount <# processes> /path/to/your/document/collection/");
return -1;
} else {
doc_path = process.argv[3];
}
var file_list = fs.readdirSync(doc_path);
var splitPattern = new RegExp(/[ \n,\.\\/_\-\<\>:\;\!\@\#\$\%\&\*\(\)=\[\]|\"\'\{\}\?\—]/);
// Loop over the files in parallel, counting words
var startTime = Date.now();
ems.parForEach(0, file_list.length, function (fileNum) {
try {
var text = fs.readFileSync(doc_path + file_list[fileNum], 'utf8', "r");
var words = text.replace(/[\n\r]/g, ' ').toLowerCase().split(splitPattern);
// Iterate over all the words in the document
words.forEach(function (word) {
// Ignore words over 15 characters long as non-English
if (word.length < 15 && word.length > 0) {
// Atomically increment the count of times this word was seen
var count = wordCounts.faa(word, 1);
}
});
// Accumulate some statistics: word and data counters, print progress
var nWords_read = stats.faa('nWords', words.length);
var nBytes_read = stats.faa('nBytesRead', text.length);
var now = Date.now();
if (ems.myID === 0 && fileNum % 10 === 0) {
console.log("Average Words/sec=" + Math.floor((1000 * nWords_read) / (now - startTime)) +
" MBytes/sec=" + Math.floor((100000 * nBytes_read) / ((now - startTime) * (1 << 20)))/100);
}
}
catch (Err) {
ems.diag("This is not a text file:" + file_list[fileNum]);
}
});
// ===============================================================================
// Output the Most Frequently Occurring Words
//
// First perform many sequential insertion sorts in parallel,
// then serially perform a merge sort of all the partial lists.
//
// Start by printing the total amount of data processed during word counting
ems.master(function() {
console.log("Totals: ", stats.read('nWords'), " words parsed, ",
stats.read('nBytesRead'), "bytes read.");
console.log("Starting sort....");
});
// Divide the array across all the processes, each process keeps track
// of the "local_sort_len" most frequent word it encounters.
var local_sort_len = Math.max(10, process.argv[2]);
var biggest_counts = new Array(local_sort_len).fill({"key": 0, "count": 0});
ems.parForEach(0, maxNKeys, function (keyN) {
if (keyN % (maxNKeys / 10) === 0) { ems.diag("Sorting key ID " + keyN); }
var key = wordCounts.index2key(keyN);
if (key) {
// Perform an insertion sort of the new key into the biggest_counts
// list, deleting the last (smallest) element to preserve length.
var keyCount = wordCounts.read(key);
var idx = local_sort_len - 1;
while (idx >= 0 && biggest_counts[idx].count < keyCount) {
idx -= 1;
}
var newBiggest = {"key": key, "count": keyCount};
if (idx < 0) {
biggest_counts = [newBiggest].concat(biggest_counts.slice(0, biggest_counts.length - 1));
} else if (idx < local_sort_len) {
var left = biggest_counts.slice(0, idx + 1);
var right = biggest_counts.slice(idx + 1);
biggest_counts = left.concat([newBiggest].concat(right)).slice(0, -1);
}
}
});
// Concatenate all the partial (one per process) lists into one list
stats.writeXF('most_frequent', []);
ems.barrier();
stats.writeEF('most_frequent', stats.readFE('most_frequent').concat(biggest_counts));
ems.barrier();
// Sort & print the list of words, only one process is needed
ems.master(function() {
biggest_counts = stats.readFF('most_frequent');
// Sort the list by word frequency
biggest_counts.sort(function (a, b) {
return b.count - a.count;
});
// Print only the first "local_sort_len" items -- assume the worst case
// of all the largest counts are discovered by a single process, the next
// largest after "local_sort_len" is no longer on any list.
console.log("Most frequently appearing terms:");
for (var index = 0; index < local_sort_len; index += 1) {
console.log(index + ': ' + biggest_counts[index].key + " " + biggest_counts[index].count);
}
});
================================================
FILE: Examples/workQ_and_TM.js
================================================
/*-----------------------------------------------------------------------------+
| Extended Memory Semantics (EMS) Version 1.0.0 |
| Synthetic Semantics http://www.synsem.com/ mogill@synsem.com |
+-----------------------------------------------------------------------------+
| Copyright (c) 2011-2014, Synthetic Semantics LLC. All rights reserved. |
| Copyright (c) 2015-2016, Jace A Mogill. All rights reserved. |
| |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions are met: |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in the |
| documentation and/or other materials provided with the distribution. |
| * Neither the name of the Synthetic Semantics nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SYNTHETIC |
| SEMANTICS LLC BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
| EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR |
| PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
| LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
| NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| |
+-----------------------------------------------------------------------------+
| Program Description: |
| |
| Enqueue the transactions from a parallel loop. |
| When all the operations have been queued they all begin |
| processing transactions. |
| |
| Unlike the concurrent_Q_and_TM.js example, randomInRange() is called |
| from every thread, so it is declared from every thread. |
| |
+-----------------------------------------------------------------------------*/
'use strict';
var ems = require('ems')(parseInt(process.argv[2]), true, 'fj');
var assert = require('assert');
var workQ;
var totalNops;
var checkNops;
var arrLen;
var heapSize;
var nTransactions;
var nTables;
var maxNops;
var tables;
//-------------------------------------------------------------------
// Timer functions
function timerStart() {
return new Date().getTime();
}
function timerStop(timer, nOps, label, myID) {
function fmtNumber(n) {
var s = ' ' + n.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
if (n < 1) return n;
else {
return s.substr(s.length - 15, s.length);
}
}
var now = new Date().getTime();
var opsPerSec = (nOps * 1000000) / ((now - timer) * 1000);
if (typeof myID === undefined || myID === 0) {
console.log(fmtNumber(nOps) + label + fmtNumber(Math.floor(opsPerSec).toString()) + " ops/sec");
}
}
//---------------------------------------------------------------------------
// Initialize shared data: global scalars, EMS buffers for statistics
// and checksums, an EMS array to be used as a work queue, and many
// tables to perform transactions on.
//
function initializeSharedData() {
arrLen = 1000000; // Maximum number of elements the EMS array can contain
heapSize = 100000; // Amount of persistent memory to reserve for transactions
nTransactions = 1000000; // Number of transactions to perform in the experiment
nTables = 6; // Number of EMS arrays to perform transactions across
maxNops = 5; // Maximum number of EMS arrays to update during a transaction
tables = [];
totalNops = ems.new(2);
checkNops = ems.new(1);
//---------------------------------------------------------------------------
// The queue of transactions to perform
// [ table#, index, read-only ]
workQ = ems.new({
dimensions: [nTransactions + ems.nThreads],
heapSize: nTransactions * 200,
useExisting: false,
dataFill: 0,
doDataFill: true,
setFEtags: 'empty'
});
//---------------------------------------------------------------------------
// Create all the tables
for (var tableN = 0; tableN < nTables; tableN++) {
tables[tableN] = ems.new({
dimensions: [arrLen],
heapSize: 0,
useExisting: false,
filename: '/tmp/EMS_tm' + tableN,
dataFill: 0,
doDataFill: true,
setFEtags: 'full'
});
}
}
//---------------------------------------------------------------------------
// Create 'nTransactions' many transactions, each having a random number (up
// to maxNops) randomly chosen values, some of which are read-only.
// Because tables and elements are chosen at random, it is necessary to
// remove duplicate elements in a single transaction, to prevent deadlocks.
//
// Transactions are pushed onto a shared queue
function generateTransactions() {
//---------------------------------------------------------------------------
// Generate a random integer within a range (inclusive) from 'low' to 'high'
var randomInRange = function (low, high) {
return ( Math.floor((Math.random() * (high - low)) + low) );
};
//---------------------------------------------------------------------------
// Generate operations involving random elements in random EMS arrays
// and enqueue them on the work queue
ems.parForEach(0, nTransactions, function (transN) {
var ops = [];
var nOps = randomInRange(1, maxNops);
for (var opN = 0; opN < nOps; opN++) {
var tableN = randomInRange(0, nTables);
var idx = randomInRange(0, arrLen);
if (transN % 2 == 0 || opN % 3 > 0) {
ops.push([tableN, idx, true]);
}
else {
ops.push([tableN, idx]);
}
}
// De-duplicate operations in a transaction which would deadlock
var indicies = [];
var uids = [];
for (var i = 0; i < ops.length; ++i) {
indicies[i] = i;
uids[i] = (ops[i][0] * 1000000000000) + ops[i][1];
}
var uniq = [];
for (opN = 0; opN < ops.length; opN++) {
var isDupl = false;
for (var checkN = 0; checkN < ops.length; checkN++) {
if (opN != checkN && uids[opN] == uids[checkN]) {
isDupl = true;
break;
}
}
if (!isDupl) {
uniq.push(ops[opN]);
}
}
workQ.enqueue(uniq);
});
// After all the work has been enqueued, add DONE semaphores to the
// end of the queue so they are processed only after all the work
// has been issued. Each thread enqueues one event and can only
// consume one before exiting.
workQ.enqueue("DONE")
}
//------------------------------------------------------------------
// Consume transactions. If there is nothing on the queue,
// keep checking until the "DONE" message is dequeued.
//
function performTransactions() {
var rwNops = 0;
var readNops = 0;
while (true) {
var ops = workQ.dequeue();
if (ops !== undefined) {
if (ops === "DONE") {
break;
} else {
for (var opN = 0; opN < ops.length; opN++) {
ops[opN][0] = tables[ops[opN][0]];
}
var transaction = ems.tmStart(ops);
ops.forEach(function (op) {
var tmp = op[0].read(op[1]);
if (op[2] != true) {
rwNops++;
op[0].write(op[1], tmp + 1);
} else {
readNops++;
}
});
ems.tmEnd(transaction, true);
}
}
}
totalNops.faa(0, rwNops);
totalNops.faa(1, readNops);
}
//------------------------------------------------------------------------
// Main program entry point
//
ems.parallel(initializeSharedData);
totalNops.writeXF(0, 0);
totalNops.writeXF(1, 0);
checkNops.writeXF(0, 0);
// Enqueue all the transaction
var startTime = timerStart();
ems.parallel(generateTransactions);
timerStop(startTime, nTransactions, " transactions enqueued ", ems.myID);
// Perform all the transactions
startTime = timerStart();
ems.parallel(performTransactions);
timerStop(startTime, nTransactions, " transactions performed", ems.myID);
timerStop(startTime, totalNops.readFF(0), " table updates ", ems.myID);
timerStop(startTime, totalNops.readFF(0) + totalNops.readFF(1), " elements referenced ", ems.myID);
// Validate the results by summing all the updates
startTime
gitextract__ryzapkl/
├── .gitignore
├── .travis.yml
├── Docs/
│ ├── docs.css
│ ├── index.html
│ └── reference.html
├── Examples/
│ ├── Interlanguage/
│ │ ├── README.md
│ │ ├── interlanguage.js
│ │ └── interlanguage.py
│ ├── KeyValueStore/
│ │ ├── README.md
│ │ └── kv_store.js
│ ├── README.md
│ ├── STREAMS/
│ │ ├── README.md
│ │ ├── streams_bulk_sync_parallel.js
│ │ └── streams_fork_join.js
│ ├── WebServer/
│ │ ├── README.md
│ │ └── webServer.js
│ ├── concurrent_Q_and_TM.js
│ ├── wordCount.js
│ └── workQ_and_TM.js
├── LICENSE
├── Makefile
├── Python/
│ ├── README.md
│ ├── __init__.py
│ ├── ems.py
│ └── setup.py
├── README.md
├── Tests/
│ ├── 3dSpace.js
│ ├── CASdataFlow.js
│ ├── ES6/
│ │ └── harmony_proxies.js
│ ├── accum_omp.c
│ ├── barrier.js
│ ├── check_image_files.sh
│ ├── fj_args.js
│ ├── fj_args.py
│ ├── fork_join.js
│ ├── fullArrays.js
│ ├── issue11_bsp.js
│ ├── issue11_fj.js
│ ├── loopScheduling.js
│ ├── mapped_test.js
│ ├── py_api.py
│ ├── readers-writer.js
│ ├── refactoring.js
│ ├── stack_and_queue.js
│ ├── strcpy.js
│ ├── stringTags.js
│ ├── testUtils.js
│ ├── test_alloc.c
│ ├── tm.js
│ ├── tm_noq.js
│ ├── tm_noq_strings.js
│ └── v8Types.js
├── Vagrantfile
├── binding.gyp
├── index.html
├── nodejs/
│ ├── ems.js
│ ├── nodejs.cc
│ └── nodejs.h
├── package.json
└── src/
├── collectives.cc
├── ems.cc
├── ems.h
├── ems_alloc.cc
├── ems_alloc.h
├── ems_proto.h
├── ems_types.h
├── loops.cc
├── primitives.cc
└── rmw.cc
SYMBOL INDEX (197 symbols across 33 files)
FILE: Examples/Interlanguage/interlanguage.js
function barrier (line 112) | function barrier(message) {
function incrementCounter (line 200) | function incrementCounter() {
FILE: Examples/Interlanguage/interlanguage.py
function barrier (line 92) | def barrier(message):
FILE: Examples/KeyValueStore/kv_store.js
function handleRequest (line 58) | function handleRequest(request, response) {
FILE: Examples/STREAMS/streams_bulk_sync_parallel.js
function stopTimer (line 51) | function stopTimer(timer, nOps, label) {
FILE: Examples/WebServer/webServer.js
function handleRequest (line 77) | function handleRequest(request, response) {
FILE: Examples/concurrent_Q_and_TM.js
function randomInRange (line 59) | function randomInRange(low, high) {
function timerStart (line 65) | function timerStart() {
function timerStop (line 69) | function timerStop(timer, nOps, label, myID) {
function initializeSharedData (line 91) | function initializeSharedData() {
function generateTransactions (line 134) | function generateTransactions() {
function performTransactions (line 187) | function performTransactions() {
FILE: Examples/workQ_and_TM.js
function timerStart (line 57) | function timerStart() {
function timerStop (line 61) | function timerStop(timer, nOps, label, myID) {
function initializeSharedData (line 83) | function initializeSharedData() {
function generateTransactions (line 129) | function generateTransactions() {
function performTransactions (line 188) | function performTransactions() {
FILE: Python/ems.py
function emsThreadStub (line 106) | def emsThreadStub(conn, taskn):
function initialize (line 132) | def initialize(nThreadsArg, pinThreadsArg=False, threadingType='bsp',
function _loop_chunk (line 193) | def _loop_chunk():
function diag (line 203) | def diag(text):
function parallel (line 210) | def parallel(func, *kargs):
function parForEach (line 231) | def parForEach(start, # First iteration's index
function tmStart (line 275) | def tmStart(emsElems):
function tmEnd (line 310) | def tmEnd(
function critical (line 338) | def critical(func, timeout=1000000):
function master (line 347) | def master(func):
function single (line 354) | def single(func):
function barrier (line 371) | def barrier(timeout=10000):
function new (line 380) | def new(arg0=None, # Maximum number of elements the EMS region can hold
function _new_EMSval (line 526) | def _new_EMSval(val):
class EMSarray (line 581) | class EMSarray(object):
method __init__ (line 582) | def __init__(self,
method destroy (line 620) | def destroy(self, unlink_file):
method _returnData (line 628) | def _returnData(self, emsval):
method sync (line 656) | def sync(self):
method index2key (line 661) | def index2key(self, index):
method push (line 669) | def push(self, value):
method pop (line 673) | def pop(self):
method dequeue (line 678) | def dequeue(self):
method enqueue (line 683) | def enqueue(self, value):
method write (line 692) | def write(self, indexes, value):
method writeEF (line 698) | def writeEF(self, indexes, value):
method writeXF (line 704) | def writeXF(self, indexes, value):
method writeXE (line 710) | def writeXE(self, indexes, value):
method read (line 717) | def read(self, indexes):
method readFE (line 723) | def readFE(self, indexes):
method readFF (line 729) | def readFF(self, indexes):
method readRW (line 735) | def readRW(self, indexes):
method releaseRW (line 741) | def releaseRW(self, indexes):
method setTag (line 745) | def setTag(self, indexes, fe):
method faa (line 751) | def faa(self, indexes, val):
method cas (line 762) | def cas(self, indexes, oldVal, newVal):
method _idx (line 774) | def _idx(self, indexes):
method __getattr__ (line 790) | def __getattr__(self, attr):
method __setattr__ (line 793) | def __setattr__(self, attr, value):
method __getitem__ (line 800) | def __getitem__(self, item):
method __setitem__ (line 803) | def __setitem__(self, key, value):
class EMSelement (line 809) | class EMSelement(object):
method __init__ (line 810) | def __init__(self, ems_array, index):
method write (line 814) | def write(self, value):
method writeXE (line 817) | def writeXE(self, value):
method writeXF (line 820) | def writeXF(self, value):
method writeEF (line 823) | def writeEF(self, value):
method read (line 826) | def read(self):
method readFF (line 829) | def readFF(self):
method readFE (line 832) | def readFE(self):
method releaseRW (line 835) | def releaseRW(self):
method readRW (line 838) | def readRW(self):
method setTag (line 841) | def setTag(self, fe):
method faa (line 844) | def faa(self, value):
method cas (line 847) | def cas(self, oldVal, newVal):
FILE: Tests/3dSpace.js
function val3d (line 47) | function val3d(i, j, k) {
function stopTimer (line 54) | function stopTimer(timer, nOps, label) {
FILE: Tests/CASdataFlow.js
function stopTimer (line 40) | function stopTimer(timer, nOps, label) {
FILE: Tests/accum_omp.c
function main (line 36) | int main() {
FILE: Tests/fj_args.js
function attach (line 90) | function attach(useExisting) {
function check_for_file (line 110) | function check_for_file(fname) {
FILE: Tests/fj_args.py
function fj_test (line 47) | def fj_test(a, b, c, taskN=None):
FILE: Tests/fork_join.js
function timerStart (line 36) | function timerStart() { return new Date().getTime(); }
function timerStop (line 37) | function timerStop(timer, nOps, label, myID) {
FILE: Tests/loopScheduling.js
function doWork (line 44) | function doWork(idx) {
FILE: Tests/py_api.py
function sum (line 54) | def sum(left, right):
function target (line 63) | def target(idx):
function parfun (line 125) | def parfun(idx):
function triangular (line 136) | def triangular(max = 10):
function check_tri (line 145) | def check_tri(msg):
function reset_tri (line 154) | def reset_tri():
function plus_one (line 206) | def plus_one():
function check_master (line 225) | def check_master():
function check_single (line 232) | def check_single():
FILE: Tests/stack_and_queue.js
function stopTimer (line 61) | function stopTimer(timer, nOps, label) {
FILE: Tests/strcpy.js
function stringFill (line 49) | function stringFill(x, n) {
FILE: Tests/testUtils.js
function fmtNumber (line 55) | function fmtNumber(n) {
FILE: Tests/test_alloc.c
function test_size (line 36) | static void
function test_alloc (line 43) | static int64_t
function test_free (line 52) | static void
function main (line 61) | int main() {
FILE: Tests/tm_noq.js
function makeWork (line 66) | function makeWork() {
FILE: Tests/tm_noq_strings.js
function makeWork (line 66) | function makeWork() {
FILE: nodejs/ems.js
function EMSidx (line 48) | function EMSidx(indexes, // Index given by application
function EMSdiag (line 71) | function EMSdiag(text) {
function EMSparallel (line 78) | function EMSparallel() {
function EMSparForEach (line 96) | function EMSparForEach(start, // First iteration's index
function EMStmStart (line 151) | function EMStmStart(emsElems) {
function EMStmEnd (line 186) | function EMStmEnd(tmHandle, // The returned value from tmStart
function EMSreturnData (line 206) | function EMSreturnData(value) {
function EMSsync (line 231) | function EMSsync(emsArr) {
function EMSindex2key (line 238) | function EMSindex2key(index) {
function EMSpush (line 250) | function EMSpush(value) {
function EMSpop (line 258) | function EMSpop() {
function EMSdequeue (line 262) | function EMSdequeue() {
function EMSenqueue (line 266) | function EMSenqueue(value) {
function EMSwrite (line 280) | function EMSwrite(indexes, value) {
function EMSwriteEF (line 289) | function EMSwriteEF(indexes, value) {
function EMSwriteXF (line 298) | function EMSwriteXF(indexes, value) {
function EMSwriteXE (line 307) | function EMSwriteXE(indexes, value) {
function EMSread (line 316) | function EMSread(indexes) {
function EMSreadFE (line 320) | function EMSreadFE(indexes) {
function EMSreadFF (line 324) | function EMSreadFF(indexes) {
function EMSreadRW (line 328) | function EMSreadRW(indexes) {
function EMSreleaseRW (line 332) | function EMSreleaseRW(indexes) {
function EMSsetTag (line 336) | function EMSsetTag(indexes, fe) {
function EMSfaa (line 340) | function EMSfaa(indexes, val) {
function EMScas (line 349) | function EMScas(indexes, oldVal, newVal) {
function EMScritical (line 361) | function EMScritical(func, timeout) {
function EMSmaster (line 374) | function EMSmaster(func) {
function EMSsingle (line 389) | function EMSsingle(func) {
function EMSbarrier (line 401) | function EMSbarrier(timeout) {
function EMSisArray (line 419) | function EMSisArray(a) {
function EMSisObject (line 423) | function EMSisObject(o) {
function EMSisDefined (line 427) | function EMSisDefined(x) {
function EMSdestroy (line 434) | function EMSdestroy(unlink_file) {
function EMSnew (line 445) | function EMSnew(arg0, // Maximum number of elements the EMS regi...
function ems_wrapper (line 637) | function ems_wrapper(nThreadsArg, pinThreadsArg, threadingType, filename) {
FILE: nodejs/nodejs.cc
type rlimit (line 68) | struct rlimit
FILE: src/collectives.cc
function EMSsingleTask (line 38) | bool EMSsingleTask(int mmapID) {
function EMScriticalEnter (line 59) | int EMScriticalEnter(int mmapID, int timeout) {
function EMScriticalExit (line 77) | bool EMScriticalExit(int mmapID) {
function EMSbarrier (line 93) | int EMSbarrier(int mmapID, int timeout) {
FILE: src/ems.cc
function emsMutexMem_alloc (line 51) | size_t emsMutexMem_alloc(struct emsMem *heap, // Base of EMS malloc st...
function emsMutexMem_free (line 66) | void emsMutexMem_free(struct emsMem *heap, // Base of EMS malloc structs
function EMSkey2index (line 86) | int64_t EMSkey2index(void *emsBuf, EMSvalueType *key, bool is_mapped) {
function EMSarrFinalize (line 177) | static void EMSarrFinalize(char *data, void *hint) {
function EMStransitionFEtag (line 208) | unsigned char EMStransitionFEtag(EMStag_t volatile *tag, EMStag_t volati...
function EMShashString (line 240) | int64_t EMShashString(const char *key) {
function EMSwriteIndexMap (line 261) | int64_t EMSwriteIndexMap(const int mmapID, EMSvalueType *key) {
function EMSreadUsingTags (line 380) | bool EMSreadUsingTags(const int mmapID,
function EMSreadRW (line 504) | bool EMSreadRW(const int mmapID, EMSvalueType *key, EMSvalueType *return...
function EMSreadFE (line 511) | bool EMSreadFE(const int mmapID, EMSvalueType *key, EMSvalueType *return...
function EMSreadFF (line 518) | bool EMSreadFF(const int mmapID, EMSvalueType *key, EMSvalueType *return...
function EMSread (line 525) | bool EMSread(const int mmapID, EMSvalueType *key, EMSvalueType *returnVa...
function EMSreleaseRW (line 532) | int EMSreleaseRW(const int mmapID, EMSvalueType *key) {
function EMSwriteUsingTags (line 580) | bool EMSwriteUsingTags(int mmapID,
function EMSwriteXF (line 684) | bool EMSwriteXF(int mmapID, EMSvalueType *key, EMSvalueType *value) {
function EMSwriteXE (line 690) | bool EMSwriteXE(int mmapID, EMSvalueType *key, EMSvalueType *value) {
function EMSwriteEF (line 696) | bool EMSwriteEF(int mmapID, EMSvalueType *key, EMSvalueType *value) {
function EMSwrite (line 702) | bool EMSwrite(int mmapID, EMSvalueType *key, EMSvalueType *value) {
function EMSsetTag (line 710) | bool EMSsetTag(int mmapID, EMSvalueType *key, bool is_full) {
function EMSdestroy (line 734) | bool EMSdestroy(int mmapID, bool do_unlink) {
function EMSindex2key (line 758) | bool EMSindex2key(int mmapID, int64_t idx, EMSvalueType *key) {
function EMSsync (line 807) | bool EMSsync(int mmapID) {
function EMSinitialize (line 817) | int EMSinitialize(int64_t nElements, // 0
FILE: src/ems.h
type emsMem (line 199) | struct emsMem
type emsMem (line 202) | struct emsMem
FILE: src/ems_alloc.cc
type emsMem (line 46) | struct emsMem
type emsMem (line 49) | struct emsMem
type emsMem (line 49) | struct emsMem
type emsMem (line 49) | struct emsMem
function emsMem_delete (line 57) | void emsMem_delete(struct emsMem *self) {
function emsNextPow2 (line 64) | size_t emsNextPow2(int64_t x) {
function EMS_index_offset (line 72) | static inline int64_t EMS_index_offset(int64_t index, int32_t level, int...
function EMS_mark_parent (line 79) | static void EMS_mark_parent(struct emsMem *self, int64_t index) {
function emsMem_alloc (line 94) | size_t emsMem_alloc(struct emsMem *self, size_t bytesRequested) {
function EMS_combine (line 154) | static void EMS_combine(struct emsMem *self, int64_t index) {
function emsMem_free (line 171) | void emsMem_free(struct emsMem *self, size_t offset) {
function emsMem_size (line 203) | size_t emsMem_size(struct emsMem *self, size_t offset) {
function EMS_dump (line 233) | static void EMS_dump(struct emsMem *self, size_t index, int32_t level) {
function emsMem_dump (line 258) | void emsMem_dump(struct emsMem *self) {
FILE: src/ems_alloc.h
type emsMem (line 43) | struct emsMem {
type emsMem (line 49) | struct emsMem
type emsMem (line 50) | struct emsMem
type emsMem (line 51) | struct emsMem
type emsMem (line 52) | struct emsMem
type emsMem (line 53) | struct emsMem
type emsMem (line 54) | struct emsMem
FILE: src/ems_types.h
type EMStag_t (line 38) | typedef union {
type EMSulong_double (line 51) | typedef union {
type EMSvalueType (line 58) | typedef struct {
FILE: src/loops.cc
function EMSloopInit (line 38) | bool EMSloopInit(int mmapID, int32_t start, int32_t end, int32_t minChun...
function EMSloopChunk (line 71) | bool EMSloopChunk(int mmapID, int32_t *start, int32_t *end) {
FILE: src/primitives.cc
function EMSpush (line 37) | int EMSpush(int mmapID, EMSvalueType *value) { // TODO: Eventually prom...
function EMSpop (line 95) | bool EMSpop(int mmapID, EMSvalueType *returnValue) {
function EMSenqueue (line 158) | int EMSenqueue(int mmapID, EMSvalueType *value) { // TODO: Eventually p...
function EMSdequeue (line 210) | bool EMSdequeue(int mmapID, EMSvalueType *returnValue) {
FILE: src/rmw.cc
function EMSfaa (line 37) | bool EMSfaa(int mmapID, EMSvalueType *key, EMSvalueType *value, EMSvalue...
function EMScas (line 301) | bool EMScas(int mmapID, EMSvalueType *key,
Condensed preview — 69 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (599K chars).
[
{
"path": ".gitignore",
"chars": 70,
"preview": "**/#*#\n**~\nbuild/\nnode_modules/\n.vagrant\n.local\n*.o\npackage-lock.json\n"
},
{
"path": ".travis.yml",
"chars": 871,
"preview": "language: node_js\n\nos:\n - linux\n\ndist: trusty\n\nbranches:\n only:\n - gh-pages\n - /.*/\n\nnode_js:\n - \"14.7\"\n - \"12.18\""
},
{
"path": "Docs/docs.css",
"chars": 11658,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Docs/index.html",
"chars": 20561,
"preview": "<html>\n <!-- --------------------------------------------------------------------------+\n | Extended Memory Sema"
},
{
"path": "Docs/reference.html",
"chars": 61317,
"preview": "<html>\n <!-- --------------------------------------------------------------------------+\n | Extended Memory Sema"
},
{
"path": "Examples/Interlanguage/README.md",
"chars": 4856,
"preview": "# Inter-Language Persistent Shared Memory\n\nIn *The Future™*, memory will have new capabilities that\nprogrammers will ma"
},
{
"path": "Examples/Interlanguage/interlanguage.js",
"chars": 9831,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Examples/Interlanguage/interlanguage.py",
"chars": 8349,
"preview": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\"\"\"\n +--------------------------------------------------------------------"
},
{
"path": "Examples/KeyValueStore/README.md",
"chars": 3479,
"preview": "# Key-Value Store Example\n\nThis example implements a simple REST server, the API is a 1-to-1 \nmapping of REST commands t"
},
{
"path": "Examples/KeyValueStore/kv_store.js",
"chars": 7556,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Examples/README.md",
"chars": 8272,
"preview": "# Extended Memory Semantics (EMS) Examples\n#### Table of Contents\n* [Simple Loop Benchmarks](#Simple-Loop-Benchmarks)\n* "
},
{
"path": "Examples/STREAMS/README.md",
"chars": 3684,
"preview": "# EMS.js STREAMS Bandwidth Benchmark\n\nThe original [STREAMS](https://www.cs.virginia.edu/stream/)\nbenchmark measures the"
},
{
"path": "Examples/STREAMS/streams_bulk_sync_parallel.js",
"chars": 6989,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Examples/STREAMS/streams_fork_join.js",
"chars": 4013,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Examples/WebServer/README.md",
"chars": 6299,
"preview": "# Parallelism Generating a Single HTTP Response\n\nUsing Fork-Join (FJ) parallelism a conventional \nHTTP server can use mu"
},
{
"path": "Examples/WebServer/webServer.js",
"chars": 8962,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Examples/concurrent_Q_and_TM.js",
"chars": 11920,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Examples/wordCount.js",
"chars": 8561,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Examples/workQ_and_TM.js",
"chars": 10884,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "LICENSE",
"chars": 2431,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Makefile",
"chars": 4490,
"preview": "#make\n# *-----------------------------------------------------------------------------+\n# | Extended Memory Semantics ("
},
{
"path": "Python/README.md",
"chars": 280,
"preview": "# Python 2 and 3\n\nThe FFI differs slightly between Python 2 and 3, these are papered over\nat runtime. There are, of cou"
},
{
"path": "Python/__init__.py",
"chars": 19,
"preview": "from ems import *\n\n"
},
{
"path": "Python/ems.py",
"chars": 33945,
"preview": "\"\"\"\n +-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS)"
},
{
"path": "Python/setup.py",
"chars": 4004,
"preview": "\"\"\"\n +-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS)"
},
{
"path": "README.md",
"chars": 16320,
"preview": "OSX | Linux | Node 4.1-14.x, Python2/3:\n[](https://tr"
},
{
"path": "Tests/3dSpace.js",
"chars": 9237,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/CASdataFlow.js",
"chars": 6323,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/ES6/harmony_proxies.js",
"chars": 3108,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/accum_omp.c",
"chars": 2907,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/barrier.js",
"chars": 4805,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/check_image_files.sh",
"chars": 1347,
"preview": "#!/bin/bash\n# -*- coding: utf-8, tab-width: 2 -*-\n\n\nfunction check_image_files () {\n local SELFPATH=\"$(readlink -m \"$BA"
},
{
"path": "Tests/fj_args.js",
"chars": 6687,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/fj_args.py",
"chars": 3679,
"preview": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\n +-------------------------------------------------------------------"
},
{
"path": "Tests/fork_join.js",
"chars": 5481,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/fullArrays.js",
"chars": 3440,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/issue11_bsp.js",
"chars": 3039,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/issue11_fj.js",
"chars": 3484,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/loopScheduling.js",
"chars": 4382,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/mapped_test.js",
"chars": 3170,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/py_api.py",
"chars": 11241,
"preview": "# -*- coding: utf-8 -*-\n\"\"\"\n +-----------------------------------------------------------------------------+\n | Extende"
},
{
"path": "Tests/readers-writer.js",
"chars": 4139,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/refactoring.js",
"chars": 5412,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/stack_and_queue.js",
"chars": 8372,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/strcpy.js",
"chars": 4024,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/stringTags.js",
"chars": 3341,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/testUtils.js",
"chars": 4125,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/test_alloc.c",
"chars": 4170,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/tm.js",
"chars": 7028,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/tm_noq.js",
"chars": 5920,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/tm_noq_strings.js",
"chars": 6107,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Tests/v8Types.js",
"chars": 7218,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "Vagrantfile",
"chars": 3214,
"preview": "# -*- mode: ruby -*-\n# vi: set ft=ruby :\n\n# All Vagrant configuration is done below. The \"2\" in Vagrant.configure\n# conf"
},
{
"path": "binding.gyp",
"chars": 619,
"preview": "{\n \"targets\": [\n {\n \"target_name\": \"ems\",\n \"sources\": [\n \"src/collectives.cc\", \"src/ems.cc\", \"src/e"
},
{
"path": "index.html",
"chars": 135,
"preview": "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta http-equiv=\"REFRESH\" content=\"0; URL=Docs/index.html\">\n</head>\n<body><"
},
{
"path": "nodejs/ems.js",
"chars": 28021,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "nodejs/nodejs.cc",
"chars": 22365,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "nodejs/nodejs.h",
"chars": 7581,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "package.json",
"chars": 1266,
"preview": "{\n \"name\": \"ems\",\n \"version\": \"1.6.1\",\n \"author\": \"Jace A Mogill <jace@mogill.com>\",\n \"description\": \"Persistent Sha"
},
{
"path": "src/collectives.cc",
"chars": 5274,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "src/ems.cc",
"chars": 43151,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "src/ems.h",
"chars": 10893,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "src/ems_alloc.cc",
"chars": 9803,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "src/ems_alloc.h",
"chars": 3219,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "src/ems_proto.h",
"chars": 5686,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "src/ems_types.h",
"chars": 3326,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "src/loops.cc",
"chars": 4677,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "src/primitives.cc",
"chars": 12740,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
},
{
"path": "src/rmw.cc",
"chars": 21614,
"preview": "/*-----------------------------------------------------------------------------+\n | Extended Memory Semantics (EMS) "
}
]
About this extraction
This page contains the full source code of the SyntheticSemantics/ems GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 69 files (561.9 KB), approximately 140.2k tokens, and a symbol index with 197 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.