Showing preview only (494K chars total). Download the full file or copy to clipboard to get everything.
Repository: eleweek/inside_python_dict
Branch: master
Commit: 8d7cda3a5324
Files: 70
Total size: 470.3 KB
Directory structure:
gitextract__zrkfi9_/
├── .babelrc
├── .gitignore
├── .prettierignore
├── LICENSE
├── README.md
├── package.json
├── patches/
│ ├── python32_debug.diff
│ ├── smooth-scrollbar+8.3.1.patch
│ └── subscribe-ui-event+2.0.4.patch
├── python_code/
│ ├── actual_dict_factory_test.py
│ ├── build_autogenerated_chapter1_hash.py
│ ├── build_autogenerated_chapter2.py
│ ├── build_autogenerated_chapter3_chapter4.py
│ ├── chapter1_linear_search_reimplementation_test.py
│ ├── chapter4_probing_python_reimplementation_test.py
│ ├── common.py
│ ├── dict32_reimplementation_test_v2.py
│ ├── dict_reimpl_common.py
│ ├── dict_reimplementation.py
│ ├── dictinfo.py
│ ├── dictinfo32.py
│ ├── dictinfo33.py
│ ├── hash_chapter1_impl.py
│ ├── hash_chapter1_reimpl_js.py
│ ├── hash_chapter1_reimplementation_test.py
│ ├── hash_chapter2_impl.py
│ ├── hash_chapter2_impl_test.py
│ ├── hash_chapter2_reimpl_js.py
│ ├── hash_chapter2_reimplementation_test.py
│ ├── hash_chapter3_class_impl.py
│ ├── hash_chapter3_class_impl_test.py
│ ├── interface_test.py
│ ├── js_reimpl_common.py
│ └── js_reimplementation_interface.py
├── scripts/
│ ├── extractPythonCode.js
│ ├── pyReimplWrapper.js
│ └── ssr.js
├── src/
│ ├── app.js
│ ├── autogenerated/
│ │ ├── chapter1.html
│ │ ├── chapter2.html
│ │ ├── chapter3.html
│ │ └── chapter4.html
│ ├── chapter1_simplified_hash.js
│ ├── chapter2_hash_table_functions.js
│ ├── chapter3_and_4_common.js
│ ├── chapter3_hash_class.js
│ ├── chapter4_real_python_dict.js
│ ├── code_blocks.js
│ ├── common_formatters.js
│ ├── hash_impl_common.js
│ ├── hash_impl_common.test.js
│ ├── index.js
│ ├── inputs.js
│ ├── mustache/
│ │ ├── chapter1.json
│ │ ├── chapter2.json
│ │ ├── chapter3.json
│ │ └── chapter4.json
│ ├── page.html.template
│ ├── probing_visualization.js
│ ├── py_obj_parsing.js
│ ├── py_obj_parsing.test.js
│ ├── store.js
│ ├── styles.css
│ └── util.js
├── ssr-all.sh
├── stress_test_python.sh
├── unittest_python.sh
├── webpack.common.js
├── webpack.dev.js
└── webpack.prod.js
================================================
FILE CONTENTS
================================================
================================================
FILE: .babelrc
================================================
{
"presets": [
[
"@babel/preset-env",
{
"modules": false
}
],
"@babel/preset-react"
],
"plugins": [
[
"@babel/plugin-proposal-decorators",
{
"legacy": true
}
],
"@babel/plugin-proposal-class-properties",
"@babel/plugin-syntax-dynamic-import",
"@babel/plugin-syntax-import-meta",
"@babel/plugin-proposal-json-strings",
"@babel/plugin-proposal-function-sent",
"@babel/plugin-proposal-export-namespace-from",
"@babel/plugin-proposal-numeric-separator",
"@babel/plugin-proposal-throw-expressions",
"@babel/plugin-proposal-optional-chaining"
],
"env": {
"test": {
"presets": [
[
"@babel/preset-env",
{
"modules": "auto"
}
],
"@babel/preset-react"
]
}
}
}
================================================
FILE: .gitignore
================================================
### PYTHON
#
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
.static_storage/
.media/
local_settings.py
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
### NODE
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
# nyc test coverage
.nyc_output
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (https://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Typescript v1 declaration files
typings/
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
# next.js build output
.next
# vim custom added
*.swp
*.swo
================================================
FILE: .prettierignore
================================================
package-lock.json
package.json
================================================
FILE: LICENSE
================================================
The MIT License (MIT)
Copyright (c) 2017-2018 Alexander Putilin
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
# Inside Python Dict - an explorable explanation
This repository contains the code for ["Inside Python Dict"](https://just-taking-a-ride.com/inside_python_dict/), a explorable explanation of python dicts.
## Code
The code is quite messy, the build system works on my laptop but might not work on your machine. That's all fixable, but I am not sure if there is any interest from people in using the actual code. If so, let me know, I'll clean things up a bit and write an overview of the codebase.
Meanwhile, try running `npm install && npm start` and see if if works.
Bugfixes will be gladly accepted.
### Disclaimer
I am providing code in the repository to you under an open source license. Because this is my personal repository, the license you receive to my code is from me and not my employer (Facebook)
================================================
FILE: package.json
================================================
{
"name": "inside_python_dict",
"version": "1.0.0",
"description": "Inside python dict - an explorable explanation",
"main": "index.js",
"scripts": {
"jest": "jest --env=node",
"test:pystress": "npm run extractcode && ./stress_test_python.sh",
"test:pyunit": "npm run extractcode && ./unittest_python.sh",
"test": "npm run jest && npm run test:pyunit && npm run test:pystress",
"build:ssr": "./ssr-all.sh",
"update:html": "mkdir -p build && (for i in {chapter1,chapter2,chapter3,chapter4}; do mustache src/mustache/$i.json src/page.html.template > src/autogenerated/$i.html; done)",
"start": "webpack-dev-server --config webpack.dev.js --host 0.0.0.0",
"serve": "http-server -p 9090 dist/",
"build": "npm run build:ssr && webpack --config webpack.prod.js",
"babel-node": "npx babel-node --presets '@babel/env'",
"extractcode": "mkdir -p build && npm run babel-node scripts/extractPythonCode.js",
"dictserver": "rm pynode.sock ; npm run babel-node scripts/pyReimplWrapper.js; rm pynode.sock",
"postinstall": "patch-package"
},
"repository": {
"type": "git",
"url": "git+https://github.com/eleweek/inside_python_dict.git"
},
"author": "Alexander Putilin",
"license": "MIT",
"bugs": {
"url": "https://github.com/eleweek/inside_python_dict/issues"
},
"homepage": "https://github.com/eleweek/inside_python_dict#readme",
"prettier": {
"printWidth": 120,
"tabWidth": 4,
"useTabs": false,
"singleQuote": true,
"bracketSpacing": false,
"semi": true,
"trailingComma": "es5"
},
"dependencies": {
"@fortawesome/fontawesome-svg-core": "^1.2.10",
"@fortawesome/free-brands-svg-icons": "^5.6.1",
"@fortawesome/free-solid-svg-icons": "^5.6.1",
"@fortawesome/react-fontawesome": "^0.1.3",
"bignumber.js": "^8.0.1",
"bootstrap": "^4.1.3",
"bowser": "^2.0.0-beta.3",
"classnames": "^2.2.6",
"d3": "^5.7.0",
"d3-selection-multi": "^1.0.1",
"i": "^0.3.6",
"immutable": "^4.0.0-rc.12",
"lodash": "^4.17.11",
"lowlight": "^1.11.0",
"memoize-one": "^4.1.0",
"mobx": "^5.8.0",
"mobx-react": "^5.4.3",
"rc-slider": "^8.6.4",
"react": "^16.6.3",
"react-css-transition-replace": "^3.0.3",
"react-dom": "^16.6.3",
"react-error-boundary": "^1.2.3",
"react-input-autosize": "^2.2.1",
"react-popper": "^1.3.2",
"react-smooth-scrollbar": "^8.0.6",
"react-stickynode": "^2.1.0",
"rehype": "^7.0.0",
"smooth-scrollbar": "8.3.1"
},
"devDependencies": {
"@babel/cli": "^7.2.0",
"@babel/core": "^7.2.2",
"@babel/node": "^7.2.2",
"@babel/plugin-proposal-class-properties": "^7.2.1",
"@babel/plugin-proposal-decorators": "^7.2.2",
"@babel/plugin-proposal-export-namespace-from": "^7.2.0",
"@babel/plugin-proposal-function-sent": "^7.2.0",
"@babel/plugin-proposal-json-strings": "^7.2.0",
"@babel/plugin-proposal-numeric-separator": "^7.2.0",
"@babel/plugin-proposal-object-rest-spread": "^7.2.0",
"@babel/plugin-proposal-optional-chaining": "^7.2.0",
"@babel/plugin-proposal-throw-expressions": "^7.2.0",
"@babel/plugin-syntax-dynamic-import": "^7.2.0",
"@babel/plugin-syntax-import-meta": "^7.2.0",
"@babel/plugin-transform-destructuring": "^7.2.0",
"@babel/preset-env": "^7.2.0",
"@babel/preset-react": "^7.0.0",
"babel-core": "^7.0.0-bridge.0",
"babel-jest": "^23.4.2",
"babel-loader": "^8.0.0",
"babel-plugin-lodash": "^3.3.4",
"clean-webpack-plugin": "^1.0.0",
"css-loader": "^2.0.1",
"dotenv": "^6.2.0",
"html-webpack-plugin": "^3.2.0",
"http-server": "^0.11.1",
"husky": "^1.2.1",
"ignore-styles": "^5.0.1",
"jest": "^23.6.0",
"mini-css-extract-plugin": "^0.5.0",
"mustache": "^3.0.1",
"npm": "^6.5.0",
"patch-package": "^5.1.1",
"prettier": "^1.15.3",
"pretty-quick": "^1.8.0",
"split": "^1.0.1",
"style-loader": "^0.23.1",
"unminified-webpack-plugin": "^2.0.0",
"webpack": "^4.27.1",
"webpack-bundle-analyzer": "^3.0.3",
"webpack-cli": "^3.1.2",
"webpack-dev-server": "^3.1.10",
"webpack-merge": "^4.1.5"
},
"husky": {
"hooks": {
"pre-commit": "pretty-quick --staged"
}
}
}
================================================
FILE: patches/python32_debug.diff
================================================
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
index c10bfccdce..3734a08281 100644
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -321,6 +321,8 @@ lookdict(PyDictObject *mp, PyObject *key, register Py_hash_t hash)
PyObject *startkey;
i = (size_t)hash & mask;
+ fprintf(stderr, "lookdict hash = %ld\n", hash);
+ fprintf(stderr, "initial i = %zu\n", i);
ep = &ep0[i];
if (ep->me_key == NULL || ep->me_key == key)
return ep;
@@ -355,7 +357,9 @@ lookdict(PyDictObject *mp, PyObject *key, register Py_hash_t hash)
least likely outcome, so test for that last. */
for (perturb = hash; ; perturb >>= PERTURB_SHIFT) {
i = (i << 2) + i + perturb + 1;
+ fprintf(stderr, "next i = %zu perturb = %zu\n", i, perturb);
ep = &ep0[i & mask];
+ fprintf(stderr, "next i & mask = %zu perturb = %zu\n", i & mask, perturb);
if (ep->me_key == NULL)
return freeslot == NULL ? ep : freeslot;
if (ep->me_key == key)
@@ -648,6 +652,7 @@ dictresize(PyDictObject *mp, Py_ssize_t minused)
}
}
else {
+ fprintf(stderr, "PyMem_NEW branch");
newtable = PyMem_NEW(PyDictEntry, newsize);
if (newtable == NULL) {
PyErr_NoMemory();
@@ -693,6 +698,7 @@ PyObject *
_PyDict_NewPresized(Py_ssize_t minused)
{
PyObject *op = PyDict_New();
+ fprintf(stderr, "_PyDict_NewPresized() %p %d\n", op, (int)minused);
if (minused>5 && op != NULL && dictresize((PyDictObject *)op, minused) == -1) {
Py_DECREF(op);
diff --git a/Objects/longobject.c b/Objects/longobject.c
index e2a4ef9c5e..7d72c88417 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -2611,6 +2611,7 @@ long_hash(PyLongObject *v)
sign = -1;
i = -(i);
}
+ fprintf(stderr, "i = %ld\n", i);
while (--i >= 0) {
/* Here x is a quantity in the range [0, _PyHASH_MODULUS); we
want to compute x * 2**PyLong_SHIFT + v->ob_digit[i] modulo
================================================
FILE: patches/smooth-scrollbar+8.3.1.patch
================================================
patch-package
--- a/node_modules/smooth-scrollbar/events/touch.js
+++ b/node_modules/smooth-scrollbar/events/touch.js
@@ -1,7 +1,7 @@
import { eventScope, TouchRecord, } from '../utils/';
var activeScrollbar;
export function touchHandler(scrollbar) {
- var MIN_EAING_MOMENTUM = 50;
+ var MIN_EAING_MOMENTUM = 3;
var EASING_MULTIPLIER = /Android/.test(navigator.userAgent) ? 3 : 2;
var target = scrollbar.options.delegateTo || scrollbar.containerEl;
var touchRecord = new TouchRecord();
--- a/node_modules/smooth-scrollbar/geometry/update.js
+++ b/node_modules/smooth-scrollbar/geometry/update.js
@@ -4,6 +4,9 @@ export function update(scrollbar) {
x: Math.max(newSize.content.width - newSize.container.width, 0),
y: Math.max(newSize.content.height - newSize.container.height, 0),
};
+ // hack for a weird chrome on windows bug
+ if (limit.x <= 2) limit.x = 0;
+ if (limit.y <= 2) limit.y = 0;
// metrics
var containerBounding = scrollbar.containerEl.getBoundingClientRect();
var bounding = {
--- a/node_modules/smooth-scrollbar/scrollbar.js
+++ b/node_modules/smooth-scrollbar/scrollbar.js
@@ -322,6 +322,10 @@ var Scrollbar = /** @class */ (function () {
if (limit.x === 0 && limit.y === 0) {
this._updateDebounced();
}
+ if (Math.abs(deltaY) > Math.abs(deltaX)) {
+ if (deltaY > 0 && offset.y === limit.y) return true;
+ if (deltaY < 0 && offset.y === 0) return true;
+ }
var destX = clamp(deltaX + offset.x, 0, limit.x);
var destY = clamp(deltaY + offset.y, 0, limit.y);
var res = true;
--- a/node_modules/smooth-scrollbar/track/track.js
+++ b/node_modules/smooth-scrollbar/track/track.js
@@ -41,8 +41,9 @@ var ScrollbarTrack = /** @class */ (function () {
this.element.classList.remove('show');
};
ScrollbarTrack.prototype.update = function (scrollOffset, containerSize, pageSize) {
+ // -2 is a hack for a weird chrome on windows bug
setStyle(this.element, {
- display: pageSize <= containerSize ? 'none' : 'block',
+ display: pageSize - 2 <= containerSize ? 'none' : 'block',
});
this.thumb.update(scrollOffset, containerSize, pageSize);
};
deleted file mode 100644
--- a/node_modules/smooth-scrollbar/track/track.js.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"file":"track.js","sourceRoot":"","sources":["../src/track/track.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EACL,QAAQ,GACT,MAAM,WAAW,CAAC;AAEnB;IAUE,wBACE,SAAyB,EACzB,YAAwB;QAAxB,6BAAA,EAAA,gBAAwB;QAT1B;;WAEG;QACM,YAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAEzC,aAAQ,GAAG,KAAK,CAAC;QAMvB,IAAI,CAAC,OAAO,CAAC,SAAS,GAAG,qCAAmC,SAAW,CAAC;QAExE,IAAI,CAAC,KAAK,GAAG,IAAI,cAAc,CAC7B,SAAS,EACT,YAAY,CACb,CAAC;QAEF,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;;;OAIG;IACH,iCAAQ,GAAR,UAAS,kBAA+B;QACtC,kBAAkB,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,6BAAI,GAAJ;QACE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YAClB,MAAM,CAAC;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,6BAAI,GAAJ;QACE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YACnB,MAAM,CAAC;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;IAED,+BAAM,GAAN,UACE,YAAoB,EACpB,aAAqB,EACrB,QAAgB;QAEhB,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE;YACrB,OAAO,EAAE,QAAQ,IAAI,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;SACtD,CAAC,CAAC;QAEH,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;IAC3D,CAAC;IACH,qBAAC;AAAD,CAAC,AApED,IAoEC"}
\ No newline at end of file
================================================
FILE: patches/subscribe-ui-event+2.0.4.patch
================================================
patch-package
new file mode 100644
Binary files /dev/null and b/node_modules/subscribe-ui-event/.index.es.js.swp differ
--- a/node_modules/subscribe-ui-event/index.es.js
+++ b/node_modules/subscribe-ui-event/index.es.js
@@ -7,7 +7,7 @@ import listenLib from './dist-es/lib/listen';
import subscribeLib from './dist-es/subscribe';
import unsubscribeLib from './dist-es/unsubscribe';
-const IS_CLIENT = typeof window !== 'undefined';
+var IS_CLIENT = typeof window !== 'undefined';
function warn() {
if (process.env.NODE_ENV !== 'production') {
@@ -15,6 +15,6 @@ function warn() {
}
}
-export const listen = IS_CLIENT ? listenLib : warn;
-export const subscribe = IS_CLIENT ? subscribeLib : warn;
-export const unsubscribe = IS_CLIENT ? unsubscribeLib : warn;
+export var listen = IS_CLIENT ? listenLib : warn;
+export var subscribe = IS_CLIENT ? subscribeLib : warn;
+export var unsubscribe = IS_CLIENT ? unsubscribeLib : warn;
================================================
FILE: python_code/actual_dict_factory_test.py
================================================
import unittest
from dict32_reimplementation_test_v2 import dict_factory
from dictinfo import dump_py_dict
def table_size(d):
return len(dump_py_dict(d)[0])
class TestDictFactory(unittest.TestCase):
def test_dict_factory(self):
self.assertEqual(table_size(dict_factory([])), 8)
self.assertEqual(table_size(dict_factory([(1, 1)])), 8)
self.assertEqual(table_size(dict_factory([(1, 1), (1, 2), (1, 3), (1, 4)])), 8)
self.assertEqual(table_size(dict_factory([(1, 1), (1, 2), (1, 3), (1, 4), (1, 5)])), 8)
self.assertEqual(table_size(dict_factory([(1, 1), (1, 2), (1, 3), (1, 4), (1, 5), (1, 6), (1, 7), (1, 8)])), 16)
self.assertEqual(table_size({1: 1, 1: 2, 1: 3, 1: 4, 1: 5, 1: 6, 1: 7, 1: 8}), 16)
self.assertEqual(table_size(dict([(1, 1), (1, 2), (1, 3), (1, 4), (1, 5), (1, 6), (1, 7), (1, 8)])), 8)
self.assertEqual(table_size({"x": "y", "abde": 1, "cdef": 4, "world": 9, "hmmm": 16, "hello": 25, "xxx": 36, "ya": 49, "hello,world!": 64, "well": 81, "meh": 100}), 64)
def main():
unittest.main()
if __name__ == "__main__":
main()
================================================
FILE: python_code/build_autogenerated_chapter1_hash.py
================================================
import os
import sys
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'build'))
from hash_chapter1_extracted import *
def create_new(numbers):
return build_insert_all(numbers)
def create_new_broken(numbers):
return build_not_quite_what_we_want(numbers)
def has_key(keys, key):
return has_number(keys, key)
def linear_search(numbers, number):
return simple_search(numbers, number)
================================================
FILE: python_code/build_autogenerated_chapter2.py
================================================
import os
import sys
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'build'))
from hash_chapter2_extracted import *
================================================
FILE: python_code/build_autogenerated_chapter3_chapter4.py
================================================
import os
import sys
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'build'))
from dict32js_extracted import Dict32Extracted
from hash_class_recycling_extracted import HashClassRecyclingExtracted
from hash_class_no_recycling_extracted import HashClassNoRecyclingExtracted
================================================
FILE: python_code/chapter1_linear_search_reimplementation_test.py
================================================
import random
import argparse
import hash_chapter1_reimpl_js
import hash_chapter1_impl
import build_autogenerated_chapter1_hash
IMPLEMENTATIONS = {
'ref': hash_chapter1_impl.linear_search,
'js': hash_chapter1_reimpl_js.linear_search,
'py_extracted': build_autogenerated_chapter1_hash.linear_search
}
def run(test_implementation, size):
MAX_VAL = 5000
ref_search = IMPLEMENTATIONS['ref']
test_search = IMPLEMENTATIONS[test_implementation]
numbers = [random.randint(-MAX_VAL, MAX_VAL) for _ in range(size)]
for number in numbers:
assert ref_search(numbers, number)
assert test_search(numbers, number)
for i in range(size * 3):
number = random.randint(-MAX_VAL, MAX_VAL)
assert ref_search(numbers, number) == test_search(numbers, number)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Stress-test chapter1 reimplementation')
parser.add_argument('--test-implementation', choices=['py_extracted', 'js'], required=True)
parser.add_argument('--size', type=int, default=100)
args = parser.parse_args()
run(test_implementation=args.test_implementation,
size=args.size)
================================================
FILE: python_code/chapter4_probing_python_reimplementation_test.py
================================================
import json
from common import AllKeyValueFactory
from js_reimpl_common import _init_sock_stuff, dump_simple_py_obj
from pprint import pprint
sock, sockfile = _init_sock_stuff()
def probe_all_js(key, slots_count):
global sockfile
global sock
data = {
"dict": "pythonProbing",
"args": {
'key': dump_simple_py_obj(key),
'slotsCount': slots_count
},
}
sock.send(bytes(json.dumps(data) + "\n", 'UTF-8'))
response = json.loads(sockfile.readline())
return response['result']
def probe_all(key, slots_count=8):
PERTURB_SHIFT = 5
links = [[] for _ in range(slots_count)]
hash_code = hash(key)
perturb = 2**64 + hash_code if hash_code < 0 else hash_code
idx = hash_code % slots_count
start_idx = idx
visited = set()
while len(visited) < slots_count:
visited.add(idx)
next_idx = (idx * 5 + perturb + 1) % slots_count
links[idx].append({'nextIdx': next_idx, 'perturbLink': perturb != 0})
idx = next_idx
perturb >>= PERTURB_SHIFT
return {'startIdx': start_idx, 'links': links}
def test():
factory = AllKeyValueFactory(100)
for slots_count in [8, 16, 32]:
for i in range(300):
key = factory.generate_key()
assert probe_all(key, slots_count) == probe_all_js(key, slots_count)
if __name__ == "__main__":
test()
================================================
FILE: python_code/common.py
================================================
import random
import string
class EmptyValueClass(object):
def __str__(self):
return "EMPTY"
def __repr__(self):
return "<EMPTY>"
class DummyValueClass(object):
def __str__(self):
return "<DUMMY>"
def __repr__(self):
return "<DUMMY>"
EMPTY = EmptyValueClass()
DUMMY = DummyValueClass()
def get_object_field_or_null(obj, field_name):
try:
return getattr(obj, field_name)
except ValueError:
return EMPTY
def get_object_field_or_none(obj, field_name):
try:
return getattr(obj, field_name)
except ValueError:
return None
def generate_random_string(str_len=5):
# FROM: https://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python
return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(str_len))
_unicode_chars = string.ascii_uppercase + string.digits + "йцукенгшщзхъфывапролджэячсмитьбю"
def generate_random_unicode(str_len):
# FROM: https://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python
return ''.join(random.choice(_unicode_chars) for _ in range(str_len))
class IntKeyValueFactory(object):
def __init__(self, n_inserts):
self.n_inserts = n_inserts
self._insert_count = 0
self._key_range = list(range(n_inserts))
def generate_key(self):
return random.choice(self._key_range)
def generate_value(self):
self._insert_count += 1
return self._insert_count
class AllKeyValueFactory(object):
def __init__(self, n_inserts, int_chance=0.1, long_chance=0.1, len0_chance=0.01, len1_chance=0.1, len2_chance=0.3, len3_chance=0.2, len_random_chance=0.17):
self.int_pbf = int_chance
self.long_pbf = self.int_pbf + long_chance
self.len0_pbf = self.int_pbf + len0_chance
self.len1_pbf = self.len0_pbf + len1_chance
self.len2_pbf = self.len1_pbf + len2_chance
self.len3_pbf = self.len2_pbf + len3_chance
self.len_random_pbf = self.len3_pbf + len_random_chance
assert 0.0 <= self.len3_pbf <= 1.0
half_range = int(n_inserts / 2)
self._int_range = [i - half_range for i in range(2 * half_range)]
def _generate_obj(self):
r = random.random()
if r <= self.int_pbf:
return random.choice(self._int_range)
if r <= self.long_pbf:
sign = "-" if random.random() < 0.5 else ""
first_digit = random.choice("123456789")
return sign + first_digit + ''.join(random.choice("0123456789") for _ in range(random.randint(20, 50)))
if r <= self.len0_pbf:
return ""
if r <= self.len1_pbf:
return generate_random_unicode(1)
if r <= self.len2_pbf:
return generate_random_unicode(2)
if r <= self.len3_pbf:
return generate_random_unicode(3)
if r <= self.len_random_pbf:
return generate_random_unicode(random.randint(4, 25))
return None
def generate_key(self):
return self._generate_obj()
def generate_value(self):
return self._generate_obj()
================================================
FILE: python_code/dict32_reimplementation_test_v2.py
================================================
import random
import argparse
import json
from common import EMPTY, AllKeyValueFactory, IntKeyValueFactory
from dictinfo import dump_py_dict
from dict_reimplementation import PyDictReimplementation32, dump_reimpl_dict
from js_reimplementation_interface import Dict32JsImpl, AlmostPythonDictRecyclingJsImpl, AlmostPythonDictNoRecyclingJsImpl
import hash_chapter3_class_impl
import build_autogenerated_chapter3_chapter4
def dict_factory(pairs=None):
if not pairs:
return {}
# quick&dirty
def to_string(x):
return json.dumps(x) if x is not None else "None"
d = eval("{" + ", ".join("{}:{}".format(to_string(k), to_string(v)) for [k, v] in pairs) + "}")
return d
IMPLEMENTATIONS = {
"dict_actual": (dict_factory, dump_py_dict),
"dict32_reimpl_py": (PyDictReimplementation32, dump_reimpl_dict),
"dict32_reimpl_js": (Dict32JsImpl, dump_reimpl_dict),
"dict32_reimpl_py_extracted": (build_autogenerated_chapter3_chapter4.Dict32Extracted, dump_reimpl_dict),
"almost_python_dict_recycling_py": (hash_chapter3_class_impl.AlmostPythonDictImplementationRecycling, dump_reimpl_dict),
"almost_python_dict_no_recycling_py": (hash_chapter3_class_impl.AlmostPythonDictImplementationNoRecycling, dump_reimpl_dict),
"almost_python_dict_no_recycling_py_simpler": (hash_chapter3_class_impl.AlmostPythonDictImplementationNoRecyclingSimplerVersion, dump_reimpl_dict),
"almost_python_dict_recycling_js": (AlmostPythonDictRecyclingJsImpl, dump_reimpl_dict),
"almost_python_dict_no_recycling_js": (AlmostPythonDictNoRecyclingJsImpl, dump_reimpl_dict),
"almost_python_dict_recycling_py_extracted": (build_autogenerated_chapter3_chapter4.HashClassRecyclingExtracted, dump_reimpl_dict),
"almost_python_dict_no_recycling_py_extracted": (build_autogenerated_chapter3_chapter4.HashClassNoRecyclingExtracted, dump_reimpl_dict),
}
def verify_same(d, dump_d_func, dreimpl, dump_dreimpl_func):
dump_d = dump_d_func(d)
dump_reimpl = dump_dreimpl_func(dreimpl)
if dump_d != dump_reimpl:
hashes_orig, keys_orig, values_orig, fill_orig, used_orig = dump_d
hashes_new, keys_new, values_new, fill_new, used_new = dump_reimpl
print("ORIG SIZE", len(hashes_orig))
print("NEW SIZE", len(hashes_new))
print("ORIG fill/used: ", fill_orig, used_orig)
print("NEW fill/used: ", fill_new, used_new)
if len(hashes_orig) == len(hashes_new):
size = len(hashes_orig)
print("NEW | ORIG")
for i in range(size):
if hashes_new[i] is not EMPTY or hashes_orig[i] is not EMPTY:
print(i, " " * 3,
hashes_new[i], keys_new[i], values_new[i], " " * 3,
hashes_orig[i], keys_orig[i], values_orig[i])
assert dump_d == dump_reimpl
def run(ref_impl_factory, ref_impl_dump, test_impl_factory, test_impl_dump, n_inserts, extra_checks, key_value_factory, initial_state, verbose):
SINGLE_REMOVE_CHANCE = 0.3
MASS_REMOVE_CHANCE = 0.002
MASS_REMOVE_COEFF = 0.8
removed = set()
if initial_state:
d = ref_impl_factory(initial_state)
else:
d = ref_impl_factory()
if initial_state:
dreimpl = test_impl_factory(initial_state)
else:
dreimpl = test_impl_factory()
if verbose:
print("Starting test")
for i in range(n_inserts):
should_remove = (random.random() < SINGLE_REMOVE_CHANCE)
if should_remove and d and d.keys(): # TODO: ugly, written while on a plane
to_remove = random.choice(list(d.keys()))
if verbose:
print("Removing {}".format(to_remove))
del d[to_remove]
del dreimpl[to_remove]
if verbose:
print(d)
verify_same(d, ref_impl_dump, dreimpl, test_impl_dump)
removed.add(to_remove)
should_mass_remove = (random.random() < MASS_REMOVE_CHANCE)
if should_mass_remove and len(d) > 10:
to_remove_list = random.sample(list(d.keys()), int(MASS_REMOVE_COEFF * len(d)))
if verbose:
print("Mass-Removing {} elements".format(len(to_remove_list)))
for k in to_remove_list:
del d[k]
del dreimpl[k]
removed.add(k)
if extra_checks:
for k in d.keys():
assert d[k] == dreimpl[k]
for r in removed:
try:
dreimpl[r]
assert False
except KeyError:
pass
key_to_insert = key_value_factory.generate_key()
value_to_insert = key_value_factory.generate_value()
_keys_set = getattr(d, '_keys_set', None)
# TODO: ugly code written on a plane
# TODO: properly implement in/not in when I land
if _keys_set is not None:
key_present = key_to_insert in _keys_set
else:
key_present = key_to_insert in d
if not key_present:
if verbose:
print("Inserting ({key}, {value})".format(key=key_to_insert, value=value_to_insert))
try:
dreimpl[key_to_insert]
assert False
except KeyError:
pass
else:
if verbose:
print("Replacing ({key}, {value1}) with ({key}, {value2})".format(key=key_to_insert, value1=d[key_to_insert], value2=value_to_insert))
removed.discard(key_to_insert)
d[key_to_insert] = value_to_insert
dreimpl[key_to_insert] = value_to_insert
if verbose:
print(d)
verify_same(d, ref_impl_dump, dreimpl, test_impl_dump)
assert dreimpl[key_to_insert] == value_to_insert
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Stress-test dict-like reimplementations')
parser.add_argument('--reference-implementation', choices=IMPLEMENTATIONS.keys(), required=True)
parser.add_argument('--test-implementation', choices=IMPLEMENTATIONS.keys(), required=True)
parser.add_argument('--no-extra-getitem-checks', dest='extra_checks', action='store_false')
parser.add_argument('--num-inserts', type=int, default=500)
parser.add_argument('--forever', action='store_true')
parser.add_argument('--kv', choices=["numbers", "all"], required=True)
parser.add_argument('--initial-size', type=int, default=-1)
parser.add_argument('--verbose', action='store_true')
args = parser.parse_args()
if args.kv == "numbers":
kv_factory = IntKeyValueFactory(args.num_inserts)
elif args.kv == "all":
kv_factory = AllKeyValueFactory(args.num_inserts)
ref_impl = IMPLEMENTATIONS[args.reference_implementation]
test_impl = IMPLEMENTATIONS[args.test_implementation]
def test_iteration():
initial_size = args.initial_size if args.initial_size >= 0 else random.randint(0, 100)
initial_state = [(kv_factory.generate_key(), kv_factory.generate_value()) for _ in range(initial_size)]
run(*(ref_impl + test_impl),
n_inserts=args.num_inserts,
extra_checks=args.extra_checks,
key_value_factory=kv_factory,
initial_state=initial_state,
verbose=args.verbose)
if args.forever:
while True:
test_iteration()
else:
test_iteration()
================================================
FILE: python_code/dict_reimpl_common.py
================================================
from common import EMPTY
class Slot(object):
def __init__(self, hash_code=EMPTY, key=EMPTY, value=EMPTY):
self.hash_code = hash_code
self.key = key
self.value = value
class BaseDictImpl(object):
def __init__(self):
self.slots = [Slot() for _ in range(self.START_SIZE)]
self.fill = 0
self.used = 0
def find_nearest_size(self, minused):
new_size = 8
while new_size <= minused:
new_size *= 2
return new_size
================================================
FILE: python_code/dict_reimplementation.py
================================================
from common import DUMMY, EMPTY
from dict_reimpl_common import BaseDictImpl, Slot
from operator import attrgetter
class PyDictReimplementationBase(BaseDictImpl):
START_SIZE = 8
PERTURB_SHIFT = 5
def __init__(self, pairs=None):
BaseDictImpl.__init__(self)
start_size = self.find_nearest_size(len(pairs)) if pairs else self.START_SIZE
self.slots = [Slot() for _ in range(start_size)]
if pairs:
for k, v in pairs:
self[k] = v
def __setitem__(self, key, value):
hash_code = hash(key)
perturb = self.signed_to_unsigned(hash_code)
idx = hash_code % len(self.slots)
target_idx = None
while self.slots[idx].key is not EMPTY:
if self.slots[idx].hash_code == hash_code and self.slots[idx].key == key:
target_idx = idx
break
if target_idx is None and self.slots[idx].key is DUMMY:
target_idx = idx
idx = (idx * 5 + perturb + 1) % len(self.slots)
perturb >>= self.PERTURB_SHIFT
if target_idx is None:
target_idx = idx
if self.slots[target_idx].key is EMPTY:
self.used += 1
self.fill += 1
elif self.slots[target_idx].key is DUMMY:
self.used += 1
self.slots[target_idx] = Slot(hash_code, key, value)
if self.fill * 3 >= len(self.slots) * 2:
self.resize()
def __delitem__(self, key):
idx = self.lookdict(key)
self.used -= 1
self.slots[idx].key = DUMMY
self.slots[idx].value = EMPTY
def __getitem__(self, key):
idx = self.lookdict(key)
return self.slots[idx].value
@staticmethod
def signed_to_unsigned(hash_code):
return 2**64 + hash_code if hash_code < 0 else hash_code
def lookdict(self, key):
hash_code = hash(key)
perturb = self.signed_to_unsigned(hash_code)
idx = hash_code % len(self.slots)
while self.slots[idx].key is not EMPTY:
if self.slots[idx].hash_code == hash_code and self.slots[idx].key == key:
return idx
idx = (idx * 5 + perturb + 1) % len(self.slots)
perturb >>= self.PERTURB_SHIFT
raise KeyError()
def resize(self):
old_slots = self.slots
new_size = self.find_nearest_size(self._next_size())
self.slots = [Slot() for _ in range(new_size)]
self.fill = self.used
for slot in old_slots:
if slot.key is not EMPTY and slot.key is not DUMMY:
perturb = self.signed_to_unsigned(slot.hash_code)
idx = slot.hash_code % len(self.slots)
while self.slots[idx].key is not EMPTY:
idx = (idx * 5 + perturb + 1) % len(self.slots)
perturb >>= self.PERTURB_SHIFT
self.slots[idx] = Slot(slot.hash_code, slot.key, slot.value)
class PyDictReimplementation32(PyDictReimplementationBase):
def _next_size(self):
return self.used * (4 if self.used <= 50000 else 2)
def dump_reimpl_dict(d):
def extract_fields(field_name):
return list(map(attrgetter(field_name), d.slots))
return extract_fields('hash_code'), extract_fields('key'), extract_fields('value'), d.fill, d.used
================================================
FILE: python_code/dictinfo.py
================================================
import sys
def dump_py_dict(d):
vi = sys.version_info
if vi.major != 3:
raise Exception("Unsupported major version")
if vi.minor < 2:
raise Exception("Unsupported minor version (too old)")
if vi.minor > 3:
raise Exception("Unsupported minor version (too new)")
if vi.minor == 2:
import dictinfo32
return dictinfo32.dump_py_dict(d)
else:
import dictinfo33
return dictinfo33.dump_py_dict(d)
================================================
FILE: python_code/dictinfo32.py
================================================
from ctypes import Structure, c_ulong, POINTER, cast, py_object, c_long
from common import get_object_field_or_null, EMPTY, DUMMY
class PyDictEntry(Structure):
_fields_ = [
('me_hash', c_long),
('me_key', py_object),
('me_value', py_object),
]
class PyDictObject(Structure):
_fields_ = [
('ob_refcnt', c_ulong),
('ob_type', c_ulong),
('ma_fill', c_ulong),
('ma_used', c_ulong),
('ma_mask', c_ulong),
('ma_table', POINTER(PyDictEntry)),
]
def dictobject(d):
return cast(id(d), POINTER(PyDictObject)).contents
d = {0: 0}
del d[0]
dummy_internal = dictobject(d).ma_table[0].me_key
del d
def dump_py_dict(d):
do = dictobject(d)
keys = []
hashes = []
values = []
size = do.ma_mask + 1
for i in range(size):
key = get_object_field_or_null(do.ma_table[i], 'me_key')
keys.append(key if key is not dummy_internal else DUMMY)
for i, key in enumerate(keys):
if key is EMPTY:
hashes.append(EMPTY)
values.append(EMPTY)
else:
hashes.append(do.ma_table[i].me_hash)
values.append(get_object_field_or_null(do.ma_table[i], 'me_value'))
return hashes, keys, values, do.ma_fill, do.ma_used
================================================
FILE: python_code/dictinfo33.py
================================================
from ctypes import Structure, c_ulong, POINTER, cast, addressof, py_object, c_long, c_void_p
from common import get_object_field_or_null, EMPTY, DUMMY
class PyDictKeyEntry(Structure):
_fields_ = [
('me_hash', c_long),
('me_key', py_object),
('me_value', py_object),
]
class PyDictKeysObject(Structure):
_fields_ = [
('dk_refcnt', c_long),
('dk_size', c_long),
('dict_lookup_func', POINTER(c_void_p)),
('dk_usable', c_long),
('dk_entries', PyDictKeyEntry),
]
class PyDictObject(Structure):
_fields_ = [
('ob_refcnt', c_ulong),
('ob_type', c_ulong),
('ma_used', c_long),
('ma_keys', POINTER(PyDictKeysObject)),
# Not actually a void*, split tables are not supported right now
('ma_values', POINTER(c_void_p))
]
def dictobject(d):
return cast(id(d), POINTER(PyDictObject)).contents
d = {0: 0}
del d[0]
dummy_internal = dictobject(d).ma_keys.contents.dk_entries.me_key
del d
def usable_fraction(size):
return (size * 2 + 1) // 3
def dump_py_dict(d):
do = dictobject(d)
keys = []
hashes = []
values = []
size = do.ma_keys.contents.dk_size
entries = cast(addressof(do.ma_keys.contents.dk_entries), POINTER(PyDictKeyEntry))
for i in range(size):
key = get_object_field_or_null(entries[i], 'me_key')
keys.append(key if key is not dummy_internal else DUMMY)
for i, key in enumerate(keys):
if key is EMPTY:
hashes.append(EMPTY)
values.append(EMPTY)
else:
hashes.append(entries[i].me_hash)
values.append(get_object_field_or_null(entries[i], 'me_value'))
return hashes, keys, values, usable_fraction(do.ma_keys.contents.dk_size) - do.ma_keys.contents.dk_usable, do.ma_used
================================================
FILE: python_code/hash_chapter1_impl.py
================================================
def create_new(numbers):
n = len(numbers)
keys = [None for i in range(2 * n)]
for num in numbers:
idx = num % len(keys)
while keys[idx] is not None:
idx = (idx + 1) % len(keys)
keys[idx] = num
return keys
def create_new_broken(numbers):
n = len(numbers)
keys = [None for i in range(n)]
for num in numbers:
idx = num % len(keys)
keys[idx] = num
return keys
def has_key(keys, key):
idx = key % len(keys)
while keys[idx] is not None:
if keys[idx] == key:
return True
idx = (idx + 1) % len(keys)
return False
def linear_search(numbers, number):
return number in numbers
================================================
FILE: python_code/hash_chapter1_reimpl_js.py
================================================
from js_reimpl_common import run_op_chapter1_chapter2
def run_op(keys, op, **kwargs):
return run_op_chapter1_chapter2("chapter1", None, keys, op, **kwargs)
def create_new(numbers):
return run_op(None, "create_new", array=numbers)
def create_new_broken(numbers):
return run_op(None, "create_new_broken", array=numbers)
def has_key(keys, key):
return run_op(keys, "has_key", key=key)
def linear_search(numbers, key):
return run_op(None, "linear_search", key=key, array=numbers)
================================================
FILE: python_code/hash_chapter1_reimplementation_test.py
================================================
import random
import argparse
import hash_chapter1_reimpl_js
import hash_chapter1_impl
import build_autogenerated_chapter1_hash
def get_implementation(is_broken, impl):
if impl == "js":
module = hash_chapter1_reimpl_js
elif impl == "py_ref":
module = hash_chapter1_impl
elif impl == "py_extracted":
module = build_autogenerated_chapter1_hash
else:
assert False
return (module.create_new_broken if is_broken else module.create_new, module.has_key)
def run(test_implementation, is_broken, n_inserts):
MAX_VAL = 5000
ref_create_new, ref_has_key = get_implementation(is_broken, "py_ref")
test_create_new, test_has_key = get_implementation(is_broken, test_implementation)
numbers = list(set(random.randint(-MAX_VAL, MAX_VAL) for _ in range(n_inserts)))
ref_keys = ref_create_new(numbers)
test_keys = test_create_new(numbers)
for number in numbers:
if not is_broken:
assert ref_has_key(ref_keys, number)
assert test_has_key(test_keys, number)
else:
assert ref_has_key(ref_keys, number) == test_has_key(test_keys, number)
for i in range(n_inserts * 3):
number = random.randint(-MAX_VAL, MAX_VAL)
assert ref_has_key(ref_keys, number) == test_has_key(test_keys, number)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Stress-test chapter1 reimplementation')
parser.add_argument('--is-broken', action='store_true')
parser.add_argument('--test-implementation', choices=['py_extracted', 'js'], required=True)
parser.add_argument('--num-inserts', type=int, default=500)
args = parser.parse_args()
run(test_implementation=args.test_implementation,
is_broken=args.is_broken,
n_inserts=args.num_inserts)
================================================
FILE: python_code/hash_chapter2_impl.py
================================================
from common import DUMMY, EMPTY
def create_new(from_keys):
n = len(from_keys)
hash_codes = [EMPTY for i in range(2 * n)]
keys = [EMPTY for i in range(2 * n)]
for key in from_keys:
hash_code = hash(key)
idx = hash_code % len(keys)
while keys[idx] is not EMPTY:
if hash_codes[idx] == hash_code and keys[idx] == key:
break
idx = (idx + 1) % len(keys)
hash_codes[idx] = hash_code
keys[idx] = key
return hash_codes, keys
def insert(hash_codes, keys, key):
hash_code = hash(key)
idx = hash_code % len(keys)
while hash_codes[idx] is not EMPTY:
if hash_codes[idx] == hash_code and keys[idx] == key:
return
idx = (idx + 1) % len(keys)
hash_codes[idx] = hash_code
keys[idx] = key
def remove(hash_codes, keys, key):
hash_code = hash(key)
idx = hash_code % len(keys)
while hash_codes[idx] is not EMPTY:
if hash_codes[idx] == hash_code and keys[idx] == key:
keys[idx] = DUMMY
return
idx = (idx + 1) % len(keys)
raise KeyError()
def has_key(hash_codes, keys, key):
hash_code = hash(key)
idx = hash_code % len(keys)
while hash_codes[idx] is not EMPTY:
if hash_codes[idx] == hash_code and keys[idx] == key:
return True
idx = (idx + 1) % len(keys)
return False
def resize(hash_codes, keys):
new_hash_codes = [EMPTY for i in range(len(hash_codes) * 2)]
new_keys = [EMPTY for i in range(len(keys) * 2)]
for hash_code, key in zip(hash_codes, keys):
if key is EMPTY or key is DUMMY:
continue
idx = hash_code % len(new_keys)
while new_hash_codes[idx] is not EMPTY:
idx = (idx + 1) % len(new_keys)
new_hash_codes[idx] = hash_code
new_keys[idx] = key
return new_hash_codes, new_keys
================================================
FILE: python_code/hash_chapter2_impl_test.py
================================================
import unittest
from hash_chapter2_impl import create_new, has_key, insert, remove, resize, DUMMY
from common import generate_random_string
class MyHashTest(unittest.TestCase):
def test_handcrafted(self):
expected_len = 6
hashes, keys = create_new([42, 43, 12])
self.assertEqual(len(hashes), expected_len)
self.assertEqual(len(keys), expected_len)
insert(hashes, keys, 42)
self.assertEqual(hashes[42 % expected_len], 42)
self.assertEqual(keys[42 % expected_len], 42)
self.assertEqual(hashes[43 % expected_len], 43)
self.assertEqual(keys[43 % expected_len], 43)
self.assertEqual(hashes[42 % expected_len], 42)
self.assertEqual(keys[42 % expected_len], 42)
self.assertEqual(hashes[12 % expected_len], 42)
self.assertEqual(keys[12 % expected_len], 42)
self.assertEqual(hashes[12 % expected_len + 1], 43)
self.assertEqual(keys[12 % expected_len + 1], 43)
self.assertEqual(hashes[12 % expected_len + 2], 12)
self.assertEqual(keys[12 % expected_len + 2], 12)
self.assertTrue(has_key(hashes, keys, 42))
self.assertTrue(has_key(hashes, keys, 43))
self.assertTrue(has_key(hashes, keys, 12))
self.assertFalse(has_key(hashes, keys, 45))
# table: [42, 43, 12, None, None, None]
insert(hashes, keys, "") # hash("") == 0
self.assertEqual(hashes[3], 0)
self.assertEqual(keys[3], "")
self.assertTrue(has_key(hashes, keys, ""))
self.assertTrue(has_key(hashes, keys, 42))
insert(hashes, keys, "aba") # hash("aba") % 6 == 5
self.assertEqual(hashes[5], hash("aba"))
self.assertEqual(keys[5], "aba")
self.assertTrue(has_key(hashes, keys, 12))
remove(hashes, keys, 12)
self.assertFalse(has_key(hashes, keys, 12))
self.assertEqual(hashes[12 % expected_len], 42)
self.assertEqual(keys[12 % expected_len], 42)
self.assertEqual(keys[12 % expected_len + 2], DUMMY)
with self.assertRaises(KeyError):
remove(hashes, keys, 12)
with self.assertRaises(KeyError):
remove(hashes, keys, 45)
self.assertFalse(has_key(hashes, keys, 12))
self.assertFalse(has_key(hashes, keys, 45))
self.assertTrue(has_key(hashes, keys, 42))
self.assertTrue(has_key(hashes, keys, 43))
self.assertTrue(has_key(hashes, keys, ""))
self.assertTrue(has_key(hashes, keys, "aba"))
insert(hashes, keys, "abg")
self.assertTrue(has_key(hashes, keys, "abg"))
self.assertEqual(hashes[4], hash("abg"))
self.assertEqual(keys[4], "abg")
hashes, keys = resize(hashes, keys)
self.assertTrue(has_key(hashes, keys, 42))
self.assertTrue(has_key(hashes, keys, 43))
self.assertTrue(has_key(hashes, keys, ""))
self.assertTrue(has_key(hashes, keys, "aba"))
self.assertTrue(has_key(hashes, keys, "abg"))
self.assertFalse(has_key(hashes, keys, 12))
self.assertFalse(has_key(hashes, keys, 45))
self.assertEqual(hashes[6], 42)
self.assertEqual(keys[6], 42)
self.assertEqual(hashes[7], 43)
self.assertEqual(keys[7], 43)
self.assertEqual(hashes[0], 0)
self.assertEqual(keys[0], "")
for h in hashes:
self.assertTrue(h != 12)
self.assertEqual(hashes[5], hash("aba"))
self.assertEqual(keys[5], "aba")
self.assertEqual(hashes[11], hash("abg"))
self.assertEqual(keys[11], "abg")
def test_all(self):
n = 10
initial_keys = [generate_random_string() for _ in range(n)]
more_keys = [generate_random_string() for _ in range(n // 3)]
myhashes, mykeys = create_new(initial_keys)
for key in more_keys:
insert(myhashes, mykeys, key)
insert(myhashes, mykeys, key)
existing_keys = initial_keys + more_keys
for key in existing_keys:
self.assertTrue(has_key(myhashes, mykeys, key))
myhashes, mykeys = resize(myhashes, mykeys)
for key in existing_keys:
self.assertTrue(has_key(myhashes, mykeys, key))
missing_keys = [generate_random_string() for _ in range(3 * n)]
for key in set(missing_keys) - set(existing_keys):
self.assertFalse(has_key(myhashes, mykeys, key))
with self.assertRaises(KeyError):
remove(myhashes, mykeys, key)
for key in existing_keys:
self.assertTrue(has_key(myhashes, mykeys, key))
remove(myhashes, mykeys, key)
self.assertFalse(has_key(myhashes, mykeys, key))
for key in more_keys:
self.assertFalse(has_key(myhashes, mykeys, key))
insert(myhashes, mykeys, key)
self.assertTrue(has_key(myhashes, mykeys, key))
remove(myhashes, mykeys, key)
self.assertFalse(has_key(myhashes, mykeys, key))
def main():
unittest.main()
if __name__ == "__main__":
main()
================================================
FILE: python_code/hash_chapter2_reimpl_js.py
================================================
from js_reimpl_common import run_op_chapter1_chapter2
def run_op(hash_codes, keys, op, **kwargs):
return run_op_chapter1_chapter2("chapter2", hash_codes, keys, op, **kwargs)
def create_new(from_keys):
return run_op(None, None, "create_new", array=from_keys)
def insert(hash_codes, keys, key):
new_hash_codes, new_keys = run_op(hash_codes, keys, "insert", key=key)
hash_codes[:] = new_hash_codes
keys[:] = new_keys
def remove(hash_codes, keys, key):
new_hash_codes, new_keys = run_op(hash_codes, keys, "remove", key=key)
hash_codes[:] = new_hash_codes
keys[:] = new_keys
def has_key(hash_codes, keys, key):
return run_op(hash_codes, keys, "has_key", key=key)
def resize(hash_codes, keys):
return run_op(hash_codes, keys, "resize")
================================================
FILE: python_code/hash_chapter2_reimplementation_test.py
================================================
import random
import argparse
from common import DUMMY, EMPTY, AllKeyValueFactory, IntKeyValueFactory
import hash_chapter2_reimpl_js
import hash_chapter2_impl
import build_autogenerated_chapter2
TEST_IMPLEMENTATIONS = {
'js_reimpl': hash_chapter2_reimpl_js,
'py_extracted': build_autogenerated_chapter2
}
def verify_same(ref_hash_codes, ref_keys, hash_codes, keys):
if (ref_hash_codes, ref_keys) != (hash_codes, keys):
print("ORIG SIZES", len(ref_hash_codes), len(ref_keys))
print("NEW SIZES", len(hash_codes), len(keys))
if len(ref_hash_codes) == len(hash_codes) == len(ref_keys) == len(keys):
size = len(hash_codes)
print("NEW | ORIG")
for i in range(size):
if ref_hash_codes[i] is not EMPTY or hash_codes[i] is not EMPTY:
print(i, " " * 3,
ref_hash_codes[i], ref_keys[i], " " * 3,
hash_codes[i], keys[i], " " * 3)
assert ref_hash_codes == hash_codes and ref_keys == keys
def run(ref_impl, test_impl, n_inserts, key_value_factory, initial_state, extra_checks, verbose):
SINGLE_REMOVE_CHANCE = 0.3
ref_hash_codes, ref_keys = ref_impl.create_new(initial_state)
test_hash_codes, test_keys = test_impl.create_new(initial_state)
def vs():
verify_same(ref_hash_codes, ref_keys, test_hash_codes, test_keys)
vs()
if verbose:
print("Starting test")
for i in range(n_inserts):
key_to_insert = key_value_factory.generate_key()
existing_keys = set([k for k in ref_keys if k is not DUMMY and k is not EMPTY])
fill = sum(1 for k in ref_keys if k is not EMPTY)
if existing_keys and random.random() < SINGLE_REMOVE_CHANCE:
key_to_remove = random.choice(list(existing_keys))
assert ref_impl.has_key(ref_hash_codes, ref_keys, key_to_remove)
assert test_impl.has_key(test_hash_codes, test_keys, key_to_remove)
ref_impl.remove(ref_hash_codes, ref_keys, key_to_remove)
test_impl.remove(test_hash_codes, test_keys, key_to_remove)
existing_keys.remove(key_to_remove)
assert not ref_impl.has_key(ref_hash_codes, ref_keys, key_to_remove)
assert not test_impl.has_key(test_hash_codes, test_keys, key_to_remove)
is_key_present = ref_impl.has_key(ref_hash_codes, ref_keys, key_to_insert)
assert (key_to_insert in existing_keys) == is_key_present
if not is_key_present:
if verbose:
print("Inserting {}".format(key_to_insert))
assert not test_impl.has_key(test_hash_codes, test_keys, key_to_insert)
else:
if verbose:
print("Re-Inserting {}".format(key_to_insert))
ref_impl.insert(ref_hash_codes, ref_keys, key_to_insert)
test_impl.insert(test_hash_codes, test_keys, key_to_insert)
vs()
assert test_impl.has_key(test_hash_codes, test_keys, key_to_insert)
assert ref_impl.has_key(ref_hash_codes, ref_keys, key_to_insert)
if fill / len(ref_keys) > 0.66:
ref_hash_codes, ref_keys = ref_impl.resize(ref_hash_codes, ref_keys)
test_hash_codes, test_keys = test_impl.resize(test_hash_codes, test_keys)
vs()
if extra_checks:
for k in existing_keys:
assert test_impl.has_key(test_hash_codes, test_keys, k)
assert ref_impl.has_key(ref_hash_codes, ref_keys, k)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Stress-test chapter2 reimplementation')
parser.add_argument('--test-implementation', choices=TEST_IMPLEMENTATIONS.keys(), required=True)
parser.add_argument('--num-inserts', type=int, default=500)
parser.add_argument('--forever', action='store_true')
parser.add_argument('--kv', choices=["numbers", "all"], required=True)
parser.add_argument('--initial-size', type=int, default=-1)
parser.add_argument('--extra-getitem-checks', action='store_true', default=False)
parser.add_argument('--verbose', action='store_true', default=False)
args = parser.parse_args()
if args.kv == "numbers":
kv_factory = IntKeyValueFactory(args.num_inserts)
elif args.kv == "all":
kv_factory = AllKeyValueFactory(args.num_inserts)
def test_iteration():
initial_size = args.initial_size if args.initial_size >= 0 else random.randint(0, 100)
initial_state = [kv_factory.generate_key() for _ in range(initial_size)]
run(hash_chapter2_impl,
TEST_IMPLEMENTATIONS[args.test_implementation],
n_inserts=args.num_inserts,
key_value_factory=kv_factory,
initial_state=initial_state,
extra_checks=args.extra_getitem_checks,
verbose=args.verbose)
if args.forever:
while True:
test_iteration()
else:
test_iteration()
================================================
FILE: python_code/hash_chapter3_class_impl.py
================================================
from common import DUMMY, EMPTY
from dict_reimpl_common import BaseDictImpl, Slot
class AlmostPythonDictBase(BaseDictImpl):
START_SIZE = 8
def __init__(self, pairs=None):
BaseDictImpl.__init__(self)
self._keys_set = set()
if pairs:
for k, v in pairs:
self[k] = v
def lookdict(self, key):
hash_code = hash(key)
idx = hash_code % len(self.slots)
while self.slots[idx].key is not EMPTY:
if self.slots[idx].hash_code == hash_code and self.slots[idx].key == key:
return idx
idx = (idx + 1) % len(self.slots)
raise KeyError()
def __getitem__(self, key):
idx = self.lookdict(key)
return self.slots[idx].value
def __delitem__(self, key):
idx = self.lookdict(key)
self.used -= 1
self.slots[idx].key = DUMMY
self.slots[idx].value = EMPTY
self._keys_set.remove(key)
def resize(self):
old_slots = self.slots
new_size = self.find_nearest_size(2 * self.used)
self.slots = [Slot() for _ in range(new_size)]
for slot in old_slots:
if slot.key is not EMPTY and slot.key is not DUMMY:
idx = slot.hash_code % len(self.slots)
while self.slots[idx].key is not EMPTY:
idx = (idx + 1) % len(self.slots)
self.slots[idx] = Slot(slot.hash_code, slot.key, slot.value)
self.fill = self.used
def keys(self):
return self._keys_set
def __len__(self):
return len(self.keys())
class AlmostPythonDictImplementationRecycling(AlmostPythonDictBase):
def __setitem__(self, key, value):
hash_code = hash(key)
idx = hash_code % len(self.slots)
target_idx = None
while self.slots[idx].key is not EMPTY:
if self.slots[idx].hash_code == hash_code and self.slots[idx].key == key:
target_idx = idx
break
if target_idx is None and self.slots[idx].key is DUMMY:
target_idx = idx
idx = (idx + 1) % len(self.slots)
if target_idx is None:
target_idx = idx
if self.slots[target_idx].key is EMPTY:
self.used += 1
self.fill += 1
elif self.slots[target_idx].key is DUMMY:
self.used += 1
self.slots[target_idx] = Slot(hash_code, key, value)
if self.fill * 3 >= len(self.slots) * 2:
self.resize()
self._keys_set.add(key)
class AlmostPythonDictImplementationNoRecycling(AlmostPythonDictBase):
def __setitem__(self, key, value):
hash_code = hash(key)
idx = hash_code % len(self.slots)
target_idx = None
while self.slots[idx].key is not EMPTY:
if self.slots[idx].hash_code == hash_code and\
self.slots[idx].key == key:
target_idx = idx
break
idx = (idx + 1) % len(self.slots)
if target_idx is None:
target_idx = idx
if self.slots[target_idx].key is EMPTY:
self.used += 1
self.fill += 1
self.slots[target_idx] = Slot(hash_code, key, value)
if self.fill * 3 >= len(self.slots) * 2:
self.resize()
self._keys_set.add(key)
class AlmostPythonDictImplementationNoRecyclingSimplerVersion(AlmostPythonDictBase):
def __setitem__(self, key, value):
hash_code = hash(key)
idx = hash_code % len(self.slots)
while self.slots[idx].key is not EMPTY:
if self.slots[idx].hash_code == hash_code and\
self.slots[idx].key == key:
break
idx = (idx + 1) % len(self.slots)
if self.slots[idx].key is EMPTY:
self.used += 1
self.fill += 1
self.slots[idx] = Slot(hash_code, key, value)
if self.fill * 3 >= len(self.slots) * 2:
self.resize()
self._keys_set.add(key)
================================================
FILE: python_code/hash_chapter3_class_impl_test.py
================================================
import unittest
from common import DUMMY, EMPTY
from hash_chapter3_class_impl import AlmostPythonDictImplementationRecycling, AlmostPythonDictImplementationNoRecycling
class HashDictImplementationTest(unittest.TestCase):
def test_handcrafted(self):
d = AlmostPythonDictImplementationRecycling()
self.assertEqual(len(d.slots), 8)
def assert_contains(i, h, k, v):
self.assertEqual(d.slots[i].hash_code, h)
self.assertEqual(d.slots[i].key, k)
self.assertEqual(d.slots[i].value, v)
d[""] = 1
d[17] = 2
d[18] = 3
self.assertEqual(d[""], 1)
self.assertEqual(d[17], 2)
self.assertEqual(d[18], 3)
assert_contains(0, 0, "", 1)
assert_contains(1, 17, 17, 2)
assert_contains(2, 18, 18, 3)
self.assertEqual(d.fill, 3)
self.assertEqual(d.used, 3)
with self.assertRaises(KeyError):
del d[1]
del d[17]
assert_contains(1, 17, DUMMY, EMPTY)
self.assertEqual(d.fill, 3)
self.assertEqual(d.used, 2)
# hash("abcd") % 8 == 0
# py 3.2 hash()
d["abcd"] = 4
self.assertEqual(d["abcd"], 4)
assert_contains(1, -2835746963027601024, "abcd", 4)
self.assertEqual(d.fill, 3)
self.assertEqual(d.used, 3)
d["abcd"] = 5
self.assertEqual(d["abcd"], 5)
assert_contains(1, -2835746963027601024, "abcd", 5)
self.assertEqual(d.fill, 3)
self.assertEqual(d.used, 3)
del d["abcd"]
with self.assertRaises(KeyError):
d["abcd"]
d[15] = 6
d[14] = 7
assert_contains(7, 15, 15, 6)
assert_contains(6, 14, 14, 7)
self.assertEqual(len(d.slots), 8)
self.assertEqual(d.fill, 5)
self.assertEqual(d.used, 4)
d[13] = 8
self.assertEqual(len(d.slots), 16)
self.assertEqual(d.fill, 5)
self.assertEqual(d.used, 5)
assert_contains(0, 0, "", 1)
assert_contains(2, 18, 18, 3)
assert_contains(13, 13, 13, 8)
assert_contains(14, 14, 14, 7)
assert_contains(15, 15, 15, 6)
def test_handcrafted_simple_setitem(self):
d = AlmostPythonDictImplementationNoRecycling()
self.assertEqual(len(d.slots), 8)
def assert_contains(i, h, k, v):
self.assertEqual(d.slots[i].hash_code, h)
self.assertEqual(d.slots[i].key, k)
self.assertEqual(d.slots[i].value, v)
d[""] = 1
d[17] = 2
d[18] = 3
self.assertEqual(d[""], 1)
self.assertEqual(d[17], 2)
self.assertEqual(d[18], 3)
assert_contains(0, 0, "", 1)
assert_contains(1, 17, 17, 2)
assert_contains(2, 18, 18, 3)
self.assertEqual(d.fill, 3)
self.assertEqual(d.used, 3)
with self.assertRaises(KeyError):
del d[1]
del d[17]
assert_contains(1, 17, DUMMY, EMPTY)
self.assertEqual(d.fill, 3)
self.assertEqual(d.used, 2)
# hash("abcd") % 8 == 0
# py 3.2 hash()
d["abcd"] = 4
self.assertEqual(d["abcd"], 4)
assert_contains(3, -2835746963027601024, "abcd", 4)
self.assertEqual(d.fill, 4)
self.assertEqual(d.used, 3)
d["abcd"] = 5
self.assertEqual(d["abcd"], 5)
assert_contains(3, -2835746963027601024, "abcd", 5)
self.assertEqual(d.fill, 4)
self.assertEqual(d.used, 3)
del d["abcd"]
with self.assertRaises(KeyError):
d["abcd"]
self.assertEqual(len(d.slots), 8)
self.assertEqual(d.fill, 4)
self.assertEqual(d.used, 2)
d[15] = 6
self.assertEqual(len(d.slots), 8)
self.assertEqual(d.fill, 5)
self.assertEqual(d.used, 3)
assert_contains(7, 15, 15, 6)
d[13] = 8
self.assertEqual(len(d.slots), 16)
self.assertEqual(d.fill, 4)
self.assertEqual(d.used, 4)
assert_contains(0, 0, "", 1)
assert_contains(2, 18, 18, 3)
assert_contains(13, 13, 13, 8)
assert_contains(15, 15, 15, 6)
def main():
unittest.main()
if __name__ == "__main__":
main()
================================================
FILE: python_code/interface_test.py
================================================
import unittest
from dict32_reimplementation import PyDictReimplementation
from hash_chapter3_class_impl import AlmostPythonDictImplementationRecycling, AlmostPythonDictImplementationNoRecycling
from js_reimplementation_interface import Dict32JsImpl, AlmostPythonDictRecyclingJsImpl, AlmostPythonDictNoRecyclingJsImpl
class Interface(unittest.TestCase):
def test_all(self):
self.do_simple_test_single_class(PyDictReimplementation)
self.do_simple_test_single_class(AlmostPythonDictImplementationRecycling)
self.do_simple_test_single_class(AlmostPythonDictImplementationNoRecycling)
self.do_simple_test_single_class(Dict32JsImpl)
self.do_simple_test_single_class(AlmostPythonDictRecyclingJsImpl)
self.do_simple_test_single_class(AlmostPythonDictNoRecyclingJsImpl)
def do_simple_test_single_class(self, klass):
d = klass()
for i in range(100):
d[i] = i
self.assertEqual(d[i], i)
for i in range(50):
del d[i]
with self.assertRaises(KeyError):
d[i]
for i in range(200):
d[i] = i + 1
self.assertEqual(d[i], i + 1)
def main():
unittest.main()
if __name__ == "__main__":
main()
================================================
FILE: python_code/js_reimpl_common.py
================================================
import socket
import json
from common import DUMMY, EMPTY
none_info = {
"type": "None",
"hash": str(hash(None))
}
def dump_simple_py_obj(obj):
if obj is DUMMY:
return {
"type": "DUMMY"
}
elif obj is EMPTY:
return None
elif obj is None:
return none_info
elif isinstance(obj, int):
return {
'type': 'int',
'value': str(obj)
}
return obj
def dump_pairs(pairs):
res = []
for k, v in pairs:
res.append([dump_simple_py_obj(k), dump_simple_py_obj(v)])
return res
def dump_array(array):
return list(map(dump_simple_py_obj, array))
def parse_array(array):
return list(map(parse_simple_py_obj, array))
def parse_simple_py_obj(obj):
if isinstance(obj, dict):
assert obj["type"] in ["DUMMY", "None", "int"]
if obj["type"] == "DUMMY":
return DUMMY
if obj["type"] == "None":
return None
return int(obj["value"])
elif obj is None:
return EMPTY
return obj
sock = None
sockfile = None
def _init_sock_stuff():
global sock
global sockfile
# TODO: unhardcode?
SOCK_FILENAME = 'pynode.sock'
if sock is None:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(SOCK_FILENAME)
sockfile = sock.makefile('r')
return sock, sockfile
def run_op_chapter1_chapter2(chapter, hash_codes, keys, op, **kwargs):
_init_sock_stuff()
for name in kwargs:
if name != 'array':
kwargs[name] = dump_simple_py_obj(kwargs[name])
else:
kwargs[name] = dump_array(kwargs[name])
data = {
"dict": chapter,
"op": op,
"args": kwargs,
"hashCodes": dump_array(hash_codes) if hash_codes is not None else None,
"keys": dump_array(keys) if keys is not None else None,
}
sock.send(bytes(json.dumps(data) + "\n", 'UTF-8'))
response = json.loads(sockfile.readline())
if "exception" in response and response["exception"]:
raise KeyError()
if 'result' in response and response['result'] is not None:
# TODO: this is pretty hacky
return response["result"]
elif "hashCodes" in response:
return parse_array(response["hashCodes"]), parse_array(response["keys"])
else:
return parse_array(response["keys"])
================================================
FILE: python_code/js_reimplementation_interface.py
================================================
import socket
import json
from common import DUMMY, EMPTY
from js_reimpl_common import dump_simple_py_obj, parse_simple_py_obj, dump_pairs
from dict_reimpl_common import Slot
class JsImplBase(object):
# TODO: unhardcode?
SOCK_FILENAME = 'pynode.sock'
def __init__(self, pairs=None):
pairs = pairs or []
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.sock.connect(self.SOCK_FILENAME)
self.sockfile = self.sock.makefile('r')
self.slots = None
self.fill = None
self.used = None
self.run_op("__init__", pairs=pairs)
def __del__(self):
self.sock.close()
def dump_slots(self):
def dump_slot(slot):
key = dump_simple_py_obj(slot.key)
value = dump_simple_py_obj(slot.value)
hash_code = slot.hash_code
if hash_code is EMPTY:
hash_code = None
return {
"hashCode": str(hash_code) if hash_code is not None else None,
"key": key,
"value": value,
}
if self.slots is None:
return None
return list(map(dump_slot, self.slots))
def restore_slots(self, slots):
def restore_slot(slot):
key = parse_simple_py_obj(slot["key"])
value = parse_simple_py_obj(slot["value"])
assert value is not DUMMY
hash_code = int(slot["hashCode"]) if slot["hashCode"] is not None else None
if hash_code is None:
hash_code = EMPTY
return Slot(hash_code, key, value)
self.slots = list(map(restore_slot, slots))
def run_op(self, op, **kwargs):
for name in kwargs:
if name != 'pairs':
kwargs[name] = dump_simple_py_obj(kwargs[name])
else:
kwargs[name] = dump_pairs(kwargs[name])
data = {
"dict": self.dict_type,
"op": op,
"args": kwargs,
"self": {
"slots": self.dump_slots(),
"used": self.used,
"fill": self.fill
}
}
# pprint(("<< sending", data, op, kwargs))
self.sock.send(bytes(json.dumps(data) + "\n", 'UTF-8'))
response = json.loads(self.sockfile.readline())
# pprint((">> receiving", response))
self.restore_slots(response["self"]["slots"])
self.fill = response["self"]["fill"]
self.used = response["self"]["used"]
if response["exception"]:
raise KeyError("whatever")
return parse_simple_py_obj(response["result"])
class Dict32JsImpl(JsImplBase):
dict_type = "dict32"
def __setitem__(self, key, value):
return self.run_op("__setitem__", key=key, value=value)
def __delitem__(self, key):
return self.run_op("__delitem__", key=key)
def __getitem__(self, key):
return self.run_op("__getitem__", key=key)
class AlmostPythonDictBaseJsImpl(JsImplBase):
dict_type = "almost_python_dict"
def __delitem__(self, key):
return self.run_op("__delitem__", key=key)
def __getitem__(self, key):
return self.run_op("__getitem__", key=key)
class AlmostPythonDictRecyclingJsImpl(AlmostPythonDictBaseJsImpl):
def __setitem__(self, key, value):
return self.run_op("__setitem__recycling", key=key, value=value)
class AlmostPythonDictNoRecyclingJsImpl(AlmostPythonDictBaseJsImpl):
def __setitem__(self, key, value):
return self.run_op("__setitem__no_recycling", key=key, value=value)
================================================
FILE: scripts/extractPythonCode.js
================================================
import 'ignore-styles';
import {
DICT32_INIT,
DICT32_SETITEM,
DICT32_RESIZE_CODE,
_DICT32_GETITEM_ONLY,
_DICT32_DELITEM_ONLY,
DICT32_LOOKDICT,
STATICMETHOD_SIGNED_TO_UNSIGNED,
PROBING_PYTHON_CODE,
} from '../src/chapter4_real_python_dict';
import {
HASH_CLASS_INIT_CODE,
HASH_CLASS_SETITEM_RECYCLING_CODE,
HASH_CLASS_SETITEM_SIMPLIFIED_CODE,
_HASH_CLASS_GETITEM_ONLY,
_HASH_CLASS_DELITEM_ONLY,
HASH_CLASS_LOOKDICT,
HASH_CLASS_RESIZE_CODE,
FIND_NEAREST_SIZE_CODE_STRING,
SLOT_CLASS_CODE_STRING,
} from '../src/chapter3_hash_class';
import {
HASH_CREATE_NEW_CODE,
HASH_SEARCH_CODE,
HASH_REMOVE_CODE,
HASH_RESIZE_CODE,
HASH_INSERT_CODE,
} from '../src/chapter2_hash_table_functions';
import {
SIMPLIFIED_INSERT_ALL_BROKEN_CODE,
SIMPLIFIED_INSERT_ALL_CODE,
SIMPLIFIED_SEARCH_CODE,
SIMPLE_LIST_SEARCH,
} from '../src/chapter1_simplified_hash';
import fs from 'fs';
import * as path from 'path';
function extractCodeLines(codeWithBpAndLevels) {
return codeWithBpAndLevels.map(([line, bp, level]) => line);
}
function outputCode(filename, headers, importedCode, indent4 = true) {
let allLines = [];
for (let part of importedCode) {
let lines;
if (typeof part !== 'string') {
lines = extractCodeLines(part);
} else {
lines = part.split('\n');
}
allLines.push(...lines);
if (lines[lines.length - 1] !== '') {
allLines.push('');
}
}
const joinedLines = allLines.map(line => (line.length > 0 && indent4 ? ' ' + line : line)).join('\n');
fs.writeFileSync(filename, headers.join('\n') + '\n' + joinedLines);
}
const commonImports = `import sys
import os
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'python_code'))
from common import DUMMY, EMPTY
`;
const dict32def = `
class Dict32Extracted(object):`;
const DIR = 'build';
outputCode(
path.join(DIR, 'dict32js_extracted.py'),
[commonImports, SLOT_CLASS_CODE_STRING, dict32def],
[
DICT32_INIT,
FIND_NEAREST_SIZE_CODE_STRING,
STATICMETHOD_SIGNED_TO_UNSIGNED,
DICT32_SETITEM,
DICT32_RESIZE_CODE,
_DICT32_GETITEM_ONLY,
_DICT32_DELITEM_ONLY,
DICT32_LOOKDICT,
]
);
const hashClassRecyclingDef = `
class HashClassRecyclingExtracted(object):`;
outputCode(
path.join(DIR, 'hash_class_recycling_extracted.py'),
[commonImports, SLOT_CLASS_CODE_STRING, hashClassRecyclingDef],
[
HASH_CLASS_INIT_CODE,
FIND_NEAREST_SIZE_CODE_STRING,
HASH_CLASS_SETITEM_RECYCLING_CODE,
HASH_CLASS_RESIZE_CODE,
_HASH_CLASS_GETITEM_ONLY,
_HASH_CLASS_DELITEM_ONLY,
HASH_CLASS_LOOKDICT,
]
);
const hashClassNoRecyclingDef = `
class HashClassNoRecyclingExtracted(object):`;
outputCode(
path.join(DIR, 'hash_class_no_recycling_extracted.py'),
[commonImports, SLOT_CLASS_CODE_STRING, hashClassNoRecyclingDef],
[
HASH_CLASS_INIT_CODE,
FIND_NEAREST_SIZE_CODE_STRING,
HASH_CLASS_SETITEM_SIMPLIFIED_CODE,
HASH_CLASS_RESIZE_CODE,
_HASH_CLASS_GETITEM_ONLY,
_HASH_CLASS_DELITEM_ONLY,
HASH_CLASS_LOOKDICT,
]
);
outputCode(
path.join(DIR, 'hash_chapter2_extracted.py'),
[commonImports],
[HASH_CREATE_NEW_CODE, HASH_SEARCH_CODE, HASH_REMOVE_CODE, HASH_RESIZE_CODE, HASH_INSERT_CODE],
false
);
outputCode(
path.join(DIR, 'hash_chapter1_extracted.py'),
[commonImports],
[SIMPLIFIED_INSERT_ALL_CODE, SIMPLIFIED_INSERT_ALL_BROKEN_CODE, SIMPLIFIED_SEARCH_CODE, SIMPLE_LIST_SEARCH],
false
);
outputCode(path.join(DIR, 'chapter4_probing_python_code.py'), [commonImports], [PROBING_PYTHON_CODE], false);
================================================
FILE: scripts/pyReimplWrapper.js
================================================
const net = require('net');
const split = require('split');
import 'ignore-styles';
import {BigNumber} from 'bignumber.js';
import {DUMMY, EMPTY, None} from '../src/hash_impl_common';
import {Dict32} from '../src/chapter4_real_python_dict';
import {GenerateProbingLinks} from '../src/probing_visualization.js';
import {AlmostPythonDict} from '../src/chapter3_hash_class';
import {Ops as Chapter2Ops} from '../src/chapter2_hash_table_functions';
import {Ops as Chapter1Ops} from '../src/chapter1_simplified_hash';
import {Slot} from '../src/chapter3_and_4_common';
import {List as ImmutableList} from 'immutable';
function parseSimplePyObj(obj) {
if (obj === null || typeof obj === 'string') {
return obj;
} else if (typeof obj === 'object' && obj.type === 'None') {
let res = None;
// TODO FIXME: this does not support multiple clients
res._hashCode = obj.hash;
return res;
} else if (typeof obj === 'object' && obj.type === 'DUMMY') {
return DUMMY;
} else if (typeof obj === 'object' && obj.type === 'EMPTY') {
return EMPTY;
} else if (typeof obj === 'object' && obj.type === 'int') {
return BigNumber(obj.value);
} else {
throw new Error(`Unknown obj ${JSON.stringify(obj)}`);
}
}
function parseArray(array) {
return array.map(parseSimplePyObj);
}
function dumpArray(array) {
return array.map(dumpSimplePyObj);
}
function parsePairs(pairs) {
return pairs.map(([k, v]) => [parseSimplePyObj(k), parseSimplePyObj(v)]);
}
function dumpSimplePyObj(obj) {
if (obj === DUMMY) {
return {
type: 'DUMMY',
};
} else if (obj === None) {
return {
type: 'None',
};
} else if (BigNumber.isBigNumber(obj)) {
return {
type: 'int',
value: obj.toString(),
};
} else {
return obj;
}
}
function restorePyDictState(state) {
let {pySelf} = Dict32.__init__();
if (state.slots != null) {
pySelf = pySelf.set(
'slots',
new ImmutableList(
state.slots.map(slot => {
let key = parseSimplePyObj(slot.key);
let value = parseSimplePyObj(slot.value);
return Slot({
pyHashCode: slot.hashCode ? new BigNumber(slot.hashCode) : null,
key: key,
value: value,
});
})
)
);
} else {
pySelf = pySelf.set('slots', null);
}
pySelf = pySelf.set('used', state.used);
pySelf = pySelf.set('fill', state.fill);
return pySelf;
}
function dumpPyDictState(pySelf) {
let data = {};
data.slots = pySelf.get('slots').map(slot => {
return {
hashCode: slot.pyHashCode != null ? slot.pyHashCode.toString() : null,
key: dumpSimplePyObj(slot.key),
value: dumpSimplePyObj(slot.value),
};
});
data.used = pySelf.get('used');
data.fill = pySelf.get('fill');
return data;
}
function dict32RunOp(pySelf, op, key, value, pairs) {
switch (op) {
case '__init__':
pySelf = Dict32.__init__(pairs).pySelf;
return {pySelf};
case '__getitem__': {
const {result, isException} = Dict32.__getitem__(pySelf, key);
return {pySelf, result, isException};
}
case '__setitem__': {
({pySelf} = Dict32.__setitem__(pySelf, key, value));
return {pySelf};
}
case '__delitem__': {
let isException;
({pySelf, isException} = Dict32.__delitem__(pySelf, key));
return {pySelf, isException};
}
default:
throw new Error('Unknown op: ' + op);
}
}
function almostPyDictRunOp(pySelf, op, key, value, pairs) {
switch (op) {
case '__init__':
pySelf = AlmostPythonDict.__init__(pairs).pySelf;
return {pySelf};
case '__getitem__': {
const {result, isException} = AlmostPythonDict.__getitem__(pySelf, key);
return {pySelf, result, isException};
}
case '__setitem__recycling': {
({pySelf} = AlmostPythonDict.__setitem__recycling(pySelf, key, value));
return {pySelf};
}
case '__setitem__no_recycling': {
({pySelf} = AlmostPythonDict.__setitem__no_recycling(pySelf, key, value));
return {pySelf};
}
case '__delitem__': {
let isException;
({pySelf, isException} = AlmostPythonDict.__delitem__(pySelf, key));
return {pySelf, isException};
}
default:
throw new Error('Unknown op: ' + op);
}
}
function chapter1run(keys, op, key, numbers) {
switch (op) {
case 'create_new':
({keys} = Chapter1Ops.createNew(numbers));
return {keys};
case 'create_new_broken':
({keys} = Chapter1Ops.createNewBroken(numbers));
return {keys};
case 'has_key': {
let result;
({keys, result} = Chapter1Ops.hasKey(keys, key));
return {keys, result};
}
case 'linear_search': {
let {result} = Chapter1Ops.linearSearch(numbers, key);
return {result};
}
default:
throw new Error('Unknown op: ' + op);
}
}
function chapter2run(hashCodes, keys, op, key, array) {
switch (op) {
case 'create_new':
({hashCodes, keys} = Chapter2Ops.createNew(array));
return {hashCodes, keys};
case 'insert':
({hashCodes, keys} = Chapter2Ops.insert(hashCodes, keys, key));
return {hashCodes, keys};
case 'remove': {
let isException;
({hashCodes, keys, isException} = Chapter2Ops.remove(hashCodes, keys, key));
return {hashCodes, keys, isException};
}
case 'has_key': {
let result;
({hashCodes, keys, result} = Chapter2Ops.hasKey(hashCodes, keys, key));
return {hashCodes, keys, result};
}
case 'resize':
({hashCodes, keys} = Chapter2Ops.resize(hashCodes, keys));
return {hashCodes, keys};
default:
throw new Error('Unknown op: ' + op);
}
}
const server = net.createServer(c => {
console.log('Client connected');
c.on('end', () => {
console.log('Client disconnected');
});
c.pipe(split()).on('data', line => {
console.log('Received line of length ' + line.length);
if (!line) return;
const data = JSON.parse(line);
const dictType = data.dict;
const op = data.op;
let {key, value, pairs, array} = data.args;
if (key !== undefined) {
key = parseSimplePyObj(key);
}
if (value !== undefined) {
value = parseSimplePyObj(value);
}
if (pairs !== undefined) {
pairs = parsePairs(pairs);
}
if (array !== undefined) {
array = parseArray(array);
}
console.log(op, data.args);
let isException, result;
let response;
if (dictType === 'dict32' || dictType === 'almost_python_dict') {
let pySelf = restorePyDictState(data.self);
if (dictType === 'dict32') {
({pySelf, isException, result} = dict32RunOp(pySelf, op, key, value, pairs));
} else if (dictType === 'almost_python_dict') {
({pySelf, isException, result} = almostPyDictRunOp(pySelf, op, key, value, pairs));
} else {
throw new Error('Unknown dict type');
}
response = {
exception: isException || false,
result: result !== undefined ? dumpSimplePyObj(result) : null,
self: dumpPyDictState(pySelf),
};
} else if (dictType === 'chapter2') {
let hashCodes = data.hashCodes != null ? new ImmutableList(parseArray(data.hashCodes)) : undefined;
let keys = data.keys != null ? new ImmutableList(parseArray(data.keys)) : undefined;
({hashCodes, keys, isException, result} = chapter2run(hashCodes, keys, op, key, array));
response = {
exception: isException || false,
result: result !== undefined ? result : null,
hashCodes: dumpArray(hashCodes),
keys: dumpArray(keys),
};
} else if (dictType === 'chapter1') {
let keys = data.keys != null ? new ImmutableList(parseArray(data.keys)) : undefined;
({keys, result} = chapter1run(keys, op, key, array));
response = {
result: result !== undefined ? result : null,
keys: keys !== undefined ? dumpArray(keys) : null,
};
} else if (dictType === 'pythonProbing') {
let g = new GenerateProbingLinks();
const result = g.run(data.args.slotsCount, key, 'python');
response = {
result,
};
} else {
throw new Error('Unknown dict type');
}
c.write(JSON.stringify(response) + '\n');
});
});
server.on('error', err => {
throw err;
});
server.on('listening', () => {
console.log(`Listening`);
});
server.listen('pynode.sock', () => {
console.log('Starting listening...');
});
================================================
FILE: scripts/ssr.js
================================================
require('dotenv').config();
process.env.NODE_ENV = 'ssr';
console.log = () => {}; // Do not log to stdout
global.performance = {now: () => 0};
import 'ignore-styles';
import * as React from 'react';
import ReactDOMServer from 'react-dom/server';
import {CHAPTER_ID_TO_COMPONENT} from '../src/index';
import {App} from '../src/app';
import fs from 'fs';
const filename = process.argv[2];
const chapterIds = JSON.parse(process.argv[3]);
const chapters = chapterIds.map(id => CHAPTER_ID_TO_COMPONENT[id]);
let selectedChapterId;
if (chapterIds.length === 1) {
selectedChapterId = chapterIds[0];
}
fs.readFile(filename, 'utf8', function(err, file) {
if (err) {
throw new Error(`Cannot read source html: ${err}`);
}
const renderedComponent = ReactDOMServer.renderToString(
<App chapters={chapters} selectedChapterId={selectedChapterId} />
);
let fullHtml = file.replace(/<div id="root"><\/div>/, `<div id="root">${renderedComponent}</div>`);
const gaId = process.env.GA_ID;
console.warn('Google analytics ID is', gaId);
if (gaId) {
let GA_SCRIPT = `<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=__GA_CODE_HERE__"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', '__GA_CODE_HERE__');
</script>`;
GA_SCRIPT = GA_SCRIPT.replace(/__GA_CODE_HERE__/g, gaId);
fullHtml = fullHtml.replace('</head>', `${GA_SCRIPT}</head>`);
}
process.stdout.write(fullHtml);
});
================================================
FILE: src/app.js
================================================
import _ from 'lodash';
import Bootstrap from 'bootstrap/dist/css/bootstrap.min.css';
import stylesCss from './styles.css';
import * as React from 'react';
import ReactDOM from 'react-dom';
import {MyErrorBoundary, initUxSettings, getUxSettings, BootstrapAlert, doubleRAF} from './util';
import {win, globalSettings} from './store';
import {faDesktop} from '@fortawesome/free-solid-svg-icons/faDesktop';
import {faSpinner} from '@fortawesome/free-solid-svg-icons/faSpinner';
import {faSyncAlt} from '@fortawesome/free-solid-svg-icons/faSyncAlt';
import {faEnvelope} from '@fortawesome/free-solid-svg-icons/faEnvelope';
import {faChevronRight} from '@fortawesome/free-solid-svg-icons/faChevronRight';
import {faFirefox} from '@fortawesome/free-brands-svg-icons/faFirefox';
import {faGithub} from '@fortawesome/free-brands-svg-icons/faGithub';
import {faTwitter} from '@fortawesome/free-brands-svg-icons/faTwitter';
import {faMailchimp} from '@fortawesome/free-brands-svg-icons/faMailchimp';
import {library, config as fontAwesomeConfig} from '@fortawesome/fontawesome-svg-core';
fontAwesomeConfig.autoAddCss = false;
library.add(faDesktop);
library.add(faFirefox);
library.add(faSpinner);
library.add(faSyncAlt);
library.add(faEnvelope);
library.add(faChevronRight);
library.add(faGithub);
library.add(faMailchimp);
library.add(faTwitter);
import {FontAwesomeIcon} from '@fortawesome/react-fontawesome';
import '@fortawesome/fontawesome-svg-core/styles.css';
function getWindowDimensions() {
const width = document.documentElement.clientWidth;
const height = document.documentElement.clientHeight;
return {width, height};
}
function logViewportStats() {
console.log(`DIMENSIONS: window inner: ${window.innerWidth}x${window.innerHeight}`);
console.log(
`DIMENSIONS: document.documentElement: ${document.documentElement.clientWidth}x${
document.documentElement.clientHeight
}`
);
const vv = window.visualViewport;
console.log(`DIMENSIONS: visualViewport: ${vv != null ? vv.width + 'x' + vv.height : vv}`);
const {width, height} = getWindowDimensions();
console.log(`DIMENSIONS: used: ${width}x${height}`);
// TODO FIXME: this is for debugging only
/*const url = `/viewports?wi=${window.innerWidth}x${window.innerHeight}&de=${document.documentElement.clientWidth}x${document.documentElement.clientHeight}&vv=${vv.width}x${vv.height}`;
const Http = new XMLHttpRequest();
Http.open("GET", url);
Http.send();*/
}
const GITHUB_REPO_URL = 'https://github.com/eleweek/inside_python_dict';
const MAILCHIMP_URL = 'http://eepurl.com/gbzhvn';
const TWITTER_LINK = 'https://twitter.com/SashaPutilin';
const EMAIL = 'avp-13@yandex.ru';
function GithubRibbon() {
return (
<a href={GITHUB_REPO_URL}>
<img
style={{position: 'absolute', top: 0, right: 0, border: 0}}
src="https://s3.amazonaws.com/github/ribbons/forkme_right_darkblue_121621.png"
alt="Fork me on GitHub"
/>
</a>
);
}
function GithubCorner() {
// FROM: http://tholman.com/github-corners/
return (
<div
dangerouslySetInnerHTML={{
__html: `<a href="${GITHUB_REPO_URL}" class="github-corner" aria-label="View source on GitHub"><svg width="80" height="80" viewBox="0 0 250 250" style="fill:#151513; color:#fff; position: absolute; top: 0; border: 0; right: 0;" aria-hidden="true"><path d="M0,0 L115,115 L130,115 L142,142 L250,250 L250,0 Z"></path><path d="M128.3,109.0 C113.8,99.7 119.0,89.6 119.0,89.6 C122.0,82.7 120.5,78.6 120.5,78.6 C119.2,72.0 123.4,76.3 123.4,76.3 C127.3,80.9 125.5,87.3 125.5,87.3 C122.9,97.6 130.6,101.9 134.4,103.2" fill="currentColor" style="transform-origin: 130px 106px;" class="octo-arm"></path><path d="M115.0,115.0 C114.9,115.1 118.7,116.5 119.8,115.4 L133.7,101.6 C136.9,99.2 139.9,98.4 142.2,98.6 C133.8,88.0 127.5,74.4 143.8,58.0 C148.5,53.4 154.0,51.2 159.7,51.0 C160.3,49.4 163.2,43.6 171.4,40.1 C171.4,40.1 176.1,42.5 178.8,56.2 C183.1,58.6 187.2,61.8 190.9,65.4 C194.5,69.0 197.7,73.2 200.1,77.6 C213.8,80.2 216.3,84.9 216.3,84.9 C212.7,93.1 206.9,96.0 205.4,96.6 C205.1,102.4 203.0,107.8 198.3,112.5 C181.9,128.9 168.3,122.5 157.7,114.1 C157.9,116.9 156.7,120.9 152.7,124.9 L141.0,136.5 C139.8,137.7 141.6,141.9 141.8,141.8 Z" fill="currentColor" class="octo-body"></path></svg></a><style>.github-corner:hover .octo-arm{animation:octocat-wave 560ms ease-in-out}@keyframes octocat-wave{0%,100%{transform:rotate(0)}20%,60%{transform:rotate(-25deg)}40%,80%{transform:rotate(10deg)}}@media (max-width:500px){.github-corner:hover .octo-arm{animation:none}.github-corner .octo-arm{animation:octocat-wave 560ms ease-in-out}}</style>`,
}}
/>
);
}
function GithubForkMe({windowWidth}) {
/*if (windowWidth != null && windowWidth > 1150) {
return <GithubRibbon />;
} else {*/
return <GithubCorner />;
/*}*/
}
const CONTENTS_DATA = [
[1, 'chapter1.html', 'Searching efficiently in a list'],
[2, 'chapter2.html', 'Why are hash tables called hash tables?'],
[3, 'chapter3.html', 'Putting it all together to make an "almost"-python-dict'],
[4, 'chapter4.html', 'How python dict *really* works internally'],
];
function chapterIdDotHtml(chapterId) {
if (chapterId && !chapterId.endsWith('.html')) {
return chapterId + '.html';
} else {
return null;
}
}
function NextPrev({selectedChapterId}) {
const selectedChapter = chapterIdDotHtml(selectedChapterId);
if (selectedChapter == null) {
return null;
}
let prevHref, prevTitle;
let nextHref, nextTitle;
for (let i = 0; i < CONTENTS_DATA.length; ++i) {
if (CONTENTS_DATA[i][1] === selectedChapter) {
if (i > 0) {
prevHref = CONTENTS_DATA[i - 1][1];
prevTitle = CONTENTS_DATA[i - 1][2];
}
if (i < CONTENTS_DATA.length - 1) {
nextHref = CONTENTS_DATA[i + 1][1];
nextTitle = CONTENTS_DATA[i + 1][2];
}
break;
}
}
if (nextHref) {
return (
<div className="next-prev mt-4">
<a href={nextHref} key={nextHref}>
<h6 key={nextHref}>
Next: {nextTitle} <FontAwesomeIcon key="chevron-right" icon="chevron-right" />{' '}
</h6>
</a>
</div>
);
} else {
return null;
}
}
class Contents extends React.PureComponent {
static EXTRA_ERROR_BOUNDARY = true;
render() {
const {selectedChapterId} = this.props;
const selectedChapter = chapterIdDotHtml(selectedChapterId);
const CIRCLE_SIZE = 30;
return (
<div className="mb-3">
<div className="d-inline-flex flex-column">
{CONTENTS_DATA.map(([i, href, title]) => {
const contentRow = (
<React.Fragment>
<div
key="circle-number"
className="rounded-circle d-flex align-items-center justify-content-center mr-2 toc-number"
style={{
width: CIRCLE_SIZE,
height: CIRCLE_SIZE,
minWidth: CIRCLE_SIZE,
maxWidth: CIRCLE_SIZE,
minHeight: CIRCLE_SIZE,
maxHeight: CIRCLE_SIZE,
backgroundColor: '#7FDBFF',
color: 'white',
}}
>
{i}
</div>
<div key="title" className="d-flex align-items-center mr-3 toc-title">
<h6 className="mb-0">{title}</h6>
</div>
</React.Fragment>
);
return (
<div
key={`toc-row-${i}`}
className="d-flex p-1 align-items-center"
style={{backgroundColor: href === selectedChapter ? 'rgba(0,0,0,.05)' : undefined}}
>
{selectedChapter === href ? (
contentRow
) : (
<a
key="toc-a"
href={href}
style={{color: '#0074D9', fontWeight: 700}}
className="d-flex toc-a"
>
{contentRow}
</a>
)}
</div>
);
})}
</div>
</div>
);
}
}
class LoadingAlert extends React.PureComponent {
constructor() {
super();
this.state = {
loaded: false,
};
}
render() {
return (
<BootstrapAlert
nondismissible={true}
sticky={true}
alertType="info"
key="js-loading"
hide={this.state.loaded && this.props.isRunningInBrowser}
extraclassName="mb-0"
>
<FontAwesomeIcon key="js-loading-spinner" icon="spinner" spin /> JavaScript code is loading...
</BootstrapAlert>
);
}
componentDidMount() {
this.setState({loaded: true});
}
}
class Alerts extends React.Component {
constructor() {
super();
this.state = {
mounted: false,
};
}
render() {
const alerts = [];
const isRunningInBrowser = typeof window !== 'undefined';
alerts.push(<LoadingAlert isRunningInBrowser={isRunningInBrowser} key="loading-warning" />);
if (this.state.mounted) {
const {browser, windowWidth, windowHeight} = this.props;
if (browser) {
if (browser.platform.type === 'mobile') {
alerts.push(
<BootstrapAlert key="mobile-device-warning">
<FontAwesomeIcon icon="desktop" /> <strong>Mobile device detected.</strong> For the best
experience use a desktop browser
</BootstrapAlert>
);
if (windowWidth < windowHeight) {
alerts.push(
<BootstrapAlert key="mobile-device-rotate-warning">
<FontAwesomeIcon icon="sync-alt" /> <strong>Rotating your device is recommended</strong>{' '}
- animations are better with a wider viewport
</BootstrapAlert>
);
}
} else if (browser.browser.name === 'Firefox' && browser.os.name !== 'Linux') {
alerts.push(
<BootstrapAlert key="ff-warning">
<FontAwesomeIcon icon={['fab', 'firefox']} /> <strong>Firefox detected.</strong> Heavy
animations may lag sometimes. If this happens, Chrome or Safari is recommended.
</BootstrapAlert>
);
}
}
}
return <React.Fragment>{alerts}</React.Fragment>;
}
componentDidMount() {
this.setState({mounted: true});
}
}
function Footer() {
return (
<footer className="footer">
<div className="footer-container container-fluid">
<hr />
<div className="footer-list">
<div className="footer-list-item">
<a className="text-muted" href={GITHUB_REPO_URL} target="_blank">
<FontAwesomeIcon icon={['fab', 'github']} /> GitHub repo
</a>
</div>
<div className="footer-list-item">
<a className="text-muted" href={MAILCHIMP_URL} target="_blank">
<FontAwesomeIcon icon={['fab', 'mailchimp']} /> Get notified about new chapters
</a>
</div>
<div className="footer-list-item">
<a className="text-muted" href={TWITTER_LINK} target="_blank">
<FontAwesomeIcon icon={['fab', 'twitter']} /> My Twitter
</a>
</div>
<div className="footer-list-item">
<a
className="text-muted"
href={`mailto:${EMAIL}?subject=Inside Python Dict feedback`}
target="_blank"
>
<FontAwesomeIcon icon="envelope" /> My Email
</a>
</div>
</div>
</div>
</footer>
);
}
// mainly to prevent addressbar stuff on mobile changing things excessively
const SIGNIFICANT_HEIGHT_CHANGE = 100;
export class App extends React.Component {
constructor() {
super();
this.state = {
mounted: false,
windowWidth: null,
windowHeight: null,
};
}
windowSizeChangeHandle = () => {
logViewportStats();
const dimensions = getWindowDimensions();
const windowWidth = dimensions.width;
const windowHeight = dimensions.height;
if (this.state.windowWidth !== windowWidth || this.state.windowHeight !== windowHeight) {
console.log('Processing window size change', windowWidth, windowHeight);
if (
this.state.windowWidth != windowWidth ||
this.state.windowHeight > windowHeight ||
windowHeight - this.state.windowHeight > SIGNIFICANT_HEIGHT_CHANGE
) {
console.log('App size changed from', this.state);
this.setState({
windowWidth,
windowHeight,
});
if (win.width !== windowWidth || win.height !== windowHeight) {
win.setWH(windowWidth, windowHeight);
}
}
fixStickyResize(windowWidth, windowHeight);
}
};
componentDidMount() {
const MEANINGFUL_Y_DIFF = 50; // components that depend on scroll should allow some leeway
let lastScrollY = null;
const onScroll = _.throttle(() => {
if (!lastScrollY || Math.abs(lastScrollY - window.scrollY) > MEANINGFUL_Y_DIFF) {
console.log('onScroll triggered', window.scrollY);
win.setScrollY(window.scrollY);
lastScrollY = window.scrollY;
}
}, 100);
window.addEventListener('scroll', onScroll);
const dimensions = getWindowDimensions();
const windowWidth = dimensions.width;
const windowHeight = dimensions.height;
console.log('componentDidMount() window geometry', windowWidth, windowHeight);
window.addEventListener('resize', _.throttle(this.windowSizeChangeHandle, 500));
globalSettings.maxCodePlaySpeed = getUxSettings().MAX_CODE_PLAY_SPEED;
this.setState({
windowWidth,
windowHeight,
mounted: true,
});
win.setAll(windowWidth, windowHeight, window.scrollY, true);
}
componentWillUnmount() {
window.removeEventListener('resize', this.windowSizeChangeHandle);
}
render() {
console.log('App.render()');
const contents = <Contents selectedChapterId={this.props.selectedChapterId} />;
const independentContents = this.props.selectedChapterId === 'chapter1';
// Make sure SSR works
const {windowWidth, windowHeight} = this.state.mounted ? this.state : {};
let chapters = [];
for (let [i, Chapter] of this.props.chapters.entries()) {
chapters.push(
<MyErrorBoundary key={`error-boundary-${i}`}>
<Chapter windowWidth={windowWidth} windowHeight={windowHeight} contents={contents} />
</MyErrorBoundary>
);
}
return (
<React.Fragment>
<div className="app-container container-fluid">
<MyErrorBoundary>
<GithubForkMe windowWidth={windowWidth} />
</MyErrorBoundary>
<h1> Inside python dict — an explorable explanation</h1>
<MyErrorBoundary>
<Alerts browser={this.props.browser} windowWidth={windowWidth} windowHeight={windowHeight} />
</MyErrorBoundary>
{!independentContents && <MyErrorBoundary>{contents}</MyErrorBoundary>}
{chapters}
<MyErrorBoundary>
<NextPrev selectedChapterId={this.props.selectedChapterId} />
</MyErrorBoundary>
</div>
<Footer />
</React.Fragment>
);
}
}
let _fsrW, _fsrH;
function fixStickyResize(windowWidth, windowHeight) {
// FIXME: this is a hack. This generates a fake resize event that react-stickynode seems to listen to
if (_fsrW !== windowWidth || _fsrH !== windowHeight) {
_fsrW = windowWidth;
_fsrH = windowHeight;
setTimeout(() => window.dispatchEvent(new Event('resize')), 500);
}
}
function fixSticky() {
// Nudges react-stickynode just a little bit
window.requestAnimationFrame(() => {
window.scrollBy(0, -1);
window.requestAnimationFrame(() => {
window.scrollBy(0, 1);
});
});
}
export function initAndRender(chapters, chapterIds) {
if (typeof window !== 'undefined') {
initUxSettings();
window.addEventListener('load', () => {
logViewportStats();
const root = document.getElementById('root');
const isSSR = root.hasChildNodes();
let selectedChapterId;
if (chapterIds.length === 1) {
selectedChapterId = chapterIds[0];
}
const props = {
chapters,
selectedChapterId,
browser: window.insidePythonDictBrowser,
};
if (isSSR) {
console.log('Rehydrating');
ReactDOM.hydrate(<App {...props} />, root);
} else {
console.log('Rendering from scratch');
ReactDOM.render(<App {...props} />, root);
}
// Seems to fix stickynode not stickying on page reload
fixSticky();
});
}
}
================================================
FILE: src/autogenerated/chapter1.html
================================================
<!DOCTYPE html>
<html lang="en">
<head>
<title>Inside python dict — an explorable explanation</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
<link href="https://fonts.googleapis.com/css?family=Montserrat:400,700,700i" rel="stylesheet" />
<link href="https://fonts.googleapis.com/css?family=Roboto:400,400i,700,700i" rel="stylesheet" />
<link href="https://fonts.googleapis.com/css?family=Roboto+Mono:400,400i,500,500i,700,700i" rel="stylesheet" />
</head>
<body>
<div id="root"></div>
<script type="text/javascript">
window.insidePythonDictChapters = ['chapter1'];
</script>
</body>
</html>
================================================
FILE: src/autogenerated/chapter2.html
================================================
<!DOCTYPE html>
<html lang="en">
<head>
<title>Inside python dict — an explorable explanation</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
<link href="https://fonts.googleapis.com/css?family=Montserrat:400,700,700i" rel="stylesheet" />
<link href="https://fonts.googleapis.com/css?family=Roboto:400,400i,700,700i" rel="stylesheet" />
<link href="https://fonts.googleapis.com/css?family=Roboto+Mono:400,400i,500,500i,700,700i" rel="stylesheet" />
</head>
<body>
<div id="root"></div>
<script type="text/javascript">
window.insidePythonDictChapters = ['chapter2'];
</script>
</body>
</html>
================================================
FILE: src/autogenerated/chapter3.html
================================================
<!DOCTYPE html>
<html lang="en">
<head>
<title>Inside python dict — an explorable explanation</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
<link href="https://fonts.googleapis.com/css?family=Montserrat:400,700,700i" rel="stylesheet" />
<link href="https://fonts.googleapis.com/css?family=Roboto:400,400i,700,700i" rel="stylesheet" />
<link href="https://fonts.googleapis.com/css?family=Roboto+Mono:400,400i,500,500i,700,700i" rel="stylesheet" />
</head>
<body>
<div id="root"></div>
<script type="text/javascript">
window.insidePythonDictChapters = ['chapter3'];
</script>
</body>
</html>
================================================
FILE: src/autogenerated/chapter4.html
================================================
<!DOCTYPE html>
<html lang="en">
<head>
<title>Inside python dict — an explorable explanation</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no" />
<link href="https://fonts.googleapis.com/css?family=Montserrat:400,700,700i" rel="stylesheet" />
<link href="https://fonts.googleapis.com/css?family=Roboto:400,400i,700,700i" rel="stylesheet" />
<link href="https://fonts.googleapis.com/css?family=Roboto+Mono:400,400i,500,500i,700,700i" rel="stylesheet" />
</head>
<body>
<div id="root"></div>
<script type="text/javascript">
window.insidePythonDictChapters = ['chapter4'];
</script>
</body>
</html>
================================================
FILE: src/chapter1_simplified_hash.js
================================================
import * as React from 'react';
import _ from 'lodash';
import {List as ImmutableList} from 'immutable';
import {EQ, BreakpointFunction, displayStr} from './hash_impl_common';
import {
LineOfBoxesComponent,
HashBoxesComponent,
HashBoxesBrokenComponent,
TetrisFactory,
VisualizedCode,
SimpleCodeInline,
} from './code_blocks';
import {PyListInput, ParsableInput, BlockInputToolbar, InputTryAnother} from './inputs';
import {
ChapterComponent,
DynamicP,
Subcontainerize,
singularOrPlural,
CrossFade,
COLOR_FOR_READ_OPS,
randint,
randomChoice,
} from './util';
import {chapter1_2_FormatCheckCollision, commonFormatCheckNotFound} from './common_formatters';
import {ProbingVisualization, GenerateProbingLinks} from './probing_visualization';
import {parsePyNumber} from './py_obj_parsing';
import {BigNumber} from 'bignumber.js';
import memoizeOne from 'memoize-one';
import {observer} from 'mobx-react';
import {win} from './store';
const CHAPTER1_MAXNUM = 999;
export const SIMPLE_LIST_SEARCH = [
['def simple_search(simple_list, key):', '', 0],
[' idx = 0', 'start-from-zero', 1],
[' while idx < len(simple_list):', 'check-boundary', 2],
[' if simple_list[idx] == key:', 'check-found', 2],
[' return True', 'found-key', 2],
[' idx += 1', 'next-idx', 2],
[' return False', 'found-nothing', 1],
];
function _parseSmallInt(value) {
const b = parsePyNumber(value);
const error = chapter1valueRangeValidator(b);
if (error) {
throw new Error(error);
}
return +b.toString();
}
function chapter1valueRangeValidator(num) {
if (!BigNumber.isBigNumber(num) && typeof num !== 'number') {
return 'Expected an integer';
}
if (num.lt(-CHAPTER1_MAXNUM) || num.gt(CHAPTER1_MAXNUM)) {
return `In chapter 1 only small integers are supported (between -${CHAPTER1_MAXNUM} and ${CHAPTER1_MAXNUM})`;
}
}
export function PySmallIntInput({inputComponentRef, ...restProps}) {
return (
<ParsableInput {...restProps} dumpValue={JSON.stringify} parseValue={_parseSmallInt} ref={inputComponentRef} />
);
}
function simpleListSearch(l, key) {
let defaultBPInfo = {
type: 'breakpoint',
arg: key,
data: l,
size: l.length,
};
let breakpoints = [];
let newBP = (point, idx, extraInfo) => {
return {...defaultBPInfo, ...{point: point, idx: idx, atIdx: l[idx]}, ...extraInfo};
};
let idx = 0;
breakpoints.push(newBP('start-from-zero', idx));
while (true) {
breakpoints.push(newBP('check-boundary', idx));
if (idx >= l.length) {
break;
}
if (EQ(l[idx], key)) {
breakpoints.push(newBP('check-found', idx, {found: true}));
breakpoints.push(newBP('found-key', idx));
return {bp: breakpoints, result: true};
} else {
breakpoints.push(newBP('check-found', idx, {found: false}));
}
idx += 1;
breakpoints.push(newBP('next-idx', idx));
}
breakpoints.push(newBP('found-nothing', idx));
return {bp: breakpoints, result: false};
}
let formatSimpleListSearchBreakpointDescription = function(bp) {
switch (bp.point) {
case 'iteration':
return `Check element in slot ${bp.idx} (<code>${bp.atIdx}</code>)`;
case 'start-from-zero':
return `Start from the beginning of the list`;
case 'check-boundary':
return bp.idx < bp.size
? `[Try #${bp.idx + 1}] <code>${bp.idx} < ${
bp.size
}</code>, so some elements have not been processed yet, and the number may be there`
: `<code>${bp.idx} == ${bp.size}</code>, so all elements were processed`;
case 'check-found':
return bp.found
? `<code>${bp.atIdx} == ${bp.arg}</code> — the wanted number is found`
: `<code>${bp.atIdx} != ${bp.arg}</code> — the wanted number has not been found so far`;
case 'found-key':
return `so return <code>True</code>`;
case 'found-nothing':
return `The wanted number <code>${bp.arg}</code> was not found, so return <code>False</code>`;
case 'next-idx':
return `Go to the next index: <code>${bp.idx}</code> == <code>${bp.idx - 1} + 1</code>`;
}
};
const SimpleListSearchStateVisualization = TetrisFactory([
[LineOfBoxesComponent, [{labels: ['simple_list']}, 'data', 'idx']],
]);
const UnnamedListVisualizationImpl = TetrisFactory([[LineOfBoxesComponent, [{labels: [null]}, 'data']]]);
function UnnamedListVisualization(props) {
const serverSide = props.windowHeight == null;
return <UnnamedListVisualizationImpl {...props} overflow={serverSide} />;
}
export const SIMPLIFIED_INSERT_ALL_BROKEN_CODE = [
['def build_not_quite_what_we_want(original_list):', 'start-execution', 0],
[' new_list = [None] * len(original_list)', 'create-new-list', 1],
['', ''],
[' for number in original_list:', 'for-loop', 2],
[' idx = number % len(new_list)', 'compute-idx', 2],
[' new_list[idx] = number', 'assign-elem', 2],
[' return new_list', 'return-created-list', 1],
];
export const SIMPLIFIED_INSERT_ALL_CODE = [
['def build_insert_all(original_list):', 'start-execution', 0],
[' new_list = [None] * (2 * len(original_list))', 'create-new-list', 1],
['', ''],
[' for number in original_list:', 'for-loop', 2],
[' idx = number % len(new_list)', 'compute-idx', 2],
[' while new_list[idx] is not None:', 'check-collision', 3],
[' idx = (idx + 1) % len(new_list)', 'next-idx', 3],
[' new_list[idx] = number', 'assign-elem', 2],
[' return new_list', 'return-created-list', 1],
];
class SimplifiedInsertAll extends BreakpointFunction {
constructor() {
super();
this._overwritten = [];
}
run(_originalList, isBroken = false) {
this.fmtIsBroken = isBroken;
this.originalList = new ImmutableList(_originalList);
this.newList = new ImmutableList();
if (isBroken) {
this.fmtMissingNumbers = new ImmutableList();
this.newListWithReplacements = new ImmutableList();
}
const startSize = (isBroken ? 1 : 2) * this.originalList.size;
for (let i = 0; i < startSize; ++i) {
this.newList = this.newList.push(null);
if (isBroken) {
this.newListWithReplacements = this.newListWithReplacements.push(new ImmutableList());
}
}
this.addBP('create-new-list', true);
for ([this.originalListIdx, this.number] of this.originalList.entries()) {
this.fmtCollisionCount = 0;
this.addBP('for-loop');
this.newListIdx = ((this.number % this.newList.size) + this.newList.size) % this.newList.size;
this.addBP('compute-idx');
if (!isBroken) {
while (true) {
this.addBP('check-collision');
if (this.newList.get(this.newListIdx) === null) {
break;
}
this.fmtCollisionCount += 1;
this.newListIdx = (this.newListIdx + 1) % this.newList.size;
this.addBP('next-idx');
}
}
const prevNumber = this.newList.get(this.newListIdx);
if (prevNumber != null) {
if (!isBroken) {
throw new Error(`!isBroken and overwriting a number - this should not happen`);
}
this.fmtMissingNumbers = this.fmtMissingNumbers.push(prevNumber);
this._overwritten.push([this.originalListIdx, prevNumber, this.number]);
}
this.newList = this.newList.set(this.newListIdx, this.number);
if (isBroken) {
this.newListWithReplacements = this.newListWithReplacements.updateIn([this.newListIdx], arr =>
arr.insert(0, this.number)
);
}
this.newList = this.newList.set(this.newListIdx, this.number);
this.addBP('assign-elem', true);
}
this.addBP('return-created-list');
return this.newList;
}
overwrittenNumbers() {
return this._overwritten;
}
}
let formatSimplifiedInsertAllDescription = function(bp, prevBp) {
switch (bp.point) {
case 'create-new-list':
return `Create a new list of <code>${bp.newList.size}</code> empty slots`;
case 'for-loop':
return `[${bp.originalListIdx + 1}/${bp.originalList.size}] The number to insert is <code>${
bp.number
}</code>`;
case 'compute-idx':
return `Compute the slot index: <code>${bp.newListIdx}</code> == <code>${bp.number} % ${
bp.newList.size
}</code>`;
case 'check-collision':
return chapter1_2_FormatCheckCollision(bp.newList, bp.newListIdx, bp.fmtCollisionCount);
case 'next-idx':
return `Keep probing, the next slot will be <code>${bp.newListIdx}</code> == <code>(${
prevBp.newListIdx
} + 1) % ${bp.newList.size}</code>`;
case 'assign-elem': {
const prevNumber = prevBp.newList.get(bp.newListIdx);
if (prevNumber != null) {
return `Collision of <code>${bp.number}</code> with <code>${prevNumber}</code> in slot <code>${
bp.newListIdx
}</code> - the number is overwritten`;
} else {
return `Put <code>${bp.number}</code> in slot <code>${bp.newListIdx}</code>`;
}
}
case 'return-created-list':
if (bp.fmtMissingNumbers && bp.fmtMissingNumbers.size > 0) {
return `Return created list with some numbers missing: ${bp.fmtMissingNumbers
.map(number => `<code>${number}</code>`)
.join(', ')}`;
} else {
return `Return created list with all original numbers present`;
}
}
};
const SimplifiedInsertStateVisualization = TetrisFactory([
[
LineOfBoxesComponent,
[
{labels: ['original_list']},
'originalList',
'originalListIdx',
undefined,
{selection1color: COLOR_FOR_READ_OPS},
],
],
[HashBoxesComponent, [{labels: ['new_list']}, 'newList', 'newListIdx']],
]);
const SimplifiedInsertBrokenStateVisualization = TetrisFactory([
[
LineOfBoxesComponent,
[
{labels: ['original_list']},
'originalList',
'originalListIdx',
undefined,
{selection1color: COLOR_FOR_READ_OPS},
],
],
[HashBoxesBrokenComponent, [{labels: ['new_list']}, 'newListWithReplacements', 'newListIdx']],
]);
export const SIMPLIFIED_SEARCH_CODE = [
['def has_number(new_list, number):', 'start-execution', 0],
[' idx = number % len(new_list)', 'compute-idx', 1],
[' while new_list[idx] is not None:', 'check-not-found', 2],
[' if new_list[idx] == number:', 'check-found', 2],
[' return True', 'found-key', 2],
[' idx = (idx + 1) % len(new_list)', 'next-idx', 2],
[' return False', 'found-nothing', 1],
];
const HideSpanWhenJsLoaded = observer(function HideWhenJsLoaded({children, tag}) {
return <span style={{opacity: win.jsLoaded ? 0 : 1, transition: 'opacity 1s ease'}}>{children}</span>;
});
class SimplifiedSearch extends BreakpointFunction {
run(_newList, _number) {
this.newList = new ImmutableList(_newList);
this.number = _number;
this.fmtCollisionCount = 0;
this.newListIdx = ((this.number % this.newList.size) + this.newList.size) % this.newList.size;
this.addBP('compute-idx');
while (true) {
this.addBP('check-not-found');
if (this.newList.get(this.newListIdx) === null) {
break;
}
this.addBP('check-found');
if (EQ(this.newList.get(this.newListIdx), this.number)) {
this.addBP('found-key');
return true;
}
this.fmtCollisionCount += 1;
this.newListIdx = (this.newListIdx + 1) % this.newList.size;
this.addBP('next-idx');
}
this.addBP('found-nothing');
return false;
}
}
let formatSimplifiedSearchDescription = function(bp) {
switch (bp.point) {
case 'compute-idx':
return `Compute the slot index: <code>${bp.newListIdx}</code> == <code>${bp.number} % ${
bp.newList.size
}</code>`;
case 'check-not-found':
return commonFormatCheckNotFound(bp.newList, bp.newListIdx, bp.fmtCollisionCount);
case 'check-found':
let found = EQ(bp.newList.get(bp.newListIdx), bp.number);
if (found) {
return `The number is found: <code>${bp.newList.get(bp.newListIdx)} == ${bp.number}</code>`;
} else {
return `The number has not been found yet: <code>${bp.newList.get(bp.newListIdx)} != ${
bp.number
}</code>`;
}
case 'found-key':
return 'Now simply return <code>True</code>';
case 'found-nothing':
return 'Now simply return <code>False</code>';
case 'next-idx':
return `Keep retracing probing steps, the next slot will be <code>${bp.newListIdx}</code>`;
case 'return-created-list':
return `Return created list`;
}
};
const SimplifiedSearchStateVisualization = TetrisFactory([
[HashBoxesComponent, [{labels: ['new_list']}, 'newList', 'newListIdx']],
]);
function DynamicSimplifiedInsertAllBrokenOverwrittenExample({originalNumbers, addedNumber, overwrittenNumbers}) {
let exampleOverwrite;
let includedList;
let [idx, n1, n2] = overwrittenNumbers[0];
n1 = displayStr(n1);
n2 = displayStr(n2);
if (addedNumber === null) {
includedList = false;
exampleOverwrite = (
<React.Fragment>
For example, <code>{n1}</code> will get the same slot index (<code>{idx}</code>) as <code>{n2}</code>,
and it will be overwritten.
</React.Fragment>
);
} else {
includedList = true;
const anStr = displayStr(addedNumber);
exampleOverwrite = (
<React.Fragment>
For the current list (
<code dangerouslySetInnerHTML={{__html: '[' + originalNumbers.join(', ') + ']'}} />) it would work. But
if we append a single number to it, for example <code>{anStr}</code>, then <code>{n1}</code> would get
overwritten by <code>{n2}</code>, and the simple algorithm breaks.
</React.Fragment>
);
}
return (
<DynamicP>
<p
className="dynamic-p"
key={`oe-p-${addedNumber}-${n1}-${n2}-${idx}-${includedList ? JSON.stringify(originalNumbers) : null}`}
>
Would this approach work, however? Not entirely. {exampleOverwrite} Situations like these are called{' '}
<em>collisions</em>.
</p>
</DynamicP>
);
}
export class Ops {
static createNew(numbers) {
let sia = new SimplifiedInsertAll();
const keys = sia.run(numbers);
const bp = sia.getBreakpoints();
return {keys, bp};
}
static createNewBroken(numbers) {
let sia = new SimplifiedInsertAll();
const keys = sia.run(numbers, true);
return {bp: sia.getBreakpoints(), overwrittenNumbers: sia.overwrittenNumbers(), keys};
}
static hasKey(keys, number) {
let ss = new SimplifiedSearch();
const result = ss.run(keys, number);
const bp = ss.getBreakpoints();
return {bp, result, keys};
}
static linearSearch(numbers, searchedNumber) {
return simpleListSearch(numbers, searchedNumber);
}
}
function anotherValue(array, RANDOM_NUMBER_CHANCE = 0.3) {
if (Math.random() > RANDOM_NUMBER_CHANCE) {
return +randomChoice(array).toString();
} else {
return randint(Math.floor(-CHAPTER1_MAXNUM / 10), Math.floor(CHAPTER1_MAXNUM / 10));
}
}
export class Chapter1_SimplifiedHash extends ChapterComponent {
constructor() {
super();
this.state = {
numbers: [
BigNumber(1),
BigNumber(56),
BigNumber(50),
BigNumber(2),
BigNumber(44),
BigNumber(25),
BigNumber(17),
BigNumber(4),
],
simpleSearchNumber: 25,
simplifiedHashSearchNumber: 2,
};
}
runSimplifiedInsertAll = memoizeOne(numbers => {
return Ops.createNew(numbers);
});
generateAlternativeDataForInsertAllBroken = memoizeOne(numbers => {
let {bp, overwrittenNumbers} = this.runSimplifiedInsertAllBroken(numbers);
if (overwrittenNumbers.length > 0) {
return {originalNumbers: numbers, numbers, bp, addedNumber: null, overwrittenNumbers};
} else {
const minNum = BigNumber.max(...numbers);
const addedNumber = minNum.plus(numbers.length + 1);
const newNumbers = [...numbers, addedNumber];
({bp, overwrittenNumbers} = this.runSimplifiedInsertAllBroken(newNumbers));
return {originalNumbers: numbers, numbers: newNumbers, bp, addedNumber, overwrittenNumbers};
}
});
runProbingSimple = memoizeOne(slotsCount => {
let g = new GenerateProbingLinks();
const {links} = g.run(slotsCount, '', 'i+1');
return {
links,
bp: g.getBreakpoints(),
};
});
runSimplifiedInsertAllBroken = memoizeOne(numbers => {
return Ops.createNewBroken(numbers);
});
runSimplifiedSearch = memoizeOne((keys, number) => {
return Ops.hasKey(keys, number);
});
runSimpleListSearch = memoizeOne((numbers, searchedNumber) => {
const {bp} = Ops.linearSearch(numbers, searchedNumber);
return {bp};
});
render() {
const slsRes = this.runSimpleListSearch(this.state.numbers, this.state.simpleSearchNumber);
const siaBrokenRes = this.generateAlternativeDataForInsertAllBroken(this.state.numbers);
const siaRes = this.runSimplifiedInsertAll(this.state.numbers);
const ssRes = this.runSimplifiedSearch(siaRes.keys, this.state.simplifiedHashSearchNumber);
const probingVisSlotsCount = 8;
const probingSimple = this.runProbingSimple(probingVisSlotsCount);
return (
<div className="chapter chapter1">
<h5>Contents</h5>
{this.props.contents}
<h2>Introduction</h2>
<Subcontainerize>
<p>
Hi! This is <em>an explorable explanation</em> of Python dictionaries. This page is dynamic and
interactive — you can plug in your data and see how the algorithms work on it.{' '}
<HideSpanWhenJsLoaded>(Once the javascript loads)</HideSpanWhenJsLoaded>
</p>
<p className="mb-2">
To start with, let's say we have a simple list of distinct integers (change it if you want - the
page will update):
</p>
<BlockInputToolbar
input={PyListInput}
inputProps={{
allowDuplicates: false,
minSize: 1,
extraValueValidator: chapter1valueRangeValidator,
}}
initialValue={this.state.numbers}
onChange={this.setter('numbers', true)}
bottomBoundary=".chapter1"
{...this.props}
/>
<p className="mb-0">
Python lists are actually arrays — contiguous chunks of memory. The name "list" may be
misleading to people who know about double-linked lists but are unfamiliar with Python. You can
picture a Python list as a row of slots, where each slot can hold a Python object:
</p>
<UnnamedListVisualization
bp={{data: this.state.numbers}}
compensateTopPadding={25}
windowWidth={this.props.windowWidth}
windowHeight={this.props.windowHeight}
/>
<p>
To check if an element is present in a list, we can use the <code>in</code> operator like this:{' '}
<code className="text-nowrap">number in simple_list</code>, which returns either{' '}
<code>True</code> or <code>False</code>. Under the hood this short snippet does a linear scan.
This can be a lot of work. To see this, let's reimplement it in Python.
</p>
<div className="div-p">
Let's say we're looking for the following number in the original list:
<PySmallIntInput
inline={true}
value={this.state.simpleSearchNumber}
onChange={this.setter('simpleSearchNumber')}
anotherValue={() => anotherValue(this.state.numbers)}
/>
</div>
<VisualizedCode
code={SIMPLE_LIST_SEARCH}
breakpoints={slsRes.bp}
formatBpDesc={formatSimpleListSearchBreakpointDescription}
stateVisualization={SimpleListSearchStateVisualization}
autoplayByDefault={true}
{...this.props}
/>
<p>
(The visualization is interactive. The buttons allow you to step through the code. Notice here
that the time slider is draggable - feel free to rewind the time or move it forward. Also, feel
free to mess with the input and the original list - the visualization will update automatically)
</p>
<p>
What's not so great about linear scans? If we have a million distinct numbers, in the worst case
scenario, we may need to scan the whole list. But scanning over a few elements is no big deal.
We need to have some order and predictability to make the search fast. We need to have some idea
of where the searched element is located.
</p>
<p>
A Python dict implementation is basically a scan of a list (but a pretty weird scan). We'll
build the actual algorithm and data structure inside Python dictionary step by step, starting
with the code above, which is intentionally verbose.{' '}
</p>
<h2>Chapter 1: searching efficiently in a list</h2>
<p>
A Python dict is a collection of key-value pairs. And, the most important part of it is handling
keys. Keys need to be organized in such a way that efficient searching, inserting and deleting
is possible.
</p>
<p>
In this chapter, to keep things simple, we won't have any values, and "keys" will just be plain
integers. So, the simplified problem is to check if a number is present in a list, but we have
to do this{' '}
<em>
<strong>fast</strong>
</em>
. We'll tackle the real problem in the following chapters. But for now, bear with me.
</p>
<p>
Accessing a single element by index is very fast. Accessing only a few elements would be fast
too. We don't want to be doing a linear scan over the whole list every time we look up a number, so
we need to organize our data in a clever way.
</p>
<p> Here's how. </p>
<p>
Let's begin by creating a new list of slots. Each slot will either hold a number from the
original list or be empty (empty slots will hold <code>None</code>). We'll use the number itself
to compute an index of a slot. The simplest way to do this is to take the remainder of{' '}
<code>number</code> divided by <code>len(the_list)</code>:{' '}
<code className="text-nowrap">number % len(the_list)</code> and put our number in slot with this
index. To check if the number is there we could compute the slot index again and see if it is
empty.
</p>
<DynamicSimplifiedInsertAllBrokenOverwrittenExample
key="overwritten-example-component"
{...siaBrokenRes}
/>
<VisualizedCode
code={SIMPLIFIED_INSERT_ALL_BROKEN_CODE}
breakpoints={siaBrokenRes.bp}
formatBpDesc={formatSimplifiedInsertAllDescription}
stateVisualization={SimplifiedInsertBrokenStateVisualization}
{...this.props}
/>
<p>
To make this approach viable, we need to somehow <em>resolve collisions</em>. Let's do the
following: if the slot is already occupied by some other number, we'll just check the slot that
comes right after it. And if that slot is empty, we'll put the number there. But, what if that
slot is also occupied? Once again, we'll go ahead and check the next slot. We'll keep repeating
this process until we finally hit an empty slot. This process is called <em>probing</em>. And
because we do it linearly, it is called <em>linear probing</em>. In code, we would write this as{' '}
<code className="text-nowrap">(idx + 1) % len(simple_list)</code>, so it wraps around back to
the beginning at the last index:
</p>
<ProbingVisualization
slotsCount={probingVisSlotsCount}
links={probingSimple.links}
adjustTop={-70}
fixedHeight={170}
{...this.props}
/>
<p>
If we make the new list the same size as the original list, we'll have too many collisions. If
we make it 10x larger, we'll have very few collisions, but we'll waste a lot of memory. So what
size should it be? We want to hit the sweet spot where we don't use up too much memory but also
don't have too many collisions. Twice the size of the original list is reasonable.
</p>
<p>
Let's transform the original list using this method (when reading this code, keep in mind that{' '}
<code>original_list</code> is a list of <em>distinct numbers</em>, so we don't need to handle
duplicates just yet).
</p>
<VisualizedCode
code={SIMPLIFIED_INSERT_ALL_CODE}
breakpoints={siaRes.bp}
formatBpDesc={formatSimplifiedInsertAllDescription}
stateVisualization={SimplifiedInsertStateVisualization}
{...this.props}
/>
<p>
To search for a number, we retrace all the steps necessary to insert it: we start from the slot{' '}
<code className="text-nowrap">number % len(new_list)</code> and do linear probing. We either end
up finding the number or hitting an empty slot. The latter situation means that the number is
not present.
</p>
<div className="div-p">
Let's say we want to search for
<PySmallIntInput
inline={true}
value={this.state.simplifiedHashSearchNumber}
onChange={this.setter('simplifiedHashSearchNumber')}
anotherValue={() => anotherValue(this.state.numbers, 0.4)}
/>
</div>
<VisualizedCode
code={SIMPLIFIED_SEARCH_CODE}
breakpoints={ssRes.bp}
formatBpDesc={formatSimplifiedSearchDescription}
stateVisualization={SimplifiedSearchStateVisualization}
{...this.props}
/>
<p>
Calculating an index based on the value of the number and resolving collisions by linear probing
is an incredibly powerful idea. What we've just implemented is a simple <em>hash table</em>{' '}
(more about the term in the next chapter). Python dict uses a hash table internally, albeit a
more complex variant.
</p>
<p>
We still haven't discussed adding more elements (what happens if a table overflows?), removing
elements (removing an element without a trace would cause a hole to appear, wouldn't this cause
the search algorithm to stop prematurely in many cases?), and perhaps most importantly, handling
objects other than integers - strings, tuples, floats. We'll do this in the next chapters.
</p>
<h6>Collision resolution via separate chaining</h6>
<p>
There is a different method of collision resolution, called{' '}
<a href="https://en.wikipedia.org/wiki/Hash_table#Separate_chaining" target="_blank">
separate chaining
</a>
. It is also a good strategy which is commonly used. But that's not how Python resolves
collision in dicts, so it is beyond the scope of this explanation.{' '}
</p>
<h6>A couple of the notes about the explanation</h6>
<p>
First, this explanation discusses <code>dict</code> as it is implemented in{' '}
<a href="http://python.org/" target="_blank">
CPython
</a>{' '}
— the "default" and most common implementation of the Python language (if you are not sure
what implementation you use, it is almost certainly CPython). Some other implementations are{' '}
<a href="https://pypy.org/" target="_blank">
PyPy
</a>
,{' '}
<a href="http://www.jython.org/" target="_blank">
Jython
</a>{' '}
and{' '}
<a href="http://ironpython.net/" target="_blank">
IronPython
</a>
. The way dicts works in each of these implementations may be similar to CPython (in the case of
PyPy) or very different from CPython (in the case of Jython).
</p>
<p>
Second, even though dict in CPython is implemented in C, this explanation uses Python for code
snippets. The goal of this page is to help you understand the algorithms and the underlying data
structures, not the minutiae of the C code (these details are interesting too - they are just
are beyond the scope of this explanation).
</p>
</Subcontainerize>
</div>
);
}
}
================================================
FILE: src/chapter2_hash_table_functions.js
================================================
import * as React from 'react';
import {BigNumber} from 'bignumber.js';
import {List} from 'immutable';
import {pyHash, pyHashUnicode, pyHashLong, HashBreakpointFunction, DUMMY, EQ, displayStr} from './hash_impl_common';
import {
HashBoxesComponent,
LineOfBoxesComponent,
TetrisFactory,
SimpleCodeBlock,
VisualizedCode,
SMALLER_BOX_GEOMETRY,
} from './code_blocks';
import {PyStringInput, PyNumberInput, PyListInput, PySNNInput, BlockInputToolbar} from './inputs';
import {
ChapterComponent,
Subcontainerize,
COLOR_FOR_READ_OPS,
randomMeaningfulString,
randomString3len,
randint,
randomChoice,
} from './util';
import {chapter1_2_FormatCheckCollision, commonFormatCheckNotFound} from './common_formatters';
import {library} from '@fortawesome/fontawesome-svg-core';
import {FontAwesomeIcon} from '@fortawesome/react-fontawesome';
import {faRedoAlt} from '@fortawesome/free-solid-svg-icons/faRedoAlt';
import {faTrashAlt} from '@fortawesome/free-solid-svg-icons/faTrashAlt';
library.add(faTrashAlt);
import memoizeOne from 'memoize-one';
export const HASH_CREATE_NEW_CODE = [
['def create_new(from_keys):', 'start-execution', 0],
[' hash_codes = [EMPTY] * (2 * len(from_keys))', 'create-new-empty-hashes', 1],
[' keys = [EMPTY] * (2 * len(from_keys))', 'create-new-empty-keys', 1],
['', '', -1],
[' for key in from_keys:', 'for-loop', 2],
[' hash_code = hash(key)', 'compute-hash', 2],
[' idx = hash_code % len(keys)', 'compute-idx', 2],
[' while hash_codes[idx] is not EMPTY:', 'check-collision', 3],
[' if hash_codes[idx] == hash_code and \\', 'check-dup-hash', 3],
[' keys[idx] == key:', 'check-dup-key', 3],
[' break', 'check-dup-break', 4],
[' idx = (idx + 1) % len(keys)', 'next-idx', 3],
['', '', -1],
[' hash_codes[idx], keys[idx] = hash_code, key', 'assign-elem', 2],
['', '', -1],
[' return hash_codes, keys', 'return-lists', 1],
];
function anotherValue(array, ARRAY_CHANCE = 0.5, MEANINGFUL_CHANCE = 0.25, NUMBER_CHANCE = 0.2) {
const roll = Math.random();
if (roll < ARRAY_CHANCE) {
return randomChoice(array);
} else if (roll < ARRAY_CHANCE + MEANINGFUL_CHANCE) {
return randomMeaningfulString();
} else if (roll < ARRAY_CHANCE + MEANINGFUL_CHANCE + NUMBER_CHANCE) {
return BigNumber(randint(-100, 100));
} else {
return randomString3len();
}
}
class HashCreateNew extends HashBreakpointFunction {
run(_fromKeys) {
this.fromKeys = new List(_fromKeys);
this.hashCodes = new List();
this.keys = new List();
for (let i = 0; i < this.fromKeys.size * 2; ++i) {
this.hashCodes = this.hashCodes.push(null);
}
this.addBP('create-new-empty-hashes');
for (let i = 0; i < this.fromKeys.size * 2; ++i) {
this.keys = this.keys.push(null);
}
this.addBP('create-new-empty-keys');
for ([this.fromKeysIdx, this.key] of this.fromKeys.entries()) {
this.addBP('for-loop');
this.hashCode = pyHash(this.key);
this.addBP('compute-hash');
this.idx = this.computeIdx(this.hashCode, this.keys.size);
this.addBP('compute-idx');
this.fmtCollisionCount = 0;
while (true) {
this.addBP('check-collision');
if (this.keys.get(this.idx) === null) {
break;
}
this.addBP('check-dup-hash');
if (this.hashCodes.get(this.idx).eq(this.hashCode)) {
this.addBP('check-dup-key');
if (EQ(this.keys.get(this.idx), this.key)) {
this.addBP('check-dup-break');
break;
}
}
this.fmtCollisionCount += 1;
this.idx = (this.idx + 1) % this.keys.size;
this.addBP('next-idx');
}
this.hashCodes = this.hashCodes.set(this.idx, this.hashCode);
this.keys = this.keys.set(this.idx, this.key);
this.addBP('assign-elem');
}
this.fromKeysIdx = null;
this.key = null;
this.addBP('return-lists');
return [this.hashCodes, this.keys];
}
}
const HashCreateNewStateVisualization = TetrisFactory([
[
LineOfBoxesComponent,
[
{labels: ['from_keys'], marginBottom: 20},
'fromKeys',
'fromKeysIdx',
undefined,
{selection1color: COLOR_FOR_READ_OPS},
],
],
[HashBoxesComponent, [{labels: ['keys'], marginBottom: 7}, 'keys', 'idx']],
[HashBoxesComponent, [{labels: ['hash_codes']}, 'hashCodes', 'idx']],
]);
function formatHashCreateNewAndInsert(bp, prevBp) {
switch (bp.point) {
case 'create-new-empty-hashes':
return `Create a new list of size <code>${bp.hashCodes.size}</code> for hash codes`;
case 'create-new-empty-keys':
return `Create a new list of size <code>${bp.keys.size}</code> for keys`;
case 'for-loop':
return `[${bp.fromKeysIdx + 1}/${bp.fromKeys.size}] The key to insert is <code>${displayStr(
bp.key
)}</code>`;
case 'compute-hash':
return `Compute the hash code: <code>${bp.hashCode}</code>`;
case 'compute-idx':
return `Compute the starting slot index: <code>${bp.idx}</code> == <code>${bp.hashCode} % ${bp.keys.size}</code>`;
case 'check-collision':
return chapter1_2_FormatCheckCollision(bp.keys, bp.idx, bp.fmtCollisionCount);
case 'check-dup-hash':
if (EQ(bp.hashCodes.get(bp.idx), bp.hashCode)) {
return `<code>${bp.hashCodes.get(bp.idx)} == ${
bp.hashCode
}</code>, we cannot rule out the slot being occupied by the same key`;
} else {
return `<code>${bp.hashCodes.get(bp.idx)} != ${
bp.hashCode
}</code>, so there is a collision with a different key`;
}
case 'check-dup-key':
if (EQ(bp.keys[bp.idx], bp.key)) {
return `<code>${displayStr(bp.keys.get(bp.idx))} == ${displayStr(
bp.key
)}</code>, so the key is already in the table`;
} else {
return `<code>${displayStr(bp.keys.get(bp.idx))} != ${displayStr(
bp.key
)}</code>, so there is a collision`;
}
case 'check-dup-break':
return 'Because the key is found, stop';
case 'check-dup-return':
return 'Because the key is found, stop';
case 'next-idx':
return `Keep probing, the next slot will be <code>${bp.idx}</code> == <code>(${prevBp.idx} + 1) % ${bp.keys.size}</code>`;
case 'assign-elem':
if (prevBp.keys.get(bp.idx) === null) {
return `Put <code>${displayStr(bp.key)}</code> and its hash <code>${
bp.hashCode
}</code> in the empty slot <code>${bp.idx}</code>`;
} else {
return `<code>${displayStr(bp.key)}</code> and its hash <code>${
bp.hashCode
}</code> is already in the slot, overwriting it anyway`;
}
case 'return-lists':
return `The hash table is complete, return the lists`;
}
}
export const HASH_SEARCH_CODE = [
['def has_key(hash_codes, keys, key):', 'start-execution', 0],
[' hash_code = hash(key)', 'compute-hash', 1],
[' idx = hash_code % len(keys)', 'compute-idx', 1],
[' while hash_codes[idx] is not EMPTY:', 'check-not-found', 2],
[' if hash_codes[idx] == hash_code and \\', 'check-hash', 2],
[' keys[idx] == key:', 'check-key', 2],
[' return True', 'return-true', 3],
[' idx = (idx + 1) % len(keys)', 'next-idx', 2],
[' return False', 'return-false', 1],
];
function formatHashRemoveSearch(bp, prevBp) {
switch (bp.point) {
case 'compute-hash':
return `Compute the hash code: <code>${bp.hashCode}</code>`;
case 'compute-idx':
return `Compute the starting slot index: <code>${bp.hashCode} % ${bp.keys.size}</code> == <code>${bp.idx}</code>`;
case 'check-not-found':
return commonFormatCheckNotFound(bp.keys, bp.idx, bp.fmtCollisionCount);
case 'check-hash':
if (bp.hashCodes.get(bp.idx).eq(bp.hashCode)) {
return `<code>${bp.hashCodes.get(bp.idx)} == ${
bp.hashCode
}</code>, so the slot might contain the same key`;
} else {
return `<code>${bp.hashCodes.get(bp.idx)} != ${
bp.hashCode
}</code>, so the slot definitely contains a different key`;
}
case 'check-key':
if (EQ(bp.keys.get(bp.idx), bp.key)) {
return `<code>${displayStr(bp.keys.get(bp.idx))} == ${displayStr(bp.key)}</code>, so the key is found`;
} else {
return `<code>${displayStr(bp.keys.get(bp.idx))} != ${
bp.key
}</code>, so there is a different key with the same hash`;
}
case 'assign-dummy':
return `Replace key in slot <code>${bp.idx}</code> with <code>DUMMY</code> placeholder`;
case 'return':
return `The key is removed, now return`;
case 'next-idx':
return `Keep retracing probing steps, the next slot will be <code>${bp.idx}</code> == <code>(${prevBp.idx} + 1) % ${bp.keys.size}</code>`;
case 'throw-key-error':
return `Throw an exception, because no key was found`;
/* search */
case 'return-true':
return `so return <code>True</code>`;
case 'return-false':
return `We hit an empty slot, so the key is not there. Return <code>False</code>`;
}
}
const HashNormalStateVisualization = TetrisFactory([
[HashBoxesComponent, [{labels: ['keys'], marginBottom: 7}, 'keys', 'idx']],
[HashBoxesComponent, [{labels: ['hash_codes']}, 'hashCodes', 'idx']],
]);
const HashNormalStateVisualizationSmallBoxes = TetrisFactory(
[
[HashBoxesComponent, [{labels: ['keys'], marginBottom: 7}, 'keys', 'idx']],
[HashBoxesComponent, [{labels: ['hash_codes']}, 'hashCodes', 'idx']],
],
{fixedGeometry: SMALLER_BOX_GEOMETRY}
);
export const HASH_REMOVE_CODE = [
['def remove(hash_codes, keys, key):', 'start-execution', 0],
[' hash_code = hash(key)', 'compute-hash', 1],
[' idx = hash_code % len(keys)', 'compute-idx', 1],
['', '', -1],
[' while hash_codes[idx] is not EMPTY:', 'check-not-found', 2],
[' if hash_codes[idx] == hash_code and \\', 'check-hash', 2],
[' keys[idx] == key:', 'check-key', 2],
[' keys[idx] = DUMMY', 'assign-dummy', 2],
[' return', 'return', 3],
[' idx = (idx + 1) % len(keys)', 'next-idx', 2],
['', ''],
[' raise KeyError()', 'throw-key-error', 1],
];
class HashRemoveOrSearch extends HashBreakpointFunction {
run(_hashCodes, _keys, _key, isRemoveMode) {
this.hashCodes = new List(_hashCodes);
this.keys = new List(_keys);
this.key = _key;
this.fmtCollisionCount = 0;
this.hashCode = pyHash(this.key);
this.addBP('compute-hash');
this.idx = this.computeIdx(this.hashCode, this.keys.size);
this.addBP('compute-idx');
while (true) {
this.addBP('check-not-found');
if (this.keys.get(this.idx) === null) {
break;
}
this.addBP('check-hash');
if (this.hashCodes.get(this.idx).eq(this.hashCode)) {
this.addBP('check-key');
if (EQ(this.keys.get(this.idx), this.key)) {
if (isRemoveMode) {
this.keys = this.keys.set(this.idx, DUMMY);
this.addBP('assign-dummy');
this.addBP('return');
return {hashCodes: this.hashCodes, keys: this.keys, isException: false, result: null};
} else {
this.addBP('return-true');
return {hashCodes: this.hashCodes, keys: this.keys, isException: false, result: true};
}
}
}
this.fmtCollisionCount += 1;
this.idx = (this.idx + 1) % this.keys.size;
this.addBP('next-idx');
}
let result, isException;
if (isRemoveMode) {
this.addBP('throw-key-error');
isException = true;
} else {
this.addBP('return-false');
isException = false;
result = false;
}
return {hashCodes: this.hashCodes, keys: this.keys, isException, result};
}
}
export const HASH_RESIZE_CODE = [
['def resize(hash_codes, keys):', 'start-execution', 0],
[' new_hash_codes = [EMPTY] * (2 * len(hash_codes))', 'create-new-empty-hashes', 1],
[' new_keys = [EMPTY] * (2 * len(keys))', 'create-new-empty-keys', 1],
[' for hash_code, key in zip(hash_codes, keys):', 'for-loop', 2],
[' if key is EMPTY or key is DUMMY:', 'check-skip-empty-dummy', 2],
[' continue', 'continue', 3],
[' idx = hash_code % len(new_keys)', 'compute-idx', 2],
[' while new_hash_codes[idx] is not EMPTY:', 'check-collision', 3],
[' idx = (idx + 1) % len(new_keys)', 'next-idx', 3],
[' new_hash_codes[idx], new_keys[idx] = hash_code, key', 'assign-elem', 2],
['', ''],
[' return new_hash_codes, new_keys', 'return-lists', 1],
];
class HashResize extends HashBreakpointFunction {
run(_hashCodes, _keys) {
this.hashCodes = _hashCodes;
this.keys = _keys;
this.newHashCodes = new List();
this.newKeys = new List();
for (let i = 0; i < this.hashCodes.size * 2; ++i) {
this.newHashCodes = this.newHashCodes.push(null);
}
this.addBP('create-new-empty-hashes');
for (let i = 0; i < this.hashCodes.size * 2; ++i) {
this.newKeys = this.newKeys.push(null);
}
this.addBP('create-new-empty-keys');
for ([this.oldIdx, [this.hashCode, this.key]] of this.hashCodes.zip(this.keys).entries()) {
this.addBP('for-loop');
this.addBP('check-skip-empty-dummy');
if (this.key === null || this.key === DUMMY) {
this.addBP('continue');
continue;
}
this.fmtCollisionCount = 0;
this.idx = this.computeIdx(this.hashCode, this.newKeys.size);
this.addBP('compute-idx');
while (true) {
this.addBP('check-collision');
if (this.newKeys.get(this.idx) === null) {
break;
}
this.fmtCollisionCount += 1;
this.idx = (this.idx + 1) % this.newKeys.size;
this.addBP('next-idx');
}
this.newHashCodes = this.newHashCodes.set(this.idx, this.hashCode);
this.newKeys = this.newKeys.set(this.idx, this.key);
this.addBP('assign-elem');
}
this.addBP('return-lists');
return [this.newHashCodes, this.newKeys];
}
}
function formatHashResize(bp, prevBp) {
switch (bp.point) {
case 'create-new-empty-hashes':
return `Create a new list of size <code>${bp.newHashCodes.size}</code> for hash codes`;
case 'create-new-empty-keys':
return `Create a new list of size <code>${bp.newKeys.size}</code> for keys`;
case 'for-loop':
return `[${bp.oldIdx + 1}/${bp.keys.size}] The current key to insert is <code>${
bp.key === null ? 'EMPTY' : bp.key
}</code>, its hash is <code>${bp.hashCode === null ? 'EMPTY' : bp.hashCode}</code>`;
case 'compute-idx':
return `Compute the starting slot index: <code>${bp.idx} == ${bp.hashCode} % ${bp.newKeys.size}</code>`;
case 'check-skip-empty-dummy':
if (bp.keys.get(bp.oldIdx) === null) {
return `The current slot is empty`;
} else if (bp.keys.get(bp.oldIdx) === DUMMY) {
return `The current slot contains a <code>DUMMY</code> placeholder`;
} else {
return `The current slot is occupied by an actually existing key`;
}
case 'continue':
return 'So skip it';
case 'check-collision':
return chapter1_2_FormatCheckCollision(bp.newKeys, bp.idx, bp.fmtCollisionCount);
case 'next-idx':
return `Keep probing, the next slot will be <code>${bp.idx}</code> == <code>(${prevBp.idx} + 1) % ${bp.keys.size}</code>`;
case 'assign-elem':
return `Put <code>${displayStr(bp.key)}</code> and its hash <code>${bp.hashCode}</code> in the empty slot ${
bp.idx
}`;
case 'return-lists':
return `The hash table has been rebuilt, return the lists`;
}
}
const HashResizeStateVisualization = TetrisFactory([
[
HashBoxesComponent,
[{labels: ['keys'], marginBottom: 7}, 'keys', 'oldIdx', undefined, {selection1color: COLOR_FOR_READ_OPS}],
],
[
HashBoxesComponent,
[
{labels: ['hash_codes'], marginBottom: 20},
'hashCodes',
'oldIdx',
undefined,
{selection1color: COLOR_FOR_READ_OPS},
],
],
[HashBoxesComponent, [{labels: ['new_keys'], marginBottom: 7}, 'newKeys', 'idx']],
[HashBoxesComponent, [{labels: ['new_hash_codes']}, 'newHashCodes', 'idx']],
]);
export const HASH_INSERT_CODE = [
['def insert(hash_codes, keys, key):', 'start-execution'],
[' hash_code = hash(key)', 'compute-hash'],
[' idx = hash_code % len(keys)', 'compute-idx'],
['', ''],
[' while keys[idx] is not EMPTY:', 'check-collision'],
[' if hash_codes[idx] == hash_code and\\', 'check-dup-hash'],
[' keys[idx] == key:', 'check-dup-key'],
[' break', 'check-dup-break'],
[' idx = (idx + 1) % len(keys)', 'next-idx'],
['', ''],
[' hash_codes[idx], keys[idx] = hash_code, key', 'assign-elem'],
];
class HashInsert extends HashBreakpointFunction {
run(_hashCodes, _keys, _key) {
this.hashCodes = new List(_hashCodes);
this.keys = new List(_keys);
this.key = _key;
this.hashCode = pyHash(this.key);
this.addBP('compute-hash');
this.idx = this.computeIdx(this.hashCode, this.keys.size);
this.addBP('compute-idx');
while (true) {
this.addBP('check-collision');
if (this.keys.get(this.idx) === null) {
break;
}
this.addBP('check-dup-hash');
if (this.hashCodes.get(this.idx).eq(this.hashCode)) {
this.addBP('check-dup-key');
if (EQ(this.keys.get(this.idx), this.key)) {
this.addBP('check-dup-break');
break;
}
}
this.idx = (this.idx + 1) % this.keys.size;
this.addBP('next-idx');
}
this.hashCodes = this.hashCodes.set(this.idx, this.hashCode);
this.keys = this.keys.set(this.idx, this.key);
this.addBP('assign-elem');
return [this.hashCodes, this.keys];
}
}
class HashExamples extends React.Component {
constructor(props) {
super(props);
this.state = {
string: 'Hello',
integer: 42,
};
}
render() {
const COMMON_WIDTH = 72;
const style = {
minWidth: COMMON_WIDTH,
widdth: COMMON_WIDTH,
display: 'inline-block',
};
return (
<div>
<div className="div-p">
<span style={style}>Strings:</span>
<code>hash(</code>
<PyStringInput
inline={true}
value={this.state.string}
onChange={value => this.setState({string: value})}
/>
<code>)</code> = <code>{pyHashUnicode(this.state.string)}</code>
</div>
<div className="div-p">
<span style={style}>Integers:</span>
<code>hash(</code>
<PyNumberInput
inline={true}
value={this.state.integer}
onChange={value => this.setState({integer: value})}
/>
<code>)</code> = <code>{pyHashLong(BigNumber(this.state.integer)).toFixed()}</code>
</div>
<p>
<span style={style}>Floats:</span>
<code>hash(42.5)</code> = <code>1426259968</code>
</p>
<p>
<span style={style}>Tuples: </span>
<code>hash(("Hi", 11))</code> = <code>4421265786515608844</code>
</p>
<p>
<span style={style}>None: </span>
<code>hash(None)</code> = <code>-9223372036581563745</code>
</p>
</div>
);
}
}
export class Ops {
static createNew(array) {
const hcn = new HashCreateNew();
const [hashCodes, keys] = hcn.run(array);
const bp = hcn.getBreakpoints();
return {hashCodes, keys, bp};
}
static hasKey(hashCodes, keys, searchedObj) {
const hs = new HashRemoveOrSearch();
const {result, hashCodes: newHashCodes, keys: newKeys} = hs.run(hashCodes, keys, searchedObj, false);
const bp = hs.getBreakpoints();
return {result, bp, hashCodes: newHashCodes, keys: newKeys};
}
static remove(hashCodes, keys, objToRemove) {
const hr = new HashRemoveOrSearch();
const {isException, hashCodes: newHashCodes, keys: newKeys} = hr.run(hashCodes, keys, objToRemove, true);
const bp = hr.getBreakpoints();
return {isException, bp, hashCodes: newHashCodes, keys: newKeys};
}
static resize(hashCodes, keys) {
const hres = new HashResize();
const [newHashCodes, newKeys] = hres.run(hashCodes, keys);
const bp = hres.getBreakpoints();
return {bp, hashCodes: newHashCodes, keys: newKeys};
}
static insert(hashCodes, keys, objToInsert) {
const hi = new HashInsert();
const [newHashCodes, newKeys] = hi.run(hashCodes, keys, objToInsert);
const bp = hi.getBreakpoints();
return {bp, hashCodes: newHashCodes, keys: newKeys};
}
}
function ListDescription({array}) {
let countStr = array.filter(e => typeof e === 'string').length;
let countNum = array.length - countStr;
if (countStr > 0 && countNum > 0) {
return 'a mixed list of integers and strings';
} else if (countStr > 0) {
return 'a list of strings';
} else {
return 'a list of integers';
}
}
class HashResizeInPlaceAnimation extends React.PureComponent {
static FULL_WIDTH = true;
static EXTRA_ERROR_BOUNDARY = true;
constructor() {
super();
this.state = {
isStart: true,
breakpointsUpdatedCounter: 0,
};
}
static getDerivedStateFromProps(props, state) {
if (props.breakpoints !== state.breakpoints) {
return {
...state,
breakpoints: props.breakpoints,
breakpointsUpdatedCounter: state.breakpointsUpdatedCounter + 1,
};
} else {
return null;
}
}
runAnimation = () => {
console.log('runAnimation');
this.setState(state => ({
isStart: !state.isStart,
}));
};
render() {
let hashCodes, keys;
let buttonIcon, buttonLabel;
if (this.state.isStart) {
({hashCodes, keys} = this.state.breakpoints[0]);
buttonLabel = 'Throw away the old table';
buttonIcon = <FontAwesomeIcon icon="trash-alt" />;
} else {
({newHashCodes: hashCodes, newKeys: keys} = this.state.breakpoints[this.state.breakpoints.length - 1]);
buttonLabel = 'Reset';
buttonIcon = <FontAwesomeIcon icon="redo-alt" />;
}
return (
<div className="hl-left" style={{paddingLeft: 20}}>
<button
type="button"
className="btn btn-primary btn-sm"
onClick={this.runAnimation}
style={{minWidth: 180}}
>
{buttonIcon} {buttonLabel}
</button>
<HashNormalStateVisualizationSmallBoxes
bp={{hashCodes, keys}}
epoch={this.state.breakpointsUpdatedCounter}
fixedGeometry={SMALLER_BOX_GEOMETRY}
/>
</div>
);
}
}
export class Chapter2_HashTableFunctions extends ChapterComponent {
constructor() {
super();
this.state = {
array: ['uname', 'mv', 1, 'time', -6, 'ps', 'mkdir', 'less'],
searchedObj: 'uname',
objToRemove: 'mv',
};
}
runCreateNew = memoizeOne(array => {
return Ops.createNew(array);
});
runSearch = memoizeOne((hashCodes, keys, searchedObj) => {
const bp = Ops.hasKey(hashCodes, keys, searchedObj).bp;
return {bp};
});
runRemove = memoizeOne((hashCodes, keys, objToRemove) => {
const {bp, hashCodes: newHashCodes, keys: newKeys} = Ops.remove(hashCodes, keys, objToRemove);
return {bp, hashCodes: newHashCodes, keys: newKeys};
});
runResize = memoizeOne((hashCodes, keys) => {
const bp = Ops.resize(hashCodes, keys).bp;
return {bp};
});
/*runInsert = memoizeOne((hashCodes, keys, objToInsert) => {
const bp = Ops.insert(hashCodes, keys, objToInsert).bp;
return {bp};
});*/
render() {
const t1 = performance.now();
const newRes = this.runCreateNew(this.state.array);
let {hashCodes, keys} = newRes;
const searchRes = this.runSearch(hashCodes, keys, this.state.searchedObj);
const removeRes = this.runRemove(hashCodes, keys, this.state.objToRemove);
hashCodes = removeRes.hashCodes;
keys = removeRes.keys;
const resizeRes = this.runResize(hashCodes, keys);
/*const insertRes = this.runInsert(hashCodes, keys, this.state.objToInsert);*/
console.log('Chapter2 render timing', performance.now() - t1);
return (
<div className="chapter chapter2">
<h2> Chapter 2. Why are hash tables called hash tables? </h2>
<Subcontainerize>
<p>
Now that we have the solution for searching in a list of numbers, can we use it for non-integer
objects? We can if we find a way to turn objects into numbers for indexing.
</p>
<p>
{' '}
We don't need a perfect one-to-one correspondence between objects and integers. In fact, it is
totally fine if two unrelated objects are turned into the same number — we can use linear
probing to resolve this collision anyway!
</p>
<p>
However, if we turn all objects into the same number, for example, <code>42</code>, our hash
table would work, but its performance would severely degrade. So, for performance reasons it is
desirable to get distinct numbers for distinct objects usually.{' '}
</p>
<p>
The transformation also needs to be completely deterministic, i.e. we need to always get the
same value for the same object. In other words, something like <code>random()</code> would not
work, because we would "forget" where we placed our objects and we wouldn't be able to locate
them during a search.
</p>
<p>
Functions that do this kind of transformation are called <em>hash functions</em>. Since it is
not required to preserve any order in the input domain, a typical hash function "mixes up" its
input domain, hence the name "hash".
</p>
<p>
In Python, there are built-in implementations of hash functions for many built-in types. They
are all available through a single interface: the function <code>hash()</code>. This function
can take any Python object as an input and call an appropriate implementation (if it exists).
</p>
<p> Here are some examples of hash function values for some of the built-in types. </p>
<HashExamples />
<p>
In the case of strings, <code>hash()</code> returns fairly unpredictable results, as it should.
However, for small integers <code className="text-nowrap">hash(x) == x</code>. This fact may
seem surprising to people familiar with hash functions, but it is a deliberate design decision
by Python core developers.
</p>
<p>
For "long" integers Python uses a different algorithm. Try typing a relatively big number, for
example, <code>12345678901234567890</code> to see this.
</p>
<p>
Another fact: <code>hash()</code> never returns <code>-1</code>, because <code>-1</code> used
internally as an indicator of an error. That's why <code>hash(-1)</code> is <code>-2</code>.
</p>
<h5>hash() implementation notes</h5>
<p>
This chapter and the next two chapters will use <code>hash()</code> implementation from Python
3.2 (running on an x86_64 system). So if you run Python 3.2 on your x86_64 system, you should
see the same hash values for integers and strings (and the same data structure states). One
exception is <code>hash(None)</code>. It changes between runs, but in this explanation{' '}
<code>hash(None)</code> is always <code>-9223372036581563745</code>.
</p>
<p>
Why Python 3.2? Because dict implementation has changed over time but Python 3.2's dict
implements most major ideas, and, thus, Python 3.2's dict is a good starting point. Later
versions of Python extend (rather than completely replace) Python 3.2's implementation.
Eventually, we will get to these implementations as well.
</p>
<h5> Unhashable types </h5>
<p>
Not all types are hashable. For example, lists aren't and if you call{' '}
<code className="text-nowrap">hash(["some", "values"])</code> you will get{' '}
<code>TypeError: unhashable type: 'list'</code>. Why can't we use the same hash function for
lists as for tuples? The answer is because lists are mutable and tuples are not.
</p>
<p>
We could still define a hash function for lists and other mutable objects. However changing a
list would change the value of the hash function as well, and therefore we will not be able to
locate the mutated list! Hashing and using lists as keys in dicts would lead to many accidental
bugs, so core developers of Python chose not to allow this.
</p>
<h5>A note on the word "hash"</h5>
<p>
Because hash tables use hash functions and because hash tables mix up inserted elements, they
are called hash tables. Sometimes people shorten "hash table" to just "hash". The output of a
hash function is sometimes called "hash value" or "hash code", but very often it is also
shortened to just "hash". Also, Python's built-in hash function is called <code>hash()</code>.
</p>
<p>
Because people like to shorten things, several different (but related) concepts end up having
the same shortened name. This can get a bit confusing sometimes.
</p>
<h5> Using hash functions for hash tables </h5>
<p>
Recall that we started with a simple problem: searching efficiently in a list of distinct
numbers. Now, let's make this problem harder: our hash table needs to handle duplicates, support
types other than integers, support removing and adding keys (and therefore resizing). We will
see how to handle values in the next chapter, but for now let's assume we only need to search
for keys.
</p>
<p>How does using hash functions change the insertion algorithm?</p>
<p>
Obviously, we have to use <code>hash()</code> function to convert objects into integers for
indexing.
</p>
<p>
Because <code>None</code> is hashable too, we will need to use some other value as a placeholder
for an empty slot. The cleanest way to do this is to create a new type and use a value of this
type. In Python, this is quite simple:
</p>
<SimpleCodeBlock>
{`
class EmptyValueClass(object):
pass
EMPTY = EmptyValueClass()
`.trim()}
</SimpleCodeBlock>
<p>
We will now use <code>EMPTY</code> to denote an empty slot. After we do this, we will be able to
insert <code>None</code> in the hash table safely.
</p>
<p>
But here is one critical thing: checking for equality of objects can be expensive. For example,
comparing strings of length 10000 may require up to 10000 comparison operations - one per each
pair of corresponding characters. And, we may end up doing several equality checks when doing
linear probing.
</p>
<p>
When we only had integers, we didn't have this problem, because comparing integers is cheap. But
here is a neat trick we can use to improve the performance in the case of arbitrary objects. We
still get numbers from hash functions. So, we can save these numbers and compare them before
comparing actual objects. When comparing hashes, there are two different outcomes. First, the
hashes are different; in this case, we can safely conclude that the objects are different as
well. Second, the hashes are equal; in this case, there is a good chance that the objects are
equal but there is still a possibility of two distinct objects having the same hash, so we have
to compare the actual objects.
</p>
<p>
This optimization is a space-time tradeoff. We spend extra memory to make the algorithm faster.
</p>
<p>
In this chapter, we'll allow duplicates. Remember how search works in the previous chapter? We
retrace the steps necessary to insert the element and check if any slot along the way contains
it. We also do all these steps when we are actually inserting an element. This means that we're
effectively doing a search while inserting an element. So, handling duplicates is
straightforward — we can terminate the insertion process if we find the element. And if we
hit an empty slot without finding the element, then the element is not in the table and it means
that we can safely insert it.
</p>
<p>
Now, let's see this algorithm in action. We'll use a separate list called{' '}
<code>hash_codes</code> for caching values of hash functions.
</p>
<p>
Let's say we have <ListDescription array={this.state.array} />:
</p>
<BlockInputToolbar
input={PyListInput}
inputProps={{minSize: 1}}
initialValue={this.state.array}
onChange={this.setter('array', true)}
bottomBoundary=".chapter2"
{...this.props}
/>
<p>Let's build a hash table from it:</p>
<VisualizedCode
code={HASH_CREATE_NEW_CODE}
breakpoints={newRes.bp}
formatBpDesc={formatHashCreateNewAndInsert}
stateVisualization={HashCreateNewStateVisualization}
{...this.props}
/>
<h5> Searching </h5>
<p>
The search algorithm hasn't changed much. We get the <code>hash()</code> function value for the
object, and do linear probing. Just like in the first chapter, we essentially have a bunch of
"islands", separated by empty slots. Each island usually contains only a few elements, and a
linear scan over a small "island" isn't expensive.
</p>
<div className="div-p">
For instance, let's search for
<PySNNInput
inline={true}
value={this.state.searchedObj}
onChange={this.setter('searchedObj')}
anotherValue={() => anotherValue(this.state.array)}
/>
and see what happens:
</div>
<VisualizedCode
code={HASH_SEARCH_CODE}
breakpoints={searchRes.bp}
formatBpDesc={formatHashRemoveSearch}
stateVisualization={HashNormalStateVisualization}
{...this.props}
/>
<h5> Removing objects </h5>
<p>
If we removed a key without a trace, it'd leave a hole, and this would break the search
algorithm. Then, how do we remove a key?
</p>
<p>
The answer is that if we can't remove a key without a trace, we should leave a trace. When
removing a key, we replace it with a "dummy" object (another term for this object is
"tombstone"). This object acts as a placeholder that indicates we shouldn't stop probing during
a search.
</p>
<p>
In code, we can create this placeholder object just like we created <code>EMPTY</code>:
</p>
<SimpleCodeBlock>
{`
class DummyValueClass(object):
pass
DUMMY = DummyValueClass()
`.trim()}
</SimpleCodeBlock>
<div className="div-p">
Let's see removing in action. Let's say we want to remove:
<PySNNInput
inline={true}
value={this.state.objToRemove}
onChange={this.setter('objToRemove')}
anotherValue={() => anotherValue(this.state.array, 0.7, 0.2, 0.1)}
/>
</div>
<VisualizedCode
code={HASH_REMOVE_CODE}
breakpoints={removeRes.bp}
formatBpDesc={formatHashRemoveSearch}
stateVisualization={HashNormalStateVisualization}
{...this.props}
/>
<p>
Removing a lot of objects may lead to a table being filled with these dummy objects. What if a
table overflows with dummy objects? There is a way to clean them up. But first, let's see what
happens if a table overflows with ordinary objects.
</p>
<h5>Resizing hash tables</h5>
<p>
How do we resize a hash table? The index of each element depends on the table size, so it may
change if the size of a table changes. Moreover, because of collisions and linear probing, each
index may depend on the indexes of other objects (which, in turn, also depend on the size of a
table and the indexes of other objects). This is a tangled mess.
</p>
<p>
There is a way to disentangle this Gordian Knot, however. We can create a new, bigger table and
re-insert all the elements from the smaller table (skipping dummy placeholders). This may sound
expensive. And, it <em>is</em> expensive. But, the thing is, we don't have to resize the table
after every operation. If we make the new table size 1.5x, 2x or even 4x the size of the old
table, it'll take a while until it fills up again. We will do resizes rarely enough that the
heavy cost of it will amortize (spread out) over many insertions/deletions.
</p>
<HashResizeInPlaceAnimation breakpoints={resizeRes.bp} />
<p className="mt-2">
Since we're re-building the table, the code is fairly similar to the code for building it from
scratch. The differences are:
</p>
<ul>
<li>the hash codes are already there, so we don't need to compute them the second time;</li>
<li>
we don't have to check for duplicates, because we know that each object is present only once
in the original table;
</li>
<li>we skip empty and dummy slots.</li>
</ul>
<VisualizedCode
code={HASH_RESIZE_CODE}
breakpoints={resizeRes.bp}
formatBpDesc={formatHashResize}
stateVisualization={HashResizeStateVisualization}
{...this.props}
/>
<p>
There is still one more important question. What condition should trigger the resizing
operation? If we postpone resizing until a table is nearly full, the performance will severely
degrade. If we resize a table when it is still sparse, we will waste memory. Typically, a hash
table is resized when it is around 66% full.
</p>
<p>
The number of non-empty slots (including dummy/tombstone slots) is called <em>fill</em>. The
ratio between fill and table capacity is called the <em>load factor</em> (also, sometimes:{' '}
<em>fill factor</em> or <em>fill ratio</em>
). So, using the new terms, we can say that a hash table is resized when the load factor reaches
66%. When a table is resized, it's size usually goes up, but sometimes it can actually go down,
if there are a lot of dummy slots.
</p>
<p>
To implement this efficiently, we need to track the load factor. We will need two counters for
tracking fill and "real" usage. With the current code structure, tracking these counters would
be messy because we would need to pass these counters to and from every function. A much cleaner
solution would be using classes. More on that in the next chapter.
</p>
</Subcontainerize>
</div>
);
}
}
================================================
FILE: src/chapter3_and_4_common.js
================================================
import _ from 'lodash';
import * as React from 'react';
import {Map, List, Record} from 'immutable';
import {BigNumber} from 'bignumber.js';
import {
COLOR_FOR_READ_OPS,
randomMeaningfulString,
randomString3len,
randint,
randomChoice,
singularOrPlural,
fixFirstValues,
} from './util';
import {HashSlotsComponent, LineOfBoxesComponent, TetrisFactory, SimpleCodeBlock, VisualizedCode} from './code_blocks';
import {BreakpointFunction, HashBreakpointFunction, pyHash, DUMMY, EQ, displayStr} from './hash_impl_common';
import {commonFormatCheckCollisionLoopEndedPart, commonFormatCheckNotFound} from './common_formatters';
export const DEFAULT_STATE = {
pairs: [
['git', 'stash'],
['cp', 'a.py b.py'],
['ed', BigNumber(36)],
['uniq', '-c'],
['ls', '/'],
['su', BigNumber(0)],
['du', '-h'],
['cut', '-f 1'],
[BigNumber(-27), BigNumber(42)],
],
keyToDel: 'rm',
};
export function singleFormatCheckCollision(slots, idx, fmtCollisionCount) {
if (slots.get(idx).key == null) {
return commonFormatCheckCollisionLoopEndedPart(idx, fmtCollisionCount);
} else {
return `[Try #${fmtCollisionCount + 1}] Slot <code>${idx}</code> is occupied: a collision occurred`;
}
}
const _lookdictIsEmpty = (slots, idx) => slots.get(idx).key == null;
export function formatHashClassLookdictRelated(bp) {
switch (bp.point) {
case 'check-not-found':
return commonFormatCheckNotFound(bp.self.get('slots'), bp.idx, bp.fmtCollisionCount, _lookdictIsEmpty);
case 'check-hash': {
const slotHash = bp.self.get('slots').get(bp.idx).pyHashCode;
if (slotHash.eq(bp.hashCode)) {
return `<code>${slotHash} == ${bp.hashCode}</code>, so the slot might be occupied by the same key`;
} else {
return `<code>${slotHash} != ${bp.hashCode}</code>, so the slot definitely contains a different key`;
}
}
case 'check-key': {
const slotKey = bp.self.get('slots').get(bp.idx).key;
if (EQ(slotKey, bp.key)) {
return `<code>${displayStr(slotKey)} == ${displayStr(bp.key)}</code>, so the key is found`;
} else {
return `<code>${displayStr(slotKey)} != ${displayStr(
bp.key
)}</code>, so the slot contains a different key but with the same hash`;
}
}
case 'return-idx':
return `Return <code>${bp.idx}</code>, the index of the current slot`;
case 'raise':
return `Throw an exception, because no key was found`;
case 'call-lookdict':
return `Call <code>self.lookdict(${displayStr(bp.key)})</code>`;
/* __delitem__ */
case 'dec-used':
return `We're about to put the <code>DUMMY</code> placeholder in the slot, so set the counter of <code>used</code> slots to <code>${bp.self.get(
'used'
)}</code>`;
case 'replace-key-dummy':
return `Replace key at <code>${bp.idx}</code> with the <code>DUMMY</code> placeholder`;
case 'replace-value-empty':
return `Replace value at <code>${bp.idx}</code> with <code>EMPTY</code>`;
/* __getitem__ */
case 'return-value': {
const slotValue = bp.self.get('slots').get(bp.idx).value;
return `Return <code>${displayStr(slotValue)}</code>, value from slot <code>${bp.idx}</code>`;
}
/* misc common */
case 'start-execution-lookdict':
case 'start-execution-getitem':
case 'start-execution-delitem':
return '';
}
}
export function formatHashClassSetItemAndCreate(bp) {
switch (bp.point) {
case 'target-idx-none':
return `Initialize <code>target_idx</code> - this is the index of the slot where we'll put the item`;
case 'check-collision':
return singleFormatCheckCollision(bp.self.get('slots'), bp.idx, bp.fmtCollisionCount);
case 'check-dup-hash': {
const slotHash = bp.self.get('slots').get(bp.idx).pyHashCode;
if (slotHash.eq(bp.hashCode)) {
return `<code>${slotHash} == ${bp.hashCode}</code>, we cannot rule out the slot being occupied by the same key`;
} else {
return `<code>${slotHash} != ${bp.hashCode}</code>, so there is a collision with a different key`;
}
}
case 'check-dup-key': {
const slotKey = bp.self.get('slots').get(bp.idx).key;
if (EQ(slotKey, bp.key)) {
return `<code>${displayStr(slotKey)} == ${displayStr(
bp.key
)}</code>, so the key is already present in the table`;
} else {
return `<code>${displayStr(slotKey)} != ${displayStr(bp.key)}</code>, so there is a collision`;
}
}
case 'check-should-recycle-target-idx':
if (bp.targetIdx !== null) {
return `<code>target_idx</code> isn't <code>None</code> — we've already found a <code>DUMMY</code> slot that we can recycle`;
} else {
return `<code>target_idx</code> is currently <code>None</code> - we are still looking for a <code>DUMMY</code> slot to recycle`;
}
break;
case 'check-should-recycle-dummy': {
const slotKey = bp.self.get('slots').get
gitextract__zrkfi9_/ ├── .babelrc ├── .gitignore ├── .prettierignore ├── LICENSE ├── README.md ├── package.json ├── patches/ │ ├── python32_debug.diff │ ├── smooth-scrollbar+8.3.1.patch │ └── subscribe-ui-event+2.0.4.patch ├── python_code/ │ ├── actual_dict_factory_test.py │ ├── build_autogenerated_chapter1_hash.py │ ├── build_autogenerated_chapter2.py │ ├── build_autogenerated_chapter3_chapter4.py │ ├── chapter1_linear_search_reimplementation_test.py │ ├── chapter4_probing_python_reimplementation_test.py │ ├── common.py │ ├── dict32_reimplementation_test_v2.py │ ├── dict_reimpl_common.py │ ├── dict_reimplementation.py │ ├── dictinfo.py │ ├── dictinfo32.py │ ├── dictinfo33.py │ ├── hash_chapter1_impl.py │ ├── hash_chapter1_reimpl_js.py │ ├── hash_chapter1_reimplementation_test.py │ ├── hash_chapter2_impl.py │ ├── hash_chapter2_impl_test.py │ ├── hash_chapter2_reimpl_js.py │ ├── hash_chapter2_reimplementation_test.py │ ├── hash_chapter3_class_impl.py │ ├── hash_chapter3_class_impl_test.py │ ├── interface_test.py │ ├── js_reimpl_common.py │ └── js_reimplementation_interface.py ├── scripts/ │ ├── extractPythonCode.js │ ├── pyReimplWrapper.js │ └── ssr.js ├── src/ │ ├── app.js │ ├── autogenerated/ │ │ ├── chapter1.html │ │ ├── chapter2.html │ │ ├── chapter3.html │ │ └── chapter4.html │ ├── chapter1_simplified_hash.js │ ├── chapter2_hash_table_functions.js │ ├── chapter3_and_4_common.js │ ├── chapter3_hash_class.js │ ├── chapter4_real_python_dict.js │ ├── code_blocks.js │ ├── common_formatters.js │ ├── hash_impl_common.js │ ├── hash_impl_common.test.js │ ├── index.js │ ├── inputs.js │ ├── mustache/ │ │ ├── chapter1.json │ │ ├── chapter2.json │ │ ├── chapter3.json │ │ └── chapter4.json │ ├── page.html.template │ ├── probing_visualization.js │ ├── py_obj_parsing.js │ ├── py_obj_parsing.test.js │ ├── store.js │ ├── styles.css │ └── util.js ├── ssr-all.sh ├── stress_test_python.sh ├── unittest_python.sh ├── webpack.common.js ├── webpack.dev.js └── webpack.prod.js
SYMBOL INDEX (595 symbols across 39 files)
FILE: python_code/actual_dict_factory_test.py
function table_size (line 7) | def table_size(d):
class TestDictFactory (line 11) | class TestDictFactory(unittest.TestCase):
method test_dict_factory (line 12) | def test_dict_factory(self):
function main (line 24) | def main():
FILE: python_code/build_autogenerated_chapter1_hash.py
function create_new (line 7) | def create_new(numbers):
function create_new_broken (line 11) | def create_new_broken(numbers):
function has_key (line 15) | def has_key(keys, key):
function linear_search (line 19) | def linear_search(numbers, number):
FILE: python_code/chapter1_linear_search_reimplementation_test.py
function run (line 15) | def run(test_implementation, size):
FILE: python_code/chapter4_probing_python_reimplementation_test.py
function probe_all_js (line 9) | def probe_all_js(key, slots_count):
function probe_all (line 27) | def probe_all(key, slots_count=8):
function test (line 45) | def test():
FILE: python_code/common.py
class EmptyValueClass (line 5) | class EmptyValueClass(object):
method __str__ (line 6) | def __str__(self):
method __repr__ (line 9) | def __repr__(self):
class DummyValueClass (line 13) | class DummyValueClass(object):
method __str__ (line 14) | def __str__(self):
method __repr__ (line 17) | def __repr__(self):
function get_object_field_or_null (line 25) | def get_object_field_or_null(obj, field_name):
function get_object_field_or_none (line 32) | def get_object_field_or_none(obj, field_name):
function generate_random_string (line 39) | def generate_random_string(str_len=5):
function generate_random_unicode (line 47) | def generate_random_unicode(str_len):
class IntKeyValueFactory (line 52) | class IntKeyValueFactory(object):
method __init__ (line 53) | def __init__(self, n_inserts):
method generate_key (line 58) | def generate_key(self):
method generate_value (line 61) | def generate_value(self):
class AllKeyValueFactory (line 66) | class AllKeyValueFactory(object):
method __init__ (line 67) | def __init__(self, n_inserts, int_chance=0.1, long_chance=0.1, len0_ch...
method _generate_obj (line 80) | def _generate_obj(self):
method generate_key (line 100) | def generate_key(self):
method generate_value (line 103) | def generate_value(self):
FILE: python_code/dict32_reimplementation_test_v2.py
function dict_factory (line 13) | def dict_factory(pairs=None):
function verify_same (line 42) | def verify_same(d, dump_d_func, dreimpl, dump_dreimpl_func):
function run (line 65) | def run(ref_impl_factory, ref_impl_dump, test_impl_factory, test_impl_du...
function test_iteration (line 169) | def test_iteration():
FILE: python_code/dict_reimpl_common.py
class Slot (line 4) | class Slot(object):
method __init__ (line 5) | def __init__(self, hash_code=EMPTY, key=EMPTY, value=EMPTY):
class BaseDictImpl (line 11) | class BaseDictImpl(object):
method __init__ (line 12) | def __init__(self):
method find_nearest_size (line 17) | def find_nearest_size(self, minused):
FILE: python_code/dict_reimplementation.py
class PyDictReimplementationBase (line 6) | class PyDictReimplementationBase(BaseDictImpl):
method __init__ (line 10) | def __init__(self, pairs=None):
method __setitem__ (line 18) | def __setitem__(self, key, value):
method __delitem__ (line 46) | def __delitem__(self, key):
method __getitem__ (line 53) | def __getitem__(self, key):
method signed_to_unsigned (line 59) | def signed_to_unsigned(hash_code):
method lookdict (line 62) | def lookdict(self, key):
method resize (line 76) | def resize(self):
class PyDictReimplementation32 (line 92) | class PyDictReimplementation32(PyDictReimplementationBase):
method _next_size (line 93) | def _next_size(self):
function dump_reimpl_dict (line 97) | def dump_reimpl_dict(d):
FILE: python_code/dictinfo.py
function dump_py_dict (line 4) | def dump_py_dict(d):
FILE: python_code/dictinfo32.py
class PyDictEntry (line 5) | class PyDictEntry(Structure):
class PyDictObject (line 13) | class PyDictObject(Structure):
function dictobject (line 24) | def dictobject(d):
function dump_py_dict (line 34) | def dump_py_dict(d):
FILE: python_code/dictinfo33.py
class PyDictKeyEntry (line 5) | class PyDictKeyEntry(Structure):
class PyDictKeysObject (line 13) | class PyDictKeysObject(Structure):
class PyDictObject (line 23) | class PyDictObject(Structure):
function dictobject (line 35) | def dictobject(d):
function usable_fraction (line 45) | def usable_fraction(size):
function dump_py_dict (line 49) | def dump_py_dict(d):
FILE: python_code/hash_chapter1_impl.py
function create_new (line 1) | def create_new(numbers):
function create_new_broken (line 16) | def create_new_broken(numbers):
function has_key (line 27) | def has_key(keys, key):
function linear_search (line 37) | def linear_search(numbers, number):
FILE: python_code/hash_chapter1_reimpl_js.py
function run_op (line 4) | def run_op(keys, op, **kwargs):
function create_new (line 8) | def create_new(numbers):
function create_new_broken (line 12) | def create_new_broken(numbers):
function has_key (line 16) | def has_key(keys, key):
function linear_search (line 20) | def linear_search(numbers, key):
FILE: python_code/hash_chapter1_reimplementation_test.py
function get_implementation (line 9) | def get_implementation(is_broken, impl):
function run (line 22) | def run(test_implementation, is_broken, n_inserts):
FILE: python_code/hash_chapter2_impl.py
function create_new (line 4) | def create_new(from_keys):
function insert (line 24) | def insert(hash_codes, keys, key):
function remove (line 37) | def remove(hash_codes, keys, key):
function has_key (line 50) | def has_key(hash_codes, keys, key):
function resize (line 60) | def resize(hash_codes, keys):
FILE: python_code/hash_chapter2_impl_test.py
class MyHashTest (line 6) | class MyHashTest(unittest.TestCase):
method test_handcrafted (line 7) | def test_handcrafted(self):
method test_all (line 100) | def test_all(self):
function main (line 138) | def main():
FILE: python_code/hash_chapter2_reimpl_js.py
function run_op (line 4) | def run_op(hash_codes, keys, op, **kwargs):
function create_new (line 8) | def create_new(from_keys):
function insert (line 12) | def insert(hash_codes, keys, key):
function remove (line 18) | def remove(hash_codes, keys, key):
function has_key (line 24) | def has_key(hash_codes, keys, key):
function resize (line 28) | def resize(hash_codes, keys):
FILE: python_code/hash_chapter2_reimplementation_test.py
function verify_same (line 16) | def verify_same(ref_hash_codes, ref_keys, hash_codes, keys):
function run (line 32) | def run(ref_impl, test_impl, n_inserts, key_value_factory, initial_state...
function test_iteration (line 107) | def test_iteration():
FILE: python_code/hash_chapter3_class_impl.py
class AlmostPythonDictBase (line 5) | class AlmostPythonDictBase(BaseDictImpl):
method __init__ (line 8) | def __init__(self, pairs=None):
method lookdict (line 15) | def lookdict(self, key):
method __getitem__ (line 27) | def __getitem__(self, key):
method __delitem__ (line 32) | def __delitem__(self, key):
method resize (line 40) | def resize(self):
method keys (line 55) | def keys(self):
method __len__ (line 58) | def __len__(self):
class AlmostPythonDictImplementationRecycling (line 62) | class AlmostPythonDictImplementationRecycling(AlmostPythonDictBase):
method __setitem__ (line 63) | def __setitem__(self, key, value):
class AlmostPythonDictImplementationNoRecycling (line 93) | class AlmostPythonDictImplementationNoRecycling(AlmostPythonDictBase):
method __setitem__ (line 94) | def __setitem__(self, key, value):
class AlmostPythonDictImplementationNoRecyclingSimplerVersion (line 118) | class AlmostPythonDictImplementationNoRecyclingSimplerVersion(AlmostPyth...
method __setitem__ (line 119) | def __setitem__(self, key, value):
FILE: python_code/hash_chapter3_class_impl_test.py
class HashDictImplementationTest (line 6) | class HashDictImplementationTest(unittest.TestCase):
method test_handcrafted (line 7) | def test_handcrafted(self):
method test_handcrafted_simple_setitem (line 77) | def test_handcrafted_simple_setitem(self):
function main (line 148) | def main():
FILE: python_code/interface_test.py
class Interface (line 7) | class Interface(unittest.TestCase):
method test_all (line 8) | def test_all(self):
method do_simple_test_single_class (line 17) | def do_simple_test_single_class(self, klass):
function main (line 34) | def main():
FILE: python_code/js_reimpl_common.py
function dump_simple_py_obj (line 12) | def dump_simple_py_obj(obj):
function dump_pairs (line 29) | def dump_pairs(pairs):
function dump_array (line 37) | def dump_array(array):
function parse_array (line 41) | def parse_array(array):
function parse_simple_py_obj (line 45) | def parse_simple_py_obj(obj):
function _init_sock_stuff (line 62) | def _init_sock_stuff():
function run_op_chapter1_chapter2 (line 77) | def run_op_chapter1_chapter2(chapter, hash_codes, keys, op, **kwargs):
FILE: python_code/js_reimplementation_interface.py
class JsImplBase (line 9) | class JsImplBase(object):
method __init__ (line 13) | def __init__(self, pairs=None):
method __del__ (line 26) | def __del__(self):
method dump_slots (line 29) | def dump_slots(self):
method restore_slots (line 49) | def restore_slots(self, slots):
method run_op (line 63) | def run_op(self, op, **kwargs):
class Dict32JsImpl (line 95) | class Dict32JsImpl(JsImplBase):
method __setitem__ (line 98) | def __setitem__(self, key, value):
method __delitem__ (line 101) | def __delitem__(self, key):
method __getitem__ (line 104) | def __getitem__(self, key):
class AlmostPythonDictBaseJsImpl (line 108) | class AlmostPythonDictBaseJsImpl(JsImplBase):
method __delitem__ (line 111) | def __delitem__(self, key):
method __getitem__ (line 114) | def __getitem__(self, key):
class AlmostPythonDictRecyclingJsImpl (line 118) | class AlmostPythonDictRecyclingJsImpl(AlmostPythonDictBaseJsImpl):
method __setitem__ (line 119) | def __setitem__(self, key, value):
class AlmostPythonDictNoRecyclingJsImpl (line 123) | class AlmostPythonDictNoRecyclingJsImpl(AlmostPythonDictBaseJsImpl):
method __setitem__ (line 124) | def __setitem__(self, key, value):
FILE: scripts/extractPythonCode.js
function extractCodeLines (line 44) | function extractCodeLines(codeWithBpAndLevels) {
function outputCode (line 48) | function outputCode(filename, headers, importedCode, indent4 = true) {
constant DIR (line 77) | const DIR = 'build';
FILE: scripts/pyReimplWrapper.js
function parseSimplePyObj (line 15) | function parseSimplePyObj(obj) {
function parseArray (line 34) | function parseArray(array) {
function dumpArray (line 38) | function dumpArray(array) {
function parsePairs (line 42) | function parsePairs(pairs) {
function dumpSimplePyObj (line 46) | function dumpSimplePyObj(obj) {
function restorePyDictState (line 65) | function restorePyDictState(state) {
function dumpPyDictState (line 92) | function dumpPyDictState(pySelf) {
function dict32RunOp (line 108) | function dict32RunOp(pySelf, op, key, value, pairs) {
function almostPyDictRunOp (line 131) | function almostPyDictRunOp(pySelf, op, key, value, pairs) {
function chapter1run (line 158) | function chapter1run(keys, op, key, numbers) {
function chapter2run (line 180) | function chapter2run(hashCodes, keys, op, key, array) {
FILE: src/app.js
function getWindowDimensions (line 38) | function getWindowDimensions() {
function logViewportStats (line 44) | function logViewportStats() {
constant GITHUB_REPO_URL (line 63) | const GITHUB_REPO_URL = 'https://github.com/eleweek/inside_python_dict';
constant MAILCHIMP_URL (line 64) | const MAILCHIMP_URL = 'http://eepurl.com/gbzhvn';
constant TWITTER_LINK (line 65) | const TWITTER_LINK = 'https://twitter.com/SashaPutilin';
constant EMAIL (line 66) | const EMAIL = 'avp-13@yandex.ru';
function GithubRibbon (line 68) | function GithubRibbon() {
function GithubCorner (line 80) | function GithubCorner() {
function GithubForkMe (line 91) | function GithubForkMe({windowWidth}) {
constant CONTENTS_DATA (line 99) | const CONTENTS_DATA = [
function chapterIdDotHtml (line 106) | function chapterIdDotHtml(chapterId) {
function NextPrev (line 114) | function NextPrev({selectedChapterId}) {
class Contents (line 152) | class Contents extends React.PureComponent {
method render (line 155) | render() {
class LoadingAlert (line 213) | class LoadingAlert extends React.PureComponent {
method constructor (line 214) | constructor() {
method render (line 222) | render() {
method componentDidMount (line 237) | componentDidMount() {
class Alerts (line 242) | class Alerts extends React.Component {
method constructor (line 243) | constructor() {
method render (line 251) | render() {
method componentDidMount (line 288) | componentDidMount() {
function Footer (line 293) | function Footer() {
constant SIGNIFICANT_HEIGHT_CHANGE (line 330) | const SIGNIFICANT_HEIGHT_CHANGE = 100;
class App (line 331) | class App extends React.Component {
method constructor (line 332) | constructor() {
method componentDidMount (line 366) | componentDidMount() {
method componentWillUnmount (line 394) | componentWillUnmount() {
method render (line 398) | render() {
function fixStickyResize (line 436) | function fixStickyResize(windowWidth, windowHeight) {
function fixSticky (line 445) | function fixSticky() {
function initAndRender (line 455) | function initAndRender(chapters, chapterIds) {
FILE: src/chapter1_simplified_hash.js
constant CHAPTER1_MAXNUM (line 35) | const CHAPTER1_MAXNUM = 999;
constant SIMPLE_LIST_SEARCH (line 37) | const SIMPLE_LIST_SEARCH = [
function _parseSmallInt (line 47) | function _parseSmallInt(value) {
function chapter1valueRangeValidator (line 57) | function chapter1valueRangeValidator(num) {
function PySmallIntInput (line 66) | function PySmallIntInput({inputComponentRef, ...restProps}) {
function simpleListSearch (line 72) | function simpleListSearch(l, key) {
function UnnamedListVisualization (line 141) | function UnnamedListVisualization(props) {
constant SIMPLIFIED_INSERT_ALL_BROKEN_CODE (line 146) | const SIMPLIFIED_INSERT_ALL_BROKEN_CODE = [
constant SIMPLIFIED_INSERT_ALL_CODE (line 156) | const SIMPLIFIED_INSERT_ALL_CODE = [
class SimplifiedInsertAll (line 168) | class SimplifiedInsertAll extends BreakpointFunction {
method constructor (line 169) | constructor() {
method run (line 175) | run(_originalList, isBroken = false) {
method overwrittenNumbers (line 234) | overwrittenNumbers() {
constant SIMPLIFIED_SEARCH_CODE (line 306) | const SIMPLIFIED_SEARCH_CODE = [
class SimplifiedSearch (line 320) | class SimplifiedSearch extends BreakpointFunction {
method run (line 321) | run(_newList, _number) {
function DynamicSimplifiedInsertAllBrokenOverwrittenExample (line 383) | function DynamicSimplifiedInsertAllBrokenOverwrittenExample({originalNum...
class Ops (line 423) | class Ops {
method createNew (line 424) | static createNew(numbers) {
method createNewBroken (line 431) | static createNewBroken(numbers) {
method hasKey (line 437) | static hasKey(keys, number) {
method linearSearch (line 444) | static linearSearch(numbers, searchedNumber) {
function anotherValue (line 449) | function anotherValue(array, RANDOM_NUMBER_CHANCE = 0.3) {
class Chapter1_SimplifiedHash (line 457) | class Chapter1_SimplifiedHash extends ChapterComponent {
method constructor (line 458) | constructor() {
method render (line 518) | render() {
FILE: src/chapter2_hash_table_functions.js
constant HASH_CREATE_NEW_CODE (line 34) | const HASH_CREATE_NEW_CODE = [
function anotherValue (line 53) | function anotherValue(array, ARRAY_CHANCE = 0.5, MEANINGFUL_CHANCE = 0.2...
class HashCreateNew (line 67) | class HashCreateNew extends HashBreakpointFunction {
method run (line 68) | run(_fromKeys) {
function formatHashCreateNewAndInsert (line 142) | function formatHashCreateNewAndInsert(bp, prevBp) {
constant HASH_SEARCH_CODE (line 199) | const HASH_SEARCH_CODE = [
function formatHashRemoveSearch (line 211) | function formatHashRemoveSearch(bp, prevBp) {
constant HASH_REMOVE_CODE (line 266) | const HASH_REMOVE_CODE = [
class HashRemoveOrSearch (line 281) | class HashRemoveOrSearch extends HashBreakpointFunction {
method run (line 282) | run(_hashCodes, _keys, _key, isRemoveMode) {
constant HASH_RESIZE_CODE (line 337) | const HASH_RESIZE_CODE = [
class HashResize (line 352) | class HashResize extends HashBreakpointFunction {
method run (line 353) | run(_hashCodes, _keys) {
function formatHashResize (line 402) | function formatHashResize(bp, prevBp) {
constant HASH_INSERT_CODE (line 456) | const HASH_INSERT_CODE = [
class HashInsert (line 470) | class HashInsert extends HashBreakpointFunction {
method run (line 471) | run(_hashCodes, _keys, _key) {
class HashExamples (line 508) | class HashExamples extends React.Component {
method constructor (line 509) | constructor(props) {
method render (line 517) | render() {
class Ops (line 563) | class Ops {
method createNew (line 564) | static createNew(array) {
method hasKey (line 571) | static hasKey(hashCodes, keys, searchedObj) {
method remove (line 579) | static remove(hashCodes, keys, objToRemove) {
method resize (line 587) | static resize(hashCodes, keys) {
method insert (line 595) | static insert(hashCodes, keys, objToInsert) {
function ListDescription (line 604) | function ListDescription({array}) {
class HashResizeInPlaceAnimation (line 616) | class HashResizeInPlaceAnimation extends React.PureComponent {
method constructor (line 620) | constructor() {
method getDerivedStateFromProps (line 628) | static getDerivedStateFromProps(props, state) {
method render (line 647) | render() {
class Chapter2_HashTableFunctions (line 679) | class Chapter2_HashTableFunctions extends ChapterComponent {
method constructor (line 680) | constructor() {
method render (line 714) | render() {
FILE: src/chapter3_and_4_common.js
constant DEFAULT_STATE (line 19) | const DEFAULT_STATE = {
function singleFormatCheckCollision (line 34) | function singleFormatCheckCollision(slots, idx, fmtCollisionCount) {
function formatHashClassLookdictRelated (line 43) | function formatHashClassLookdictRelated(bp) {
function formatHashClassSetItemAndCreate (line 93) | function formatHashClassSetItemAndCreate(bp) {
function formatHashClassResize (line 208) | function formatHashClassResize(bp) {
function formatHashClassInit (line 253) | function formatHashClassInit(bp) {
function hashClassConstructor (line 285) | function hashClassConstructor(size = 8) {
class HashClassInitEmpty (line 300) | class HashClassInitEmpty extends BreakpointFunction {
method run (line 301) | run(initStartSize = null, pairsLength = null) {
function findClosestSize (line 341) | function findClosestSize(minused) {
function findClosestSizeBN (line 350) | function findClosestSizeBN(minused) {
class HashClassSetItemBase (line 359) | class HashClassSetItemBase extends HashBreakpointFunction {
method run (line 360) | run(_self, _key, _value, useRecycling, Resize, optimalSizeQuot) {
method getResize (line 464) | getResize() {
class HashClassLookdictBase (line 469) | class HashClassLookdictBase extends HashBreakpointFunction {
method run (line 470) | run(_self, _key) {
class HashClassGetItem (line 509) | class HashClassGetItem extends HashBreakpointFunction {
method run (line 510) | run(_self, _key, Lookdict) {
class HashClassDelItem (line 527) | class HashClassDelItem extends HashBreakpointFunction {
method run (line 528) | run(_self, _key, Lookdict) {
class HashClassInsertAll (line 550) | class HashClassInsertAll extends HashBreakpointFunction {
method constructor (line 551) | constructor() {
method run (line 557) | run(_self, _pairs, useRecycling, SetItem, Resize, optimalSizeQuot) {
method getResizes (line 578) | getResizes() {
class HashClassResizeBase (line 618) | class HashClassResizeBase extends HashBreakpointFunction {
method run (line 619) | run(_self, optimalSizeQuot) {
function anotherKey (line 676) | function anotherKey(pairs, ARRAY_CHANCE = 0.5, MEANINGFUL_CHANCE = 0.25,...
function generateNonPresentKey (line 705) | function generateNonPresentKey(pySelf, getitem) {
function addPairsUntilResize (line 714) | function addPairsUntilResize(pySelf, getitem, setitem) {
function selectOrCreateResize (line 734) | function selectOrCreateResize(pySelf, resizes, getitem, setitem) {
FILE: src/chapter3_hash_class.js
method computeIdxAndSave (line 40) | computeIdxAndSave(hashCode, len) {
method nextIdxAndSave (line 45) | nextIdxAndSave() {
class HashClassSetItem (line 51) | class HashClassSetItem extends chapter3Extend(HashClassSetItemBase) {}
class HashClassLookdict (line 52) | class HashClassLookdict extends chapter3Extend(HashClassLookdictBase) {}
class HashClassResize (line 53) | class HashClassResize extends chapter3Extend(HashClassResizeBase) {}
class AlmostPythonDict (line 55) | class AlmostPythonDict {
method __init__ (line 56) | static __init__(pairs) {
method __delitem__ (line 74) | static __delitem__(pySelf, key) {
method __getitem__ (line 83) | static __getitem__(pySelf, key) {
method __setitem__base (line 92) | static __setitem__base(pySelf, key, value, isRecycling) {
method __setitem__recycling (line 100) | static __setitem__recycling(pySelf, key, value) {
method __setitem__no_recycling (line 104) | static __setitem__no_recycling(pySelf, key, value) {
function formatHashClassChapter3IdxRelatedBp (line 109) | function formatHashClassChapter3IdxRelatedBp(bp, prevBp) {
constant HASH_CLASS_SETITEM_SIMPLIFIED_CODE (line 124) | const HASH_CLASS_SETITEM_SIMPLIFIED_CODE = [
constant HASH_CLASS_INIT_CODE (line 144) | const HASH_CLASS_INIT_CODE = [
constant HASH_CLASS_SETITEM_SIMPLIFIED_WITH_INIT_CODE (line 155) | const HASH_CLASS_SETITEM_SIMPLIFIED_WITH_INIT_CODE = [...HASH_CLASS_INIT...
constant HASH_CLASS_SETITEM_RECYCLING_CODE (line 157) | const HASH_CLASS_SETITEM_RECYCLING_CODE = [
constant HASH_CLASS_RESIZE_CODE (line 186) | const HASH_CLASS_RESIZE_CODE = [
constant HASH_CLASS_LOOKDICT (line 202) | const HASH_CLASS_LOOKDICT = [
constant HASH_CLASS_GETITEM (line 224) | const HASH_CLASS_GETITEM = [...HASH_CLASS_LOOKDICT, ..._HASH_CLASS_GETIT...
constant HASH_CLASS_DELITEM (line 235) | const HASH_CLASS_DELITEM = [...HASH_CLASS_LOOKDICT, ..._HASH_CLASS_DELIT...
constant FIND_NEAREST_SIZE_CODE_STRING (line 237) | const FIND_NEAREST_SIZE_CODE_STRING = `def find_closest_size(self, minus...
constant SLOT_CLASS_CODE_STRING (line 244) | const SLOT_CLASS_CODE_STRING = `class Slot(object):
function DynamicPartResize (line 251) | function DynamicPartResize({extraPairs, resize, pairsCount, resizesCount...
function DynamicPartSetItemRecycling (line 277) | function DynamicPartSetItemRecycling({hasDummy, outcome, otherOutcomes, ...
class FindClosestSizeExample (line 352) | class FindClosestSizeExample extends React.PureComponent {
method constructor (line 354) | constructor() {
method render (line 373) | render() {
class Chapter3_HashClass (line 388) | class Chapter3_HashClass extends ChapterComponent {
method constructor (line 389) | constructor() {
method render (line 565) | render() {
FILE: src/chapter4_real_python_dict.js
method computeIdxAndSave (line 39) | computeIdxAndSave(hashCode, len) {
method nextIdxAndSave (line 46) | nextIdxAndSave() {
class SideBySideDicts (line 59) | class SideBySideDicts extends React.Component {
method render (line 63) | render() {
class Dict32SetItem (line 86) | class Dict32SetItem extends chapter4Extend(HashClassSetItemBase) {}
class Dict32Lookdict (line 87) | class Dict32Lookdict extends chapter4Extend(HashClassLookdictBase) {}
class Dict32Resize (line 88) | class Dict32Resize extends chapter4Extend(HashClassResizeBase) {
function formatDict32IdxRelatedBp (line 93) | function formatDict32IdxRelatedBp(bp, prevBp) {
function formatPythonProbing (line 112) | function formatPythonProbing(bp, prevBp) {
constant STATICMETHOD_SIGNED_TO_UNSIGNED (line 160) | const STATICMETHOD_SIGNED_TO_UNSIGNED = [
constant DICT32_INIT (line 170) | const DICT32_INIT = [
constant DICT32_SETITEM (line 185) | const DICT32_SETITEM = [
constant DICT32_SETITEM_WITH_INIT (line 218) | const DICT32_SETITEM_WITH_INIT = [...STATICMETHOD_SIGNED_TO_UNSIGNED, .....
constant DICT32_RESIZE_CODE (line 220) | const DICT32_RESIZE_CODE = [
constant DICT32_LOOKDICT (line 239) | const DICT32_LOOKDICT = [
constant DICT32_GETITEM (line 263) | const DICT32_GETITEM = [...DICT32_LOOKDICT, ..._DICT32_GETITEM_ONLY];
constant DICT32_DELITEM (line 274) | const DICT32_DELITEM = [...DICT32_LOOKDICT, ..._DICT32_DELITEM_ONLY];
class Dict32 (line 276) | class Dict32 {
method __init__ (line 277) | static __init__(pairs) {
method __delitem__ (line 315) | static __delitem__(pySelf, key) {
method __getitem__ (line 324) | static __getitem__(pySelf, key) {
method __setitem__ (line 333) | static __setitem__(pySelf, key, value) {
constant PROBING_PYTHON_CODE (line 352) | const PROBING_PYTHON_CODE = [
function DynamicPartResize (line 365) | function DynamicPartResize({extraPairs, resize}) {
class Chapter4_RealPythonDict (line 395) | class Chapter4_RealPythonDict extends ChapterComponent {
method constructor (line 396) | constructor() {
method render (line 461) | render() {
FILE: src/code_blocks.js
function reflow (line 52) | function reflow(node) {
function renderPythonCode (line 60) | function renderPythonCode(codeString) {
function dummyFormat (line 70) | function dummyFormat(bp) {
function SimpleCodeBlock (line 75) | function SimpleCodeBlock(props) {
function SimpleCodeInline (line 83) | function SimpleCodeInline(props) {
constant DEFAULT_BOX_GEOMETRY (line 87) | const DEFAULT_BOX_GEOMETRY = {
constant SMALLER_BOX_GEOMETRY (line 91) | const SMALLER_BOX_GEOMETRY = {
function computeBoxTransformProperty (line 96) | function computeBoxTransformProperty(idx, y, boxSize, spacingX) {
function pyObjToReactKey (line 101) | function pyObjToReactKey(obj) {
class ActiveBoxSelectionUnthrottled (line 105) | class ActiveBoxSelectionUnthrottled extends React.PureComponent {
method render (line 106) | render() {
class ActiveBoxSelectionThrottledHelper (line 139) | class ActiveBoxSelectionThrottledHelper extends React.Component {
method render (line 144) | render() {
method componentDidUpdate (line 166) | componentDidUpdate() {
function SingleBoxSelection (line 176) | function SingleBoxSelection({idx, status, boxSize, spacingX, spacingY, b...
class SelectionGroup (line 190) | class SelectionGroup extends React.Component {
method constructor (line 193) | constructor() {
method render (line 202) | render() {
method componentDidUpdate (line 233) | componentDidUpdate() {
method handleStartingTransition (line 244) | handleStartingTransition() {
method getDerivedStateFromProps (line 259) | static getDerivedStateFromProps(props, state) {
class Box (line 281) | class Box extends React.PureComponent {
method shortDisplayedString (line 282) | shortDisplayedString(value) {
method render (line 321) | render() {
class SlotSelection (line 414) | class SlotSelection extends React.PureComponent {
method render (line 415) | render() {
class LineOfBoxesSelection (line 448) | class LineOfBoxesSelection extends React.PureComponent {
method render (line 449) | render() {
class BaseBoxesComponent (line 488) | class BaseBoxesComponent extends React.PureComponent {
method constructor (line 493) | constructor() {
method markRemoved (line 515) | static markRemoved(state, targetModId) {
method getDerivedStateFromProps (line 546) | static getDerivedStateFromProps(nextProps, state) {
method notSoDeepDel (line 969) | static notSoDeepDel(m, [e1, e2, e3]) {
method garbageCollect (line 982) | static garbageCollect(state, targetModId) {
method debugLogState (line 1024) | debugLogState() {
method staticDebugLogState (line 1028) | static staticDebugLogState(state) {
method render (line 1053) | render() {
method componentDidUpdate (line 1100) | componentDidUpdate() {
function isImmutableListOrMap (line 1146) | function isImmutableListOrMap(obj) {
function deepGet (line 1150) | function deepGet(obj, path) {
function TetrisFactory (line 1164) | function TetrisFactory(lines, opts) {
function selectGeometry (line 1183) | function selectGeometry(windowWidth, windowHeight) {
class Tetris (line 1191) | class Tetris extends React.PureComponent {
method _getExpectedHeight (line 1194) | static _getExpectedHeight(windowWidth, windowHeight, lines, fixedGeome...
method constructor (line 1208) | constructor() {
method render (line 1221) | render() {
method componentDidUpdate (line 1304) | componentDidUpdate() {
method componentDidMount (line 1308) | componentDidMount() {
class CodeBlockWithActiveLineAndAnnotations (line 1313) | class CodeBlockWithActiveLineAndAnnotations extends React.Component {
method constructor (line 1314) | constructor() {
method getCodeWithExplanationHtmlLines (line 1332) | getCodeWithExplanationHtmlLines(visibleBreakpoints, activeBp) {
method getVisibleBreakpoints (line 1402) | getVisibleBreakpoints(activeBp) {
method render (line 1439) | render() {
method getScrollTopTarget (line 1465) | getScrollTopTarget() {
method componentDidUpdate (line 1512) | componentDidUpdate() {
class TimeSliderWithControls (line 1518) | @observer
method constructor (line 1522) | constructor() {
method unixtimestamp (line 1529) | unixtimestamp() {
method getDerivedStateFromProps (line 1533) | static getDerivedStateFromProps(nextProps, state) {
method getDerivedStateFromProps (line 1629) | static getDerivedStateFromProps(props, state) {
method componentDidUpdate (line 1637) | componentDidUpdate() {
method componentDidMount (line 1647) | componentDidMount() {
method render (line 1653) | render() {
class VisualizedCode (line 1744) | class VisualizedCode extends React.Component {
method constructor (line 1748) | constructor(props) {
method getDerivedStateFromProps (line 1774) | static getDerivedStateFromProps(props, state) {
method render (line 1795) | render() {
class HashBoxesComponent (line 1879) | class HashBoxesComponent extends React.PureComponent {
method getExpectedGeometry (line 1880) | static getExpectedGeometry({boxGeometry: {boxSize, spacingY}}) {
method getKeys (line 1884) | static getKeys(array) {
method boxFactory (line 1901) | static boxFactory(keys, value) {
method render (line 1905) | render() {
class HashBoxesBrokenComponent (line 1919) | class HashBoxesBrokenComponent extends React.PureComponent {
method height (line 1922) | static height(boxSize) {
method getExpectedGeometry (line 1926) | static getExpectedGeometry({boxGeometry: {boxSize, spacingY}}) {
method getKeys (line 1935) | static getKeys(array) {
method boxFactory (line 1945) | static boxFactory(keys, value) {
method render (line 1964) | render() {
class HashSlotsComponent (line 1979) | class HashSlotsComponent extends React.PureComponent {
method getExpectedGeometry (line 1980) | static getExpectedGeometry({boxGeometry: {boxSize, spacingY}}) {
method boxFactory (line 1984) | static boxFactory(keys, value) {
method getKeys (line 1994) | static getKeys(array) {
method render (line 2013) | render() {
class CounterList (line 2027) | class CounterList {
method constructor (line 2028) | constructor() {
method inc (line 2032) | inc(group, value) {
class LineOfBoxesComponent (line 2054) | class LineOfBoxesComponent extends React.PureComponent {
method getKeys (line 2055) | static getKeys(array) {
method getKeysForKVPairs (line 2087) | static getKeysForKVPairs(array) {
method boxFactory (line 2103) | static boxFactory(keys, values) {
method getExpectedGeometry (line 2113) | static getExpectedGeometry(props) {
method render (line 2124) | render() {
FILE: src/common_formatters.js
function commonFormatCheckCollisionLoopEndedPart (line 3) | function commonFormatCheckCollisionLoopEndedPart(idx, fmtCollisionCount) {
function chapter1_2_FormatCheckCollision (line 19) | function chapter1_2_FormatCheckCollision(l, idx, fmtCollisionCount) {
function commonFormatCheckNotFound (line 28) | function commonFormatCheckNotFound(l, idx, fmtCollisionCount, isEmpty = ...
FILE: src/hash_impl_common.js
function repr (line 3) | function repr(obj, allowNull = false) {
function displayStr (line 24) | function displayStr(obj, quoteString = true) {
class Int64 (line 40) | class Int64 {
method constructor (line 44) | constructor(jsNumInt32 = 0) {
method xor (line 60) | xor(other) {
method sign (line 68) | sign() {
method inc (line 72) | inc() {
method eq (line 77) | eq(other) {
method add (line 86) | add(other) {
method complement (line 97) | complement() {
method mul (line 105) | mul(other) {
method toNumber (line 128) | toNumber() {
method toStringDestroying (line 139) | toStringDestroying() {
method _carryOverAll (line 184) | _carryOverAll() {
function pyHashStringAndUnicode (line 194) | function pyHashStringAndUnicode(s) {
function pyHashString (line 211) | function pyHashString(s) {
function pyHashUnicode (line 216) | function pyHashUnicode(s) {
function pyHashLong (line 220) | function pyHashLong(num) {
function pyHash (line 264) | function pyHash(o) {
class BreakpointFunction (line 281) | class BreakpointFunction {
method constructor (line 282) | constructor() {
method addBP (line 287) | addBP(point) {
method setExtraBpContext (line 310) | setExtraBpContext(extraBpContext) {
method getBreakpoints (line 314) | getBreakpoints() {
function computeIdx (line 319) | function computeIdx(hashCodeBig, len) {
class HashBreakpointFunction (line 327) | class HashBreakpointFunction extends BreakpointFunction {
method computeIdx (line 328) | computeIdx(hashCodeBig, len) {
class DummyClass (line 333) | class DummyClass {
method toString (line 334) | toString() {
class NoneClass (line 339) | class NoneClass {
method toString (line 342) | toString() {
constant DUMMY (line 347) | const DUMMY = new DummyClass();
function isNone (line 350) | function isNone(o) {
function isDummy (line 354) | function isDummy(o) {
function EQ (line 358) | function EQ(o1, o2) {
function signedToUnsigned (line 368) | function signedToUnsigned(num) {
function computePerturb (line 376) | function computePerturb(hashCode) {
function nextIdxPerturb (line 380) | function nextIdxPerturb(idx, perturb, size) {
function perturbShift (line 387) | function perturbShift(perturb) {
FILE: src/index.js
constant CHAPTER_ID_TO_COMPONENT (line 7) | const CHAPTER_ID_TO_COMPONENT = {
FILE: src/inputs.js
class ParsableInputBase (line 32) | class ParsableInputBase extends React.Component {
method constructor (line 33) | constructor(props) {
method forceSetValue (line 47) | forceSetValue(value) {
class ParsableInputBlock (line 77) | class ParsableInputBlock extends ParsableInputBase {
method formatErrorMessage (line 103) | formatErrorMessage(e) {
method render (line 134) | render() {
method componentDidMount (line 173) | componentDidMount() {
class ParsableInputInline (line 180) | class ParsableInputInline extends ParsableInputBase {
method getDerivedStateFromProps (line 181) | static getDerivedStateFromProps(props, state) {
method render (line 219) | render() {
class ParsableInputInlineImpl (line 242) | class ParsableInputInlineImpl extends React.Component {
method render (line 243) | render() {
class ParsableInputAutogrowing (line 308) | class ParsableInputAutogrowing extends ParsableInputBase {
method render (line 309) | render() {
function ParsableInput (line 322) | function ParsableInput(props) {
function PyListInput (line 336) | function PyListInput({inputComponentRef, extraValueValidator, allowDupli...
function PyDictInput (line 347) | function PyDictInput({inputComponentRef, minSize, ...restProps}) {
function _dumpStringOrNumOrNone (line 358) | function _dumpStringOrNumOrNone(obj) {
function PyNumberInput (line 368) | function PyNumberInput({inputComponentRef, ...restProps}) {
function PyStringInput (line 379) | function PyStringInput({inputComponentRef, ...restProps}) {
function PyStringOrNumberInput (line 390) | function PyStringOrNumberInput({inputComponentRef, ...restProps}) {
function PySNNInput (line 401) | function PySNNInput({inputComponentRef, ...restProps}) {
class BlockInputToolbar (line 412) | class BlockInputToolbar extends React.Component {
method constructor (line 416) | constructor() {
method render (line 425) | render() {
method updateHeight (line 451) | updateHeight() {
method componentDidMount (line 460) | componentDidMount() {
method componentDidUpdate (line 464) | componentDidUpdate() {
class BlockInputToolbarImpl (line 469) | class BlockInputToolbarImpl extends React.Component {
method constructor (line 470) | constructor() {
method getDerivedStateFromProps (line 487) | static getDerivedStateFromProps(props, state) {
method hackyPossibleWorkingDeepEqual (line 497) | hackyPossibleWorkingDeepEqual(o1, o2) {
method _updateStack (line 519) | _updateStack(value) {
method render (line 591) | render() {
FILE: src/probing_visualization.js
constant DEFAULT_PROBING_BOX_GEOMETRY (line 24) | const DEFAULT_PROBING_BOX_GEOMETRY = {
constant SMALLER_PROBING_BOX_GEOMETRY (line 29) | const SMALLER_PROBING_BOX_GEOMETRY = {
class ProbingVisualizationImpl (line 34) | class ProbingVisualizationImpl extends React.Component {
method constructor (line 42) | constructor(props) {
method shouldComponentUpdate (line 60) | shouldComponentUpdate(nextProps, nextState) {
method render (line 87) | render() {
method transitionEnd (line 126) | transitionEnd() {
method d3render (line 139) | d3render() {
method _initOrUpd (line 346) | _initOrUpd() {
method componentDidUpdate (line 357) | componentDidUpdate() {
method componentDidMount (line 361) | componentDidMount() {
function selectProbingGeometry (line 366) | function selectProbingGeometry(windowWidth, windowHeight) {
class ProbingStateVisualization (line 374) | class ProbingStateVisualization extends React.Component {
method getExpectedHeight (line 375) | static getExpectedHeight() {
method render (line 379) | render() {
class ProbingVisualization (line 393) | class ProbingVisualization extends React.Component {
method render (line 397) | render() {
class GenerateProbingLinks (line 411) | class GenerateProbingLinks extends BreakpointFunction {
method run (line 412) | run(_slotsCount, _key, algo) {
FILE: src/py_obj_parsing.js
class PyParsingError (line 5) | class PyParsingError extends Error {
method constructor (line 6) | constructor(text, pos) {
class PyObjParser (line 17) | class PyObjParser {
method constructor (line 18) | constructor(literal) {
method skipWhitespace (line 23) | skipWhitespace() {
method current (line 29) | current() {
method next (line 33) | next() {
method isWhiteSpaceOrEol (line 37) | isWhiteSpaceOrEol(c) {
method isCurrentWhitespaceOrEol (line 41) | isCurrentWhitespaceOrEol() {
method consume (line 45) | consume(expectedChar) {
method consumeWS (line 57) | consumeWS(expectedChar) {
method throwErr (line 62) | throwErr(text, pos) {
method _parseStringOrNumberOrNone (line 70) | _parseStringOrNumberOrNone(allowedSeparators, fromDict, allowNonesInEr...
method _parseStringOrNumber (line 79) | _parseStringOrNumber(allowedSeparators, fromDict = true, allowNonesInE...
method parseDict (line 105) | parseDict(minSize = null) {
method parseList (line 137) | parseList(allowDuplicates = true, minSize = null, extraValueValidator) {
method parseNumber (line 180) | parseNumber(allowedSeparators = '') {
method parseString (line 217) | parseString() {
method isNextNone (line 246) | isNextNone(allowedSeparators = '') {
method _parseNoneOrThrowUnknownIdentifier (line 255) | _parseNoneOrThrowUnknownIdentifier(allowedSeparators) {
method checkTrailingChars (line 265) | checkTrailingChars() {
function _checkTrailingChars (line 273) | function _checkTrailingChars(parser, parseFunc) {
function parsePyString (line 279) | function parsePyString(s) {
function parsePyNumber (line 284) | function parsePyNumber(s) {
function parsePyDict (line 289) | function parsePyDict(s, minSize = null) {
function parsePyList (line 294) | function parsePyList(s, allowDuplicates = true, minSize = null, extraVal...
function parsePyStringOrNumber (line 299) | function parsePyStringOrNumber(s) {
function parsePyStringOrNumberOrNone (line 304) | function parsePyStringOrNumberOrNone(s) {
function dumpSimplePyObj (line 311) | function dumpSimplePyObj(o) {
function dumpPyList (line 321) | function dumpPyList(l) {
function dumpPyDict (line 329) | function dumpPyDict(d) {
FILE: src/util.js
constant OLIVE (line 13) | const OLIVE = '#3D9970';
constant RED (line 14) | const RED = '#FF4136';
constant BLUE (line 15) | const BLUE = '#0074D9';
constant COLOR_FOR_READ_OPS (line 17) | const COLOR_FOR_READ_OPS = '#13920b';
function doubleRAF (line 19) | function doubleRAF(callback) {
class CrossFade (line 25) | class CrossFade extends React.Component {
method render (line 26) | render() {
class DynamicP (line 39) | class DynamicP extends React.PureComponent {
method constructor (line 43) | constructor() {
method getDerivedStateFromProps (line 52) | static getDerivedStateFromProps(props, state) {
method render (line 64) | render() {
method componentDidUpdate (line 87) | componentDidUpdate() {
method componentDidMount (line 97) | componentDidMount() {
class DebounceWhenOutOfView (line 103) | class DebounceWhenOutOfView extends React.Component {
method render (line 104) | render() {
constant LEEWAY_Y (line 109) | const LEEWAY_Y = 100;
class DebounceWhenOutOfViewImpl (line 110) | class DebounceWhenOutOfViewImpl extends React.Component {
method constructor (line 113) | constructor() {
method getDerivedStateFromProps (line 122) | static getDerivedStateFromProps(props, state) {
method render (line 143) | render() {
method componentDidUpdate (line 148) | componentDidUpdate() {
method componentDidMount (line 153) | componentDidMount() {
method updateGeometry (line 158) | updateGeometry() {
method checkDebounceProps (line 186) | checkDebounceProps() {
function linebreaks (line 196) | function linebreaks(s) {
function MyFallbackComponent (line 203) | function MyFallbackComponent({componentStack, error}) {
function MyErrorBoundary (line 217) | function MyErrorBoundary(props) {
function squashUpdates (line 231) | function squashUpdates(func) {
class ChapterComponent (line 247) | class ChapterComponent extends React.Component {
method setter (line 250) | setter(name, throttled = false, incId = false) {
function Subcontainerize (line 275) | function Subcontainerize({children}) {
class BootstrapAlert (line 311) | class BootstrapAlert extends React.Component {
method constructor (line 314) | constructor() {
method render (line 328) | render() {
function initUxSettings (line 369) | function initUxSettings() {
function getUxSettings (line 452) | function getUxSettings() {
function singularOrPlural (line 459) | function singularOrPlural(num, singular, plural) {
function randint (line 463) | function randint(a, b) {
function randomChoice (line 472) | function randomChoice(array) {
function randomMeaningfulString (line 477) | function randomMeaningfulString() {
constant LETTERS (line 481) | const LETTERS = 'abcdefghijklmnopqrstuvwxyz';
function randomString3len (line 483) | function randomString3len() {
function fixFirstValues (line 491) | function fixFirstValues(func, values) {
function isDefinedSmallBoxScreen (line 507) | function isDefinedSmallBoxScreen(windowWidth, windowHeight) {
constant RANDOM_STRINGS (line 511) | const RANDOM_STRINGS = [
Condensed preview — 70 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (501K chars).
[
{
"path": ".babelrc",
"chars": 1062,
"preview": "{\n \"presets\": [\n [\n \"@babel/preset-env\",\n {\n \"modules\": false\n "
},
{
"path": ".gitignore",
"chars": 2173,
"preview": "### PYTHON\n#\n# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribut"
},
{
"path": ".prettierignore",
"chars": 31,
"preview": "package-lock.json\npackage.json\n"
},
{
"path": "LICENSE",
"chars": 1089,
"preview": "The MIT License (MIT)\n\nCopyright (c) 2017-2018 Alexander Putilin\n\nPermission is hereby granted, free of charge, to any p"
},
{
"path": "README.md",
"chars": 819,
"preview": "# Inside Python Dict - an explorable explanation\n\nThis repository contains the code for [\"Inside Python Dict\"](https://j"
},
{
"path": "package.json",
"chars": 4275,
"preview": "{\n \"name\": \"inside_python_dict\",\n \"version\": \"1.0.0\",\n \"description\": \"Inside python dict - an explorable explanation"
},
{
"path": "patches/python32_debug.diff",
"chars": 2025,
"preview": "diff --git a/Objects/dictobject.c b/Objects/dictobject.c\nindex c10bfccdce..3734a08281 100644\n--- a/Objects/dictobject.c\n"
},
{
"path": "patches/smooth-scrollbar+8.3.1.patch",
"chars": 3774,
"preview": "patch-package\n--- a/node_modules/smooth-scrollbar/events/touch.js\n+++ b/node_modules/smooth-scrollbar/events/touch.js\n@@"
},
{
"path": "patches/subscribe-ui-event+2.0.4.patch",
"chars": 943,
"preview": "patch-package\nnew file mode 100644\nBinary files /dev/null and b/node_modules/subscribe-ui-event/.index.es.js.swp differ\n"
},
{
"path": "python_code/actual_dict_factory_test.py",
"chars": 1123,
"preview": "import unittest\n\nfrom dict32_reimplementation_test_v2 import dict_factory\nfrom dictinfo import dump_py_dict\n\n\ndef table_"
},
{
"path": "python_code/build_autogenerated_chapter1_hash.py",
"chars": 434,
"preview": "import os\nimport sys\nsys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'build'))\nfrom hash_"
},
{
"path": "python_code/build_autogenerated_chapter2.py",
"chars": 148,
"preview": "import os\nimport sys\nsys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'build'))\nfrom hash_"
},
{
"path": "python_code/build_autogenerated_chapter3_chapter4.py",
"chars": 304,
"preview": "import os\nimport sys\nsys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'build'))\nfrom dict3"
},
{
"path": "python_code/chapter1_linear_search_reimplementation_test.py",
"chars": 1193,
"preview": "import random\nimport argparse\n\nimport hash_chapter1_reimpl_js\nimport hash_chapter1_impl\nimport build_autogenerated_chapt"
},
{
"path": "python_code/chapter4_probing_python_reimplementation_test.py",
"chars": 1405,
"preview": "import json\nfrom common import AllKeyValueFactory\nfrom js_reimpl_common import _init_sock_stuff, dump_simple_py_obj\nfrom"
},
{
"path": "python_code/common.py",
"chars": 3227,
"preview": "import random\nimport string\n\n\nclass EmptyValueClass(object):\n def __str__(self):\n return \"EMPTY\"\n\n def __re"
},
{
"path": "python_code/dict32_reimplementation_test_v2.py",
"chars": 7447,
"preview": "import random\nimport argparse\nimport json\n\nfrom common import EMPTY, AllKeyValueFactory, IntKeyValueFactory\nfrom dictinf"
},
{
"path": "python_code/dict_reimpl_common.py",
"chars": 507,
"preview": "from common import EMPTY\n\n\nclass Slot(object):\n def __init__(self, hash_code=EMPTY, key=EMPTY, value=EMPTY):\n "
},
{
"path": "python_code/dict_reimplementation.py",
"chars": 3340,
"preview": "from common import DUMMY, EMPTY\nfrom dict_reimpl_common import BaseDictImpl, Slot\nfrom operator import attrgetter\n\n\nclas"
},
{
"path": "python_code/dictinfo.py",
"chars": 474,
"preview": "import sys\n\n\ndef dump_py_dict(d):\n vi = sys.version_info\n\n if vi.major != 3:\n raise Exception(\"Unsupported "
},
{
"path": "python_code/dictinfo32.py",
"chars": 1289,
"preview": "from ctypes import Structure, c_ulong, POINTER, cast, py_object, c_long\nfrom common import get_object_field_or_null, EMP"
},
{
"path": "python_code/dictinfo33.py",
"chars": 1840,
"preview": "from ctypes import Structure, c_ulong, POINTER, cast, addressof, py_object, c_long, c_void_p\nfrom common import get_obje"
},
{
"path": "python_code/hash_chapter1_impl.py",
"chars": 707,
"preview": "def create_new(numbers):\n n = len(numbers)\n keys = [None for i in range(2 * n)]\n\n for num in numbers:\n i"
},
{
"path": "python_code/hash_chapter1_reimpl_js.py",
"chars": 506,
"preview": "from js_reimpl_common import run_op_chapter1_chapter2\n\n\ndef run_op(keys, op, **kwargs):\n return run_op_chapter1_chapt"
},
{
"path": "python_code/hash_chapter1_reimplementation_test.py",
"chars": 1819,
"preview": "import random\nimport argparse\n\nimport hash_chapter1_reimpl_js\nimport hash_chapter1_impl\nimport build_autogenerated_chapt"
},
{
"path": "python_code/hash_chapter2_impl.py",
"chars": 1906,
"preview": "from common import DUMMY, EMPTY\n\n\ndef create_new(from_keys):\n n = len(from_keys)\n hash_codes = [EMPTY for i in ran"
},
{
"path": "python_code/hash_chapter2_impl_test.py",
"chars": 5081,
"preview": "import unittest\nfrom hash_chapter2_impl import create_new, has_key, insert, remove, resize, DUMMY\nfrom common import gen"
},
{
"path": "python_code/hash_chapter2_reimpl_js.py",
"chars": 782,
"preview": "from js_reimpl_common import run_op_chapter1_chapter2\n\n\ndef run_op(hash_codes, keys, op, **kwargs):\n return run_op_ch"
},
{
"path": "python_code/hash_chapter2_reimplementation_test.py",
"chars": 4965,
"preview": "import random\nimport argparse\n\nfrom common import DUMMY, EMPTY, AllKeyValueFactory, IntKeyValueFactory\n\nimport hash_chap"
},
{
"path": "python_code/hash_chapter3_class_impl.py",
"chars": 4035,
"preview": "from common import DUMMY, EMPTY\nfrom dict_reimpl_common import BaseDictImpl, Slot\n\n\nclass AlmostPythonDictBase(BaseDictI"
},
{
"path": "python_code/hash_chapter3_class_impl_test.py",
"chars": 4246,
"preview": "import unittest\nfrom common import DUMMY, EMPTY\nfrom hash_chapter3_class_impl import AlmostPythonDictImplementationRecyc"
},
{
"path": "python_code/interface_test.py",
"chars": 1265,
"preview": "import unittest\nfrom dict32_reimplementation import PyDictReimplementation\nfrom hash_chapter3_class_impl import AlmostPy"
},
{
"path": "python_code/js_reimpl_common.py",
"chars": 2408,
"preview": "import socket\nimport json\n\nfrom common import DUMMY, EMPTY\n\nnone_info = {\n \"type\": \"None\",\n \"hash\": str(hash(None)"
},
{
"path": "python_code/js_reimplementation_interface.py",
"chars": 3618,
"preview": "import socket\nimport json\n\nfrom common import DUMMY, EMPTY\nfrom js_reimpl_common import dump_simple_py_obj, parse_simple"
},
{
"path": "scripts/extractPythonCode.js",
"chars": 3819,
"preview": "import 'ignore-styles';\n\nimport {\n DICT32_INIT,\n DICT32_SETITEM,\n DICT32_RESIZE_CODE,\n _DICT32_GETITEM_ONLY,"
},
{
"path": "scripts/pyReimplWrapper.js",
"chars": 9572,
"preview": "const net = require('net');\nconst split = require('split');\nimport 'ignore-styles';\n\nimport {BigNumber} from 'bignumber."
},
{
"path": "scripts/ssr.js",
"chars": 1670,
"preview": "require('dotenv').config();\n\nprocess.env.NODE_ENV = 'ssr';\n\nconsole.log = () => {}; // Do not log to stdout\nglobal.perfo"
},
{
"path": "src/app.js",
"chars": 19496,
"preview": "import _ from 'lodash';\nimport Bootstrap from 'bootstrap/dist/css/bootstrap.min.css';\nimport stylesCss from './styles.cs"
},
{
"path": "src/autogenerated/chapter1.html",
"chars": 773,
"preview": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <title>Inside python dict — an explorable explanation</title>\n"
},
{
"path": "src/autogenerated/chapter2.html",
"chars": 773,
"preview": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <title>Inside python dict — an explorable explanation</title>\n"
},
{
"path": "src/autogenerated/chapter3.html",
"chars": 773,
"preview": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <title>Inside python dict — an explorable explanation</title>\n"
},
{
"path": "src/autogenerated/chapter4.html",
"chars": 773,
"preview": "<!DOCTYPE html>\n<html lang=\"en\">\n <head>\n <title>Inside python dict — an explorable explanation</title>\n"
},
{
"path": "src/chapter1_simplified_hash.js",
"chars": 33649,
"preview": "import * as React from 'react';\nimport _ from 'lodash';\nimport {List as ImmutableList} from 'immutable';\n\nimport {EQ, Br"
},
{
"path": "src/chapter2_hash_table_functions.js",
"chars": 46671,
"preview": "import * as React from 'react';\n\nimport {BigNumber} from 'bignumber.js';\n\nimport {List} from 'immutable';\nimport {pyHash"
},
{
"path": "src/chapter3_and_4_common.js",
"chars": 27869,
"preview": "import _ from 'lodash';\nimport * as React from 'react';\nimport {Map, List, Record} from 'immutable';\nimport {BigNumber} "
},
{
"path": "src/chapter3_hash_class.js",
"chars": 39577,
"preview": "import * as React from 'react';\n\nimport {HashBreakpointFunction, pyHash, EQ, computeIdx, displayStr, DUMMY} from './hash"
},
{
"path": "src/chapter4_real_python_dict.js",
"chars": 39369,
"preview": "import * as React from 'react';\n\nimport {BigNumber} from 'bignumber.js';\nimport {\n hashClassConstructor,\n HashClas"
},
{
"path": "src/code_blocks.js",
"chars": 77628,
"preview": "import _ from 'lodash';\nimport classNames from 'classnames';\nimport memoizeOne from 'memoize-one';\nimport * as React fro"
},
{
"path": "src/common_formatters.js",
"chars": 1498,
"preview": "import {singularOrPlural} from './util';\n\nexport function commonFormatCheckCollisionLoopEndedPart(idx, fmtCollisionCount"
},
{
"path": "src/hash_impl_common.js",
"chars": 8938,
"preview": "import {BigNumber} from 'bignumber.js';\n\nexport function repr(obj, allowNull = false) {\n let res;\n if (typeof obj "
},
{
"path": "src/hash_impl_common.test.js",
"chars": 1167,
"preview": "import {BigNumber} from 'bignumber.js';\nimport {pyHashLong} from './hash_impl_common';\n\ntest('pyHashLong() from short in"
},
{
"path": "src/index.js",
"chars": 744,
"preview": "import {initAndRender} from './app';\nimport {Chapter1_SimplifiedHash} from './chapter1_simplified_hash.js';\nimport {Chap"
},
{
"path": "src/inputs.js",
"chars": 22370,
"preview": "import * as React from 'react';\nimport _ from 'lodash';\nimport {BigNumber} from 'bignumber.js';\nimport Sticky from 'reac"
},
{
"path": "src/mustache/chapter1.json",
"chars": 35,
"preview": "{\n \"chapters\": \"['chapter1']\"\n}\n"
},
{
"path": "src/mustache/chapter2.json",
"chars": 35,
"preview": "{\n \"chapters\": \"['chapter2']\"\n}\n"
},
{
"path": "src/mustache/chapter3.json",
"chars": 35,
"preview": "{\n \"chapters\": \"['chapter3']\"\n}\n"
},
{
"path": "src/mustache/chapter4.json",
"chars": 35,
"preview": "{\n \"chapters\": \"['chapter4']\"\n}\n"
},
{
"path": "src/page.html.template",
"chars": 704,
"preview": "<!doctype html>\n<html lang=\"en\">\n <head>\n <title>Inside python dict — an explorable explanation</title>\n <m"
},
{
"path": "src/probing_visualization.js",
"chars": 16975,
"preview": "import * as React from 'react';\nimport {Map as ImmutableMap, List as ImmutableList, Set as ImmutableSet} from 'immutable"
},
{
"path": "src/py_obj_parsing.js",
"chars": 10882,
"preview": "import {None, isNone, EQ} from './hash_impl_common';\n\nimport {BigNumber} from 'bignumber.js';\n\nclass PyParsingError exte"
},
{
"path": "src/py_obj_parsing.test.js",
"chars": 13520,
"preview": "import {BigNumber} from 'bignumber.js';\nimport {\n parsePyString,\n parsePyNumber,\n parsePyStringOrNumber,\n pa"
},
{
"path": "src/store.js",
"chars": 986,
"preview": "import {observable, action} from 'mobx';\n\nexport let globalSettings = observable({\n codePlaySpeed: 1,\n maxCodePlay"
},
{
"path": "src/styles.css",
"chars": 7433,
"preview": "/* TODO: remove unnessary fonts/font-weights */\n\n/* FROM: https://stackoverflow.com/questions/12502234/how-to-prevent-we"
},
{
"path": "src/util.js",
"chars": 16425,
"preview": "import _ from 'lodash';\nimport * as React from 'react';\nimport {observer} from 'mobx-react';\nimport {reaction} from 'mob"
},
{
"path": "ssr-all.sh",
"chars": 187,
"preview": "#!/bin/bash\nmkdir -p build\nfor i in {chapter1,chapter2,chapter3,chapter4}; do npm run --silent babel-node scripts/ssr.js"
},
{
"path": "stress_test_python.sh",
"chars": 3175,
"preview": "#!/bin/bash\nset -e -o pipefail\n\nNUM_INSERTS=200\nNUM_INSERTS_SMALLER=100\n\neval \"`pyenv init -`\"\n\npyenv shell 3.2.6\n\nfor k"
},
{
"path": "unittest_python.sh",
"chars": 259,
"preview": "#!/bin/bash\nset -e -o pipefail\n\neval \"`pyenv init -`\"\npyenv shell 3.2.6\n\npython3 python_code/hash_chapter2_impl_test.py\n"
},
{
"path": "webpack.common.js",
"chars": 1531,
"preview": "const webpack = require('webpack');\nconst path = require('path');\nconst MiniCssExtractPlugin = require('mini-css-extract"
},
{
"path": "webpack.dev.js",
"chars": 890,
"preview": "const merge = require('webpack-merge');\nconst common = require('./webpack.common.js');\nconst HtmlWebpackPlugin = require"
},
{
"path": "webpack.prod.js",
"chars": 1097,
"preview": "const webpack = require('webpack');\nconst merge = require('webpack-merge');\nconst common = require('./webpack.common.js'"
}
]
About this extraction
This page contains the full source code of the eleweek/inside_python_dict GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 70 files (470.3 KB), approximately 112.6k tokens, and a symbol index with 595 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.