[ { "path": ".gitignore", "content": "# Logs\nlogs\n*.log\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\nlerna-debug.log*\n.pnpm-debug.log*\n\n*database.csv\n# Diagnostic reports (https://nodejs.org/api/report.html)\nreport.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json\n\n# Runtime data\npids\n*.pid\n*.seed\n*.pid.lock\n\n# Directory for instrumented libs generated by jscoverage/JSCover\nlib-cov\n\n# Coverage directory used by tools like istanbul\ncoverage\n*.lcov\n\n# nyc test coverage\n.nyc_output\n\n# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)\n.grunt\n\n# Bower dependency directory (https://bower.io/)\nbower_components\n\n# node-waf configuration\n.lock-wscript\n\n# Compiled binary addons (https://nodejs.org/api/addons.html)\nbuild/Release\n\n# Dependency directories\nnode_modules/\njspm_packages/\n\n# Snowpack dependency directory (https://snowpack.dev/)\nweb_modules/\n\n# TypeScript cache\n*.tsbuildinfo\n\n# Optional npm cache directory\n.npm\n\n# Optional eslint cache\n.eslintcache\n\n# Optional stylelint cache\n.stylelintcache\n\n# Microbundle cache\n.rpt2_cache/\n.rts2_cache_cjs/\n.rts2_cache_es/\n.rts2_cache_umd/\n\n# Optional REPL history\n.node_repl_history\n\n# Output of 'npm pack'\n*.tgz\n\n# Yarn Integrity file\n.yarn-integrity\n\n# dotenv environment variable files\n.env\n.env.development.local\n.env.test.local\n.env.production.local\n.env.local\n\n# parcel-bundler cache (https://parceljs.org/)\n.cache\n.parcel-cache\n\n# Next.js build output\n.next\nout\n\n# Nuxt.js build / generate output\n.nuxt\ndist\n\n# Gatsby files\n.cache/\n# Comment in the public line in if your project uses Gatsby and not Next.js\n# https://nextjs.org/blog/next-9-1#public-directory-support\n# public\n\n# vuepress build output\n.vuepress/dist\n\n# vuepress v2.x temp and cache directory\n.temp\n.cache\n\n# Docusaurus cache and generated files\n.docusaurus\n\n# Serverless directories\n.serverless/\n\n# FuseBox cache\n.fusebox/\n\n# DynamoDB Local files\n.dynamodb/\n\n# TernJS port file\n.tern-port\n\n# Stores VSCode versions used for testing VSCode extensions\n.vscode-test\n\n# yarn v2\n.yarn/cache\n.yarn/unplugged\n.yarn/build-state.yml\n.yarn/install-state.gz\n.pnp.*\n" }, { "path": "README.md", "content": "# processing-large-reports-in-the-browser\n\n## About\n\nThis is the examples shown in my live screen about [Performance and multithreading in browsers + ECMAScript Modules in Web Workers (pt-br)](https://www.youtube.com/live/-wXPxJYhZeI?feature=share)\n\nFirst of all, leave your star 🌟 on this repo.\n\nAccess our [**exclusive telegram channel**](https://bit.ly/canalerickwendel) so I'll let you know about all the content I've been producing \n\n## Complete source code\n- Access it in [app](./recorded/)\n- Checkout the [live demo](https://erickwendel.github.io/processing-large-reports-in-the-browser/recorded) (preferentially on Chrome browser)\n - Upload the file [database-small.csv](./recorded/assets/database-small.csv) or download the [SLC_Police_Calls_2013__2016_cleaned_geocoded.csv(210.83 MB)](https://www.kaggle.com/datasets/foenix/slc-crime?select=SLC_Police_Calls_2013__2016_cleaned_geocoded.csv)on Kaggle.\n\n\n## Have fun!\n\n![Erick_Wendel_-_Thumbnail_Performance_e_multithreading](https://user-images.githubusercontent.com/8060102/221061262-f9425071-0080-48ff-b69c-55d9036937ba.jpg)\n\n## References:\n- https://caniuse.com/?search=workers\n" }, { "path": "preclass/annotations.txt", "content": "https://www.kaggle.com/datasets/foenix/slc-crime?select=SLC_Police_Calls_2013__2016_cleaned_geocoded.csv\n\n\nnpm init -y\nnpm i -D browser-sync@2.27.11\n\npackage.json \n start: browser-sync -w\n\ntouch index.html \n all html\n\nmkdir src \n touch \n view -> falar que é tudo que é api de tela \n export default class View {}\n service -> tudo que é regra de negocio\n export default class Service {}\n worker -> tudo que vai rodar em segundo plano no browser\n console.log(`I'm ready`)\n onmessage = (msg) => console.log('hello from worker')\n controller -> o intermedio entre as camadas\n export default class Controller {}\n index -> factory, inicializacao das intancias\n all\nindex.html \n import index.js \n\n-> should see a message on worker\n\n view \n setFileSize\n configureOnFileChange\n controller \n constructor\n all \n configureOnFileChange\n \n init \n configureOnFileChange\n-> should show raw bytes \n formatBytes\n all \n configureOnFileChange\n add formatBytes\n-> should show bytes \n view \n configureOnFormSubmit\n all \n controller \n configureOnFormSubmit\n init\n configureOnFormSubmit\n log form\n> after submit should log results \n view \n isWorkerEnabled\n updateProgress\n\n controller \n configureOnFormSubmit\n isWorkerEnabled\n only the if \n-> should receive a message on worker\n\n controller \n #events \n progress all \n onOcurrenceUpdate only log results \n worker \n postMessage({ eventType: 'progress', 100 })\n postMessage({ eventType: 'ocurrenceUpdate', })\n-> progress should be at 100% and ocurrenceUpdate should be logged \n\n service\n processFile only signature\n worker \n all\n \n service \n file.stream \n .pipeThrough(new TextDecoderStream())\n .pipeTo(new WritableStream({\n write(chunk) {\n console.log(' chunk', chunk)\n }\n }))\n> should print file contents \n service \n #csvToJSON (linesLength)\n all without progress \n\n> writable should print results as json \n\n service \n #setupProgress\n all \n processFile\n progressFn\n csvToJSON\n progressFn\n> should print progress correctly \n\n service \n #findOcurrencies\n all \n controller\n #events\n ocurrenceUpdate\n all \n> all application should work \n> try uploading with the big file and show that only measuring by the break line is not enough, thats why the linesLength and ocurrences fail on big files \n - task for you to figure out how to handle this \n " }, { "path": "preclass/demo/assets/database-small.csv", "content": "case,case,date cleared,call description,location,police zone,police grid,city council,x-coordinate,y-coordinate,x_gps_coords,y_gps_coords\nSL2016463,SL2016463,01/01/2016 12:00:00 AM,VEHICLE THEFT,5XX S 900 E ,Z5,151,4.0,1899098.0,883261.0,-111.86496146662066,40.757620896145816\nSL2016117,SL2016117,01/01/2016 12:00:00 AM,DOMESTIC/PHYSICAL INVESTIGATION,3XX W 300 S ,Z3,132,4.0,1889023.0,885223.0,-111.90135834600376,40.76288754694132\nSL2016427,SL2016427,01/01/2016 12:00:00 AM,TRESPASSING/UNWANTED - IND/SMALL GROUP,9XX N CHURCHILL DR ,Z3,131,3.0,1891429.0,895330.0,-111.89283367183442,40.790656783850956\nSL2016253,SL2016253,01/01/2016 12:00:00 AM,SUICIDE - THREAT,8XX W 300 N ,Z1,112,2.0,1885076.0,889825.0,-111.91568312403221,40.77546891450854\nSL20162,SL20162,01/01/2016 12:00:00 AM,HOLD LOG,1XX S 200 W ,Z3,132,4.0,1890028.0,886559.0,-111.89775203515366,40.76656669982441\nSL2016451,SL2016451,01/01/2016 12:00:00 AM,DUI DRIVER,2XX E 700 S ,Z3,135,4.0,1893682.0,881986.0,-111.8844904059753,40.754059238407514\nSL20164,SL20164,01/01/2016 12:00:00 AM,HOLD LOG,6XX E 500 S ,Z4,144,4.0,1897180.0,883741.0,-111.8718915856916,40.75891666684364\nSL20167,SL20167,01/01/2016 12:00:00 AM,HOLD LOG,1XX S MAIN ST ,Z3,133,4.0,1891897.0,886421.0,-111.89100292664365,40.766210538859816\nSL20168,SL20168,01/01/2016 12:00:00 AM,PHYSICAL (FIGHT) INDIVIDUAL/SMALL GROUP,16XX W 700 N ,Z1,112,1.0,1878776.0,893043.0,-111.93848548107479,40.784218588076925\nSL20169,SL20169,01/01/2016 12:00:00 AM,TRAFFIC STOP,1XX E 100 S ,Z3,133,4.0,1892875.0,886743.0,-111.88747748637616,40.767105984796935\nSL201610,SL201610,01/01/2016 12:00:00 AM,TRAFFIC STOP,,113,,,,,,\nSL20166,SL20166,01/01/2016 12:00:00 AM,CELL 911 HANGUP,8XX W 100 S ,Z2,121,2.0,1885393.0,886772.0,-111.91448753269357,40.76709359244312\nSL201615,SL201615,01/01/2016 12:00:00 AM,HOLD LOG,1XX S MAIN ST ,Z3,133,4.0,1891897.0,886421.0,-111.89100292664365,40.766210538859816\nSL201617,SL201617,01/01/2016 12:00:00 AM,HOLD LOG,1XX S 200 W ,Z3,132,4.0,1890028.0,886559.0,-111.89775203515366,40.76656669982441\nSL201618,SL201618,01/01/2016 12:00:00 AM,HOLD LOG,0XX W 300 S ,Z3,133,4.0,1891718.0,885293.0,-111.89163126027671,40.763112433818634\nSL201623,SL201623,01/01/2016 12:00:00 AM,HOLD LOG,,SOSL,SOSL,,1887422.0,869203.0,-111.9068745235229,40.71889818951389\nSL201611,SL201611,01/01/2016 12:00:00 AM,TRAFFIC STOP,3XX W PAXTON AVE ,Z5,152,5.0,1889383.0,878176.0,-111.89994509279057,40.74355040073993\nSL201621,SL201621,01/01/2016 12:00:00 AM,911 HANGUP CALL,1XX E WESTMINSTER AVE ,Z6,161,5.0,1893124.0,872787.0,-111.88636102134953,40.72880447201603\nSL201622,SL201622,01/01/2016 12:00:00 AM,HOLD LOG,2XX N 2200 W ,Z1,113,1.0,1874833.0,889644.0,-111.95266088662085,40.774835770400095" }, { "path": "preclass/demo/index.html", "content": "\n\n\n\n \n \n \n Document\n \n\n\n\n
\n
\n \n \n
\n\n
\n \n \n
\n
\n \n \n
\n\n
\n \n \n
\n
\n \n
\n
\n \n 0% \n
\n
\n \n
\n
\n\n\n \n\n\n" }, { "path": "preclass/demo/package.json", "content": "{\n \"devDependencies\": {\n \"browser-sync\": \"^2.27.11\"\n },\n \"name\": \"live-workers-report\",\n \"version\": \"0.0.1\",\n \"main\": \"index.js\",\n \"dependencies\": {\n \"accepts\": \"^1.3.8\",\n \"ansi-regex\": \"^2.1.1\",\n \"ansi-styles\": \"^2.2.1\",\n \"async\": \"^2.6.4\",\n \"anymatch\": \"^3.1.3\",\n \"async-each-series\": \"^0.1.1\",\n \"batch\": \"^0.6.1\",\n \"base64id\": \"^2.0.0\",\n \"brace-expansion\": \"^1.1.11\",\n \"balanced-match\": \"^1.0.2\",\n \"binary-extensions\": \"^2.2.0\",\n \"browser-sync-ui\": \"^2.27.11\",\n \"braces\": \"^3.0.2\",\n \"bs-recipes\": \"^1.3.4\",\n \"bs-snippet-injector\": \"^2.0.1\",\n \"browser-sync-client\": \"^2.27.11\",\n \"axios\": \"^0.21.4\",\n \"bytes\": \"^3.1.2\",\n \"chokidar\": \"^3.5.3\",\n \"color-convert\": \"^2.0.1\",\n \"chalk\": \"^1.1.3\",\n \"color-name\": \"^1.1.4\",\n \"cliui\": \"^8.0.1\",\n \"commander\": \"^2.20.3\",\n \"connect-history-api-fallback\": \"^1.6.0\",\n \"cookie\": \"^0.4.2\",\n \"call-bind\": \"^1.0.2\",\n \"debug\": \"^2.6.9\",\n \"cors\": \"^2.8.5\",\n \"depd\": \"^2.0.0\",\n \"easy-extender\": \"^2.3.4\",\n \"destroy\": \"^1.0.4\",\n \"concat-map\": \"^0.0.1\",\n \"dlv\": \"^1.1.3\",\n \"connect\": \"^3.6.6\",\n \"eazy-logger\": \"^3.1.0\",\n \"encodeurl\": \"^1.0.2\",\n \"ee-first\": \"^1.1.1\",\n \"engine.io-parser\": \"^5.0.6\",\n \"engine.io\": \"^6.4.1\",\n \"dev-ip\": \"^1.0.1\",\n \"engine.io-client\": \"^6.4.0\",\n \"escape-html\": \"^1.0.3\",\n \"etag\": \"^1.8.1\",\n \"emoji-regex\": \"^8.0.0\",\n \"escalade\": \"^3.1.1\",\n \"escape-string-regexp\": \"^1.0.5\",\n \"fill-range\": \"^7.0.1\",\n \"eventemitter3\": \"^4.0.7\",\n \"finalhandler\": \"^1.1.0\",\n \"fs-extra\": \"^3.0.1\",\n \"fresh\": \"^0.5.2\",\n \"fsevents\": \"^2.3.2\",\n \"function-bind\": \"^1.1.1\",\n \"get-intrinsic\": \"^1.2.0\",\n \"follow-redirects\": \"^1.15.2\",\n \"get-caller-file\": \"^2.0.5\",\n \"glob-parent\": \"^5.1.2\",\n \"graceful-fs\": \"^4.2.10\",\n \"has\": \"^1.0.3\",\n \"has-ansi\": \"^2.0.0\",\n \"http-errors\": \"^2.0.0\",\n \"iconv-lite\": \"^0.4.24\",\n \"has-symbols\": \"^1.0.3\",\n \"immutable\": \"^3.8.2\",\n \"http-proxy\": \"^1.18.1\",\n \"is-binary-path\": \"^2.1.0\",\n \"inherits\": \"^2.0.4\",\n \"is-extglob\": \"^2.1.1\",\n \"is-fullwidth-code-point\": \"^3.0.0\",\n \"is-number\": \"^7.0.0\",\n \"is-glob\": \"^4.0.3\",\n \"is-number-like\": \"^1.0.8\",\n \"is-wsl\": \"^1.1.0\",\n \"jsonfile\": \"^3.0.1\",\n \"lodash\": \"^4.17.21\",\n \"lodash.isfinite\": \"^3.3.2\",\n \"localtunnel\": \"^2.0.2\",\n \"limiter\": \"^1.1.5\",\n \"mime-db\": \"^1.52.0\",\n \"mime\": \"^1.4.1\",\n \"mime-types\": \"^2.1.35\",\n \"minimatch\": \"^3.1.2\",\n \"micromatch\": \"^4.0.5\",\n \"ms\": \"^2.0.0\",\n \"mitt\": \"^1.2.0\",\n \"negotiator\": \"^0.6.3\",\n \"normalize-path\": \"^3.0.0\",\n \"object-assign\": \"^4.1.1\",\n \"object-inspect\": \"^1.12.3\",\n \"on-finished\": \"^2.3.0\",\n \"openurl\": \"^1.1.1\",\n \"opn\": \"^5.3.0\",\n \"parseurl\": \"^1.3.3\",\n \"picomatch\": \"^2.3.1\",\n \"portscanner\": \"^2.2.0\",\n \"qs\": \"^6.11.0\",\n \"raw-body\": \"^2.5.2\",\n \"range-parser\": \"^1.2.1\",\n \"readdirp\": \"^3.6.0\",\n \"require-directory\": \"^2.1.1\",\n \"requires-port\": \"^1.0.0\",\n \"resp-modifier\": \"^6.0.2\",\n \"rx\": \"^4.1.0\",\n \"rxjs\": \"^5.5.12\",\n \"send\": \"^0.16.2\",\n \"safer-buffer\": \"^2.1.2\",\n \"serve-static\": \"^1.13.2\",\n \"serve-index\": \"^1.9.1\",\n \"server-destroy\": \"^1.0.1\",\n \"setprototypeof\": \"^1.2.0\",\n \"side-channel\": \"^1.0.4\",\n \"socket.io-adapter\": \"^2.5.2\",\n \"socket.io-client\": \"^4.6.1\",\n \"statuses\": \"^1.3.1\",\n \"socket.io-parser\": \"^4.2.2\",\n \"socket.io\": \"^4.6.1\",\n \"strip-ansi\": \"^3.0.1\",\n \"supports-color\": \"^2.0.0\",\n \"tfunk\": \"^4.0.0\",\n \"string-width\": \"^4.2.3\",\n \"to-regex-range\": \"^5.0.1\",\n \"stream-throttle\": \"^0.1.3\",\n \"toidentifier\": \"^1.0.1\",\n \"ua-parser-js\": \"^1.0.2\",\n \"symbol-observable\": \"^1.0.1\",\n \"typescript\": \"^4.9.5\",\n \"vary\": \"^1.1.2\",\n \"unpipe\": \"^1.0.0\",\n \"universalify\": \"^0.1.2\",\n \"utils-merge\": \"^1.0.1\",\n \"wrap-ansi\": \"^7.0.0\",\n \"ws\": \"^8.11.0\",\n \"y18n\": \"^5.0.8\",\n \"xmlhttprequest-ssl\": \"^2.0.0\",\n \"yargs\": \"^17.7.1\",\n \"yargs-parser\": \"^21.1.1\"\n },\n \"scripts\": {\n \"start\": \"npx browser-sync -w\",\n \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n },\n \"keywords\": [],\n \"author\": \"\",\n \"license\": \"ISC\",\n \"description\": \"\"\n}\n" }, { "path": "preclass/demo/src/controller.js", "content": "export default class Controller {\n #view\n #service\n #worker\n #events = {\n alive: () => { },\n ocurrenceUpdate: ({ found, linesLength, took }) => {\n const [[key, value]] = Object.entries(found)\n this.#view.updateDebugLog(`found ${value} ocurrencies of ${key} - over ${linesLength} lines - took: ${took}`)\n },\n progress: ({ total }) => {\n this.#view.updateProgress(total)\n }\n }\n\n constructor({ view, service, worker }) {\n this.#service = service\n this.#view = view\n this.#worker = this.#configureWorker(worker)\n\n }\n\n static init(deps) {\n const controller = new Controller(deps)\n return controller.init()\n }\n\n init() {\n this.#view.configureOnFileChange(this.#configureOnFileChange.bind(this))\n this.#view.configureOnFormSubmit(this.#configureOnFormSubmit.bind(this))\n }\n\n #configureWorker(worker) {\n worker.onmessage = ({ data }) => this.#events[data.eventType](data)\n return worker\n }\n\n #formatBytes(bytes) {\n const units = ['B', 'KB', 'MB', 'GB', 'TB']\n\n let i = 0\n\n for (i; bytes >= 1024 && i < 4; i++) {\n bytes /= 1024\n }\n\n return `${bytes.toFixed(2)} ${units[i]}`\n }\n\n #configureOnFileChange(file) {\n const size = file.size\n this.#view.setFileSize(this.#formatBytes(size))\n }\n\n #configureOnFormSubmit({ description, file }) {\n const query = {}\n query[\"call description\"] = new RegExp(description, 'i')\n if (this.#view.isWorkerEnabled()) {\n console.log('executing on worker thread!')\n this.#worker.postMessage({ query, file })\n return\n }\n\n console.log('executing on main thread!')\n // sem worker\n this.#service.processFile({\n query,\n file,\n onProgress: (total) => {\n this.#events.progress({ total })\n },\n onOcurrenceUpdate: (...args) => {\n this.#events.ocurrenceUpdate(...args)\n }\n })\n }\n}" }, { "path": "preclass/demo/src/index.js", "content": "import Controller from \"./controller.js\"\nimport Service from \"./service.js\"\nimport View from \"./view.js\"\n\n// somente no chrome por enquanto\nconst worker = new Worker('./src/worker.js', { type: \"module\"})\n\nController.init({\n view: new View(),\n service: new Service(),\n worker,\n})" }, { "path": "preclass/demo/src/service.js", "content": "export default class Service {\n\n #setupProgress(totalBytes, onProgress) {\n let totalUploaded = 0\n onProgress(0)\n return (chunkLength) => {\n totalUploaded += chunkLength\n const total = 100 / totalBytes * totalUploaded\n onProgress(total)\n }\n }\n\n processFile({ query, file, onOcurrenceUpdate, onProgress }) {\n const startedAt = performance.now()\n const elapsed = () => `${(Math.round(performance.now() - startedAt) / 1000)} secs`\n const progressFn = this.#setupProgress(file.size, onProgress)\n const linesLength = { counter: 0 }\n const onUpdate = () => {\n return (found) => {\n onOcurrenceUpdate({\n found,\n took: elapsed(),\n linesLength: linesLength.counter,\n })\n }\n }\n\n file.stream()\n .pipeThrough(new TextDecoderStream())\n .pipeThrough(this.#csvToJSON({ progressFn, linesLength }))\n .pipeTo(this.#findOcurrencies({ query, onUpdate: onUpdate() }))\n }\n\n #findOcurrencies({ query, onUpdate }) {\n const queryKeys = Object.keys(query)\n let found = {}\n\n return new WritableStream({\n write(chunk) {\n\n for (const keyIndex in queryKeys) {\n const key = queryKeys[keyIndex]\n const queryValue = query[key]\n found[queryValue] = found[queryValue] ?? 0\n if (queryValue.test(chunk[key])) {\n found[queryValue]++\n onUpdate(found)\n }\n }\n },\n\n close: () => onUpdate(found)\n })\n }\n\n #csvToJSON({ linesLength, progressFn }) {\n let columns = []\n return new TransformStream({\n transform(chunk, controller) {\n progressFn(chunk.length)\n const lines = chunk.split('\\n')\n linesLength.counter += lines.length\n\n if (!columns.length) {\n const firstLine = lines.shift()\n columns = firstLine.split(',')\n linesLength.counter--\n }\n\n for (const line of lines) {\n if (!line.length) continue\n\n const currentColumns = line.split(',')\n let currentItem = {}\n for (const columIndex in currentColumns) {\n const columnItem = currentColumns[columIndex]\n currentItem[columns[columIndex]] = columnItem.trimEnd()\n }\n\n controller.enqueue(currentItem)\n }\n },\n })\n }\n}" }, { "path": "preclass/demo/src/view.js", "content": "export default class View {\n #debugElement = document.getElementById('debug')\n #fileSizeElement = document.getElementById('file-size')\n #processingProgress = document.getElementById('progress')\n #csvFile = document.querySelector('#csv-file')\n #form = document.querySelector('#form')\n #workerChecker = document.querySelector('#worker')\n\n configureOnFileChange(fn) {\n this.#csvFile.addEventListener('change', e => {\n fn(e.target.files[0])\n })\n }\n setFileSize(size) {\n this.#fileSizeElement.innerText = `File size: ${size}\\n`\n }\n isWorkerEnabled() {\n return this.#workerChecker.checked\n }\n updateProgress(value) {\n this.#processingProgress.value = value\n }\n \n configureOnFormSubmit(fn) {\n this.#form.reset()\n this.#form.addEventListener('submit', (e) => {\n e.preventDefault()\n const file = this.#csvFile.files[0]\n if (!file) {\n alert('Please select a file')\n return\n }\n\n this.updateDebugLog(\"\")\n\n const form = new FormData(e.currentTarget)\n const description = form.get('description')\n\n fn({ description, file })\n\n })\n }\n\n updateDebugLog(text, reset = true) {\n if (reset) {\n this.#debugElement.innerText = text\n return\n }\n\n this.#debugElement.innerText += text\n }\n\n}" }, { "path": "preclass/demo/src/worker.js", "content": "import Service from './service.js'\nconst service = new Service()\n\nconsole.log(`I'm alive!`)\npostMessage({ eventType: 'alive' })\n\nonmessage = ({ data }) => {\n const { query, file } = data\n service.processFile({\n query,\n file,\n onProgress: (total) => postMessage({ eventType: 'progress', total }),\n onOcurrenceUpdate: (args) => {\n postMessage({ eventType: 'ocurrenceUpdate', ...args })\n }\n })\n}" }, { "path": "preclass/prova-real/index.js", "content": "import { createReadStream } from 'node:fs'\nimport csvtojson from 'csvtojson'\nimport { pipeline } from 'node:stream/promises'\nimport { Writable } from 'node:stream'\nlet lines = 0\nlet ocurrences = 0\nconst word = process.argv[2]?.trim()\nconsole.time('search')\nconsole.log('searching for...', word)\nawait pipeline(\n createReadStream('./database.csv'),\n csvtojson(),\n new Writable({\n write(chunk, enc, cb) {\n lines++\n if (new RegExp(word, 'i').test(JSON.parse(chunk)['call description'])) {\n ocurrences++\n }\n cb()\n }\n })\n)\n\nconsole.log({ lines, ocurrences })\n\nconsole.timeEnd('search')" }, { "path": "preclass/prova-real/package.json", "content": "{\n \"name\": \"app\",\n \"version\": \"0.0.1\",\n \"description\": \"\",\n \"main\": \"index.js\",\n \"type\": \"module\",\n \"scripts\": {\n \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n },\n \"keywords\": [],\n \"author\": \"\",\n \"license\": \"ISC\",\n \"dependencies\": {\n \"csvtojson\": \"^2.0.10\"\n }\n}\n" }, { "path": "recorded/.vscode/settings.json", "content": "{\n \"window.zoomLevel\": 3\n}" }, { "path": "recorded/assets/database-small.csv", "content": "case,case,date cleared,call description,location,police zone,police grid,city council,x-coordinate,y-coordinate,x_gps_coords,y_gps_coords\nSL2016463,SL2016463,01/01/2016 12:00:00 AM,VEHICLE THEFT,5XX S 900 E ,Z5,151,4.0,1899098.0,883261.0,-111.86496146662066,40.757620896145816\nSL2016117,SL2016117,01/01/2016 12:00:00 AM,DOMESTIC/PHYSICAL INVESTIGATION,3XX W 300 S ,Z3,132,4.0,1889023.0,885223.0,-111.90135834600376,40.76288754694132\nSL2016427,SL2016427,01/01/2016 12:00:00 AM,TRESPASSING/UNWANTED - IND/SMALL GROUP,9XX N CHURCHILL DR ,Z3,131,3.0,1891429.0,895330.0,-111.89283367183442,40.790656783850956\nSL2016253,SL2016253,01/01/2016 12:00:00 AM,SUICIDE - THREAT,8XX W 300 N ,Z1,112,2.0,1885076.0,889825.0,-111.91568312403221,40.77546891450854\nSL20162,SL20162,01/01/2016 12:00:00 AM,HOLD LOG,1XX S 200 W ,Z3,132,4.0,1890028.0,886559.0,-111.89775203515366,40.76656669982441\nSL2016451,SL2016451,01/01/2016 12:00:00 AM,DUI DRIVER,2XX E 700 S ,Z3,135,4.0,1893682.0,881986.0,-111.8844904059753,40.754059238407514\nSL20164,SL20164,01/01/2016 12:00:00 AM,HOLD LOG,6XX E 500 S ,Z4,144,4.0,1897180.0,883741.0,-111.8718915856916,40.75891666684364\nSL20167,SL20167,01/01/2016 12:00:00 AM,HOLD LOG,1XX S MAIN ST ,Z3,133,4.0,1891897.0,886421.0,-111.89100292664365,40.766210538859816\nSL20168,SL20168,01/01/2016 12:00:00 AM,PHYSICAL (FIGHT) INDIVIDUAL/SMALL GROUP,16XX W 700 N ,Z1,112,1.0,1878776.0,893043.0,-111.93848548107479,40.784218588076925\nSL20169,SL20169,01/01/2016 12:00:00 AM,TRAFFIC STOP,1XX E 100 S ,Z3,133,4.0,1892875.0,886743.0,-111.88747748637616,40.767105984796935\nSL201610,SL201610,01/01/2016 12:00:00 AM,TRAFFIC STOP,,113,,,,,,\nSL20166,SL20166,01/01/2016 12:00:00 AM,CELL 911 HANGUP,8XX W 100 S ,Z2,121,2.0,1885393.0,886772.0,-111.91448753269357,40.76709359244312\nSL201615,SL201615,01/01/2016 12:00:00 AM,HOLD LOG,1XX S MAIN ST ,Z3,133,4.0,1891897.0,886421.0,-111.89100292664365,40.766210538859816\nSL201617,SL201617,01/01/2016 12:00:00 AM,HOLD LOG,1XX S 200 W ,Z3,132,4.0,1890028.0,886559.0,-111.89775203515366,40.76656669982441\nSL201618,SL201618,01/01/2016 12:00:00 AM,HOLD LOG,0XX W 300 S ,Z3,133,4.0,1891718.0,885293.0,-111.89163126027671,40.763112433818634\nSL201623,SL201623,01/01/2016 12:00:00 AM,HOLD LOG,,SOSL,SOSL,,1887422.0,869203.0,-111.9068745235229,40.71889818951389\nSL201611,SL201611,01/01/2016 12:00:00 AM,TRAFFIC STOP,3XX W PAXTON AVE ,Z5,152,5.0,1889383.0,878176.0,-111.89994509279057,40.74355040073993\nSL201621,SL201621,01/01/2016 12:00:00 AM,911 HANGUP CALL,1XX E WESTMINSTER AVE ,Z6,161,5.0,1893124.0,872787.0,-111.88636102134953,40.72880447201603\nSL201622,SL201622,01/01/2016 12:00:00 AM,HOLD LOG,2XX N 2200 W ,Z1,113,1.0,1874833.0,889644.0,-111.95266088662085,40.774835770400095\n" }, { "path": "recorded/index.html", "content": "\n\n\n\n \n \n \n Document\n \n\n\n\n
\n
\n \n \n
\n
\n \n \n
\n\n
\n \n \n
\n
\n \n \n
\n\n
\n \n
\n\n
\n \n 0%\n
\n\n
\n \n
\n\n
\n\n \n\n\n" }, { "path": "recorded/package.json", "content": "{\n \"name\": \"recorded\",\n \"version\": \"0.0.1\",\n \"description\": \"\",\n \"main\": \"index.js\",\n \"scripts\": {\n \"start\": \"npx browser-sync -w\"\n },\n \"keywords\": [],\n \"author\": \"\",\n \"license\": \"ISC\",\n \"devDependencies\": {\n \"browser-sync\": \"^2.27.11\"\n }\n}\n" }, { "path": "recorded/src/controller.js", "content": "export default class Controller {\n #view\n #worker\n #service\n #events = {\n alive: () => { },\n progress: ({ total }) => {\n this.#view.updateProgress(total)\n },\n ocurrenceUpdate: ({ found, linesLength, took }) => {\n const [[key, value]] = Object.entries(found)\n this.#view.updateDebugLog(\n `found ${value} ocurrencies of ${key} - over ${linesLength} lines - took: ${took}`\n )\n }\n }\n\n constructor({ view, worker, service }) {\n this.#view = view\n this.#service = service\n this.#worker = this.#configureWorker(worker)\n }\n\n static init(deps) {\n const controller = new Controller(deps)\n controller.init()\n return controller\n }\n\n init() {\n this.#view.configureOnFileChange(\n this.#configureOnFileChange.bind(this)\n )\n\n this.#view.configureOnFormSubmit(\n this.#configureOnFormSubmit.bind(this)\n )\n }\n\n #configureWorker(worker) {\n worker.onmessage = ({ data }) => this.#events[data.eventType](data)\n\n return worker\n }\n\n #formatBytes(bytes) {\n const units = ['B', 'KB', 'MB', 'GB', 'TB']\n\n let i = 0\n\n for (i; bytes >= 1024 && i < 4; i++) {\n bytes /= 1024\n }\n\n return `${bytes.toFixed(2)} ${units[i]}`\n }\n\n #configureOnFileChange(file) {\n this.#view.setFileSize(\n this.#formatBytes(file.size)\n )\n }\n\n #configureOnFormSubmit({ description, file }) {\n const query = {}\n query['call description'] = new RegExp(\n description, 'i'\n )\n\n if (this.#view.isWorkerEnabled()) {\n console.log('executing on worker thread!')\n this.#worker.postMessage({ query, file })\n return\n }\n\n console.log('executing on main thread!')\n this.#service.processFile({\n query,\n file,\n onProgress: (total) => {\n this.#events.progress({ total })\n },\n onOcurrenceUpdate: (...args) => {\n this.#events.ocurrenceUpdate(...args)\n }\n })\n }\n}\n" }, { "path": "recorded/src/index.js", "content": "import Controller from \"./controller.js\"\nimport Service from \"./service.js\"\nimport View from \"./view.js\"\n\n// worker modules só funciona no chrome por enquanto\n// ou seja worker funciona, mas import/export nao\nconst worker = new Worker('./src/worker.js', {\n type: \"module\"\n})\n\n\nController.init({\n view: new View(),\n service: new Service(),\n worker\n})" }, { "path": "recorded/src/service.js", "content": "export default class Service {\n\n processFile({ query, file, onOcurrenceUpdate, onProgress }) {\n const linesLength = { counter: 0 }\n const progressFn = this.#setupProgress(file.size, onProgress)\n const startedAt = performance.now()\n const elapsed = () => `${((performance.now() - startedAt) / 1000).toFixed(2)} secs`\n\n const onUpdate = () => {\n return (found) => {\n onOcurrenceUpdate({\n found,\n took: elapsed(),\n linesLength: linesLength.counter\n })\n\n }\n }\n\n file.stream()\n .pipeThrough(new TextDecoderStream())\n .pipeThrough(this.#csvToJSON({ linesLength, progressFn }))\n .pipeTo(this.#findOcurrencies({ query, onOcurrenceUpdate: onUpdate() }))\n // .pipeTo(new WritableStream({\n // write(chunk) {\n // console.log('chunk', chunk)\n // }\n // }))\n }\n\n #csvToJSON({ linesLength, progressFn }) {\n let columns = []\n return new TransformStream({\n transform(chunk, controller) {\n progressFn(chunk.length)\n const lines = chunk.split('\\n')\n linesLength.counter += lines.length\n\n if (!columns.length) {\n const firstLine = lines.shift()\n columns = firstLine.split(',')\n linesLength.counter--\n }\n\n for (const line of lines) {\n if (!line.length) continue\n let currentItem = {}\n const currentColumsItems = line.split(',')\n for (const columnIndex in currentColumsItems) {\n const columnItem = currentColumsItems[columnIndex]\n currentItem[columns[columnIndex]] = columnItem.trimEnd()\n }\n controller.enqueue(currentItem)\n }\n }\n })\n }\n\n #findOcurrencies({ query, onOcurrenceUpdate }) {\n const queryKeys = Object.keys(query)\n let found = {}\n\n return new WritableStream({\n write(jsonLine) {\n for (const keyIndex in queryKeys) {\n const key = queryKeys[keyIndex]\n const queryValue = query[key]\n found[queryValue] = found[queryValue] ?? 0\n if (queryValue.test(jsonLine[key])) {\n found[queryValue]++\n onOcurrenceUpdate(found)\n }\n }\n },\n close: () => onOcurrenceUpdate(found)\n })\n }\n #setupProgress(totalBytes, onProgress) {\n let totalUploaded = 0\n onProgress(0)\n\n return (chunkLength) => {\n totalUploaded += chunkLength\n const total = 100 / totalBytes * totalUploaded\n onProgress(total)\n }\n }\n}" }, { "path": "recorded/src/view.js", "content": "export default class View {\n #csvFile = document.querySelector('#csv-file')\n #fileSize = document.querySelector('#file-size')\n #form = document.querySelector('#form')\n #debug = document.querySelector('#debug')\n #progress = document.querySelector('#progress')\n #worker = document.querySelector('#worker')\n\n setFileSize(size) {\n this.#fileSize.innerText = `File size: ${size}\\n`\n }\n\n configureOnFileChange(fn) {\n this.#csvFile.addEventListener('change', e => {\n fn(e.target.files[0])\n })\n }\n\n configureOnFormSubmit(fn) {\n this.#form.reset()\n this.#form.addEventListener('submit', (e) => {\n e.preventDefault()\n const file = this.#csvFile.files[0]\n // isso aqui deveria estar na controller\n if(!file) {\n alert('Please select a file!')\n return\n }\n this.updateDebugLog(\"\")\n const form = new FormData(e.currentTarget)\n const description = form.get('description')\n fn({ description, file })\n })\n }\n\n updateDebugLog(text, reset = true) {\n if(reset) {\n this.#debug.innerText = text\n return;\n }\n\n this.#debug.innerText += text\n }\n\n updateProgress(value) {\n this.#progress.value = value\n }\n\n isWorkerEnabled() {\n return this.#worker.checked\n }\n}\n" }, { "path": "recorded/src/worker.js", "content": "import Service from './service.js'\nconsole.log(`I'm alive!`)\nconst service = new Service()\n\npostMessage({ eventType: 'alive' })\nonmessage = ({ data }) => {\n const { query, file } = data\n service.processFile({\n query,\n file,\n onOcurrenceUpdate: (args) => {\n postMessage({ eventType: 'ocurrenceUpdate', ...args })\n },\n onProgress: (total) => postMessage({ eventType: 'progress', total })\n })\n}" } ]