Repository: ErickWendel/processing-large-reports-in-the-browser Branch: main Commit: 2f7cd76554ec Files: 22 Total size: 34.1 KB Directory structure: gitextract_ro4ow9zx/ ├── .gitignore ├── README.md ├── preclass/ │ ├── annotations.txt │ ├── demo/ │ │ ├── assets/ │ │ │ └── database-small.csv │ │ ├── index.html │ │ ├── package.json │ │ └── src/ │ │ ├── controller.js │ │ ├── index.js │ │ ├── service.js │ │ ├── view.js │ │ └── worker.js │ └── prova-real/ │ ├── index.js │ └── package.json └── recorded/ ├── .vscode/ │ └── settings.json ├── assets/ │ └── database-small.csv ├── index.html ├── package.json └── src/ ├── controller.js ├── index.js ├── service.js ├── view.js └── worker.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ # Logs logs *.log npm-debug.log* yarn-debug.log* yarn-error.log* lerna-debug.log* .pnpm-debug.log* *database.csv # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json # Runtime data pids *.pid *.seed *.pid.lock # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage *.lcov # nyc test coverage .nyc_output # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) .grunt # Bower dependency directory (https://bower.io/) bower_components # node-waf configuration .lock-wscript # Compiled binary addons (https://nodejs.org/api/addons.html) build/Release # Dependency directories node_modules/ jspm_packages/ # Snowpack dependency directory (https://snowpack.dev/) web_modules/ # TypeScript cache *.tsbuildinfo # Optional npm cache directory .npm # Optional eslint cache .eslintcache # Optional stylelint cache .stylelintcache # Microbundle cache .rpt2_cache/ .rts2_cache_cjs/ .rts2_cache_es/ .rts2_cache_umd/ # Optional REPL history .node_repl_history # Output of 'npm pack' *.tgz # Yarn Integrity file .yarn-integrity # dotenv environment variable files .env .env.development.local .env.test.local .env.production.local .env.local # parcel-bundler cache (https://parceljs.org/) .cache .parcel-cache # Next.js build output .next out # Nuxt.js build / generate output .nuxt dist # Gatsby files .cache/ # Comment in the public line in if your project uses Gatsby and not Next.js # https://nextjs.org/blog/next-9-1#public-directory-support # public # vuepress build output .vuepress/dist # vuepress v2.x temp and cache directory .temp .cache # Docusaurus cache and generated files .docusaurus # Serverless directories .serverless/ # FuseBox cache .fusebox/ # DynamoDB Local files .dynamodb/ # TernJS port file .tern-port # Stores VSCode versions used for testing VSCode extensions .vscode-test # yarn v2 .yarn/cache .yarn/unplugged .yarn/build-state.yml .yarn/install-state.gz .pnp.* ================================================ FILE: README.md ================================================ # processing-large-reports-in-the-browser ## About This is the examples shown in my live screen about [Performance and multithreading in browsers + ECMAScript Modules in Web Workers (pt-br)](https://www.youtube.com/live/-wXPxJYhZeI?feature=share) First of all, leave your star 🌟 on this repo. Access our [**exclusive telegram channel**](https://bit.ly/canalerickwendel) so I'll let you know about all the content I've been producing ## Complete source code - Access it in [app](./recorded/) - Checkout the [live demo](https://erickwendel.github.io/processing-large-reports-in-the-browser/recorded) (preferentially on Chrome browser) - Upload the file [database-small.csv](./recorded/assets/database-small.csv) or download the [SLC_Police_Calls_2013__2016_cleaned_geocoded.csv(210.83 MB)](https://www.kaggle.com/datasets/foenix/slc-crime?select=SLC_Police_Calls_2013__2016_cleaned_geocoded.csv)on Kaggle. ## Have fun! ![Erick_Wendel_-_Thumbnail_Performance_e_multithreading](https://user-images.githubusercontent.com/8060102/221061262-f9425071-0080-48ff-b69c-55d9036937ba.jpg) ## References: - https://caniuse.com/?search=workers ================================================ FILE: preclass/annotations.txt ================================================ https://www.kaggle.com/datasets/foenix/slc-crime?select=SLC_Police_Calls_2013__2016_cleaned_geocoded.csv npm init -y npm i -D browser-sync@2.27.11 package.json start: browser-sync -w touch index.html all html mkdir src touch view -> falar que é tudo que é api de tela export default class View {} service -> tudo que é regra de negocio export default class Service {} worker -> tudo que vai rodar em segundo plano no browser console.log(`I'm ready`) onmessage = (msg) => console.log('hello from worker') controller -> o intermedio entre as camadas export default class Controller {} index -> factory, inicializacao das intancias all index.html import index.js -> should see a message on worker view setFileSize configureOnFileChange controller constructor all configureOnFileChange init configureOnFileChange -> should show raw bytes formatBytes all configureOnFileChange add formatBytes -> should show bytes view configureOnFormSubmit all controller configureOnFormSubmit init configureOnFormSubmit log form > after submit should log results view isWorkerEnabled updateProgress controller configureOnFormSubmit isWorkerEnabled only the if -> should receive a message on worker controller #events progress all onOcurrenceUpdate only log results worker postMessage({ eventType: 'progress', 100 }) postMessage({ eventType: 'ocurrenceUpdate', }) -> progress should be at 100% and ocurrenceUpdate should be logged service processFile only signature worker all service file.stream .pipeThrough(new TextDecoderStream()) .pipeTo(new WritableStream({ write(chunk) { console.log(' chunk', chunk) } })) > should print file contents service #csvToJSON (linesLength) all without progress > writable should print results as json service #setupProgress all processFile progressFn csvToJSON progressFn > should print progress correctly service #findOcurrencies all controller #events ocurrenceUpdate all > all application should work > try uploading with the big file and show that only measuring by the break line is not enough, thats why the linesLength and ocurrences fail on big files - task for you to figure out how to handle this ================================================ FILE: preclass/demo/assets/database-small.csv ================================================ case,case,date cleared,call description,location,police zone,police grid,city council,x-coordinate,y-coordinate,x_gps_coords,y_gps_coords SL2016463,SL2016463,01/01/2016 12:00:00 AM,VEHICLE THEFT,5XX S 900 E ,Z5,151,4.0,1899098.0,883261.0,-111.86496146662066,40.757620896145816 SL2016117,SL2016117,01/01/2016 12:00:00 AM,DOMESTIC/PHYSICAL INVESTIGATION,3XX W 300 S ,Z3,132,4.0,1889023.0,885223.0,-111.90135834600376,40.76288754694132 SL2016427,SL2016427,01/01/2016 12:00:00 AM,TRESPASSING/UNWANTED - IND/SMALL GROUP,9XX N CHURCHILL DR ,Z3,131,3.0,1891429.0,895330.0,-111.89283367183442,40.790656783850956 SL2016253,SL2016253,01/01/2016 12:00:00 AM,SUICIDE - THREAT,8XX W 300 N ,Z1,112,2.0,1885076.0,889825.0,-111.91568312403221,40.77546891450854 SL20162,SL20162,01/01/2016 12:00:00 AM,HOLD LOG,1XX S 200 W ,Z3,132,4.0,1890028.0,886559.0,-111.89775203515366,40.76656669982441 SL2016451,SL2016451,01/01/2016 12:00:00 AM,DUI DRIVER,2XX E 700 S ,Z3,135,4.0,1893682.0,881986.0,-111.8844904059753,40.754059238407514 SL20164,SL20164,01/01/2016 12:00:00 AM,HOLD LOG,6XX E 500 S ,Z4,144,4.0,1897180.0,883741.0,-111.8718915856916,40.75891666684364 SL20167,SL20167,01/01/2016 12:00:00 AM,HOLD LOG,1XX S MAIN ST ,Z3,133,4.0,1891897.0,886421.0,-111.89100292664365,40.766210538859816 SL20168,SL20168,01/01/2016 12:00:00 AM,PHYSICAL (FIGHT) INDIVIDUAL/SMALL GROUP,16XX W 700 N ,Z1,112,1.0,1878776.0,893043.0,-111.93848548107479,40.784218588076925 SL20169,SL20169,01/01/2016 12:00:00 AM,TRAFFIC STOP,1XX E 100 S ,Z3,133,4.0,1892875.0,886743.0,-111.88747748637616,40.767105984796935 SL201610,SL201610,01/01/2016 12:00:00 AM,TRAFFIC STOP,,113,,,,,, SL20166,SL20166,01/01/2016 12:00:00 AM,CELL 911 HANGUP,8XX W 100 S ,Z2,121,2.0,1885393.0,886772.0,-111.91448753269357,40.76709359244312 SL201615,SL201615,01/01/2016 12:00:00 AM,HOLD LOG,1XX S MAIN ST ,Z3,133,4.0,1891897.0,886421.0,-111.89100292664365,40.766210538859816 SL201617,SL201617,01/01/2016 12:00:00 AM,HOLD LOG,1XX S 200 W ,Z3,132,4.0,1890028.0,886559.0,-111.89775203515366,40.76656669982441 SL201618,SL201618,01/01/2016 12:00:00 AM,HOLD LOG,0XX W 300 S ,Z3,133,4.0,1891718.0,885293.0,-111.89163126027671,40.763112433818634 SL201623,SL201623,01/01/2016 12:00:00 AM,HOLD LOG,,SOSL,SOSL,,1887422.0,869203.0,-111.9068745235229,40.71889818951389 SL201611,SL201611,01/01/2016 12:00:00 AM,TRAFFIC STOP,3XX W PAXTON AVE ,Z5,152,5.0,1889383.0,878176.0,-111.89994509279057,40.74355040073993 SL201621,SL201621,01/01/2016 12:00:00 AM,911 HANGUP CALL,1XX E WESTMINSTER AVE ,Z6,161,5.0,1893124.0,872787.0,-111.88636102134953,40.72880447201603 SL201622,SL201622,01/01/2016 12:00:00 AM,HOLD LOG,2XX N 2200 W ,Z1,113,1.0,1874833.0,889644.0,-111.95266088662085,40.774835770400095 ================================================ FILE: preclass/demo/index.html ================================================ Document
0%
================================================ FILE: preclass/demo/package.json ================================================ { "devDependencies": { "browser-sync": "^2.27.11" }, "name": "live-workers-report", "version": "0.0.1", "main": "index.js", "dependencies": { "accepts": "^1.3.8", "ansi-regex": "^2.1.1", "ansi-styles": "^2.2.1", "async": "^2.6.4", "anymatch": "^3.1.3", "async-each-series": "^0.1.1", "batch": "^0.6.1", "base64id": "^2.0.0", "brace-expansion": "^1.1.11", "balanced-match": "^1.0.2", "binary-extensions": "^2.2.0", "browser-sync-ui": "^2.27.11", "braces": "^3.0.2", "bs-recipes": "^1.3.4", "bs-snippet-injector": "^2.0.1", "browser-sync-client": "^2.27.11", "axios": "^0.21.4", "bytes": "^3.1.2", "chokidar": "^3.5.3", "color-convert": "^2.0.1", "chalk": "^1.1.3", "color-name": "^1.1.4", "cliui": "^8.0.1", "commander": "^2.20.3", "connect-history-api-fallback": "^1.6.0", "cookie": "^0.4.2", "call-bind": "^1.0.2", "debug": "^2.6.9", "cors": "^2.8.5", "depd": "^2.0.0", "easy-extender": "^2.3.4", "destroy": "^1.0.4", "concat-map": "^0.0.1", "dlv": "^1.1.3", "connect": "^3.6.6", "eazy-logger": "^3.1.0", "encodeurl": "^1.0.2", "ee-first": "^1.1.1", "engine.io-parser": "^5.0.6", "engine.io": "^6.4.1", "dev-ip": "^1.0.1", "engine.io-client": "^6.4.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "emoji-regex": "^8.0.0", "escalade": "^3.1.1", "escape-string-regexp": "^1.0.5", "fill-range": "^7.0.1", "eventemitter3": "^4.0.7", "finalhandler": "^1.1.0", "fs-extra": "^3.0.1", "fresh": "^0.5.2", "fsevents": "^2.3.2", "function-bind": "^1.1.1", "get-intrinsic": "^1.2.0", "follow-redirects": "^1.15.2", "get-caller-file": "^2.0.5", "glob-parent": "^5.1.2", "graceful-fs": "^4.2.10", "has": "^1.0.3", "has-ansi": "^2.0.0", "http-errors": "^2.0.0", "iconv-lite": "^0.4.24", "has-symbols": "^1.0.3", "immutable": "^3.8.2", "http-proxy": "^1.18.1", "is-binary-path": "^2.1.0", "inherits": "^2.0.4", "is-extglob": "^2.1.1", "is-fullwidth-code-point": "^3.0.0", "is-number": "^7.0.0", "is-glob": "^4.0.3", "is-number-like": "^1.0.8", "is-wsl": "^1.1.0", "jsonfile": "^3.0.1", "lodash": "^4.17.21", "lodash.isfinite": "^3.3.2", "localtunnel": "^2.0.2", "limiter": "^1.1.5", "mime-db": "^1.52.0", "mime": "^1.4.1", "mime-types": "^2.1.35", "minimatch": "^3.1.2", "micromatch": "^4.0.5", "ms": "^2.0.0", "mitt": "^1.2.0", "negotiator": "^0.6.3", "normalize-path": "^3.0.0", "object-assign": "^4.1.1", "object-inspect": "^1.12.3", "on-finished": "^2.3.0", "openurl": "^1.1.1", "opn": "^5.3.0", "parseurl": "^1.3.3", "picomatch": "^2.3.1", "portscanner": "^2.2.0", "qs": "^6.11.0", "raw-body": "^2.5.2", "range-parser": "^1.2.1", "readdirp": "^3.6.0", "require-directory": "^2.1.1", "requires-port": "^1.0.0", "resp-modifier": "^6.0.2", "rx": "^4.1.0", "rxjs": "^5.5.12", "send": "^0.16.2", "safer-buffer": "^2.1.2", "serve-static": "^1.13.2", "serve-index": "^1.9.1", "server-destroy": "^1.0.1", "setprototypeof": "^1.2.0", "side-channel": "^1.0.4", "socket.io-adapter": "^2.5.2", "socket.io-client": "^4.6.1", "statuses": "^1.3.1", "socket.io-parser": "^4.2.2", "socket.io": "^4.6.1", "strip-ansi": "^3.0.1", "supports-color": "^2.0.0", "tfunk": "^4.0.0", "string-width": "^4.2.3", "to-regex-range": "^5.0.1", "stream-throttle": "^0.1.3", "toidentifier": "^1.0.1", "ua-parser-js": "^1.0.2", "symbol-observable": "^1.0.1", "typescript": "^4.9.5", "vary": "^1.1.2", "unpipe": "^1.0.0", "universalify": "^0.1.2", "utils-merge": "^1.0.1", "wrap-ansi": "^7.0.0", "ws": "^8.11.0", "y18n": "^5.0.8", "xmlhttprequest-ssl": "^2.0.0", "yargs": "^17.7.1", "yargs-parser": "^21.1.1" }, "scripts": { "start": "npx browser-sync -w", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], "author": "", "license": "ISC", "description": "" } ================================================ FILE: preclass/demo/src/controller.js ================================================ export default class Controller { #view #service #worker #events = { alive: () => { }, ocurrenceUpdate: ({ found, linesLength, took }) => { const [[key, value]] = Object.entries(found) this.#view.updateDebugLog(`found ${value} ocurrencies of ${key} - over ${linesLength} lines - took: ${took}`) }, progress: ({ total }) => { this.#view.updateProgress(total) } } constructor({ view, service, worker }) { this.#service = service this.#view = view this.#worker = this.#configureWorker(worker) } static init(deps) { const controller = new Controller(deps) return controller.init() } init() { this.#view.configureOnFileChange(this.#configureOnFileChange.bind(this)) this.#view.configureOnFormSubmit(this.#configureOnFormSubmit.bind(this)) } #configureWorker(worker) { worker.onmessage = ({ data }) => this.#events[data.eventType](data) return worker } #formatBytes(bytes) { const units = ['B', 'KB', 'MB', 'GB', 'TB'] let i = 0 for (i; bytes >= 1024 && i < 4; i++) { bytes /= 1024 } return `${bytes.toFixed(2)} ${units[i]}` } #configureOnFileChange(file) { const size = file.size this.#view.setFileSize(this.#formatBytes(size)) } #configureOnFormSubmit({ description, file }) { const query = {} query["call description"] = new RegExp(description, 'i') if (this.#view.isWorkerEnabled()) { console.log('executing on worker thread!') this.#worker.postMessage({ query, file }) return } console.log('executing on main thread!') // sem worker this.#service.processFile({ query, file, onProgress: (total) => { this.#events.progress({ total }) }, onOcurrenceUpdate: (...args) => { this.#events.ocurrenceUpdate(...args) } }) } } ================================================ FILE: preclass/demo/src/index.js ================================================ import Controller from "./controller.js" import Service from "./service.js" import View from "./view.js" // somente no chrome por enquanto const worker = new Worker('./src/worker.js', { type: "module"}) Controller.init({ view: new View(), service: new Service(), worker, }) ================================================ FILE: preclass/demo/src/service.js ================================================ export default class Service { #setupProgress(totalBytes, onProgress) { let totalUploaded = 0 onProgress(0) return (chunkLength) => { totalUploaded += chunkLength const total = 100 / totalBytes * totalUploaded onProgress(total) } } processFile({ query, file, onOcurrenceUpdate, onProgress }) { const startedAt = performance.now() const elapsed = () => `${(Math.round(performance.now() - startedAt) / 1000)} secs` const progressFn = this.#setupProgress(file.size, onProgress) const linesLength = { counter: 0 } const onUpdate = () => { return (found) => { onOcurrenceUpdate({ found, took: elapsed(), linesLength: linesLength.counter, }) } } file.stream() .pipeThrough(new TextDecoderStream()) .pipeThrough(this.#csvToJSON({ progressFn, linesLength })) .pipeTo(this.#findOcurrencies({ query, onUpdate: onUpdate() })) } #findOcurrencies({ query, onUpdate }) { const queryKeys = Object.keys(query) let found = {} return new WritableStream({ write(chunk) { for (const keyIndex in queryKeys) { const key = queryKeys[keyIndex] const queryValue = query[key] found[queryValue] = found[queryValue] ?? 0 if (queryValue.test(chunk[key])) { found[queryValue]++ onUpdate(found) } } }, close: () => onUpdate(found) }) } #csvToJSON({ linesLength, progressFn }) { let columns = [] return new TransformStream({ transform(chunk, controller) { progressFn(chunk.length) const lines = chunk.split('\n') linesLength.counter += lines.length if (!columns.length) { const firstLine = lines.shift() columns = firstLine.split(',') linesLength.counter-- } for (const line of lines) { if (!line.length) continue const currentColumns = line.split(',') let currentItem = {} for (const columIndex in currentColumns) { const columnItem = currentColumns[columIndex] currentItem[columns[columIndex]] = columnItem.trimEnd() } controller.enqueue(currentItem) } }, }) } } ================================================ FILE: preclass/demo/src/view.js ================================================ export default class View { #debugElement = document.getElementById('debug') #fileSizeElement = document.getElementById('file-size') #processingProgress = document.getElementById('progress') #csvFile = document.querySelector('#csv-file') #form = document.querySelector('#form') #workerChecker = document.querySelector('#worker') configureOnFileChange(fn) { this.#csvFile.addEventListener('change', e => { fn(e.target.files[0]) }) } setFileSize(size) { this.#fileSizeElement.innerText = `File size: ${size}\n` } isWorkerEnabled() { return this.#workerChecker.checked } updateProgress(value) { this.#processingProgress.value = value } configureOnFormSubmit(fn) { this.#form.reset() this.#form.addEventListener('submit', (e) => { e.preventDefault() const file = this.#csvFile.files[0] if (!file) { alert('Please select a file') return } this.updateDebugLog("") const form = new FormData(e.currentTarget) const description = form.get('description') fn({ description, file }) }) } updateDebugLog(text, reset = true) { if (reset) { this.#debugElement.innerText = text return } this.#debugElement.innerText += text } } ================================================ FILE: preclass/demo/src/worker.js ================================================ import Service from './service.js' const service = new Service() console.log(`I'm alive!`) postMessage({ eventType: 'alive' }) onmessage = ({ data }) => { const { query, file } = data service.processFile({ query, file, onProgress: (total) => postMessage({ eventType: 'progress', total }), onOcurrenceUpdate: (args) => { postMessage({ eventType: 'ocurrenceUpdate', ...args }) } }) } ================================================ FILE: preclass/prova-real/index.js ================================================ import { createReadStream } from 'node:fs' import csvtojson from 'csvtojson' import { pipeline } from 'node:stream/promises' import { Writable } from 'node:stream' let lines = 0 let ocurrences = 0 const word = process.argv[2]?.trim() console.time('search') console.log('searching for...', word) await pipeline( createReadStream('./database.csv'), csvtojson(), new Writable({ write(chunk, enc, cb) { lines++ if (new RegExp(word, 'i').test(JSON.parse(chunk)['call description'])) { ocurrences++ } cb() } }) ) console.log({ lines, ocurrences }) console.timeEnd('search') ================================================ FILE: preclass/prova-real/package.json ================================================ { "name": "app", "version": "0.0.1", "description": "", "main": "index.js", "type": "module", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], "author": "", "license": "ISC", "dependencies": { "csvtojson": "^2.0.10" } } ================================================ FILE: recorded/.vscode/settings.json ================================================ { "window.zoomLevel": 3 } ================================================ FILE: recorded/assets/database-small.csv ================================================ case,case,date cleared,call description,location,police zone,police grid,city council,x-coordinate,y-coordinate,x_gps_coords,y_gps_coords SL2016463,SL2016463,01/01/2016 12:00:00 AM,VEHICLE THEFT,5XX S 900 E ,Z5,151,4.0,1899098.0,883261.0,-111.86496146662066,40.757620896145816 SL2016117,SL2016117,01/01/2016 12:00:00 AM,DOMESTIC/PHYSICAL INVESTIGATION,3XX W 300 S ,Z3,132,4.0,1889023.0,885223.0,-111.90135834600376,40.76288754694132 SL2016427,SL2016427,01/01/2016 12:00:00 AM,TRESPASSING/UNWANTED - IND/SMALL GROUP,9XX N CHURCHILL DR ,Z3,131,3.0,1891429.0,895330.0,-111.89283367183442,40.790656783850956 SL2016253,SL2016253,01/01/2016 12:00:00 AM,SUICIDE - THREAT,8XX W 300 N ,Z1,112,2.0,1885076.0,889825.0,-111.91568312403221,40.77546891450854 SL20162,SL20162,01/01/2016 12:00:00 AM,HOLD LOG,1XX S 200 W ,Z3,132,4.0,1890028.0,886559.0,-111.89775203515366,40.76656669982441 SL2016451,SL2016451,01/01/2016 12:00:00 AM,DUI DRIVER,2XX E 700 S ,Z3,135,4.0,1893682.0,881986.0,-111.8844904059753,40.754059238407514 SL20164,SL20164,01/01/2016 12:00:00 AM,HOLD LOG,6XX E 500 S ,Z4,144,4.0,1897180.0,883741.0,-111.8718915856916,40.75891666684364 SL20167,SL20167,01/01/2016 12:00:00 AM,HOLD LOG,1XX S MAIN ST ,Z3,133,4.0,1891897.0,886421.0,-111.89100292664365,40.766210538859816 SL20168,SL20168,01/01/2016 12:00:00 AM,PHYSICAL (FIGHT) INDIVIDUAL/SMALL GROUP,16XX W 700 N ,Z1,112,1.0,1878776.0,893043.0,-111.93848548107479,40.784218588076925 SL20169,SL20169,01/01/2016 12:00:00 AM,TRAFFIC STOP,1XX E 100 S ,Z3,133,4.0,1892875.0,886743.0,-111.88747748637616,40.767105984796935 SL201610,SL201610,01/01/2016 12:00:00 AM,TRAFFIC STOP,,113,,,,,, SL20166,SL20166,01/01/2016 12:00:00 AM,CELL 911 HANGUP,8XX W 100 S ,Z2,121,2.0,1885393.0,886772.0,-111.91448753269357,40.76709359244312 SL201615,SL201615,01/01/2016 12:00:00 AM,HOLD LOG,1XX S MAIN ST ,Z3,133,4.0,1891897.0,886421.0,-111.89100292664365,40.766210538859816 SL201617,SL201617,01/01/2016 12:00:00 AM,HOLD LOG,1XX S 200 W ,Z3,132,4.0,1890028.0,886559.0,-111.89775203515366,40.76656669982441 SL201618,SL201618,01/01/2016 12:00:00 AM,HOLD LOG,0XX W 300 S ,Z3,133,4.0,1891718.0,885293.0,-111.89163126027671,40.763112433818634 SL201623,SL201623,01/01/2016 12:00:00 AM,HOLD LOG,,SOSL,SOSL,,1887422.0,869203.0,-111.9068745235229,40.71889818951389 SL201611,SL201611,01/01/2016 12:00:00 AM,TRAFFIC STOP,3XX W PAXTON AVE ,Z5,152,5.0,1889383.0,878176.0,-111.89994509279057,40.74355040073993 SL201621,SL201621,01/01/2016 12:00:00 AM,911 HANGUP CALL,1XX E WESTMINSTER AVE ,Z6,161,5.0,1893124.0,872787.0,-111.88636102134953,40.72880447201603 SL201622,SL201622,01/01/2016 12:00:00 AM,HOLD LOG,2XX N 2200 W ,Z1,113,1.0,1874833.0,889644.0,-111.95266088662085,40.774835770400095 ================================================ FILE: recorded/index.html ================================================ Document
0%
================================================ FILE: recorded/package.json ================================================ { "name": "recorded", "version": "0.0.1", "description": "", "main": "index.js", "scripts": { "start": "npx browser-sync -w" }, "keywords": [], "author": "", "license": "ISC", "devDependencies": { "browser-sync": "^2.27.11" } } ================================================ FILE: recorded/src/controller.js ================================================ export default class Controller { #view #worker #service #events = { alive: () => { }, progress: ({ total }) => { this.#view.updateProgress(total) }, ocurrenceUpdate: ({ found, linesLength, took }) => { const [[key, value]] = Object.entries(found) this.#view.updateDebugLog( `found ${value} ocurrencies of ${key} - over ${linesLength} lines - took: ${took}` ) } } constructor({ view, worker, service }) { this.#view = view this.#service = service this.#worker = this.#configureWorker(worker) } static init(deps) { const controller = new Controller(deps) controller.init() return controller } init() { this.#view.configureOnFileChange( this.#configureOnFileChange.bind(this) ) this.#view.configureOnFormSubmit( this.#configureOnFormSubmit.bind(this) ) } #configureWorker(worker) { worker.onmessage = ({ data }) => this.#events[data.eventType](data) return worker } #formatBytes(bytes) { const units = ['B', 'KB', 'MB', 'GB', 'TB'] let i = 0 for (i; bytes >= 1024 && i < 4; i++) { bytes /= 1024 } return `${bytes.toFixed(2)} ${units[i]}` } #configureOnFileChange(file) { this.#view.setFileSize( this.#formatBytes(file.size) ) } #configureOnFormSubmit({ description, file }) { const query = {} query['call description'] = new RegExp( description, 'i' ) if (this.#view.isWorkerEnabled()) { console.log('executing on worker thread!') this.#worker.postMessage({ query, file }) return } console.log('executing on main thread!') this.#service.processFile({ query, file, onProgress: (total) => { this.#events.progress({ total }) }, onOcurrenceUpdate: (...args) => { this.#events.ocurrenceUpdate(...args) } }) } } ================================================ FILE: recorded/src/index.js ================================================ import Controller from "./controller.js" import Service from "./service.js" import View from "./view.js" // worker modules só funciona no chrome por enquanto // ou seja worker funciona, mas import/export nao const worker = new Worker('./src/worker.js', { type: "module" }) Controller.init({ view: new View(), service: new Service(), worker }) ================================================ FILE: recorded/src/service.js ================================================ export default class Service { processFile({ query, file, onOcurrenceUpdate, onProgress }) { const linesLength = { counter: 0 } const progressFn = this.#setupProgress(file.size, onProgress) const startedAt = performance.now() const elapsed = () => `${((performance.now() - startedAt) / 1000).toFixed(2)} secs` const onUpdate = () => { return (found) => { onOcurrenceUpdate({ found, took: elapsed(), linesLength: linesLength.counter }) } } file.stream() .pipeThrough(new TextDecoderStream()) .pipeThrough(this.#csvToJSON({ linesLength, progressFn })) .pipeTo(this.#findOcurrencies({ query, onOcurrenceUpdate: onUpdate() })) // .pipeTo(new WritableStream({ // write(chunk) { // console.log('chunk', chunk) // } // })) } #csvToJSON({ linesLength, progressFn }) { let columns = [] return new TransformStream({ transform(chunk, controller) { progressFn(chunk.length) const lines = chunk.split('\n') linesLength.counter += lines.length if (!columns.length) { const firstLine = lines.shift() columns = firstLine.split(',') linesLength.counter-- } for (const line of lines) { if (!line.length) continue let currentItem = {} const currentColumsItems = line.split(',') for (const columnIndex in currentColumsItems) { const columnItem = currentColumsItems[columnIndex] currentItem[columns[columnIndex]] = columnItem.trimEnd() } controller.enqueue(currentItem) } } }) } #findOcurrencies({ query, onOcurrenceUpdate }) { const queryKeys = Object.keys(query) let found = {} return new WritableStream({ write(jsonLine) { for (const keyIndex in queryKeys) { const key = queryKeys[keyIndex] const queryValue = query[key] found[queryValue] = found[queryValue] ?? 0 if (queryValue.test(jsonLine[key])) { found[queryValue]++ onOcurrenceUpdate(found) } } }, close: () => onOcurrenceUpdate(found) }) } #setupProgress(totalBytes, onProgress) { let totalUploaded = 0 onProgress(0) return (chunkLength) => { totalUploaded += chunkLength const total = 100 / totalBytes * totalUploaded onProgress(total) } } } ================================================ FILE: recorded/src/view.js ================================================ export default class View { #csvFile = document.querySelector('#csv-file') #fileSize = document.querySelector('#file-size') #form = document.querySelector('#form') #debug = document.querySelector('#debug') #progress = document.querySelector('#progress') #worker = document.querySelector('#worker') setFileSize(size) { this.#fileSize.innerText = `File size: ${size}\n` } configureOnFileChange(fn) { this.#csvFile.addEventListener('change', e => { fn(e.target.files[0]) }) } configureOnFormSubmit(fn) { this.#form.reset() this.#form.addEventListener('submit', (e) => { e.preventDefault() const file = this.#csvFile.files[0] // isso aqui deveria estar na controller if(!file) { alert('Please select a file!') return } this.updateDebugLog("") const form = new FormData(e.currentTarget) const description = form.get('description') fn({ description, file }) }) } updateDebugLog(text, reset = true) { if(reset) { this.#debug.innerText = text return; } this.#debug.innerText += text } updateProgress(value) { this.#progress.value = value } isWorkerEnabled() { return this.#worker.checked } } ================================================ FILE: recorded/src/worker.js ================================================ import Service from './service.js' console.log(`I'm alive!`) const service = new Service() postMessage({ eventType: 'alive' }) onmessage = ({ data }) => { const { query, file } = data service.processFile({ query, file, onOcurrenceUpdate: (args) => { postMessage({ eventType: 'ocurrenceUpdate', ...args }) }, onProgress: (total) => postMessage({ eventType: 'progress', total }) }) }