Repository: Estheryu991/SwissGerman_Dictionary Branch: main Commit: 2aaed151241d Files: 125 Total size: 1.8 MB Directory structure: gitextract_ywkx76s7/ ├── .eslintrc ├── .github/ │ ├── FUNDING.yml │ └── workflows/ │ └── intro.yml ├── .gitignore ├── .prettier ├── Cloud/ │ └── Speech-to-text/ │ └── README.md ├── Datensätze/ │ └── Bern/ │ └── Sandkeusch.py ├── Docs/ │ ├── Sound/ │ │ ├── sound.ogg │ │ └── untterregle.ogg │ └── images/ │ └── untterregle.ogg ├── Geschichte/ │ ├── EIDGENOSSENSCHAFT.md │ └── geschichte.md ├── Gewaltenteilung/ │ └── Gewaltenteilung_Im_Rechtsstaat.md ├── Install ├── LICENSE ├── Lied/ │ ├── .gitignore │ ├── Geburi.md │ ├── Patent Ochsner/ │ │ └── Für immer uf di.md │ ├── cake.less │ └── loading.css ├── NLP/ │ └── Examples/ │ └── one/ │ ├── assignment.md │ ├── cl.md │ ├── example.py │ └── number.py ├── Notebook/ │ ├── Intro.ipynb │ ├── K_mean_clustering.ipynb │ └── text_data.ipynb ├── README.md ├── SwissGermanBot/ │ └── login.txt ├── datasets/ │ ├── Buendnertuetsch/ │ │ └── buendnertuetsch.txt │ ├── Jugendsprache/ │ │ └── jugendsprache.txt │ ├── bern/ │ │ ├── W Nuss Vo Buempliz.py │ │ └── bern.txt │ ├── dictionary.py │ ├── english/ │ │ └── festivalLove-eng.ipynb │ ├── french/ │ │ └── french.txt │ ├── img/ │ │ └── map.txt │ ├── korean/ │ │ ├── festivalLovekr.ipynb │ │ ├── high-kr.py │ │ └── malLuege.py │ ├── luzern/ │ │ └── luzern.txt │ ├── zuerich/ │ │ ├── BlueBike.txt │ │ ├── FestivalLove.ipynb │ │ ├── JohnJuno.ipynb │ │ ├── Sie het en traum gha.txt │ │ ├── Teil_Vo_Dir.ipynb │ │ ├── fremdi_fürenand.ipynb │ │ ├── high.py │ │ ├── i_ha's_immer_gwüsst.ipynb │ │ ├── lena.ipynb │ │ ├── malleuge.txt │ │ ├── panama.txt │ │ ├── teleskop.py │ │ ├── valencia.ipynb │ │ ├── zuerich.txt │ │ ├── zuerichII.txt │ │ └── zuerichbs.txt │ └── zug/ │ └── zug.txt ├── dist/ │ ├── siriwave.esm.js │ ├── siriwave.umd.js │ └── typ/ │ ├── curve.d.ts │ ├── index.d.ts │ └── ios.d.ts ├── facts/ │ ├── ASR/ │ │ ├── DeepSpeech.md │ │ ├── data/ │ │ │ └── alphabet.txt │ │ └── pre-processing/ │ │ ├── audio.py │ │ ├── ch_De_prep.py │ │ ├── corpus_cleaning.py │ │ ├── finden_files.py │ │ ├── folien.py │ │ ├── prep_vocab.py │ │ ├── prepare_data.py │ │ ├── shuffle_split.py │ │ └── utf_8.sh │ ├── GAN.md │ ├── chrütertee.java │ ├── semantiks.md │ └── word2vec.md ├── src/ │ ├── context.ts │ ├── ios9_curve.ts │ ├── klassich.ts │ ├── pramp/ │ │ └── algodaten/ │ │ ├── DecodeVariations/ │ │ │ ├── answer.java │ │ │ ├── answer.txt │ │ │ ├── answerII.py │ │ │ ├── hints.txt │ │ │ └── question.txt │ │ ├── Diff Between Two Strings/ │ │ │ ├── answer.py │ │ │ └── question.txt │ │ ├── Island Count/ │ │ │ ├── answer.c │ │ │ ├── hints.txt │ │ │ └── question.txt │ │ ├── Number of Paths/ │ │ │ ├── ans.py │ │ │ ├── answer.txt │ │ │ ├── hints.txt │ │ │ └── question.txt │ │ ├── Number_Of_Paths/ │ │ │ ├── answer.txt │ │ │ ├── hints.txt │ │ │ └── question.txt │ │ ├── Root of Number/ │ │ │ ├── ans.c │ │ │ ├── ans.java │ │ │ ├── answer.txt │ │ │ ├── hints.txt │ │ │ └── question.txt │ │ ├── Root_of_Number/ │ │ │ ├── hints.txt │ │ │ ├── problem.txt │ │ │ └── pseudocode.py │ │ └── pairs_with_specific_difference/ │ │ └── answer.py │ ├── read-Swiss-German.py │ └── srf/ │ ├── comedy/ │ │ └── renatoKaiser.txt │ └── dok/ │ ├── Kampf gegen die Suchttxt │ └── KampfSucht.txt ├── swissDeutschBot/ │ ├── .gitignore │ ├── .idee/ │ │ └── Inspektion/ │ │ ├── .gitignore │ │ ├── GPT3api.iml │ │ ├── miscellanious.xml │ │ ├── modules.xml │ │ ├── profil_einstellungen.xml │ │ ├── projeckt_vorgaben.xml │ │ ├── requirements.md │ │ └── vcs.xml │ ├── Dockerfile │ ├── mini.py │ └── openAI.py └── trials/ ├── bünzli.md └── quiz.md ================================================ FILE CONTENTS ================================================ ================================================ FILE: .eslintrc ================================================ { "parser": "@typescript-eslint/parser", "extends": [ "plugin:@typescript-eslint/recommended", // Uses the recommended rules from the @typescript-eslint/eslint-plugin "prettier/@typescript-eslint", // Uses eslint-config-prettier to disable ESLint rules from @typescript-eslint/eslint-plugin that would conflict with prettier "plugin:prettier/recommended" // Enables eslint-plugin-prettier and eslint-config-prettier. This will display prettier errors as ESLint errors. Make sure this is always the last configuration in the extends array.], ], "parserOptions": { "ecmaVersion": 2018, "sourceType": "module" }, "plugins": ["prettier"], "env": { "browser": true } } ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms Wir danken Ihnen für Ihre Hilfe. github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] ================================================ FILE: .github/workflows/intro.yml ================================================ # Dieser Workflow führt Tests mit node aus und veröffentlicht dann ein Paket in GitHub Packages, wenn eine Version erstellt wird # luege https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages name: Node.js Package on: release: types: [created] jobs: publish-npm: needs: build runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: node-version: 12 registry-url: https://registry.npmjs.org/ - run: yarn install - run: yarn build - run: yarn publish env: NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} publish-gpr: needs: build runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: node-version: 12 registry-url: https://npm.pkg.github.com/ - run: yarn install - run: yarn build - run: yarn publish env: NODE_AUTH_TOKEN: ${{secrets.GH_PKG_TOKEN}} ================================================ FILE: .gitignore ================================================ node_modules/ yarn-error.log ================================================ FILE: .prettier ================================================ { "semi": true, "trailingComma": "all", "singleQuote": false, "printWidth": 120, "tabWidth": 2 } ================================================ FILE: Cloud/Speech-to-text/README.md ================================================ # Step 1: Choose [Awesome-Swiss-German Project](https://cloud.google.com/speech-to-text) # Step 2: Enable the Speech-to-Text API You'll be using the Speech-to-Text API in this project. Enable it by clicking Enable APIs: Unable to enable required APIs speech.googleapis.com When you see a green checkmark next to all of the APIs listed above, click Next. # Step 3:Open Cloud Shell Cloud Shell is a built-in command-line tool for the console. In this tutorial, you use Cloud Shell to deploy your app. Open Cloud Shell by clicking Activate Cloud Shell. # Step 4: Create an application To create a new Python application, run the following command: ` mkdir speech-to-text-python && \ touch \ speech-to-text-python/app.py ` To set the speech-to-text-python directory as your Cloud Shell workspace and open the Cloud Shell Editor, run the following command: cd speech-to-text-python cloudshell open-workspace . To set your project ID in the Cloud Shell terminal, run the following command: export PROJECT_ID= To learn how to authenticate requests to this API, click Next. # Step 5 : Set up authentication To call the Speech-to-Text API, your application needs to authenticate its identity to the Speech-to-Text service and obtain authorization to perform tasks. Create a service account to authenticate your API requests: gcloud iam service-accounts create \ speech-to-text-quickstart \ --project Grant your service account the Speech-to-Text API User role: gcloud projects \ add-iam-policy-binding \ --member \ serviceAccount:speech-to-text-quickstart@.iam.gserviceaccount.com \ --role roles/speech.serviceAgent Create a service account key: gcloud iam service-accounts keys \ create speech-to-text-key.json \ --iam-account \ speech-to-text-quickstart@.iam.gserviceaccount.com Set the key as your default credentials: export \ GOOGLE_APPLICATION_CREDENTIALS=speech-to-text-key.json To learn how to call the Speech-to-Text API, click Next. Call the Speech-to-Text API Open `app.py` in the [Cloud Shell Editor](https://cloud.google.com/shell/docs/editor-overview?hl=en-GB) in the Cloud Shell Editor by running the following command in your terminal: ` cloudshell open app.py` Install the Speech-to-Text client library: ` pip3 install --upgrade \ google-cloud-speech` In `app.py`, import the Speech-to-Text client library at the beginning of the file: from google.cloud import speech Create a Speech-to-Text API client and add a variable that points to the path of the example audio file provided: ` # Instantiates a client client = speech.SpeechClient() # The name of the audio file to transcribe gcs_uri = "gs://cloud-samples-data/speech/brooklyn_bridge.raw" ` Add the following code, which transcribes your audio file using the specified configuration, then prints the transcription: ` def transcribe_speech(): audio = speech.RecognitionAudio(uri=gcs_uri) config = speech.RecognitionConfig( encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16, sample_rate_hertz=16000, language_code="en-US", ) # Detects speech in the audio file response = client.recognize(config=config, audio=audio) for result in response.results: print("Transcript: {}".format(result.alternatives[0].transcript)) ` At the end of your app.py file, call transcribeSpeech(): ` transcribe_speech() ` From your terminal, run your application. ` python3 app.py ` You should now see the transcribed message in your terminal: ` Transcript: Wie alt isch die ETH Zurich. ` To avoid incurring charges to your account and learn about next steps, click Next. # Step 7: Clean up Delete the file containing your service account key. ` rm speech-to-text-key.json ` If you created a project specifically for this tutorial, you can delete it on the Google Cloud console Projects page. ================================================ FILE: Datensätze/Bern/Sandkeusch.py ================================================ Damals bin ich jede Tag ist Studio Gange, Gfange i dem Bunker Homie immer wenn ich schriibe Chömet die Erinnerige zrugg Ich weiss no ganz genau die ganzi Nacht, han immer welle blibe Am nächsten Tag go schaffe han immer verschlafe de Chef mich am hasse mer hend emmer müesse striite Doch han d Lehr bestande wärs nach mim Lehrer gange wär ich scho eher gange doch ich han grisse Trotzdem ben ich denn weg gange Min Weg gange Im Weg gstande send alli Doch han sie denn weggschlage Wie Ali, bechunnsch Herzrase vo Panik If ich em Benz fahr dur de Abig Nenn mich Xen Abi, mini Erfahrige zeigt Du machsch ja gern Party und du lengsch alli in scheiss Met dem wo t' hengsch amigs Zu dem wirsch Schlicht Homie Well du Wiis ziehsch Homie Besch de lieblings Homie du weisch Ich vermisse das Sandchaste-Lebe Sorgefrei da mit de Kids vom Nachbar am Henge Alles schiint guet doch es cha sich schlagartig wende Wien Schlag idie Fressi Mit 6i echo ahfange merke 13i ganz Tag am schwänze Sie Herr Lehrernei es esch ned anstrengend zlerne Doch ich han ahfange Rappe D Jugend lebt nachem Yolo Motto bis sie met 18i sterbe Hoff dass sie nur vo mim Sound abhängig werdend Mal unabhängig werdet Anstatt nur unahständig umehenget Ständig einheitlich Ich ha das lebe so satt Nassdoch no chli lebe be lang nonig satt Ich muess ja fast glaube was am mäntig und ziistig bis am friitig i de ziitig so staht, drum fick uf de Staat! ================================================ FILE: Docs/Sound/sound.ogg ================================================ ================================================ FILE: Geschichte/EIDGENOSSENSCHAFT.md ================================================ Im Mittelalter schufen sich die verschiedenen Territorien, die oft auch zusammen arbeiteten, langsam, aber stetig mehr Freiraum von den imperialen Herrschern, deren Untertanen sie nominell lange waren. Die Geschichte der Schweiz wurde stark geprägt von der Reformation und den darauf folgenden Kämpfen und Kriegen zwischen Katholiken und Protestanten. Gegen Ende des 18. Jahrhunderts besetzte das revolutionäre Frankreich die Schweiz und errichtete die Helvetische Republik, einen zentralistischen Staat nach französischem Vorbild. Bald brachen innere Kämpfe aus, französische Truppen kamen erneut in die Schweiz. ================================================ FILE: Geschichte/geschichte.md ================================================ Vorläufer der modernen Schweiz waren die seit dem Ende des 13. Jahrhunderts als lockerer Bund organisierte Alte Eidgenossenschaft, die von 1798 bis 1803 bestehende zentralistisch aufgebaute Helvetische Republik sowie die 1803 gegründete und 1815 neu organisierte «Schweizerische Eidgenossenschaft». ================================================ FILE: Gewaltenteilung/Gewaltenteilung_Im_Rechtsstaat.md ================================================ Der Staat ist in drei Bereiche aufgeteilt: Parlament, Regierung und Gericht. Für jeden Bereich gibt es auf den Ebenen Bund, Kantone und Gemeinden voneinander getrennte Institutionen (Staatsorgane). Gerichte gibt es zusätzlich auf der Ebene Bezirk. # Das Parlament (Legislative) Gesetze erlassen (- Seite 13), Regierung und Verwaltung kontrollieren # Die Regierung (Exekutive) Gesetze ausführen, re- gieren, verwalten, den Staat in der Innen- und Aussenpolitik vertreten # Das Gericht (Judikative, Justiz) Urteilen, richten, strafen, schützen Die Gewaltenteilung auf den Ebenen Bund, Kantone, Gemeinden # Der Bund # Die Schweiz Bundesversammlung Bundesrat Nationalrat: 200 Mitglieder 7 Mitglieder Ständerat: 46 Mitglieder (Wahl durch die Vereinigte (Wahl durch Stimmberechtigte) Bundesversammlung) Bundesgericht 41 Mitglieder (Anzahl variierend) (Wahl durch die Vereinigte Bundesversammlung) # Der Kanton Der Stand Kantonsrat, Grosser Rat oder Landrat (von den Stimmberechtigten des Kantons gewählt) Regierungsrat oder Staatsrat (von den Stimmberechtigten des Kantons gewählt) Kantonsgericht oder Obergericht (vom Kantonsrat, Grossen Rat, Landrat gewählt) # Die Gemeinde\ Die Stadt Gemeinde- oder Gemeinderat oder Stadtparlament Stadtrat (Wahl durch Stimmberechtigte Wahl durch Stimmberechtigte der Gemeinde/Stadt) der Gemeinde/Stadt) Gemeinde- oder Bürgerversammlung (Stimmberechtigte der Gemeinde) Kantonsgericht oder Obergericht (vom Kantonsrat, Grossen Rat, Landrat gewählt) # Bezirk: Amtsgericht, Bezirksgericht, Kreisgericht (Wahl durch Stimmberechtigte des Bezirks/Kreises) Gemeinde: Vermittler/in, Friedensrichter/in (Wahl durch Stimmberechtigte der Gemeinde/Stadt) ================================================ FILE: Install ================================================ conda create -n nlp_lss python=3.8 conda activate nlp_lss pip install spacy==3.2.2 pip install pandas==1.4.1 pip install tqdm==4.62.3 pip install gensim==4.1.2 pip install beautifulsoup4==4.10.0 pip install googletrans==3.0.0 pip install matplotlib==3.5.1 pip install spacytextblob pip install lxml==4.8.0 pip install transformers[torch] pip install unidecode==2.8 pip install seaborn==0.11.2 pip install sklearn==0.24.1 pip install wordcloud==1.8.1 pip install fightin-words==1.0.5 pip install eli5==0.11.0 pip install flair==0.7.0 pip install tensorflow==2.4.1 pip install keras==2.4.3 pip install sentencepiece==0.1.91 pip install scikit-learn-extra==0.1.0b2 pip install pyldavis==3.2.2 pip install hdbscan==0.8.27 pip install pydot # spacy python -m spacy download en_core_web_sm # nltk pip install nltk python -c "import nltk; nltk.download('wordnet')" python -c "import nltk; nltk.download('stopwords')" python -c "import nltk; nltk.download('vader_lexicon')" # install jupyter notebook conda install -c conda-forge jupyterlab sudo apt install graphviz ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2022 Esther Hoeun Yu Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Lied/.gitignore ================================================ # Logs logs *.log npm-debug.log* yarn-debug.log* yarn-error.log* # Runtime data pids *.pid *.seed *.pid.lock # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage # nyc test coverage .nyc_output # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) .grunt # Bower dependency directory (https://bower.io/) bower_components # node-waf configuration .lock-wscript # Compiled binary addons (https://nodejs.org/api/addons.html) build/Release # Dependency directories node_modules/ jspm_packages/ # TypeScript v1 declaration files typings/ # Optional npm cache directory .npm # Optional eslint cache .eslintcache # Optional REPL history .node_repl_history # Output of 'npm pack' *.tgz # Yarn Integrity file .yarn-integrity # dotenv environment variables file .env # parcel-bundler cache (https://parceljs.org/) .cache # next.js build output .next # nuxt.js build output .nuxt # vuepress build output .vuepress/dist # Serverless directories .serverless/ # FuseBox cache .fusebox/ #DynamoDB Local files .dynamodb/ ================================================ FILE: Lied/Geburi.md ================================================ Happy Birthday A Happy Birthday animation design in CSS3, HTML5. URL: http://estheryu991.github.io/geburi/ Technology Used: HTML5 CSS3 jQuery GNU/Linux Digital Ocean as VPS GIMP # Setup ## If you have python installed: ``` cd Geburi ``` && ``` python -m SimpleHTTPServer --port 8081 ``` visit http://localhost:8081 in your browser. ## If you have nodejs installed ``` npm install ``` && ``` npm run server-node ``` visit http://localhost:8081 in your browser. ================================================ FILE: Lied/Patent Ochsner/Für immer uf di.md ================================================ Schweizer Lied findet [hier](https://www.youtube.com/watch?v=-U_awHiE2e0) # Lyrics: >>> "uf mueters seel wo hüt furt isch vo dr ärde uf au die schöne ching, wo hüt z nacht gebore wärde uf au die zyt wo isch vergange, uf au die zyt wo mir no blybt uf au die grüene triebe, uf die zuckersüesse frücht i de böim uf au die grosse plän & uf all die grosse tröim uf au die wo fyre & wo singe, uf au die wo sueche & wo villech sogar finge es glas uf d liebi & eis uf ds voue läbe & eis uf au das, wo mir nid chöi häbe es tor geit uuf & nes angers geit zue blybsch i mym härz - sogar no denn - we's afat weh tue uf au die wo chöi vergässe, uf au die wo chöi vergäh uf au die wones grosses härz hei & wo sech das nid lö la näh uf ne gränzelose himu, uf nes uferloses meer & für immer uf di es glas uf d liebi & eis uf ds voue läbe & eis uf au das, wo mir nid chöi häbe es tor geit uuf & nes angers geit zue blybsch i mym härz - sogar no denn - we's afat weh tue uf mueters seel wo hüt furt isch vo dr ärde uf au die schöne ching, wo hüt z nacht gebore wärde uf au die zyt wo isch vergange, uf au die zyt wo mir no blybt" Contributor: May ================================================ FILE: Lied/cake.less ================================================ //////////////////////////////////////////// var @D:300px; // Control diameter /////////////////////////////////////////// /* ============================================== POSITION */ .cake{ position:absolute; display:none; // top:30%; left:50%; margin-left:-(@D/2); width:@D; height:@D; } /* ============================================== BASE */ .cake:after{ background:rgba(255,255,255,1); border-radius:@D; content:""; position:absolute; bottom:0; left:0; width:@D; height:@D/50; } .cake:before{ } /* ============================================== Candle */ .velas{ background:rgba(255,255,255,1); border-radius:100%; position:absolute; top:50%; left:50%; margin-left:-(@D/2)/20; margin-top:-(@D/2)/6; width:@D/20; height:@D/6; } .velas:after, .velas:before{ background:rgba(255,0,0,0.4); content:""; position:absolute; width:100%; height:@D/45; } .velas:after{ top:25%; left:0; } .velas:before{ top:45%; left:0; } /* ============================================== Fire */ .fuego{ display:none; border-radius:100%; box-shadow:0 0 40px 10px rgba(248,233,209,0.2); position:absolute; top:-12px; left:50%; margin-left:-(@D/2)/15; //margin-top:-(@D/2)/10; width:@D/15; height:@D/8; } .fuego:nth-child(1){ -webkit-animation:fuego 2s infinite; -moz-animation:fuego 2s infinite; -o-animation:fuego 2s infinite; -ms-animation:fuego 2s infinite; animation:fuego 2s infinite; } .fuego:nth-child(2){ -webkit-animation:fuego 1.5s infinite; -moz-animation:fuego 1.5s infinite; -o-animation:fuego 1.5s infinite; -ms-animation:fuego 1.5s infinite; animation:fuego 1.5s infinite; } .fuego:nth-child(3){ -webkit-animation:fuego 1s infinite; -moz-animation:fuego 1s infinite; -o-animation:fuego 1s infinite; -ms-animation:fuego 1s infinite; animation:fuego 1s infinite; } .fuego:nth-child(4){ -webkit-animation:fuego 0.5s infinite; -moz-animation:fuego 0.5s infinite; -o-animation:fuego 0.5s infinite; -ms-animation:fuego 0.5s infinite; animation:fuego 0.5s infinite; } .fuego:nth-child(5){ -webkit-animation:fuego 0.2s infinite; -moz-animation:fuego 0.2s infinite; -o-animation:fuego 0.2s infinite; -ms-animation:fuego 0.2s infinite; animation:fuego 0.2s infinite; } /* ============================================== Animation Fire */ @-webkit-keyframes fuego{ 0%{ background:rgba(254,248,97,0.5); -webkit-transform:translateY(0) scale(1); } 50%{ background:rgba(255,50,0,0.1); -webkit-transform:translateY(-(@D/5)) scale(0); } 100%{ background:rgba(254,248,97,0.5); -webkit-transform:translateY(0) scale(1); } } @-moz-keyframes fuego{ 0%{ background:rgba(254,248,97,0.5); -moz-transform:translateY(0) scale(1); } 50%{ background:rgba(255,50,0,0.1); -moz-transform:translateY(-(@D/5)) scale(0); } 100%{ background:rgba(254,248,97,0.5); -moz-transform:translateY(0) scale(1); } } @-o-keyframes fuego{ 0%{ background:rgba(254,248,97,0.5); -o-transform:translateY(0) scale(1); } 50%{ background:rgba(255,50,0,0.1); -o-transform:translateY(-(@D/5)) scale(0); } 100%{ background:rgba(254,248,97,0.5); -o-transform:translateY(0) scale(1); } } @-ms-keyframes fuego{ 0%{ background:rgba(254,248,97,0.5); -ms-transform:translateY(0) scale(1); } 50%{ background:rgba(255,50,0,0.1); -ms-transform:translateY(-(@D/5)) scale(0); } 100%{ background:rgba(254,248,97,0.5); -ms-transform:translateY(0) scale(1); } } @keyframes fuego{ 0%{ background:rgba(254,248,97,0.5); transform:translateY(0) scale(1); } 50%{ background:rgba(255,50,0,0.1); transform:translateY(-(@D/5)) scale(0); } 100%{ background:rgba(254,248,97,0.5); transform:translateY(0) scale(1); } } /* ============================================== Frosting */ .cobertura{ background:rgba(236,231,227,1); border-radius:@D/2; position:absolute; top:60%; left:50%; margin-left:-(@D/2)/1.8; margin-top:-(@D/2)/10; width:@D/1.8; height:@D/8; z-index:10; } .cobertura:after, .cobertura:before{ background:rgba(236,231,227,1); border-radius:@D; content:""; position:absolute; width:@D/20; height:@D/10; } .cobertura:after{ top:@D/15; right:@D/7; } .cobertura:before{ top:@D/10; right:@D/11; } /* ============================================== BIZCOCHO */ .bizcocho{ background:rgba(109,56,38,1); position:absolute; bottom:0; left:50%; margin-left:-(@D/2)/2; width:@D/2; height:@D/3; } .bizcocho:after, .bizcocho:before{ background:rgba(236,231,227,0.6); content:""; position:absolute; width:100%; height:@D/20; } .bizcocho:after{ top:30%; left:0; } .bizcocho:before{ top:60%; left:0; } /* ============================================== TEXT */ h1, p{ font-family: 'Lato', sans-serif; font-weight: 300; font-style:italic; text-align:center; width:100%; -webkit-user-select: none; -moz-user-select: none; -ms-user-select: none; user-select: none; } ================================================ FILE: Lied/loading.css ================================================ /* Absolute Center CSS Spinner */ .loading { position: fixed; z-index: 99999; height: 2em; width: 2em; overflow: show; margin: auto; top: 0; left: 0; bottom: 0; right: 0; } /* Transparent Overlay */ .loading:before { content: ''; display: block; position: fixed; top: 0; left: 0; width: 100%; height: 100%; background-color: rgba(0,0,0,0.3); } /* :not(:required) hides these rules from IE9 and below */ .loading:not(:required) { /* hide "loading..." text */ font: 0/0 a; color: transparent; text-shadow: none; background-color: transparent; border: 0; } .loading:not(:required):after { content: ''; display: block; font-size: 10px; width: 1em; height: 1em; margin-top: -0.5em; -webkit-animation: spinner 1500ms infinite linear; -moz-animation: spinner 1500ms infinite linear; -ms-animation: spinner 1500ms infinite linear; -o-animation: spinner 1500ms infinite linear; animation: spinner 1500ms infinite linear; border-radius: 0.5em; -webkit-box-shadow: #FFF 1.5em 0 0 0, rgba(0, 0, 0, 0.75) 1.1em 1.1em 0 0, rgba(0, 0, 0, 0.75) 0 1.5em 0 0, #FFF -1.1em 1.1em 0 0, rgba(0, 0, 0, 0.5) -1.5em 0 0 0, rgba(0, 0, 0, 0.5) -1.1em -1.1em 0 0, rgba(0, 0, 0, 0.75) 0 -1.5em 0 0, rgba(0, 0, 0, 0.75) 1.1em -1.1em 0 0; box-shadow: #FFF 1.5em 0 0 0, rgba(0, 0, 0, 0.75) 1.1em 1.1em 0 0, rgba(0, 0, 0, 0.75) 0 1.5em 0 0, #FFF -1.1em 1.1em 0 0, rgba(0, 0, 0, 0.5) -1.5em 0 0 0, rgba(0, 0, 0, 0.5) -1.1em -1.1em 0 0, rgba(0, 0, 0, 0.75) 0 -1.5em 0 0, rgba(0, 0, 0, 0.75) 1.1em -1.1em 0 0; } /* Animation */ @-webkit-keyframes spinner { 0% { -webkit-transform: rotate(0deg); -moz-transform: rotate(0deg); -ms-transform: rotate(0deg); -o-transform: rotate(0deg); transform: rotate(0deg); } 100% { -webkit-transform: rotate(360deg); -moz-transform: rotate(360deg); -ms-transform: rotate(360deg); -o-transform: rotate(360deg); transform: rotate(360deg); } } @-moz-keyframes spinner { 0% { -webkit-transform: rotate(0deg); -moz-transform: rotate(0deg); -ms-transform: rotate(0deg); -o-transform: rotate(0deg); transform: rotate(0deg); } 100% { -webkit-transform: rotate(360deg); -moz-transform: rotate(360deg); -ms-transform: rotate(360deg); -o-transform: rotate(360deg); transform: rotate(360deg); } } @-o-keyframes spinner { 0% { -webkit-transform: rotate(0deg); -moz-transform: rotate(0deg); -ms-transform: rotate(0deg); -o-transform: rotate(0deg); transform: rotate(0deg); } 100% { -webkit-transform: rotate(360deg); -moz-transform: rotate(360deg); -ms-transform: rotate(360deg); -o-transform: rotate(360deg); transform: rotate(360deg); } } @keyframes spinner { 0% { -webkit-transform: rotate(0deg); -moz-transform: rotate(0deg); -ms-transform: rotate(0deg); -o-transform: rotate(0deg); transform: rotate(0deg); } 100% { -webkit-transform: rotate(360deg); -moz-transform: rotate(360deg); -ms-transform: rotate(360deg); -o-transform: rotate(360deg); transform: rotate(360deg); } } ================================================ FILE: NLP/Examples/one/assignment.md ================================================ Natural Language Processing Fall 2022 Prof. Ryan Cotterell Assignment 1: Backpropagation 29/09/2022 - 16:21h The first half of the assignment is a series of theoretical questions about the material. When you have answered the questions to your satisfaction, you should upload a pdf file with your solutions (preferably written in LATEX) to Moodle. The second question is a coding task that is to be solved in the released Google Colab notebook. You have to copy the notebook to your own drive in order to edit it. When submitting your solution, please execute all cells in your copied notebook, then save it and include a link to your notebook in your PDF file submission. We will run software on your submission to test for plagiarism. Important: Please ensure, that all cells in the notebook are already executed and that the notebook is accessible via the shared link in Editor mode! The notebook must also be executable from top to bottom without throwing errors. 1 Theory Question 1: Efficiently Computing the Hessian (50 pts) Backpropagation on a computation graph can be used to efficiently obtain the derivatives of a function with respect to the input. In this problem, we consider a differentiable map f : Rn → R. This question makes heavy use of the operator ∇ (pronounced nabla). We write ∇f(x) to denote the Jacobian of f evaluated at point x. Importantly, in the special case that the co-domain of f is simply R, we refer to the Jacobian of f as the gradient of f. When ∇f gives us a gradient, we will take it to be a column vector. Furthermore, we write ∇∇f(x) or ∇2f(x) to indicate the Hessian of f. Recall from Lecture 2 that f(x)’s Hessian is the matrix of second derivatives at point x. This notation is intuitive because the Hessian of f is simply the Jacobian of the gradient of f. You can assume that f is twice differentiable. You will now derive an algorithm to compute the Hessian ∇2f(x) using repeated calls to backpropagation. a) (10 pts) Show that the following identity holds true: ∇2f(x) =  (∇(e⊤1 ∇f(x)))⊤ ... (∇(e⊤n ∇f(x)))⊤   (1) where {e1, . . . , en} are the standard basis vectors of Rn, which are taken to be column vectors. In words, you are asked to demonstrate that (∇(e⊤i ∇f(x)))⊤ computes the ith row of the Hessian ∇2f(x). Then, give an expression to compute the ith column of the Hessian that is similar to one for the ith row derived above. 1 b) (15 pts) Suppose f can be computed in O(m) time. Use the identity in equation 1, to briefly describe an efficient algorithm, based on backpropogation, for computing ∇2f(x). Discuss the runtime of your algorithm. Explain why this constitutes a faster algorithm than the more na¨ıve way of computing the Hessian entry by entry and also give the runtime of this more na¨ıve algorithm. c) (15 pts) Now, briefly describe an algorithm to compute the tensor of 3rd-order deriva tives and discuss its runtime in terms of m. Then, explain how to produce the tensor of kth-order derivatives and discuss its runtime in terms of m. The answer for the kth-order derivative can be qualitative as it is tricky to come up with good notation. d) (10 pts) Describe an O(m) algorithm for computing the diagonal of ∇2f(x). Hint: This is a tricky problem unless you look at it the right way. The general idea is to develop an analogue of the multivariate chain rule for elements of the Hessian’s diagonal. Then, you can develop a dynamic program very similar to backpropagation itself that has the correct runtime. Recall that the multivariate chain rule says ∂xj=X ∂zk ∂xj and our goal is to compute ∂2yi ∂yi M k=1 ∂yj ∂zk Hessian. ∂xj∂xj, another notation for the diagonal elements of the 2 2 Practice Question 2: Coding Backpropagation (50 pts) In this question, we ask you to code backpropagation in Python in a Google Colab notebook: https://colab.research.google.com/drive/1lSgujLn0ETFUMssy ClRAb3PlbhtUoQU?usp= sharing There are a several cells, marked with “[do not change]”, that you are not allowed to modify! In other other cells, you may of course add more (class) functions. However, do not change the signatures of existing functions, such as class initializations. Also, you are not allowed to import any libraries other than the ones that are being automatically imported in the cell “Library Imports”. The notebook allows you to select one of four math problems in a drop-down menu. Each math problem is represented as dict and consists of 4 key-value pairs: problem: math equation as string, in vars: dict of input variables, output: true output solution of the math problem as float and derivative: dict of the input variables’ partial derivatives. The provided Parser class is converting the math problem into infix notation that we can perform subsequent operations on. In particular, it is taking a math problem as a string input (problem), as well as optional initialization values in vars of the problem’s input variables and returning a list infix. As a simplification, you can expect operations to be grouped, e.g. x + y + z will be (x + y) + z, so that the parser always returns lists (and sublists) of length 3 at most and individual operations (such as +) will expect 2 variables at most, as discussed in item (ii). See for instance: exp(x) - (y * 2) −→ [[’exp’, ’x’], ’-’, [’y’, ’*’, 2]] (2) There are three cells for three classes with open ToDos for you to implement: (i) Building: the Builder class is taking the infix notation math problem as input and instantiates the class variable self.graph with a computation graph. Class initializa tion accepts the input variables as an additional, optional argument to avoid naming conflicts with the intermediate and output variables which may be arbitrarily chosen from the set of unicode chars. You are free in choosing your own data structure for representing the computation graph. As a simplification, we do not require optimized parallelization — the order of operations should be correct and free of conflicts per taining variable-sharing. However, operations that can be executed in parallel may also be consecutively executed. g = Builder(infix: list, in_vars: dict) print(g.graph) (ii) Operations: the Operator class defines functions that are used by the Executor class (item (iii)), which is defined next. You are asked to implement several child classes that all inherit from the abstract Operator class and that define the following functions: self.fn_map = {"log": Log(), "exp": Exp(), "+": Add(),\ "-": Sub(), "ˆ": Pow(), "sin": Sin(), "*": Mult(), "/": Div()} 3 where log is the natural logarithm. In particular, each class inheriting from Operator must have a function method self.f() and a first-order partial derivative method self.df(). Both take one or two variables as an input for unary or binary operations respectively. While self.f() always returns a single float, self.df() returns a single element list of floats for unary and a two-element list of floats, i.e. a Jacobian vector, for binary operations. (iii) Executing: the Executor class takes two dictionaries as input: the computation graph and the initialized input variables. The class should implement the functions forward and backward, which use the input variables to run a forward and backward pass on the computation graph respectively. After executing both, two class variables of the Executor class should be initialized: self.output should hold the output of the forward pass as a float value; self.derivative should hold the dictionary of first-order partial derivatives of all input variables. Given the input variables in - vars = {’x’: 2.0, ’y’: -2.0} and the computation graph b.graph as constructed in the Builder class (item (i)), the outputs should be: e = executing.Executor(graph: dict = b.graph, in_vars: dict = in_vars) e.forward() e.backward() print(e.output) ## 11.39 print(e.derivative) ## {'x': 7.39, 'y': -2.0} For grading your submission, we will run all cells in your notebook from top to bottom. In particular, we will consider the cell “Test Function for Grading”. This loads a set of math problems and executes the full pipeline of parsing, building and executing. In particular, we compare if the Executor class variables self.derivative and self.output match the true solution. Therefore, we round all floats to two decimal places. Note: Please ensure that you share your copied notebook via a link in Editor mode. The notebook must be fully executable and all cells should be executed already! 4 ================================================ FILE: NLP/Examples/one/cl.md ================================================ zb. **Computerlinguistik und Sprachtechnologie** Was ist Computerlinguistik und Sprachtechnologie? Das Studienprogramm Computerlinguistik und Sprachtechnologie bietet Veranstaltungen an in den Bereichen Computerlinguistik, Sprachtechnologie, Phonetik und Sprachsignalverarbeitung, sowie kognitiver Linguistik. Das heisst, es werden allgemeine Kenntnisse in Linguistik und Informatik mit spezifischen Kenntnissen und Fähigkeiten in Computerlinguistik verknüpft. Die Grundlagenkurse in maschinellem Lernen und automatischer Sprachverarbeitung bauen auf dem Maturawissen des Bereichs Mathematik auf. **Wann sind Sie bei uns richtig?** Vertrautheit im Umgang mit Computern, aber keine spezifischen Vorkenntnisse Programmieren: Keine Vorkenntnisse erforderlich, aber Interesse! Python wird obligatorisch gelehrt, später können weitere Programmiersprachen gewählt werden Fremdsprachen: Bereitschaft, englischsprachige Fachtexte zu lesen (Kein Lateinobligatorium) Ausgeprägte Freude am Gebrauch und der Analyse von Sprachen Interesse an der automatischen Verarbeitung der menschlichen Sprache Spass am Basteln von sprachtechnologischen Anwendungen und Umsetzen eigener Ideen zur Sprachverarbeitung Interesse an Fragen der modernen Informationsgesellschaft Mitwirken an der Flexibilisierung der Mensch-Maschine-Kommunikation Spass am Computer als mächtiges Modellierungsinstrument Das sagen unsere Studierenden «Computerlinguistik: Wo Sprachen geliebt, Professor:innen geduzt und Grafikkarten ins Schwitzen gebracht werden.» **«Die Computerlinguistik fasziniert mich, da sie Sprache und Technologie auf eine interessante Weise verbindet und es immer wieder neue Themenfelder zu entdecken gibt.»** «Die Computerlinguistik verbindet die beiden Bereiche, die mich schon immer interessiert haben: Sprache und Informatik, eine äusserst aktuelle Studienrichtung. Wir lernen die Sachen und Technologien kennen, die frisch auf den Markt gekommen sind. Die digitale Sprachverarbeitung wird immer allgegenwärtiger, von Alexa über DeepL bis hin zu Smart Homes. Mit dem Computerlinguistik-Studium stehen einem alle Türen offen. Ob lieber Richtung Forschung, Softwareentwicklung oder Webdesign – es ist alles dabei.» «Am CL-Studium begeistert mich, dass ich nach jeder Vorlesung mehr weiss und kann als davor. Man muss nicht einfach büffeln und auswendig lernen, sondern eignet sich Fertigkeiten an, bei denen man immer weiss, warum sie einem später etwas nützen werden. Und dass man am Puls der Zeit ist – die Forschung entwickelt sich unglaublich schnell weiter und man ist mittendrin. Man denke nur, wie Google Translate noch vor 10 Jahren übersetzte und im Gegensatz dazu, was er heute kann!» «Ich habe das CL-Studium gewählt, weil die Berufsperspektiven gut und vielfältig sind. Man wird sich immer weiterentwickeln und neuen Herausforderungen stellen können (und auch müssen).» ================================================ FILE: NLP/Examples/one/example.py ================================================ example n is always more than n Lowerbound how you define an automic automation every node to be an dependent N and the calculation of the output ================================================ FILE: NLP/Examples/one/number.py ================================================ import sys import re from pyparsing import * import random ROUND = 2 ## we round results to two decimal places for checking MATH_PROBLEMS = [{'problem': 'x/y', 'in_vars': {'x': 1.0, 'y': 1.0}, 'output': 1.0, 'derivative': {'x': 1.0, 'y': -1.0}}, {'problem': 'exp(x) - (y * 2)', 'in_vars': {'x': 2.0, 'y': -2.0},'output': 11.39, 'derivative': {'x': 7.39, 'y': -2.0}}, {'problem': '(x^2 - 1) * (y+2)', 'in_vars': {'x': 3.0, 'y': 2.0}, 'output': 32.0, 'derivative': {'x': 24.0, 'y': 8.0}}, {'problem': 'z + sin(x^(2) + (y * exp(z)))', 'in_vars': {'x': 2.0, 'y': -1.0, 'z': 0.0}, 'output': 0.14, 'derivative': {'x': -3.96, 'y': -0.99, 'z': 1.99}}] class Parser(): def __init__(self, print_res: bool = False): self.print_res = print_res self.expr = self.infix_notation_parser() def parse(self, math_string: str, in_vars: dict = {}): """ math_string: str describing math equation, e.g. 'z + sin(x^(2) + y * exp(z))' in_vars: optional dict of mapping between input variables and numbers RETURN: list of infix parsed string, returns dict of input used_vars """ if self.print_res: print(f"\nproblem: {math_string} with input {in_vars}") in_vars = self.get_input_variables(math_string, in_vars) infix_list = self.expr.parseString(math_string).asList()[0] return infix_list, in_vars def infix_notation_parser(self, ): """ RETURN: expr py_parse object """ ppc = pyparsing_common ParserElement.enablePackrat() sys.setrecursionlimit(3000) integer = ppc.integer variable = Word(alphas, exact=1) operand = integer | variable expop = Literal("^") factop = Literal("!") ident = Word(alphas, alphanums + "_$") signop = oneOf("+ -") multop = oneOf("* /") plusop = oneOf("+ -") #e = CaselessKeyword("E") #pi = CaselessKeyword("PI") expr = infixNotation( operand, [ (ident, 1, opAssoc.RIGHT), (factop, 1, opAssoc.LEFT), (expop, 2, opAssoc.RIGHT), (signop, 1, opAssoc.RIGHT), (multop, 2, opAssoc.LEFT), (plusop, 2, opAssoc.LEFT), ], ) return expr def get_input_variables(self, math_string: str, in_vars: dict = {}): """ math_string: str describing math equation, e.g. 'z + sin(x^(2) + y * exp(z))' in_vars: optional dict of mapping between input variables and numbers RETURN: returns dict of input used_vars """ in_var_list = list(set(re.findall(r'(?i)(? float comparison print(f"\n{str(i)}: problem: {math_problem['problem']}, in_vars: {math_problem['in_vars']}") if round(e.output,2) == round(math_problem["output"], ROUND): print(f"SUCCESS output: {round(e.output, ROUND)}") else: print(f"FAILURE output: {round(e.output, ROUND)} != {math_problem['output']}") ## first derivative --> dict comparison true_deriv = {k: round(v, ROUND) for k, v in math_problem["derivative"].items()} e.derivative = {k: round(v,ROUND) for k,v in e.derivative.items()} if e.derivative == true_deriv: print(f"SUCCESS derivative: {e.derivative}") else: print(f"FAILURE derivative: {e.derivative} != {math_problem['derivative']}") if __name__ == '__main__': pass ================================================ FILE: Notebook/Intro.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "toc": true }, "source": [ "

Week 01. Text Data Essentials

\n", "
" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Week 01. Introduction to Text Data\n", "\n", "Natural Language Processing for Law and Social Science
\n", "Elliott Ash, ETH Zurich" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:19.066847Z", "start_time": "2022-03-03T18:54:18.994527Z" } }, "outputs": [], "source": [ "# set random seed\n", "import numpy as np\n", "np.random.seed(4)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Loading and Inspecting Data with Pandas" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:19.954764Z", "start_time": "2022-03-03T18:54:19.506368Z" } }, "outputs": [], "source": [ "#import warnings; warnings.simplefilter('ignore')\n", "# !pip install pandas\n", "import pandas as pd\n", "df = pd.read_csv('sc_cases.zip',compression='gzip')" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:19.961024Z", "start_time": "2022-03-03T18:54:19.955791Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "RangeIndex: 820 entries, 0 to 819\n", "Data columns (total 10 columns):\n", " # Column Non-Null Count Dtype \n", "--- ------ -------------- ----- \n", " 0 case_name 786 non-null object \n", " 1 opinion_type 820 non-null object \n", " 2 date_standard 820 non-null object \n", " 3 authorship 820 non-null object \n", " 4 x_republican 803 non-null float64\n", " 5 maj_judges 786 non-null object \n", " 6 dissent_judges 786 non-null object \n", " 7 topic_id 786 non-null float64\n", " 8 cite_count 812 non-null float64\n", " 9 opinion_text 820 non-null object \n", "dtypes: float64(3), object(7)\n", "memory usage: 64.2+ KB\n" ] } ], "source": [ "df.info() " ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:19.970882Z", "start_time": "2022-03-03T18:54:19.961700Z" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
case_nameopinion_typedate_standardauthorshipx_republicanmaj_judgesdissent_judgestopic_idcite_countopinion_text
0ERICK CORNELL CLAY v. UNITED STATESmajority2003-03-04GINSBURG0.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...[]1.02926.0JUSTICE GINSBURG delivered the opinion of the ...
1HILLSIDE DAIRY INC., A&A DAIRY, L&S DAIRY, AND...majority2003-06-09STEVENS1.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...['THOMAS, CLARENCE']8.0117.0Justice Stevens delivered the opinion of the C...
2CHARLES RUSSELL RHINES v. DOUGLAS WEBER, WARDENmajority2005-03-30O'CONNOR1.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...[]1.023364.0Justice O'Connor delivered the opinion of the ...
3NaNmajority2005-04-15KENNEDY1.0NaNNaNNaN6.0Justice Kennedy, Circuit Justice. \\n\\n This is...
4STATE OF ALASKA v. UNITED STATES OF AMERICAmajority2005-06-06KENNEDY1.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...['REHNQUIST, WILLIAM', 'SCALIA, ANTONIN', 'THO...10.084.0Justice Kennedy delivered the opinion of the C...
\n", "
" ], "text/plain": [ " case_name opinion_type \\\n", "0 ERICK CORNELL CLAY v. UNITED STATES majority \n", "1 HILLSIDE DAIRY INC., A&A DAIRY, L&S DAIRY, AND... majority \n", "2 CHARLES RUSSELL RHINES v. DOUGLAS WEBER, WARDEN majority \n", "3 NaN majority \n", "4 STATE OF ALASKA v. UNITED STATES OF AMERICA majority \n", "\n", " date_standard authorship x_republican \\\n", "0 2003-03-04 GINSBURG 0.0 \n", "1 2003-06-09 STEVENS 1.0 \n", "2 2005-03-30 O'CONNOR 1.0 \n", "3 2005-04-15 KENNEDY 1.0 \n", "4 2005-06-06 KENNEDY 1.0 \n", "\n", " maj_judges \\\n", "0 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "1 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "2 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "3 NaN \n", "4 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "\n", " dissent_judges topic_id cite_count \\\n", "0 [] 1.0 2926.0 \n", "1 ['THOMAS, CLARENCE'] 8.0 117.0 \n", "2 [] 1.0 23364.0 \n", "3 NaN NaN 6.0 \n", "4 ['REHNQUIST, WILLIAM', 'SCALIA, ANTONIN', 'THO... 10.0 84.0 \n", "\n", " opinion_text \n", "0 JUSTICE GINSBURG delivered the opinion of the ... \n", "1 Justice Stevens delivered the opinion of the C... \n", "2 Justice O'Connor delivered the opinion of the ... \n", "3 Justice Kennedy, Circuit Justice. \\n\\n This is... \n", "4 Justice Kennedy delivered the opinion of the C... " ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.head()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:19.979689Z", "start_time": "2022-03-03T18:54:19.972244Z" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
case_nameopinion_typedate_standardauthorshipx_republicanmaj_judgesdissent_judgestopic_idcite_countopinion_text
0ERICK CORNELL CLAY v. UNITED STATESmajority2003-03-04GINSBURG0.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...[]1.02926.0JUSTICE GINSBURG delivered the opinion of the ...
1HILLSIDE DAIRY INC., A&A DAIRY, L&S DAIRY, AND...majority2003-06-09STEVENS1.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...['THOMAS, CLARENCE']8.0117.0Justice Stevens delivered the opinion of the C...
2CHARLES RUSSELL RHINES v. DOUGLAS WEBER, WARDENmajority2005-03-30O'CONNOR1.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...[]1.023364.0Justice O'Connor delivered the opinion of the ...
4STATE OF ALASKA v. UNITED STATES OF AMERICAmajority2005-06-06KENNEDY1.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...['REHNQUIST, WILLIAM', 'SCALIA, ANTONIN', 'THO...10.084.0Justice Kennedy delivered the opinion of the C...
5REGINALD A. WILKINSON, DIRECTOR, OHIO DEPARTME...majority2005-06-13KENNEDY1.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...[]4.04230.0Justice Kennedy delivered the opinion of the C...
\n", "
" ], "text/plain": [ " case_name opinion_type \\\n", "0 ERICK CORNELL CLAY v. UNITED STATES majority \n", "1 HILLSIDE DAIRY INC., A&A DAIRY, L&S DAIRY, AND... majority \n", "2 CHARLES RUSSELL RHINES v. DOUGLAS WEBER, WARDEN majority \n", "4 STATE OF ALASKA v. UNITED STATES OF AMERICA majority \n", "5 REGINALD A. WILKINSON, DIRECTOR, OHIO DEPARTME... majority \n", "\n", " date_standard authorship x_republican \\\n", "0 2003-03-04 GINSBURG 0.0 \n", "1 2003-06-09 STEVENS 1.0 \n", "2 2005-03-30 O'CONNOR 1.0 \n", "4 2005-06-06 KENNEDY 1.0 \n", "5 2005-06-13 KENNEDY 1.0 \n", "\n", " maj_judges \\\n", "0 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "1 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "2 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "4 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "5 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "\n", " dissent_judges topic_id cite_count \\\n", "0 [] 1.0 2926.0 \n", "1 ['THOMAS, CLARENCE'] 8.0 117.0 \n", "2 [] 1.0 23364.0 \n", "4 ['REHNQUIST, WILLIAM', 'SCALIA, ANTONIN', 'THO... 10.0 84.0 \n", "5 [] 4.0 4230.0 \n", "\n", " opinion_text \n", "0 JUSTICE GINSBURG delivered the opinion of the ... \n", "1 Justice Stevens delivered the opinion of the C... \n", "2 Justice O'Connor delivered the opinion of the ... \n", "4 Justice Kennedy delivered the opinion of the C... \n", "5 Justice Kennedy delivered the opinion of the C... " ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# drop missing\n", "df = df.dropna()\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 6, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:20.078271Z", "start_time": "2022-03-03T18:54:20.075074Z" } }, "outputs": [ { "data": { "text/plain": [ "count 781\n", "unique 27\n", "top SCALIA\n", "freq 86\n", "Name: authorship, dtype: object" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Number of label categories (e.g. judges)\n", "df['authorship'].describe()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:20.238731Z", "start_time": "2022-03-03T18:54:20.227653Z" } }, "outputs": [ { "data": { "text/plain": [ "SCALIA 86\n", "GINSBURG 81\n", "THOMAS 79\n", "KENNEDY 79\n", "BREYER 73\n", "SOUTER 72\n", "STEVENS 72\n", "O'CONNOR 52\n", "REHNQUIST 49\n", "ROBERTS 28\n", "ALITO 23\n", "Breyer 12\n", "Roberts 10\n", "Alito 9\n", "Scalia 8\n", "Thomas 8\n", "Ginsburg 8\n", "Stevens 7\n", "Kennedy 7\n", "SOTOMAYOR 6\n", "Souter 5\n", "STEVENS AND O'CONNOR ; REHNQUIST ; BREYER 2\n", "STEVENS (IN PART), BREYER (IN PART) 1\n", "JUSTICE ALITO 1\n", "BREYER ; 1\n", "ROBERTS , SCALIA , THOMAS , ALITO 1\n", "Sotomayor 1\n", "Name: authorship, dtype: int64" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# tabulations of label categories \n", "df['authorship'].value_counts()" ] }, { "cell_type": "code", "execution_count": 8, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:20.382374Z", "start_time": "2022-03-03T18:54:20.377383Z" } }, "outputs": [], "source": [ "df['authorship'] = df['authorship'].str.upper()" ] }, { "cell_type": "code", "execution_count": 9, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:20.524682Z", "start_time": "2022-03-03T18:54:20.515558Z" } }, "outputs": [ { "data": { "text/plain": [ "SCALIA 94\n", "GINSBURG 89\n", "THOMAS 87\n", "KENNEDY 86\n", "BREYER 85\n", "STEVENS 79\n", "SOUTER 77\n", "O'CONNOR 52\n", "REHNQUIST 49\n", "ROBERTS 38\n", "ALITO 32\n", "SOTOMAYOR 7\n", "STEVENS AND O'CONNOR ; REHNQUIST ; BREYER 2\n", "STEVENS (IN PART), BREYER (IN PART) 1\n", "JUSTICE ALITO 1\n", "BREYER ; 1\n", "ROBERTS , SCALIA , THOMAS , ALITO 1\n", "Name: authorship, dtype: int64" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df['authorship'].value_counts()" ] }, { "cell_type": "code", "execution_count": 10, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:20.658055Z", "start_time": "2022-03-03T18:54:20.648459Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Index(['SCALIA', 'GINSBURG', 'THOMAS', 'KENNEDY', 'BREYER', 'STEVENS',\n", " 'SOUTER', 'O'CONNOR', 'REHNQUIST', 'ROBERTS', 'ALITO'],\n", " dtype='object')\n" ] } ], "source": [ "# keep all judges through ALITO\n", "keep_judges = df['authorship'].value_counts().index[:11]\n", "print(keep_judges)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:20.811481Z", "start_time": "2022-03-03T18:54:20.796104Z" } }, "outputs": [ { "data": { "text/plain": [ "SCALIA 94\n", "GINSBURG 89\n", "THOMAS 87\n", "KENNEDY 86\n", "BREYER 85\n", "STEVENS 79\n", "SOUTER 77\n", "O'CONNOR 52\n", "REHNQUIST 49\n", "ROBERTS 38\n", "ALITO 32\n", "Name: authorship, dtype: int64" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df[df['authorship'].isin(keep_judges)]\n", "df['authorship'].value_counts()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:20.955563Z", "start_time": "2022-03-03T18:54:20.943541Z" } }, "outputs": [ { "data": { "text/plain": [ "0 2003-03-04\n", "1 2003-06-09\n", "2 2005-03-30\n", "4 2005-06-06\n", "5 2005-06-13\n", " ... \n", "815 2001-04-18\n", "816 2001-04-24\n", "817 2001-04-24\n", "818 2001-05-14\n", "819 2001-05-14\n", "Name: date_standard, Length: 768, dtype: object" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.date_standard" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:21.096221Z", "start_time": "2022-03-03T18:54:21.080282Z" } }, "outputs": [ { "data": { "text/plain": [ "0 2003-03-04\n", "1 2003-06-09\n", "2 2005-03-30\n", "4 2005-06-06\n", "5 2005-06-13\n", " ... \n", "815 2001-04-18\n", "816 2001-04-24\n", "817 2001-04-24\n", "818 2001-05-14\n", "819 2001-05-14\n", "Name: date_standard, Length: 768, dtype: datetime64[ns]" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df['date_standard'] = pd.to_datetime(df['date_standard'])\n", "df['date_standard']" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:21.243003Z", "start_time": "2022-03-03T18:54:21.229193Z" } }, "outputs": [ { "data": { "text/plain": [ "2001 77\n", "2000 76\n", "2009 74\n", "2002 74\n", "2004 73\n", "2003 70\n", "2005 70\n", "2007 65\n", "2010 64\n", "2006 63\n", "2008 62\n", "Name: year, dtype: int64" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df['year'] = df['date_standard'].dt.year\n", "df['year'].value_counts()" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:21.830847Z", "start_time": "2022-03-03T18:54:21.364628Z" } }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUuElEQVR4nO3df4zc9X3n8ef7cCA5Nuc1kK4s2zoT1UqEQkPNijhKVO0GJQekivmDIiKrONSVT1daJUpOh2mlO1U66ZyTaC7QE83qiM5UbjaUFtlyaFN3YVXxByR2QjA/QlmoudhyvQoYpxvo3dF73x/zWTLZ7u7Mzs7Mej/7fEij+X4/3893Pp/35pvXfPnM7DoyE0lSXf7FSk9AktR9hrskVchwl6QKGe6SVCHDXZIqtG6lJwBwxRVX5NatWzs696c//SmXXnppdyd0gVortVpnfdZKrf2u8/jx4z/OzPfNd+yCCPetW7dy7Nixjs6dnJxkZGSkuxO6QK2VWq2zPmul1n7XGRGvLnTMZRlJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SarQBfEbqstx4vR5PrfvWysy9sn9n16RcSWpFe/cJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mqkOEuSRUy3CWpQoa7JFXIcJekCrUM94j4QEQ83fT4SUR8ISIui4ijEfFSed5Q+kdE3BsRUxHxTERs730ZkqRmLcM9M1/MzGsy8xrgWuBN4BFgHzCRmduAibIPcCOwrTz2Avf3YN6SpEUsdVnmeuDlzHwV2AkcKO0HgJvL9k7gwWx4EhiMiI3dmKwkqT2Rme13jvg68L3M/MOIeCMzB0t7AOcyczAijgD7M/OJcmwCuCszj815rb007uwZGhq6dnx8vKMCpl8/z9m3Ojp12a7etL6v483MzDAwMNDXMVeCddZnrdTa7zpHR0ePZ+bwfMfa/nvuEXEx8Bng7rnHMjMjov13icY5Y8AYwPDwcI6MjCzl9Hfcd/AQ95xYmT9Lf3LXSF/Hm5ycpNOf02pinfVZK7VeSHUuZVnmRhp37WfL/tnZ5ZbyPF3aTwNbms7bXNokSX2ylHD/LPCNpv3DwO6yvRs41NR+e/nWzA7gfGaeWfZMJUlta2s9IyIuBT4J/Num5v3AQxGxB3gVuLW0PwrcBEzR+GbNHV2brSSpLW2Fe2b+FLh8TttrNL49M7dvAnd2ZXaSpI74G6qSVCHDXZIqZLhLUoUMd0mqkOEuSRUy3CWpQoa7JFXIcJekChnuklQhw12SKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkirUVrhHxGBEPBwRP4yIFyLioxFxWUQcjYiXyvOG0jci4t6ImIqIZyJie29LkCTN1e6d+1eBv8zMDwIfBl4A9gETmbkNmCj7ADcC28pjL3B/V2csSWqpZbhHxHrgV4AHADLz/2TmG8BO4EDpdgC4uWzvBB7MhieBwYjY2OV5S5IWEZm5eIeIa4Ax4Hkad+3Hgc8DpzNzsPQJ4FxmDkbEEWB/Zj5Rjk0Ad2XmsTmvu5fGnT1DQ0PXjo+Pd1TA9OvnOftWR6cu29Wb1vd1vJmZGQYGBvo65kqwzvqslVr7Xefo6OjxzBye79i6Ns5fB2wHficzn4qIr/KzJRgAMjMjYvF3iTkyc4zGmwbDw8M5MjKylNPfcd/BQ9xzop0yuu/krpG+jjc5OUmnP6fVxDrrs1ZqvZDqbGfN/RRwKjOfKvsP0wj7s7PLLeV5uhw/DWxpOn9zaZMk9UnLcM/Mvwd+FBEfKE3X01iiOQzsLm27gUNl+zBwe/nWzA7gfGae6e60JUmLaXc943eAgxFxMfAKcAeNN4aHImIP8Cpwa+n7KHATMAW8WfpKkvqorXDPzKeB+Rbtr5+nbwJ3Lm9akqTl8DdUJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mqkOEuSRUy3CWpQoa7JFXIcJekChnuklQhw12SKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAq1Fe4RcTIiTkTE0xFxrLRdFhFHI+Kl8ryhtEdE3BsRUxHxTERs72UBkqR/bil37qOZeU1mzv5D2fuAiczcBkyUfYAbgW3lsRe4v1uTlSS1ZznLMjuBA2X7AHBzU/uD2fAkMBgRG5cxjiRpiSIzW3eK+DvgHJDA1zJzLCLeyMzBcjyAc5k5GBFHgP2Z+UQ5NgHclZnH5rzmXhp39gwNDV07Pj7eUQHTr5/n7FsdnbpsV29a39fxZmZmGBgY6OuYK8E667NWau13naOjo8ebVlN+zro2X+PjmXk6In4BOBoRP2w+mJkZEa3fJX7+nDFgDGB4eDhHRkaWcvo77jt4iHtOtFtGd53cNdLX8SYnJ+n057SaWGd91kqtF1KdbS3LZObp8jwNPAJcB5ydXW4pz9Ol+2lgS9Ppm0ubJKlPWoZ7RFwaEe+d3QY+BTwLHAZ2l267gUNl+zBwe/nWzA7gfGae6frMJUkLamc9Ywh4pLGszjrgTzLzLyPiu8BDEbEHeBW4tfR/FLgJmALeBO7o+qwlSYtqGe6Z+Qrw4XnaXwOun6c9gTu7MjtJUkf8DVVJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFTLcJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mqkOEuSRVqO9wj4qKI+H5EHCn7V0bEUxExFRHfjIiLS/slZX+qHN/ao7lLkhawlDv3zwMvNO1/GfhKZv4icA7YU9r3AOdK+1dKP0lSH7UV7hGxGfg08D/KfgCfAB4uXQ4AN5ftnWWfcvz60l+S1CeRma07RTwM/BfgvcC/Bz4HPFnuzomILcBfZOaHIuJZ4IbMPFWOvQx8JDN/POc19wJ7AYaGhq4dHx/vqIDp189z9q2OTl22qzet7+t4MzMzDAwM9HXMlWCd9Vkrtfa7ztHR0eOZOTzfsXWtTo6IXwWmM/N4RIx0a1KZOQaMAQwPD+fISGcvfd/BQ9xzomUZPXFy10hfx5ucnKTTn9NqYp31WSu1Xkh1tpOKHwM+ExE3Ae8G/hXwVWAwItZl5tvAZuB06X8a2AKcioh1wHrgta7PXJK0oJZr7pl5d2ZuzsytwG3AY5m5C3gcuKV02w0cKtuHyz7l+GPZztqPJKlrlvM997uAL0bEFHA58EBpfwC4vLR/Edi3vClKkpZqSYvVmTkJTJbtV4Dr5unzj8CvdWFukqQO+RuqklQhw12SKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFTLcJalChrskVchwl6QKGe6SVCHDXZIq1DLcI+LdEfGdiPhBRDwXEb9f2q+MiKciYioivhkRF5f2S8r+VDm+tcc1SJLmaOfO/X8Dn8jMDwPXADdExA7gy8BXMvMXgXPAntJ/D3CutH+l9JMk9VHLcM+GmbL7rvJI4BPAw6X9AHBz2d5Z9inHr4+I6NaEJUmttbXmHhEXRcTTwDRwFHgZeCMz3y5dTgGbyvYm4EcA5fh54PIuzlmS1MK6djpl5j8B10TEIPAI8MHlDhwRe4G9AENDQ0xOTnb0OkPvgS9d/Xbrjj3Q6Zw7NTMz0/cxV4J11met1Hoh1dlWuM/KzDci4nHgo8BgRKwrd+ebgdOl22lgC3AqItYB64HX5nmtMWAMYHh4OEdGRjoq4L6Dh7jnxJLK6JqTu0b6Ot7k5CSd/pxWE+usz1qp9UKqs51vy7yv3LETEe8BPgm8ADwO3FK67QYOle3DZZ9y/LHMzC7OWZLUQju3vBuBAxFxEY03g4cy80hEPA+MR8R/Br4PPFD6PwD8cURMAa8Dt/Vg3pKkRbQM98x8BvjledpfAa6bp/0fgV/ryuwkSR3xN1QlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFTLcJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mqkOEuSRUy3CWpQoa7JFXIcJekCrUM94jYEhGPR8TzEfFcRHy+tF8WEUcj4qXyvKG0R0TcGxFTEfFMRGzvdRGSpJ/Xzp3728CXMvMqYAdwZ0RcBewDJjJzGzBR9gFuBLaVx17g/q7PWpK0qJbhnplnMvN7ZfsfgBeATcBO4EDpdgC4uWzvBB7MhieBwYjY2O2JS5IWFpnZfueIrcDfAB8C/ldmDpb2AM5l5mBEHAH2Z+YT5dgEcFdmHpvzWntp3NkzNDR07fj4eEcFTL9+nrNvdXTqsl29aX1fx5uZmWFgYKCvY64E66zPWqm133WOjo4ez8zh+Y6ta/dFImIA+DPgC5n5k0aeN2RmRkT77xKNc8aAMYDh4eEcGRlZyunvuO/gIe450XYZXXVy10hfx5ucnKTTn9NqYp31WSu1Xkh1tvVtmYh4F41gP5iZf16az84ut5Tn6dJ+GtjSdPrm0iZJ6pN2vi0TwAPAC5n5B02HDgO7y/Zu4FBT++3lWzM7gPOZeaaLc5YktdDOesbHgF8HTkTE06Xtd4H9wEMRsQd4Fbi1HHsUuAmYAt4E7ujmhCVJrbUM9/LBaCxw+Pp5+idw5zLnJUlaBn9DVZIqZLhLUoUMd0mqkOEuSRUy3CWpQoa7JFXIcJekChnuklQhw12SKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShdr5B7K/HhHTEfFsU9tlEXE0Il4qzxtKe0TEvRExFRHPRMT2Xk5ekjS/du7c/ydww5y2fcBEZm4DJso+wI3AtvLYC9zfnWlKkpaiZbhn5t8Ar89p3gkcKNsHgJub2h/MhieBwYjY2KW5SpLaFJnZulPEVuBIZn6o7L+RmYNlO4BzmTkYEUeA/Zn5RDk2AdyVmcfmec29NO7uGRoaunZ8fLyjAqZfP8/Ztzo6ddmu3rS+r+PNzMwwMDDQ1zFXgnXWZ63U2u86R0dHj2fm8HzH1i33xTMzI6L1O8Q/P28MGAMYHh7OkZGRjsa/7+Ah7jmx7DI6cnLXSF/Hm5ycpNOf02pinfVZK7VeSHV2+m2Zs7PLLeV5urSfBrY09dtc2iRJfdRpuB8Gdpft3cChpvbby7dmdgDnM/PMMucoSVqilusZEfENYAS4IiJOAf8J2A88FBF7gFeBW0v3R4GbgCngTeCOHsxZktRCy3DPzM8ucOj6efomcOdyJyVJWh5/Q1WSKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFTLcJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mqUE/CPSJuiIgXI2IqIvb1YgxJ0sJa/gPZSxURFwH/HfgkcAr4bkQczsznuz3WStu671t9He9LV7/N5/Z9i5P7P93XcSWtPl0Pd+A6YCozXwGIiHFgJ1BduK+Ufr+p9Nvsm1gz39DUS936/9R8124rvbq2IzO7+4IRtwA3ZOZvlv1fBz6Smb89p99eYG/Z/QDwYodDXgH8uMNzV5u1Uqt11met1NrvOv91Zr5vvgO9uHNvS2aOAWPLfZ2IOJaZw12Y0gVvrdRqnfVZK7VeSHX24gPV08CWpv3NpU2S1Ce9CPfvAtsi4sqIuBi4DTjcg3EkSQvo+rJMZr4dEb8NfBu4CPh6Zj7X7XGaLHtpZxVZK7VaZ33WSq0XTJ1d/0BVkrTy/A1VSaqQ4S5JFVrV4b5a/sxBRHw9IqYj4tmmtssi4mhEvFSeN5T2iIh7S03PRMT2pnN2l/4vRcTupvZrI+JEOefeiIjFxuhhnVsi4vGIeD4inouIz9dYa0S8OyK+ExE/KHX+fmm/MiKeKnP7ZvlCARFxSdmfKse3Nr3W3aX9xYj4N03t817bC43RaxFxUUR8PyKO1FprRJws19bTEXGstK3eazczV+WDxoe1LwPvBy4GfgBctdLzWmCuvwJsB55tavuvwL6yvQ/4ctm+CfgLIIAdwFOl/TLglfK8oWxvKMe+U/pGOffGxcboYZ0bge1l+73A3wJX1VZrGXugbL8LeKrM6SHgttL+R8C/K9u/BfxR2b4N+GbZvqpct5cAV5br+aLFru2FxujDNfxF4E+AI4vNYzXXCpwErpjTtmqv3Z5fFD38H+KjwLeb9u8G7l7peS0y3638fLi/CGws2xuBF8v214DPzu0HfBb4WlP710rbRuCHTe3v9FtojD7WfIjG3xiqtlbgXwLfAz5C4zcT1829Pml8c+yjZXtd6Rdzr9nZfgtd2+WcecfocY2bgQngE8CRxeaxmmtl/nBftdfual6W2QT8qGn/VGlbLYYy80zZ/ntgqGwvVNdi7afmaV9sjJ4r/zn+yzTuaqurtSxTPA1MA0dp3H2+kZlvzzO3d+opx88Dl7P0+i9fZIxe+m/AfwD+X9lfbB6rudYE/ioijkfjz6PAKr52V+zPD+hnMjMjoqffSe3HGLMiYgD4M+ALmfmTsrTYt3n0aYx/Aq6JiEHgEeCDvRxvpUTErwLTmXk8IkZWeDq99vHMPB0RvwAcjYgfNh9cbdfuar5zX+1/5uBsRGwEKM/TpX2huhZr3zxP+2Jj9ExEvItGsB/MzD9vMY9VXStAZr4BPE5j2WAwImZvmJrn9k495fh64DWWXv9ri4zRKx8DPhMRJ4FxGkszX11kHqu21sw8XZ6nabxhX8cqvnZXc7iv9j9zcBiY/SR9N4316dn228un8TuA8+U/2b4NfCoiNpRP0z9FYw3yDPCTiNhRPn2/fc5rzTdGT5TxHwBeyMw/aDpUVa0R8b5yx05EvIfG5wov0Aj5Wxaoc3ZutwCPZWOB9TBwW/mGyZXANhofus17bZdzFhqjJzLz7szcnJlbyzwey8xdi8xjVdYaEZdGxHtnt2lcc8+ymq/dXn5A0esHjU+s/5bGeufvrfR8FpnnN4AzwP+lsda2h8aa4gTwEvDXwGWlb9D4x05eBk4Aw02v8xvAVHnc0dQ+TONCfBn4Q372m8fzjtHDOj9OY93yGeDp8riptlqBXwK+X+p8FviPpf39NAJrCvhT4JLS/u6yP1WOv7/ptX6v1PIi5dsTi13bC43Rp+t4hJ99W6aqWstYPyiP52bnsZqvXf/8gCRVaDUvy0iSFmC4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAr9f06u23fLO4nWAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "df['cite_count'].hist()" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:21.882734Z", "start_time": "2022-03-03T18:54:21.831876Z" } }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAO6ElEQVR4nO3df2xd9XnH8fcz0m00rggoncWSaOaPiCkjKwWLsXWa7LEfFKqGSRMCMZq0TNkfdKNTpDVsf7TS1CnSRrdV3diywkhVhocoiAho1yirhSqNrQlDhB9lRCUwvDRpRwgY0LrQZ3/ck8wNDrZz7/XxfXi/JOue8/0en/s8yvXH5x6fexKZiSSplh9puwBJUu8Z7pJUkOEuSQUZ7pJUkOEuSQUta7sAgJUrV+bIyMiJ9ddee43ly5e3V9AiqN5j9f6gfo/V+4PB73Hv3r3fy8z3zja3JMJ9ZGSEPXv2nFifnJxkbGysvYIWQfUeq/cH9Xus3h8Mfo8R8fyp5jwtI0kFGe6SVJDhLkkFGe6SVJDhLkkFGe6SVJDhLkkFGe6SVJDhLkkFLYlPqEpzGdn6YGvPfWDbla09t3S6PHKXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpILmDPeIWBMRX4+IpyLiyYi4qRk/JyJ2RcSzzePZzXhExOciYn9EPB4RF/W7CUnSD5vPkfsxYEtmrgMuBW6MiHXAVmB3Zq4FdjfrAB8E1jZfm4Fbe161JOltzRnumXkwMx9tll8FngZWARuAHc1mO4CrmuUNwBez4xFgRUSc2+vCJUmnFpk5/40jRoCHgQuAFzJzRTMewJHMXBERDwDbMvMbzdxu4JOZueekfW2mc2TP8PDwxRMTEyfmpqenGRoa6qKtpa96j73ub9/U0Z7ta6HWrzpr1nH/DQffoPc4Pj6+NzNHZ5tbNt+dRMQQ8GXgE5n5SifPOzIzI2L+vyU637Md2A4wOjqaY2NjJ+YmJyeZuV5R9R573d+mrQ/2bF8LdeC6sVnH/TccfJV7nNfVMhHxLjrBfmdm3tsMHzp+uqV5PNyMTwFrZnz76mZMkrRI5nO1TAC3AU9n5mdnTO0ENjbLG4H7Z4x/pLlq5lLgaGYe7GHNkqQ5zOe0zAeA64F9EfFYM/aHwDbg7oi4AXgeuLqZewi4AtgPvA58tJcFS5LmNme4N38YjVNMXzbL9gnc2GVdkqQu+AlVSSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSrIcJekggx3SSpoPv/NnvSONrL1wVnHt6w/xqZTzPXCgW1X9m3fqs8jd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIK8cZgW5FQ30TpZv2+qJenteeQuSQUZ7pJUkOEuSQXNGe4RcXtEHI6IJ2aMfToipiLisebrihlzN0fE/oh4JiJ+vV+FS5JObT5H7ncAl88y/ueZeWHz9RBARKwDrgF+pvmev46IM3pVrCRpfuYM98x8GHhpnvvbAExk5v9k5nPAfuCSLuqTJJ2GyMy5N4oYAR7IzAua9U8Dm4BXgD3Alsw8EhGfBx7JzC81290GfCUz75lln5uBzQDDw8MXT0xMnJibnp5maGioq8aWukHtcd/U0XltN3wmHHqjz8W0rN89rl91Vv92Pg+D+hpdiEHvcXx8fG9mjs42d7rXud8K/DGQzeMtwMcWsoPM3A5sBxgdHc2xsbETc5OTk8xcr2hQe5zvtetb1h/jln21P0bR7x4PXDfWt33Px6C+Rheico+ndbVMZh7KzDcz8wfA3/H/p16mgDUzNl3djEmSFtFphXtEnDtj9TeA41fS7ASuiYgfi4jzgLXAv3VXoiRpoeZ8TxkRdwFjwMqIeBH4FDAWERfSOS1zAPgdgMx8MiLuBp4CjgE3ZuabfalcknRKc4Z7Zl47y/Btb7P9Z4DPdFOUJKk7fkJVkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgqaM9wj4vaIOBwRT8wYOycidkXEs83j2c14RMTnImJ/RDweERf1s3hJ0uzmc+R+B3D5SWNbgd2ZuRbY3awDfBBY23xtBm7tTZmSpIWYM9wz82HgpZOGNwA7muUdwFUzxr+YHY8AKyLi3B7VKkmap8jMuTeKGAEeyMwLmvWXM3NFsxzAkcxcEREPANsy8xvN3G7gk5m5Z5Z9bqZzdM/w8PDFExMTJ+amp6cZGhrqsrWlbVB73Dd1dF7bDZ8Jh97oczEt63eP61ed1b+dz8OgvkYXYtB7HB8f35uZo7PNLet255mZETH3b4i3ft92YDvA6Ohojo2NnZibnJxk5npFg9rjpq0Pzmu7LeuPccu+rl9eS1q/ezxw3Vjf9j0fg/oaXYjKPZ7u1TKHjp9uaR4PN+NTwJoZ261uxiRJi+h0w30nsLFZ3gjcP2P8I81VM5cCRzPzYJc1SpIWaM73lBFxFzAGrIyIF4FPAduAuyPiBuB54Opm84eAK4D9wOvAR/tQsyRpDnOGe2Zee4qpy2bZNoEbuy1KktQdP6EqSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUUO3/wVgaYCPz/M/Ie+3AtitbeV71lkfuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ95YZQG3dc0TS4PDIXZIKMtwlqSDDXZIKMtwlqSDDXZIK6upqmYg4ALwKvAkcy8zRiDgH+EdgBDgAXJ2ZR7orU5K0EL04ch/PzAszc7RZ3wrszsy1wO5mXZK0iPpxWmYDsKNZ3gFc1YfnkCS9jcjM0//miOeAI0ACf5uZ2yPi5cxc0cwHcOT4+knfuxnYDDA8PHzxxMTEibnp6WmGhoZOu65B0E2P+6aO9ria3hs+Ew690XYV/VW1x/WrzgL8ORwE4+Pje2ecNfkh3X5C9RczcyoifgLYFRHfmjmZmRkRs/72yMztwHaA0dHRHBsbOzE3OTnJzPWKuulx0wB8QnXL+mPcsq/2B6Cr9njgujHAn8NB19Vpmcycah4PA/cBlwCHIuJcgObxcLdFSpIW5rTDPSKWR8R7ji8DvwY8AewENjabbQTu77ZISdLCdPOechi4r3NanWXAP2TmVyPim8DdEXED8DxwdfdlSpIW4rTDPTO/DbxvlvH/Bi7rpihJUnf8hKokFWS4S1JBhrskFWS4S1JBhrskFWS4S1JBhrskFWS4S1JBhrskFWS4S1JBhrskFWS4S1JBhrskFWS4S1JBhrskFWS4S1JBhrskFVTvv26X1JWRrQ8CsGX9MTY1y4vlwLYrF/X5KjPcuzDSxQu/jR8cSe8cnpaRpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIMd0kqyHCXpIIG/q6Q3dyZUZKqGvhwl1THYh+sHb/1dsX7yBvukt7x2jwD0K9fLH075x4Rl0fEMxGxPyK29ut5JElv1Zdwj4gzgL8CPgisA66NiHX9eC5J0lv168j9EmB/Zn47M78PTAAb+vRckqSTRGb2fqcRvwlcnpm/3axfD/xcZn58xjabgc3N6vnAMzN2sRL4Xs8LW1qq91i9P6jfY/X+YPB7/KnMfO9sE639QTUztwPbZ5uLiD2ZObrIJS2q6j1W7w/q91i9P6jdY79Oy0wBa2asr27GJEmLoF/h/k1gbUScFxE/ClwD7OzTc0mSTtKX0zKZeSwiPg78E3AGcHtmPrmAXcx6uqaY6j1W7w/q91i9PyjcY1/+oCpJapc3DpOkggx3SSpoSYV7RKyJiK9HxFMR8WRE3NR2Tf0QEWdExL9HxANt19IPEbEiIu6JiG9FxNMR8fNt19RLEfH7zevziYi4KyJ+vO2auhURt0fE4Yh4YsbYORGxKyKebR7PbrPGbp2ixz9tXqePR8R9EbGixRJ7akmFO3AM2JKZ64BLgRuL3rbgJuDptovoo78EvpqZPw28j0K9RsQq4PeA0cy8gM4FA9e0W1VP3AFcftLYVmB3Zq4Fdjfrg+wO3trjLuCCzPxZ4D+Amxe7qH5ZUuGemQcz89Fm+VU6obCq3ap6KyJWA1cCX2i7ln6IiLOAXwJuA8jM72fmy60W1XvLgDMjYhnwbuC/Wq6na5n5MPDSScMbgB3N8g7gqsWsqddm6zEzv5aZx5rVR+h8JqeEJRXuM0XECPB+4F9bLqXX/gL4A+AHLdfRL+cB3wX+vjn19IWIWN52Ub2SmVPAnwEvAAeBo5n5tXar6pvhzDzYLH8HGG6zmEXwMeArbRfRK0sy3CNiCPgy8InMfKXtenolIj4EHM7MvW3X0kfLgIuAWzPz/cBrDP7b+ROa884b6PwS+0lgeUT8VrtV9V92rpkue910RPwRndPCd7ZdS68suXCPiHfRCfY7M/PetuvpsQ8AH46IA3TulPnLEfGldkvquReBFzPz+Duue+iEfRW/AjyXmd/NzP8F7gV+oeWa+uVQRJwL0DwebrmevoiITcCHgOuy0Ad/llS4R0TQOVf7dGZ+tu16ei0zb87M1Zk5QuePcP+cmaWO+jLzO8B/RsT5zdBlwFMtltRrLwCXRsS7m9frZRT6g/FJdgIbm+WNwP0t1tIXEXE5ndOkH87M19uup5eWVLjTObK9ns4R7WPN1xVtF6UF+13gzoh4HLgQ+JN2y+md5h3JPcCjwD46P0MD/xH2iLgL+Bfg/Ih4MSJuALYBvxoRz9J5x7KtzRq7dYoePw+8B9jV5M3ftFpkD3n7AUkqaKkduUuSesBwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKuj/AMq6AVKCQr12AAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "import numpy as np\n", "df['log_cite_count'] = np.log(df['cite_count'])\n", "df['log_cite_count'].hist()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Save what we have done so far." ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:23.031287Z", "start_time": "2022-03-03T18:54:21.883786Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " case_name opinion_type \\\n", "0 ERICK CORNELL CLAY v. UNITED STATES majority \n", "1 HILLSIDE DAIRY INC., A&A DAIRY, L&S DAIRY, AND... majority \n", "2 CHARLES RUSSELL RHINES v. DOUGLAS WEBER, WARDEN majority \n", "4 STATE OF ALASKA v. UNITED STATES OF AMERICA majority \n", "5 REGINALD A. WILKINSON, DIRECTOR, OHIO DEPARTME... majority \n", ".. ... ... \n", "815 MICHAEL F. EASLEY, * GOVERNOR OF NORTH CAROLIN... majority \n", "816 GAIL ATWATER, et al. v. CITY OF LAGO VISTA et al. majority \n", "817 JAMES ALEXANDER, DIRECTOR, ALABAMA DEPARTMENT ... majority \n", "818 UNITED STATES v. OAKLAND CANNABIS BUYERS' COOP... majority \n", "819 WILBERT K. ROGERS v. TENNESSEE majority \n", "\n", " date_standard authorship x_republican \\\n", "0 2003-03-04 GINSBURG 0.0 \n", "1 2003-06-09 STEVENS 1.0 \n", "2 2005-03-30 O'CONNOR 1.0 \n", "4 2005-06-06 KENNEDY 1.0 \n", "5 2005-06-13 KENNEDY 1.0 \n", ".. ... ... ... \n", "815 2001-04-18 BREYER 0.0 \n", "816 2001-04-24 SOUTER 1.0 \n", "817 2001-04-24 SCALIA 1.0 \n", "818 2001-05-14 THOMAS 1.0 \n", "819 2001-05-14 O'CONNOR 1.0 \n", "\n", " maj_judges \\\n", "0 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "1 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "2 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "4 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "5 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", ".. ... \n", "815 ['BREYER, STEPHEN', 'GINSBURG, RUTH', \"O'CONNO... \n", "816 ['KENNEDY, ANTHONY', 'REHNQUIST, WILLIAM', 'SC... \n", "817 ['KENNEDY, ANTHONY', \"O'CONNOR, SANDRA\", 'REHN... \n", "818 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "819 ['GINSBURG, RUTH', 'KENNEDY, ANTHONY', \"O'CONN... \n", "\n", " dissent_judges topic_id cite_count \\\n", "0 [] 1.0 2926.0 \n", "1 ['THOMAS, CLARENCE'] 8.0 117.0 \n", "2 [] 1.0 23364.0 \n", "4 ['REHNQUIST, WILLIAM', 'SCALIA, ANTONIN', 'THO... 10.0 84.0 \n", "5 [] 4.0 4230.0 \n", ".. ... ... ... \n", "815 ['KENNEDY, ANTHONY', 'REHNQUIST, WILLIAM', 'SC... 2.0 1236.0 \n", "816 ['BREYER, STEPHEN', 'GINSBURG, RUTH', \"O'CONNO... 1.0 3120.0 \n", "817 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'SOUTER,... 9.0 4986.0 \n", "818 [] 1.0 736.0 \n", "819 ['BREYER, STEPHEN', 'SCALIA, ANTONIN', 'STEVEN... 4.0 2100.0 \n", "\n", " opinion_text year log_cite_count \n", "0 JUSTICE GINSBURG delivered the opinion of the ... 2003 7.981392 \n", "1 Justice Stevens delivered the opinion of the C... 2003 4.762174 \n", "2 Justice O'Connor delivered the opinion of the ... 2005 10.058952 \n", "4 Justice Kennedy delivered the opinion of the C... 2005 4.430817 \n", "5 Justice Kennedy delivered the opinion of the C... 2005 8.349957 \n", ".. ... ... ... \n", "815 JUSTICE BREYER delivered the opinion of the Co... 2001 7.119636 \n", "816 JUSTICE SOUTER delivered the opinion of the Co... 2001 8.045588 \n", "817 JUSTICE SCALIA delivered the opinion of the Co... 2001 8.514389 \n", "818 JUSTICE THOMAS delivered the opinion of the Co... 2001 6.601230 \n", "819 JUSTICE O'CONNOR delivered the opinion of the ... 2001 7.649693 \n", "\n", "[768 rows x 12 columns]\n" ] } ], "source": [ "df.to_pickle('sc_cases_cleaned.pkl',compression='gzip')\n", "print(df)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Iterating over documents in a dataframe" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "In the following, we show how to iterate over a dataframe and three different ways of how to tokenize documents." ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:24.535487Z", "start_time": "2022-03-03T18:54:23.692986Z" } }, "outputs": [], "source": [ "import spacy\n", "# more infos at https://spacy.io/\n", "nlp = spacy.load('en_core_web_sm')" ] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:34.516885Z", "start_time": "2022-03-03T18:54:24.536688Z" } }, "outputs": [], "source": [ "processed = {} # empty python dictionary for processed data\n", "# iterate over rows\n", "for i, row in df.iterrows():\n", " if i >= 10:\n", " break\n", " docid = i # make document identifier\n", " text = row['opinion_text'] # get text snippet\n", " document = nlp(text) # get sentences/tokens\n", " processed[docid] = document # add to dictionary " ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:34.519332Z", "start_time": "2022-03-03T18:54:34.517544Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "opinion 1: JUSTICE GINSBURG delivered the opinion of the Court.\n", "\n", " A motion by a federal prisoner for postconviction relief under 28 U.S.C. § 2255 is subject to a one-year time limitation that generally runs from \"the date on which the judgment of conviction becomes final.\" \n", "\n", " opinion 2: Justice Stevens delivered the opinion of the Court. \n", "\n", "In most of the United States, not including California, the minimum price paid to dairy farmers producing raw milk is regulated pursuant to federal marketing orders. Those orders guarantee a uniform price for the producers,\n" ] } ], "source": [ "# first and second opinions\n", "print (\"opinion 1:\", processed[0][:50], \"\\n\\n\", \"opinion 2:\", processed[1][:50])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Let's see in more detail what information we can extract from documents procesesd using spaCy: " ] }, { "cell_type": "code", "execution_count": 21, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:34.525070Z", "start_time": "2022-03-03T18:54:34.521139Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "JUSTICE PROPN compound\n", "GINSBURG PROPN nsubj\n", "delivered VERB ROOT\n", "the DET det\n", "opinion NOUN dobj\n", "of ADP prep\n", "the DET det\n", "Court PROPN pobj\n", ". PUNCT punct\n", "\n", "\n", " SPACE dep\n", "A DET det\n", "motion NOUN nsubj\n", "by ADP prep\n", "a DET det\n", "federal ADJ amod\n", "prisoner NOUN pobj\n", "for ADP prep\n", "postconviction NOUN compound\n", "relief NOUN pobj\n", "under ADP prep\n", "28 NUM nummod\n", "U.S.C. PROPN compound\n", "§ PROPN pobj\n", "2255 NUM nummod\n", "is AUX ROOT\n", "subject ADJ acomp\n", "to ADP prep\n", "a DET det\n", "one NUM nummod\n", "- PUNCT punct\n", "year NOUN compound\n", "time NOUN compound\n", "limitation NOUN pobj\n", "that PRON nsubj\n", "generally ADV advmod\n", "runs VERB relcl\n", "from ADP prep\n", "\" PUNCT punct\n", "the DET det\n", "date NOUN pobj\n", "on ADP prep\n", "which PRON pobj\n", "the DET det\n", "judgment NOUN nsubj\n", "of ADP prep\n", "conviction NOUN pobj\n", "becomes VERB relcl\n", "final ADJ acomp\n", ". PUNCT punct\n", "\" PUNCT punct\n" ] } ], "source": [ "for token in processed[0][:50]:\n", " print(token.text, token.pos_, token.dep_)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "alternatively, we can preprocess with gensim" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:35.152483Z", "start_time": "2022-03-03T18:54:34.525947Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "opinion 1: ['justice', 'ginsburg', 'delivered', 'the', 'opinion', 'of', 'the', 'court', 'motion', 'by', 'federal', 'prisoner', 'for', 'postconviction', 'relief', 'under', 'is', 'subject', 'to', 'one', 'year', 'time', 'limitation', 'that', 'generally', 'runs', 'from', 'the', 'date', 'on', 'which', 'the', 'judgment', 'of', 'conviction', 'becomes', 'final', 'this', 'case', 'concerns', 'the', 'starting', 'date', 'for', 'the', 'one', 'year', 'limitation', 'it', 'presents'] \n", "\n", " opinion 2: ['justice', 'stevens', 'delivered', 'the', 'opinion', 'of', 'the', 'court', 'in', 'most', 'of', 'the', 'united', 'states', 'not', 'including', 'california', 'the', 'minimum', 'price', 'paid', 'to', 'dairy', 'farmers', 'producing', 'raw', 'milk', 'is', 'regulated', 'pursuant', 'to', 'federal', 'marketing', 'orders', 'those', 'orders', 'guarantee', 'uniform', 'price', 'for', 'the', 'producers', 'but', 'through', 'pooling', 'mechanisms', 'require', 'the', 'processors', 'of']\n" ] } ], "source": [ "from gensim.utils import simple_preprocess\n", "\n", "processed = {} # empty python dictionary for processed data\n", "# iterate over rows\n", "for i, row in df.iterrows():\n", " docid = i # make document identifier\n", " text = row['opinion_text'] # get text snippet\n", " document = simple_preprocess(text) # get sentences/tokens\n", " processed[docid] = document # add to dictionary \n", " if i > 100:\n", " break\n", "# first and second opinions\n", "print (\"opinion 1:\", processed[0][:50], \"\\n\\n\", \"opinion 2:\", processed[1][:50]) # note how simple preprocess drops punctuation" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "or with nltk" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:37.494062Z", "start_time": "2022-03-03T18:54:35.153364Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "opinion 1: ['justice', 'ginsburg', 'delivered', 'the', 'opinion', 'of', 'the', 'court', '.', 'a', 'motion', 'by', 'a', 'federal', 'prisoner', 'for', 'postconviction', 'relief', 'under', '28', 'u.s.c', '.', '§', '2255', 'is', 'subject', 'to', 'a', 'one-year', 'time', 'limitation', 'that', 'generally', 'runs', 'from', '``', 'the', 'date', 'on', 'which', 'the', 'judgment', 'of', 'conviction', 'becomes', 'final', '.', \"''\", '§', '2255'] \n", "\n", " opinion 2: ['justice', 'stevens', 'delivered', 'the', 'opinion', 'of', 'the', 'court', '.', 'in', 'most', 'of', 'the', 'united', 'states', ',', 'not', 'including', 'california', ',', 'the', 'minimum', 'price', 'paid', 'to', 'dairy', 'farmers', 'producing', 'raw', 'milk', 'is', 'regulated', 'pursuant', 'to', 'federal', 'marketing', 'orders', '.', 'those', 'orders', 'guarantee', 'a', 'uniform', 'price', 'for', 'the', 'producers', ',', 'but', 'through']\n" ] } ], "source": [ "from nltk.tokenize import word_tokenize\n", "processed = {} # empty python dictionary for processed data\n", "# iterate over rows\n", "for i, row in df.iterrows():\n", " docid = i # make document identifier\n", " text = row['opinion_text'] # get text snippet\n", " document = word_tokenize(text.lower()) # get sentences/tokens\n", " processed[docid] = document # add to dictionary \n", " if i > 100:\n", " break\n", "# first and second opinions\n", "print (\"opinion 1:\", processed[0][:50], \"\\n\\n\", \"opinion 2:\", processed[1][:50]) # note that we just tokenize and keep all tokens\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Saving data" ] }, { "cell_type": "code", "execution_count": 24, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:37.554721Z", "start_time": "2022-03-03T18:54:37.494841Z" } }, "outputs": [], "source": [ "# save as python pickle\n", "pd.to_pickle(processed, 'processed_corpus.pkl')\n", "# delete it\n", "import os \n", "os.remove('processed_corpus.pkl')" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:37.557086Z", "start_time": "2022-03-03T18:54:37.555589Z" } }, "outputs": [], "source": [ "# Merging Data-frames Example\n", "# Perform a left join:\n", "# df_merged = pd.merge(df1,df2,on='id', how='left', validation='m:1')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Web Scraping" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Downloading URL's" ] }, { "cell_type": "code", "execution_count": 26, "metadata": { "ExecuteTime": { "end_time": "2022-03-03T18:54:39.084595Z", "start_time": "2022-03-03T18:54:37.557840Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "b'\\n\\n\\n\\n \\n \\n \\n \\n\\n \\n \\n \\n \\n\\n\\n\\n'\n", "\n", "78512 characters in string.\n" ] } ], "source": [ "import urllib.request as urllib # Python's module for accessing web pages\n", "url = 'https://goo.gl/VRF8Xs' # shortened URL for court case\n", "page = urllib.urlopen(url) # open the web page\n", "\n", "html = page.read() # read web page contents as a string\n", "print(html[:400]) # print first 400 characters\n", "print()\n", "print(html[-400:]) # print last 400 characters\n", "print()\n", "print(len(html),'characters in string.') # print length of string" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Parsing HTML" ] }, { "cell_type": "code", "execution_count": 291, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:32:09.647832Z", "start_time": "2022-02-25T09:32:09.602699Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "People v. Germany, 674 P.2d 345 – CourtListener.com\n" ] } ], "source": [ "# Parse raw HTML\n", "# !pip install beautifulsoup4\n", "from bs4 import BeautifulSoup # package for parsing HTML\n", "soup = BeautifulSoup(html, 'lxml') # parse html of web page\n", "print(soup.title) # example usage: print title item" ] }, { "cell_type": "code", "execution_count": 292, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:32:18.769526Z", "start_time": "2022-02-25T09:32:18.758006Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "519\n" ] } ], "source": [ "# extract text\n", "text = soup.get_text() # get text (remove HTML markup)\n", "lines = text.splitlines() # split string into separate lines\n", "print(len(lines)) # print number of lines" ] }, { "cell_type": "code", "execution_count": 293, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:32:26.434807Z", "start_time": "2022-02-25T09:32:26.422384Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "187\n" ] } ], "source": [ "lines = [line for line in lines if line != ''] # drop empty lines\n", "print(len(lines)) # print number of lines" ] }, { "cell_type": "code", "execution_count": 294, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:32:30.847356Z", "start_time": "2022-02-25T09:32:30.839684Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['People v. Germany, 674 P.2d 345 – CourtListener.com', 'Toggle navigation', 'About', 'FAQ', 'Donate', 'Sign in / Register', 'From Free Law Project, a 501(c)(3) non-profit.', 'Opinions\\xa0', 'Advanced Search', 'Citation Look Up', 'Citation Visualizations', 'RECAP Archive', 'Oral Arguments', 'Judges', 'Financial Disclosures', '\\xa0Donate', 'Your Notes', ' (edit)', ' ', ' (none)']\n" ] } ], "source": [ "print(lines[:20]) # print first 20 lines" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Removing unicode characters" ] }, { "cell_type": "code", "execution_count": 295, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:32:53.962331Z", "start_time": "2022-02-25T09:32:53.952376Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['Visualizations\\xa0'] ['Visualizations ']\n" ] } ], "source": [ "# !pip install unidecode\n", "from unidecode import unidecode # package for removing unicode\n", "uncode_str = 'Visualizations\\xa0'\n", "fixed = unidecode(uncode_str) # example usage\n", "print([uncode_str],[fixed]) # print cleaned string (replaced with a space)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Quantity of Text" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Count words per document." ] }, { "cell_type": "code", "execution_count": 301, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:35:39.259306Z", "start_time": "2022-02-25T09:35:39.016620Z" } }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 301, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAURUlEQVR4nO3dbYxcV33H8e+fJKRWlsZxE1auY3VDcVuFWJhkFYJAaDcISMILB4lGQYjYkMptFSRQ3SqGviiUIqUPAYlC0y5KGgcoSxqIYuWhNDVZRXkRUpsaPySELMSUrIwtiOOwQNM6/ffFHKfDsusdz+zM7J79fqTR3Hvuwzn/veOf7965MxuZiSSpLi/r9wAkSQvPcJekChnuklQhw12SKmS4S1KFTu/3AADOPffcHBoaamvbn/70p5x11lkLO6BFzHrrt9xqtt727d69+0eZed5syxZFuA8NDbFr1662tp2YmGBkZGRhB7SIWW/9llvN1tu+iPj+XMu8LCNJFTLcJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mqkOEuSRVaFJ9QXaqGtt3X8z63rj/OSM97lbTUeOYuSRWaN9wj4lci4rGI+FZEHIiIj5X22yPi6YjYUx4bSntExKcjYjIi9kbExV2uQZI0QyuXZV4ALs/M6Yg4A3gkIh4oy/4kM++asf6VwLryeD1wS3mWJPXIvGfu2TBdZs8oj5P9Ve2NwB1lu0eBlRGxuvOhSpJaFZkny+myUsRpwG7g1cBnM/PGiLgdeAONM/udwLbMfCEi7gVuysxHyrY7gRszc9eMfW4BtgAMDg5eMj4+3lYB09PTDAwMtLVtp/ZNHet5n4Mr4JWrzu55v/3Sz+PbL8utZutt3+jo6O7MHJ5tWUt3y2Tmi8CGiFgJ3B0RFwEfBn4IvBwYA24E/rzVQWXmWNmO4eHhbPf7jfv5XdCb+3S3zDV+93XVllvN1tsdp3S3TGY+BzwEXJGZh8qllxeAfwQuLatNAWubNju/tEmSeqSVu2XOK2fsRMQK4K3At09cR4+IAK4G9pdNdgDXlbtmLgOOZeahLoxdkjSHVi7LrAa2l+vuLwPuzMx7I+LrEXEeEMAe4A/K+vcDVwGTwM+A9y34qCVJJzVvuGfmXuB1s7RfPsf6CdzQ+dAkSe3yE6qSVCHDXZIqZLhLUoUMd0mqkOEuSRUy3CWpQoa7JFXIcJekChnuklQhw12SKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SarQvOEeEb8SEY9FxLci4kBEfKy0XxAR34iIyYj4ckS8vLSfWeYny/KhLtcgSZqhlTP3F4DLM/O1wAbgioi4DPhL4FOZ+WrgKHB9Wf964Ghp/1RZT5LUQ/OGezZMl9kzyiOBy4G7Svt24OoyvbHMU5a/JSJioQYsSZpfZOb8K0WcBuwGXg18Fvhr4NFydk5ErAUeyMyLImI/cEVmPlOWfRd4fWb+aMY+twBbAAYHBy8ZHx9vq4Dp6WkGBgba2rZT+6aO9bzPwRXwylVn97zffunn8e2X5Vaz9bZvdHR0d2YOz7bs9FZ2kJkvAhsiYiVwN/A7nQ4qM8eAMYDh4eEcGRlpaz8TExO0u22nNm+7r+d9bl1/nGv6VG8/9PP49styq9l6u+OU7pbJzOeAh4A3ACsj4sR/DucDU2V6ClgLUJafDfx4IQYrSWpNK3fLnFfO2ImIFcBbgSdohPy7ymqbgHvK9I4yT1n+9Wzl2o8kacG0cllmNbC9XHd/GXBnZt4bEY8D4xHxF8B/ALeW9W8FPh8Rk8CzwLVdGLck6STmDffM3Au8bpb27wGXztL+X8DvLsjoJElt8ROqklQhw12SKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFTLcJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mq0LzhHhFrI+KhiHg8Ig5ExAdL+0cjYioi9pTHVU3bfDgiJiPiyYh4ezcLkCT9stNbWOc4sDUzvxkRrwB2R8SDZdmnMvNvmleOiAuBa4HXAL8O/FtE/FZmvriQA5ckzW3eM/fMPJSZ3yzTPwGeANacZJONwHhmvpCZTwOTwKULMVhJUmsiM1tfOWIIeBi4CPgjYDPwPLCLxtn90Yj4DPBoZn6hbHMr8EBm3jVjX1uALQCDg4OXjI+Pt1XA9PQ0AwMDbW3bqX1Tx3re5+AKeOWqs3veb7/08/j2y3Kr2XrbNzo6ujszh2db1splGQAiYgD4CvChzHw+Im4BPg5keb4ZeH+r+8vMMWAMYHh4OEdGRlrd9BdMTEzQ7rad2rztvp73uXX9ca7pU7390M/j2y/LrWbr7Y6W7paJiDNoBPsXM/OrAJl5ODNfzMz/BT7H/196mQLWNm1+fmmTJPVIK3fLBHAr8ERmfrKpfXXTau8E9pfpHcC1EXFmRFwArAMeW7ghS5Lm08plmTcC7wX2RcSe0vYR4N0RsYHGZZmDwO8DZOaBiLgTeJzGnTY3eKeMJPXWvOGemY8AMcui+0+yzSeAT3QwLklSB/yEqiRVyHCXpAoZ7pJUIcNdkipkuEtShVr+hKoWj6E+fDL2hIM3vaNvfUtqnWfuklQhw12SKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFTLcJalC84Z7RKyNiIci4vGIOBARHyztqyLiwYh4qjyfU9ojIj4dEZMRsTciLu52EZKkX9TKmftxYGtmXghcBtwQERcC24CdmbkO2FnmAa4E1pXHFuCWBR+1JOmk5g33zDyUmd8s0z8BngDWABuB7WW17cDVZXojcEc2PAqsjIjVCz1wSdLcIjNbXzliCHgYuAj4z8xcWdoDOJqZKyPiXuCmzHykLNsJ3JiZu2bsawuNM3sGBwcvGR8fb6uA6elpBgYG2tq2U/umjvW8z8EVcPjnPe/2JevXnN3T/vp5fPtludVsve0bHR3dnZnDsy1r+c/sRcQA8BXgQ5n5fCPPGzIzI6L1/yUa24wBYwDDw8M5MjJyKpu/ZGJigna37dTmPvy5u63rj3Pzvv79dcSD7xnpaX/9PL79stxqtt7uaOlumYg4g0awfzEzv1qaD5+43FKej5T2KWBt0+bnlzZJUo+0crdMALcCT2TmJ5sW7QA2lelNwD1N7deVu2YuA45l5qEFHLMkaR6t/H7/RuC9wL6I2FPaPgLcBNwZEdcD3weuKcvuB64CJoGfAe9byAFLkuY3b7iXN0ZjjsVvmWX9BG7ocFySpA74CVVJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFTLcJalChrskVchwl6QKGe6SVCHDXZIqZLhLUoUMd0mqkOEuSRUy3CWpQvOGe0TcFhFHImJ/U9tHI2IqIvaUx1VNyz4cEZMR8WREvL1bA5ckza2VM/fbgStmaf9UZm4oj/sBIuJC4FrgNWWbv4uI0xZqsJKk1swb7pn5MPBsi/vbCIxn5guZ+TQwCVzawfgkSW3o5Jr7ByJib7lsc05pWwP8oGmdZ0qbJKmHIjPnXyliCLg3My8q84PAj4AEPg6szsz3R8RngEcz8wtlvVuBBzLzrln2uQXYAjA4OHjJ+Ph4WwVMT08zMDDQ1rad2jd1rOd9Dq6Awz/vebcvWb/m7J7218/j2y/LrWbrbd/o6OjuzByebdnp7ewwMw+fmI6IzwH3ltkpYG3TqueXttn2MQaMAQwPD+fIyEg7Q2FiYoJ2t+3U5m339bzPreuPc/O+tg7bgjj4npGe9tfP49svy61m6+2Oti7LRMTqptl3AifupNkBXBsRZ0bEBcA64LHOhihJOlXzngJGxJeAEeDciHgG+DNgJCI20LgscxD4fYDMPBARdwKPA8eBGzLzxa6MXJI0p3nDPTPfPUvzrSdZ/xPAJzoZlCSpM35CVZIqZLhLUoUMd0mqkOEuSRUy3CWpQoa7JFXIcJekChnuklQhw12SKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SaqQ4S5JFZo33CPitog4EhH7m9pWRcSDEfFUeT6ntEdEfDoiJiNib0Rc3M3BS5Jm18qZ++3AFTPatgE7M3MdsLPMA1wJrCuPLcAtCzNMSdKpmDfcM/Nh4NkZzRuB7WV6O3B1U/sd2fAosDIiVi/QWCVJLYrMnH+liCHg3sy8qMw/l5kry3QARzNzZUTcC9yUmY+UZTuBGzNz1yz73ELj7J7BwcFLxsfH2ypgenqagYGBtrbt1L6pYz3vc3AFHP55z7t9yfo1Z/e0v34e335ZbjVbb/tGR0d3Z+bwbMtO73TnmZkRMf//EL+83RgwBjA8PJwjIyNt9T8xMUG723Zq87b7et7n1vXHuXlfx4etbQffM9LT/vp5fPtludVsvd3R7t0yh09cbinPR0r7FLC2ab3zS5skqYfaDfcdwKYyvQm4p6n9unLXzGXAscw81OEYJUmnaN7f7yPiS8AIcG5EPAP8GXATcGdEXA98H7imrH4/cBUwCfwMeF8XxixJmse84Z6Z755j0VtmWTeBGzod1KnYN3WsL9e+JWkx8xOqklQhw12SKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRVyHCXpAoZ7pJUIcNdkipkuEtShQx3SapQ//7qg5akoR5/SdvW9cfZvO0+Dt70jp72Ky11nrlLUoUMd0mqkOEuSRUy3CWpQoa7JFXIcJekChnuklShju5zj4iDwE+AF4HjmTkcEauALwNDwEHgmsw82tkwJUmnYiHO3Eczc0NmDpf5bcDOzFwH7CzzkqQe6sZlmY3A9jK9Hbi6C31Ikk4iMrP9jSOeBo4CCfxDZo5FxHOZubIsD+DoifkZ224BtgAMDg5eMj4+3tYYjjx7jMM/b2/8S9HgCpZlvevXnN3vofTM9PQ0AwMD/R5Gz1hv+0ZHR3c3XTX5BZ1+t8ybMnMqIl4JPBgR325emJkZEbP+75GZY8AYwPDwcI6MjLQ1gL/94j3cvG/5fEXO1vXHl2W9B98z0u+h9MzExATt/ntYiqy3Ozq6LJOZU+X5CHA3cClwOCJWA5TnI50OUpJ0atoO94g4KyJecWIaeBuwH9gBbCqrbQLu6XSQkqRT08nv94PA3Y3L6pwO/FNm/ktE/DtwZ0RcD3wfuKbzYUqSTkXb4Z6Z3wNeO0v7j4G3dDIoSVJn/ISqJFXIcJekChnuklQhw12SKmS4S1KFDHdJqtDy+Ry7lrShbff1re+DN72jb31L7fLMXZIqZLhLUoUMd0mqkOEuSRUy3CWpQoa7JFXIcJekChnuklQhw12SKmS4S1KF/PoBaR69/uqDreuPs3nbfX7tgTrimbskVchwl6QKdS3cI+KKiHgyIiYjYlu3+pEk/bKuhHtEnAZ8FrgSuBB4d0Rc2I2+JEm/rFtvqF4KTGbm9wAiYhzYCDzepf6k6vTzO+x76cQbyMvFzHq79cZ5ZObC7zTiXcAVmfl7Zf69wOsz8wNN62wBtpTZ3waebLO7c4EfdTDcpcZ667fcarbe9v1GZp4324K+3QqZmWPAWKf7iYhdmTm8AENaEqy3fsutZuvtjm69oToFrG2aP7+0SZJ6oFvh/u/Auoi4ICJeDlwL7OhSX5KkGbpyWSYzj0fEB4CvAacBt2XmgW70xQJc2llirLd+y61m6+2CrryhKknqLz+hKkkVMtwlqUJLNtxr+nqDiDgYEfsiYk9E7CptqyLiwYh4qjyfU9ojIj5d6t4bERc37WdTWf+piNjUr3pmExG3RcSRiNjf1LZgNUbEJeVnOFm2jd5W+IvmqPejETFVjvOeiLiqadmHy9ifjIi3N7XP+jovNyt8o7R/udy40DcRsTYiHoqIxyPiQER8sLRXeYxPUu/iOcaZueQeNN6k/S7wKuDlwLeAC/s9rg7qOQicO6Ptr4BtZXob8Jdl+irgASCAy4BvlPZVwPfK8zll+px+19ZUz5uBi4H93agReKysG2XbKxdhvR8F/niWdS8sr+EzgQvKa/u0k73OgTuBa8v03wN/2Od6VwMXl+lXAN8pdVV5jE9S76I5xkv1zP2lrzfIzP8GTny9QU02AtvL9Hbg6qb2O7LhUWBlRKwG3g48mJnPZuZR4EHgih6PeU6Z+TDw7IzmBamxLPvVzHw0G/8S7mjaV1/MUe9cNgLjmflCZj4NTNJ4jc/6Oi9nrJcDd5Xtm392fZGZhzLzm2X6J8ATwBoqPcYnqXcuPT/GSzXc1wA/aJp/hpP/YBe7BP41InZH42sZAAYz81CZ/iEwWKbnqn0p/kwWqsY1ZXpm+2L0gXIZ4rYTlyg49Xp/DXguM4/PaF8UImIIeB3wDZbBMZ5RLyySY7xUw702b8rMi2l8i+YNEfHm5oXlTKXqe1aXQ43ALcBvAhuAQ8DNfR1NF0TEAPAV4EOZ+XzzshqP8Sz1LppjvFTDvaqvN8jMqfJ8BLibxq9qh8uvopTnI2X1uWpfij+ThapxqkzPbF9UMvNwZr6Ymf8LfI7GcYZTr/fHNC5jnD6jva8i4gwaQffFzPxqaa72GM9W72I6xks13Kv5eoOIOCsiXnFiGngbsJ9GPSfuFNgE3FOmdwDXlbsNLgOOlV97vwa8LSLOKb8Kvq20LWYLUmNZ9nxEXFauVV7XtK9F40TIFe+kcZyhUe+1EXFmRFwArKPx5uGsr/NyBvwQ8K6yffPPri/Kz/1W4InM/GTToiqP8Vz1Lqpj3K93mzt90Hi3/Ts03mn+036Pp4M6XkXjHfJvAQdO1ELjmttO4Cng34BVpT1o/CGU7wL7gOGmfb2fxhs1k8D7+l3bjDq/ROPX1P+hcf3w+oWsERgu/5C+C3yG8unrRVbv50s9e8s/9tVN6/9pGfuTNN0FMtfrvLxuHis/h38GzuxzvW+iccllL7CnPK6q9RifpN5Fc4z9+gFJqtBSvSwjSToJw12SKmS4S1KFDHdJqpDhLkkVMtwlqUKGuyRV6P8AE8dw10s3oFsAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "def get_words_per_doc(txt):\n", " # split text into words and count them.\n", " return len(txt.split()) \n", "\n", "# apply to our dataframe\n", "df['num_words'] = df['opinion_text'].apply(get_words_per_doc)\n", "df['num_words'].hist()" ] }, { "cell_type": "code", "execution_count": 302, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:35:42.316770Z", "start_time": "2022-02-25T09:35:42.234336Z" } }, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEGCAYAAACUzrmNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA+rUlEQVR4nO3deXyU9bX48c/JRiBkYUlCSNghECABJVL3FVyqVettrVrr0lbr0tbe3trqrb/aanu72sW2Wq1L1brUuu8KClp3gkACCWFHEkIWIJOQkHXO7495BseYZZLMmpz36/W8MvOdZ545D8ucfHdRVYwxxpjexIQ7AGOMMZHPkoUxxpg+WbIwxhjTJ0sWxhhj+mTJwhhjTJ/iwh1AsIwfP16nTp0a7jCMMSZqrF69uk5V07t7bcgmi6lTp1JUVBTuMIwxJmqIyM6eXrNmKGOMMX2yZGGMMaZPliyMMcb0yZKFMcaYPlmyMMYY0ydLFsYYY/pkycIYY0yfLFkYY4akmsYWnlu3O9xhDBmWLIwxQ9IdK7by3UfXUN3QEu5QQuaf7+/kfx5fh9sd+H2KLFkYY4akFeU1ABRXuMIcSei8sbGG4op6YmIk4Ne2ZGGMGXK21zWxc28zACUV9eENJkRUleIKF/k5qUG5viULY8yQs9KpVYxLSqC4cnjULPY0tFB3oJWCbEsWxhjjlxXltUxPT+LkORmUVLhQDXwbfqTxNrfl56QF5fqWLIwxQ8rBtk7e37aXE3MzKMhJZW9TG7tdQ7+Tu7iintgYYd7ElKBcf8guUW6MGZ7e37aXtg43J85OJznR8xVXUuEiO21kmCMLruIKF7mZySTGxwbl+lazMMYMKSvKaxgZH8viaWPJy0ohLkYoqawPd1hBpaqUVLqC1l8BQU4WIpImIk+IyEYRKRORo0Tkt87zYhF5WkTSnHOnishBEVnrHH/zuc4iESkRkS0icruIBH5cmDEm6qkqK8trOXrGOBLjY0mMjyU3M3nID5+t2H+Q+uZ2CiZFabIA/gS8oqpzgAVAGbAMmK+qBcAm4Eaf87eq6kLnuMqn/E7gCmCWc5we5LiNMVFoe10TH+9r5sTZn+wMWpCTSknl0O7kXucMDy7ITgvaZwQtWYhIKnA8cC+Aqrapar2qvqaqHc5p7wM5fVwnC0hR1ffV87f9IHBusOI2xkSvleW1AJw4O+NQWX5OKvXN7VTsPxiusIKupMJFQmwMuRNGB+0zglmzmAbUAveLyBoRuUdEkrqc83XgZd/3OOe+KSLHOWXZQIXPORVO2WeIyJUiUiQiRbW1tQG6DWNMtFhRXsP09CQmjR11qMz72/ZQbooqrnAxJyuZEXHB6dyG4CaLOOBw4E5VPQxoAm7wvigiPwY6gIedoipgsnPu94FHRKRfY8BU9W5VLVTVwvT09L7fYIwZMg62dfLB9n2c5FOrAMidMJqE2BiKh2gnt9utrK90URCkmdtewUwWFUCFqn7gPH8CT/JARC4DzgK+6jQtoaqtqrrXebwa2ArkApV8uqkqxykzxphD3ttWd2jIrK8RcbHMyUqmZIjWLLbvbaKxtSOo/RUQxGShqnuAXSIy2yk6BSgVkdOBHwJnq2qz93wRSReRWOfxdDwd2dtUtQpoEJEjnVFQlwDPBituY0x0WrGx9tCQ2a7ysz2d3MFYjTXcSg7N3I7emgXAd4CHRaQYWAj8H/AXIBlY1mWI7PFAsYisxVMLuUpV9zmvXQPcA2zBU+Pw7ecwxgxzqsrKTTUcM3Nct+32BTmpNLZ0sHNfczfvjm7FFS4S42OYlRG8zm0I8gxuVV0LFHYpntnDuU8CT/bwWhEwP6DBGWOGjG11Tezad5Arj5/R7ev5hzq565k2vus4m+hWXFHPvImpxMUG93d/m8FtjIl6h4bM5nY/sGVW5mhGxMUMuX6Ljk43G3Y3kB/EmdteliyMMVFvZXkNM7oMmfUVHxvD3IkpQ2658q21TRxs7wz6SCiwZGGMiXLNbR18sO2zQ2a7KshOZUOli84h1Mld7J25bcnCGGN6997WvbR1uj81a7s7+TlpNLV1sr3uQIgiC77iChdJCbFMHx/czm2wZGGMiXIrymsYlRDLEdPG9Hqe97fvoTSTu7jSxfzs1KDsud2VJQtjTNT6ZJXZ8X0udTEjfTQj42OHTLJo63BTVtUQkiYosGRhjIliW2ubqNh/8DOztrsTGyPMz06hZIh0cm+qbqStwx20bVS7smRhjIlaK8trAPxKFuCZb7Fht4uOTncwwwoJbw1pgdUsjDGmdyvLa5mZMZqcMd0Pme2qICeVlnY3W2qjv5O7pLKelMQ4JvcwXDjQLFkYY6JSU2sHH27fx0l+1irgk/WThkK/RXGFi4KcNEK1caglC2NMVPJ3yKyvaeOSGD0iLupncre0d1K+pzFkndtgycIYE6W8Q2YLp/Y+ZNZXjNPJHe0zucuqGuhwqyULY4zpTX+GzHZVkJNGWVUDbR3R28ntHdEVqpFQYMnCGBOFttYeoLL+ICfN6f+OmPnZqbR1uNlU3RiEyEKjuMLFuKQEJqYmhuwzLVkYY6LOoVVm+9Ff4eVtuonm+RYlFZ5tVEPVuQ2WLIwxUWhFeQ2zMkaTnTay3++dPHYUKYlxUTsiqrmtg801jSFtggJLFsaYKNPU2sGq7fs5aU7/axUAIkJBThollfWBDSxENuxuwK2eVXRDyZKFMSaqvOsdMtvDRkf+yM9JpXxPIy3tnQGMLDS8NaJQjoQCSxbGmCizsryGpIRYCqeOHfA1CrJTae9UyvdEXyd3cUU9E1ISyUgJXec2BDlZiEiaiDwhIhtFpExEjhKRsSKyTEQ2Oz/HOOeKiNwuIltEpFhEDve5zqXO+ZtF5NJgxmyMiVyHhszOHE9C3MC/vg7N5I7CTu6SCteh+EMp2DWLPwGvqOocYAFQBtwAvK6qs4DXnecAZwCznONK4E4AERkL3Ax8DlgM3OxNMMaY4WVLjTNkdgCjoHxlp41kbFICJc5Oc9GioaWdbXVNIe+vgCAmCxFJBY4H7gVQ1TZVrQfOAR5wTnsAONd5fA7woHq8D6SJSBZwGrBMVfep6n5gGXB6sOI2xkSuT4bMDry/Ajyd3PnZqVE3Imr9ocl4QyhZANOAWuB+EVkjIveISBKQqapVzjl7gEzncTawy+f9FU5ZT+WfISJXikiRiBTV1tYG8FaMMZFgRXkNuZmjmTiAIbNdFeSksrnmAAfboqeT+5PO7bSQf3Ywk0UccDhwp6oeBjTxSZMTAKqqQMB2T1fVu1W1UFUL09MH95uHMSayHGjtYNWOfQOaiNed/OxUOt1KaVVDQK4XCiUVLnLGeJrQQs2vZCEiR4vIRSJyiffw420VQIWqfuA8fwJP8qh2mpdwftY4r1cCk3zen+OU9VRujBlG3t1SR3unDroJysv723k09VsUV9aHfMisV5/JQkQeAn4HHAsc4RyFfb1PVfcAu0RktlN0ClAKPAd4RzRdCjzrPH4OuMQZFXUk4HKaq14FThWRMU7H9qlOmTFmGFm5qdYzZHbKwIfM+spMGUF68oioGRG1v6mNXfsOhqUJCjxNRX0pBOY6TUb99R3gYRFJALYBl+NJUI+LyDeAncD5zrkvAZ8HtgDNzrmo6j4RuRVY5Zx3i6ruG0Asxpgopaqs3FjDMYMcMutLRCjITo2avS28SS0cI6HAv2SxHpgAVPV1YlequpbuayGndHOuAtf2cJ37gPv6+/nGmKFhc80Bdrta+M4pswJ63fycVN4or6GptYOkEf58HYaPt7lsXqQlCxF5Hk/nczJQKiIfAq3e11X17OCHZ4wxnlnbMPghs10V5KSi6llvafG0wDRvBUtxhYtp45NIHRkfls/vLZX+LmRRGGNML1ZsrGV2ZjJZqYMfMutrfrZ3T+76iE8WJZWusMbYY+Ofqr6pqm8Cn/c+9i0LXYjGmOHsQGsHRTv3BbxWAZCRnEhWamLE721R09hClauF/DA1QYF/Q2eXdlN2RqADMcaY7rxzaMhsYOZXdJUfBZ3cJWGcjOfVY7IQkatFpASY7Szs5z22A8WhC9EYM5ytLK9l9Ig4CqcGZ0m4gpxUttU10dDSHpTrB0JxhYsYgXkTU8IWQ299Fo8ALwO/5NMzrxtt6KoxJhQ8q8zWcMzMccTHBmfBCe+Oc+srXRw9Y3xQPmOwSipdzMwYHdYRW731WbhUdQee4ayNPgciEp7ueGPMsLKp+gBVrpagNUEBh/oB1kdov4WqUlxRT352Wljj8CdVf4RnQcBNwGbn8Q4R+UhEFgUzOGPM8BasIbO+xiYlkJ02MmJXoK1ytVB3oC1sy3x4+ZMsluEZETVeVcfh6dx+AbgGuCOYwRljhrcV5TXMmRD4IbNdFeSkRuyIKG8SC8ey5L78SRZHquqhtZhU9TXgKGfPiRFBi8wYM6w1trRTtGM/JwSxVuGVn5PKzr3NuJojr5O7pLKeuBhhblb4OrfBv2RRJSI/EpEpzvFDPCvHxgLuIMdnjBmm3tmylw63DnpXPH8UOP0BkVi7KK5wkZuZTGJ8bFjj8CdZXIRnWfBnnGOyUxbLJ4sAGmNMQL25qYbkEXEsmhL8XZS9ndzFlfVB/6z+8HRuu8LeXwF+LCSoqnV4Vo/tzpbAhmOMMZ4vyRUbazlm5vigDZn1lToqninjRkXc5Lxd+w7iOtge1sl4Xn0mCxHJBX4ATPU9X1VPDl5YxpjhrLy6kT0NLUEdBdVVfnYqaz6uD9nn+WOds9JsVNQsgH8DfwPuAaJns1pjTNRaWV4LENT5FV0V5KTyQnEVew+0Mm50ZIzdKal0kRAbQ25mcrhD8StZdKjqnUGPxBhjHCs2eobMTkhNDNln5vt0cocySfWmuKKevKzkgG34NBj+RPC8iFwjIlkiMtZ7BD0yY8yw1NjSzuqd+0P+hT0/2zM0NVL6LdxuZX1lQ0T0V4B/NQvvftnX+5QpMD3w4Rhjhrt3ttTR4daQ9lcAJCfGMz09KWL25N5W18SB1o6wT8bz8mc01LRQBGKMMeDprwjVkNmuCrJTeX9bZKyTWuIM442Ezm3woxlKREaJyE0icrfzfJaInBX80Iwxw41nldlajp0VmiGzXeXnpLGnoYWahpaQf3ZXxRUuEuNjmJk+OtyhAP71WdwPtAFHO88rgZ/7c3ER2SEiJSKyVkSKnLJ/Oc/XOq+vdcqnishBn9f+5nOdRc51tojI7SIi/blJY0x02Lgn9ENmfXl/i4+EmdwlFS7mT0wlLgxJszv+RDFDVX8DtAOoajPQny/rk1R1oaoWOu//ivN8IfAk8JTPuVu9r6nqVT7ldwJXALOc4/R+fL4xJkqEY8isr7lZKcQIYV+BtqPTzfrdrojprwD/kkWbiIzE06mNiMwAWgf7wU7t4Hzg0T7OywJSVPV9VVXgQeDcwX6+MSbyrCyvIS8rhcyU0A2Z9ZU0Io6ZGaPDXrPYUnuAlnZ3xPRXgH/J4mbgFWCSiDwMvA780M/rK/CaiKwWkSu7vHYcUK2qm33KponIGhF5U0SOc8qygQqfcyqcss8QkStFpEhEimpra/0M0RgTCRpa2inauT9sTVBe+dlpFFe48PxuGh6HliUP84ZHvvpMFqq6DDgPuAxPLaAQOOjn9Y9V1cPx7IFxrYgc7/PahXy6VlEFTFbVw4DvA4+ISL/W5FXVu1W1UFUL09PD+w/OGNM/72yuo9OtnJgb3v+7BTmp1B1oZU8YO7lLKlyMHhHH9PFJYYuhK782dFXVvcCL3uci8hGe1Wf7el+l87NGRJ4GFgNviUgcngS0yOfcVpzmLVVdLSJbgVw8Heo5PpfNccqMMUPIyvJakhPjODwMQ2Z9efsJiitcQd90qSfFFfXMz04hJiZyxvIMtJu9zzsQkSQRSfY+Bk4F1jsvLwE2qmqFz/npzh4ZiMh0PB3Z21S1CmgQkSOdfo5LgGcHGLcxJgKpKis31XBcmIbM+pqblUJsjIRtJndbh5uyqsaImbnt5VfNohv+NOZlAk87o1zjgEdU9RXntQv4bMf28cAtItKOZ1Olq1TVOzvmGuAfwEjgZecwxgwRZVWNVDe0cmJu+NdkSoyPJTczOWwzuTdVN9LW6T60x0ak6DFZiMjzdJ8UBBjX14VVdRuwoIfXLuum7Ek8Q2m7O78ImN/XZxpjotPKTTUAIdlC1R8F2am8VroHVSXU07q8ndsLoqhm8bsBvmaMMf2ysryWuWEcMttVfk4q/yraRcX+g0waOyqkn11cUU/qyHgmjQ1Pf0lPekwWqvpmKAMxxgxProOeVWa/dXzkrE3qO5M79MnCs41qpC1UERnzyI0xw9Y7W5whsxGyhwTA7AnJxMdKyGdyt7R3sqm6MaIm43lZsjDGhNXK8hrPkNnJaeEO5ZARcbHMmZByaOXXUCmtaqDDrRE1Gc/LkoUxJmy8q8wePys9YhbM88rPSQ35TG7vcN2orFmISK6I/F1EXhORN7xHKIIzxgxtpVUN1DS2RswoKF8F2ak0tnSwc29zyD6zuMLF+NEJZIVwO1l/+TPP4t/A34C/A53BDccYM5wcWmU2zEt8dOfQTO5KF1NDtOxGSWU9BTlpEde5Df4liw5VvTPokRhjhp03y2uZNzGFjAgZMusrNzOZhLgYSirqOXvBxKB/XlNrB1tqDnDG/Kygf9ZA+NNI+LyIXCMiWSIy1nsEPTJjzJDmOtjO6o/Dv8psT+JjY5iblRKyEVEbdjfg1sjsrwD/ahaXOj+v9ylTIHIGRRtjos7bmyNvyGxXBTmpPLm6Ardbg76oX3FFPUBEbXjky58lyqd1c1iiMMYMysryGlIS4zhsUlq4Q+lRfnYqTW2dbKtrCvpnlVS6yEpNJCM58prkwL/RUPEi8l0RecI5vi0i8aEIzhgzNLndyspNtRyXG3lDZn15V34NxXyL4gpXxC0e6Mufv6U78ew7cYdzLHLKjDFmQEqrGqhtbI3IUVC+ZqQnMTI+Nuj9Fq6D7Wyva4rY/grwr8/iCFX1XT32DRFZF6yAjDFD35ubPENmI3F+ha+42BjmTUwJ+t4WG5zl0PMjbKVZX/7ULDpFZIb3ibMxkc23MMYM2MryGuZnp0Rs+7yv/JxUNuxuoKPTHbTP8O6dURDlzVDXAytEZKWIvAm8AfxPcMMyxgxVrmbPKrORsNGRPwpyUjnY3snW2uB1chdX1DNp7EjGJCUE7TMGq89mKFV9XURmAbOdonJnv2xjjOm3/2ypxa1E7PyKrryL+hVX1DN7QnJQPqO4whVxmx111WPNQkROdn6eB5wJzHSOM50yY4zpt5XltaQkxrEwgofM+po+PomkhFhKgrTN6r6mNir2H4zozm3ovWZxAp4mpy9085oCTwUlImPMkOV2O6vMRviQWV8xMcL87NSgjYgqOdS5HaXJQlVvdn5ePtCLi8gOoBFPh3iHqhaKyE+BK4Ba57T/VdWXnPNvBL7hnP9dVX3VKT8d+BMQC9yjqr8aaEzGmPAprWqg7kBrRM/a7k5BTioPvLeT9k438QFOcsW76gGYH8Gd2+BHn4WIjAD+C5jqe76q3uLnZ5ykqnVdyv6gqp/ax1tE5gIXAPOAicByEcl1Xv4rsBSoAFaJyHOqWurn5xtjIsTK8hoATojw+RVd5eek0daxnU3VjcybGNgv9eJKF9PHJ5GSGNlznf1Jkc8C5wAdQJPPEWjnAI+paquqbge2AIudY4uqblPVNuAx51xjTJRZWV5LfnYq6ckjwh1Kv3hnVgdjvkWJs+d2pPNnUl6Oqp4+wOsr8JqIKHCXqt7tlH9bRC4BioD/UdX9QDbwvs97K5wygF1dyj/X3YeJyJXAlQCTJ08eYMjGmGCob27jo4/3c+1JM8MdSr9NGTuK5MQ4iitdXBDA69Y0tLCnoSWiJ+N5+VOzeFdE8gd4/WNV9XDgDOBaETkez1IhM4CFQBVw2wCv/RmqereqFqpqYXp6/6u5HZ1uVmysYcPu0G7Sbsxw8J/NdVE1ZNZXTIyQn53K+gCPiCqO4G1Uu/InWRwLrBaRchEpFpESESn25+KqWun8rAGeBhararWqdqqqG8/ue4ud0yuBST5vz3HKeioPOLfCdx5dw0Pv7QzG5Y0Z1laW15I6Mp6Fk8aEO5QByc9JpayqgdaOwC1gUVzpIkZg3sSUgF0zWPxphjpjIBcWkSQgRlUbncenAreISJaqVjmnfRFY7zx+DnhERH6Pp4N7FvAhIMAsEZmGJ0lcAFw0kJj6khAXwwmz01leVhOS9euNGS7cbuXNTZ4hs7FR+v+qIDuN9k5l054DARvmWlJRz6yMZEYl+PNVHF69TcrzprrGHo6+ZAJvO4sOfgi8qKqvAL/xqZ2cBPw3gKpuAB4HSoFXgGudGkgH8G3gVaAMeNw5NyiW5mVSd6CVtc5GJMaYwduw2xkyG2WjoHwVHNqTuz4g11NVz7LkUdAEBb3XLB4BzgJW4+mo9v11oM+d8lR1G7Cgm/Kv9fKeXwC/6Kb8JeCl3j4vUE6c7fnNZ3lpNYdPjs7qsjGRxjtk9vgoThY5Y0aSNireMyKq2yE2/bPb1cLeprao6K+AXmoWqnqW83Oaqk4fLjvlpY1KYPHUsSwvqw53KMYMGSs3ReeQWV8ink7uQM3kLvFuoxrhk/G8/JqKKCLnicjvReQ2ETk3yDGF3ZK5mWyqPsDOvcHfStGYoa6+uY01H+/npCgcBdVVQU4qm6obaWkffCd3cYWLuBghLyvyO7fBv21V7wCuAkrwdEZfJSJ/DXZg4bQ0LxOAZaVWuzBmsN5yhsyeEGVLfHQnPzuNDrdSVtUw6GsVV7iYPSGZxPjYAEQWfP7ULE4GTlPV+1X1fuDzTtmQNXncKGZnJltTlDEBsLK8hrRR8VGzymxvvP0Lg12B1tO5XR81/RXgX7LYAvhOh57klA1pS+ZmsGrHfuqb28IdijFRa0ddE8s2VHNCFA+Z9ZWVmsj40QmD7rf4eF8zDS0dFETBzG0vf5JFMlDm7JS3Es/Q1hQReU5EngtqdGG0JC+TTreywhnFYYzpn8aWdr75YBGxscL3l+b2/YYo4O3kHuwaUd5kEy2d2+DfpLyfBD2KCLQgJ4305BEsL63hi4flhDscY6JKp1u57rG1bK9r4qGvL2bKuKRwhxQw+TlpvLlpM81tHQOeTFdcUU9CXAy5mcHZeS8Y/LnTj/EsGw5Q6syfGPJiYoQleRk8v66K1o5ORsRFRyeUMZHgt6+W88bGGm49Zx5Hzxwf7nACqiA7FbdC6e4GCqeOHdA1iitc5GWlkBAXHRtAQR8zuEXkcWA58HXnWC4i//aZ3T2kLcnL5EBrB+9v2xfuUIyJGk+vqeBvb27los9N5uIjp4Q7nIDzzrgeaL+F262sr3SxIIo6t6H3Povb8fRPzFLV81T1PDyrxZYAfwlFcOF2zMzxjIyPZbkNoTXGL2t31fOjJ0v43LSx/PQL8xCJ/k7trjJTEslMGTHgEVHb6ppoauuMqv4K6D1ZHKOqP3VWhwVAPW4Bjgp+aOGXGB/LcbPGs7ysGlUNdzjGRLQ9rhaufLCIjOQR3HnxoqhqYumv/Ow0ige4fpz3fdE0Egr8nMHdjaH360IPlszNpMrVwobdg5+EY8xQ1dLeybceKuJAawf3XFrI2KSEcIcUVAU5qWyra6Kxpb3f7y2ucDEyPpYZ6dHV6d9bsnhXRH4iXeqRIvL/gPeCG1bkOHlOBiI2m9uYnqgqP3qymHUVLv7wlYXMmTD0uzTzc1JRZUC/RJZUupifnUJcbHTVvHqL9jtAPrBFRJ50jq14VpL9dkiiiwDjR49g0eQxNpvbmB787c1tPLt2Nz84NZfT5k0IdzghMdA9uTs63WzY7SI/Oy0IUQVXj0NnVbUB+LKIzADmOsWlqro1JJFFkCVzM/nVyxvZXX+QiWkjwx2OMRFjeWk1v3l1I2cVZEXl3toDNX70CLLTRlLcz07uzTUHaGl3R9UyH1591oNUdauqPu8cwy5RgGcILWC1C2N8bKpu5LrH1jBvYgq//dKCITnyqTeemdz1/XqPtyYSLRse+YquRrMwmZkxmunjk6zfwhjH/qY2vvlAESMT4rj7a4WMTBh+k1bzc1LZsbcZV7P/ndzFlfUkj4hjWhTOaLdk4aclczN5f9veAY1+MGYoae90c83DH7HH1cJdX1s0bJtmvU1J63f73xRVUuFifnYqMVG4qKK/mx8dKyKXO4/TRWRacMOKPEvyMmnv9Gw6b8xwdusLpby3bS//d14+i6YM362HvZ3c/s7kbutwU1bVGJX9FeDf5kc3Az8CbnSK4oF/BjOoSHT45DTGjIq32dxmWHv4g508+N5OrjhuGl9aNLwX2EwblcDksaMoqaz36/zyPY20dbqjsr8C/KtZfBE4G2gCUNXdeJYt75OI7BCREhFZKyJFTtlvRWSjiBSLyNMikuaUTxWRg865a0Xkbz7XWeRcZ4uI3N517kcoxMXGcPKcTN7YWEN7p7vvNxgzxLy/bS83P7uBE3LTueGMvHCHExHyc/zfk7vYSSoLomzmtpc/yaJNPWtdKICI9Ldn5iRVXaiqhc7zZcB8VS0ANvFJjQVgq3PuQlW9yqf8TuAKYJZznN7PGAJi6dwMGlo6WLXDFhY0w8uufc1c/c/VTB43itsvPGxIbGQUCAXZqVTsP8i+pr43SSve5SJtVDw5Y6Kzj8efZPG4iNwFpInIFXhWof37QD9QVV9T1Q7n6ftAr3VZEckCUlT1fSdpPQicO9DPH4zjZqWTEBfD8lLbEMkMHwdaO7jiwSI63co9lxSSOjI+3CFFjPx+bLNaXOkiPzs1aocY+zPP4nfAE8CTwGzgJ6r6Zz+vr8BrIrJaRK7s5vWvAy/7PJ8mImtE5E0ROc4pywYqfM6pcMo+Q0SuFJEiESmqrQ18R3TSiDiOmTGOZWV7bGFBMyy43cr3/7WWTdWN/OWiw5mePjrcIUWU+Ydmctf3el5LeyebqhujtgkK/Nv8CFVdhqf5qL+OVdVKEckAlonIRlV9C0BEfgx0AA8751YBk1V1r4gsAp4RkXndX7bHOO8G7gYoLCwMyrf5krmZrHi6ls01B6JqlytjBuIPyzfxWmk1PzlrLsfnpoc7nIiTkhjP9PFJffZblFY10OnWqO3cBv9GQzWKSEOXY5fTOT29t/eqaqXzswZ4GljsXPMy4Czgq07TEqraqqp7ncerga1ALlDJp5uqcpyysPDO5rYJemaoe37dbv78xhbOL8zh8mOmhjuciJWfk9pnM1TxrnqAqB02C/71WfwRuB5P008O8APgEeAx4L6e3iQiSSKS7H0MnAqsF5HTgR8CZ6tqs8/56SIS6zyejqcje5uqVgENInKkMwrqEuDZ/t5ooGSmJLIgJ9WShRnS1le6uP6JdRROGcOt586P2nb2UMjPTqXK1UJNY0uP5xRXuhg/egQTUhJDGFlg+ZMszlbVu1S1UVUbnKae01T1X0BvM3IygbdFZB3wIfCiqr6CZ5e9ZDzNUr5DZI8HikVkLZ4+kqtU1Tvs6BrgHmALnhqHbz9HyC3Jy2Ttrvpe/3EYE61qGlu44sEixo5K4M6LF9n+833wbmK0vpfaRUmFZxvVaE66/vRZNIvI+Xi+wAG+BHi/JXvsF1DVbXiWM+9a3u3SlKr6JJ5O9O5eKwLm+xFrSCyZm8ltyzbxelkNFy6eHO5wjAmY1o5OrnpoNfXN7Txx9VGkJ48Id0gRb97EFEQ8M7lPnpP5mdebWjvYUnuAMwuywhBd4PhTs/gq8DWgBqh2Hl8sIiMZRvta+JozIZmcMSNtNrcZUlSVHz+9no8+rue28xcwb2L0tq+HUtKIOGamj+5xb4v1lS5Uo7u/AvyoWTg1hC/08PLbgQ0nOogIS/IyefTDj2lu62BUgl+DyoyJaPe+vZ0nVldw3Smz+Hx+dP8WHGr5Oan8Z3MdqvqZpiZv53c0bnjky5/RUIkicq2I3CEi93mPUAQXyZbOzaS1w81/NteFOxRjBm1leQ3/91IZZ8yfwHWnzAp3OFGnIDuV2sZWqhtaP/NacYWLiamJUd+k508z1EPABOA04E08I6IagxlUNFg8bSzJiXHWFGWi3tbaA3zn0TXMnpDCbecviMrls8Mt3+nkLu5mcl5JpSuq51d4+ZMsZqrq/wOaVPUB4Ezgc8ENK/LFx8Zw0uwM3thYQ6fbZnOb6ORqbueKB4pIiI3h75cssibVAZqblUJsjHxmvoXrYDvb65oOjZiKZv4kC+9uP/UiMh9IBTKCF1L0WDI3k71NbazdtT/coRjTbx2dbr7z2Bp27W/mb19bRM6YUeEOKWqNTIhlVsboz8zkXn+ov2J41CzuFpExwE3Ac0Ap8OugRhUlTshNJy5GeM2aokwU+uXLG3lrUy23njOfI6aODXc4Ua/Amcntu26cN3lE+0go6CNZiEgM0KCq+1X1LVWdrqoZqnpXiOKLaKkj4zly+jjrtzBR5/GiXdz79nYuO3oqF9hcoYDIz0ljX1MblfUHD5WVVNYzeewo0kYlhDGywOg1WaiqG8/SHKYHS/Iy2FrbxLbaA+EOxRi/rN65j5ueXs+xM8dz05m2iVGg5B9agfaTpqh1u4ZG5zb41wy1XER+ICKTRGSs9wh6ZFFiyVzPjM3lZVa7MJGvsv4g33poNRPTEvnLRYcRF+vPV4Dxx5wJycTFCMVOP8XeA61U1h9kwTBKFl8BrgXeAlY7R1Ewg4omOWNGkZeVYhsimYjX3NbBlQ8W0dru5p5LC4dE00gkSYyPZfaE5EM1i6EyGc/Ln82PpnVz9Lo0+XCzNC+Dop37/Npa0ZhwUFWu/3cxpVUN3H7hYczMsL1YgsG3k9vbuT0/OyXMUQWGPzO4R4nITSJyt/N8loicFfzQoseSuZm4Fd7YaLULE3laOzr5n3+v48WSKm44fQ4nzbGR78GSn52G62A7u/YdpLjCxfT0JJITh8Y2tP40Q90PtAFHO88rgZ8HLaIoNH9iKpkpI2xUlIk4dQda+erfP+Cpjyr53pJZXHm8NQoEk3eIbHFlPSWV9VG9jWpX/kzXnKGqXxGRCwFUtVmieVH2IIiJ8Sws+PSaSlraO0mMt/X/Tfht3NPAN/5RxN6mVv560eFRv0R2NMjNTCYhNoblpdVUN7QOicl4Xv7ULNqc5cgVQERmAJ9dLWuYWzI3k+a2Tt7btjfcoRjD62XV/Ncd79LhdvP4t46yRBEiCXEx5GUl81LJHmBoTMbz8idZ/BR4BZgkIg8Dr2NzLz7jqOnjGJUQa9utmrBSVf7+1ja++WAR09KTePbaY4fEukTRJD8nlbZONzECcycOjc5t8G8/i9dEZDVwJCDAdapq63J3kRgfywm56bxeVo37nPm2cqcJubYONzc9U8LjRRV8Pn8Ct315ISMTrEk01Aqy04CPyc1MHlILM/ozGup54FRgpaq+YImiZ0vyMqluaGX97p734o00brfy2IcfU+U62PfJJmLta2rj4ns/4PGiCr578kz+cuHhlijCxDtjeyj1V4B/zVC/A44DSkXkCRH5kogk+nNxEdkhIiUislZEipyysSKyTEQ2Oz/HOOUiIreLyBYRKRaRw32uc6lz/mYRuXQA9xkSJ83JIEaIqqaoRz78mBueKuGr93zAfpsnEpU2Vzdy7l/fYe2uev50wUK+f+psq9mG0ayM0Ryfm85ZCyaGO5SA8mdS3puqeg0wHbgLOB/Pftz+OklVF6pqofP8BuB1VZ2Fp//jBqf8DGCWc1wJ3Ame5ALcjGcPjcXAzd4EE2nGJiVQOHVs1CSLPa4Wfv3yRvKyUqjYf5BvPLCKlvbOcIdl+mFFeQ3n3fEuzW2d/OvKIzlnYXa4Qxr24mJjePDrizkhNz3coQSUXwvDOKOh/gu4CjgCeGAQn3mOz/sfAM71KX9QPd4H0kQkC88OfctUdZ+q7geWAacP4vODamleJhv3NLJrX3O4Q+nTT55dT1unm79dfDi3X7CQNbvq+e6ja2wzpyigqtz39na+8Y9VTBo7iue+fQyHTY7I36HMEOFPn8XjQBlwMvAXPPMuvuPn9RV4TURWi8iVTlmmqlY5j/cAmc7jbGCXz3srnLKeyruL9UoRKRKRotraWj9DDKxoWVjwlfVVvFZazX8vzWXKuCROn5/FzWfN5bXSan72/IZPrclvIkt7p5v/fXo9t7xQytK5mTxx9VFMTBsZ7rDMEOdPV/29wIWq2gkgIseKyIWqeq0f7z1WVStFJANYJiIbfV9UVRWRgH0rqerdwN0AhYWFYfm2mzY+iRnpSSwvq+byY6aFI4Q+uQ6285NnNzA3K4VvHvtJjJcdM40qVwt3vbWNrNSRXH3ijDBGabqzv6mNqx9ezfvb9nHtSTP4n6XWP2FCw58+i1eBAhH5jYjsAG4FNvb+rkPvrXR+1gBP4+lzqHaal3B+evs/KoFJPm/Pccp6Ko9YS+dO4INt+3AdbO/75DD41csbqTvQyq//q+AzS1T/6PQ5nL1gIr9+ZSNPr6kIU4SmO1tqDvDFO97ho531/P78BVx/2hxLFCZkekwWIpIrIjc7tYE/42kKElU9SVX/3NeFRSRJRJK9j/EMv12PZ2tW74imS4FnncfPAZc4o6KOBFxOc9WrwKkiMsbp2D7VKYtYS+dm0OFW3twUnqaw3nywbS+Pfvgx3zh2WrebssTECL/9cgFHTR/H9f8u5u3NNlI6Ery1qZYv3vEOB1o7ePTKz3He4TnhDskMM73VLDbi6ac4S1WPdRJEf4bKZAJvi8g64EPgRVV9BfgVsFRENgNLnOcALwHbgC3A34FrAFR1H57azCrnuMUpi1gLJ41hXFJCxI2Kamnv5ManSsgZM5L/Xprb43kj4mK565JFzMwYzVX/XM2GKJo3MhQ98O4OLv/HKrLTRvLMtcewaIrtPWZCr7c+i/OAC4AVIvIK8BieGdx+UdVtwIJuyvcCp3RTrng2WeruWvcB9/n72eEWGyOckpfBy+v30NbhJiEuMnYj++uKLWyra+LBry/uc2ZpSmI8919+BOfd8S6X37+Kp645mpwxo0IUqQFPR/Ytz5fy0Ps7WZKXwR8vOIzRI4bOjGATXXr8FlPVZ1T1AmAOsAL4HpAhIneKyKkhii9qLcnLpLGlg1U7IqMStHFPA3eu3Mp5h2VzvJ/jv7NSR/LA1xdzsL2Ty+5fRX2zTdoLFVdzO5ffv4qH3t/Jt06Yzl1fK7REYcLKnw7uJlV9RFW/gKdzeQ3wo6BHFuWOnTWeEXExEdEU1elWbniyhJSR8dx01tx+vTc3M5m/X1LIx3ubufLB1TZpLwS21Xo6sj/YvpfffqmAG8/II9Y6sk2Y9at9RFX3q+rdqvqZZiTzaaMS4jhu1niWlVaHfc7CQ+/tYO2uen5y1lzGJvV/3+Ujp4/jtvMX8OGOfXz/8bW4bdJe0LyzpY5z//oO9QfbeeSKI/ly4aS+32RMCERGY/oQtSQvk8r6g2zc0xi2GCrrD/LbV8s5PjedcxYOfK2aLyyYyE1n5vFSyR5ufbE07AlwMFSVDbtd1DS0RNR9PPzBTi6570MmpCby7LXHcMRU68g2kcMaQYPo5LwMxFlYMC8r9Ovaqyr/75n1uBV+ce58BrvB4TePm87u+hbue2c7E1NHckUUbtFZ09jCDU+WHNovfWxSAnlZyeRNSGFOVgp5WcnMzBjNiLjQrdja0enm5y+W8Y93d3DS7HRuv/CwIbNvsxk6LFkEUUZyIgsnpbG8rJrvnjIr5J//QnEVb2ys4aYz85g0NjAjmW46M4/qhhZ+8VIZmamJnB1FK2u+XFLF/z5dQnNbJ9efNpuR8bFs3NNAWVUjD76/k7YONwBxMcKM9NHkZSU7CSSFvAnJpCePGHTC7cp1sJ1vP/IR/9lcxzePncaNn7f+CROZLFkE2ZK8TH77ajl7XC1MSPVrZfeAqG9u42fPb6AgJzWgy47ExAi3nb+A2gOt/ODxdaSPHsFRM8YF7PrB0NDSzk+f28BTH1WSn53KH76ygJkZyZ86p6PTzY69TZRVNVJW1UBZVQMfbN/HM2t3HzpnXFICeVkpzJmQ7Pk5yFrIjromvvHAKnbubeZX5+VzweLJg7pPY4JJIqnNNpAKCwu1qKgo3GGwqbqRU//wFr/44ny++rkpIfvc6/+9jqfWVPL8t48NytaOruZ2vvS3d9nT0MK/rzqKORMic/vId7fU8YN/r6O6sZVrT5zBd06ZRXys/111+5va2LjHk0C8tZDy6sZP1UJmZoz2SSCepqyM5N5/MXhv616ufng1AHd+dVHEJ1wzPIjIap/tJD79miWL4FJVTvjtSqanJ/GPyxeH5DPf3VLHRfd8wNUnzuBHp88J2udU1h/kvDveQRCevvZoslIjZ+XTlvZOfvNKOfe9s51p45P4/fkLAraEt7cWUlrVyEanFrJxTyNVrpZD54wfncCcCZ7E4fmZwsyM0STExfDYhx9z0zPrmTJuFPdeegRTxycFJC5jBsuSRZjd+kIpD723kzU/WUpSkCdWtbR3ctof30KAV753PInxwe2oLd3dwPl3vUd22kgev+ooUkeGv2N2faWL//7XWjbXHOBrR07hxs/PCcleyPub2ijb08DGqsZDCaRrLWTy2FFsq2vi+Nx0/nLRYaRYR7aJIL0lC+uzCIEleZnc+/Z2/rO5ltPnZwX1s/64fDM79zbzyDc/F/REATB3Ygp3fW0Rl93/Id96qIgHvr44pCOJfHV0uvnbm1v54/LNjE1K4IEQ71Y2JimBo2eM5+gZ4z8Vk28tZOOeRs4syOK6U2Z9ZsVfYyKZJYsQOGLqGFJHxvNaaXVQk8WG3S7+/p9tnF+Yw9Ezx/f9hgA5ZuZ4fvulBXzvX2v5wb+L+dNXFoZ86eztdU18//G1rPm4nrMKsvj5ufNJG9X/CYiBFhcbw8yMZGZmJEfVyDFjurJkEQJxsTGcPCeDFRtr6Oh0B+U3yo5ONzc+VcKYUfH87+fzAn79vpx7WDZVrhZ+/cpGslITQxaDqvLPDz7m/14sIz5W+NMFC20famOCwJJFiCzJy+TpNZV89HE9i6cFfmbuP97dQXGFiz9feFjYfqO+6oTpVLkOcvdb28hKTQz6ToHVDS388Ili3txUy3GzxvObLxVEVCe7MUOJJYsQOT53PPGxwrLSPQFPFrv2NXPba5s4ZU4GZxUEt0+kNyLCzV+YR3VDC7e8UMqElETOyA9OPC8WV/HjZ0poae/kZ2fP42tHTrFd44wJIuthC5HkxHiOmhH4hQVVlR8/s54YgVsDsKTHYMXGCH+64DAOnzyG6/61lg+3B3aJdldzO997bA3XPvIRU8aO4sXvHselR0+1RGFMkFmyCKGleRns2NvM1tqmgF3zmbWVvLWplh+ePoeJaZHRBJMYH8s9lxSSM2YkVzxYxJaawCyk+PbmOk7741s8X1zF95bM4omrj2ZG+uiAXNsY0ztLFiF0Sl4mAMvLArPHxb6mNm59oYzDJqdx8ZGhmx3ujzFJCTxw+WLiY2O49L5VVDe09P2mHhxs6+Snz23g4ns/YNSIWJ66+mi+tyS3XzOxjTGDY//bQmhi2kjmTUwJ2IZIP3+hlMaWdn51XkFELj43aewo/nH5EdQ3t3HZ/atobGnv9zWKK+o568//4R/v7uCyo6fy4neOY8GktMAHa4zplSWLEFs6N5OPPt5P3YHWQV3nzU21PLWmkqtPmMHsCcl9vyFM5mencsfFi9hc3cjV//zo0GzmvrR3uvnT8s2cd8e7NLd18s9vfI6fnj2PkQnhmfBnzHAX9GQhIrEiskZEXnCe/0dE1jrHbhF5xik/UURcPq/9xOcap4tIuYhsEZEbgh1zMC3Jy0SVQ/spDERzWwc/frqE6elJXHPSzABGFxwn5Kbzy/PyeXtLHTc8WdxnB//W2gN86c53+cPyTZxZkMUr1x3PsbNCN8nQGPNZoRg6ex1QBqQAqOpx3hdE5EngWZ9z/6OqZ/m+WURigb8CS4EKYJWIPKeqpcEOPBjmTUxhYmoiy0qrOX+AW2b+YdkmKvYf5PFvHRWSJT0C4cuFk9jjauG2ZZuYkJrID7tZ4FBVeej9nfzfS2WMiIvlLxcdxlkFNuvZmEgQ1GQhIjnAmcAvgO93eS0FOBm4vI/LLAa2qOo2532PAecAUZksRIQlczN5vGgXLe2d/f6yL66o5963t3PR5yYHZXJfMH375JnsdrVwx8qtZKWN5Gs+nfJ7XC1c/8Q6/rO5jhNy0/nNlwrITAnd/h/GmN4Fuxnqj8APge4aqs8FXlfVBp+yo0RknYi8LCLznLJsYJfPORVO2WeIyJUiUiQiRbW1tYMOPliW5GXS0u7mnS11/Xpfe6ebG54sYfzoEdxwRvCWHg8WEeHWc+axJC+Dm59dz2sb9gDw3LrdnPbHtyjasZ+fnzuff1x+hCUKYyJM0JKFiJwF1Kjq6h5OuRB41Of5R8AUVV0A/Bl4pr+fqap3q2qhqhamp4dutdH++tz0sYweEdfvUVH3vr2d0qoGbjlnXtQubR0XG8PtFx5Gfk4a33l0Dd98YBXffXQN08Yn8dJ1x3HxkVPCPrHQGPNZwaxZHAOcLSI7gMeAk0XknwAiMh5P89KL3pNVtUFVDziPXwLinfMqAd/G/RynLGqNiIvlhNnpLC+rwe32bzb3jrom/rBsE6fNywz6MufBNiohjvsuLSQrNZGV5bX84NRcnrjqKKbZJkDGRKyg9Vmo6o3AjeAZ6QT8QFUvdl7+EvCCqh6aqSUiE4BqVVURWYwnke0F6oFZIjINT5K4ALgoWHGHytK8TF4srmJdRX2fO7h5lvQoISE2hlvOmR+iCINr3OgRPH3NMTS0tDNlnCUJYyJduOZZXMCnm6DAk0DWi8g64HbgAvXoAL4NvIpnVNXjqrohpNEGwYmz04mNEb9mcz+xuoJ3tuzlR2fMGVJt+WOSEixRGBMlbFvVMLrg7vfY19TGa/99Qo/n1Da2suT3b5KbOZp/XXmULZhnjAma3rZVtRncYbR07gQ2VR9g596eFxa85YVSDrZ18svzCixRGGPCxpJFGC3JywBgeVn3s7nf2FjN8+t2c+1JM5mZYaurGmPCx5JFGE0Zl0Ru5miWle75zGsHWju46en15GaO5uoTZ4QhOmOM+YQlizBbOjeTVTv2U9/c9qny371aTlVDC788r4CEOPtrMsaEl30LhdmSvEw63crK8k9mnH/08X4eeG8HXztyCoum9D6s1hhjQsGSRZgtyEkjPXnEodncbR1ubnyyhAkpiVx/2uwwR2eMMR6WLMIsJkZYkpfBm5tqae3o5O63tlJe3cit58wnOUqX9DDGDD2WLCLAkrxMDrR28OgHH3P761s4Mz+LJXMzwx2WMcYcYskiAhwzczyJ8TH87IVSEuNjuPnsueEOyRhjPsWSRQRIjI/luFnpqMKPz8wjI3noLOlhjBkaQrFTnvHDtSfNZG5WyoB3zzPGmGCyZBEhFk5KY+GktHCHYYwx3bJmKGOMMX2yZGGMMaZPliyMMcb0yZKFMcaYPlmyMMYY0ydLFsYYY/pkycIYY0yfLFkYY4zpk6hquGMIChGpBXYO8O3jgboAhhMN7J6HvuF2v2D33F9TVDW9uxeGbLIYDBEpUtXCcMcRSnbPQ99wu1+wew4ka4YyxhjTJ0sWxhhj+mTJont3hzuAMLB7HvqG2/2C3XPAWJ+FMcaYPlnNwhhjTJ8sWRhjjOnTsEgWIjJJRFaISKmIbBCR65zysSKyTEQ2Oz/HOOUiIreLyBYRKRaRw32udalz/mYRuTRc99SXQN2ziCwUkfecaxSLyFfCeV+9CeTfs/N6iohUiMhfwnE//gjwv+3JIvKaiJQ515saptvqUYDv9zfONcqccyRc99WbAdzzHOf/bKuI/KDLtU4XkXLnz+OGfgWiqkP+ALKAw53HycAmYC7wG+AGp/wG4NfO488DLwMCHAl84JSPBbY5P8c4j8eE+/6CfM+5wCzn8USgCkgL9/0F8559rvcn4BHgL+G+t1DcM7ASWOo8Hg2MCvf9BfHf9dHAO0Csc7wHnBju+wvQPWcARwC/AH7gc51YYCswHUgA1gFz/Y4j3H8QYfrDfxZYCpQDWT5/IeXO47uAC33OL3devxC4y6f8U+dF8jHQe+7mOutwkkekH4O5Z2AR8BhwGRGcLAJ1z86Xz9vhjj+E93sUsBoYCYwCioC8cN9PIO7Z57yfdkkWRwGv+jy/EbjR388dFs1Qvpyq9WHAB0CmqlY5L+0BMp3H2cAun7dVOGU9lUe0Qd6z73UW4/mNZGsw4w2EwdyziMQAtwGfqsJHukH+PecC9SLylIisEZHfikhsaCIfmMHcr6q+B6zAU1OuwvMlWhaKuAfDz3vuyaC+v4ZVshCR0cCTwPdUtcH3NfWk2iE3jjhQ9ywiWcBDwOWq6g54oAEUgHu+BnhJVSuCFGLABeCe44Dj8CTII/A0VVwW+EgDY7D3KyIzgTwgB88X5skiclyQwg2IcH9/DZtkISLxeP6gH1bVp5ziaudL0PtlWOOUVwKTfN6e45T1VB6RAnTPiEgK8CLwY1V9PxSxD1SA7vko4NsisgP4HXCJiPwqBOEPSIDuuQJYq6rbVLUDeAb4VId/pAjQ/X4ReF9VD6jqATz9GkeFIv6B6Oc992RQ31/DIlk4oxzuBcpU9fc+Lz0HeEc0XYqnLdBbfokzkuJIwOVU914FThWRMc7Ig1OdsogTqHsWkQTgaeBBVX0iROEPSKDuWVW/qqqTVXUqnt+0H1TV/o0cCZEA/tteBaSJiHfF0ZOB0qDfQD8F8H4/Bk4QkTjni/gEICKboQZwzz1ZBcwSkWnO/+sLnGv4J9ydNaE4gGPxVNGKgbXO8XlgHPA6sBlYDox1zhfgr3ja5kuAQp9rfR3Y4hyXh/vegn3PwMVAu8811gILw31/wf579rnmZURwB3eA/20vda5TAvwDSAj3/QXx33Usns7vMjxJ8ffhvrcA3vMEPDXFBqDeeZzivPZ5PKOptuJpKfA7DlvuwxhjTJ+GRTOUMcaYwbFkYYwxpk+WLIwxxvTJkoUxxpg+WbIwxhjTJ0sWxhhj+mTJwpgIFelrM5nhxZKFMQEgIreIyPd8nv9CRK4TketFZJWzl8LPfF5/RkRWO/sTXOlTfkBEbhORdUTw8hNm+LFkYUxg3AdcAuCsWnsBnpVAZwGLgYXAIhE53jn/66q6CCgEvisi45zyJDx7LixQ1bdDGL8xvYoLdwDGDAWqukNE9orIYXiWil6DZ/XWU53H4NlQaBbwFp4E8UWnfJJTvhfoxLNgnDERxZKFMYFzD561pCbgqWmcAvxSVe/yPUlETgSWAEeparOIrAQSnZdbVLUzRPEa4zdrhjImcJ4GTsdTo3jVOb7u7EOAiGSLSAaQCux3EsUcPNt9GhPRrGZhTICoapuIrADqndrBayKSB7znWWWaA3hW8X0FuEpEyvBsjRnRe4QYA9iqs8YEitOx/RHwZVXdHO54jAkka4YyJgBEZC6ePU5et0RhhiKrWRhjjOmT1SyMMcb0yZKFMcaYPlmyMMYY0ydLFsYYY/pkycIYY0yf/j8cKoIfKd2drAAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "# plot length by year\n", "ax = df.groupby('year')['num_words'].mean().plot()\n", "ax.set_ylabel('Average Opinion Length')\n", "import matplotlib.pyplot as plt\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 303, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:35:44.161062Z", "start_time": "2022-02-25T09:35:43.959415Z" } }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 303, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaUAAAGoCAYAAADmTPpwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABn6ElEQVR4nO3dd5wcV5Xo8d+t0HGiRqMsWbJsWU44ycY2OGHsNTyWsEvGS37ksAF22X089gHLkpecM5jgJRiDCQYMLMbYxnLCtmRZki0rSyNNng6V7vujukcjabq7eqZmuqfnfD8ffSSNWl11u6vqVLjnHKW1RgghhGgGRqNXQAghhCiToCSEEKJpSFASQgjRNCQoCSGEaBoSlIQQQjQNq9ErUAeZJiiEaBWq0SvQrORKSQghRNOQoCSEEKJpSFCKwfKVq1BKzfiv5StXNXqoQggxo9QcqujQtCuqlOIFX/jTjC/n+tdezBz6vqpavnIVe3fvmtFlLFuxkj27ds7oMoSYInmmVMFcmuggWsje3btmPJBf/9qLZ/T9hRDxa/nbd7Nxa23WGJbcIqzHLHxeLfeZzQK53S2qafkrpZY6Iw+8mR/L6y+d3UA7k2bh8wK5IqvXbOyTIN/LXNXyQUnUSQ7kQogGavnbd0IIIeYOCUpCCCGahty+E2K6ShMqZpJpJ/Hd4owuA2QavWg8CUpCTNdsTEB57cXyrK9es3CyIEE8fhKUhBCtaZZOFkS8JCgJIY6YhasLIaqRoCSEOEKuLkSDyew7IYQQTUOCkhBCiKYhQUkIIUTTkKAkhBCiaUhQEkII0TQkKAkhhGgaEpSEEEI0DQlKQgghmoYEJSGEEE1DgpIQQoimIUFJCCFE05CgJIQQomlIUBJCCNE0JCgJIYRoGhKUhBBCNA0JSkIIIZqGBCUhhBBNQ4KSEEKIpiFBSQghRNOQoCSEEKJpSFASQgjRNCQoCSGEaBoSlIQQQjQNCUpCCCGahgQlIYQQTUOCkhBCiKYhQUkIIUTTkKAkhBCiaUhQEkII0TQkKAkhhGgaEpSEEEI0DQlKQgghmoYEJSGEEE1DgpIQQoimIUFJCCFE05CgJIQQomlIUBJCCNE0JCgJIYRoGhKUhBBCNA2ltW70OkSilPolsHAK/3UhcCjm1WkmMr65TcY3t011fIe01tfEvTKtYM4EpalSSm3UWm9o9HrMFBnf3Cbjm9tafXyNILfvhBBCNA0JSkIIIZrGfAhKX2z0CswwGd/cJuOb21p9fLOu5Z8pCSGEmDvmw5WSEEKIOUKCkhBCiKYhQUkIIUTTkKAkhBCiacyZoHTNNddoQH7JL/klv1rhV2QteuyraM4EpUOHWrlSiRBCTG6+HfvmTFASQgjR+iQoCSGEaBoSlIQQQjQNCUpCCCGahgQlIYQQTUOCkhCzTGuN1JwUYnJWo1dAiPnE9TUjjo/W0JYwSJgKpVSjV0uIpiFBSYhZEGjNqBPg+EeukEacAMuAtoSJZUhgEgIkKAkxo7TWFLyAMXfy23VeAIMFn7SlyNiGXDWJeU+eKQkxg8bcygFpoqIvz5iEAAlKQsyoIGKsMVSdBdGEaFESlIQQQjQNCUpCCCGahgQlIYRoYo89/jjXvvp1jV6NWSNBSQghmpjr+uw7NNTo1Zg1EpTmMK01eden6AVSIaBJ2YYiYdae5m0qaKXJ4Dk3oG/Mw48606PJub4m5/iyn82CGc1TUkp9FXgGcFBrfUbpZwuA64HVwA7g+VrrgZlcj1bk+ppRxyecSayxDGhPmJiShNlUUpYihcINwuTZY4/RCsjaBkmrNb43L9DsH/EYLgYAHM75LOuwaE/MzRysY5Oe855PW8Igacn5/EyZ6U/268A1x/zsHcAtWuuTgVtKfxcRBVozUvQZKpYDUsgLYKDgMyZnc01FqbCMkG0oulMm6QnBJ2kqutMmSUuNv26u0lrTn/PYethhuBiM97z2Newe8tgx6B5VzaLZle9C9Of9o9ZbE1biGCq0zlVgs5nRoKS1/gPQf8yPnwV8o/TnbwDPnsl1aCVFL2Ag71dNtMx7moG8TyCBqamUg07GNuhOmXSlTNoSBsYcD0YAjq/Z3u+yf9Qn0MfnW2kg52q2HXbIOUEjVrEufqAZLPhVk57d0klgzvVncc3mh0Zcgy7WWu8r/Xk/sLjSC5VSr1FKbVRKbezr65udtWtiOTeIlGBpm6qlnk+0EqUUpqHCZ0hzPBiVjRTDq4ko22babv4xF31N1Iu6XIRqHVMx8diXz42xY/sjM7KcZtTQG6M6vM9U8VvVWn9Ra71Ba72ht7d3FtdMiJnVKgGpTK7L4zXx2JfOZFm9dl2jV2nWNCIoHVBKLQUo/X6wAesghBCiCTUiKP0EeFnpzy8DbmzAOgghhGhCMxqUlFLfBW4HTlFK7VZKvQr4AHCVUmor8NTS34UQQoiZzVPSWr+owj9dOZPLFUIIMTdJBtgcYkR8ON5q6RNaa8m9amKWEX2251zYNqPmn2ut8QPNgVF3RlMwisXivJp9J51n55COpEHODch71XeAhDlLKzTDtA6nGY+VKiGEeT2tN3NtrutMmRgK9ox4BMHkM/EU4fY7B2ISKSvczkaKlVMwwo7CmpFi+IqBgsPydov2ZPw7XzKZnFez7yQozSFKKbIJk5QVlhhyj8lDTJhqPCFzrtNaU/Q1Y86RA8PAhLbhIMGpmbQnTdYlDPrGPA7njnxnCrAMWNFpj39vc0HCNFiQVsedBIZXRzBUDPAm7H9+ALuGPDIJn+XtNnaEeodichKU5iDTUHQkTRw/rMulVFj3rhV2BK01gYYRxz9qpy/Le5qi79OeMLAMCUzNxFCKxW02XamAPcMeBU+zKGvSkzHn5PdUPglMWprhoo8faEYcTb5Cwmx4Va/Zethh3cIEltShnBIJSnOUUoqkdaQC9Vzc6Scz6gRVyyhB+Fxi1AnoTJlSuaIJJS2DNd020BrbpWWEtQsPlMoo1aKAIECe2E+RBKU5rhV2etF6Wm27bLXxNDOJ5UII0cSKxSKbNz00b7rPSlASQogmlkwmufRtX5g33WclKAkhhGga8yooaa0pegGOH19Pl0Br9g67jBRnv6/KTIyn0coPlWsxWqx9+KgTsGfYbZnGceXxNKKvlx9o8m4Q67INBZ1JI9I21xrfYOPMm4kOXhDm9pSnGdtGQNs024cP5H029xVxSweSJW0WaxfMzlRQLwg70JYnqsUxnmaQshQpS+H4mjH3+PbhABnbOKqD61zmBZrHBhz2jngA7Bh0WL8wSU9mbu6aXqB5tN9h32h5PC6n9iZZkJ75jG6tNXkvGO9xlHMhYytS1vRbsactRardojcLu4fdin2UFNCeNLBbJIG9Eebmll8HrfWkVRDKnSMzliJt17fRFr2ARw479OePniK6b8Tj4JjHup4ki7Izk5uhdXiwLsQ4nmZSXu+ECQnTPGqs5eRgxdyfDaW15lDO5+FDRQJ9pPyO48ODB4t0pVzWL0yStObGzQytNX05ny3HjUfzwIEC3WmTU3oSMzYe19eMOEfvjxoYczUFz6c9aU7rZFGpsJSSYcIJXTajxYC9I974SWE5SXh5h002MTe+s2al5kpNsQ0bNuiNGzfW9X9cP0x6qzVCU0Fn0sSIsNHuGXbZ1u+gJ2n7XGYo6EmbnLYoGWt1BccPqpY+KTNVufTL3D5ww5Fk2kDrsMZaC4zJ8TUPHSwwXJz8SrDMUPCExUm608197lj0Ah46WGTEqTye8EQCzlqSoisV32WE1jpSbhtANqarpvJytYZ9ox5DhYDerMnC+pKEI78wmc7qjhUnY6uAU087naULO7nuy5+f2oo3j4rjb+6tfZrGnNoBCcIKCVG3pW39Ts0EukAzI4mdE0vuVFNPgcxmp1TYOtzQc//qqKxvLDyQ1foutSbWA/hM6RvzGa5xsqQJj0KdyXivIryASAEJwtJBcW1DSoXHjGWlW3qJGaymkkwmufLtnxv/+76ff3LGltUMWvo6s55rwKivjXph2Qq3mJpJq32WrTSamdjP6hH1s5yJz1wpNaMBaT5q6aAkhBBibpGgJIQQomlIUBJCCNE0WnqigxBCzHXFYpFbPvz68b/bKuDKZ79o/O8tMhtvXEsGJa01u4Zd8m5Ad7r21Gitoz8EtU1w/doPbN1AE2gd67RsQ0GUiUYzURQgCMIPqRHTzP1Aoxq47HKmQFyTLSyDcIOr8T1pwu87zjzhmRiPbYYz0WpOAtKl18Q4Ho2OPHkiAJTWsY070GFvpc19Bc5Zmp6xxPVjZ98dq9Vm47VcUBop+vx5T56BfDgd/MRumxUddtU22vU0xNywLD1p4myZAkwjrDoQ1yZ6JFcn2uvjbPAZBBpfw5ZDRRKmYu2CxKy1JA9KXT7v3J1jqODzlBPbsE01KxUztA7HvflgkUcOF3nKiVnaptlIsZwTmLEVKzos9gx7BBXy3QwFXSkDL9CxjLfcMfWhvgLbDzs85cQ2sgkjlsaQi7MmBkm2HC5W3E4NBd0pEy/QsRy8tQ6D0XAxYCDn054yMKtsl2bMgdj1NTuHHH7/2BiOr3nwYJGr1raxvMOO5f3ns5YJSl6g2XSwwCOHnaN29G39LntHPE7vTZJNGEftELZB3aV5kpbBmYtT9Od9Hi6VGCrvhIaCxVmLk3riKTVUPohNVsFhMlMZT7VlBzqsUvHogDN+hdaX81nXk2BBemZLGrm+Zs+wy28fG2PMCWtD7bx3gAtXZDhzcQpzBrvOur5mqOjz622jHMqFNQ2/ff8QZyxO8qRVWUyj/qs2rfV4p2BNmMfWljA4MOrRnz+S42Oo8MpofW98pYZcXzNY8PnVtlH68+F4rrt/kLOWpLhwZWZK45lIKcWiNosFGZPt/Q77R72j9gnLULGWGgq0puhp9gx74zlKh3MBGVvRljiyTmVxJs16fljK6NfbR9kz7I3/fKQY8JOHh1ndleCyNdk51fq92bRMUPr19lHGnGDS21s5V3PX3gKLsyan9iaxjLBczXRKnixIm1y4Ms3jgy6PD7pk7HDHa0/Gs+OVD2KV6r9NVK63lTDj2RHKpYw2HSwydkyNr/JZYVfK4PTeZOnWTXzBwQ80BU/zm+2j7Bxyj/o3L4A/7szxUF+Ra05qi73Ntlc6wbh1xxib+opH/ZsGHjhQZFu/wxWrs5y4IBFp2eXgPuoE4zUSy0xDsazDZkE6YHepffiKDovVXYlYAr5XuoX8PztyPDzJeO7bXyhdBbaxusue9mdpGYpTFiZZ1m6xua9IztWs7LA5ocuOZTxBuYrCiMdQ8fgixDk33HY6kmFX5qSpyCaMWG77lgvlbtyb4+69hUn3SS+ARwccdg65vOycLlJzpERUs2mZoFQpIE10YMxnaXvAup5oB5RaDKVY051gZadd9dbBVPTno1WjmIlad3fvzTPiVF/6YCFg+4DLup4EceYO/n7HGA/3FasG4oG8z6+3j/K3p3fEmri4pa/In3aOUahS8D3van6+dZQ3X9gT6T1HnACnxoaZsg1OWWiTTZixnVgAbDpY4PadOZwqReRzrubnj4zw+gsWxHa7uT1pcv7ydPg8LMar6f68z8HR6vtFoGG4qFnXZmAa8X2We0c8fr1t5LiTtMmWr0tXcqmWObrOrnn5scV922cmnnFEfXgbljuJd/kRq7bMiHqencVNw3gV+dmmVPzPysoTJRpBKRXrBI2yqMOZiUkxboO2jWNn3x1r4my8VpiJNy+DkhBCzBW1Zt9N1Aoz8eSmpxBCiKYhQUkIIUTTmHdB6fFBh62Hi7VfGFGgNQdGPXLVniZPQdKs3X6i3MAwF/PN7p60Gamz63DBZ8egQ1w9ubTWJE1FR4T2Blk7nllVEw0XJ889O1Y9kysso5QsW4MXaIYKfmyfJcDhnE/B0zXfM8q2Vq9dQy6/2jaKF+MDQsuAbKL2mjq+ZtPBYs0JJvXwA83CTLSZtQ169NQyWuaZ0sk9CbYedio+2C24AVsOFRkqBJgGnLYoxYvO7KRzGv1qhos+mw4Wx3Ml4pzO25Ywws6ZFRqY+YFmqBCUHsz7tCcNlrZbsTwsP3FBgjXdsGs4nO5+7HHFDzT7R1wGS59lT8bkqWvbp5WHUvQ0e0dcejIWXSmTQzmPRw4Vj3u4bCg4b1ma85alY5v1l3cDbn18jG2HnZoHlFMWJrhsdTZytY60ZZC2wvGNucf3HNJaM+ZoxlyNIuxNtKLDJjON7qVjTsDPHxlh86EiflDOXzMm3S7X9ya57ITMeL+j6Sp6AV+/d4D/fnAYpeCr9/TzzssXccai1LTfuzNp0pE0ybkBe4e947YNrcNtaNthB4DbduV4yposayNO359MoDUHxzxyrmZdT5JVXQkeOlhgZJIp6VDqPttu0ybdZ6espTrPDuR97tydY3TC9PBAa3YNuuwYdI/qFmuqME/kOae2c9mabF1n3a6v2drv0DfmHXXANlT4vqfGmPhYzsQfcXx8ffRB7Fhh8q5Jdzqe/B0/0HiBZvMhJ6yQoTWDeZ99ox7oI2eE5SoWT1ic4oIVmbqqBARa0zfmczh39FTfckWFrYeL7BsJkxRXdFg8dW0bacvAiiEiaa3Z1Ffk1sdz45UrKulOm1y9to3u9NSqOpQrEEw8ySh6etLus+W8s3pPMgKtuXtvnpu3jeFPGE+5olHGUmTscLbmgrTJ1Se10ZWaXpWKie7YleP9f+hjzAkoTPgwk6bi8jVZ3nJhz7ROAsvKn+XE7Wak6LOpr0jBPTo1xDJgUdbiyrVtdTdMHCn6k1TdCPfHg2MeWw8Xx2drWgoSluLKE9tY3Z2I8vZ1d56NwlYBT7n04rkwA6/i+FsqKEG4wT464HDf/gL9OZ+HDhZx/coHnIQJi9ts/uGinkhnpwdGXbYcciqWh4EwOPRmTNb3xtMOvfwdDRZ89o9Wv8WkgKSlWB1TwiKEwWn3kMuvHx2l6OmKy7eMsA7a80/vpCPCAWDMCdg97OIHlT9LP9A4fsDCjMnyjkRsB9DBvM8vto4wWPCrTgNXwJNWxVdFopzDsnvYpVglHwrC7WhVp0U2Ufuz7BvzuP7BIQbyfsWpy+WTh79e186ZS1Kx5dYNFnz+4/cHuW9/oWLlEdsE21B8/GlLOS2GqyYIg7DjaX61bZRdw8df0ZeVx33FmiynLEzWHLNX2t5zbuW6euWTpof7ihzOe5y9JDwhq+MkIvILO7oW6Kd/6GdRX07fzz/JLT/+buTXN0jF8bfM7bsypRRrFyRZ0WHzvv/pq1mex/HDABL1YPfwoWjt0NuT8dW+K+9Eh3O1n3loSs8IYnxIYBqK7QNF8jUSB70AOpJG5FtP+0a8mnlBpqFYkrZZ1mHF+gzprj258RJC1fRkTM5cnIrlygzC7zLn+ji1F02giVyu5nePjXFwrPqbamBJm8UZi1Ox5kT9Zvsod+/NV83jcf3wKHTKwmRsyzWU4nDeY89I5YAER/LPVnRGq1oxWPBrJsmW87BOX5RkeUd4BS3i0bI3PpOWEbnMh1LhWVcU0duhx5/UGvnkKkIF6nppHW3Zijo+y4jLnokSd/Us+/gnQdM1A8nWkbfL6N9PVFHnMszAZglA1PgaeTpHHStpKCUBKWYtG5SEEELMPRKUhBBCNI2GPVNSSr0V+N+EV/Vf0lp/vFHrIoQQzapW7btjHduZNopmqpnXkKCklDqDMCBdADjAL5VSN2mttzVifYQQolnVU/tuqpqpZl6jbt+dCtyptc5prT3gf4C/iXMBt2zcxAOPPApB7WlOrh+9bbllRHtM7frBeA+WuBgRJ2QEAbE/Sw/njNRethdozIifpamirabn68gP8qNKmSpS8q1Tx7YRXfQm3lGHnbZVpMoRec+PPDEgqraIPYvCTsI61okWlope2b3oB5EqZhhGtCkRQRAwODzCV370a4JA6jjEpVFB6UHgEqVUj1IqAzwdWHnsi5RSr1FKbVRKbezr64v0xnv7BnjBOz/Ly9/7ZX730/9mx5YH8D2XyXZvRZjtvqjNmvTfJ3PesjSdKaPqjh1ozQMHizx0sDDeaG06HF/Tn/P43gOD3L8vj+tXLh0T5ilNa3FH8XzNaNHnt48McPfOYTw/qDjVq5yoGLW0zKoum45U5anzWofJu3fuyfPjh4fDxMiYAv2TTshy3rJ01ZMMrTUHRz2+ed8ggwUfd5plawKtw8Trww6b+4p4QfXvMW0pvIjLfNrJ7Vy8MlNlPBrP9/nz1gO86mt3sm8wT8GNMC89gmtObuOtF/WQsRWVZrAbKsyfe+/vD7J7yKU4zR4h5SaY2wYc9gy5VU9cLAM6UwaeryPNiO1OGSxpM6vu44VikfseephXvu3/8X8/cx0Xv/SfuW/LY1MczfEmHvscJ76yaHNBw5JnlVKvAt4AjAEPAUWt9d9Xen2t5FnX8/n8j37Lh779c1zPDw+eJV0LF/PEq55Ftr0Tw7KBMJdnabvFy87pYmm7Xff69415bDlUxJ/Q/6e8owwXj0wiTtuKC5alWZitvwRQEGjcQPOTh0f4wUND43kgy9stnntGJ53JI/lVCkhZiuUd1rQ66k5UcANu2TbEB36/l6FS57u2pMnlJ3XTm7WxSg3pLAPStsFVa9tY3lH/Z5lzA/YMu7j+kVMDLwirR9y5J89oqa5gylL81UltPGFxKjz4xnAFM1Tw+c32UQ6OHZ0z5QeaYScY/5mp4MmrMlyyOoulwrPperi+5uCYy2+2jzFcKlFjGbCuJ0lv1iL8KMMzdKVgWbtFR7L+5o2Hcx43bB5m38iRMjy+75MrOPzhvkc4NDQKQMI0eMOV63jd5SeTsOJpiDdY8PnE7Yf5w46x8aoV5avR5R027Ykj43niijQvPLMT26y/j5TjBfTnfb56z+B4d2IF9GZNejLm+FWbARgGXLgyw1lLUnVf8XqBZt+Ix8iE/dl1XcZyed7/ma/yx7vuHX+tUpBKJHjBXz2Zd7/xWjrbMrXefsaSZ6eiAQm3zV3RQSn1n8BurfVnK72mVlC68s0fZOuuA+QKTqWFcOJp53DuJU8lnbB54ZmdXLAiPa0Dmx+E1SN2D3vhQaxYOZt+aZvFE1ekSURsHx5ozcN9RT51Zz8HRr3jh0O4Y19zcju2qaZ8EJuMH2j2Djv86y938Zd9uUlfs6o7xVNO7iZlGVywIs05S9PTqiChteZwzmf/qIfjazbuzbN7+PhxQ3jAft7pHSyIqZyS1prtAw6/2TZaakGvyVdIuu5MGTxnfTtruqPVUwtKFRx+++gojw64k76mI2lwxqIUKdugO2WwuM2a9mf5UF+RHz00TMH1ufeRx9n8+L5JryRWLsjw0RecxwUn9sSWV/fAgQLv/t1BDox69KTNUtA9/r0ztuJ5p3dw4crwAF5r+eWyVz/aNMwfduQmvbeRMGFFh03aNljTZXP5iW3TrkM35gQ8PlDA8TU//Plv+NJ3fkihOPlxJpmwyaaS3Pv9T9YKTBKUKv1DA6+UFmmtDyqlVgG/Ai7UWg9Wen2toLTimf9AwZl8p5/oPa99Hq/860siJ9ZGcfO2EQYLtW9HnNBps2F5OtKZ4Wtv3ENfhIoDrz+/myvXtsX63OPl12/j/n25ml1Ln3laN2+/bCmdMfZ9/vHmYe7fX6i57KVtFq88tyu2q0KAO3fl+PX20artw8ve85RFkd7zN9tH2HrYqfncY3HW4hnr28jY8SVifvXW7Xz05s2MFqrvFwnT4OH3/3Ws7cO39BX41B391CioAsAHr14cqSbe7x4d5aYttVuSJ03Fe65cFEudvbL/uftB3vSBL7Fr74Gar82mU/zx6x9gzYol1V42I7XvpspWAaeedvqk/zZDM/OasszQD5VSPYALvLFaQIqTSRBrQKpHPeF/oBDtfr8m/tbPQwU/chvtuD/LQEdr4T0Tp1KmoWJ/Xy+I/iA+7nbohgpvazeCaShsU+FFiEpRP3NfU7NsWFl2Bqp0Dw2PxP6eUczG7LtqZntmXsOCktb6kkYtWwghRHOSig5CCCGahgQlIYQQTWPeBaVRUuwcrDBDbwq01tx93/309/fXfG3Kit52+sIVaVIRWpI/uPEO7r///ojvGs2GFVkWt9We2j04mueO7dHyx6J69NAYA2O18zLStoo9CXTH7v3s2rmj5usidpMAwuaAUTryDuWK3LJpP0GMCddDDqQz2ZqvyyTM6BW0IzrQP8yunY/XTFY1VPTPsy1hsKa79nbp5Me44Wc3UyjEl9/TN5TDM6P1gZJE2ulpmX5KL3jqBfz3LX8mX5x8plEyneW8y6/hEZbz4T8eYsPyNM89vXNaD0Q3PbqL173nMzyycy+Bhqc//elcffXVmObRByFTwRmLkpzck4x8IH3zhQspegGfu6uf23flj/v3wsAB/vK1d3LTA7diGXDttdfyoQ99iK6urimPp+xfLl/GP14Kn79jP9+659BxD+p1EOCM9HPTHcP8eqPBhtUL+ODzzmXFgpq5GRXtHy7yzp9t5087hvACzdpF7Zx9wkJs6+jPsjwV/qlrs7FNDBjJFfiPr/2E626+HV/D8pWruPTqZ9De0Xnca9cvTPCs9R2R26Gfvzxs3f7QwQK378wdlzIQBJp7tu3htod2YBqKVQsyfOzFGzhjedeUx5NzA7557yB/6rNZs/YkRkdH2LHjcVz3+H3jaWcu4/3PPWfKyzqW6/l8/md38NEf/AE3gLb2Ds696DI6uxcc99o13TavOCdMK9C6dmLrBSvSnLs0xea+It/+y9B4vleZ1pod9/yBO773Cb4XeHR1tvPFj/wHV13+pCmPx/F8PvvT2/nYj24laF9CMtWNc3g32ps84GVSSc446QSWLeqZ8jLnu6bIU4oiSufZP296lDd8+JscHBg+kq+kFGvPOJczL7wCy7JAHUn4tAzFC84I8yTqydEYzRV47xe+xzd+cgtF1xs/G0wmE7Rl23j5K17BunXrgDCn5vzlaWxDTSn3pOAFPD7g8sk7DrNv1CPwPR69+ets+v5HwPdK1SoglUqRSqX41Kc+xUte8pJYck4KbsChnMe//XIn9+3NhRUWCmPkB/tQ6PGzestQWKbBm65cx2svX0eijhl5XqD56h17+OQfduH6R5JVLUNhGIrzT1zECQvbUEqFScOnd9CRjKeFt9aan/7xPv7pk98lX3THUwpMw0AZJhsuvpSzzr8Q0zTpShk859QOlnfYJKawbD8Iqzn89rExtveH2+aeQ0P84q4t5IoOTmngCkjaBn9z3ir+7Rln0J6Knoystea2nTm+cFc/jq8nBEBN4Afs3buXAwcPArCqJ8t/vfBcTl/eRSYRz7npnQ/v5E2fvpFDw2PkSieHSoFhmKxeu45Tz96AbSfI2IoXnNHJOctS2Eb9fcfK+Uo3bh7md4+F+UpDB3bzp+s+Qv+eR3GLhfHXZtIpLr3ofD7zgf/H8qWL61rObQ/t4M2fvZGBkfyR8QBaB+jcIM7gAdClxO6ETTKZ4GNvexXPufKiKGNqqinh1VSbLl5NjankzZenVK+o7dA93+fzN/yOD37r56Q7FnDBVc8i09YxXsnhWAlTsaLD4k1PjNYO/Rd/vJs3/ufnyOUdCs7ktwETiQSXP+lCPv72V7KoPTntM/pyZYdP3nAr733rKygMHMAtTJ7Ums1mOe+88/jpT39KR0fHtJZbVnADfvSXA7z7x/dTLBQq3pJJJ0wWZBP89+svYcWC2reN/rJ3hLf+6BH6Rh3yFbKOLUOxrCvNh561jrOXZmKr5LDrQD+v+9DXeWD77ooJ17Ztk8m28bF3vJ6nnbkUcwqVHI7l+pod/Xn+6fp7eWjXoaMqj0yUtAyStsk3X30x55xw/FXGsQ6MenzsT4d4fNAdr6ZwLB0EeK7L01cbvPzi1SQthRFDbtLQWIG3f+ln3Hz3IxScyROeLcvEMCz+4aXP4fWXryVhTu0kbSLH1/SNFHjpv/0Xd/zyh2jfm/TWmWVZJGyLz3/4PTzvmU+ruf0MjOZ52xdv4pZ7t5GvMB5DQeD7uP27SWiHFz3tUt79hhfTkY18t6CpkmdnQo2E3KbMU5oRlmnypuc+lb+9fAPvvmUfpDuo9v07vqYrFf3M+5Xv+gT5Ctnc4+/pOFzxhDUsaU9M+yAG4YEwaSi++/5/ZGRf9fpaY2NjrFixgmQyvrbTKdvggcf2UyzkqxZGzTs+mW6L3vZo997/5Sfb2DlQqPoaL9Ccv6KNJyxJx3J1VPbhb/+cuzY9VrUuoeu6rFyQ4erTemNbtm0q/rztAJt3Vw5IAEUvwAs0T1jZHel9v/OXQbYedqrm/CjD4NSl7bzyyYtiTTr+wa0P8IuNW3Cq1NLzPJ+UrXjrlWtjS9JNmIrHtmzinl/fgO9W3ic9z8PzPJ50wXmRTmiu//393Hz31qo5XoEGDJNU7wn88j1/xxPWrZ7CCMRkWnaiw9KFXaxaspAoJyT1tEOP+iDaMo1YzkIn8rzaFSsADMOI/WFrEESr1G0ohVvlYDtR1MKtMX+MQPjsI8p3bhpG7AmogdaRd7yoodALotUdN1S05OR6+EEQKQO2nv0sqiAISNjRbnFaVrQKD34QrZo4gG1ZEpBi1rJBSQghxNwjQUkIIUTTaLlnSkII0UrqbYfeLCZryx6luKsEJSGEaGKNLsgapyjFXVv29t1djx1iy77Bil1SJ/LqaHmdTiUivXZ4LE/RnXw66VR1dHZiVZjaPlGxWIx9kkU2Ga1JYdHzSUR8oNyeNCO18M4VPYIIbe3r0ZFNk7Brn5Plii4JK95zt6Rl4Ed6kK5xI05YydpGpM+y4AWR2sDXI5uKNss0CDS6SrfdqUhn0hQidmYdGR2LtOxsKoEVYf9RhMm1P7z1gVjHNN+1XFA6PFrkTdf9mWu/+Cf+dP/DHDjUj19lNphthAfHqBvVzZ9/D2edsoZMavIp14ZhgGHyqVu28pU/bKfg+tMuHeP5mlHH55n/9iVOuOBqrGR60tcppUin0/T29sY2+84PNEUvYO0JKzhh2WIMw6BSTE7bJqt7soxUarR4jM8//1QuP2lBlfYXGu05/PBHN/CO//goo2NjuF48gf5dr3oWr/hfTyaVsCtOEzZMiwN+hn+9eRf9eW88wXWqgiAg77gcPrgPb/AABEHF2XWZhMm6xR3sHZg8H+1Yrzi3m6vWtoVNJCst3/f5y/Y9vOkrv6N/JE8xpnboL7jsLN7x/MtJJ490Iz6WMgw8I8FzPnELj/WNknemv+y863PIXsySq/43RiqLaScmfV0mnWLZkkUc6DsUaUr4S55yDm973mWkE3bFE7GwQ7Ai0Jq3felnPP2dX2XrnkPTGY4oaZnk2SDQfOeOHfznTQ/g+hpnQiDq7mjnrPVrSSbs8SuIhKnoSZu8/NwuTuiafGOuRGvN9Tffyj//19cpui7FUiUAZZjYnYtJrjkPIxkm0a1Z2MbHXnQe65d2kK4zc17rMCv/tsfH+PHmkfGkyL0P/InffeLvKY4MjCfRZrNZTjzxRL75zW9y9tln17WcShwv4OFDRb59/xBDpZIuo7k8D2zZzmguP55nk7ZN2lIWH37+uVxxatXGZpO67bFB3n7jVoYL3pEkWt/BGzrI4Vu+gjewF4BFC3v45PveydVXXEImHS0XqpbNO/byxo98k227D44n0ZqWjbISLN3wV2SXrAbCxnGvPq+b55zWQcJUdfewyhVdtu05xBs//WMeKR+8DItUz3JIZscrjSRMg4Rl8K5nnsnzzj+h7jy3HQMOn7jjMPtHvCNJtIGP47ps33Q/wwNhjcZ0wuJfnn0+1152GknLjCWfbl//MP/85Z9z64M7xst9GYZBoMHqWoqZ7UKpsGbhSy9ew788/QwSllExkFVScH0O51ze9uOt/HnncDhEt8DoxhsZ3XQrBGGVFdM0Sdg2//T6V/L2N76aZLK+/XzPoSHe9qWfcfvmnRPGoyY9yTSUImGZvPKa83n78y4jk6x5R6Plk2cnMyGhtvUrOjz3M3/goT2D5CqcgSmlOHHFUtatXk7SMnnOae1cujo7rQZ5gyNjvPNT3+I7v7gVLJvUiRdgdy+d9LXPOGs5H3zeOWSTVqSzNT/Q7B1x+do9g+wdOf7qwHcd/vLjz3Hf9z9JMmHzkY98hFe/+tWx3LbzA82IE/CNewfZ3Hf8rRGtNXsPHuLh7Y+j0Lzq0pN461XrSU2ja2rRC/j8bbv57K07cYoFhv703+QeuZ3JEmAuuXADX/vkB1i+ZHFs7dCv/82d/Mtnf0DOcelZt4EF65+IYR5/ErGy0+Zdl/eyfmEy0rI9PyBXdPjXr/6SH9z6wKSvMZJZUr0rMUybvz57Bf/+rDPpzk49+TnQmlseHeUrGwcouD77dj7Knh2PovXxV3onL+3iU696CmesWhhbO/Tf37+dN33mRvqGcthtXRidS1DG8dtGb3uS9/3t2Vx1+tLxK49q/CA82fzsrbv40u17cScJDm7/HoZ+/zX8gb1cdP45fOEj7+XEE1ZOazy33LuNN3/mRg6P5MJKDlUOmemERUcmxZ8+/gba0lW/QwlKFbRMUDrlX2+kUKFUzUT/+owz+buLT4y1M+UV772R7f0OapKD2ETPP38V73nOWZGumN792wPsG619i+OapQUuO6mb7u5omf9RfOJPh9jaX7uF9zlLbP7XyRlWdNcuKRTVS//p3Xz/xzehneOL0E507hNO51f//TXaopd1qem79xzgc3ccJEjUHs8fX31ipPf85y//nB/98UFG8tWfe5x14jI+8aa/4dTl8X2PX/jNg3z0p/cwlqv+WSYsg22feVWs7dBve2Q/r/7KH8n5tU9U7v73p7EwQhWQb921l8/ftof9I9VvD2dsxff+ZjGnnXxibIH2N/du5bUf/xGjEW5NZ1M2v/3ga1i9pGp5qDlT+65eCdti/brJ13fC7Lv5U2aoFtuIv1VyKtuOGhqq+bpiHc8kDo5Fu+e+oHcx3TEGBYDBQhCphXfKtljcMfnzralKaqdmQIKwdEzcJ1SZTBo73VaxdtxUDI0VagYkCCstTKfK+mQSBnhVyu/MpIRtYdpJ8Gs/A/QjPnN1fM3hsdpVTZQyWLd2TWwBCSBpW3Xc4ox3Jslcm31Xo+ZdTS030UEIIcTcJUFJCCFE05CgJIQQomnMu6B0cNTl0UPRcj+i0Foz0n8I7dW+d9+did7K4tTeZKRkyO3bHmHbY49Hes+oTuiyIz13C7Tm4Fi8CcKDBR9l137o3d6WxTKnPttvMgf7+sgd2lPzdfV0fVjZ28Wq3q6ar9OGxQMHirE+JxvzFKlMe83XhU0Z430Okvc0HW1tNV+nVDgNPoqOlMW6RbWfu2nf4/d3b8bz40u41ihOWjn5zNpjxV0Jfb5pmdl3b/n2XfzqwX3kKyYEKqxMO0YihW0avPi8xfzj5SeQSUz9wLZj3yHe8l/XcdfmHbiBJrHkJKzu5cc9YLVNxeuvWMcbrzyFpGVEegDr+pqcG07L3jTJtOzi2DAbf/QFdtzzBywDXvfyF/F///GNZNLTn3jgBRrP1/zgoWFu25k7blK2AnqzJgszJpahWN2d4PLV2UhNEis5PDTKv33+B/zk1ntwHBd/YC+FA4+Nd/ac6CV/+0w+9t5/o70tG0+H3aLDR75xA5/67k24PnSecCqLn/gMrNTxB9Vzl6Z452WLWJg1I6UT+H6A4/l8/qY7+NgNt06asNqx7EQWrd9AMmFzck+S/3NZL6s668upmajoBVz/4BA3bRnBcX0GDx3gsS2bJm19csmpy/n4K69gUUcmllylQGv++HiOHzw0RMENGB4d4/6Ht5MrHL8Nn7Kkg4+/+DzWLe7AjNCBNtAaxwv4+aZDvPdXjzFcOP6zdAf2Uth2JzY+S3s6+czb/o4nnr52yuPRWrNn2GV7v0PB9dm+ez+f+/7NHOyffGJTJmmzZkk3P/uPV5JOVM1VatnZdxM71Vapddf6U8IB/rDlAP90/T2M5N2jgpOyk5jp9vFEPoCUZZC2Df7zGSdx1SkL6jq4FR2Xj33vV3z6B7/B9fywnwxgmCbKTmEvPw0zHXZ9vXDtQj72wvNY0JacUh6P4wc8csjhuvsHGSyEfV623fEr/vyDz6F9F88NDzTpdIq2TIYvfOQ9PP2pl9e9nEmX7QX05Xy+es8Ae4bDK6KsrVjeYR/VAdZQYCq4eFWGMxan6sr9CoKA6375J971pRtwPG+8UZyBxvdcCnsexh8NEz5PXbeWr3/yA5x84upYgi/Ab++8nze873MMjeXIl5NnTQutDJac/zS6TzkfpQwWpE3++ckL2bA8XaUCRWUFx2VwtMCbP3sjf3ggbNSYaOti6ZkXk8i0gxFOhDUU2Ibiuad38Mpzu+tuxnf33jyfvvMweVfjlGcR6gDfD3h862b69u0GYHFnhg+/9DIuXr+s7qTuSnYOuXz9ngEO5fwjy0bj+5rH9+xj6+N7CLQmkzB5x9NP5wVPXE3CMurOFXT9gIIb8O6bH+WGv/QBEBTHKG6/C3foIHpCSap00uZpF53F+1//XHo6a1+5TTRc9NncV6Tg6fHjRhAEuL7Pz/6wkR/99s/jV2MJyyRhmfy/v7uKlzzlnCgBfl7kKVWZiTc/ghKE2d6fvmULX/qfbbiBwki3g2mhK3wGadvgnBXtfPa562lP1d45b73/EV7/oW8wPJonV6kDrTJYsupEPvKm53LZKUumvdMHWuP6mi/8+gE++N53MXxwD25x8o6tmXSaJ1+4gW9/9iO0t01/qni5qsStO0a5a0+etF35IGIZ0J40eNb6DtqTtQPw5h17ee0Hv86OfYcqtiRHByT9HP/6kr/iVS96DslEIpYE4QOHB3nz+7/AH+/dNOlZPIBpJ7Dbuvmnv38Lb7psTSwtvHNFl98/sIN3/2onZs9KDNNksv0zaSkytsGHr17M+t7atzP7cx6fvrOfzX3FKu3QfQq5MS7tdXjr084kaZux5CYVvIAfPjTMHbtyVEoV1DrAcT26gyH+7/86lWzSjFwjsZK867Pt4BjXfvRH7Hn4Xgz0pOW1ErZJwrL47NtfytMvPqvm+3qBZtthhwNjXsVEWdf1GMkV+PT1v+Cx3fu45vxTeN/Lr6GnQ9qhTzSVoNRyeUop2+Rt15zG8zas4jlfuZ+hGo968m5AR9Is3Vev7dp//wJjFQ5i43TA6686k6eetqzuEiqTMZQiaSm+9+n/pH/3o1WfO+Tyebo72iN346xFKUXCDNehLWFWbTDqBYAOA30Ur/vgN9i8Y2/1mrnK4Nl/9RRe9eK/JV1nmZhq3vel/+aWO+8fv8qdjO86rF+Q5E2XrIw8ploySRs3u4j0YhOvyriLnsb1fU7uiVbZ4Vv3D/KXA4Wq1QaUYXLGqh7+8crFsbZDv2NXntt35armtillkE0l+PDTnzDtwF6Wtk1yg30c2nY/BD6VFu+4Po7r84STVqK1rnlXZN+Ix/5Rr+q2btsWCzrbeMcrnk13MMIF66dXNUIc0bITHU5Y2MbKnmhXCqZhRC6aGvXhabJKccqpchwn0oNw0zQJJnkWMx2BjtTxGqNUpDKKoutFarFuWdGe39Sj4DhVA9LEZXuxt0NXkZ7fhKV3or2n4+uqAanMVCr+dugRv+84k1nLPD8gGaHaO4QJvVHWQetoreUBUglbAlLMWjYoCSGEmHta7vadEEK0kmbqPFutrt1kli7srHsZEpSEEKKJNVPtu+nWtYtCbt8JIYRoGi0blDYfzLNz0Ik079L1g8iJg1FaaAPk8kXcmB+QZzOZ0hTi6hzPxVDxfrUGwaSJrMfyAo0Z8YF2NpWINBOr6FaemjtV2XQKO8KUZMdxI3/nUdkmkSasaCBqA+GUZURqc+74OvZ26AmjcrfbiWYi/SSZsCi60aqK5ArRJgoZhop8YHR9zcOHorVjF9G0XFAaKfq85ze7een12/BUAssyq+4wKcvANhRulZbpE93wwbdw8orFZFKTT09WSoFSfOKmjVx/28PkHW/aZUf8QFPwAl789+/ihPVnYycr562kUyks08KL0DIg2rID8oUid/3xt9x/959xXYdKU+YsI5wOHqWvFcBX3/kqLjzjJDJVpnorw+SXm/v5zK07ybt+pBlzUfz7a1/I865+Mqkqy04mEnhmit9vPYzjB9Nua6912Fo+YSraEkY4u67Ca1OWYmm7xc5aOQ0lLz+niyetypCoEnE0Yf7Nh287xKjj48Y0De/ClRmeelIbthEm/07GVGG3519sHSHnBJHbVVRT9AIyC5bwxCuuwbLDfX0yqYTNgo4sO/ZFa4e+tM1iRadVcSwQdroecwLu2Vfg+w8O8fV7BxjMx3sSOl+1TPKs1ppfbBnkfb/dg+PpoxIIPc/DKRZQHJk2m7YNFmRsPvKsk7nghPoexvl+wNduupX3fO1GXM8fvyJShoGRSGN0Lceww/yS01f28KlXPYVVC9tJ126RfNyYvAAe6ivwi0dGyZcSW7bd+yd++ZUP4xYLuE54lpbNpFm0sIevfPw/ufj8c+taTiW5QpH7tzzG6/7jszy+9yAAy1as5HkveTmd3Quw7fCAbiqwDMUVJ2Y5aUGirqm/Wmt+cftf+PuPf4dcwaFQai1vmBYqkSZ18oVY7QvDZXcm+cAzTuLcFe2kp1EeaqKND23lde/9LHv7+seTaBO2jZ1IcO2113LOOeeglMI24KwlKU7oSmCq+qc3F72AQzmfT9x+mG39YaDxA82oE+D4R6YgWwosU/Gq87p53umdWHXm9Gw5VOQTtx9mIO+P7wOKcNr2cDEYr7TQnjB484ULeMqJbSTN2iV+ougb8/jGvYPsHHLHl2MQTmtfs8BmRYc93g59/cIkp/YmMRRTq+jgab553yAPHAi/s2IhxwO3/YZd2x/G98ITMqUUSdviZU9/Ev/6sr+u1Qn2ODk3YHNfkVEnGD9uaB0eQ7b3O+yZ0BG6XNXkkhOyPPmETJTvbU4mz8b4TKn1Kzq8/kePcs/eHPkKZ+laazyniOe5JEyDN16ygv990XLsaeQSHRwY5p8/fT2/uOMBfK2wupdjpDuO28GVghc/eT3/7wUXk05Eb4c+WPD54aZhdg8ff9XjFgvcdsPXufeWG7FNxTv/4Q28+dV/hx1D0qzr+YyM5XjrB7/IT//nruP+XSnFBRdfwjV/HVZYOLU3GZ6lTyMhczRf5P3f+ClfvemPuIEmfcITsJeuQ01yG/KKk7v50DNPpjsd7bOsxfcDvvTDm3nPF76H6wdc8uQn86xnP5tU6vgr0u6UwRNXZOhIRqth6AdhNY7r7h/kl9tGj7sNqXVYDmjUCTAUXLAiw9uetJDe7NRvGfqB5mdbRvj2XwZxfE3BC9vbT2b9wiTvuryXlZ12bK3l791X4Lr7Byl4mp60yckLE5Mm62ZsxQXL0ywqjTVK7TsvgN8/NspPt4xOeqV3eP8eNv72JxRGRzh19RI+87aXcurqZdMaT9+Yz8OHiriB5nDOZ8uhYsXKFeUmom96Yk/Vq1bmQO27yWbaVallV6/WD0rnf+oBitVS5Etec/5CnvuEHpZ01HfWVM0l//J1th/KoYzqZ+8vueRU3vPCi0lFKDv0xY397B2p/SzlvPYhLljVxdLFvfWsclXXvuOj/G7jXxir0TH17551Ne987YtZ3Fm7DE5Ur/vWXdz8yBBGonptu7OXtXHd350eW902gD9v72PjziF6emt/li84I9rV9Vfv7ufWx3MMFavfdlzTZfOKc7s5Y3F8n+V19w3w9XsHmKRu6VGSJvzqZWtiq7QAsHvI4ZZHx0hFqITxrFPaI73uDzvG+PW2Ufpy1QdkK83LTnY575QTYkvY3Xa4yE+3jDBQqH37OGHAGy5YwIJM1W2z6a+UZnimXcXxN+yZklLqH5RSDymlHlRKfVcpFd/eWEVXxo41IAGks201AxLAWNGNnCkeJSABLFu2PNaABLBt176aAQnAd4t0puLdhDo6O2sGJICir2OvTNDT3cmiRfF+loOFoGZAAjANxYkL4iujBOEt6ijbUNyTSACSlkF7Mtq2EXXxjq/pj/DcxjAMzl4XX0CC8BZjzo24pvEXrphXGhKUlFLLgbcAG7TWZwAm8MJGrIsQQojm0cjZdxaQVkpZQAbY28B1EUII0QQaEpS01nuAjwA7gX3AkNb6V8e+Tin1GqXURqXUxr6+vtleTSGEaIiJxz7HmV95UA0pM6SU6gaeBawBBoHvK6Wu1VpfN/F1WusvAl+EcKJDHMtOmArH17VmxtT3nuksljWE51XPDWpL2ZFvN6/ssNg1XPu50pgTMFTw6UzF1xr8lNXL2XPwMCO5fNXX+WaSPcMea3vjexaSSVh0pW0G88d3SZ0oTBaN9+b9aNFnzAlqPnSvZ6ndaZPulFHzAbmhYKjgk4mpRQaE6WSdKYO+XO1lx80LdKQK8BC9EnrSVPRmTPaPVX+upLXmkUNF1vcmY60u35M22TtaO/8vjrljE499yXRWz0btu2Nn202lbl0cGjL7Tin1POAarfWrSn9/KXCh1voNlf5PzXboN+7gzl2jFaeEJ03FC85eyFXrurENxcKMGbmldSWDBZ9v3z/Ig/sLOJ7HAw88wJ49e457naEUL7viNP7P315IyjYjPYD1As1I0eeHm0bYOVT5AK0A04BLo+dH1OR6PmP5Av/w4S9zwy23H79MZbDyrCex9qJrSNg2l67O8vJzuslOox26F+gw92PYxfU1Nz2wn5s3HZx0MsPVpyzg/X99Ep2pmKaEB5pbHh3lpw+P4GtY0maxticx6We5IG1y4Yp0mPwadUp4oLn+gSF+umVk0pOMFR0WF6zIkLQUvVmL85elp/VZBlrzcF+RBw+G05gf7ivyx8fHJp3GfMaiJO+6fBFL2q1YDuBaa3YOudy9N19x2nRZW8LgguVpejJmqU1HtCnhtz0+xo83j0zazNDxw/1GKVjebvOWi3pY0z31k6ZywvOoq/ECza4hl1seHas46cE2oD1p8oYLFsQ2JXy2Zt/NRl27CZprSrhS6onAV4HzgTzwdWCj1vpTlf5PlOTZW7YN8+7f7KbgBUdNDz9vRRuveeJiMrYx3uOofDBf3mHTVucBwA80v310jJ9uGcEPjswC832f0ZER7rn3XkZHRwE4a3Uvn3rVU1i2oG1K05ddX7PlUIGbHhk9bkdQHJm5ZBuQsQ3+5rSOae2EE+UKRTZt38lr3/tZtu/aB0DH4lWcfvULSWU7UZY9vmzbVLz6vG4uW52tO3m2b8xny+EigT4yE8z1AoYKLl++bSfb+sYAWNmV5EPPPJkzl7bFljy77XCRr987eFRiaZgcC+sWJlmUNceTZ89ZmmZlpz3l5NmBvM8n7jjMlkNh8mxbwuCJK9J0p83x6diK8MrltN7klM70D4553Lk7R9E7sl0GgcYJNL97dGw8cbcjafAPF/VwyerslNq7T2a46PPn3XkGC37VmZGGgtN7k6xbONXkWU3RD7juviHu2x92YA50WGGh4B1JRFaE2+UVa7L83dlddV+Flk8MJ44l0Bo/gNt35bh/f3F8WQbh8eSKNVkuWpmJMr1eglKlf2hUnpJS6t3ACwAPuBd4tda64s3TqO3Qc47Pp27bzw8e7KcjafK6i5awtiddsdOmIjw4LO+wIuVpPNrv8LV7BxgqHDmIHU3j+wGH9u3ixWcv5BkbTox8dVSJH4RnaT95eJgHDzrVO2IasK4nwbNP7Yilu2gQBBRdj09//1f89NGA7jWnY1oWk7bwNhUrOi3ecUkvPdVzNIDJM+aP5XgBD+4dImvByy5YSsI0YsmnGXUCrn9gkPv2F3Ar3A0yVZgI+dentHPhygymInKNxEqKXsAdu8b44848q7snvxorLztpKS5ZlaUrXTsAF7yAu/fm2TfiVQwInq/py3koDa/esICkqbBiuI3tB5oHDxbYetip2QxycdbkiSsysbSWL3oBjw86fPz2wxwY9VEwaffZhAm2afCPFy/gnKW125VrrRlzwwBXiVdKeL55+yj9OZ813Qmeub6djmTkkyUJShU0rHWF1vrfgX+P+30zCZN/uWI5f3NmD4/2F1nWWf1sc/ysKuIm8sk7DlfdWEFhmiavveoMnnpiWyzdZ00j3IH780HNnA43CM/i40qENAyDdDJB5ykX0muOoavsS0Vfk3d05B3zoYMFRp3qI0pYBn+1fiGnLExMq/rGsW7cPMw9ewtVz+h9HQalC1dmsGN6Bpm0DFZ0Jji5p/p36WvIu5qOiHlg9+8vsGe4egtvy1SsX5jkqWvbYrnNW/bYoMPWw07NvDFDwaWrs7E950laBgFwOOejqRwMHR8cP2BZeyJSO/SCp2vs4+Fn2ZkyeM6pHYw5QeS29aK2lu2ndPLCFIZh1Ny4oBSQNJHOXbyImYYp04i9HXrUZZuGKlVDju/A4wWqakAqs0yFF+hIB/GoSZuWoWKf1BA1+dY2FH7E8USldbjNRblJEXWpXhCthbdlqFgexE8UBPGOpR5+EN6icyNsTFFr/EX9eJRSJExYJgEpVi0blIQQohXE1Xm2VtfYRs22O5YEJSGEaGJxdZ6d5WdGU9Zy/ZSEEELMXRKUhBBCNI2WDUoDeZ+BvB+t7XQdcwKiPnD3Aj3tjrPHijpjyg9qzzCayrKjLN4PdOT1jDoBLJ5es0ezI47HC3SsLR0g+kxPiP7Q3YzYkjzQOvYKDoYi0v4zE8knpgIv4gbiBDrS8aCej0cKgsev5YKSF2hue3yM7/5lkPv25RkuVm+9HOY2RC+J8qYLF9CdNqiWu2kqePiQw8ExL5a2z1prgkBz6eoMC9Im1dKPbCP8DOJYbnnZXqDZsDzNig6LapPQynkvQ8VobaFP603RnqgeHBSQdwJyboDW0Q4qUTzr1HaesDhV83sME0JzuH48JxleoLEUqBpvZaqwJfpQhP49AGctTrG0vfr3Y5QO4IMFP9bPcnV3grXdiarLNlW4fQxHHE8UWmvWdNs859T2MO+pwvJtA9K24vEBJ9LJWspSJCPMtjQU9eQliYhapskfwGMDDrc8GnaknHj2tChrccrCJKZxJHu8nDm/vMOivc4Nyws0v9o2yi+2juIHR1qsW0Y5Sz8znvDYmTQ4tTc55WTBQGtGiwH7Rj28INwRHx90uWdfnkAzPq3ZNsJ6cM8+tT22nIlyRvudu/PjPYEOjnn8eVeO4oTP2DLCK4+Xn9PFlWvb6spD0Vqzf9Rj62EHrY9cFRmAYcDKDpu2Ul8eU4Wfr2XE074bwvbhX79ngDE3wCnFUkOF28dJPQmWtoeljJKm4txlKZa121PK8fGDMPflzj05+qrUbitvl6csTHJab7LubWb/qMufd+dxJkx5L7ckP2VhgiVt4XjKSeOJmFqhQxjs7tydY6QYjC+7PJ5Te5OsX1j/eCajdTj9vdxKvrzs79w/xKa+4lFJ7QlT8eRVGV5+bnfdlVsmq+hQlrEUaTtaqakKIv/HqJ1no8yui6lrbByar6JDvWoFpV88MsKOQafipbypygcZG1MpejIGvdnp1fs6nPP41n2D46Vbzl6a4sTuxPHt0Anrm63pToQHvAjLLJcz2TPsMjZJnS3H19y/P8/jgy5KwcUrM1y2OhtLPk152ffsy7Nj8Pi6e+Xaapv7ihgKnrgiw6vO657WWaPra7b1h1eXWkNv1qS3Qm3ChKnC2nPUX+5nMl6guXnrKL/cGta+W5S1OKknMelnuTBj8sQVabIRD0iBDk9aNvcVePiQUzU3y1Bh0c8LVmTqPoBO5AeaTX1FthwKy+AszlqcXGE8thF+llG3y1q01jw24HDv/gJaQ0/G5ILlmWnV8pv43hAmt+bcyZOPN/cV+do9A+TcgN6MxVsu6pnWSZrWmoIXjO+DtgFtCTOO4Bp7RYe5MruupPWD0uf+fDjSveVLTshwWm+KhBXf3eCbtoxgGdQs67Os3eSkBdHOFncMOOTc2gmRCUOzMGvTHaEUTVS37xpj/6hfoYzSEQvSBid2JVgbY/LgwVEXT1OzirupoDM1vYK6x7pnb5779xfI1DiAKuD5Eduh378/z84ht2bX0q6UwVmLUyxui6fILIQnTSNOECnA9aSnVwrrWEUvYMQJWBih3FRUBTcg7wU1k549X3NwzOP0RanYngeWT9RiTKKWoFTBvMxTijMgASxusxh1akfEqA9kgUmvjibTk7FiDUgQtvCuFZAgvJ++Oqbir2Vp25i0+vNs6EyZtCeNmge9etYu5+pIbbQNpejJxBeQICzQ26hTzqRlxFJ7cSIN0apwmIozF6dib4duyOOjWdFyEx2EEELMXRKUhBBCNI15eftOCCHmikq175qlU2zc5l1QMlQ5gTC++82WcXTDvWrLjippqkjPVsq9luJsRZC1FWNOtPv3rh9vcmk56TjO7ycqRfi55yJUlo8qYSoSpqr5jE5B7NXIoby9x/qWkejSrMO4E48bNZ5GqlT7bo5NbIisZW7fLW6zqNZY0lBh6+clbRb9eZ+cG63aQzWBDvMYVnRYrO9N0J6svAKL2yzW9SQjB6a1C2zWLrBJ1ZiUcTAX8Mghh8M5L7ZkyCefkOUZp7SztK3yOUtXyiRjGzw64LJvxJ12YmmgNQdGXfaN+BwaC8g5QcXx2IaiI2nGlk2vtSbv+nSkDC5YkeakBXbF76kzaXD12mzkz/qcpSmesa6dE7vtiq9ZkDY5rTfBqBswXPRjqwSSshTdpe+pElNBV8wJoI4f0J/3GSj4MzKedJV9ojwrU8xdLXOl9JxTO9jcV+TWx3NHtSgHWNxmcuHKDElTjXcOzbmagufTnjDrPjs9NndBKYWlYFWnRc4N2D3k4ZZm2mVsxWm9STJ2fR1Tw4RNWNNtM5j3OTDmVzxD1MCBUZ/+fMCKDot0nW2fj2UoRcpSXLwqw6Exjz/vzZMvjTVpKlZ22iRMhaEUGhjIBwwVHJZ1WFPKVRop+uwZ8cK+PKWfjTrhrLXOlDH+/RgKsna8CZ/HJkiahmJlh83SdotNfQ6Hc2Giq2XAmYtSnLggUVc79PKsrXOWpjm5J8mdu3MMlqoaJE3Fqb0JulJH8l4cX9Of98naipQ1reTM8f+btiBlmYwUg/G+Q4pwdl75pCeOz9MPNKOOP77tw8yMJ1zvMHl2JscjGqNlgpJSitNKB41bd4yxtd/BNhQXLE+zuN2a9PZWoGGo6JM0FdmEEemWUbUsb0MpsrbBuoUJDo36tKcMlrXbU05MLGfed6VNOlMmu4fdip1aNeEB4LEBl86UwdI2a9qtuy1DsajN4uknt7PpYIGhQkBX+vjcoPJU3d1DHmnbZ0WHHSnQu75mz4hLzjk+H6v8nv35gLQFvVlrPN8mjgNOoDVjTjDpLVLDUCRQnLkoyVDBZ6AQcPaSVNhscIqfqWUoOpMGTz2xjccGHPrzHid0VU6mHiufNCXNad+aLW9HHUkDN9A4viZjx5d8HF5pBlVve8Y9HlOF43F8jRvEOx7RWJGDklJqLbBba11USl0OPAH4ptZ6cGZWbWpSlsFVJ7Vz5ojLcNGf9CB6rHp6tA4V/KrPjsoHgBO6bdJ2tEBXi6HCujd+hDyn8VyOmPZNQ4W16RZmLEyj+gpowA00URvu7hxyI3UGtk2DbGJ6Z9nHqhSQJjINxcKMyck9yViWXT6YLu+w6EzVHo+vqVpPbirLt0sloWJNlPV1pOdwMzGehEmsV86i8eq5UvohsEEpdRLwReBG4DvA02dixaZrSbtN0lKRHtZD9MAU9e54eDCPd0eJ+oDXKM+6iHHxkceNilx1va7xNIgq3aKMdxUadxCdieU2sijMfAhGE2ffTZxx1yqz7Y5VT1AKtNaeUuo5wKe01p9SSt07UysmhBDi6Nl3rTrjbqJ6noi7SqkXAS8Dbir9rPKUIiGEEKJO9QSlVwAXAe/TWj+mlFoDfGtmVksIIcR8FPn2ndZ6E/CWCX9/DPjgTKyUEEKI+almUFJKPUCV59xa6yfEukYxmdh8L4q4H5dqwqmycT6IjfpW9bR3j7zsiK/TRF/Pet6zkeLeNhr5bL6c9DsfJgiIuSnKldIzSr+/sfR7+ZbdtTT+eHGcKDkTxwpKnSyj7KYdSYOR4uQNxo56zyD+yLC8w2L3kIfjV+6zVC5XE3dg6s1aOMHkOUUTl11un16rHxKUxjPs4dYYj+fHP56MbRDooxM9J+NryLnBeFWEOA7mSVPhRigjpQjz4uIqPaR1mFTu+AFpK8bxWAoviDYeX2ssCYh1KRaL3PrxN7N+3cktO+NuoshN/pRS92qtzznmZ/dorc+dkTU7RpR26K6vGXEqVz44lmJqLaG1Djtf5icJfIZiSlUi6ln2wCQVHlRp2cvaLdqT8eb0TDRS9Nkz7BHoo89IFLAoa9KTqa9ZnNZhxv+BUf+49zMULOuwaI85R2miohcw6tQ+yTBUuK3EmePj+mEFhMmO5SlLRe5uW0u5fXjODcbzwmZqPJX2v7SlwgRXCUhlkT+Ijq4F+vzL/6rVZt1VHH89U8KVUupJWuvbSn+5mCaqnTdS9OtqDldPFYdjKaXIJkySVnhQ8SaUFEpPs5RKlGUvyFh0pEz2jXiMFMOFd6cNFmWt2AtgHqs9abJuoUHfmMfhXLjsbMJgWbs1pUCsSs3tOpLheMrNErvTBovbpteuPoqkFZ6UVDrJKAs0DBcDbEOFQZ/pX2XYpqIrZR5VsspUxFL1AI7cqnN8fVzgLY8nztbythnWpst7wXhjQ6vUPjzOgsGitdUTlF4JfE0pVb5+HCz9rClEDUhJU5G2jVh2krB0jBlWMVBTL0Ez1WWv7LTJuQGGCitZzBZDKRa32XSlAvyAmu3Do7BNxaqucDymqt1aPk7lkwyl/JpdYt0gvLKLq823Uoq0HZ7geIGO9cql6GkKflC143G5Nl1PTN2LlVJkbJPUDIxHzA+R9iyllAlcprU+qxyUtNZDM7pmM8Q0VKxnbWGpk8btdNUqQM+0mQgcjRxPeFXWmMekxgxsRxqqBqSZNBPjEfNDpCOA1toHXlT689BcDUhCCCGaWz33IG5TSn0auB4YK/9Qa31P7GslhBACCGffbd70EFc++0UsXdjJdV/+fKNXaUbVE5TOLv3+ngk/08BTYlsbIYQQR0kmk1z6ti8AsO/nn2zw2sy8eio6XDGTKzKXhS3BmfGZYpMpd/VsxLJbzUy0lo9qptqHN+4pmRBTU08/pU7g34FLSz/6H+A9zfJ8yTJqP9RNWYq0pWKrtOAHmgOjHgOFAMuAFR022RhmokWhtR6fequArK1IzvB09FaltebgmM+WQ0UCDb1Zk97s5H24yvk9cVbrmJjfM51UhWOlLEXKMqtOdw/z6poms0OIum7ffRV4EHh+6e9/B3wN+Ju4V2oqOpNH50dMVM6VqKeNdTVaa4aLAXtHvPFeMl4Ajw+6tCUMlk4xZyeqY5MUNTDqavIxdfacT3JuwOa+IqNOMP55HhrzGcj7rOi0x7vdwpEEUIiv++2oE+BMSGco+hqn1D58uicZx7YPH5mQU1f+eVrah4smU09QWqu1/tsJf3+3Uuq+qSxUKXUK4YSJshOBd2mtPz6V9yu953h+xEgxwA3KVxAGyRh3vKKn2Tvikncnb+E94gSMHnZY3GayIF1fdYNaqrXwhrAkzmDBj7UaQKvyA82OQYfdpeoUEwVAEMCOAZf2pMGabpvOpDnltvbH0loflTB73L8TnmQUfD+WxNNyx9vOpBkGPU/TFlMCsBBxqyco5ZVST9Za/xFAKfUkID+VhWqtt1CaOFHKgdoD3DCV9zqWoRSdKROn1Ds87uS9xwacmt1sNeFBLW7DRT9S3kk9hWjnq+0DDnuHvarPWzRh3b2u1OS38qaq6OuKAWkiL4i/fXjSDG8RSjASzaqeoPR64BulZ0sK6Cds+DddVwLbtdaPx/Be4xLmzNwnj9zC24j/LDRq22k53NTmVSkCO5FhKAIdb0t2aR8u6lFuh56wLa64eEOjV2fG1TP77j7gLKVUR+nvwzGtwwuBSSsNKqVeA7wGYNWqVTEtTgghmtvEY18yneHKt3+Ovp9/suVzlKCOgqpKqe1KqW8DLwZWxrFwpVQCeCbw/cn+XWv9Ra31Bq31ht7e3jgWKYQQTW/isS+RSDZ6dWZVPfe4TgO+APQAHy4Fqek+B3oacI/W+sA030cIIUQLqCco+YBb+j0ADpZ+TceLqHDrTgghxPxTz0SHYeAB4L+AL2mtD09nwUqpLHAV8NrpvE+z0jr+dugiPvV8L/INNjfZz1pLPVdKLwL+ALwB+J5S6t1KqSunumCt9ZjWuqdZKkJEtaLDCvNVqrxGEeYzxT3Jqi1hRpoF5gXxL7vVrO6yaU8YVT9PQ4WJym7Mc+yTliIR4YtUhMsXk9Na4weaoq/D7rqNnNY4g7QO6Pv5J+dFK3Soox36+H9Qaj3hs6C/BxZprdMzsF7HidIOfbaUywsNFoLjWnibBizvOLoSQJy01uTdgNwkZWMUYffblJQbikTr8Hvc2u8QBOE9aaCUVAqrOm1O6LJnrK6g4wdHVZKYKM5yQ62mfMyaWD7JVKWqLQ2qQTkFkVeyt7dX9/X1zeS6NML026ErpX4InAVsJ7xieilw57RXbQ4yDcWyDpvudMCeYW+8TExPpnLNtLgopchMaMXulo6kchCrn1KKJe02PRmL7f0OB8Y8ADqSBusXJknPcMPBhGnQnVJHnWSEtejMGS1TNVeVg5Eb6OOCua9hqOiP7wdSrWLuqueZ0vuBe0sN/46jlLpKa/3reFZrbkjbBmsX2AwXA9KWQcKavZ3ANBSdKQvHD1AoOYhNg20q1vcmWd5h4fqa7pjLQ1Uz8STDCzQJqbZQUdHXFL3qt1OLvsbN+3TH1N5dzL56kmdr3Tv7IDCvghKEB5XOVON2gJmqXDEftScb9z2ahoq9bUWr0ZpIz/da88nS/BHnEU32KCGEENMSZ1CSExQhhIhZoehw7atf1+jVmDVy70cIIZqYMkz2HZpTmTPTEmdQ2hHjewkhRNMI29XLzaDZUM+U8Mk6zA4BD2itD2qtm6IDrRCiNaUsRdIyqza6LLerj9PETs/l7sMyQ3Lm1DMl/FXARcDvSn+/HLgbWKOUeo/W+lsxr5sQQoxTSqEIg046CNu7T4xNM9Gu/tgAmPc0Rd+nLWHIzNcZUk9QsoBTyxW9lVKLgW8CTyRMppWgJISYcUopTCPsCFzwNI4fjJfgirNdfc6dvFxXoGG4GGAbmraEIVP5Y1ZPqF95TIuJg6Wf9RNWDxdCiFmhVJhknLIUHUkT04gv6bjcrr7WEyQ30AwWJq0lEKtisciO7Y/M+HKaRT1XSr9XSt3EkYZ8zy39LAsMxr1iQghRy0w826mn/u5sTH1IJpOsXrtuFpbUHOoJSm8E/gZ4cunv3wB+qMOCVFfEvWJCCCHmn3rKDGml1B8Bh/AE4c+6VWvFCyGEaIjIz5SUUs8H/kx42+75wJ1KqefO1IoJIYSYf+q5ffd/gPO11gcBlFK9wG+AH8zEigkhhJh/6pl9Z5QDUsnhOv+/EEK0HD/mzsTH0jqYN11nob4rpV8qpW4Gvlv6+wuAn8e/SkII0TgpS+H44AW1XwswUPBntNJDKmFz3Zc/H/v7Nqt6Jjq8XSn1t8CTSj/6otb6hplZLSGEaAxDKTqTJo4fdriNch0klR7iU8+VElrrHwI/nKF1EUKIpqCUImmFHZ1zbkDBqx2aypUeEoamPSn18aaqZlBSSo0weY6YIpwp3hH7WgkhRBMwlKItYWIqnzE32rMjJ9AEGkyJSVNSMyhprdtnY0WEEKJZmYZCUbv0kJg+ufkphBBNTDrPCiGEaBrSeVYIIYRoEAlKMdBaI2UA4yGfpRDzW11TwsXRtNbk3YCcpzEVtCdNLGn4NSVa67CPjROgSi2tJd9DNAvbUCxIm4xFnB4us8GnToLSFLm+ZsTxx3uv+BoGCz4pS5GdoczuVuUFmlHHH8+g1+V8DzPs7GnIZykarLw/Z22DtBW2Yp+s4oNtKNlmp0mCUp0CHWZ5O/7kZ0sFT1P0ypnd8XXDbEVaa3JuQL7Cmafja/rzPllbkbIk0IvGU0phKuhMmuNX9howZvDqXjrPiqoGC37NzpSa8Ow/IdlzVQ0XfdwI9cUcH1KypYomopQiaYKdMil4wYzVvQPpPCtqiFoQ2FBylVRL9M9yZtdDiKkoXzVlE2ajV6WlyJNkIYQQTUOCkhBCiKYhQUkIIUTTkKAkhBBNbL7NvmtYUFJKdSmlfqCUelgptVkpdVGj1mWmSGUC0Yxku5xb5tvsu0ZeKX0C+KXWej1wFrC5gesSWdaONhXM8SP2Up7HMrZBlE/T8TVeIOWHpktrTaA1rnyWook1ZEq4UqoTuBR4OYDW2gGcRqxLvdK2ScIMKxBMlmOjgIwke0aStAxsUzHmBBQrJCNDmPc1VAxImGG2vAL5bOtQDkAFL0xW1oBlQHvCxFDyWYrm0qgrpTVAH/A1pdS9SqkvK6Wyx75IKfUapdRGpdTGvr6+2V/LCkxD0Zmy6EgefaafNBXdaZO0bcqOHpGhFO1Jk66UWbNTp+NrBvI+BU9L4dYIyp9RuQTWWCkgAXgBDBT8MEjJZ9l0Jh77HKfY6NWZVY0KShZwLvA5rfU5wBjwjmNfpLX+otZ6g9Z6Q29v72yvY00J02BB2iRjKzqTJu1JU2peTZFlqEiBSQNjbsBI0Z+V9ZrLHD8siTVY8Kl0IZr3NAMF+SybzcRjXyKRbPTqzKpGBaXdwG6t9Z2lv/+AMEjNOUopMraJLSWFpk0pFbnKuq+R1tQ1BJqqt0Unvk40L5l9Nwu01vuBXUqpU0o/uhLY1Ih1EUKIZjbfZt81svbdm4FvK6USwKPAKxq4LkIIIZpAw4KS1vo+YEOjli+EEKL5SEUHIYQQTUOCkmgq2YRBV8rEqrJlGip8nUwtqS5phS287SqTRxRhN1UhmoX0UxJNxVAKhQ47e3r6qNwaYLzdPEjSZy3l9ISOpIEbhNPDJ860k2TkuUHrgKULOxu9GrNGgpJoOuUDZNKChGUy5gT4WksFgilSSmEb0J0yybkBjq9pSxhYhjSinAtSCZvrvvz5Rq/GrJGgJJqWUgoFtCXkymi6yp9dxjbI2Ef/TIhmIjeTRdOTg2d8yp+lfKaiWUlQEnOCHETjI5+laGYSlIQQQjQNCUpCCNHELGt+PfqXoCSEEE3M87xGr8KskqAk5gTp9yPE/DC/rgvFnKO1RgNag4GWh/RCtDgJSqIpla+M8qUW3hB29s1KBQIhWpoEJdFUysHIK5XFmdijruhrnLxPNmGQNMOfSXASorXIMyXRVIqeZsQJGCoGk7bw1sCoEzDiSAtvMT8Uig7Xvvp1jV6NWSNBSTQVN9A4EVp4+4G0QxfzgzJM9h0aavRqzBoJSkIIIZqGBCUhhBBNQ4KSEEKIpiFBSQghRNOQKeGiqWQTBqlJpoNPpAj7ArXKZPBAh+N1fU3WViQtQ6a6i3lLgpJoKoZSKAO6UiaFUjv0iVopgVZrTcELGHOPRN9RV5P3fNqTJpYxt8cn4iHt0IVosHKwSVmQtMzwqinQtCdNzBZph+4FmpGiP+nVoK9hsOCTshRZW66a5jtphy5Ekyi3Q29PGEf9bK5z/IDhYlDzdQVPk7FpmduUQkQhEx1E02u1Ft5+7Xg0rjVGLER0EpTEnNAqAUkIUZ0EJSGEEE1DgpIQQjQxKcgqhBCiaUhBViEmIe3IhRCzQYKSqElrTaDDygMSnKYvYSqi5sW6gXzeYn6RPCVRUTkA5dyAvKfHy/ukSluNzIibGtNQdKdM8m5Azps86JgKqeog5iUJSuI45WDklmrQlU/WNTDmBhQ8aE+YmEb4DxKc6qeUIpMwSVqaUcdnYjWlrK1ISf07MU9JUBLHKXqaoq8r3jryNQwWfbK2QcqSA+d0mIaiM2Xh+AGOr8nYBoYEIzFBsVhkx/ZHGr0as0aCkjhO3qtcoXsiL9BIzYF4JEyDhNnotRDNKJlMsnrtukavxqyRiQ5CCCGaRsOulJRSO4ARwAc8rfWGRq2LEEKI5tDo23dXaK0PNXgdhBBCNAm5fSeEEKJpNDIoaeBXSqm7lVKvmewFSqnXKKU2KqU29vX1zfLqzV8dSZO2Gu3GDQUpS85phJgJE499TrEwrzrPqkZl6Cullmut9yilFgG/Bt6stf5Dpddv2LBBb9y4cfZWcJ7TWod5SU5A8ZipeGlLkbHDgCS5NEJMSeQdp7e3V7fgSXnF8TfsVFdrvaf0+0HgBuCCRq2LOJ5SCkMp2hIGnaU25JYB3SmTTKlFtwQkIUTcGhKUlFJZpVR7+c/A1cCDjVgXUZ1SCsuArpQZBidDgpEQYuY0avbdYuCG0sHNAr6jtf5lg9ZF1KCUQmstwUgIMeMaEpS01o8CZzVi2WJqJCAJIWaDTJ8SQogmZlmNTiedXRKUhBCiiXme1+hVmFUSlIQQQjQNCUpCCCGahgQlIYQQTUOCkhBCiKYhQUkIIZqYzL4TQgjRNGT2nRBCCNEgEpSEEEI0DQlKQgghmoYEJSGEEE1DgpIQQjQxmX0nhBCiacjsOyGEEKJBJCgJIYRoGhKUhBBCNA0JSkIIIZqGBCUhhGhiMvtOCCFE05DZd0IIIUSDSFASQgjRNCQoCSGEaBoSlIQQQjQNCUpCCNHEZPadEEKIpiGz74QQQogGkaAkhBCiaUhQEkII0TQkKAkhhGgaEpSEEKKJyew7IYQQTUNm3wkhhBANIkFJCCFE05CgJIQQomlIUBJCCNE0GhqUlFKmUupepdRNjVwPIYRoVjL7bna9Fdjc4HUQQoimJbPvZolSagXwv4AvN2odhBBCNJdGXil9HPhnIKj0AqXUa5RSG5VSG/v6+mZtxYQQopEmHvvy+XyjV2dWNSQoKaWeARzUWt9d7XVa6y9qrTdorTf09vbO0toJIURjTTz2pdPpRq/OrGrUldKTgGcqpXYA3wOeopS6rkHrIsSs01o3ehWEaEoNCUpa63/VWq/QWq8GXgj8Vmt9bSPWRYjZpLUeD0gSmEQUhaLDta9+XaNXY9bMr7mGQjSQ1hov0Iw4AVpDNmGQNEEp1ehVE01MGSb7Dg01ejVmTcODktb698DvG7waQsyI8asiYNQJcPwjV0ejTkDBgPaEiVGKSxKgxHzX8KAkRCvzAo0XQM4NmOxmnRfAQMEnYynSdqPTBoVoPAlKQsygvKePujqqxAk0KUCuk8R8J6dmQgghmoYEJSGEEE1DgpIQQjQx2zZZurCz0asxayQoCSFEE1tzwglc9+XPN3o1Zo0EJSFmUNSJC1rLJAchQIKSEDMqmzBIGNHCTSAFHoSQKeFCzCRDKTpSJo4fMOoExwUeBWRtRdIyJHFWCCQoCTErEqZBd0qRcwPyXhiZkqYimzAwJBgJMU6CkhCzRClFNmGSsjQasCLe1hNiPpGgJMQsMyUYCVGRTHQQQgjRNCQoCSGEaBoSlIQQQjQNCUpCCCGahgQlIYQQTUOCkhBCiKahyu2am51Sqg94fAr/dSFwKObVaSYyvrlNxje3TXV8h7TW10R5oVLql1Ff2wrmTFCaKqXURq31hkavx0yR8c1tMr65rdXH1why+04IIUTTkKAkhBCiacyHoPTFRq/ADJPxzW0yvrmt1cc361r+mZIQQoi5Yz5cKQkhhJgjJCgJIYRoGnMuKCmlViqlfqeU2qSUekgp9dbSzxcopX6tlNpa+r279HOllPqkUmqbUuovSqlzJ7zXy0qv36qUelmjxjRRXONTSp2tlLq99B5/UUq9oJHjKovz+yv9e4dSardS6tONGM+xYt4+VymlfqWU2lx6v9UNGta4mMf3odJ7bC69pil6ekxhjOtL+1pRKfW2Y97rGqXUltL439GI8cw5Wus59QtYCpxb+nM78AhwGvAh4B2ln78D+GDpz08HfkHYefpC4M7SzxcAj5Z+7y79ubuFxrcOOLn052XAPqCrVcY34f0+AXwH+HSjxxb3+IDfA1eV/twGZFplfMDFwG2AWfp1O3B5o8c3xTEuAs4H3ge8bcL7mMB24EQgAdwPnNbo8TX7rzl3paS13qe1vqf05xFgM7AceBbwjdLLvgE8u/TnZwHf1KE7gC6l1FLgr4Bfa637tdYDwK+BhmdNxzU+rfUjWuutpffZCxwEemdvJJOL8ftDKXUesBj41eyNoLq4xqeUOg2wtNa/Lr3XqNY6N4tDmVSM358GUoQH6yRgAwdmaxzV1DtGrfVBrfVdgHvMW10AbNNaP6q1doDvld5DVDHngtJEpdsZ5wB3Aou11vtK/7Sf8GAF4ca0a8J/2136WaWfN41pjm/i+1xAuPNvn8n1rdd0xqeUMoCPAkfdLmkm0/z+1gGDSqkfKaXuVUp9WCllzs6aRzOd8Wmtbwd+R3gFvw+4WWu9eTbWux4Rx1hJ0x9jmtGcDUpKqTbgh8Dfa62HJ/6b1loTnonNWXGNr3RW+i3gFVrrIPYVnaIYxvcG4Oda690ztIrTEsP4LOASwqB7PuEtoJfHv6ZTM93xKaVOAk4FVhAeqJ+ilLpkhlZ3Slr9GNOs5mRQUkrZhBvLt7XWPyr9+MCE2zpLCW9XAewBVk747ytKP6v084aLaXwopTqAnwH/p3TrpCnENL6LgDcppXYAHwFeqpT6wCysfk0xjW83cF/p1o8H/Bg4apJHo8Q0vucAd5RuS44SPne6aDbWP4o6x1hJ0x5jmtmcC0qlGTpfATZrrf9rwj/9BCjPoHsZcOOEn7+0NAvoQmCodAl+M3C1Uqq7NIvm6tLPGiqu8SmlEsANhPfzfzBLq19TXOPTWr9Ea71Ka72a8Grim1rrhs9uinH7vIvw+Uv5OeBTgE0zPoAaYhzfTuAypZRVCgCXET67abgpjLGSu4CTlVJrSvvjC0vvIaqJa8bEbP0Cnkx42fwX4L7Sr6cDPcAtwFbgN8CC0usV8BnC5ykPABsmvNcrgW2lX69o9NjiHB9wLeGD1/sm/Dq7VcZ3zHu+nOaZfRfn9nlV6X0eAL4OJFplfIQz075AGIg2Af/V6LFNY4xLCK9sh4HB0p87Sv/2dMLZe9sJ71g0fHzN/kvKDAkhhGgac+72nRBCiNYlQUkIIUTTkKAkhBCiaUhQEkII0TQkKAkhhGgaEpSEEEI0DQlKQkxBs9WhE6JVSFASLU8p9R6l1N9P+Pv7lFJvVUq9XSl1lwr7/Lx7wr//WCl1d6mXzmsm/HxUKfVRpdT9NFFJHCFaiQQlMR98FXgpQKm6+AsJqzyfTNhe4GzgPKXUpaXXv1JrfR6wAXiLUqqn9PMsYT+gs7TWf5zF9Rdi3rAavQJCzDSt9Q6l1GGl1DmE7QbuJay8fXXpzxA20TsZ+ANhIHpO6ecrSz8/DPiERTqFEDNEgpKYL75MWCNvCeGV05XA+7XWX5j4IqXU5cBTgYu01jml1O8Jm9EBFLTW/iytrxDzkty+E/PFDYSdhc8nrAZ/M/DKUs8clFLLlVKLgE5goBSQ1hO28BZCzBK5UhLzgtbaUUr9DhgsXe38Sil1KnB72KmAUcLK6r8EXqeU2gxsAZqmD5UQ84FUCRfzQmmCwz3A87TWWxu9PkKIycntO9HylFKnEfbMukUCkhDNTa6UhBBCNA25UhJCCNE0JCgJIYRoGhKUhBBCNA0JSkIIIZqGBCUhhBBN4/8DEF2ff/ep1rgAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "df['log_words'] = np.log(df['num_words'])\n", "import seaborn as sns\n", "sns.jointplot(data=df,x='year', y='log_words',kind='hex')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Build a frequency distribution over words with `Counter`." ] }, { "cell_type": "code", "execution_count": 304, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:37:22.448696Z", "start_time": "2022-02-25T09:37:22.317909Z" } }, "outputs": [ { "data": { "text/plain": [ "[('the', 32485),\n", " ('of', 16429),\n", " ('to', 12675),\n", " ('a', 9144),\n", " ('that', 8737),\n", " ('in', 8484),\n", " ('and', 7815),\n", " ('at', 5172),\n", " ('for', 4569),\n", " ('is', 4115),\n", " ('not', 3658),\n", " ('l.', 3297),\n", " ('ed.', 3273),\n", " ('as', 3108),\n", " ('or', 2956),\n", " ('s.', 2904),\n", " ('ct.', 2805),\n", " ('§', 2752),\n", " ('court', 2667),\n", " ('on', 2666)]" ] }, "execution_count": 304, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from collections import Counter\n", "freqs = Counter()\n", "for i, row in df.iterrows():\n", " freqs.update(row['opinion_text'].lower().split())\n", " if i > 100:\n", " break\n", "freqs.most_common()[:20] # can use most frequent words as style/function words" ] }, { "cell_type": "markdown", "metadata": { "toc-hr-collapsed": false }, "source": [ "# Dictionary / Matching Methods" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Sentiment Analysis" ] }, { "cell_type": "code", "execution_count": 261, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:49:07.616803Z", "start_time": "2022-02-25T08:49:07.610484Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3.2.2\n" ] } ], "source": [ "import spacy\n", "from spacytextblob.spacytextblob import SpacyTextBlob\n", "print (spacy.__version__)" ] }, { "cell_type": "code", "execution_count": 262, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:49:09.741266Z", "start_time": "2022-02-25T08:49:08.751772Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "polarity 0.05404135338345859\n", "subjectivity 0.5065358709273183\n" ] } ], "source": [ "# Dictionary-Based Sentiment Analysis\n", "# nltk.download('vader_lexicon')\n", "\n", "# textblob sentiment analysis: https://github.com/sloria/TextBlob\n", "# pip install spacytextblob\n", "\n", "import spacy\n", "from spacytextblob.spacytextblob import SpacyTextBlob\n", "\n", "\n", "nlp = spacy.load('en_core_web_sm')\n", "# spacy_text_blob = SpacyTextBlob()\n", "nlp.add_pipe('spacytextblob')\n", "doc = nlp(df.iloc[0][\"opinion_text\"])\n", "#from nltk.sentiment.vader import SentimentIntensityAnalyzer\n", "#sid = SentimentIntensityAnalyzer()\n", "#polarity = sid.polarity_scores(text)\n", "print(\"polarity\", doc._.polarity) # sentimentintensityanalayzer nltk: {'neg': 0.134, 'neu': 0.785, 'pos': 0.081, 'compound': -0.9999}\n", "print (\"subjectivity\", doc._.subjectivity)" ] }, { "cell_type": "code", "execution_count": 263, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:50:48.585836Z", "start_time": "2022-02-25T08:49:12.565542Z" } }, "outputs": [], "source": [ "# sample 10% of the dataset\n", "dfs = df.sample(frac=.1) \n", "# apply compound sentiment score to data-frame\n", "def get_sentiment(snippet):\n", " #return sid.polarity_scores(snippet)['compound']\n", " return nlp(snippet)._.polarity\n", "dfs['sentiment'] = dfs['opinion_text'].apply(get_sentiment)" ] }, { "cell_type": "code", "execution_count": 264, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:50:48.590377Z", "start_time": "2022-02-25T08:50:48.586905Z" } }, "outputs": [ { "data": { "text/plain": [ "[' \\n\\n This case presents the question whether uses of patented inventions in preclinical research, the',\n", " 't. \\n\\n This case requires us to decide whether the use of race as a factor in student admissions by t',\n", " ' \\n\\n The State of Washington prohibits labor unions from using the agency-shop fees of a nonmember f',\n", " ' \\n\\n The question is whether the provision of the Fair Labor Standards Act of 1938 (FLSA or Act), tha',\n", " \"t. \\n\\n We granted certiorari to resolve whether the Nevada Supreme Court's refusal to extend full fai\"]" ] }, "execution_count": 264, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dfs.sort_values('sentiment',inplace=True)\n", "# print beginning of most positive documents\n", "[x[50:150] for x in dfs[-5:]['opinion_text']]" ] }, { "cell_type": "code", "execution_count": 265, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:50:48.593796Z", "start_time": "2022-02-25T08:50:48.591101Z" } }, "outputs": [ { "data": { "text/plain": [ "['\\n\\nThe False Claims Act (FCA) imposes civil liability on any person who knowingly uses a \"false recor',\n", " ' \\n\\nWhen an alien is found ineligible to remain in the United States, the process for selecting the c',\n", " 'he Court. \\n\\n Petitioner Josue Leocal, a Haitian citizen who is a lawful permanent resident of the Un',\n", " \" \\n\\n Respondent Arturo Recuenco was convicted of assault in the second degree based on the jury's fin\",\n", " 'he Court. \\n\\n InTotten v. United States,92 U.S. 105, 23 L. Ed. 605 (1876), we held that public policy']" ] }, "execution_count": 265, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# print beginning of most negative documents\n", "[x[50:150] for x in dfs[:5]['opinion_text']]" ] }, { "cell_type": "code", "execution_count": 266, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:51:03.211610Z", "start_time": "2022-02-25T08:50:58.841088Z" } }, "outputs": [ { "data": { "text/plain": [ "[' Court. \\n\\n Under the Internal Revenue Code, individuals may subtract from their adjusted gross inco',\n", " 't and delivered an opinion, in which Justice Souter, Justice Ginsburg, and Justice Breyer join. \\n\\n T',\n", " '.\\n\\nA citizen of Hawaii comes before us claiming that an explicit, race-based voting qualification ha',\n", " 't and delivered the opinion of the Court with respect to Parts I, II, III, and VI, an opinion with r',\n", " 't. \\n\\n For private actions brought under 42 U.S.C. § 1983 and other specified measures designed to s']" ] }, "execution_count": 266, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n", "sid = SentimentIntensityAnalyzer()\n", "# sample 20% of the dataset\n", "dfs = df.sample(frac=.1) \n", "\n", "# apply compound sentiment score to data-frame\n", "def get_sentiment(snippet):\n", " return sid.polarity_scores(snippet)['compound']\n", "dfs['sentiment_vader'] = dfs['opinion_text'].apply(get_sentiment)\n", "dfs.sort_values('sentiment_vader',inplace=True)\n", "# print beginning of most positive documents\n", "[x[50:150] for x in dfs[-5:]['opinion_text']]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Sentiment Analysis with Huggingface " ] }, { "cell_type": "code", "execution_count": 206, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:06:44.820739Z", "start_time": "2022-02-25T08:06:33.006201Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "No model was supplied, defaulted to distilbert-base-uncased-finetuned-sst-2-english (https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english)\n" ] } ], "source": [ "from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer\n", "pipe = pipeline(\"sentiment-analysis\")" ] }, { "cell_type": "code", "execution_count": 225, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:12:48.077724Z", "start_time": "2022-02-25T08:12:33.578170Z" } }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4a535366d00343c9b7130ea1443f84d1", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/768 [00:00" ] }, "execution_count": 144, "metadata": {}, "output_type": "execute_result" } ], "source": [ "re.search(pattern1,'The White House tried to calm uncertainty in the markets.')" ] }, { "cell_type": "code", "execution_count": 145, "metadata": { "ExecuteTime": { "end_time": "2022-02-24T19:57:57.465095Z", "start_time": "2022-02-24T19:57:57.455425Z" } }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 145, "metadata": {}, "output_type": "execute_result" } ], "source": [ "re.search(pattern2,'The Congress tried to calm uncertainty in the economy.')" ] }, { "cell_type": "code", "execution_count": 147, "metadata": { "ExecuteTime": { "end_time": "2022-02-24T19:58:25.501487Z", "start_time": "2022-02-24T19:58:25.495322Z" } }, "outputs": [], "source": [ "re.search(pattern3,'The Congress tried to calm uncertainty in the markets.')" ] }, { "cell_type": "code", "execution_count": 148, "metadata": { "ExecuteTime": { "end_time": "2022-02-24T19:58:25.891850Z", "start_time": "2022-02-24T19:58:25.885051Z" } }, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 148, "metadata": {}, "output_type": "execute_result" } ], "source": [ "re.search(pattern3,'The Congress tried to calm uncertainty in the markets.', re.IGNORECASE)" ] }, { "cell_type": "code", "execution_count": 149, "metadata": { "ExecuteTime": { "end_time": "2022-02-24T19:58:27.988977Z", "start_time": "2022-02-24T19:58:27.983865Z" } }, "outputs": [], "source": [ "def indicates_uncertainty(doc):\n", " m1 = re.search(pattern1, doc, re.IGNORECASE)\n", " m2 = re.search(pattern2, doc, re.IGNORECASE)\n", " m3 = re.search(pattern3, doc, re.IGNORECASE)\n", " if m1 and m2 and m3:\n", " return True\n", " else:\n", " return False" ] }, { "cell_type": "code", "execution_count": 150, "metadata": { "ExecuteTime": { "end_time": "2022-02-24T19:58:31.927254Z", "start_time": "2022-02-24T19:58:31.915697Z" } }, "outputs": [ { "data": { "text/plain": [ "True" ] }, "execution_count": 150, "metadata": {}, "output_type": "execute_result" } ], "source": [ "indicates_uncertainty('The White House tried to calm uncertainty in the economy.')" ] }, { "cell_type": "code", "execution_count": 151, "metadata": { "ExecuteTime": { "end_time": "2022-02-24T19:59:06.550986Z", "start_time": "2022-02-24T19:59:06.537632Z" } }, "outputs": [ { "data": { "text/plain": [ "False" ] }, "execution_count": 151, "metadata": {}, "output_type": "execute_result" } ], "source": [ "indicates_uncertainty('The White House tried to calm uncertainty in the markets.')" ] }, { "cell_type": "code", "execution_count": 305, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:55:15.135040Z", "start_time": "2022-02-25T09:55:13.319551Z" } }, "outputs": [], "source": [ "df['uncertainty'] = df['opinion_text'].apply(indicates_uncertainty)" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.057291666666666664" ] }, "execution_count": 47, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.uncertainty.mean()" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEGCAYAAABrQF4qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA0W0lEQVR4nO3deXxU5b348c83k5WsZN+AsEOABCIiCFQqFVkFqbbazWp/v7a31ba/rtrfbeu1+2YXaxe7adt7r3pdkCWIuCcoCsoS9oQ1ZJ+EhISQdZ7fHzPxF2Igk2Rmzszk+3698nJyzplzvg8m3zn5Ps95HjHGoJRSKniFWB2AUkop79JEr5RSQU4TvVJKBTlN9EopFeQ00SulVJALtTqAvpKTk01OTo7VYSilVEB555137MaYlP72+V2iz8nJYffu3VaHoZRSAUVETl9un5ZulFIqyGmiV0qpIKeJXimlgpwmeqWUCnKa6JVSKshpoldKqSCniV4ppYKcJnqlVL8cDsP/7C6n5nyb1aGoYdJEr5TqV1GZnW88tZ+VvynitWN1VoejhkETvVKqX4X7q4iJCCU5JoI7/vY2P992hK5uh9VhqSHQRK+Uep/ObgfbDlWzdHoqG764kNuuHsPDrxznY39+i+omLeUEGk30Sqn32XminsbWTlbOyiAq3MZPPpzHrz86mwOVTaz8bRGvHq21OkQ1CJrolVLvU1hSTXS4jeum/P/JENfNyWLTPYtIjY3g03/fxU+f11JOoNBEr5S6RFe3g20Hq7l+ehqRYbZL9k1MiWHDFxdy+7yx/OHV49z+551UNV20KFLlLk30SqlLvH2ygYYLHaycmd7v/sgwGz9eP4vf3DabQ5XnWfmbIl7RUo5f00SvlLrElpIqosJsLJmaesXj1s52lnLS46O4U0s5fk0TvVLqPd0O4yzbTEslKtw24PETUmJ49gvX8rFrnKWc2x7ZSWWjlnL8jSZ6pdR7dp1qwN7SwcpZGW6/JzLMxo9udpZyDledZ9Vvi3jliJZy/IkmeqXUewpLqogMC2HJ1H6XHr2iS0o5j+7ix1sP06mlHL+giV4pBTjnttl6oJolU1KJjhjactI9pZyPXzOWP712Qks5fkITvVIKgN2nz1HX3M7KPPfLNv2JDLPxw5tn8dDtczha3czK3xbx0uEaD0WphsKtRC8iy0XkqIiUici9/ez/gIi8KyJdInJLn313iEip6+sOTwWulPKswpIqwkNDuH7alUfbuGtNfiab7llEZnwUn3lsNz8u1FKOVQZM9CJiAx4GVgC5wO0iktvnsDPAp4H/6vPeROB7wDXAPOB7IjJ6+GErpTzJWbapYsmUFGKGWLbpz/jkaJ75wrV8cv44/vT6CT76pzep0FKOz7lzRz8PKDPGnDDGdACPA2t7H2CMOWWM2Q/0/bi+EdhujGkwxpwDtgPLPRC3UsqD9pSfo+Z8+6BG27grMszG99fN5Hcfm8OxmhZW/qaIFw9pKceX3En0WUB5r+/Pura5w633ishnRWS3iOyuq9N5r5XytS37qwm3hbB0umfKNv1ZnZfJ5nsWkT06iv/1j938cMshLeX4iF90xhpjHjHGzDXGzE1JGfywLqXU0PWUbT4wJZnYyDCvXisnOZqn/+1aPrVgHH8uOslH/vQmZ8+1evWayr1EXwGM6fV9tmubO4bzXqWUD+w920hVU5tXyjb9iQyz8cDamfz+4wWU1bSw6rfFWsrxMncS/S5gsoiMF5Fw4DZgo5vn3wYsE5HRrk7YZa5tSik/sbWkijCbsHR6mk+vu3JWBpu/tIgxic5Szg82H6KjS0s53jBgojfGdAF340zQh4EnjTEHReQBEbkJQESuFpGzwK3An0TkoOu9DcD3cX5Y7AIecG1TSvkBYwyFJdUsnpxCfJR3yzb9GZfkLOXcsWAcfynWUo63iDHG6hguMXfuXLN7926rw1BqRNhX3sjah3fw81vyuHXumIHf4EWFJVV866n9iMAvbs1n2Yz+p0lW/RORd4wxc/vb5xedsUopaxQeqCI0RFiWa31S7SnljEuK5rP/fIeHXymzOqSgoYleqRHKWbapYuGkZOJH+b5s059xSdE89W8L+ND0VH7/Shltnd1WhxQUNNErNUIdrDxPecNFVs6y/m6+t4hQG3cuHM+Fjm6d7thDNNErNUJtKanC5idlm77mT0giOSaCTfsrrQ4lKGiiV2oEMsawtaSKaycmMTo63Opw3scWIqyclc7LR2q50N5ldTgBTxO9UiPQoarznKpv9dlDUkOxOi+Ttk4HL+oUx8OmiV6pEWhrSbWrbOPbh6QGY+640aTHRbJpX5XVoQQ8TfRKjTA9o23mT0gkKSbC6nAuKyREWJWXwevH6mi62Gl1OAFNE71SI8zRmmZO2C+wYqb/lm16rMnPpKPbwQsHq60OJaBpoldqhCncX0WIwI0B8ORpfnY8YxKj2LxfyzfDoYleqRGm8EA188YnkhLrv2WbHiLCqlmZ7Ciz03Chw+pwApYmeqVGkNKaZspqW1jlx6Nt+lqTn0GXw/D8AS3fDJUmeqVGkC0lVUiAlG165GbEMSE5ms368NSQaaJXagTZWlLN1eMSSY2LtDoUt4kIq/Mz2XmintrmNqvDCUia6JUaIcpqWzha0+x3c9u4Y01eBg7j/KBSg6eJXqkRYmuJc+TK8gAYVtnX5LRYpqbFavlmiDTRKzVCbCmpcj5tGh84ZZve1uRnsOvUOSobL1odSsDRRK/UCHCiroUj1c2sCKDRNn2tzssEnCtRqcHRRK/UCLDVNTRxxczAq8/3yEmOZmZWHJv04alB00Sv1AhQWFLFnLEJZCZEWR3KsKzJy2RfeSNn6nUB8cHQRK9UkDtdf4GDlecD6iGpy1mV52zD5hLtlB0MTfRKBblC15DE5QFctumRPXoUBWMTdOriQdJEr1SQKyypIn9MAtmjR1kdikeszsvkcNV5jte1WB1KwNBEr1QQK29opaSiiZVBcDffY1VeBiKwWe/q3aaJXqkgtvWAMxn685KBg5UWF8m8nEQ27a/EGGN1OAFBE71SQWxLSTWzsuIZkxgcZZseq/Mz35vSQQ1ME71SQersuVb2lTeyIgDnthnIipnp2EKETft09I07NNErFaR65m9fGYBz2wwkOSaCaycmsXl/lZZv3KCJXqkgVVhSRW5GHDnJ0VaH4hWr8zI4Xd/KgYrzVofi9zTRKxWEKhsv8u6ZxvceMApGN85IJ8wmbNIZLQekiV6pIPR8EMxtM5CEUeEsnpzCFi3fDEgTvVJBqLCkimnpsUxIibE6FK9anZdBheuvF3V5muiVCjLVTW3sPn0uqMbOX84NuWmEh4bo6JsBuJXoRWS5iBwVkTIRubef/REi8oRr/1sikuPaHiYij4lIiYgcFpH7PBy/UqqP54PwIanLiY0M44NTUygsqaLboeWbyxkw0YuIDXgYWAHkAreLSG6fwz4DnDPGTAJ+BfzUtf1WIMIYMwu4Cvhcz4eAUso7Cg9UMyUthkmpwV226bEmP5Pa5nbePtlgdSh+y507+nlAmTHmhDGmA3gcWNvnmLXAY67XTwFLRUQAA0SLSCgQBXQAOhZKKS+pbW5j16kGVgTh2PnLuX5aKlFhNl1P9grcSfRZQHmv78+6tvV7jDGmC2gCknAm/QtAFXAG+IUx5n0fuyLyWRHZLSK76+rqBt0IpZTTtgPVGENQD6vsa1R4KEunp7L1QDVd3Q6rw/FL3u6MnQd0A5nAeOBrIjKh70HGmEeMMXONMXNTUlK8HJJSwauwpJqJKdFMHiFlmx5r8jNpuNDBG8frrQ7FL7mT6CuAMb2+z3Zt6/cYV5kmHqgHPgY8b4zpNMbUAjuAucMNWin1fvaWdt46Wc+qWRk4K6cjx3VTUoiNCA3o8o3Di53J7iT6XcBkERkvIuHAbcDGPsdsBO5wvb4FeNk4n2A4A1wPICLRwHzgiCcCV0pdatvBahwGVoyA0TZ9RYbZuGFGGs8fqKajKzDLN5//1zv8fJt30uOAid5Vc78b2AYcBp40xhwUkQdE5CbXYX8FkkSkDPgq0DME82EgRkQO4vzA+LsxZr+nG6GUcj4kNSE5mmnpsVaHYok1eZmcb+uiqDTw+vm2H6rhhUM1xEeFeeX8oe4cZIwpBAr7bPtur9dtOIdS9n1fS3/blVKeVd/Szs4TDXz+ugkjrmzTY+GkZOKjwti8v4ql09OsDsdtFzu6uX/jQaakxXDnwvFeuYY+GatUEHjhUA3dDjMiHpK6nPDQEFbMTOeFg9W0dXZbHY7bHn6ljIrGizywdiZhNu+kZE30SgWBwpIqxiWNIjcjzupQLLU6L5MLHd28erTW6lDccqKuhUdeP8HNc7KYPyHJa9fRRK9UgDvnGla4cgSOtulr/oREkmPC2bTf/xcON8bwvY0HiQgN4b6V07x6LU30SgW47T1lmxH0NOzlhNpCWDEzg5cO13ChvcvqcK6osKSaolI7X1s2hdTYSK9eSxO9UgFuS0kVYxKjmJk1sss2PVbnZdDW6eClI/5bvmlp7+KBzQfJzYjjE/PHef16muiVCmBNrZ3sKLOzcqaWbXpcnZNIWlyEX09d/JsXj1Fzvp3vr5tJqJc6YHvTRK9UANt+uIYuhxmRD0ldTkiIsGpWJq8dreN8W6fV4bzP0epm/rbjFB+dO4arxo32yTU10SsVwApLqshKiCI/O97qUPzK6vwMOrodbD9YY3UolzDG8J3nDhAbGcq3Vni3A7Y3TfRKBajzbZ0UldaxYma6lm36mDMmgayEKL9bOPzZPRW8fbKBb944jcTocJ9dVxO9UgHqxUM1dHYbVo6gKYndJSKszs+guNTOuQsdVocDQNPFTn5UeJj8MQncdvWYgd/gQZrolQpQhSXVZMRHMjs7wepQ/NKavEy6HIZtB6utDgWAB184Sv2FDn6wdiYhIb79C0wTvVIBqLmtk9dL61gxM8PnSSNQzMiMY3xytF+Ubw5UNPHPnaf55PxxzLKgP0UTvVIB6OUjtXR0OVg5K93qUPyWiLA6L4M3j9dT19xuWRwOh+HfNxwgMTqcry2bakkMmuiVCkBb9leRFhdBwVjfDM8LVGvyM3EY2HrAuikRntxdzt7yRu5bMd1r0xAPRBO9UgGmpb2LV49p2cYdU9JimZIWw+Z91iT6cxc6+OnzR5iXk8j6gr5LbfuOJnqlAsz/L9voaBt3rMnLZNfpBqqaLvr82j/bdoTzbV08sG6GpUNgNdErFWC2llSREhvhs6cqA93q/EyMcZa7fGnPmXM8vqucO6/NYVq6tfMQaaJXAaW0ppnvbz7kN2Ojfa21o4tXjtayfEY6Ni3buGV8cjQzMuPY7MNE3+3qgE2NjeArN0zx2XUvRxO9Cih/LjrBX4tPsvqhYvafbbQ6HJ975UgdbZ1athmsNfmZ7C1vpLyh1SfX+9fO0xysPM+/r8olJsKtFVu9ShO9ChjGGIpL7eRlx2OM4ZY/vMnjb5+xOiyfKjxQRXJMOPPGJ1odSkBZ5fpg9MVdfV1zO7944SiLJiWz2k+eWtZErwLGSfsFKpvauHXuGDZ/aTHXTEjk3mdK+OZT+wJqjdChutjRzcuHa7lRyzaDNiZxFLPHJLDZBw9P/bjwMG2d3fzHWms7YHvTRK8CRnGZHYDFk5JJjA7n0Tvncc/1k3hy91k+/Ic3fPZnuVVeO1bLxc5uLdsM0Zr8TA5WnudEXYvXrvHWiXqe2VPB/148gYkpMV67zmBpolcBo6jUTvboKMYljQLAFiJ8bdlU/vKpuZxpaGX1Q8W84serCg3XlpJqEqPDuUbLNkOyalYGIt4r33R2O/jOcwfISoji7usneeUaQ6WJXgWErm4HO4/Xs3hy8vv+HP5Qbhqb71lEZkIUdz22iwe3H6PbYSyK1DvaOrt5+XANN85I88mKRMEoPT6Sq3MSvVa+eXTHKY7VtPDdNbmMCre+A7Y3/YlRAWHf2Uaa27tYNCml3/3jkqJ55t+uZf2cbH77Uil3PbqLxtbgGYL52rE6LnRo2Wa41uRlcKymhaPVzR49b3VTG79+8RgfnJrCstw0j57bEzTRq4BQVGpHBK6dmHTZY6LCbfzi1jx+ePNM3jxez+qHijlQ0eTDKL1na0kVCaPCmD/h8u1XA1s+M4MQweN39d/fcohOh+H+m/ynA7Y3TfQqIOwoszMrK57RA6zKIyJ8/JpxPPn5BTgchvV/eIMndgX2EMy2zm5ePFzLjbnphGnZZlhSYiO4dmIym/ZVYoxnynvFpXa27K/ii0smMS4p2iPn9DT9qVF+r6W9iz1nGlk4Kdnt98wek8DmLy1mXk4i33q6hG89tT9gh2AWldppae9ihU5J7BGr8zI4Vd/Kwcrzwz5Xe1c3333uAOOSRvG56yZ4IDrv0ESv/N7O4/V0OQyLB5HoARKjw3nsrnl88YMTeWJ3Obf8MbCGYB6raea+Z/Zzz3+/S1J0ONdOHFz7Vf+Wz0wnNEQ8siDJX4pOcsJ+gf+4aQaRYTYPROcdmuiV3ysusxMZFsJVOYOfxMsWInzjxmn8+VNzOV3fyprfFfPqUf8dgulwGF4+UsMn//oWy371Os+8W8G62Vn8z+cXEB6qv66ekDAqnMWTk9m8r2pY5ZvyhlYeermU5TPSWTI11YMRep7+5Ci/V1Rax7zxSUSEDv2O6YbcNDbdvYj0uEjufHQXv37xGA4/GoJ5ob2Lx944xdIHX+OuR3dzrKaZb9w4lTfvW8pPPpzHBD96+CYYrM7LpKLxInvKG4d8jgc2H0IQvrsm13OBeYl/DfZUqo+qposcr7vAbVePHfa5cpKjefYLC/m/z5bw6xdL2VveyK8/OpuEUVfu4PWm8oZWHnvjFE/sLqe5rYvZYxL4zW2zWTkrQzteveiGGWmEPxvCpn2VQ1ql66XDNWw/VMO3lk8jMyHKCxF6liZ65deKSp3THiya7Jn6dFS4jV9+JJ8540bzwKaDrH6omD9+4ipmZvluwWZjDG+dbODvO06y/VANISKsmJXBnQtzdGlAH4mLDGPJlBQKS6r4zqrcQa3U1dbZzf2bDjIpNYbPLBrvxSg9x61bBhFZLiJHRaRMRO7tZ3+EiDzh2v+WiOT02pcnIm+KyEERKRGRSA/Gr4LcjjI7yTHhTEuP9dg5RYRPzh/Hk59bQLdrCOaTu8o9dv7Laevs5n92l7Pqt8Xc9shO3jrZwOevm0jRtz7IQ7fP0STvY2vyM6k5386uUw2Det/vXymjvOEiD6ydETD9JgPe0YuIDXgYuAE4C+wSkY3GmEO9DvsMcM4YM0lEbgN+CnxUREKBfwGfNMbsE5EkoNPjrVBByeEw7Cizs3DS+6c98IQ5Y0ez+Z5FfOnxPXzz6f3sKT/H99Z4fvREbXMb/9p5hv966zT2lg6mpMXw4/WzWDc7i6hw/x2pEeyWTk8lKszGpv2VXOPmg2gn7Rf442snuCk/M6BGQblTupkHlBljTgCIyOPAWqB3ol8L3O96/RTwO3H+Zi4D9htj9gEYY+o9FLcaAY5UN2Nv6WDRIIdVDkZSTAT/uOsafvnCUX7/6nEOVJznD58oIHv0qGGfu+RsE3/fcZJN+yvp7DYsnZbKnQvHs3BSkl8+PTnSjAoP5frpqWwtqeb+NTMGnEPIGMP3Nh4kPDSEf1813UdReoY7iT4L6P137VngmssdY4zpEpEmIAmYAhgR2QakAI8bY37W9wIi8lngswBjxw6/000Fh+KyOgAWT+5/fhtPsYUI31w+jdljEvjak/tY/VAxv7ltDtdNGfx1u7odbD9Uw992nGTXqXOMCrfx8WvGcce1OYxP9s+nJkeyNXmZbNlfxc4TDQP2Az1/oJrXj9XxndW5pMYFVgXa252xocAi4GqgFXhJRN4xxrzU+yBjzCPAIwBz5871nzFvylJFpXYmpcaQHu+bX6plM9LZdE8sn//XO3z672/zfz40hbs/OMmtjrqm1k6e2H2Gx944TUXjRbJHR/Hvq6bzkavHEBcZ5oPo1VAsmZpCTEQom/ZVXjHRX2jv4oHNh5iWHssdC8b5MELPcCfRVwBjen2f7drW3zFnXXX5eKAe593/68YYO4CIFAIFwEsodQVtnd28fbKB2+f59i+8niGY3362hAe3H2NveSO/+shs4kf1n6yP17Xw6I5TPPXOWS52dnPN+ES+uyaXD01P01WgAkBkmI0bctN4/mA1318387Kdq799uZSqpjYeun1OQE4T7U7Eu4DJIjJeRMKB24CNfY7ZCNzhen0L8LJxPnK2DZglIqNcHwDXcWltX6l+vXP6HO1dDhZ7aFjlYESF23jwI/l8f+0MikrrWP27oktmwTTG8NqxOj7997dZ+svXeGJXOavyMtjypUU88bkFutRfgFmTn0HTxc73SoV9ldY089eik9xyVTZzcwJz0ZcB7+hdNfe7cSZtG/A3Y8xBEXkA2G2M2Qj8FfiniJQBDTg/DDDGnBORB3F+WBig0BizxUttUUGkuMxOaIi4PRrC00SETy7IYUZWPF/8z3f58B/e4P6bZtDtMDz6xinKaltIiY3g/3xoCh+7ZiwpsRGWxKmGb9GkFOKjwti8r4rrp106l7wxhu88d4DoiFDuWzHNogiHz60avTGmECjss+27vV63Abde5r3/wjnEUim3FZfamTM2gZgIa5/pKxg7mk33LOJL/72H+54pAWBWVjwPfiSfVXkZw5qWQfmH8NAQls9IZ0tJFW2d3ZcMr924r5KdJxr4wbqZJMUE7oe5Phmr/M65Cx0cqGziK0unWB0KAMkxEfzjrnlsKakiMyGKueNG6/DIILM6P4Mndpfz6tE6ls90Tgd9vq2TH2w5TF52vM/7ijwt8HoVVNDbcdyOMZ6b9sATQm0hrJ2dxdU5iZrkg9CCCUkkRYdfsvLUr7Yfw97Szg/WzQz4PhdN9MrvFJfaiY0MJT/bd/PPqJEt1BbCilnpvHS4ltaOLg5WNvHYG6f42Lyx5GUnWB3esGmiV37FGENRqZ0FE5ICchibClyr8zK56Fq28TsbDpAwKpxv3DjV6rA8Qn+TlF85Vd9KReNFS4ZVqpHt6pxE0uIieGDTQd4908i9K6ZZOoW1J2miD2BNFzv5S9EJurodVofiMcWlzrHMi7w87YFSfdlChJWzMrC3dHDVuNHcUpBtdUgeo4k+gD32xil+sOUw2w/VWB2KxxSX2clKiCInafiTiik1WLddPZbxydH8YN3MQc1R7+800QcoYwwb9jpnonj63b4zUgSmrm4HbxyvZ5GXpiVWaiBT02N55etLmJ4RZ3UoHqWJPkAdqDjPiboLZCVE8erRWupb2q0Oadj2VzTR3NblV8MqlQoGmugD1Ia9FYTbQnjwI/l0OQyb9lUO/CY/V1xqRwQWenH+eaVGIk30AajbldiXTE3hmglJ5GbE8eyewC/fFJfamZEZR2J0cIx0UMpfaKIPQG8er6e2uZ11c7IAWF+Qxb6zTZTVNlsc2dC1tHfx7plzLJqko22U8jRN9AHoub0VxEaEcv20VABump1JiMAzAdwp+9aJerocRsfPK+UFmugDTFtnN88fqGb5zPT3ZtlLjY3kA1NSeHZPBQ5HYC7QVVxmJyI0hKvGjbY6FKWCjib6APPykVqa27veK9v0WF+QTVVTGztPBOb668WlduaNT7xkilillGdoog8wG/ZUkBobwfw+C3Isy00jNiI0IMfUVze1UVrbwiIdbaOUV2iiDyBNrZ28erSONfmZ75s2NTLMxspZGWw9UEVrR5dFEQ5NcZkd8K9piZUKJproA0jhgSo6uh2sm53V7/71BVm0dnSz7WC1jyMbnuLSOpKiw5meHlxPIyrlLzTRB5ANeyqYkBLNzKz+E+LVOYlkj44KqNE3xhiKy+pZOCk5qOYWUcqfaKIPEJWNF3nrZAPrZmdddh6YkBDh5jlZ7CizU93U5uMIh+ZIdTP2lnYt2yjlRZroA8RG1xQHa2dnXvG4m+dk4TDOsfaBYEdPfV47YpXyGk30AWLDngrmjE1gXFL0FY+bkBLDnLEJPP3uWYzx/zH1RaV2JqREk5kQZXUoSgUtTfQB4Gh1M0eqmy/bCdvX+oJsjtW0cLDyvJcjG572rm7eOlnPYr2bV8qrNNEHgOf2VmALEVblZbh1/Jq8DMJs4vedsu+cPkdbp0NXk1LKyzTR+zmHw/Dc3koWT04mOSbCrfckjApn6bQ0Nu6r8OtlBotL7dhChPkTEq0ORamgponez71z5hwVjRcH7ITta31BFvaWDopK7V6KbPiKy+zMGZNAbGSY1aEoFdQ00fu5DXsqiAqzsSw3fVDvWzI1ldGjwnj63bNeimx4zl3ooKSiSYdVKuUDmuj9WEeXgy0lVdyQm0Z0ROig3hseGsKa/ExeOFRD08VOL0U4dG+eqMcYdFpipXxAE70fe/1YHY2tnaybM7iyTY/1Bdl0dDnYWlLl4ciGr6jUTkxEKHnZCVaHolTQ00TvxzbsrSAxOpzFQxyVkp8dz4SUaL8cfVNcVsf8CUmE2fRHUClv098yP9XS3sWLh2tYNStjyMlQRPhwQTZvn2rgTH2rhyMcutP1FyhvuKhlG6V8RBO9n9p2oJq2TseQyzY9ehYo8afFw3tGAmlHrFK+4VaiF5HlInJURMpE5N5+9keIyBOu/W+JSE6f/WNFpEVEvu6huIPehr0VjEmMomDs8JbWy0qIYsGEJJ7Z4z9TIhSX2smMj2RC8pWnc1BKecaAiV5EbMDDwAogF7hdRHL7HPYZ4JwxZhLwK+CnffY/CGwdfrgjQ11zOzvK7KzNv/xMlYOxviCL0/WtvHumcfjBDVO3w/DGcTuLJid7pG1KqYG5c0c/DygzxpwwxnQAjwNr+xyzFnjM9fopYKm4fotFZB1wEjjokYhHgM37K3GYgWeqdNeKWRlEhoXwjB+Mqd9/tpHzbV067YFSPuROos8Cynt9f9a1rd9jjDFdQBOQJCIxwLeA/xh+qAM7Un3eb8oTw7FhbyW5GXFMTov1yPliIkK5cUY6m/ZV0t7V7ZFzDlXPtMTXTkwa4EillKd4uzP2fuBXxpiWKx0kIp8Vkd0isruurm5IF9pRZmf5r4vYeiCwltHr66T9AvvKG4fdCdvX+oJszrd18fLhWo+ed7CKSu3kZsS5PW+PUmr43En0FcCYXt9nu7b1e4yIhALxQD1wDfAzETkFfAX4tojc3fcCxphHjDFzjTFzU1KG9if9/AlJTEmL4afPH6Gjy38n8hrIc3srEIGb8t2bkthdCycmkRobwdMWjqm/0N7Fu2fO6bBKpXzMnUS/C5gsIuNFJBy4DdjY55iNwB2u17cALxunxcaYHGNMDvBr4EfGmN95JvRL2UKE+1ZM53R9K//51mlvXMLrjHHOVDl/fBLp8ZEePXeoLYR1c7J49Wgt9S3tHj23u94+2UBnt9FhlUr52ICJ3lVzvxvYBhwGnjTGHBSRB0TkJtdhf8VZky8Dvgq8bwimLyyZmsK1E5P47Uulfjm/y0D2n23ipP2Cx8s2PdYXZNHlMGxyLUvoa0WldsJDQ7g6R6clVsqX3KrRG2MKjTFTjDETjTE/dG37rjFmo+t1mzHmVmPMJGPMPGPMiX7Ocb8x5heeDf9SIsK3V07nXGsnf3j1uDcv5RUb9lYQbgth+Uz3FhgZrGnpceRmxFn28FRxWR3zchKJDLNZcn2lRqqgezJ2ZlY8N8/J4m87TlLReNHqcNzW1e1g074qrp+WSnyU9+ZnX1+Qxb6zTZTVNnvtGv2pOd/GsZoWLdsoZYGgS/QAX1s2BYBfbjtqcSTue+N4PfaWdq+VbXrcNDsTW4jvlxnsGVa5SNeHVcrngjLRZ48exZ0Lc3h2bwUHKpqsDsctz+2tJDYylCVTU716ndTYSBZPTubZPRU4HL575qC41E5idDi5GXE+u6ZSyikoEz3AF5ZMIiEqjJ9sPeL3D1G1dXaz7WA1K2am+6R+vb4gm6qmNnaeqPf6tcA5mqi4zM61E5MICdFpD5TytaBN9PFRYdxz/WSKy+y8dmxoD2H5youHa2hp72LdbM+Onb+cZblpxEaE+mxM/bGaFmqb23X8vFIWCdpED/CJ+eMYlzSKHxceoduHZYrB2rCnkrS4CK6Z4JtpASLDbKyclcHWA1W0dnR5/XpFpc4PWp3fRilrBHWiDw8N4Zs3TuNoTTNPv2P9hF79aWzt4LVjtdyU7+wk9ZX1BVm0djhLRt5WXGZnQnI0WQlRXr+WUur9gjrRA6yclc6csQn8cvtRn9y9DtaWkio6uw1rfVS26XF1TiLZo6O8Pvqmvaubt0406LBKpSwU9Im+5yGqmvPt/LXopNXhvM9zeyqZlBrDjEzfjkYJCRHWz8liR5md6qY2r11nz5lGLnZ2s1CHVSplmaBP9OC8e12Wm8YfXztOXbM187z05+y5Vt4+1cC62ZmWLMJxc0E2DuOcSM1bikvt2EKEBTotsVKWGRGJHuBbK6bR1uXgNy8dszqU92x0zTnj67JNj/HJ0cwZm8DT73pvmcGiMjv52fHERXrvaV+l1JWNmEQ/MSWGj80by3+/XU5Z7RWnx/eZ5/ZUctW40YxJHGVZDOsLsjlW08LByvMeP3dTayclZxt1tI1SFhsxiR7gyx+aTFSYjZ8+f8TqUDhSfZ6jNc0eWy5wqNbkZRBm886UCG8ct+Mw6Ph5pSw2ohJ9ckwEn79uAtsP1fD2yQZLY9mwpxJbiLBqlndmqnRXwqhwlk5LY+O+Crq6PbtgS1GZnZiIUGaPSfDoeZVSgzOiEj3AZxZNID0ukh8WHrZsagSHw7BxbwUfmJxMkh8sqbe+IAt7SwdFpXaPnre41M78CYmE2Ubcj5lSfmXE/QZGhdv46rIp7CtvZPP+Kkti2HWqgcqmNtbNsaYTtq8lU1MZPSqMp9/13ENlZ+pbOdPQqsMqlfIDIy7RA3y4IJtp6bH8bNsR2ru6fX79DXsrGRVu44bcNJ9fuz/hoSHclJ/JC4dqPLYyV7FrWmKtzytlvRGZ6G0hwn0rp1PecJF/7Tzj02t3dDkoLKliWW4ao8JDfXrtK1lfkE1Hl4OtJZ75K6e4rI70uEgmpsR45HxKqaEbkYke4LopKSyenMxDL/t2fdlXj9bSdLGTtX5StumRlx3PhJRoj4y+6XYYdpTVs2hysiUPgimlLjViEz3AfSum03Sxk9+/Uuazaz63t5Kk6HAW+1ntWkT4cEE2b59q4Ex967DOdaCiiaaLnVq2UcpPjOhEn5sZx/o52fz9jVOcPTe85OaO5rZOXjxcw+q8DEL9cCRKT+fwcBcP76nPa0esUv7B/7KNj339xikI8AsfrC/7/IFq2rsc3GTRlAcDyUqIYsGEJJ7ZM7wpEYpK65ieEUeyHwwdVUppoicjPorPLBrPhr2VlJz17vqyG/dVMjZxFAVjE7x6neFYX5DF6fpW3j3TOKT3t3Z08c7pcyyapJOYKeUvRnyiB/j8kokkRofzIy8+RFXb3MaOMjtrLZqp0l0rZmUQGRbCM0McU//2yQY6u43Ob6OUH9FED8RFhvHlpZN580Q9rxyt9co1Nu2rwmGsm6nSXTERoSyfkc6mfZVDesaguNROuC2EeTmJXohOKTUUmuhdPnbNWMYnR/PjwiMen/MFnHO+z8yKY1Kq/48rX1+Qzfm2Ll4+PPgPveIyO3NzRhMVbvNCZEqpodBE7xJmC+Fby6dSWtvC/3h4fdkTdS3sP9vEOj+/m++xcFIyqbERPD3IMfW1zW0cqW7WZQOV8jOa6Hu5cUY6c8eN5sHtx7jQ7rn1ZTfsrUQE1uRbOyWxu2whwro5Wbx6tJb6FvdX5NrRM+3BJK3PK+VPNNH3IuKcGqGuuZ0/F53wyDmNMTy3t4JrJyaRFhfpkXP6wvqCLLochk2uVbDcUVRqJ2FUmM/Xv1VKXZkm+j6uGjeaFTPTeeT1E9Q2D3/R7L3ljZyub/X7Tti+pqXHkZsR5/bDU8YYikvtLJyYTEiI/44qUmok0kTfj28un0ZHl4NfbS8d9rme21tJeGgIy2emeyAy31pfkMW+s02U1TYPeGxZbQu1ze1an1fKD2mi78f45Gg+MX8cT+w641aSu5yubgeb91eydFpqQC6OfdPsTGwh7i0z2LNoySKd9kApv6OJ/jK+tHQy0eGh/GTr0NeXLS6zY2/pCLiyTY/U2Eg+MDmZZ/dU4HBc+UGy4jI7OUmjLF3oXCnVP7cSvYgsF5GjIlImIvf2sz9CRJ5w7X9LRHJc228QkXdEpMT13+s9HL/XJEaH828fnMiLh2vZeaJ+SOfYuLeSuMhQPjgtcEehrC/Ipqqp7Yr/Bh1dDnaeqNeyjVJ+asBELyI24GFgBZAL3C4iuX0O+wxwzhgzCfgV8FPXdjuwxhgzC7gD+KenAveFuxaOJzM+kh8VHh7wjravix3dbDtYzcpZGUSEBu7DQzfkphEbEXrFMfV7zpyjtaObRTqsUim/5M4d/TygzBhzwhjTATwOrO1zzFrgMdfrp4ClIiLGmD3GmJ7xeQeBKBEJmCkNI8NsfG3ZVPafbWLTfveHGQJsP1zDhY7ugC3b9IgMs7FyVgZbD1TR2tH/swXFZXZCBBZM1InMlPJH7iT6LKC81/dnXdv6PcYY0wU0AX1/6z8MvGuMed8TOCLyWRHZLSK76+rq3I3dJ26ek0VuRhw/33Z0UHO/PLengoz4SK4ZH/hzvqwvyKLV9RdKf4pK7eRlJxAfFXgdzkqNBD7pjBWRGTjLOZ/rb78x5hFjzFxjzNyUFP/68z8kRPj2yumcPXeRf7xx2q33NFzo4LVjddyUnxkUY8qvzkkke3RUv6Nvmi52sv9so64mpZQfcyfRVwBjen2f7drW7zEiEgrEA/Wu77OBZ4FPGWOODzdgKyyanMx1U1J46OVSGls7Bjx+S0kVXQ4T8GWbHiEhwvo5Wewos1PddOlDZG8er8dhdFilUv7MnUS/C5gsIuNFJBy4DdjY55iNODtbAW4BXjbGGBFJALYA9xpjdngoZkvct3IaLe1d/O7lgdeXfW5PBVPSYpieEeuDyHzj5oJsHMY5C2dvxWV1jAq3MWfsaIsiU0oNZMBE76q53w1sAw4DTxpjDorIAyJyk+uwvwJJIlIGfBXoGYJ5NzAJ+K6I7HV9pXq8FT4wLT2OW67K5h9vnqa84fLry5Y3tLL79DnWzs7y6wVGBmt8cjQFYxN4+t1LlxksLrUzf0IS4aH6SIZS/sqt305jTKExZooxZqIx5oeubd81xmx0vW4zxtxqjJlkjJlnjDnh2v4DY0y0MWZ2ry/vrOzhA1+9YSohIfCzK6wvu9E1CdhNATJT5WCsL8jmWE0LByvPA84PtVP1rVq2UcrP6W3YIKTHR/K/F09g075K9pU3vm+/MYYNeyqYO250UD4hujovg3BbyHudssU90xJrR6xSfk0T/SB97rqJJMeE88N+1pc9XNVMaW0La+cERydsXwmjwrl+Wiob91XQ1e2guNROWlxEQKyapdRIpol+kGIiQvnyh6bw9skGXuyz1N5zeysIDRFWzcqwKDrvW1+Qhb3FOXx0x3E7CyclB1VfhFLBSBP9ENx29RgmpETzk62H31tf1uEwbNxXyXVTUkiMDrc4Qu9ZMjWV0aPC+MnWIzS2dmrZRqkAoIl+CMJsIdy7fBrH6y7wxG7nQ8NvnWygqqktaMs2PcJDQ7gpP5PS2hbAub6sUsq/aaIfohty05iXk8ivtpfS0t7Fc3sriA63ccP0NKtD87r1BdkATEuPJTU2cJZHVGqk0kQ/RCLCt1dNx97Szu9eLqOwpIobZ6QTFR64M1W6Ky87niVTU7h17piBD1ZKWS7U6gAC2ewxCazOy+CPrzlndrhpdvCNne+PiPDonfOsDkMp5Sa9ox+mb944jTCbkBwTrg8OKaX8kt7RD9PYpFH8eH0ekWEhhNr0c1Mp5X800XvALVdlWx2CUkpdlt6CKqVUkNNEr5RSQU4TvVJKBTlN9EopFeQ00SulVJDTRK+UUkFOE71SSgU5TfRKKRXkpO8qSVYTkTrg9DBOkQzYPRROIBhp7QVt80ihbR6cccaYlP52+F2iHy4R2W2MmWt1HL4y0toL2uaRQtvsOVq6UUqpIKeJXimlglwwJvpHrA7Ax0Zae0HbPFJomz0k6Gr0SimlLhWMd/RKKaV60USvlFJBzu8TvYiMEZFXROSQiBwUkS+7tieKyHYRKXX9d7Rru4jIb0WkTET2i0hBr3Pd4Tq+VETusKpNA/FUm0Vktoi86TrHfhH5qJXtuhxP/j927Y8TkbMi8jsr2uMOD/9cjxWRF0TksOt8ORY164o83Oafuc5x2HWMWNWuKxlCm6e5fmfbReTrfc61XESOuv497h1UIMYYv/4CMoAC1+tY4BiQC/wMuNe1/V7gp67XK4GtgADzgbdc2xOBE67/jna9Hm11+7zc5inAZNfrTKAKSLC6fd5qb6/z/Qb4L+B3VrfNF20GXgVucL2OAUZZ3T4v/1xfC+wAbK6vN4ElVrfPQ21OBa4Gfgh8vdd5bMBxYAIQDuwDct2Ow+p/iCH8wz0H3AAcBTJ6/WMedb3+E3B7r+OPuvbfDvyp1/ZLjvPnr6G2uZ/z7MOV+P35azjtBa4CHgc+jR8nek+12ZU0iq2O38dtXgC8A0QBo4DdwHSr2+OJNvc67v4+iX4BsK3X9/cB97l7Xb8v3fTm+pN0DvAWkGaMqXLtqgbSXK+zgPJebzvr2na57X5tmG3ufZ55OO8Ejnsz3uEaTntFJAT4JXDJn7z+bpj/j6cAjSLyjIjsEZGfi4jNN5EP3XDabIx5E3gF51+oVTgT4GFfxD0cbrb5coaVvwIm0YtIDPA08BVjzPne+4zzIy7oxol6qs0ikgH8E7jTGOPweKAe4oH2fgEoNMac9VKIHueBNocCi3F+uF2N80/7T3s+Us8ZbptFZBIwHcjGmeyuF5HFXgrXI6zOXwGR6EUkDOc/0n8aY55xba5xJbCeRFbr2l4BjOn19mzXtstt90seajMiEgdsAf6vMWanL2IfCg+1dwFwt4icAn4BfEpEfuKD8IfEQ20+C+w1xpwwxnQBG4BLOqf9iYfafDOw0xjTYoxpwVnHX+CL+IdikG2+nGHlL79P9K7e9L8Ch40xD/batRHoGTlzB87aV8/2T7l67OcDTa4/kbYBy0RktKuHe5lrm9/xVJtFJBx4FviHMeYpH4U/aJ5qrzHm48aYscaYHJx3uP8wxgxudIKPePDneheQICI9sxZeDxzyegOGwINtPgNcJyKhriR6HeCXpZshtPlydgGTRWS86/f6Ntc53GN154QbnReLcP5Zsx/Y6/paCSQBLwGlwItAout4AR7GWYsuAeb2OtddQJnr606r2+btNgOfADp7nWMvMNvq9nnz/3Gvc34aP+6M9fDP9Q2u85QAjwLhVrfPyz/XNpwdtYdxfqg9aHXbPNjmdJx/pZ0HGl2v41z7VuIctXMc51/obsehUyAopVSQ8/vSjVJKqeHRRK+UUkFOE71SSgU5TfRKKRXkNNErpVSQ00SvlFJBThO9Ul4QCPPNqJFDE70a8UTkARH5Sq/vfygiXxaRb4jILtdc6P/Ra/8GEXnHNb/4Z3ttbxGRX4rIPvz4kXw18miiVwr+BnwKwDUD5m04ZxScDMwDZgNXicgHXMffZYy5CpgLfElEklzbo3HOmZ5vjCn2YfxKXVGo1QEoZTVjzCkRqReROTini92DcybIZa7X4FzQYzLwOs7kfrNr+xjX9nqgG+fkVUr5FU30Sjn9Bef8OOk47/CXAj82xvyp90EisgT4ELDAGNMqIq8Cka7dbcaYbh/Fq5TbtHSjlNOzwHKcd/LbXF93ueYRR0SyRCQViAfOuZL8NJxL3Cnl1/SOXinAGNMhIq8Aja678hdEZDrwpnOmWVpwzgb6PPB5ETmMczk4v53jX6keOnulUrzXCfsucKsxptTqeJTyJC3dqBFPRHJxrlHwkiZ5FYz0jl4ppYKc3tErpVSQ00SvlFJBThO9UkoFOU30SikV5DTRK6VUkPt/tvwNeiQxHFkAAAAASUVORK5CYII=\n", "text/plain": [ "
" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "df.groupby('year')['uncertainty'].mean().plot()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## WordNet" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "These examples are based on the [NLTK tutorial](https://www.nltk.org/howto/wordnet.html)." ] }, { "cell_type": "code", "execution_count": 311, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:58:47.493172Z", "start_time": "2022-02-25T09:58:47.486145Z" } }, "outputs": [], "source": [ "# nltk.download('wordnet')\n", "from nltk.corpus import wordnet as wn" ] }, { "cell_type": "code", "execution_count": 156, "metadata": { "ExecuteTime": { "end_time": "2022-02-24T20:02:33.505007Z", "start_time": "2022-02-24T20:02:33.498808Z" } }, "outputs": [ { "data": { "text/plain": [ "[Synset('judge.n.01'),\n", " Synset('evaluator.n.01'),\n", " Synset('judge.v.01'),\n", " Synset('evaluate.v.02'),\n", " Synset('estimate.v.01'),\n", " Synset('pronounce.v.02'),\n", " Synset('judge.v.05')]" ] }, "execution_count": 156, "metadata": {}, "output_type": "execute_result" } ], "source": [ "wn.synsets('judge')" ] }, { "cell_type": "code", "execution_count": 312, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:59:45.313935Z", "start_time": "2022-02-25T09:59:45.305618Z" } }, "outputs": [ { "data": { "text/plain": [ "[Synset('judge.v.01'),\n", " Synset('evaluate.v.02'),\n", " Synset('estimate.v.01'),\n", " Synset('pronounce.v.02'),\n", " Synset('judge.v.05')]" ] }, "execution_count": 312, "metadata": {}, "output_type": "execute_result" } ], "source": [ "wn.synsets('judge', pos='v') # can filter on part of speech" ] }, { "cell_type": "code", "execution_count": 313, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T09:59:55.836509Z", "start_time": "2022-02-25T09:59:55.828784Z" } }, "outputs": [ { "data": { "text/plain": [ "Synset('judge.n.01')" ] }, "execution_count": 313, "metadata": {}, "output_type": "execute_result" } ], "source": [ "judge = wn.synset('judge.n.01')\n", "judge" ] }, { "cell_type": "code", "execution_count": 314, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T10:00:03.911052Z", "start_time": "2022-02-25T10:00:03.902438Z" } }, "outputs": [ { "data": { "text/plain": [ "'a public official authorized to decide questions brought before a court of justice'" ] }, "execution_count": 314, "metadata": {}, "output_type": "execute_result" } ], "source": [ "judge.definition()" ] }, { "cell_type": "code", "execution_count": 315, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T10:00:10.315766Z", "start_time": "2022-02-25T10:00:10.304304Z" } }, "outputs": [ { "data": { "text/plain": [ "['I estimate this chicken to weigh three pounds']" ] }, "execution_count": 315, "metadata": {}, "output_type": "execute_result" } ], "source": [ "wn.synset('estimate.v.01').examples()" ] }, { "cell_type": "code", "execution_count": 171, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T07:57:50.442433Z", "start_time": "2022-02-25T07:57:50.431143Z" } }, "outputs": [ { "data": { "text/plain": [ "[Synset('adjudicator.n.01'), Synset('official.n.01')]" ] }, "execution_count": 171, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# categories to which \"judge.n.01\" belongs\n", "judge.hypernyms()" ] }, { "cell_type": "code", "execution_count": 316, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T10:00:48.120317Z", "start_time": "2022-02-25T10:00:48.113103Z" } }, "outputs": [ { "data": { "text/plain": [ "[Synset('entity.n.01')]" ] }, "execution_count": 316, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# the root category of \"judge.n.01\"\n", "judge.root_hypernyms()" ] }, { "cell_type": "code", "execution_count": 317, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T10:00:53.937497Z", "start_time": "2022-02-25T10:00:53.928251Z" } }, "outputs": [ { "data": { "text/plain": [ "[Synset('think.v.03')]" ] }, "execution_count": 317, "metadata": {}, "output_type": "execute_result" } ], "source": [ "wn.synset('estimate.v.01').root_hypernyms()" ] }, { "cell_type": "code", "execution_count": 318, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T10:01:08.703182Z", "start_time": "2022-02-25T10:01:08.696863Z" } }, "outputs": [ { "data": { "text/plain": [ "[Synset('alcalde.n.01'),\n", " Synset('chief_justice.n.01'),\n", " Synset('daniel.n.02'),\n", " Synset('doge.n.01'),\n", " Synset('justiciar.n.01'),\n", " Synset('magistrate.n.01'),\n", " Synset('ordinary.n.01'),\n", " Synset('praetor.n.01'),\n", " Synset('qadi.n.01'),\n", " Synset('recorder.n.03'),\n", " Synset('trial_judge.n.01'),\n", " Synset('trier.n.01')]" ] }, "execution_count": 318, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# members of the \"judge.n.01\" category\n", "judge.hyponyms()" ] }, { "cell_type": "code", "execution_count": 178, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:02:52.324402Z", "start_time": "2022-02-25T08:02:52.314403Z" } }, "outputs": [ { "data": { "text/plain": [ "[Synset('jury.n.01')]" ] }, "execution_count": 178, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# \"holonym\" is a part of a whole\n", "juror = wn.synset('juror.n.01')\n", "juror.member_holonyms()" ] }, { "cell_type": "code", "execution_count": 179, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:02:52.486713Z", "start_time": "2022-02-25T08:02:52.476865Z" } }, "outputs": [ { "data": { "text/plain": [ "[Synset('person.n.01')]" ] }, "execution_count": 179, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# can find \"lowest common hypernyms\":\n", "judge.lowest_common_hypernyms(juror)" ] }, { "cell_type": "code", "execution_count": 319, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T10:01:45.760265Z", "start_time": "2022-02-25T10:01:45.748916Z" } }, "outputs": [ { "data": { "text/plain": [ "[Lemma('judge.n.01.judge'),\n", " Lemma('judge.n.01.justice'),\n", " Lemma('judge.n.01.jurist')]" ] }, "execution_count": 319, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# \"lemmas\" are specific senses of a specific word.\n", "judge.lemmas()" ] }, { "cell_type": "code", "execution_count": 320, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T10:01:57.383094Z", "start_time": "2022-02-25T10:01:57.373544Z" } }, "outputs": [ { "data": { "text/plain": [ "['judge', 'justice', 'jurist']" ] }, "execution_count": 320, "metadata": {}, "output_type": "execute_result" } ], "source": [ "[lemma.name() for lemma in judge.lemmas()]" ] }, { "cell_type": "code", "execution_count": 182, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:02:54.261992Z", "start_time": "2022-02-25T08:02:54.251066Z" } }, "outputs": [ { "data": { "text/plain": [ "[Lemma('judicial.s.01.judicial'),\n", " Lemma('judicial.a.02.judicial'),\n", " Lemma('judicial.a.03.judicial'),\n", " Lemma('judgeship.n.01.judgeship'),\n", " Lemma('judge.v.05.judge'),\n", " Lemma('judge.v.05.adjudicate'),\n", " Lemma('decide.v.02.adjudicate')]" ] }, "execution_count": 182, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# lemmas have additional properties\n", "judge_lemma = judge.lemmas()[0]\n", "judge_lemma.derivationally_related_forms()" ] }, { "cell_type": "code", "execution_count": 183, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:02:55.555900Z", "start_time": "2022-02-25T08:02:55.548222Z" } }, "outputs": [ { "data": { "text/plain": [ "[Lemma('bad.a.01.bad')]" ] }, "execution_count": 183, "metadata": {}, "output_type": "execute_result" } ], "source": [ "good = wn.synset('good.a.01').lemmas()[0]\n", "good.antonyms()" ] }, { "cell_type": "code", "execution_count": 184, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:02:55.928847Z", "start_time": "2022-02-25T08:02:55.918782Z" } }, "outputs": [ { "data": { "text/plain": [ "['Somebody judge something', 'Somebody judge PP', 'Somebody judge that CLAUSE']" ] }, "execution_count": 184, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# verb frames summarize the different semantic contexts that a verb can be used\n", "judge_verb = wn.synset('estimate.v.01').lemmas()[4]\n", "judge_verb.frame_strings()" ] }, { "cell_type": "code", "execution_count": 186, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:03:04.668594Z", "start_time": "2022-02-25T08:03:04.657402Z" } }, "outputs": [ { "data": { "text/plain": [ "0.1111111111111111" ] }, "execution_count": 186, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# measure similarity in the dictionary between words\n", "judge.path_similarity(wn.synset('juror.n.01'))" ] }, { "cell_type": "code", "execution_count": 203, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:05:31.074790Z", "start_time": "2022-02-25T08:05:31.062152Z" } }, "outputs": [ { "data": { "text/plain": [ "0.08333333333333333" ] }, "execution_count": 203, "metadata": {}, "output_type": "execute_result" } ], "source": [ "judge.path_similarity(wn.synset('cat.n.01'))" ] }, { "cell_type": "code", "execution_count": 204, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:05:31.469152Z", "start_time": "2022-02-25T08:05:31.458493Z" } }, "outputs": [ { "data": { "text/plain": [ "0.5454545454545454" ] }, "execution_count": 204, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Wu-Palmer similarity.\n", "judge.wup_similarity(juror)" ] }, { "cell_type": "code", "execution_count": 205, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:05:31.752657Z", "start_time": "2022-02-25T08:05:31.741757Z" } }, "outputs": [ { "data": { "text/plain": [ "0.5217391304347826" ] }, "execution_count": 205, "metadata": {}, "output_type": "execute_result" } ], "source": [ "judge.wup_similarity(wn.synset('cat.n.01'))" ] }, { "cell_type": "code", "execution_count": 190, "metadata": { "ExecuteTime": { "end_time": "2022-02-25T08:03:26.100296Z", "start_time": "2022-02-25T08:03:23.887564Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Synset('judgeship.n.01')\n", "Synset('judgment.n.02')\n", "Synset('value_judgment.n.01')\n", "Synset('judgment.n.03')\n", "Synset('confession_of_judgment.n.01')\n", "Synset('default_judgment.n.01')\n", "Synset('final_judgment.n.01')\n", "Synset('judgment_in_personam.n.01')\n", "Synset('judgment_in_rem.n.01')\n", "Synset('judgment_of_dismissal.n.01')\n", "Synset('judgment_on_the_merits.n.01')\n", "Synset('summary_judgment.n.01')\n", "Synset('judgment.n.06')\n", "Synset('judgment.n.04')\n", "Synset('prejudgment.n.01')\n", "Synset('judgment.n.01')\n", "Synset('judges.n.01')\n", "Synset('back_judge.n.01')\n", "Synset('field_judge.n.01')\n", "Synset('judge.n.01')\n", "Synset('judge_advocate.n.02')\n", "Synset('judge_advocate.n.01')\n", "Synset('judge_advocate_general.n.01')\n", "Synset('line_judge.n.01')\n", "Synset('side_judge.n.01')\n", "Synset('trial_judge.n.01')\n", "Synset('judgment_lien.n.01')\n", "Synset('judgment_day.n.01')\n" ] } ], "source": [ "# Can iterate over all synsets; e.g., all nouns:\n", "for synset in list(wn.all_synsets('n')):\n", " if 'judg' in str(synset):\n", " print(synset)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "**Exercise**. Use wordnet to expand the set of words in the Baker-Bloom-Davis dictionary and re-compute policy uncertainty scores by year. " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" }, "toc": { "base_numbering": "1", "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Week 01. Text Data Essentials", "title_sidebar": "Contents", "toc_cell": true, "toc_position": { "height": "calc(100% - 180px)", "left": "10px", "top": "150px", "width": "165px" }, "toc_section_display": true, "toc_window_display": true }, "toc-autonumbering": false, "toc-showcode": false, "toc-showmarkdowntxt": false, "toc-showtags": false }, "nbformat": 4, "nbformat_minor": 4 } ================================================ FILE: Notebook/K_mean_clustering.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": { "toc": true }, "source": [ "

Table of Contents

\n", "
" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Natural Language Processing for Law and Social Science
\n", "Directed by Elliott Ash, ETH Zurich, edited by Estheryu991" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:40:51.192954Z", "start_time": "2022-03-11T09:40:51.186493Z" } }, "outputs": [], "source": [ "# set random seed\n", "import numpy as np\n", "np.random.seed(4)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:40:51.740668Z", "start_time": "2022-03-11T09:40:51.614198Z" } }, "outputs": [], "source": [ "# Setup\n", "import warnings; warnings.simplefilter('ignore')\n", "%matplotlib inline\n", "import pandas as pd\n", "df = pd.read_pickle('sc_cases_cleaned.pkl',compression='gzip')\n", "X = pd.read_pickle('X.pkl').toarray()\n", "X_tfidf = pd.read_pickle('X_tfidf.pkl').toarray()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:40:52.236650Z", "start_time": "2022-03-11T09:40:52.209792Z" } }, "outputs": [], "source": [ "from gensim.utils import simple_preprocess\n", "text0 = ' '.join(simple_preprocess(df['opinion_text'][0]))\n", "text1 = ' '.join(simple_preprocess(df['opinion_text'][1]))\n", "\n", "text1[:1000]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Document Distance" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Text Re-Use" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Notes on this implementation of the Smith-Waterman algorithm can be found [here](https://tiefenauer.github.io/blog/smith-waterman/#step-1-scoring-matrix)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:41:24.617382Z", "start_time": "2022-03-11T09:41:23.676571Z" } }, "outputs": [], "source": [ "import itertools\n", "import numpy as np\n", "\n", "def matrix(a, b, match_score=3, gap_cost=2):\n", " H = np.zeros((len(a) + 1, len(b) + 1), np.int)\n", "\n", " for i, j in itertools.product(range(1, H.shape[0]), range(1, H.shape[1])):\n", " match = H[i - 1, j - 1] + (match_score if a[i - 1] == b[j - 1] else - match_score)\n", " delete = H[i - 1, j] - gap_cost\n", " insert = H[i, j - 1] - gap_cost\n", " H[i, j] = max(match, delete, insert, 0)\n", " return H\n", "def traceback(H, b, b_='', old_i=0):\n", " # flip H to get index of **last** occurrence of H.max() with np.argmax()\n", " H_flip = np.flip(np.flip(H, 0), 1)\n", " i_, j_ = np.unravel_index(H_flip.argmax(), H_flip.shape)\n", " i, j = np.subtract(H.shape, (i_ + 1, j_ + 1)) # (i, j) are **last** indexes of H.max()\n", " if H[i, j] == 0:\n", " return b_, j\n", " b_ = b[j - 1] + '-' + b_ if old_i - i > 1 else b[j - 1] + b_\n", " return traceback(H[0:i, 0:j], b, b_, i)\n", "def smith_waterman(a, b, match_score=3, gap_cost=2):\n", " a, b = a.upper(), b.upper()\n", " H = matrix(a, b, match_score, gap_cost)\n", " b_, pos = traceback(H, b)\n", " return pos, pos + len(b_)\n", "\n", "start, end = smith_waterman(text0[:1000], text1[:1000])" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:41:24.865349Z", "start_time": "2022-03-11T09:41:24.852460Z" } }, "outputs": [], "source": [ "text0[start: end]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Cosine Similarity" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:41:43.952894Z", "start_time": "2022-03-11T09:41:43.919280Z" } }, "outputs": [], "source": [ "# compute pair-wise similarities between all documents in corpus\"\n", "from sklearn.metrics.pairwise import cosine_similarity\n", "\n", "sim = cosine_similarity(X[:100])\n", "sim.shape\n", "\n", "sim" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:41:49.366523Z", "start_time": "2022-03-11T09:41:49.344155Z" } }, "outputs": [], "source": [ "# TF-IDF Similarity\n", "tsim = cosine_similarity(X[:100])\n", "tsim[:3,:3]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Jensen-Shannon Divergence" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T08:38:11.995406Z", "start_time": "2022-03-11T08:38:11.985278Z" } }, "outputs": [], "source": [ "from scipy.stats import entropy\n", "def js(p, q):\n", " p /= p.sum()\n", " q /= q.sum()\n", " m = (p + q) / 2\n", " return (entropy(p, m) + entropy(q, m)) / 2\n", "js(tsim[0],tsim[1])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Clustering" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## K-means clustering" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:43:29.382083Z", "start_time": "2022-03-11T09:43:25.856420Z" } }, "outputs": [], "source": [ "# create 100 clusters of similar documents\n", "from sklearn.cluster import KMeans\n", "num_clusters = 40\n", "km = KMeans(n_clusters=num_clusters)\n", "km.fit(X)\n", "doc_clusters = km.labels_.tolist()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:43:55.174072Z", "start_time": "2022-03-11T09:43:55.149135Z" } }, "outputs": [], "source": [ "df['cluster'] = doc_clusters\n", "df[df['cluster']==3]['opinion_text']" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Silhouette Score\n", "\n", "Choose the optimal number of clusters. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:44:52.277204Z", "start_time": "2022-03-11T09:44:52.141635Z" } }, "outputs": [], "source": [ "from sklearn.metrics import silhouette_score\n", "silhouette_score(X, km.labels_)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:46:45.296285Z", "start_time": "2022-03-11T09:45:12.878930Z" } }, "outputs": [], "source": [ "sil_scores = []\n", "for n in range(2, num_clusters):\n", " km = KMeans(n_clusters=n)\n", " km.fit(X)\n", " sil_scores.append(silhouette_score(X, km.labels_))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T09:46:45.467824Z", "start_time": "2022-03-11T09:46:45.343903Z" } }, "outputs": [], "source": [ "import matplotlib.pyplot as plt \n", "plt.plot(range(2, num_clusters), sil_scores)\n", "plt.xlabel('Number of Clusters')\n", "plt.ylabel('Silhouette Score')\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:17.926048Z", "start_time": "2022-03-11T07:07:17.917927Z" } }, "outputs": [], "source": [ "opt_sil_score = max(sil_scores[5:20])\n", "sil_scores.index(opt_sil_score)\n", "opt_num_cluster = range(2, num_clusters)[sil_scores.index(opt_sil_score)]\n", "print('The optimal number of clusters is %s' %opt_num_cluster)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:19.322909Z", "start_time": "2022-03-11T07:07:17.934484Z" } }, "outputs": [], "source": [ "km = KMeans(n_clusters=opt_num_cluster)\n", "km.fit(X)\n", "doc_clusters = km.labels_.tolist()\n", "\n", "df['cluster_mean'] = doc_clusters\n", "df[df['cluster_mean']==1]['opinion_text']\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## K-Medoids" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:19.388613Z", "start_time": "2022-03-11T07:07:19.340115Z" } }, "outputs": [], "source": [ "#!pip install sklearn_extra\n", "from sklearn_extra.cluster import KMedoids\n", "\n", "kmed = KMedoids(n_clusters=opt_num_cluster)\n", "kmed.fit(X)\n", "doc_clusters = kmed.labels_.tolist()\n", "\n", "df['cluster_med'] = doc_clusters\n", "df[df['cluster_med']==1]['opinion_text']" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## DBSCAN" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:19.439357Z", "start_time": "2022-03-11T07:07:19.391327Z" } }, "outputs": [], "source": [ "from sklearn.cluster import DBSCAN\n", "\n", "dbscan = DBSCAN(eps=0.95, min_samples=5)\n", "dbscan.fit(X_tfidf)\n", "db_clusters = dbscan.labels_\n", "\n", "df['cluster_db'] = db_clusters\n", "df[df['cluster_db']==1]['opinion_text']" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Hierarchical DBSCAN\n", "\n", "Automatically chooses epsilon, performing DBSCAN over various epsilon values e returns the result that gives the best stability over epsilon. For reference see [here](https://github.com/scikit-learn-contrib/hdbscan/)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:20.304382Z", "start_time": "2022-03-11T07:07:19.445785Z" } }, "outputs": [], "source": [ "#!pip install hdbscan\n", "\n", "from hdbscan import HDBSCAN\n", "\n", "hdbscan = HDBSCAN(min_cluster_size=5)\n", "hdbscan.fit(X_tfidf)\n", "hdb_clusters = hdbscan.labels_\n", "\n", "df['cluster_hdb'] = hdb_clusters\n", "df[df['cluster_hdb']==1]['opinion_text']" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Hierarchical Clustering" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:20.394867Z", "start_time": "2022-03-11T07:07:20.305217Z" } }, "outputs": [], "source": [ "from sklearn.cluster import AgglomerativeClustering\n", "\n", "cluster = AgglomerativeClustering(n_clusters=opt_num_cluster, affinity='euclidean', linkage='ward')\n", "cluster.fit_predict(X)\n", "\n", "clusters = dbscan.labels_\n", "\n", "df['cluster_hie'] = clusters\n", "df[df['cluster_hie']==1]['opinion_text']\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Principal Component Analysis" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:20.548510Z", "start_time": "2022-03-11T07:07:20.395612Z" } }, "outputs": [], "source": [ "#%% Principal Components\n", "from sklearn.decomposition import PCA\n", "pca = PCA(n_components=3,svd_solver='randomized')\n", "Xpca = pca.fit_transform(X)\n", "pca.explained_variance_ratio_" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:20.624942Z", "start_time": "2022-03-11T07:07:20.563819Z" } }, "outputs": [], "source": [ "#%% PCA Viz\n", "plt.scatter(Xpca[:,0],Xpca[:,1], alpha=.1)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:20.771563Z", "start_time": "2022-03-11T07:07:20.636987Z" } }, "outputs": [], "source": [ "#%% PCA 3D Viz\n", "from mpl_toolkits.mplot3d import Axes3D\n", "Axes3D(plt.figure()).scatter(Xpca[:,0],Xpca[:,1], Xpca[:,2], alpha=.1)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:21.644782Z", "start_time": "2022-03-11T07:07:20.780568Z" } }, "outputs": [], "source": [ "#%% make components to explain 95% of variance\n", "pca = PCA(n_components=.95)\n", "X95 = pca.fit_transform(X)\n", "pca.n_components_" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:21.792302Z", "start_time": "2022-03-11T07:07:21.645487Z" } }, "outputs": [], "source": [ "#%% PCA Inverse Transform\n", "Xrestore = pca.inverse_transform(X95)\n", "plt.plot(Xrestore[0],X[0],'ro')" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:32.071529Z", "start_time": "2022-03-11T07:07:21.816551Z" } }, "outputs": [], "source": [ "#%% Incremental PCA\n", "X_mm = np.memmap('X.pkl',shape=(32567, 525))\n", "\n", "from sklearn.decomposition import IncrementalPCA\n", "inc_pca = IncrementalPCA(n_components=100, batch_size=1000)\n", "inc_pca.fit(X_mm)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:32.079855Z", "start_time": "2022-03-11T07:07:32.072452Z" } }, "outputs": [], "source": [ "#%% PC Regression\n", "from sklearn.linear_model import LinearRegression\n", "from sklearn.model_selection import cross_val_score\n", "Y = df['log_cite_count']\n", "lin_reg = LinearRegression()\n", "scores = cross_val_score(lin_reg,\n", " X95[:,:10],\n", " Y) \n", "scores.mean()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:36.448807Z", "start_time": "2022-03-11T07:07:32.082221Z" } }, "outputs": [], "source": [ "#%% MDS, Isomap, and T-SNE\n", "from sklearn.manifold import MDS, Isomap, TSNE\n", "mds = MDS(n_components=2)\n", "Xmds = mds.fit_transform(X[:500,:200])\n", "Axes3D(plt.figure()).scatter(Xmds[:,0],Xmds[:,1], alpha=.3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:36.765819Z", "start_time": "2022-03-11T07:07:36.472825Z" } }, "outputs": [], "source": [ "#%% Isomap\n", "iso = Isomap(n_components=2)\n", "Xiso = iso.fit_transform(X[:500,:200])\n", "Axes3D(plt.figure()).scatter(Xiso[:,0],Xiso[:,1], alpha=.3)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:07:38.158032Z", "start_time": "2022-03-11T07:07:36.781207Z" } }, "outputs": [], "source": [ "#%% t-SNE\n", "tsne = TSNE(n_components=2, n_iter=250)\n", "Xtsne = tsne.fit_transform(X[:500,:200])\n", "Axes3D(plt.figure()).scatter(Xtsne[:,0],Xtsne[:,1], alpha=.3)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Latent Dirichlet Allocation\n", "\n", "For further reference see the material from topic [modeling with gensim](https://www.machinelearningplus.com/nlp/topic-modeling-gensim-python/)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T08:04:52.181598Z", "start_time": "2022-03-11T08:02:37.752474Z" } }, "outputs": [], "source": [ "# clean document\n", "from gensim.utils import simple_preprocess\n", "import spacy\n", "from spacy.tokenizer import Tokenizer\n", "from tqdm import tqdm as tq\n", "nlp = spacy.load('en_core_web_sm')\n", "# this is faster and we don't need the whole grammatical parse analysis\n", "\n", "def tokenize(x, nlp):\n", " # lemmatize and lowercase without stopwords, punctuation and numbers\n", " return [w.lemma_.lower() for w in nlp(x) if not w.is_stop and not w.is_punct and not w.is_digit and len(w) > 2]\n", "\n", "# split into paragraphs\n", "doc_clean = []\n", "for doc in tq(df['opinion_text'][:100]):\n", " # split by paragraph\n", " for paragraph in doc.split(\"\\n\\n\"):\n", " doc_clean.append(tokenize(paragraph, nlp))\n", "print (doc_clean[:10])\n", "\n", "\n", "# randomize document order\n", "from random import shuffle\n", "shuffle(doc_clean)\n", "\n", "# creating the term dictionary\n", "from gensim import corpora\n", "dictionary = corpora.Dictionary(doc_clean)\n", "# filter extremes, drop all words appearing in less than 10 paragraphs and all words appearing in at least every third paragraph\n", "dictionary.filter_extremes(no_below=10, no_above=0.33, keep_n=1000)\n", "print (len(dictionary))\n", "\n", "\n", "# creating the document-term matrix\n", "doc_term_matrix = [dictionary.doc2bow(doc) for doc in doc_clean]\n", "\n", "# train LDA with 10 topics and print\n", "from gensim.models.ldamodel import LdaModel\n", "lda = LdaModel(doc_term_matrix, num_topics=10, \n", " id2word = dictionary, passes=3)\n", "lda.show_topics(formatted=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "start_time": "2022-03-11T10:01:55.267Z" } }, "outputs": [], "source": [ "# to get the topic proportions for a document, use\n", "# the corresponding row from the document-term matrix.\n", "lda[doc_term_matrix[1]]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "start_time": "2022-03-11T10:01:49.081Z" }, "scrolled": true }, "outputs": [], "source": [ "# or, for all documents\n", "[lda[d] for d in doc_term_matrix]" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:26:40.906548Z", "start_time": "2022-03-11T07:26:36.749256Z" }, "scrolled": true }, "outputs": [], "source": [ "###\n", "# LDA Word Clouds\n", "###\n", "\n", "from numpy.random import randint\n", "from wordcloud import WordCloud\n", "import matplotlib.pyplot as plt\n", "\n", "# make directory if not exists\n", "from os import mkdir\n", "try:\n", " mkdir('lda')\n", "except:\n", " pass\n", "\n", "# make word clouds for the topics\n", "for i,weights in lda.show_topics(num_topics=-1,\n", " num_words=100,\n", " formatted=False):\n", " \n", " #logweights = [w[0], np.log(w[1]) for w in weights]\n", " maincol = randint(0,360)\n", " def colorfunc(word=None, font_size=None, \n", " position=None, orientation=None, \n", " font_path=None, random_state=None): \n", " color = randint(maincol-10, maincol+10)\n", " if color < 0:\n", " color = 360 + color\n", " return \"hsl(%d, %d%%, %d%%)\" % (color,randint(65, 75)+font_size / 7, randint(35, 45)-font_size / 10) \n", "\n", " \n", " wordcloud = WordCloud(background_color=\"white\", \n", " ranks_only=False, \n", " max_font_size=120,\n", " color_func=colorfunc,\n", " height=600,width=800).generate_from_frequencies(dict(weights))\n", "\n", " plt.clf()\n", " plt.imshow(wordcloud,interpolation=\"bilinear\")\n", " plt.axis(\"off\")\n", " plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:16:19.464120Z", "start_time": "2022-03-11T07:16:17.547251Z" } }, "outputs": [], "source": [ "# pyLDAvis, for more details, refer to https://github.com/bmabey/pyLDAvis\n", "import pyLDAvis.gensim\n", "pyLDAvis.enable_notebook()\n", "pyLDAvis.gensim.prepare(lda, doc_term_matrix, dictionary)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Using Mallet to calculate coherence scores for different number of topics to automatically determine the best number of topics\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T08:37:27.411606Z", "start_time": "2022-03-11T08:31:06.461743Z" } }, "outputs": [], "source": [ "# you need gensim version <= 3.8.3 for this to work\n", "import gensim\n", "from gensim.corpora import Dictionary\n", "from gensim.models.wrappers import LdaMallet\n", "from gensim.models.coherencemodel import CoherenceModel\n", "\n", "mallet_path = '~/Downloads/mallet-2.0.8/bin/mallet'\n", "scores = []\n", "for num_topics in range(2, 20, 2):\n", " print (num_topics)\n", " lda = LdaMallet(mallet_path, doc_term_matrix, num_topics=num_topics, id2word=dictionary)\n", " coherence = CoherenceModel(model=lda, texts=doc_clean, corpus=doc_term_matrix, dictionary=dictionary, coherence='c_v')\n", " scores.append((num_topics, coherence.get_coherence()))\n", "pd.DataFrame(scores, columns=[\"Number of Topics\", \"Coherence Scores\"])\n", "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Singular Value Decomposition (SVD)\n", "\n", "For further reference for this and the following section see [here](https://github.com/fastai/course-nlp/blob/219d0c217bd83339e21471d31cd787e86d6ec0a0/2-svd-nmf-topic-modeling.ipynb)." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:30:57.630094Z", "start_time": "2022-03-11T07:30:56.943444Z" } }, "outputs": [], "source": [ "from scipy import linalg\n", "\n", "X = pd.read_pickle('X.pkl').todense()\n", "vec = pd.read_pickle('vec-3grams-1.pkl')\n", "vocab = np.array(vec.get_feature_names())\n", "vocab[400:500]\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:31:00.112146Z", "start_time": "2022-03-11T07:30:59.201038Z" } }, "outputs": [], "source": [ "U, s, Vh = linalg.svd(X, full_matrices=False)\n", "print(U.shape, s.shape, Vh.shape)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:31:01.693951Z", "start_time": "2022-03-11T07:31:01.399719Z" } }, "outputs": [], "source": [ "plt.plot(s)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:31:03.071259Z", "start_time": "2022-03-11T07:31:03.057501Z" } }, "outputs": [], "source": [ "num_top_words=8\n", "\n", "def show_topics(a):\n", " top_words = lambda t: [vocab[i] for i in np.argsort(t)[:-num_top_words-1:-1]]\n", " topic_words = ([top_words(t) for t in a])\n", " return [' '.join(t) for t in topic_words]\n", "\n", "show_topics(Vh[:10])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Non-negative Matrix Factorization (NMF) " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:31:06.882377Z", "start_time": "2022-03-11T07:31:04.922245Z" } }, "outputs": [], "source": [ "from sklearn import decomposition\n", "\n", "clf = decomposition.NMF(n_components=10, random_state=1)\n", "\n", "W1 = clf.fit_transform(X)\n", "H1 = clf.components_\n", "\n", "show_topics(H1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Author Topic Model" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:31:09.147221Z", "start_time": "2022-03-11T07:31:09.077828Z" } }, "outputs": [], "source": [ "from gensim.models import AuthorTopicModel\n", "from gensim.test.utils import temporary_file\n", "\n", "df = df.reset_index()\n", "df['id'] = df.index\n", "author2doc = df[:100][['authorship','id']]\n", "author2doc = author2doc.groupby('authorship').apply(lambda x: list(x['id'])).to_dict()\n", "author2doc" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2022-03-11T07:31:10.623422Z", "start_time": "2022-03-11T07:31:10.554968Z" } }, "outputs": [], "source": [ "model = AuthorTopicModel(\n", " doc_term_matrix, author2doc=author2doc, id2word=dictionary, num_topics=10)\n", "\n", "# For each author list topic distribution\n", "author_vecs = [model.get_author_topics(author) for author in model.id2author.values()]\n", "author_vecs[:2]" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": true, "toc_position": {}, "toc_section_display": true, "toc_window_display": true }, "toc-autonumbering": true }, "nbformat": 4, "nbformat_minor": 4 } ================================================ FILE: Notebook/text_data.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Week 04. Machine Learning with Text Data" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Natural Language Processing for Law and Social Science
\n", "Elliott Ash, ETH Zurich" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "# set random seed\n", "import numpy as np\n", "np.random.seed(4)" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "ename": "ValueError", "evalue": "unsupported pickle protocol: 5", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmagic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'matplotlib notebook'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_pickle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'sc_cases_cleaned.pkl'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcompression\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'gzip'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdrop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m/home/dominsta/anaconda3/envs/legal_dna/lib/python3.6/site-packages/pandas/io/pickle.py\u001b[0m in \u001b[0;36mread_pickle\u001b[0;34m(filepath_or_buffer, compression)\u001b[0m\n\u001b[1;32m 180\u001b[0m \u001b[0;31m# We want to silence any warnings about, e.g. moved modules.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 181\u001b[0m \u001b[0mwarnings\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msimplefilter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"ignore\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mWarning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 182\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 183\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mexcs_to_catch\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 184\u001b[0m \u001b[0;31m# e.g.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mValueError\u001b[0m: unsupported pickle protocol: 5" ] } ], "source": [ "# set up\n", "import warnings; warnings.simplefilter('ignore')\n", "%matplotlib notebook\n", "import pandas as pd\n", "df = pd.read_pickle('sc_cases_cleaned.pkl',compression='gzip')\n", "df=df.reset_index(drop=True)\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "SCALIA 94\n", "GINSBURG 89\n", "THOMAS 87\n", "KENNEDY 86\n", "BREYER 85\n", "STEVENS 79\n", "SOUTER 77\n", "O'CONNOR 52\n", "REHNQUIST 49\n", "ROBERTS 38\n", "ALITO 32\n", "Name: authorship, dtype: int64" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = df.assign(authorship_id=(df['authorship']).astype('category').cat.codes)\n", "df['authorship'].value_counts() " ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "array([[,\n", " ]], dtype=object)" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df[['year','log_cite_count']].hist()" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "['act', 'congress', 'district', 'state']\n" ] }, { "data": { "text/plain": [ "<768x4 sparse matrix of type ''\n", "\twith 2534 stored elements in Compressed Sparse Row format>" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.feature_extraction.text import CountVectorizer\n", "vectorizer = CountVectorizer(min_df=0.01, # at min 1% of docs\n", " max_df=.9, \n", " max_features=4,\n", " stop_words='english',\n", " ngram_range=(1,3))\n", "X = vectorizer.fit_transform(df['opinion_text'])\n", "words = vectorizer.get_feature_names()\n", "print(words)\n", "X" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
case_nameopinion_typedate_standardauthorshipx_republicanmaj_judgesdissent_judgestopic_idcite_countopinion_textyearlog_cite_countauthorship_idx_actx_congressx_districtx_state
0ERICK CORNELL CLAY v. UNITED STATESmajority2003-03-04GINSBURG0.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...[]1.02926.0JUSTICE GINSBURG delivered the opinion of the ...20037.98139220.1935480.2258060.1612900.419355
1HILLSIDE DAIRY INC., A&A DAIRY, L&S DAIRY, AND...majority2003-06-09STEVENS1.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...['THOMAS, CLARENCE']8.0117.0Justice Stevens delivered the opinion of the C...20034.76217490.2619050.0952380.0476190.595238
2CHARLES RUSSELL RHINES v. DOUGLAS WEBER, WARDENmajority2005-03-30O'CONNOR1.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...[]1.023364.0Justice O'Connor delivered the opinion of the ...200510.05895240.0128210.0256410.4615380.500000
3STATE OF ALASKA v. UNITED STATES OF AMERICAmajority2005-06-06KENNEDY1.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...['REHNQUIST, WILLIAM', 'SCALIA, ANTONIN', 'THO...10.084.0Justice Kennedy delivered the opinion of the C...20054.43081730.3225810.1290320.0000000.548387
4REGINALD A. WILKINSON, DIRECTOR, OHIO DEPARTME...majority2005-06-13KENNEDY1.0['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY...[]4.04230.0Justice Kennedy delivered the opinion of the C...20058.34995730.0192310.0000000.5961540.384615
\n", "
" ], "text/plain": [ " case_name opinion_type \\\n", "0 ERICK CORNELL CLAY v. UNITED STATES majority \n", "1 HILLSIDE DAIRY INC., A&A DAIRY, L&S DAIRY, AND... majority \n", "2 CHARLES RUSSELL RHINES v. DOUGLAS WEBER, WARDEN majority \n", "3 STATE OF ALASKA v. UNITED STATES OF AMERICA majority \n", "4 REGINALD A. WILKINSON, DIRECTOR, OHIO DEPARTME... majority \n", "\n", " date_standard authorship x_republican \\\n", "0 2003-03-04 GINSBURG 0.0 \n", "1 2003-06-09 STEVENS 1.0 \n", "2 2005-03-30 O'CONNOR 1.0 \n", "3 2005-06-06 KENNEDY 1.0 \n", "4 2005-06-13 KENNEDY 1.0 \n", "\n", " maj_judges \\\n", "0 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "1 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "2 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "3 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "4 ['BREYER, STEPHEN', 'GINSBURG, RUTH', 'KENNEDY... \n", "\n", " dissent_judges topic_id cite_count \\\n", "0 [] 1.0 2926.0 \n", "1 ['THOMAS, CLARENCE'] 8.0 117.0 \n", "2 [] 1.0 23364.0 \n", "3 ['REHNQUIST, WILLIAM', 'SCALIA, ANTONIN', 'THO... 10.0 84.0 \n", "4 [] 4.0 4230.0 \n", "\n", " opinion_text year log_cite_count \\\n", "0 JUSTICE GINSBURG delivered the opinion of the ... 2003 7.981392 \n", "1 Justice Stevens delivered the opinion of the C... 2003 4.762174 \n", "2 Justice O'Connor delivered the opinion of the ... 2005 10.058952 \n", "3 Justice Kennedy delivered the opinion of the C... 2005 4.430817 \n", "4 Justice Kennedy delivered the opinion of the C... 2005 8.349957 \n", "\n", " authorship_id x_act x_congress x_district x_state \n", "0 2 0.193548 0.225806 0.161290 0.419355 \n", "1 9 0.261905 0.095238 0.047619 0.595238 \n", "2 4 0.012821 0.025641 0.461538 0.500000 \n", "3 3 0.322581 0.129032 0.000000 0.548387 \n", "4 3 0.019231 0.000000 0.596154 0.384615 " ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X = X.todense()\n", "X = X / X.sum(axis=1) # counts to frequencies\n", "for i, word in enumerate(words):\n", " column = X[:,i]\n", " df['x_'+word] = column\n", "df.head()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "log_cite_count 1.000000\n", "x_state 0.153254\n", "x_district 0.111862\n", "x_congress -0.137626\n", "x_act -0.237354\n", "Name: log_cite_count, dtype: float64" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# inspecting data\n", "import numpy as np\n", "features = ['x_'+x for x in words]\n", "cites_features = ['log_cite_count'] + features\n", "df2 = df[cites_features]\n", "corr_matrix = df2.corr()\n", "corr_matrix['log_cite_count'].sort_values(ascending=False)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "array([[,\n", " ,\n", " ,\n", " ,\n", " ],\n", " [,\n", " ,\n", " ,\n", " ,\n", " ],\n", " [,\n", " ,\n", " ,\n", " ,\n", " ],\n", " [,\n", " ,\n", " ,\n", " ,\n", " ],\n", " [,\n", " ,\n", " ,\n", " ,\n", " ]], dtype=object)" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from pandas.plotting import scatter_matrix\n", "scatter_matrix(df2)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df2.plot(kind='scatter', x='x_act', y='log_cite_count', alpha = 0.1)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Regression" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "# create a test set \n", "from sklearn.model_selection import train_test_split\n", "train, test = train_test_split(df2, test_size=0.2)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
x_actx_congressx_districtx_state
5960.1739130.3695650.1521740.304348
900.0223880.0000000.1492540.828358
7340.7741940.0967740.0967740.032258
6940.2820510.2564100.1282050.333333
5170.0169490.0000000.0000000.983051
\n", "
" ], "text/plain": [ " x_act x_congress x_district x_state\n", "596 0.173913 0.369565 0.152174 0.304348\n", "90 0.022388 0.000000 0.149254 0.828358\n", "734 0.774194 0.096774 0.096774 0.032258\n", "694 0.282051 0.256410 0.128205 0.333333\n", "517 0.016949 0.000000 0.000000 0.983051" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "Xtrain = train[features]\n", "Ytrain = train['log_cite_count']\n", "Xtrain.head()" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "596 8.449771\n", "90 8.477620\n", "734 6.648985\n", "694 5.236442\n", "517 3.583519\n", "Name: log_cite_count, dtype: float64" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "Ytrain.head()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([-1.05443811, -0.27079188, 0.87434979, 0.45088019])" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# our first machine learning model\n", "from sklearn.linear_model import LinearRegression\n", "lin_reg = LinearRegression()\n", "\n", "lin_reg.fit(Xtrain, Ytrain)\n", "lin_reg.coef_ " ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2.1599649495666817" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# in-sample performance\n", "from sklearn.metrics import mean_squared_error\n", "Ytrain_pred = lin_reg.predict(Xtrain) \n", "train_mse = mean_squared_error(Ytrain,Ytrain_pred)\n", "train_mse" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "2.1030016631048745" ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# out-of-sample performance\n", "Xtest = test[features]\n", "Ytest = test['log_cite_count']\n", "Ytest_pred = lin_reg.predict(Xtest) \n", "test_mse = mean_squared_error(Ytest,Ytest_pred)\n", "test_mse" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import seaborn as sns\n", "\n", "p = sns.jointplot(x=Ytest, y=Ytest_pred, kind='reg')\n", "p.set_axis_labels('Log Citations', 'Predicted Log Citations', fontsize=10)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Pipelines and Cross-Validation" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "# missing values\n", "df['authorship_id'].fillna(0,inplace=True)" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "# Scaling\n", "from sklearn.preprocessing import StandardScaler\n", "scaler = StandardScaler()\n", "X = scaler.fit_transform(df2)\n", "df2 = pd.DataFrame(X,columns=df2.columns)" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 0.4273395 , -0.10042156, 0.26583686, -0.25620003, 0.08875597],\n", " [-1.69574977, 0.2303693 , -0.38976878, -0.73332927, 0.62993863],\n", " [ 1.7975002 , -0.97500173, -0.73922735, 1.00407587, 0.33689649],\n", " ...,\n", " [ 0.778854 , 0.31343437, 1.46745168, -0.54274748, -0.70067715],\n", " [-0.48288351, 0.47521029, 0.43962056, 0.85945227, -1.20157476],\n", " [ 0.20858247, 0.53243068, -0.86797524, -0.93320774, 0.87744045]])" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "X" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "# Encoding categorical variables\n", "from sklearn.preprocessing import OneHotEncoder\n", "encoder = OneHotEncoder()\n", "judge = df['authorship_id']\n", "judge_fes = encoder.fit_transform(judge.values.reshape(-1,1))\n", "judge_ids = ['j_'+str(x) for x in range(len(judge.unique()))]\n", "judge_fes = pd.DataFrame(judge_fes.todense(),columns=judge_ids)\n", "df = pd.concat([df,judge_fes],axis=1)\n", "train, test = train_test_split(df, test_size=0.2)" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "# Encoding categorical variables\n", "from sklearn.preprocessing import OneHotEncoder\n", "encoder = OneHotEncoder()\n", "judge = df['authorship_id']\n", "judge_fes = encoder.fit_transform(judge.values.reshape(-1,1))\n", "judge_ids = ['j_'+str(x) for x in range(len(judge.unique()))]\n", "judge_fes = pd.DataFrame(judge_fes.todense(),columns=judge_ids)\n", "df = pd.concat([df,judge_fes],axis=1)\n", "train, test = train_test_split(df, test_size=0.2)" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.07158439367084446 0.04427120481337193\n" ] } ], "source": [ "# Cross-validation\n", "from sklearn.ensemble import RandomForestRegressor\n", "forest_reg = RandomForestRegressor()\n", "\n", "from sklearn.model_selection import cross_val_score\n", "scores = cross_val_score(forest_reg,\n", " df[features],\n", " df['x_republican'],\n", " cv=3,\n", " n_jobs=-1)\n", "\n", "# the default score for random forest reg is negative MSE.\n", "print(np.abs(scores.mean()),scores.std())" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'bootstrap': True, 'max_features': 4, 'n_estimators': 30}\n", "0.10601703666326963\n" ] } ], "source": [ "# Grid Search\n", "from sklearn.model_selection import GridSearchCV\n", "param_grid = {'n_estimators': [3, 10, 30],\n", " 'max_features': [2, 4],\n", " 'bootstrap': [True, False]}\n", "\n", "grid_search = GridSearchCV(forest_reg, \n", " param_grid, \n", " cv=3) \n", "grid_search.fit(df[features],df['x_republican'])\n", "\n", "print(grid_search.best_params_)\n", "print(np.abs(grid_search.best_score_))" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'n_estimators': 30, 'max_features': 2, 'bootstrap': True}\n", "0.08213114206388883\n" ] } ], "source": [ "from sklearn.model_selection import RandomizedSearchCV\n", "rand_search = RandomizedSearchCV(forest_reg, param_grid, cv=3) \n", "rand_search.fit(df[features],df['x_republican'])\n", "\n", "print(rand_search.best_params_)\n", "print(np.abs(rand_search.best_score_))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Saving and loading\n", "from sklearn.externals import joblib\n", "joblib.dump(forest_reg,'forest_reg.pkl') # will overwrite local files\n", "forest_reg = joblib.load('forest_reg.pkl')" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [], "source": [ "from sklearn.pipeline import Pipeline\n", "from sklearn.feature_extraction.text import TfidfTransformer\n", "\n", "pipeline = Pipeline([\n", " ('vect', CountVectorizer()),\n", " ('tfidf', TfidfTransformer()),\n", " ('forest_reg', RandomForestRegressor()),\n", "])\n", "\n", "parameters = {\n", " 'vect__max_df': (0.5, 0.75, 1.0),\n", " 'vect__max_features': (2, 4, 8, 16),\n", " 'forest_reg__n_estimators': [3, 10, 30],\n", " 'forest_reg__max_features': [2, 4],\n", " 'forest_reg__bootstrap': [True, False]}" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Performing grid search...\n", "pipeline: ['vect', 'tfidf', 'forest_reg']\n", "parameters:\n", "{'forest_reg__bootstrap': [True, False],\n", " 'forest_reg__max_features': [2, 4],\n", " 'forest_reg__n_estimators': [3, 10, 30],\n", " 'vect__max_df': (0.5, 0.75, 1.0),\n", " 'vect__max_features': (2, 4, 8, 16)}\n", "Fitting 5 folds for each of 144 candidates, totalling 720 fits\n", "done in 554.450s\n", "\n", "Best score: 0.182\n", "Best parameters set:\n", "\tforest_reg__bootstrap: False\n", "\tforest_reg__max_features: 2\n", "\tforest_reg__n_estimators: 30\n", "\tvect__max_df: 0.75\n", "\tvect__max_features: 16\n" ] } ], "source": [ "from pprint import pprint\n", "from time import time\n", "\n", "if __name__ == \"__main__\":\n", " # multiprocessing requires the fork to happen in a __main__ protected\n", " # block\n", "\n", " # find the best parameters for both the feature extraction and the\n", " # classifier\n", " grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n", "\n", " print(\"Performing grid search...\")\n", " print(\"pipeline:\", [name for name, _ in pipeline.steps])\n", " print(\"parameters:\")\n", " pprint(parameters)\n", " t0 = time()\n", " grid_search.fit(df.opinion_text, df.log_cite_count)\n", " print(\"done in %0.3fs\" % (time() - t0))\n", " print()\n", "\n", " print(\"Best score: %0.3f\" % grid_search.best_score_)\n", " print(\"Best parameters set:\")\n", " best_parameters = grid_search.best_estimator_.get_params()\n", " for param_name in sorted(parameters.keys()):\n", " print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Nested Cross-Validation" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Average difference of 0.008041 with std. dev. of 0.021027.\n" ] } ], "source": [ "import numpy as np\n", "from sklearn.model_selection import KFold\n", "n = 30\n", "\n", "forest_reg = RandomForestRegressor()\n", "non_nested_scores = np.zeros(n)\n", "nested_scores = np.zeros(n)\n", "\n", "\n", "param_grid = {'n_estimators': [3, 10, 30],\n", " 'max_features': [2, 4],\n", " 'bootstrap': [True, False]}\n", "\n", "for i in range(n):\n", "\n", " # Choose cross-validation techniques for the inner and outer loops,\n", "\n", " inner_cv = KFold(n_splits=4, shuffle=True, random_state=i)\n", " outer_cv = KFold(n_splits=4, shuffle=True, random_state=i)\n", "\n", " # Non_nested parameter search and scoring\n", " grid_search = GridSearchCV(estimator=forest_reg, param_grid=param_grid, cv=inner_cv)\n", " grid_search.fit(Xtrain, Ytrain)\n", " non_nested_scores[i] = grid_search.best_score_\n", "\n", " # Nested CV with parameter optimization\n", " nested_score = cross_val_score(grid_search, X=Xtrain, y=Ytrain, cv=outer_cv)\n", " nested_scores[i] = nested_score.mean()\n", "\n", "score_difference = non_nested_scores - nested_scores\n", "print(\"Average difference of {:6f} with std. dev. of {:6f}.\"\n", " .format(score_difference.mean(), score_difference.std()))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Regularization" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.39751011925115076" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from sklearn.model_selection import cross_val_score\n", "import numpy as np\n", "from sklearn.linear_model import LinearRegression\n", "\n", "m = 100\n", "x = 6 * np.random.rand(m,1) - 3\n", "y = 0.5 * x ** 2 + x + 2 + np.random.randn(m,1)\n", "y = y.ravel()\n", "\n", "from sklearn.preprocessing import PolynomialFeatures\n", "poly_2 = PolynomialFeatures(degree=2) # also adds interactions\n", "X_poly_2 = poly_2.fit_transform(x)\n", "\n", "\n", "poly_300 = PolynomialFeatures(degree=300) \n", "X_poly_300 = poly_300.fit_transform(x)\n", "\n", "\n", "lin_reg = LinearRegression()\n", "cross_val_score(lin_reg, x, y, cv=3, n_jobs=3).mean()" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.7813839684415319" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cross_val_score(lin_reg, X_poly_2, y, cv=3, n_jobs=3).mean()" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "-2.4409029579050945e+25" ] }, "execution_count": 31, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cross_val_score(lin_reg, X_poly_300, y, cv=3, n_jobs=3).mean()" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.4372437481422473" ] }, "execution_count": 32, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Lasso\n", "from sklearn.linear_model import Lasso\n", "lasso_reg = Lasso(alpha=0.1)\n", "cross_val_score(lasso_reg,x,y).mean()" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.43471954056204976" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Ridge\n", "from sklearn.linear_model import Ridge, SGDRegressor\n", "ridge_reg = Ridge(alpha=1)\n", "cross_val_score(ridge_reg,x,y).mean()" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(0.1, 0.0001)" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "###\n", "# Elastic Net\n", "###\n", "from sklearn.linear_model import ElasticNetCV\n", "enet_reg = ElasticNetCV(alphas=[.0001, .001, .01,.1,1], \n", " l1_ratio=[.0001, .001, .01,.1,.5,.9, .99, 1])\n", "enet_reg.fit(x,y)\n", "enet_reg.alpha_, enet_reg.l1_ratio_" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.4323487671492522" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cross_val_score(enet_reg,x,y).mean()" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5 non-zero of 5 coefficients.\n" ] }, { "data": { "text/plain": [ "0.009194244568332843" ] }, "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ "scaler = StandardScaler()\n", "Xscale = scaler.fit_transform(X)\n", "from sklearn.linear_model import ElasticNet\n", "enet_reg = ElasticNet(alpha=.1, l1_ratio=.0001)\n", "enet_reg.fit(Xscale,df['x_republican'])\n", "nonzero = enet_reg.coef_ != 0\n", "print(nonzero.sum(),'non-zero of',len(enet_reg.coef_),'coefficients.')\n", "X_enet = X[:,nonzero]\n", "X_enet\n", "cross_val_score(enet_reg,X_enet,df['x_republican'],n_jobs=-1).mean()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Classification" ] }, { "cell_type": "code", "execution_count": 38, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
x_republicanx_000x_10x_100x_101x_102x_103x_104x_104 ctx_105...x_violatedx_violationx_waterx_wayx_westx_wordsx_workx_yearx_yearsx_york
00.00.0000000.0000000.0000000.0017090.0017090.0017090.0017090.0017090.001709...0.0000000.0034190.0000000.0017090.0000000.0119660.0000000.0170940.0000000.0
11.00.0000000.0026390.0000000.0026390.0000000.0000000.0052770.0026390.000000...0.0026390.0000000.0000000.0000000.0052770.0026390.0000000.0000000.0000000.0
21.00.0000000.0000000.0000000.0000000.0147060.0000000.0000000.0000000.000000...0.0000000.0016340.0000000.0016340.0000000.0000000.0000000.0130720.0016340.0
31.00.0000000.0036520.0014610.0000000.0000000.0007300.0000000.0000000.006574...0.0000000.0000000.0138790.0029220.0000000.0000000.0000000.0007300.0029220.0
41.00.0010030.0020060.0010030.0000000.0000000.0040120.0000000.0000000.001003...0.0020060.0020060.0000000.0000000.0000000.0000000.0010030.0030090.0010030.0
\n", "

5 rows × 501 columns

\n", "
" ], "text/plain": [ " x_republican x_000 x_10 x_100 x_101 x_102 x_103 \\\n", "0 0.0 0.000000 0.000000 0.000000 0.001709 0.001709 0.001709 \n", "1 1.0 0.000000 0.002639 0.000000 0.002639 0.000000 0.000000 \n", "2 1.0 0.000000 0.000000 0.000000 0.000000 0.014706 0.000000 \n", "3 1.0 0.000000 0.003652 0.001461 0.000000 0.000000 0.000730 \n", "4 1.0 0.001003 0.002006 0.001003 0.000000 0.000000 0.004012 \n", "\n", " x_104 x_104 ct x_105 ... x_violated x_violation x_water \\\n", "0 0.001709 0.001709 0.001709 ... 0.000000 0.003419 0.000000 \n", "1 0.005277 0.002639 0.000000 ... 0.002639 0.000000 0.000000 \n", "2 0.000000 0.000000 0.000000 ... 0.000000 0.001634 0.000000 \n", "3 0.000000 0.000000 0.006574 ... 0.000000 0.000000 0.013879 \n", "4 0.000000 0.000000 0.001003 ... 0.002006 0.002006 0.000000 \n", "\n", " x_way x_west x_words x_work x_year x_years x_york \n", "0 0.001709 0.000000 0.011966 0.000000 0.017094 0.000000 0.0 \n", "1 0.000000 0.005277 0.002639 0.000000 0.000000 0.000000 0.0 \n", "2 0.001634 0.000000 0.000000 0.000000 0.013072 0.001634 0.0 \n", "3 0.002922 0.000000 0.000000 0.000000 0.000730 0.002922 0.0 \n", "4 0.000000 0.000000 0.000000 0.001003 0.003009 0.001003 0.0 \n", "\n", "[5 rows x 501 columns]" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Make a small data set\n", "df = pd.read_pickle('sc_cases_cleaned.pkl',compression='gzip')\n", "df=df.reset_index(drop=True)\n", "\n", "from sklearn.feature_extraction.text import CountVectorizer\n", "vectorizer500 = CountVectorizer(min_df=0.01, # at min 1% of docs\n", " max_df=.9, \n", " max_features=500,\n", " stop_words='english',\n", " ngram_range=(1,3))\n", "X500 = vectorizer500.fit_transform(df['opinion_text']).todense()\n", "X500 = X500 / X500.sum(axis=1) # counts to frequencies\n", "vocab = vectorizer500.get_feature_names()\n", "for i, word in enumerate(vectorizer500.get_feature_names()):\n", " column = X500[:,i]\n", " df['x_'+word] = column\n", "\n", "\n", "features = [x for x in df.columns if x.startswith('x_') and x!=\"x_republican\"]\n", "keepcols = ['x_republican'] + features\n", "df1 = df[keepcols]\n", "y = df1['x_republican']\n", "# set up train/test split\n", "from sklearn.model_selection import train_test_split\n", "train, test = train_test_split(df1, test_size=0.2)\n", "features = [x for x in df1.columns if x.startswith('x_') and x!=\"x_republican\"]\n", "df1.head()" ] }, { "cell_type": "code", "execution_count": 39, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "0.7402597402597403" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Binary Classifier\n", "from sklearn.linear_model import SGDClassifier\n", "sgd_clf = SGDClassifier(max_iter=10) \n", "sgd_clf.fit(train[features], train['x_republican'])\n", "sgd_clf.score(test[features],test['x_republican']) # accuracy on held-out data" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "0.7734375" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# compare to picking largest category\n", "df1['x_republican'].mean()" ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [], "source": [ "from multiprocessing import cpu_count\n", "nthreads = cpu_count() - 1\n", "\n", "# Form cross-validated prdictions for all data points\n", "from sklearn.model_selection import cross_val_predict\n", "df1['x_republican_sgd'] = cross_val_predict(sgd_clf,\n", " df1[features],\n", " df1['x_republican'],\n", " cv=3,\n", " n_jobs=nthreads)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "After applying a classifier to a data set with known labels `0` and `1`:\n", "\n", "
\n", "\n", "
 Definition
\n", "
    \n", "\n", "
  • TP (true positives): labels which were predicted as 1 and actually are 1.

    \n", "\n", "\n", "
  • TN (true negatives): labels which were predicted as 0 and actually are 0.

    \n", "\n", "\n", "
  • FP (false positives): labels which were predicted as 1 and actually are 0.

    \n", "\n", "\n", "
  • FN (false negatives): labels which were predicted as 0 and actually are 1.

    \n", "\n", "
\n", "\n", "To memorize this: \n", "\n", "
    \n", "\n", "
  • The second word \"positives\"/\"negatives\" refers to the prediction computed by the classifier.\n", "
  • The first word \"true\"/\"false\" expresses if the classification was correct or not.\n", "\n", "
\n", "\n", "This is the so called Confusion Matrix:\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Predicted NPredicted P
Actual NTN FP
Actual PFN TP
\n", "\n", "
\n", "\n", "\n", "\n", "- So the total number of predictions can be expressed as `TP` + `FP` + `FN` + `TN`.\n", "\n", "\n", "- The number of correct predictions is `TP` + `TN`.\n", "\n", "\n", "- `TP` + `FN` is the number of positive examples in our data set, \n", "\n", "\n", "- `FP` + `TN` is the number of negative examples.\n", "\n", "- **precision** is computed as TP / (TP + FP).\n", "\n", "\n", "- **recall** is computed as TP / (TP + FN).\n", "\n", "- The **F1 score** is computed as F1 = 2 * (precision * recall) / (precision + recall).\n", "\n", "\n", "
\n", "
 Definition
\n", "\n", "This allows us to define accuracy as (TP + TN) / (TP + FP + FN + TN).\n", "\n", "
" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "" ] }, "execution_count": 47, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Construct confusion matrix\n", "from sklearn.metrics import confusion_matrix, plot_confusion_matrix\n", "#confusion_matrix(df1['x_republican'], df1['x_republican_sgd'])\n", "plot_confusion_matrix(sgd_clf, test[features],test['x_republican'])" ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Precision: 0.7734375\n", "Recall: 1.0\n" ] } ], "source": [ "# Precision and Recall\n", "from sklearn.metrics import precision_score, recall_score\n", "print('Precision:',precision_score(df1['x_republican'], df1['x_republican_sgd']))\n", "print('Recall:',recall_score(df1['x_republican'], df1['x_republican_sgd']))" ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.8722466960352423" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#F1 score\n", "from sklearn.metrics import f1_score\n", "f1_score(df1['x_republican'],df1['x_republican_sgd'])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Precision/Recall Trade Off" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "(array([ 4., 32., 134., 253., 205., 95., 33., 9., 1., 2.]),\n", " array([1.7415352 , 1.95265053, 2.16376586, 2.3748812 , 2.58599653,\n", " 2.79711186, 3.0082272 , 3.21934253, 3.43045786, 3.6415732 ,\n", " 3.85268853]),\n", " )" ] }, "execution_count": 48, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# scores used by SGDClassifier\n", "import matplotlib.pyplot as plt\n", "\n", "y_scores = sgd_clf.decision_function(df[features])\n", "plt.hist(y_scores) # histogram of scores" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1.0" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# prediction using default threshold ... \n", "threshold = 0\n", "(y_scores > threshold).mean()" ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1.0" ] }, "execution_count": 50, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# ... gives default model prediction\n", "ypred = sgd_clf.predict(df[features])\n", "ypred.mean()" ] }, { "cell_type": "code", "execution_count": 51, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1.0" ] }, "execution_count": 51, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# increasing threshold means more zeros are predicted\n", "threshold = 1\n", "(y_scores > threshold).mean()" ] }, { "cell_type": "code", "execution_count": 52, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Visualizing the precision/recall tradeoff\n", "y_scores = cross_val_predict(sgd_clf,\n", " df1[features],\n", " df1['x_republican'],\n", " cv=3,\n", " method='decision_function',\n", " n_jobs=3)\n", "\n", "from sklearn.metrics import precision_recall_curve\n", "precisions, recalls, thresholds = precision_recall_curve(df1['x_republican'], y_scores)\n", "\n", "import matplotlib.pyplot as plt\n", "plt.plot(thresholds, precisions[:-1], label=\"Precision\")\n", "plt.plot(thresholds, recalls[:-1], label=\"Recall\")\n", "plt.xlabel('Threshold')\n", "plt.legend()" ] }, { "cell_type": "code", "execution_count": 53, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "Text(0, 0.5, 'Precision')" ] }, "execution_count": 53, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Plot precision vs recall\n", "plt.step(recalls, precisions)\n", "plt.xlabel('Recall')\n", "plt.ylabel('Precision')" ] }, { "cell_type": "code", "execution_count": 54, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Precision for Low Threshold: 0.7734375\n", "Precision for High Threshold: 0.8346213292117465\n", "Recall for Low Threshold: 1.0\n", "Recall for High Threshold: 0.9090909090909091\n" ] } ], "source": [ "# Assessing other scoring thresholds\n", "ypred_lower = y_scores > -1\n", "ypred_higher = y_scores > 1\n", "\n", "print('Precision for Low Threshold:',precision_score(y, ypred_lower))\n", "print('Precision for High Threshold:',precision_score(y, ypred_higher))\n", "print('Recall for Low Threshold:',recall_score(y, ypred_lower))\n", "print('Recall for High Threshold:',recall_score(y, ypred_higher))" ] }, { "cell_type": "code", "execution_count": 55, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "Text(0, 0.5, 'True Positive Rate')" ] }, "execution_count": 55, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# ROC Curve\n", "from sklearn.metrics import roc_curve\n", "fpr, tpr, thresholds = roc_curve(y, y_scores)\n", "plt.plot(fpr, tpr)\n", "plt.plot([0,1],[0,1],'k--')\n", "plt.xlabel('False Positive Rate')\n", "plt.ylabel('True Positive Rate')" ] }, { "cell_type": "code", "execution_count": 56, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.6847594721157939" ] }, "execution_count": 56, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# AUC\n", "from sklearn.metrics import roc_auc_score\n", "roc_auc_score(y, y_scores)" ] }, { "cell_type": "code", "execution_count": 57, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "" ] }, "execution_count": 57, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Random Forests and predict_proba()\n", "from sklearn.ensemble import RandomForestClassifier\n", "rfc = RandomForestClassifier()\n", "y_probas_rfc = cross_val_predict(rfc,\n", " X500,\n", " y,\n", " cv=3,\n", " method='predict_proba',\n", " n_jobs=3)\n", "\n", "y_scores_rfc = y_probas_rfc[:,1]\n", "fpr_rfc, tpr_rfc, thresholds_rfc = roc_curve(y,y_scores_rfc) \n", "\n", "# compare precision/recall tradeoff for SGD and RF classifiers\n", "plt.plot(fpr, tpr, label=\"SGD\")\n", "plt.plot(fpr_rfc, tpr_rfc, label=\"RF\")\n", "plt.legend()" ] }, { "cell_type": "code", "execution_count": 58, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.8887195711908356" ] }, "execution_count": 58, "metadata": {}, "output_type": "execute_result" } ], "source": [ "roc_auc_score(y,y_scores_rfc)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Multinomial Classification" ] }, { "cell_type": "code", "execution_count": 59, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "0.21744791666666666\n" ] }, { "data": { "text/plain": [ "[('GINSBURG', 'GINSBURG'),\n", " ('STEVENS', 'STEVENS'),\n", " (\"O'CONNOR\", 'KENNEDY'),\n", " ('KENNEDY', 'SOUTER'),\n", " ('KENNEDY', 'KENNEDY'),\n", " ('SOUTER', 'SOUTER'),\n", " ('BREYER', 'SOUTER'),\n", " ('GINSBURG', 'GINSBURG')]" ] }, "execution_count": 59, "metadata": {}, "output_type": "execute_result" } ], "source": [ "author = df['authorship']\n", "authpred = cross_val_predict(sgd_clf,\n", " X500,\n", " author,\n", " cv=7)\n", "print((authpred == author).mean())\n", "list(zip(author[:8],authpred[:8]))" ] }, { "cell_type": "code", "execution_count": 60, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Multi-class confusion matrix\n", "conf_mx = confusion_matrix(author,authpred)\n", "conf_mx\n", "plt.matshow(conf_mx)\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 61, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "" ] }, "execution_count": 61, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# normalize colors\n", "conf_mx_norm = conf_mx / conf_mx.sum(axis=1, keepdims=True)\n", "plt.matshow(conf_mx_norm)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Multinomial Logistic" ] }, { "cell_type": "code", "execution_count": 64, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(0.09765625, 0.01390244564066577)" ] }, "execution_count": 64, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "from sklearn.linear_model import LogisticRegression\n", "logistic = LogisticRegression(C=1, # default L2 penalty # .01, .1, 1, 2, 10\n", " class_weight='balanced')\n", "\n", "scores = cross_val_score(logistic,\n", " Xscale[:1000],\n", " author[:1000],\n", " cv=3,\n", " n_jobs=3)\n", "\n", "scores.mean(), scores.std()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Ensemble Learning" ] }, { "cell_type": "code", "execution_count": 65, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.795586113233172" ] }, "execution_count": 65, "metadata": {}, "output_type": "execute_result" } ], "source": [ "Y = df1['x_republican'] > 0\n", "\n", "# Bagging classifier\n", "\n", "from sklearn.ensemble import BaggingClassifier\n", "from sklearn.tree import DecisionTreeClassifier\n", "\n", "bag_clf = BaggingClassifier(\n", " DecisionTreeClassifier(), n_estimators=50,\n", " max_samples=100, bootstrap=True, n_jobs=-1\n", " )\n", "\n", "cross_val_score(bag_clf, X500, Y).mean()" ] }, { "cell_type": "code", "execution_count": 66, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 2, 172],\n", " [ 0, 594]])" ] }, "execution_count": 66, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# random forest\n", "from sklearn.ensemble import RandomForestClassifier\n", "rnd_clf = RandomForestClassifier(n_estimators=500, \n", " max_leaf_nodes=16, \n", " n_jobs=-1)\n", "y_pred_rf = cross_val_predict(rnd_clf, X500, Y) \n", "confusion_matrix(Y,y_pred_rf)" ] }, { "cell_type": "code", "execution_count": 67, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[(0.0009572418864364776, 'k'),\n", " (0.0010358544276956854, 'o'),\n", " (0.0011153699385341863, 'y'),\n", " (0.004262871674066923, 'r')]" ] }, "execution_count": 67, "metadata": {}, "output_type": "execute_result" } ], "source": [ "rnd_clf.fit(X500,Y)\n", "feature_importances = rnd_clf.feature_importances_\n", "sorted(zip(feature_importances, word), reverse=False)" ] }, { "cell_type": "code", "execution_count": 68, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[18:15:13] WARNING: /home/conda/feedstock_root/build_artifacts/xgboost_1607604574104/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n", "[18:15:14] WARNING: /home/conda/feedstock_root/build_artifacts/xgboost_1607604574104/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n", "[18:15:14] WARNING: /home/conda/feedstock_root/build_artifacts/xgboost_1607604574104/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n", "[18:15:15] WARNING: /home/conda/feedstock_root/build_artifacts/xgboost_1607604574104/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n", "[18:15:15] WARNING: /home/conda/feedstock_root/build_artifacts/xgboost_1607604574104/work/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n" ] }, { "data": { "text/plain": [ "0.8749681690858161" ] }, "execution_count": 68, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# XGBoost\n", "from xgboost import XGBClassifier, XGBRegressor\n", "dfX = pd.DataFrame(X500,columns=vocab)\n", "xgb_clf = XGBClassifier()\n", "cross_val_score(xgb_clf, dfX, Y).mean()" ] }, { "cell_type": "code", "execution_count": 69, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[18:15:18] WARNING: /home/conda/feedstock_root/build_artifacts/xgboost_1607604574104/work/src/learner.cc:541: \n", "Parameters: { feature_names } might not be used.\n", "\n", " This may not be accurate due to some parameters are only used in language bindings but\n", " passed down to XGBoost core. Or some parameters are not used but slip through this\n", " verification. Please open an issue if you find above cases.\n", "\n", "\n" ] }, { "data": { "text/plain": [ "[(0.033571467, 'analysis'),\n", " (0.031830117, 'new york'),\n", " (0.029660394, 'granted certiorari'),\n", " (0.026276005, 'sought'),\n", " (0.025258552, 'related'),\n", " (0.02393513, 'stated'),\n", " (0.023086436, 'internal quotation'),\n", " (0.021765428, 'require'),\n", " (0.021112507, 'light'),\n", " (0.021009024, 'added')]" ] }, "execution_count": 69, "metadata": {}, "output_type": "execute_result" } ], "source": [ "xgb_reg = XGBRegressor(feature_names=vocab)\n", "xgb_reg.fit(dfX,Y)\n", "sorted(zip(xgb_reg.feature_importances_, vocab),reverse=True)[:10]" ] }, { "cell_type": "code", "execution_count": 70, "metadata": {}, "outputs": [ { "data": { "application/javascript": [ "/* Put everything inside the global mpl namespace */\n", "/* global mpl */\n", "window.mpl = {};\n", "\n", "mpl.get_websocket_type = function () {\n", " if (typeof WebSocket !== 'undefined') {\n", " return WebSocket;\n", " } else if (typeof MozWebSocket !== 'undefined') {\n", " return MozWebSocket;\n", " } else {\n", " alert(\n", " 'Your browser does not have WebSocket support. ' +\n", " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", " 'Firefox 4 and 5 are also supported but you ' +\n", " 'have to enable WebSockets in about:config.'\n", " );\n", " }\n", "};\n", "\n", "mpl.figure = function (figure_id, websocket, ondownload, parent_element) {\n", " this.id = figure_id;\n", "\n", " this.ws = websocket;\n", "\n", " this.supports_binary = this.ws.binaryType !== undefined;\n", "\n", " if (!this.supports_binary) {\n", " var warnings = document.getElementById('mpl-warnings');\n", " if (warnings) {\n", " warnings.style.display = 'block';\n", " warnings.textContent =\n", " 'This browser does not support binary websocket messages. ' +\n", " 'Performance may be slow.';\n", " }\n", " }\n", "\n", " this.imageObj = new Image();\n", "\n", " this.context = undefined;\n", " this.message = undefined;\n", " this.canvas = undefined;\n", " this.rubberband_canvas = undefined;\n", " this.rubberband_context = undefined;\n", " this.format_dropdown = undefined;\n", "\n", " this.image_mode = 'full';\n", "\n", " this.root = document.createElement('div');\n", " this.root.setAttribute('style', 'display: inline-block');\n", " this._root_extra_style(this.root);\n", "\n", " parent_element.appendChild(this.root);\n", "\n", " this._init_header(this);\n", " this._init_canvas(this);\n", " this._init_toolbar(this);\n", "\n", " var fig = this;\n", "\n", " this.waiting = false;\n", "\n", " this.ws.onopen = function () {\n", " fig.send_message('supports_binary', { value: fig.supports_binary });\n", " fig.send_message('send_image_mode', {});\n", " if (fig.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: fig.ratio });\n", " }\n", " fig.send_message('refresh', {});\n", " };\n", "\n", " this.imageObj.onload = function () {\n", " if (fig.image_mode === 'full') {\n", " // Full images could contain transparency (where diff images\n", " // almost always do), so we need to clear the canvas so that\n", " // there is no ghosting.\n", " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", " }\n", " fig.context.drawImage(fig.imageObj, 0, 0);\n", " };\n", "\n", " this.imageObj.onunload = function () {\n", " fig.ws.close();\n", " };\n", "\n", " this.ws.onmessage = this._make_on_message_function(this);\n", "\n", " this.ondownload = ondownload;\n", "};\n", "\n", "mpl.figure.prototype._init_header = function () {\n", " var titlebar = document.createElement('div');\n", " titlebar.classList =\n", " 'ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix';\n", " var titletext = document.createElement('div');\n", " titletext.classList = 'ui-dialog-title';\n", " titletext.setAttribute(\n", " 'style',\n", " 'width: 100%; text-align: center; padding: 3px;'\n", " );\n", " titlebar.appendChild(titletext);\n", " this.root.appendChild(titlebar);\n", " this.header = titletext;\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._root_extra_style = function (_canvas_div) {};\n", "\n", "mpl.figure.prototype._init_canvas = function () {\n", " var fig = this;\n", "\n", " var canvas_div = (this.canvas_div = document.createElement('div'));\n", " canvas_div.setAttribute(\n", " 'style',\n", " 'border: 1px solid #ddd;' +\n", " 'box-sizing: content-box;' +\n", " 'clear: both;' +\n", " 'min-height: 1px;' +\n", " 'min-width: 1px;' +\n", " 'outline: 0;' +\n", " 'overflow: hidden;' +\n", " 'position: relative;' +\n", " 'resize: both;'\n", " );\n", "\n", " function on_keyboard_event_closure(name) {\n", " return function (event) {\n", " return fig.key_event(event, name);\n", " };\n", " }\n", "\n", " canvas_div.addEventListener(\n", " 'keydown',\n", " on_keyboard_event_closure('key_press')\n", " );\n", " canvas_div.addEventListener(\n", " 'keyup',\n", " on_keyboard_event_closure('key_release')\n", " );\n", "\n", " this._canvas_extra_style(canvas_div);\n", " this.root.appendChild(canvas_div);\n", "\n", " var canvas = (this.canvas = document.createElement('canvas'));\n", " canvas.classList.add('mpl-canvas');\n", " canvas.setAttribute('style', 'box-sizing: content-box;');\n", "\n", " this.context = canvas.getContext('2d');\n", "\n", " var backingStore =\n", " this.context.backingStorePixelRatio ||\n", " this.context.webkitBackingStorePixelRatio ||\n", " this.context.mozBackingStorePixelRatio ||\n", " this.context.msBackingStorePixelRatio ||\n", " this.context.oBackingStorePixelRatio ||\n", " this.context.backingStorePixelRatio ||\n", " 1;\n", "\n", " this.ratio = (window.devicePixelRatio || 1) / backingStore;\n", " if (this.ratio !== 1) {\n", " fig.send_message('set_dpi_ratio', { dpi_ratio: this.ratio });\n", " }\n", "\n", " var rubberband_canvas = (this.rubberband_canvas = document.createElement(\n", " 'canvas'\n", " ));\n", " rubberband_canvas.setAttribute(\n", " 'style',\n", " 'box-sizing: content-box; position: absolute; left: 0; top: 0; z-index: 1;'\n", " );\n", "\n", " var resizeObserver = new ResizeObserver(function (entries) {\n", " var nentries = entries.length;\n", " for (var i = 0; i < nentries; i++) {\n", " var entry = entries[i];\n", " var width, height;\n", " if (entry.contentBoxSize) {\n", " if (entry.contentBoxSize instanceof Array) {\n", " // Chrome 84 implements new version of spec.\n", " width = entry.contentBoxSize[0].inlineSize;\n", " height = entry.contentBoxSize[0].blockSize;\n", " } else {\n", " // Firefox implements old version of spec.\n", " width = entry.contentBoxSize.inlineSize;\n", " height = entry.contentBoxSize.blockSize;\n", " }\n", " } else {\n", " // Chrome <84 implements even older version of spec.\n", " width = entry.contentRect.width;\n", " height = entry.contentRect.height;\n", " }\n", "\n", " // Keep the size of the canvas and rubber band canvas in sync with\n", " // the canvas container.\n", " if (entry.devicePixelContentBoxSize) {\n", " // Chrome 84 implements new version of spec.\n", " canvas.setAttribute(\n", " 'width',\n", " entry.devicePixelContentBoxSize[0].inlineSize\n", " );\n", " canvas.setAttribute(\n", " 'height',\n", " entry.devicePixelContentBoxSize[0].blockSize\n", " );\n", " } else {\n", " canvas.setAttribute('width', width * fig.ratio);\n", " canvas.setAttribute('height', height * fig.ratio);\n", " }\n", " canvas.setAttribute(\n", " 'style',\n", " 'width: ' + width + 'px; height: ' + height + 'px;'\n", " );\n", "\n", " rubberband_canvas.setAttribute('width', width);\n", " rubberband_canvas.setAttribute('height', height);\n", "\n", " // And update the size in Python. We ignore the initial 0/0 size\n", " // that occurs as the element is placed into the DOM, which should\n", " // otherwise not happen due to the minimum size styling.\n", " if (width != 0 && height != 0) {\n", " fig.request_resize(width, height);\n", " }\n", " }\n", " });\n", " resizeObserver.observe(canvas_div);\n", "\n", " function on_mouse_event_closure(name) {\n", " return function (event) {\n", " return fig.mouse_event(event, name);\n", " };\n", " }\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mousedown',\n", " on_mouse_event_closure('button_press')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseup',\n", " on_mouse_event_closure('button_release')\n", " );\n", " // Throttle sequential mouse events to 1 every 20ms.\n", " rubberband_canvas.addEventListener(\n", " 'mousemove',\n", " on_mouse_event_closure('motion_notify')\n", " );\n", "\n", " rubberband_canvas.addEventListener(\n", " 'mouseenter',\n", " on_mouse_event_closure('figure_enter')\n", " );\n", " rubberband_canvas.addEventListener(\n", " 'mouseleave',\n", " on_mouse_event_closure('figure_leave')\n", " );\n", "\n", " canvas_div.addEventListener('wheel', function (event) {\n", " if (event.deltaY < 0) {\n", " event.step = 1;\n", " } else {\n", " event.step = -1;\n", " }\n", " on_mouse_event_closure('scroll')(event);\n", " });\n", "\n", " canvas_div.appendChild(canvas);\n", " canvas_div.appendChild(rubberband_canvas);\n", "\n", " this.rubberband_context = rubberband_canvas.getContext('2d');\n", " this.rubberband_context.strokeStyle = '#000000';\n", "\n", " this._resize_canvas = function (width, height, forward) {\n", " if (forward) {\n", " canvas_div.style.width = width + 'px';\n", " canvas_div.style.height = height + 'px';\n", " }\n", " };\n", "\n", " // Disable right mouse context menu.\n", " this.rubberband_canvas.addEventListener('contextmenu', function (_e) {\n", " event.preventDefault();\n", " return false;\n", " });\n", "\n", " function set_focus() {\n", " canvas.focus();\n", " canvas_div.focus();\n", " }\n", "\n", " window.setTimeout(set_focus, 100);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'mpl-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'mpl-button-group';\n", " continue;\n", " }\n", "\n", " var button = (fig.buttons[name] = document.createElement('button'));\n", " button.classList = 'mpl-widget';\n", " button.setAttribute('role', 'button');\n", " button.setAttribute('aria-disabled', 'false');\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", "\n", " var icon_img = document.createElement('img');\n", " icon_img.src = '_images/' + image + '.png';\n", " icon_img.srcset = '_images/' + image + '_large.png 2x';\n", " icon_img.alt = tooltip;\n", " button.appendChild(icon_img);\n", "\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " var fmt_picker = document.createElement('select');\n", " fmt_picker.classList = 'mpl-widget';\n", " toolbar.appendChild(fmt_picker);\n", " this.format_dropdown = fmt_picker;\n", "\n", " for (var ind in mpl.extensions) {\n", " var fmt = mpl.extensions[ind];\n", " var option = document.createElement('option');\n", " option.selected = fmt === mpl.default_extension;\n", " option.innerHTML = fmt;\n", " fmt_picker.appendChild(option);\n", " }\n", "\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "};\n", "\n", "mpl.figure.prototype.request_resize = function (x_pixels, y_pixels) {\n", " // Request matplotlib to resize the figure. Matplotlib will then trigger a resize in the client,\n", " // which will in turn request a refresh of the image.\n", " this.send_message('resize', { width: x_pixels, height: y_pixels });\n", "};\n", "\n", "mpl.figure.prototype.send_message = function (type, properties) {\n", " properties['type'] = type;\n", " properties['figure_id'] = this.id;\n", " this.ws.send(JSON.stringify(properties));\n", "};\n", "\n", "mpl.figure.prototype.send_draw_message = function () {\n", " if (!this.waiting) {\n", " this.waiting = true;\n", " this.ws.send(JSON.stringify({ type: 'draw', figure_id: this.id }));\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " var format_dropdown = fig.format_dropdown;\n", " var format = format_dropdown.options[format_dropdown.selectedIndex].value;\n", " fig.ondownload(fig, format);\n", "};\n", "\n", "mpl.figure.prototype.handle_resize = function (fig, msg) {\n", " var size = msg['size'];\n", " if (size[0] !== fig.canvas.width || size[1] !== fig.canvas.height) {\n", " fig._resize_canvas(size[0], size[1], msg['forward']);\n", " fig.send_message('refresh', {});\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_rubberband = function (fig, msg) {\n", " var x0 = msg['x0'] / fig.ratio;\n", " var y0 = (fig.canvas.height - msg['y0']) / fig.ratio;\n", " var x1 = msg['x1'] / fig.ratio;\n", " var y1 = (fig.canvas.height - msg['y1']) / fig.ratio;\n", " x0 = Math.floor(x0) + 0.5;\n", " y0 = Math.floor(y0) + 0.5;\n", " x1 = Math.floor(x1) + 0.5;\n", " y1 = Math.floor(y1) + 0.5;\n", " var min_x = Math.min(x0, x1);\n", " var min_y = Math.min(y0, y1);\n", " var width = Math.abs(x1 - x0);\n", " var height = Math.abs(y1 - y0);\n", "\n", " fig.rubberband_context.clearRect(\n", " 0,\n", " 0,\n", " fig.canvas.width / fig.ratio,\n", " fig.canvas.height / fig.ratio\n", " );\n", "\n", " fig.rubberband_context.strokeRect(min_x, min_y, width, height);\n", "};\n", "\n", "mpl.figure.prototype.handle_figure_label = function (fig, msg) {\n", " // Updates the figure title.\n", " fig.header.textContent = msg['label'];\n", "};\n", "\n", "mpl.figure.prototype.handle_cursor = function (fig, msg) {\n", " var cursor = msg['cursor'];\n", " switch (cursor) {\n", " case 0:\n", " cursor = 'pointer';\n", " break;\n", " case 1:\n", " cursor = 'default';\n", " break;\n", " case 2:\n", " cursor = 'crosshair';\n", " break;\n", " case 3:\n", " cursor = 'move';\n", " break;\n", " }\n", " fig.rubberband_canvas.style.cursor = cursor;\n", "};\n", "\n", "mpl.figure.prototype.handle_message = function (fig, msg) {\n", " fig.message.textContent = msg['message'];\n", "};\n", "\n", "mpl.figure.prototype.handle_draw = function (fig, _msg) {\n", " // Request the server to send over a new figure.\n", " fig.send_draw_message();\n", "};\n", "\n", "mpl.figure.prototype.handle_image_mode = function (fig, msg) {\n", " fig.image_mode = msg['mode'];\n", "};\n", "\n", "mpl.figure.prototype.handle_history_buttons = function (fig, msg) {\n", " for (var key in msg) {\n", " if (!(key in fig.buttons)) {\n", " continue;\n", " }\n", " fig.buttons[key].disabled = !msg[key];\n", " fig.buttons[key].setAttribute('aria-disabled', !msg[key]);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_navigate_mode = function (fig, msg) {\n", " if (msg['mode'] === 'PAN') {\n", " fig.buttons['Pan'].classList.add('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " } else if (msg['mode'] === 'ZOOM') {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.add('active');\n", " } else {\n", " fig.buttons['Pan'].classList.remove('active');\n", " fig.buttons['Zoom'].classList.remove('active');\n", " }\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Called whenever the canvas gets updated.\n", " this.send_message('ack', {});\n", "};\n", "\n", "// A function to construct a web socket function for onmessage handling.\n", "// Called in the figure constructor.\n", "mpl.figure.prototype._make_on_message_function = function (fig) {\n", " return function socket_on_message(evt) {\n", " if (evt.data instanceof Blob) {\n", " /* FIXME: We get \"Resource interpreted as Image but\n", " * transferred with MIME type text/plain:\" errors on\n", " * Chrome. But how to set the MIME type? It doesn't seem\n", " * to be part of the websocket stream */\n", " evt.data.type = 'image/png';\n", "\n", " /* Free the memory for the previous frames */\n", " if (fig.imageObj.src) {\n", " (window.URL || window.webkitURL).revokeObjectURL(\n", " fig.imageObj.src\n", " );\n", " }\n", "\n", " fig.imageObj.src = (window.URL || window.webkitURL).createObjectURL(\n", " evt.data\n", " );\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " } else if (\n", " typeof evt.data === 'string' &&\n", " evt.data.slice(0, 21) === 'data:image/png;base64'\n", " ) {\n", " fig.imageObj.src = evt.data;\n", " fig.updated_canvas_event();\n", " fig.waiting = false;\n", " return;\n", " }\n", "\n", " var msg = JSON.parse(evt.data);\n", " var msg_type = msg['type'];\n", "\n", " // Call the \"handle_{type}\" callback, which takes\n", " // the figure and JSON message as its only arguments.\n", " try {\n", " var callback = fig['handle_' + msg_type];\n", " } catch (e) {\n", " console.log(\n", " \"No handler for the '\" + msg_type + \"' message type: \",\n", " msg\n", " );\n", " return;\n", " }\n", "\n", " if (callback) {\n", " try {\n", " // console.log(\"Handling '\" + msg_type + \"' message: \", msg);\n", " callback(fig, msg);\n", " } catch (e) {\n", " console.log(\n", " \"Exception inside the 'handler_\" + msg_type + \"' callback:\",\n", " e,\n", " e.stack,\n", " msg\n", " );\n", " }\n", " }\n", " };\n", "};\n", "\n", "// from http://stackoverflow.com/questions/1114465/getting-mouse-location-in-canvas\n", "mpl.findpos = function (e) {\n", " //this section is from http://www.quirksmode.org/js/events_properties.html\n", " var targ;\n", " if (!e) {\n", " e = window.event;\n", " }\n", " if (e.target) {\n", " targ = e.target;\n", " } else if (e.srcElement) {\n", " targ = e.srcElement;\n", " }\n", " if (targ.nodeType === 3) {\n", " // defeat Safari bug\n", " targ = targ.parentNode;\n", " }\n", "\n", " // pageX,Y are the mouse positions relative to the document\n", " var boundingRect = targ.getBoundingClientRect();\n", " var x = e.pageX - (boundingRect.left + document.body.scrollLeft);\n", " var y = e.pageY - (boundingRect.top + document.body.scrollTop);\n", "\n", " return { x: x, y: y };\n", "};\n", "\n", "/*\n", " * return a copy of an object with only non-object keys\n", " * we need this to avoid circular references\n", " * http://stackoverflow.com/a/24161582/3208463\n", " */\n", "function simpleKeys(original) {\n", " return Object.keys(original).reduce(function (obj, key) {\n", " if (typeof original[key] !== 'object') {\n", " obj[key] = original[key];\n", " }\n", " return obj;\n", " }, {});\n", "}\n", "\n", "mpl.figure.prototype.mouse_event = function (event, name) {\n", " var canvas_pos = mpl.findpos(event);\n", "\n", " if (name === 'button_press') {\n", " this.canvas.focus();\n", " this.canvas_div.focus();\n", " }\n", "\n", " var x = canvas_pos.x * this.ratio;\n", " var y = canvas_pos.y * this.ratio;\n", "\n", " this.send_message(name, {\n", " x: x,\n", " y: y,\n", " button: event.button,\n", " step: event.step,\n", " guiEvent: simpleKeys(event),\n", " });\n", "\n", " /* This prevents the web browser from automatically changing to\n", " * the text insertion cursor when the button is pressed. We want\n", " * to control all of the cursor setting manually through the\n", " * 'cursor' event from matplotlib */\n", " event.preventDefault();\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (_event, _name) {\n", " // Handle any extra behaviour associated with a key event\n", "};\n", "\n", "mpl.figure.prototype.key_event = function (event, name) {\n", " // Prevent repeat events\n", " if (name === 'key_press') {\n", " if (event.which === this._key) {\n", " return;\n", " } else {\n", " this._key = event.which;\n", " }\n", " }\n", " if (name === 'key_release') {\n", " this._key = null;\n", " }\n", "\n", " var value = '';\n", " if (event.ctrlKey && event.which !== 17) {\n", " value += 'ctrl+';\n", " }\n", " if (event.altKey && event.which !== 18) {\n", " value += 'alt+';\n", " }\n", " if (event.shiftKey && event.which !== 16) {\n", " value += 'shift+';\n", " }\n", "\n", " value += 'k';\n", " value += event.which.toString();\n", "\n", " this._key_event_extra(event, name);\n", "\n", " this.send_message(name, { key: value, guiEvent: simpleKeys(event) });\n", " return false;\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onclick = function (name) {\n", " if (name === 'download') {\n", " this.handle_save(this, null);\n", " } else {\n", " this.send_message('toolbar_button', { name: name });\n", " }\n", "};\n", "\n", "mpl.figure.prototype.toolbar_button_onmouseover = function (tooltip) {\n", " this.message.textContent = tooltip;\n", "};\n", "mpl.toolbar_items = [[\"Home\", \"Reset original view\", \"fa fa-home icon-home\", \"home\"], [\"Back\", \"Back to previous view\", \"fa fa-arrow-left icon-arrow-left\", \"back\"], [\"Forward\", \"Forward to next view\", \"fa fa-arrow-right icon-arrow-right\", \"forward\"], [\"\", \"\", \"\", \"\"], [\"Pan\", \"Left button pans, Right button zooms\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-arrows icon-move\", \"pan\"], [\"Zoom\", \"Zoom to rectangle\\nx/y fixes axis, CTRL fixes aspect\", \"fa fa-square-o icon-check-empty\", \"zoom\"], [\"\", \"\", \"\", \"\"], [\"Download\", \"Download plot\", \"fa fa-floppy-o icon-save\", \"download\"]];\n", "\n", "mpl.extensions = [\"eps\", \"jpeg\", \"pdf\", \"png\", \"ps\", \"raw\", \"svg\", \"tif\"];\n", "\n", "mpl.default_extension = \"png\";/* global mpl */\n", "\n", "var comm_websocket_adapter = function (comm) {\n", " // Create a \"websocket\"-like object which calls the given IPython comm\n", " // object with the appropriate methods. Currently this is a non binary\n", " // socket, so there is still some room for performance tuning.\n", " var ws = {};\n", "\n", " ws.close = function () {\n", " comm.close();\n", " };\n", " ws.send = function (m) {\n", " //console.log('sending', m);\n", " comm.send(m);\n", " };\n", " // Register the callback with on_msg.\n", " comm.on_msg(function (msg) {\n", " //console.log('receiving', msg['content']['data'], msg);\n", " // Pass the mpl event to the overridden (by mpl) onmessage function.\n", " ws.onmessage(msg['content']['data']);\n", " });\n", " return ws;\n", "};\n", "\n", "mpl.mpl_figure_comm = function (comm, msg) {\n", " // This is the function which gets called when the mpl process\n", " // starts-up an IPython Comm through the \"matplotlib\" channel.\n", "\n", " var id = msg.content.data.id;\n", " // Get hold of the div created by the display call when the Comm\n", " // socket was opened in Python.\n", " var element = document.getElementById(id);\n", " var ws_proxy = comm_websocket_adapter(comm);\n", "\n", " function ondownload(figure, _format) {\n", " window.open(figure.canvas.toDataURL());\n", " }\n", "\n", " var fig = new mpl.figure(id, ws_proxy, ondownload, element);\n", "\n", " // Call onopen now - mpl needs it, as it is assuming we've passed it a real\n", " // web socket which is closed, not our websocket->open comm proxy.\n", " ws_proxy.onopen();\n", "\n", " fig.parent_element = element;\n", " fig.cell_info = mpl.find_output_cell(\"
\");\n", " if (!fig.cell_info) {\n", " console.error('Failed to find cell for figure', id, fig);\n", " return;\n", " }\n", " fig.cell_info[0].output_area.element.one(\n", " 'cleared',\n", " { fig: fig },\n", " fig._remove_fig_handler\n", " );\n", "};\n", "\n", "mpl.figure.prototype.handle_close = function (fig, msg) {\n", " var width = fig.canvas.width / fig.ratio;\n", " fig.cell_info[0].output_area.element.off(\n", " 'cleared',\n", " fig._remove_fig_handler\n", " );\n", "\n", " // Update the output cell to use the data from the current canvas.\n", " fig.push_to_output();\n", " var dataURL = fig.canvas.toDataURL();\n", " // Re-enable the keyboard manager in IPython - without this line, in FF,\n", " // the notebook keyboard shortcuts fail.\n", " IPython.keyboard_manager.enable();\n", " fig.parent_element.innerHTML =\n", " '';\n", " fig.close_ws(fig, msg);\n", "};\n", "\n", "mpl.figure.prototype.close_ws = function (fig, msg) {\n", " fig.send_message('closing', msg);\n", " // fig.ws.close()\n", "};\n", "\n", "mpl.figure.prototype.push_to_output = function (_remove_interactive) {\n", " // Turn the data on the canvas into data in the output cell.\n", " var width = this.canvas.width / this.ratio;\n", " var dataURL = this.canvas.toDataURL();\n", " this.cell_info[1]['text/html'] =\n", " '';\n", "};\n", "\n", "mpl.figure.prototype.updated_canvas_event = function () {\n", " // Tell IPython that the notebook contents must change.\n", " IPython.notebook.set_dirty(true);\n", " this.send_message('ack', {});\n", " var fig = this;\n", " // Wait a second, then push the new image to the DOM so\n", " // that it is saved nicely (might be nice to debounce this).\n", " setTimeout(function () {\n", " fig.push_to_output();\n", " }, 1000);\n", "};\n", "\n", "mpl.figure.prototype._init_toolbar = function () {\n", " var fig = this;\n", "\n", " var toolbar = document.createElement('div');\n", " toolbar.classList = 'btn-toolbar';\n", " this.root.appendChild(toolbar);\n", "\n", " function on_click_closure(name) {\n", " return function (_event) {\n", " return fig.toolbar_button_onclick(name);\n", " };\n", " }\n", "\n", " function on_mouseover_closure(tooltip) {\n", " return function (event) {\n", " if (!event.currentTarget.disabled) {\n", " return fig.toolbar_button_onmouseover(tooltip);\n", " }\n", " };\n", " }\n", "\n", " fig.buttons = {};\n", " var buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " var button;\n", " for (var toolbar_ind in mpl.toolbar_items) {\n", " var name = mpl.toolbar_items[toolbar_ind][0];\n", " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", " var image = mpl.toolbar_items[toolbar_ind][2];\n", " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", "\n", " if (!name) {\n", " /* Instead of a spacer, we start a new button group. */\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", " buttonGroup = document.createElement('div');\n", " buttonGroup.classList = 'btn-group';\n", " continue;\n", " }\n", "\n", " button = fig.buttons[name] = document.createElement('button');\n", " button.classList = 'btn btn-default';\n", " button.href = '#';\n", " button.title = name;\n", " button.innerHTML = '';\n", " button.addEventListener('click', on_click_closure(method_name));\n", " button.addEventListener('mouseover', on_mouseover_closure(tooltip));\n", " buttonGroup.appendChild(button);\n", " }\n", "\n", " if (buttonGroup.hasChildNodes()) {\n", " toolbar.appendChild(buttonGroup);\n", " }\n", "\n", " // Add the status bar.\n", " var status_bar = document.createElement('span');\n", " status_bar.classList = 'mpl-message pull-right';\n", " toolbar.appendChild(status_bar);\n", " this.message = status_bar;\n", "\n", " // Add the close button to the window.\n", " var buttongrp = document.createElement('div');\n", " buttongrp.classList = 'btn-group inline pull-right';\n", " button = document.createElement('button');\n", " button.classList = 'btn btn-mini btn-primary';\n", " button.href = '#';\n", " button.title = 'Stop Interaction';\n", " button.innerHTML = '';\n", " button.addEventListener('click', function (_evt) {\n", " fig.handle_close(fig, {});\n", " });\n", " button.addEventListener(\n", " 'mouseover',\n", " on_mouseover_closure('Stop Interaction')\n", " );\n", " buttongrp.appendChild(button);\n", " var titlebar = this.root.querySelector('.ui-dialog-titlebar');\n", " titlebar.insertBefore(buttongrp, titlebar.firstChild);\n", "};\n", "\n", "mpl.figure.prototype._remove_fig_handler = function (event) {\n", " var fig = event.data.fig;\n", " fig.close_ws(fig, {});\n", "};\n", "\n", "mpl.figure.prototype._root_extra_style = function (el) {\n", " el.style.boxSizing = 'content-box'; // override notebook setting of border-box.\n", "};\n", "\n", "mpl.figure.prototype._canvas_extra_style = function (el) {\n", " // this is important to make the div 'focusable\n", " el.setAttribute('tabindex', 0);\n", " // reach out to IPython and tell the keyboard manager to turn it's self\n", " // off when our div gets focus\n", "\n", " // location in version 3\n", " if (IPython.notebook.keyboard_manager) {\n", " IPython.notebook.keyboard_manager.register_events(el);\n", " } else {\n", " // location in version 2\n", " IPython.keyboard_manager.register_events(el);\n", " }\n", "};\n", "\n", "mpl.figure.prototype._key_event_extra = function (event, _name) {\n", " var manager = IPython.notebook.keyboard_manager;\n", " if (!manager) {\n", " manager = IPython.keyboard_manager;\n", " }\n", "\n", " // Check for shift+enter\n", " if (event.shiftKey && event.which === 13) {\n", " this.canvas_div.blur();\n", " // select the cell after this one\n", " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", " IPython.notebook.select(index + 1);\n", " }\n", "};\n", "\n", "mpl.figure.prototype.handle_save = function (fig, _msg) {\n", " fig.ondownload(fig, null);\n", "};\n", "\n", "mpl.find_output_cell = function (html_output) {\n", " // Return the cell and output element which can be found *uniquely* in the notebook.\n", " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", " // IPython event is triggered only after the cells have been serialised, which for\n", " // our purposes (turning an active figure into a static one), is too late.\n", " var cells = IPython.notebook.get_cells();\n", " var ncells = cells.length;\n", " for (var i = 0; i < ncells; i++) {\n", " var cell = cells[i];\n", " if (cell.cell_type === 'code') {\n", " for (var j = 0; j < cell.output_area.outputs.length; j++) {\n", " var data = cell.output_area.outputs[j];\n", " if (data.data) {\n", " // IPython >= 3 moved mimebundle to data attribute of output\n", " data = data.data;\n", " }\n", " if (data['text/html'] === html_output) {\n", " return [cell, data, j];\n", " }\n", " }\n", " }\n", " }\n", "};\n", "\n", "// Register the function which deals with the matplotlib target/channel.\n", "// The kernel may be null if the page has been refreshed.\n", "if (IPython.notebook.kernel !== null) {\n", " IPython.notebook.kernel.comm_manager.register_target(\n", " 'matplotlib',\n", " mpl.mpl_figure_comm\n", " );\n", "}\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "" ] }, "execution_count": 70, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from xgboost import plot_importance\n", "plot_importance(xgb_reg, max_num_features=20)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.12" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": {}, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 4 } ================================================ FILE: README.md ================================================ # Hoi, wie gäähts? ## Wilkommen zu [Swiss German Dictionary](https://www.apple.com/chde/) Possible Google [Speech-to-text](https://cloud.google.com/speech-to-text) ### [Googletrans](https://www.makeuseof.com/python-translator-build/) is a module that uses the Google Translate API to detect and translate sentences from one language to another, which you can use Python Translator to build as well. [Speech-to-text](http://dialektkarten.ch/trans/index.html) [![Speech-to-text](http://dialektkarten.ch/trans/index.html)](image) image # ![](https://i.imgur.com/dLI4HYM.jpg) drawing Circularity: If a compiler uses attribute grammars, it must handle circularity. Normally, existing translation tools are: 1. [Deepl](https://www.deepl.com/translator) 2. [Google Translator](https://translate.google.com/) 3. [Papago](https://papago.naver.com/) 4. [Existing SwissGerman Dictionary](https://www.m-translate.com/translator/swiss#) [App Demo](https://www.swiss-german-online.com/app.html) VERSION WHICH SHOWS SWISS-GERMAN PHRASES EASILY IN SHORT VIDEOS. Feel free to drop a message on [Personal Blog](www.togom11.com), [Linkedin](https://ch.linkedin.com/in/esthiyu?trk=public_profile_locale-url) or [Instagram](https://www.instagram.com/esthicodes/?hl=en). Using Backpropagation and Log-Linear Modeling, to do probabilistic NLP and logistic regression. Used Tools: Cloud Run, BigQuery, Virtual Machines with GPUs and Machine Learning APIs, Google Cloud [Speech-to-Text](https://cloud.google.com/speech-to-text) You can find an overview [here](https://cloud.google.com/products) # vscode-stories-api https://github.com/ide-stories/vscode-stories - [Discord](https://discord.gg/ABpGdRxvaA) # How to run on your computer 1. Have PostgreSQL running on your computer 2. Create a database called `stories` 3. Copy `.env.example` to `.env` and fill in `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` (you will have to register a GitHub OAuth account and set the callback url to: http://localhost:8080/auth/github/callback) 4. Fill in database credentials to `.env` ([typeorm docs options](https://typeorm.io/#/connection-options/postgres--cockroachdb-connection-options)) 5. Don't forget to run `yarn` 6. `yarn dev` to startup server [26 Different Cantonal Swiss German:](https://studyinginswitzerland.com/cantons-of-switzerland/) 1. The Canton of Zurich https://en.wikipedia.org/wiki/File:Wappen_Zürich_matt.svg![image](https://user-images.githubusercontent.com/78131082/216808934-27729b66-06eb-4a2b-a5be-a2fc79d83da2.png) ![istockphoto-1061784708-612x612](https://user-images.githubusercontent.com/78131082/216808346-8d7ef036-0482-4612-a2ac-9ae2d30442e5.jpg) --- [![github](https://cloud.githubusercontent.com/assets/17016297/18839843/0e06a67a-83d2-11e6-993a-b35a182500e0.png)][1][![facebook](https://cloud.githubusercontent.com/assets/17016297/18839836/0a06deb4-83d2-11e6-8078-1d0974af0f63.png)][2][![linkedin](https://cloud.githubusercontent.com/assets/17016297/18839848/0fc7e74e-83d2-11e6-8c6a-277fc9d6e067.png)][3] --- 2. The Canton of Berne ![kantone-schweiz-bern-blog](https://user-images.githubusercontent.com/78131082/216809005-e21b667b-b4f1-42d5-a8e4-53d9fccbef35.jpg) 3. The Canton of Lucerne ![Luzern](https://user-images.githubusercontent.com/78131082/216809012-786ed59e-069d-4649-af90-1af9a0bf8c15.jpg) 4. The Canton of Uri ![1200px-Flag_of_Canton_of_Uri svg](https://user-images.githubusercontent.com/78131082/216809035-3fecfd74-19af-42a3-b27e-97c85d81379d.png) 5. The Canton of Schwyz ![st,small,507x507-pad,600x600,f8f8f8](https://user-images.githubusercontent.com/78131082/216809062-b65a119b-1431-459b-9b44-516d354f7625.jpg) 6. The Canton of Obwalden ![Wappen_Obwalden_matt svg](https://user-images.githubusercontent.com/78131082/216809074-2319744a-0539-4f45-99d7-2a9aa52ed3f2.png) 7. The Canton of Nidwalden ![1200px-Wappen_Nidwalden_matt svg](https://user-images.githubusercontent.com/78131082/216809081-75b793a1-8d75-44fb-b0c8-7313f7c6775a.png) 8. The Canton of Glarus![images](https://user-images.githubusercontent.com/78131082/216809087-e357198c-b633-4bfa-b50d-89137aaf15f1.jpeg) 9. The Zug Canton ![images](https://user-images.githubusercontent.com/78131082/216809096-a7d1a819-3c14-432d-adcd-45e2f95e7c8c.png) 10. The Canton of Fribourg ![Freiburg](https://user-images.githubusercontent.com/78131082/216809104-8b549fec-d526-46e7-9e7b-d9c31216af79.jpg) 11. The Canton of Solothurn![images](https://user-images.githubusercontent.com/78131082/216809109-2fab4807-595e-4ca7-b9f7-ebbaeb58a5af.png) 12. The Canton of Basel-Stadt![images](https://user-images.githubusercontent.com/78131082/216809116-e64b9f21-d9b9-4b7a-a0f5-4fa2090993a2.png) 13. The Canton of Basel-Landschaft![images](https://user-images.githubusercontent.com/78131082/216809121-35adbde4-a200-4cac-beca-cb4cabcdb58b.jpeg) 14. The Canton of Schaffhausen![images](https://user-images.githubusercontent.com/78131082/216809132-ffa5884d-9462-4716-91c4-5c3ac32a60f5.jpeg) 15. The Canton of Appenzell-Ausserrhoden ![unnamed](https://user-images.githubusercontent.com/78131082/216809146-7c705126-a518-4608-986d-323d3a806ee8.jpg) 16. The Canton of Appenzell-Innerrhoden ![unnamed](https://user-images.githubusercontent.com/78131082/216809169-ee1f2a29-d0e9-4048-9216-45622fcdf037.jpg) 17. The Canton of St.Gallen ![unnamed](https://user-images.githubusercontent.com/78131082/216809183-17a75dc7-bdc1-44f8-9cc7-65966e042f70.jpg) 18. The Canton of Graubünden ![Wappen_Graubünden svg](https://user-images.githubusercontent.com/78131082/216809190-e3985ab9-06b1-4fdf-ba72-1bf0e765fb8f.png) 19. The Canton of Aargau![images](https://user-images.githubusercontent.com/78131082/216809196-285377f5-bf8c-48d0-8ed7-aeafbf666a3f.png) 20. The Canton of Thurgau ![121686_1](https://user-images.githubusercontent.com/78131082/216809204-118b6900-7a4d-49b1-9c02-30b84ef0f822.jpg) 21. The Canton of Ticino ![st,small,507x507-pad,600x600,f8f8f8](https://user-images.githubusercontent.com/78131082/216809213-3190c031-51ec-4531-a916-87fc3d87d716.jpg) 22. The Canton of Vaud ![images](https://user-images.githubusercontent.com/78131082/216809219-1c8face1-4324-4e58-ad45-414287206d71.png) 23. The Canton of Valais ![langfr-200px-Wappen_Wallis_matt svg](https://user-images.githubusercontent.com/78131082/216809231-618fe7d9-9ee2-4acf-b3c9-0e95dfc3a6ae.png) 24. The Canton of Neuchatel ![images](https://user-images.githubusercontent.com/78131082/216809246-8ec63686-fb14-47f0-9fdb-9c299137c022.jpeg) 25. The Canton of Geneva ![Wappen_Genf_matt svg](https://user-images.githubusercontent.com/78131082/216809254-5f979d82-5464-4bd2-bb97-7105bda28ae1.png) 26. Canton of Jura ![6122927_001](https://user-images.githubusercontent.com/78131082/216809260-d060ca23-81c9-4a2d-a29d-9779dc9c78c8.jpg) [1]: http://www.github.com/your_contact_info [2]: https://www.linkedin.com/in/your_contact_info [3]: https://www.facebook.com/your_contact_info Design: [Figma](https://www.figma.com/) figma arbeitet Collaboration: [FigJam](https://www.figma.com/figjam/) [wonder.me](http://wonder.me/) [conceptboard.com](http://conceptboard.com) [jamboard.google.com](http://jamboard.google.com/) [menti.com](http://menti.com/) [retrotool.io](http://retrotool.io/) [kahoot.it](http://kahoot.it/) https://meta.stackexchange.com/questions/38915/creating-an-image-link-in-markdown-format An audio tool(Siri Annotation Analyst to help us improve the way people and machines interact.) that allows you to manually load up [Swiss German](https://www.youtube.com/shorts/lVCv6C8dTSI) in Italian, Chinese, Korean, Norwegian, Swedish, Danish, Finnish, Dutch, French Swiss French, Swiss Italian, Austrian German, Flemish, Hebrew, Russian, [Irish](https://www.youtube.com/watch?v=K7tKje_5M3M). We have collected 8 different dialects, and 26 other dialects are covered in the discussions. ## Interacting with your Devices Once your device has been added to SwissGermanBot, you should be able to tell Siri to control your devices. One final thing to remember is that Siri will almost always prefer its default phrase handling over SwissGermanBot devices. For instance, if you name your Sonos device "Radio" and try saying "Siri, turn on the Radio" then Siri will probably start playing an iTunes Radio station on your phone. Even if you name it "Esthi" and say "Siri, turn on Esthi", Siri will probably just launch the Esthi app instead. This is why, for instance, the suggested `name` for the Esthi accessory is "Speakers". We have collected 8 major dialects from [MTC Project Hub](https://projects.mtc.ethz.ch/projects/swiss-voice/swissdial). # Culture Have a look at [Swiss National Day](https://www.youtube.com/watch?v=GHepwehZmD4&t=15s) ## Contents The repository contains four python scripts: **ANTLR (ANother Tool for Language Recognition)** Here is a brief description: * ANTLR (ANother Tool for Language Recognition) is a powerful parser generator for reading, processing, executing, or translating structured text or binary files. It's widely used to build languages, tools, and frameworks. From a grammar, ANTLR generates a parser that can build and walk parse trees. * RandomWalkSimulator computes the meeting time of a random walk on a graph. * RandomWalkSimulatorCUDA computes the meeting time of random walks on a graph using CUDA and GPUs (much faster for large graphs). It requires Cudatoolkit to run. * MeetingTimeEstimator is a class that makes educated guesses of the meeting times of two walks which have not met, based on the meeting times of walks which have met. Each script is described in detail in the documentation provided [here](https://rse-distance.readthedocs.io). If you are interested in a **quick start tutorial** see the section **Tutorial** below. ## Installation The scripts are provided in the form of a python package called [structural_diversity_index](https://pypi.org/project/structural-diversity-index/). To install the package and its dependencies type into the terminal ```rb pip install structural_diversity_index==0.0.3 ``` This will install the 0.0.3 version (latest) of the package in your python packages directory. ### Installation for GPUs If you are not interested in running computations on GPUs you can ignore this section. Installing the structural_diversity_index package via pip does not enable you to run computations on GPUs. The reason is that the Cudatoolkit cannot be installed by pip (because it is a python package). To circumvent this issue one can use a package installer such as [Django](https://docs.djangoproject.com/en/4.1/intro/install/). Once you have installed conda on your computer, download the file **environment.yml** from the GitHub. In the terminal, go to the directory containing the environment.yml file you downloaded and type: ```rb conda env create -f environment.yml ``` This will create a environment called **sd_index** and install all the dependencies necessary to Automatic Speech Recognition (ASR) - DeepSpeech Swiss German. Now you can (see Examples.ipynb in [GitHub](https://github.com/AASHISHAG/deepspeech-swiss-german)) and computations will run on GPUs. ## Tutorial The Jupyter notebook **Example.ipynb** contains a detailed tutorial explaining how to use the package. ## Pre-processing the code If you are interested in extending, modifying or simply playing around with the code, I have created a detailed documentation with Pre-processing which is available [here](https://github.com/AASHISHAG/deepspeech-swiss-german/tree/master/pre-processing). # https://github.com/AASHISHAG/deepspeech-swiss-german/tree/master/pre-processing

## Create a custom AI model using AutoML Natural Language ### Introduction This walkthrough shows you how to use AutoML Natural Language to create a custom machine learning model. You can create a model to classify documents, identify entities in documents or analyse the prevailing emotional attitude in a document. ### Learn how to: Set up a project and workspace. Learn about different model objectives. Import data for a data set. Train and use your custom model. Clean up the resources that you created for this walkthrough. ### How to Start By using the Cloud ML API to train custom machine learning models with minimum effort, and the Cloud Storage API to store and access your data. ## Step 2: Model objectives AutoML Natural Language can train custom models for four distinct tasks, known as model objectives: Single label classification classifies documents by assigning a label to them. Multi-label classification allows a document to be assigned multiple labels. Entity extraction identifies entities in documents. Sentiment analysis analyses attitudes within documents. For this walkthrough, you'll create a Single-label classification model by using the 'happy moments' sample data set. The resulting model classifies happy moments into categories reflecting the causes of happiness. ## Step 3: Import data for a data set Click the Navigation menu icon, then click Natural Language. You can see where it is by clicking the following button: Natural Language Within the AutoML text and document classification section, click Get started. Click the New data set button. Enter a data set name. Leave the Location set to Global. Select the Single-label classification as your model objective. Click Create data set. ## Step 4: Import data to create a data set Verify that you are on the Import tab of your new data set details page. In the Select files to import section, mark the Select a CSV file on Cloud Storage option. In the Select a CSV file on Cloud Storage section, enter the following [PATH] to the public data set into the text field. cloud-ml-data/NL-classification/happiness.csv Click Import. The import can take approximately 10 minutes per 1,000 documents. Once the data set import is complete, the Items tab becomes the active window. During training, a progress bar indicates the progress of the training. ## Step 5: Train your model The Items tab shows a list of available items to include in your training model, a summary of statistics and an example set of labels for the data set selected. When you have finished reviewing the data set, switch to the Train tab. Click Start training. In the new panel, enter a model name for the new model. Mark the Deploy model after training finishes tick box. Click Start training. Training a model can take several hours to complete. After the model is successfully trained, you will receive a message at the email address associated with your project. The progress panel changes to display the results in the panel. After training, the bottom of the Train tab shows high-level metrics for the model, such as precision and recall percentages. To see more granular detail, click the See full evaluation option or the Evaluate tab. ## Step 6: Use the custom model After your model has been successfully trained, you can use it to analyse other documents. AutoML Natural Language analyses the text using your model and displays the annotations. Click the Test & use tab. Click inside the Input text below box and add some sample text. Click Predict to review the results of the analysis. The prediction results are displayed with their predicted labels. Explore the resulting annotated code shown in the Use the custom model section. 🎉 Success You've successfully created and trained a sample data set using public data using the AutoML Natural Language API! ## Step 7: Next steps Delete the project If you've created a project specifically for this tutorial, you can delete it using the Projects page in the Cloud Console to avoid incurring charges to your account for resources used in this tutorial. This also deletes all underlying resources. Delete data set If you'd rather delete just the data sets that you created during this tutorial: In the Natural Language menu, click Data sets. On the row containing your data set, click More actions > Delete . Click Delete to finalise the data set removal. Vertex AI brings AutoML and AI Platform together into a unified API, client library, and user interface. AutoML lets you train models on image, tabular, text, and video datasets without writing code, while training in AI Platform lets you run custom training code. With Vertex AI, both AutoML training and custom training are available options. Whichever option you choose for training, you can save models, deploy models, and request predictions with Vertex AI. More examples on Google Cloud and NLU can be seen in [YouTube](https://www.youtube.com/watch?v=2w7nYI9MaYM) . ### Usage 1. Clone the repository ```bash git clone https://github.com/Estheryu991/SwissGerman_Dictionary cd SwissGerman_Dictionary ``` 2. Download the Pre trained model at [BaiduDrive](https://pan.baidu.com/s/10vuV7m00OHLcsihaC-Adsw) or [GoogleDrive](https://drive.google.com/file/d/1UoE-XuW1SDLUjZmJPkIZ1MLxvQFgmTFH/view?usp=sharing), and put it into `Data/net-data` 3. Run the test code.(test AFLW2000 images) `python run_basics.py #Can run only with python and tensorflow` 4. Run with your own images `python demo.py -i -o --isDlib True ` run `python demo.py --help` for more details. 5. For Texture Editing Apps: `python demo_texture.py -i image_path_1 -r image_path_2 -o output_path ` run `python demo_texture.py --help` for more details. ## Citation If you use this code, please consider citing: ``` @inProceedings{feng2018prn, title = {Swiss German Language in Social Science Reconstruction and The Distributional Hypothesis and Word Vectors}, author = {Hoeun Yu, Dawid}, booktitle = {ECCV}, year = {2022} } ``` ## Contacts Please contact _hoeuyu@ethz.ch_ or open an issue for any questions or suggestions. Danke Vilmals! (●'◡'●) ## Acknowledgements - Thanks [Andrea Musso](https://faces.dmi.unibas.ch/bfm/) for Introducing (https://coss.ethz.ch/people/phd/amusso.html), [Tim](http://www.cbsr.ia.ac.cn/users/xiangyuzhu/projects/3DDFA/main.htm), and [Feng Yao](https://github.com/YadiraF/PRNet/blob/master/README.md) for sharing NLP Data, Vilem for the [Poetry Translation](https://github.com/zouharvi/mean-poet) - Thanks Siri for sharing the work [Siri](https://github.com/Xinglab/siri), which helps me a lot in studying Siri response. - Thanks for giving the lecture on [Computational Semantics for Natural Language Processing](http://www.mrinmaya.io/teaching_csnlp22) and the [presentation](https://app.gather.town/events/oyd9OJWmuXtEyeK3F61j) - Thanks the authors of [Elliott Ash, (ashe@ethz.ch)](https://github.com/anilbas/3DMMasSTN), [Siri](https://github.com/kopiro/siriwave), [3dmm_cnn](https://github.com/anhttran/3dmm_cnn), [vrn](https://github.com/AaronJackson/vrn), [Afra Amini, (afra.amini@inf.ethz.ch)](https://github.com/elliottash/nlp_lss_2022), [face-alignment](https://github.com/1adrianb/face-alignment) for making their excellent works publicly available. - [References](https://docs.google.com/document/d/1JtSKVVGjJ3oIMoSE8FHION--Xi1RXUk0xp7MhE_J_CM/edit) _For Example_ ``` sd_list = ['Dubel','Erdnüssli', tschutte] Gnüss es! gramm c = wo/wenn treffed mir üs v = 我们在哪里见面？ c = Chunnsch mit mir go Znacht ässe? ``` v = 你想和我吃饭吗？ English (bag) a - > ä c = Chum gli hei! (come home soon!) b = bald == gli d = Fläsche == bottle de - > male di - << Frau ``` >>> Der -- Die -- Das mehr und mehr: 🇬🇧: Wanna go out drinking? 🇨🇭: Wämmer eis go ziie? 🇩🇪: Wollen wir einen trinken? 🇬🇧: I'm cold - Ich ha chalt 🇨🇭: Mir ist kalt I have a bit of a headache 🇨🇭:Ich han es bitzeli Chopfweh Ich habe ein bisschen Kopfweh Now the fun is over! 🇨🇭:Jetzt isch färtig luschtig Region of Switzerland North: Thurgau South: West East A little 🇨🇭: Es bitzeli Ein bisschen Approximately Öppe Etwa Someone Öpper Jemand Something Öppis Etwas Not Nööd Nicht Nothing Nüüt Nichts Here Da Hier There Det Dort ...right? ...gäll? ...nicht wahr? Otherwise Susch Sonst Disgusting Gruusig Grausig Very (not a very nice expression)(Uu) huere Some times Mängisch Manchmal Well, yes Mol Doch Yeah, right Äbä Eben  Work Schaffe Arbeiten Work hard Chrampfe == bügle Hart arbeiten Sunbathe Sünnele Sich sonnen Go shopping Poschte Einkaufen Look Luege Sehen Call Aalüte Anrufen I call you Ich lüte dir aa Ich rufe dich an You know Weisch Weisst du Are you coming? Chunnsch? Kommst du? Do we have...? Hämmer...? Haben wir...? D Bevölkereg esch gege d noi Schtrasse. 0.33271761714670817 D Bevölkerig esch gege d noi Schtrasse. 0.20469203806629924 D Bevölkereg isch gege d noi Schtrasse. 0.20469203806629924 D Bevölkerig isch gege d noi Schtrasse. 0.1259291011009509 D Bevölkereg esch gege d nöi Schtrasse. 0.04989357708977294 D Bevölkerig esch gege d nöi Schtrasse. 0.030695152449413003 D Bevölkereg isch gege d nöi Schtrasse. 0.030695152449413003 D Bevölkerig isch gege d nöi Schtrasse. 0.018884041571070945

Let's go Gömmer Gehen wir [NLP](https://cloud.google.com/natural-language/docs/samples/language-entity-sentiment-text?hl=de) ================================================ FILE: SwissGermanBot/login.txt ================================================ aiohttp==3.7.4.post0 async-timeout==3.0.1 attrs==21.2.0 certifi==2021.10.8 chardet==4.0.0 charset-normalizer==2.0.7 discord==1.7.3 discord.py==1.7.3 et-xmlfile==1.1.0 idna==3.3 multidict==5.2.0 numpy==1.21.2 openai==0.10.5 openpyxl==3.0.9 pandas==1.3.3 pandas-stubs==1.2.0.27 python-dateutil==2.8.2 pytz==2021.3 requests==2.26.0 six==1.16.0 tqdm==4.62.3 typing-extensions==3.10.0.2 urllib3==1.26.7 yarl==1.7.0 ================================================ FILE: datasets/Buendnertuetsch/buendnertuetsch.txt ================================================ allpot - viel, oft Bagaaschi - eine Menge Leute Fretschni - Kälte Goof - Kind gööla - spielen (Kinder) Gaggalaari - Dummkopf Mungg - Murmeltier Nani - Grossmutter Neni - Grossvater patschifig - gemütlich / locker Poppi - Baby / Säugling Pölli - Kopf Gutschi - Couch / Sofa Guttla - Wasserpfütze gschpunna - unglaublich Huttla - Kleider khoga guat - sehr gut moniz - hallo / tschüss Scarnuz - Papier(Sack) Schesawaaga - Kinderwagen Schparz - Fusstritt tschent - toll / super Viva - Prost Zeeba - Zehen ================================================ FILE: datasets/Jugendsprache/jugendsprache.txt ================================================ Allround-Laie : Versager Augentinnitus : Das unangenehme Gefühl, von dummen Menschen umgeben zu sein bambus : cool / krass Beckenrandschwimmer : Weichling Bewegungslegastheniker : unsportlicher Mensch BolognaFlüchting : Studienabbrecher Cloudophobie : Angst vor Datenklau Dadster : szeniger Papa dönieren : Döner essen Earthporn : schöne Landschaft Enterbrainment : niveauvolle Unterhaltung entspannungsorientiert : faul Erzeugnisfraktion : Eltern ferierbar : toll / ehrwürdig flittern : über Twitter flirten Gemüse-Taliban : radikaler Veganer Genussoptimierer : Koch Hirnpimper : Lehrer krimmen : jem. etwas wegnehmenm was man ihm vorher geschenkt hat Läuft! : Geht klar! merkeln : krinr Entscheidungen treffen Münzenmallorca : Solarium Smombie : exzessiver Smartphonenutzer auch im Gehen Swaggetarier : Jemand der nur aus Imagegründen Vegetarier ist ================================================ FILE: datasets/bern/W Nuss Vo Buempliz.py ================================================ D'w. nuss vo bümpliz geit dür d'strass Liecht und flüchtig, wie nes gas So unerreichbar höch Bockstössigi himbeerbuebe Schüüch und brav wie schaf Schön fönfrisiert Chöme tubetänzig nöch Und d'spargle wachse I bluetjung morge D'sunne chunnt's wird langsam warm Sie het meh als hundert ching Und jede früehlig git's es nöis Het d'chiuchefänschterouge off Und macht se zue bi jedem kuss Und we sie lachet wärde bärge zu schtoub Und jedes zäihe läderhärz wird weich D'w. nuss vo bümpliz Isch schön win es füür I dr nacht Win e rose im schnee We se gseh duss in bümpliz De schlat mir mys härz hert im hals Und I gseh win I ungergah, ja, ja Sie wohnt im ne huus us glas Hinger türe ohni schloss Gseht dür jedi muur Dänkt wi nes füürwärch Win e zuckerschtock Läbt win e wasserfau Für sie git's nüt, wo's nid git Und aus wo's git, git's nid für ging Sie nimmt's wie's chunnt und lat's la gah D'w. nuss vo bümpliz Isch schön win es füür I dr nacht Win e rose im schnee We se gseh duss in bümpliz De schlat mir mys härz hert im hals Und I gseh win I ungergah D'w. nuss vo bümpliz Isch schön win es füür I dr nacht Win e rose im schnee We se gseh duss in bümpliz De schlat mir mys härz hert im hals Und I gseh win I ungergah d'w. nuss vo buempliz geit duer d'strass liecht & fluechtig, wie nes gas so unerreichbar hoech bockstoessigi himbeerbuebe schuech & brav wie schaf schoen foenfrisiert choeme tubetaenzig noech & d'spargle wachse i bluetjung morge d'sunne chunnt 's wird langsam warm sie het meh als hundert ching & jede fruehlig git's es noeis het d'chiuchefaenschterouge off & macht se zue bi jedem kuss & we sie lachet waerde baerge zu schtoub & jedes zaeihe laederhaerz wird weich d'w. nuss vo buempliz isch schoen win es fuehr i dr nacht win e rose im schnee we se gseh duss in buempliz de schlat mir mys haerz hert i hals & i gseh win i ungergah sie wohnt im ne huus us glas hinger tuere ohni schloss gseht duer jedi muur daenkt wi nes fuehrwaerch win e zuckerstock laebt win e wasserfau fuer sie git's nuet, wo's nid git & aus wo's git, git's nid fuer ging sie nimmt's wie's chunnt & lat's la gah ================================================ FILE: datasets/bern/bern.txt ================================================ Bern Chatz: Chätzli Darf ich ein Kekse haben?: Dörfti bitte es Guetzli ha? Eine frische Milch bitte: E früschi Müuch bitte. hallo: hoi danke: merci Chrüsimüsi: durchneinander füdleblutt: komplett nackt äckegstabi:nackenstaber gigetschi: apfelkern wenni hinech eis wott go zieh, (dem muessi)sötti bim Schaffe fürschi mache. Wenn ich etwas trinken gehe, muss ich beim Afbeit etwas schneller machen. Hey Löu, was hesch wider bote? Wenn jemanden etwas verbogt hatt. Entschuldigung, wo finde ich die WC? : Tschuldigung, wo fingi s WC? ================================================ FILE: datasets/dictionary.py ================================================ # Swiss German Dictionary (alphabetically ordered) # AAabig (evening)aacho (to arrive)aadüte (to hint at sth) also: aatöneAakunft (arrival)Aaruef (phone call)aatöne (to hint) also: adüüteabartig (kinky)abegheie (to fall from something)Abflug (takeoff)abflüge (to takeoff)abhaue (to crop or to bunk)Abreis (departure)abreise (to depart)abverheit (failed)abverreckt (totally failed)Ade, Adjö (formal goodbye)Advent (advent, season before Christmas)Adventschranz (advent wreath)Adventskalender (Christmas calendar)Adventsmärt (advent market)Agända (diary)Alles Guete zum Geburi! (Happy Birthday!) also: Häppi Börsdei!alpott (again and again)amigs (sometimes)ängstirnig (narrow-minded) Anke (butter) also: Butteraper (snow free) # Arschbombe (cannonball, jumping butt first into the pool) # Ätti (dad in Bernese dialect)äuä (fat change!, come on!)Auto (car)autofahre (to drive a car, to ride a car)autostöpple (to hitchhike, lit. to stop a car)äxgüsi (from french "excusez", sorry) # BBaby (baby) also: Bébé, Buschi, Poppi # Badchappe (bathing cap) # Badchleid (bathing suit for women) # bade (to take a bath)Badefeerie (beach vacation)bädele (to swim, to play in the water) # Badhose (bathing trunks) # Badi (public pool) also: Schwümmbi # Badtüechli (towel) # Balkonie (staycation, lit. imaginary land on your balcony)Bäredräck (licorice)Bébé (baby) also: Baby, BuschiBerg (mountain)Beschprächig (meeting)Bett (bed)Bettfläsche (hot water bottle)Bettmümpfeli (bedtime snack)Bewärbig (application)bewundere (to admire)Bibeli (chick, pimple) # Binggis (children) # bisle (to pee) also: brünzlebiwakiere (to bivouac)bizzeli (a little bit)Blätter - leavesBleischtift (pencil) # Blödian (idiot) also: Tubelblöterle (to lollygag)Blueme (flower)Blüemli (little flower)blüttle (walk around undressed)Böllä (ball, onion (only in Zurich dialect))Bratwurscht (bratwurst)briegge (to cry) also: greine, gränne, hüüle, brüeleBrüeder (brother)Brüederli (little brother)brüele (to cry) also: greine, gränne, hüüle, brieggeBrünneli (washbasin, sink) also: Lawabo # brünzle (to pee) also: bislebruuchbar (useful)bsinne (to remember)bsoffe (drunk)Bsoffene (a drunk)Bsuecher (visitor)büätzä (to work) also: chrampfe, schaffeBüätzer (worker)bueche (to book)Buggel (bump)Buggeli (little bumps, bumps on a ski slope)Buggelpischte (mogul slope)Büro (office)Bürzi (chignon) also: Hüppi, PfürziBus (bus)Buschi (baby) also: Bébé, BabyBuschiwägeli (pram)busfahre (to ride a bus, to drive a bus)Büsi (cat) also: Busle, ChatzBusle (cat) also: Büsi, ChatzBütschgi ((apple) core) also: Groibschi, GigetschiBuuch (belly)Buuchweh (tummy ache)Buur (farmer)Buurehof (farm)Buurehof-Feerie (vacation on a farm)Ccampiere (to camp out)Campingplatz (campground) also: ZältplatzChalet (chalet)chalt (cold)Chälti (coldness)Chappe (touque, hat)Charte (map, card)Chäs (cheese)Chaschte (wardrobe, strong and heavy man)Chäschüechli (cheese tart)Chaugummi (bubble gum) also: CheutschgiCheib (bloke)chere (to turn)Cherze (candle)Cherzezieh (making candles)Cheutschgi (chewing gum) also: Chaugummi # chiischtrig (to be hoarse)Chilbi (fun fair, annual fair)Chind (child, children)Chindergeburi (child's birthday)Chlapf (smack)Chleider azieh (to put on clothes) also: sich aalegechlöpfe (to bang) # Chlöpfer (a Cervelat, a typical Swiss sausage) # chlüübe (to pinch)Chnebel (large piece of wood)chneble (to gag someone)choche (to cook)Chochtopf (cooking pot)Chopf (head)Chopfhörer (head phone) # Chöpfler (dive with head first) also: Schpiessli # Chopfweh (headache) also: Grindweh # chrable (to crawl) # chräble (to scratch)Chrälleli (beads)chrampfe (to work hard) Chriesi (cherry, cherries)Chriesischtei (cherry stone)Chrippe (manger, day care center)Chrischtbaum (Christmas tree)Chrischtchindli (Christ child)Chrone (crown)Chrotepösche (dandelion) also: SoibluemeChrüsimüsi (Chaos) also: DurenandChuchi (kitchen)Chuchichäschtli (kitchen cupboard)Chuchischublade (cupboard drawer) Chue (cow)Chueche (cake1)Chueflade (cowpat)Chuehnagel (frostbite on finger or toes, lit. cow nail)Chugelschriiber (ballpen) also: ChugiChugi (ballpen) also: Chugelschriiber # Chugele (globe, ball) # Chugelibahn (marble run) # chugele (to roll)Chum gli hei! (come home soon!)Chündigung (letter of dismissal)Chunnsch mit mir go Znacht ässe? (Will you go out to dinner with me?)Chürbis - pumpkinchüschele (to whisper in someone's ear)chützele (to tickle)chützelig (ticklish)chuum (unlikely, improbable)Cou-Cousin (male second cousin)Cou-Cousine (female second cousin)Cousin (male cousin)Cousine (female cousin)Dd'Hose ahaa (to wear the pants, e.g. to be in charge)dinne (inside)dischtanziert (aloof) Donnschtig (Thursday)driigheie (to fall for sth, to fall into sth)Du bisch de Beschti! (you're the best (for men))Du bisch di Beschti! (you're the best (for women))Du gsehsch mega hübsch us. (you look so pretty)Du machsch mi glücklich (you make me happy)Dubel (idiot) also: IdiotDurenand (chaos) also: Chrüsimüsodurenand sii (to be messy, confused)dusse (outside)EEgoischt (egoist)eh (anyway)Eichörnli (squirrel)Eier färbe (to color eggs, e.g. by cooking them in dye or drawing on them)Eier sueche (to look for the eggs)Eiertätsch (egg bump, i.e. an Easter game involving eggs)eimal (once)Eimal Kaffi bittschön (One coffee please)Eltere (parents)En Guete! (enjoy your meal, bon appetit)En Huffe Gäld (a pile of money)En Schöne (have a nice day)En schöne Valentinstag (happy valentinesday)Erdnüssli (peanut, typical gift from Santa)Erläbnis (adventure, experience)erschte Auguscht (1st of August = Switzerland's national day)erstuunlich (astounding)Es fägt (it's fun) Es guets Nöis (happy new year)Es menschelet (there are interpersonal problems)Es schöns Wucheänd (have a nice weekend)Eus gohts guet (we're all right)FFähli (fur)Familie (family)farbig - colorfulFäscht (party)fasle (to maunder, to babble) # Faul (foul) # Feerie (holiday, vacation)Feschtplatte (hard disk)figuretle (busywork)Finke (slipper)Fisimatänte mache (make a fuss about sth. when it's not really necessary)Fitze (Santa's stick that he uses to hit bad behaving children) # Flangge (cross pass in soccer) # Flitterwuche (honey moon)Flughafe (airport)Flugi (airplane) also: FlugzüügFlugzüüg (airplane) also: FlugiFluug (flight)Fluugangscht (fear of flying)flüüge (to fly)Fluughafe (airport)fötele (to take photos)Föteli (photo) also: FotiFoti (photo) also: FöteliFötzeli (scrap of paper) # Fotzelschnitte (lit. slice of paper scraps, French Toast)Frau (woman, wife)Frauefurz (firecracker, lit. A woman's fart)Friitig (friday)Früehlig (spring)Fründ (boyfriend, friend)Fründin (girlfriend, friend)früsch (fresh)fruschtriert (frustrated)Fudi (butt) also: FüdliFüdli (butt) also: Fudifüdliblutt (buttnaked) Füess (feet) # Fuessball (football, soccer, also lit. the soccer ball) # Fuessball spiele (to play soccer) also: tschutte # Fuessballplatz (soccer field) also: Tschuttiplatz # füessle (to touch each others feet)fulänze (to relax, to do nothing)Furz (fart)Füür (fire)füürle (to make a fire, to make a bonfire)Füürwärk (fireworks)Gga blüemle (to go pick flowers)ga bügle (to go to work)Garracho (high speed)Gascht (guest)Gäscht (guests)Geburi (birthday)genialschtens (awesome)Gepäck (luggage, baggage)gfroore (frozen)gfrüüre (to freeze)gfürchig (creepy)gheie (to fall)ghöre (to hear)Gigampfi (seesaw)Gigetschi (apple core) also: Bütschgi, Groibschi # gigele (to chuckle, to giggle)Glattiis (black ice)gleitschirmflüge (paragliding)glette (to iron)Glöggliböögg (silly, annoying person)Glugsi (hiccup) also: HitzgiGlunge (puddle)glüschtle (to look with envie) Gmüetlichkeit (coziness)Gnüss es! (have fun!)gnüsse (enjoy sth)Gnusswuche (week of indugence)goisse (to shriek)Gomfi (jam) also: GonfiGonfi (jam) also: GomfiGoof (brat) # Gool (goal) # Gooli (goalie) # göötsche (to play with water)Gopfriedstutz (darn it)Göppel (old car, old vehicle, old bicycle) # Gorner (corner kick) # göötsche (to play with water) # gränne (to cry, to weep) also: greine, hüüle, briegge, brüelegrau (grey)Grind (head)Grittibänz (bread roll in the shape of a man)Gromi (grandma) also: Grosi, GrosmamiGropi (grandpa) also: Grospapi, GrosättiGrosätti (grandpa) also: Gropi, GrospapiGroschind (grandchild, grandchildren)Groseltere (grandparents)Grosi (grandma) also: Gromi, GrosmamiGrosmami (grandma) also: Gromi, GrosiGrospapi (grandpa) also: Gropi, GrosättiGrossmami (grandma)Grossmeitli (granddaughter)Grübschi (apple core) also: Bütschgi, GigetschiGrücht (rumor)Grüezi (formal hello) also: Griezi, Grüessech Grüüsch (sound)Gschänkli (gift, gifts))Gschwüschterti (sibblings)gsund (healthy)Gsundheit! (Bless you! (after sneezing))guet (good)Guet Nacht (good night)Guete Aabig (good evening)Guete Morge (good morning)Guete Tag (good day)Guetzli (cookie, cookies)Güfeli (pin, needle pin)Gummistiefel (rain boots)gumpe (to jump)günschtig (cheap, good prize)Güsel (trash)Güselchübel (trash can)Gutsch (sip or spill) # güxle (to peek, to look)Gwitter (thunderstorm)HHaar (hair)Haargummeli (elastic hair band)Haarschnitt (haircut)habere (to eat)Hackbrätt (zither)häimelig (cozy)Halbpension (half board)Hallo (hello)Hampfle (handful)Händsche (gloves, mittens)Häppi Börsdei! (Happy Birthday!) also: Alles Guete zum Geburi!Härdöpfel (potato)Härdöpfelstock (mashed potatoes)heimlifeiss (to be more than can be seen at first sight (in a good or bad way))Hemli (shirt)Herbscht (fall, autumn)Herbschtferie (fall vacation)hiilegge (to fork out)hindersi fahre (to drive in reverse)Hochzig (wedding)Hoi (informal hello)Hoi zäme (informal hello to a group)Hoigümper (grasshopper)Hose (pants)Hosesack (pocket)Hostel (hostel)Hotel (hotel)Hotelzimmer (hotel room)Hudigägeler (Swiss folk music)Hueschte (to cough, a cough)Hülle (casing, covering)Hund (dog)Hündli (puppy)Hundsverlochete (party in the hills)Hüppi (chignon) also: Bürzi, Pfürzihüüle (to weep) also: greine, gränne, brüele, brieggeHuus (hous)Huusfrau (housewife)Huustür (door (of a house))IIch be ned Schwizer. (I am not Swiss (male))Ich be ned Schwizerin. (I am not Swiss (female))Ich be nur chli am umeluege. (I'm just looking.)Ich bi voll. (I'm full, I ate enough)Ich ha Milch gärn (I like milk)Ich has kapiert (I got it, I understood) also: Ich has verstande, Ich has tscheggtIch has tscheggt (I got it, I understood) also: Ich has kapiert, Ich has verstandeIch has verstande (I got it, I understood) also: Ich has kapiert, Ich has tscheggtIch liebe dich (I love you) Ich mag nüm. (I've had enough)Ich vermiss di. (I miss you)ID (Swiss identity card)Iglu (igloo)Iibruch (burglary)Iichaufe (shopping)iigschneit (to be snowed in, stuck in a place because of snow fall)iipacke (to bag something)iipacke (to bag something) # Iis (ice) # Iis (ice)iischlöfernd (soporific)iischalt (ice cold)iischneebele (to cover someone with snow) # Iishockey (hockey) # Iisräge (ice rain)Iiszapfe (icicle)iiu (sure)in Usgang goh (to go out) # inegumpe (dive in, jump in) # Jjödele (to yodel) also: jodlejodle (to yodel) also: jödeleJoggel (clumsy person, strange person, funny person)jöggele (to play table football) also: töggeleJöggelichaschte (table football) also: TöggelichaschteJugi (youth hostel)jung (young)jungi Lüt (young people) # KKaff (hicktown)Kafi (coffee)Karfriitig (Good Friday) # Karsumpel (stuff, junk)Kater (hangover, male cat)kei Ahnig (no idea)Koffer (suitcase)Kolleeg (friend)Kroki (sketch of a terrain)Kuckucksuhr (cuckoo clock)Kunscht (art)Künschtler (artist)Kunschtsammlig (art collection)LLampe (lamp)Lämpe (brawl)langi Unterhose (long johns)Langwili (boredom)Lätzli (bib)lauffe (to walk) Lawabo (washbasin, sink) also: BrünneliLawine (avalanche)Lawinegfohr (avalanche danger)lehre (to study, to teach)lerne (to study)Liebe Gruess (kind regards) also: Liebi GrüessLiebi (love)Liebi Grüess (kind regards) also: Liebe GruessLiecht (light) Liechterchetti (chain of lights)Liechtli (little light)Liegestuehl (deck chair)links (left, the left side)lisme (to knit)Löli (fool, student driver)Los mir zue! (listen to me!)lose (to listen)luege (to look)Luft (air)Luftibus (wanton person)Lunch (picnic lunch)lüüchte (to shine, to glow)Lüüchtstift (highlighter)Luune (mood)luunisch (moody)lüüte (ring)Lüütispiili (ring and run)MMaa (man, husband) # Mägerlimucki (a skinny child)Mami (mom) also: Mueti, Muttimampfe (to eat)Mandarinli (mandarine)Mäntig (Monday)Marroni (edible chestnuts)Marronitschtand (Marroni booth)Marroniverchäufer (Marroni vendor)Marzipaneili (marzipan eggs)Masseentlassig (collective dismissal)Meer (sea, ocean)mega fein (delicious)Menü (menu)messi (thank you, from french "merci") also: dankemischi maschi (gallimaufry)mitspile (to play along)Mittag (noon)Mittwuch (Wednesday)Morge (morning)Moscht (apple juice)moschte (to make applesauce)motze (to complain, to baa) also: mötzelemötzele (to complain, to baa) also: motzemüed si (to be tired)Mueter (mother)Mueti (mom) also: MamiNNäbel (fog)Näbelmeer (sea of fog)näblig (foggy)Nacht (night)Nachzügler (straggler)Nami (afternoon) also: NamitagNamitag (afternoon) also: NamiNatel (mobile phone)Natuur (nature, outdoors)Neffe (nephew)neutral (neutral) Nichte (niece)nigelnagelneu (brand spanking new)nonig (not yet)notiere (to annotate)nüüsse  (to sneeze)Oöbbe (approximately, sometimes)öbber (someone)öbber bewundere (to admire sb)öbber speiche (to kick sb)öbberem Honig ums Muul schmire (to butter sb up)öbbis (something)öbbis bewundere (to admire sth)öbbis ufgäh (to jettison sth, to give up sth)Ohregrübler (earwig)Öpfel (apple, apples)Oschtere (easter)Oschtereili (easter egg)Oschterhaas (easter bunny)Oschterhäsli (little easter bunny)Oschtermäntig (Easter Monday, a national holiday)Oschternäschtli (Easter nest,little basket you put eggs and bunnies in)PPapi (dad)Partner (partner, boyfriend, girlfriend)Pass (passport)Pass uf dich uf! (take care!) Passaschier (passenger)Passkontrolle (passport control)Pauschalreis (package tour, all-inclusive vacation) # Pedale (pedal) # penne (to sleep)Pfnüsel (cold, runny nose)Pfüderi (little rascal)Pfürzi (chignon) also: Bürzi, HüppiPilz (mushroom, mushrooms) # Pnö (tire) # Pögg (puck) # Poppi (baby) also: Baby, Bebé, Buschi # Poscht (mail, the post office)poschte (to shop)Poschtiwägeli (shopping cart)Proscht! (cheers!) # Proviant (provisions) # Prüefig (exam)Publikum (audience)Puff (mess, brothel)Qquängele (to be querulous)quängelig (querulous)Quartier (neighborhood)quirlig (lively)quätsche (to squeeze) # RRäbe - turnip, beetrootRäbeliechtli (beetroot lantern)Räbeliechtliumzug (beetroot lantern walk)rächts (right, the right side)rächtzittig (on time)Rad (wheel)Räge (rain)Rägeboge (rainbow)Rägemantel (rain coat)Rägeschirm (umbrella)Rägetropfe (rain drops)Rahm (cream) also: Nidle # Ränzler: (lit. belly-er or paunch-er; when you jump off the diving board and land smack on your tummy)Rappespalter (penny pincher)Reis (trip, journey)reise (to travel, to take a trip)Reisebüro (travel agency)Reisefieber (wanderlust)Reisefüdli (someone who travels a lot)Reisefüehrer (travel book, guide book)Reiseziel (destination)reläxe (to relax)Reschti (restaurant)Riitseili (swing) also: RiittiRiitti (swing) also: Riitseili # ring (easy, easily)Rootwii (red wine)Röschti (Swiss potato pancake)Röschtigrabe (imaginary line that separates the French and German speaking parts of Switzerland)Ross (horse)Rössli (small horse)Rossöpfel (road apple)Rucksack (backpack)Rüebli (carrot)Rüeblitorte (carrot cake)rüere (to stir, to throw)Rüerei (scrambled egg) # Rutschbahn (slide) # Ruum (room)Rüüm (rooms)SSAC Hütte (cottage of the Swiss Alpine Club)Sack (bag)säiche (to rain, to piss)Samichlaus (Santa Claus)Samschtig (Saturday)sändele (to play with sand)Schaal (scarf)schaffe (to work) also: chrampfeSchale (suit, coffee with milk, peel)Schiff (boat, ship)schifffahre (to ride a boat, to ride a ship) # Schii (ski) # Schiifahre (to ski) # Schiilift (ski lift) # Schiri (referee) # Schiss (fear) # Schisshaas (scaredy cat, lit. fear rabbit)Schläckzüg (sweets, candy) also: Zeltli, Täfeli # Schläger (hockey stick) # Schlamassel (mess)Schlirg (scrawl)Schlitte (sledge) # Schlittschueh (skate, ice skate) # schlittschüehle (to ice skate) # Schluck (sip)Schlückli (a little sip)schmöcke (to smell) schmuggle (to bootleg, to smuggle)Schmutzli (lit: dirty one, Santa's helper)schmuuse (to spoon with sb, to canoodle)schnaagge (to crawl)Schnee (snow)Schneeball (snow ball)Schneeball-Schlacht (snow ball fight)schneebele (to play in the snow)Schneeflocke (snowflake)Schneeflöckli (small snowflake)Schneeflöckli (snowflake)Schneefrau (snow woman)Schneemaa (snowman)Schneeschturm (snow storm)schneie (to snow)schnorchle (to snorkle) # Schnudderlumpe (lit. snot cloth, handkerchief)Schnuddernaase (running nose)Schnüggel (cuty)schnütze (to blow one's nose)schockierend (shocking)Schöggeler (someone who has an easy life)Schöggeli (small chocolates)Schoggi (chocolate)Schoggichueche (chocolate cake)Schoggihaas (chocolate bunny)Schoggischtängel (chocolate stick)Schöni Ferie (happy holidays)Schöni Wiehnacht (marry Xmas)Schoppe (baby milk bottle) # Schpelsache (toys, lit. playing things) # Schpiessli (dive with head first) also: Chöpfler # Schpinnhupele (cobweb)Schpoiz-Chnebel (recorder, flute)Schpriisse (splitter, splinter)schpringe (to run) also: seckle # Schprungbrätt (diving board) # Schprütze (injection, syringe)schprütze (to slosh, to besplatter)schriibe (to write)Schtäge (stairs) Schtange (bar, half a liter of beer)Schtärn (star) Schtärnli (little star)Schtell (job, place)Schtelleazeig (job advert)Schtern (star)Schtern vo Bethlehem (Chistmas star)schtier (broke, flat broke)Schtift (a pen or an apprentice)Schtraass (street)Schtrand (beach)Schtrasselampe (street lamp)schtrub (weird) Schtube (living room, sitting room)Schtuck (piece)Schtückli (morsel)schtudiere (to reflect or to study)Schtutz (Swiss Franc or a steep ascent) schüch (timid)Schuel (school)Schuelig (training)Schuum (foam)Schwager (brother-in-law)schwänze (to play hooky)Schwiegereltere (parents-in-law)Schwiegermueter (mother-in-law)Schwiegervatter (father-in-law)Schwiiz (Switzerland)Schwiizerdütsch (Swiss German)Schwö (sis)Schwögerin (sister-in-law)Schwöschter (sister)Schwöschterli (little sister)Schwümmbi (public pool) also: Badi # Schwümmbrüle (goggles) # schwümme (to swim)seckle (to run) also: schpringeSeegfrörni (freezing of a lake)Sehenswürdigkeit (tourist attraction)singe (to sing)Sitzig (meeting)Soiblueme (dandelion) also: ChrotepöscheSouvenir (souvenir) # Spazifizottel (stroll, walk)Stadtplan (city map)Stadtrundfahrt (sightseeing tour in a city)Ströfzgi (extra exercise)Summer (summer)Summerfeerie (summer vacation)sumpfig (boggy)Sunne (sun)Sunnebrand (sunburn)Sunnegräm (sunscreen)sünnele (to sun tan)Sunneschirm (sunshade)Sunntig (Sunday)süüferli (gradually, carefully)Suuser (young wine)Ttäderle (to sneak, to squeal, to tell on someone)Täderlisack (someone who tells on someone)Täfeli (candy, sweets) also: Zeltlitanke (to refuel)Tannebaum (fir tree)Tante (aunt)Täsche (bag)täubele (to outdare)Tee (tea)Teebüteli (tea bag)telefoniere (to make a phone call)tifig (quickly)töggele (to play table football) also: jöggeleTöggelichaschte (table football) also: Jöggelichaschtetönt guet (sounds good)tööne (to sound) # trampe (to pedal) # Truube (grape, grapes)Trüübeli (small grapes)tschalpe (to walk)tschättere (to judder)Tschüss (bye [to a friend])Tschüss zämme (bye all)Tschüssli (cheerio, bye)tschutte (to boot, to play soccer) # tschutte (to play soccer) also: Fuessball spiele # tschüttele (to play with a soccer ball, easy going soccer playing) ================================================ FILE: datasets/english/festivalLove-eng.ipynb ================================================ *I dem riise menschemeer* *look at you as if it were deserted* *And while crowd freaks out around you* *You only relaxed din daiquiri sippsch* Festival, festival love When I'm away from you *nüme* sleep *wants jedi summer night will there be duregmacht* festival festival love I chan you *ned always ha* *It doesn't matter if it's dirt and *mud* or sunne am* *Stürchled* zeme for camping hi *Hend the cup filled with cheap wii* *Pull me in the trekking tent* *The sweat is beading on the inside* Festival, festival love When I'm away from you, sleep *wants jedi summer night will there be duregmacht* festival festival love Chan I need you always ha It doesn't matter *öb* dirt and mud or sunne am festival festival love And you *laugh* me ah *My heart stays* My heart stays *stah* And want *nüm gah* how you *move* me if you when you *move* Festival, festival love When I'm away from you *nüme* sleep Jedi wants summer night there *duregmacht* festival festival love I chan you *no* always ha *No matter if dirt and mud or sunne am* festival festival love ================================================ FILE: datasets/french/french.txt ================================================ - Bonjour. - Bonjour. Je m'appelle Martin. Et vous ? - Je m'appelle Sarah. - Enchanté. ================================================ FILE: datasets/img/map.txt ================================================ http://dialektkarten.ch/trans/index.html ================================================ FILE: datasets/korean/festivalLovekr.ipynb ================================================ *I dem riise menschmeer* *버려진 널 바라봐* *그리고 군중이 당신 주위를 놀라게 하는 동안* *당신은 편안한 din daiquiri sippsch* 축제, 축제의 사랑 내가 당신에게서 떨어져 있을 때 *nüme* 잠 *제다이 여름 밤을 원합니다. duregmacht가 있을 것입니다* 축제 축제 사랑 I chan you *ned always ha* *흙이든 *진흙*이든 햇빛이든 상관없어요* *Stürchled* zeme for camping 안녕하세요. *싸구려 wii로 가득 찬 잔을 들어라* *트레킹 텐트 안으로 끌어당겨 주세요* *땀이 속에서 흥건하다* 축제, 축제의 사랑 내가 당신에게서 떨어져있을 때, 잠 *제다이 여름 밤을 원합니다. duregmacht가 있을 것입니다* 축제 축제 사랑 찬 항상 니가 필요해 ha 그것은 중요하지 않습니다 *öb* 먼지와 진흙 또는 sunne am 축제 축제 사랑 그리고 당신은 *웃음* 나를 아 *내 마음은 머물다* 내 마음은 *stah* 그리고 원해 *num gah* 당신이 나를 *움직이는* 방법 만약 너라면 당신이 *이동*할 때 축제, 축제의 사랑 내가 당신에게서 떨어져 있을 때 *nüme* 잠 제다이는 그곳에서 여름밤을 원한다 *duregmacht* 축제 축제 사랑 I chan you *no* 항상 ha *먼지나 진흙이나 햇빛이 있더라도* 축제 축제 사랑 ================================================ FILE: datasets/korean/high-kr.py ================================================ 나 취 했어 나 취 했어 아 *땀*이 난다 내가 *uh 금단*인 것처럼 *자이트레의 작은 손* *비메네 급강하처럼* 난 *그냥 코 풀고* 잠시 *진정* *Chum 나에게 당신에게서 기차를 줘* 그리고 난 다시 기분이 좋아 나 취 했어 너와 함께 기분이 너무 좋아 난 너무 *ghypet* 너와 함께 기분이 좋아 내가 밤새 널 깨뜨린다는 걸 알잖아 나를 그렇게 중독되게 만들고 싶니? 너무 높다 나 취 했어 *최소 심박수 경주* 너무 많은 아드레날린 *오픈차트 게임* 나는 엔돌핀을 끊는다 난 너와만 느껴 *내가 모든 것을 찬 것처럼* *당신에게로 가게 해주세요* *곧 다시 할 수 있을 거예요* 나 취 했어 너와 함께 기분이 너무 좋아 난 너무 흥분해 너와 함께 기분이 좋아 내가 밤새 널 깨뜨린다는 걸 알잖아 나를 그렇게 중독되게 만들고 싶니? 너무 높다 나 취 했어 높은 나 취 했어 나 취 했어 너와 함께 기분이 너무 좋아 난 너무 흥분해 너와 함께 기분이 좋아 내가 밤새 널 깨뜨린다는 걸 알잖아 나를 그렇게 중독되게 만들고 싶니? 너무 높다 나 취 했어 ================================================ FILE: datasets/korean/malLuege.py ================================================ 그것이 그녀가 의미하는 것입니다. 가장 아름다운 바 큰 꿀꺽 뮤트 그리고 그녀에게 물어볼게 "어떻게 누가, ois 2와 함께 진정해, 둘 다 일치하는 것은 친절합니다 내꺼였어?" 가끔 그녀는 거짓말을 해, 아마 내일 아니면 뭔가 때때로 그녀는 거짓말을 하고 난 uf and devo 가끔 그녀는 거짓말을 해, 아마 내일 아니면 뭔가 그리고 그녀는 전혀 필요하지 않습니다 당신은 나를 참조 Uf 틴더가 다시 잡는다 사진을 확인하세요 게임하려면 오른쪽으로 스와이프하세요. 나는 "당신은 모른다. 빈 프리티그 이데 바 헤쉬 당신은 gha를 의미합니까 나는 선택했다 당신은 좋은 당신은 멋지다 진정해, 둘 다 너와 나 유리 Wii 좁은 동사의 Chönted 세그가 내꺼였어?" 가끔 그녀는 거짓말을 해, 아마 내일 아니면 뭔가 때때로 그녀는 거짓말을 하고 난 uf and devo 가끔 그녀는 거짓말을 해, 아마 내일 아니면 뭔가 그리고 그녀는 전혀 필요하지 않습니다 대학에서 또 다른 플랫 쉐어 파티 z'seebach 오 젠장, 당신에게 유용합니다 그녀는 기꺼이, 그리고 z'luut가 많이 있습니다 Chum gömer use, 내게는 랑데뷰처럼 들리네 D'wimpere klimperet, 그녀는 내가 무슨 뜻인지 묻습니다. 그리고 난 정말 일시적인 전환이 없어 가끔 난 거짓말을 해, 아마 내일 아니면 뭔가 난 거짓말을 할 때 거짓말을 하고 난 uf와 devo 가끔 난 거짓말을 해, 아마 내일 아니면 뭔가 그리고 나는 그것을 전혀 필요로하지 않습니다 Det staht sie Die schönst ade bar En grosse schluck muet Und denn frög ich sie mal "Wie wers, mit ois zwei Mal chille nur mir beid Macheds ois gmüetlich Was meinsch?" Mal luege het sie gseit, villicht morn oder so Mal luege het sie gseit und isch uf und devo Mal luege het sie gseit, villicht morn oder so Und siither isch vo ihre gar nüt meh cho Sie het mich Uf tinder wieder catcht Han d'bilder uscheckt Swipe nach rechts für de match Ich schrieb "weisch du no Am fritig ide bar Do hesch du gmeint gha Mir chönted mal Du bisch nice Du bisch geil Mal chille nur mir beid Du und ich Es glas wii Chönted im schmale verbi Seg was meinsch?" Mal luege het sie gseit, villicht morn oder so Mal luege het sie gseit und isch uf und devo Mal luege het sie gseit, villicht morn oder so Und siither isch vo ihre gar nüt meh cho Anere wg party z'seebach bim kolleg Ohni scheiss, usem nüt staht sie au det Sie seg so glangwillt, und da ine segs viel z'luut Chum gömer use, tönt für mich nach rendez-vous D'wimpere klimperet, sie frögt ja was ich mein? Und ich han würklich kein momentlang überleit Mal luege han ich gseit, villicht morn oder so Mal luege han ich gseit und bin uf und devo Mal luege han ich gseit, villicht morn oder so Und siither isch vo mir gar nüt meh cho That's what she stands for The nicest ade bar A big gulp muet And then I'll ask her "How who, with ois two Just chill, both of me Macheds ois cozy What do you mean?" Sometimes she lies, maybe tomorrow or something Sometimes she lies and I uf and devo Sometimes she lies, maybe tomorrow or something And there is no need for her at all she hates me Uf tinder catches again Check out the pictures Swipe right for the match I wrote "you don't know At the fritig ide bar Do you mean gha I chönted you are nice you are horny Just chill, both of me You and me It glass wii Chönted in the narrow verb Say what do you mean?" Sometimes she lies, maybe tomorrow or something Sometimes she lies and I uf and devo Sometimes she lies, maybe tomorrow or something And there is no need for her at all Another flat share party z'seebach at college Oh no shit, it's useful for you She's so willing, and there's a lot of z'luut Chum gömer use, sounds like a rendez-vous to me D'wimpere klimperet, she asks what I mean? And I really don't have a momentary transition Sometimes I'm lying, maybe tomorrow or something I'm lying when I'm lying and I'm uf and devo Sometimes I'm lying, maybe tomorrow or something And then I don't need it at all ================================================ FILE: datasets/luzern/luzern.txt ================================================ luzerndütsch los gahts ei biuspiel Los gohts ================================================ FILE: datasets/zuerich/BlueBike.txt ================================================ Letschte summer hesch dis velo kauft Han denkt *seg* eige, so ganz in blau Hinderem bahnhof hesch din *stammplatz* gha Wenni mis velo *abgschlüss* vo tag zu tag Denn hani *Ade* velostation *amel* chli länger Vode lüüt hinder mir *gspüri* scho de *ärger* Jedes mal wenn ich im zug sitz, sueched mini auge dis blaue bike Jedes mal wenn ich im zug sitz, sueched mini auge dis blaue bike Ich würs so gern gseh, ich *gäbt* so viel meh Blaui velos wie sand am meer Jedes mal wenn du im zug bisch, sueched dini auge mis blaue bike Suech die ganz stadt nach dim velo ab A jedem *ecke*, ich fahre bis spat id nacht Früehner hemer das no zeme gmacht Jetz wenni mis velo hol, hoffi ich lauf i dich *dri* Denn hani Ade velostation *amel* chli länger Vode *lüüt* hinder mir gspüri *scho* de ärger Jedes mal wenn ich im zug sitz, *sueched* mini auge dis blaue bike Jedes mal wenn ich im zug sitz, sueched mini auge dis blaue bike Ich würs so gern gseh, ich *gäbt* so viel meh Blaui velos wie sand am meer Jedes mal wenn du im zug bisch, sueched dini auge mis blaue bike Ich sitze uf mim bike und ich fahre dur d'nacht Fahre, fahre dur d'nacht, ganz allei dur d'kulturstadt Ich sitze uf mim bike und ich fahre dur d'nacht Fahre, fahre dur d'nacht, ganz allei dur d'kulturstadt Ich sitze uf mim bike und ich fahre dur d'nacht Fahre, fahre dur d'nacht, ganz allei dur d'kulturstadt Ich sitze uf mim bike und ich fahre dur d'nacht Fahre, fahre dur d'nacht, ganz allei dur d'kulturstadt Ich sitze uf mim bike und ich fahre dur d'nacht Fahre, fahre dur d'nacht, ganz allei dur d'kulturstadt Ich sitze uf mim bike und ich fahre dur d'nacht Fahre, fahre dur d'nacht, ganz allei dur d'kulturstadt Ich sitze uf mim bike und ich fahre dur d'nacht Fahre, fahre dur d'nacht, ganz allei dur d'kulturstadt Ich sitze uf mim bike und ich fahre dur d'nacht Fahre, fahre dur d'nacht, ganz allei dur d'kulturstadt ================================================ FILE: datasets/zuerich/FestivalLove.ipynb ================================================ *I dem riise menschemeer* *Gsehn nur dich als wärs menscheleer* *Und während crowd rund um dich usflippt* *Du nur relaxt din daiquiri sippsch* Festival, festival love Wenn ich weg dir *nüme* schlaf *Will jedi summernacht wird da duregmacht* Festival festival love Chan ich dich *nöd immer ha* *Egal öb dreck und *schlamm* oder sunne am* *Stürchled* zeme zum camping hi *Hend d’becher gfüllt mit billigwii* *Ziehsch mich i dis trekkingzelt* *Es perlt de schweiss vo de innewänd* Festival, festival love Wenn ich weg dir nüme schlaf *Will jedi summernacht wird da duregmacht* Festival festival love Chan ich dich nöd immer ha Egal *öb* dreck und schlamm oder sunne am Festival festival love Und du *lachsch* mich ah *Mis herz bliibt* Mis herz bliibt *stah* Und will *nüm gah* Wie du mich *bewegsch* Wenn du dich Wenn du dich *bewegsch* Festival, festival love Wenn ich weg dir *nüme* schlaf Will jedi summernacht wird da *duregmacht* Festival festival love Chan ich dich *nöd* immer ha *Egal öb dreck und schlamm oder sunne am* Festival festival love ================================================ FILE: datasets/zuerich/JohnJuno.ipynb ================================================ Ich *hanen* *letschti* ja no gseh gha So richtig *erchennt* het er mich aber nöd *Hüfig* het er vom mars *verzellt* gha Dass er irgendwenn de *erst wird si und mir werded scho gseh* John Juno here Und jedes mal wenn ich in *himmel ufe lueg* Weiss ich er isch no da Er *triibt ganz allei durs weltall* Irgendwo in outer space Er isch en einsame astronaut John Juno can you hear me? Ja *ghörsch* du, ja *ghörsch* du mich? *Er isch en planet ohni mond gsi* *Z'nöch ade sunne, glich ohni strom gsi* *Liechtjahr hend ois trennt gha* So viel sterne, so viel liechter *Aber glich het de himmel über ois no gar kei farb* John Juno here *Und jedes mal wenn ich in himmel ufe lueg* Weiss ich er isch no da *Er triibt ganz allei durs weltall* Irgendwo in outer space Er isch en einsame astronaut John Juno can you hear me? Ja ghörsch du, ja ghörsch du mich? Und jedes mal wenn ich *ufe* lueg Weiss ich er isch no da und *triibt* durs weltall Det isch kei *chraft* meh wo in *zruggzieht* Kein gschmack meh wo im d'lust *verdirbt* Er chan eifach echli si, ganz allei i *sinre* galaxie Und jedes mal wenn ich in himmel ufe lueg Weiss ich er isch no da *Er triibt ganz allei durs weltall* Irgendwo in outer space Er isch en einsame astronaut En einsame, einsame astronaut Er isch en einsame astronaut ================================================ FILE: datasets/zuerich/Sie het en traum gha.txt ================================================ Sie het en traum gha Und de traum het nöd *nah glah* Sie het immer de wunsch gha Dass es *allne mal glich gaht* Oh sie het en traum gha De ganzi weg isch sie *aleige gsi* Nöd gnau gwüsst wos *dure gaht*, nur ihres smartphone debi Dur *d'wüesti* schritt für schritt D'bei mached doch scho lang lang nüme mit Sie wanderet barfuess dur de schnee Und *triibt ufme floss über d'welle vom mittelmeer* Sie het en traum gha Und de traum het *nöd nah glah* Sie het immer de wunsch gha Dass es allne mal glich gaht Sie het en traum gha Und de traum het nöd nah glah Es *müesst so viel andersch gha* Dass es allne mal glich gaht Sie het en traum gha Sie het en traum gha Und de traum het *nöd nah glah* Sie het immer de wunsch gha Dass es allne mal glich gaht Sie het en traum gha Trotz wind und welle, *het sies da ane gschafft* *Mir gebed alles, gebed ihre s'gfühl dass sie nöd da ane passt* Sie kämpft glich wiiter tag für tag *Uf wer chan sie sich scho ve*rlah?* *Nur will sie d'sprach da nöd verstaht* *Heisst das nöd, dass sie nöd gspürt, was du uselahsch* Sie het en traum gha Und de traum het nöd nah glah Sie het immer de wunsch gha Dass es allne mal glich gaht Sie het en traum gha Und de traum het nöd nah glah *Es müesst so viel andersch gha* Dass es allne mal glich gaht *Sie git nöd uuf* Sie het es ziel *Git alles uf* Für das was sie will Sie git nöd uuf Sie kämpft fürs ziel Git defür alles uf Für es neus menschebild Sie het en traum gha Und de traum het nöd nah glah Sie het immer de wunsch gha Dass es *allne* mal glich gaht Sie het en traum gha Und de traum het nöd nah glah Es *müesst* so viel andersch gha *Dass es allne mal glich gaht* ================================================ FILE: datasets/zuerich/Teil_Vo_Dir.ipynb ================================================ *Ich wär so gern en Teil vo dir gsi* *Stell di ufs Podescht du bisch min Mittelpunkt* *Ich find bi dir nid statt ich bliib im Hintergund* *Was chan ich geh was du no nid hesch* Bin nur en Punkt i dim Kalender *De Mond rotiert umd Erde will sie zieht ihn ah* *Du schriibsch nur ide Nacht du gsehsch mi nid am Tag* Wieso *schwümm* ich no *geg de Strom* Und *sie seit*: oh es isch verbii Und *alles womer bliibt egal wie chli ich bhalt das bi mir* *Zwüschet eus uf eimal Liechtjahr* Eigentlich scho immer *ich hans nur nid gseh* Und wär au nomal uf eus beidi *inegheit* *Obwohl das Schiff scho länger sinkt* *Und dänn luegi nomal hindre* *Gseh din Punkt verschwinde det am Horizont* *Ghei us allne Wulche s isch en schöne Flug* *Schwäbe* dur das *Abigrot* Und sie seit oh es isch verbii Und sie seit oh es isch verbii Und alles wo mer bliibt isch eigentlich au gar nid so *chlii* ================================================ FILE: datasets/zuerich/fremdi_fürenand.ipynb ================================================ *Gäbts e ziitmaschine ich würd sie neh* *Mach denn d'türe uf* im 2010 *D'hitz stauts ide altstadthüüserschlucht* Und mich *ziehts wieder zum salzhuus hi* Und denn fangt alles *namal* vo vorne ah Chönt ich nur namal *zrugg zum afang gha* Zum erste mal *Ide summerbar* *Alles uf afang* *Wüssed nüt vonenand* Zum zweite mal *Gits en neuafang* *Denn hebts es lebelang* *Wüssed nüt vonenand* Mir sind fremdi fürenand *Nach 4 jahr gymi simmer direkt ad uni* Zeme ine wohnig zoge, hend alles teilt Für de job bin ich uf züri und du uf bern Gäbts e ziitmaschine ich würd sie neh *Will denn fangt alles namal vo vorne ah* *Chönt ich nur namal zrugg zum afang gha* Zum erste mal Ide summerbar Alles uf afang *Wüssed nüt vonenand* Zum zweite mal *Gits en neuafang* *Denn hebts es lebelang* *Wüssed nüt vonenand* Mir sind *fremdi fürenand* Mir sind *fremdi fürenand* *Aber villicht ischs au guet wies gsi isch* Und besser wenn ich dich *dasmal gar nöd triff* Und alles *wieder vergiss* Und alles wieder vergiss Und alles wieder vergiss Zum erste mal Ide summerbar *Alles uf afang* *Wüssed nüt vonenand* Zum zweite mal *Gits en neuafang* *Denn hebts es lebelang* *Wüssed nüt vonenand* *Mir sind fremdi fürenand* Mir sind fremdi fürenand Mir sind fremdi fürenand ================================================ FILE: datasets/zuerich/high.py ================================================ Ich bin so high Ich bin so high Ich fang ah *schwitze* Als wär ich *uf entzug* *Mini händ am ziitre* *Wie bimene sturzflug* Ich *bruch nur e nase voll* Für en augeblick *rueh* *Chum gib mer en zug vo dir* Und ich fühl mich wieder guet Ich bin so high Fühl mich so high mit dir Ich bin so *ghypet* Fühl mich so ghypet mit dir Du weisch ich bruch dich die ganzi nacht Will du mich so süchtig machsch So high Ich bin so high *Min puls am rase* So viel adrenalin *Spiel mit offne charte* Ich bruche endorphin Ich han nur mit dir das gfühl *Als ob ich alles chan* *Mach mich uf de weg zu dir* *Dass ich das bald wieder han* Ich bin so high Fühl mich so high mit dir Ich bin so ghypet Fühl mich so ghypet mit dir Du weisch ich bruch dich die ganzi nacht Will du mich so süchtig machsch So high Ich bin so high High Ich bin so high Ich bin so high Fühl mich so high mit dir Ich bin so ghypet Fühl mich so ghypet mit dir Du weisch ich bruch dich die ganzi nacht Will du mich so süchtig machsch So high Ich bin so high # korean: https://github.com/esthicodes/Awesome-Swiss-German/blob/main/datasets/korean/high-kr.py ================================================ FILE: datasets/zuerich/i_ha's_immer_gwüsst.ipynb ================================================ *Scho woni di vo jahre zerst mau gseh ha hesch mi iritiert.* *Ha nid genau gwüsst wases isch wo mi a dir het fasziniert.* *Mir hei üs verlore u immer wider gfunge.* *Wöu ds schiksau hed das für üs bestumme.* Und i has immer gwüsst, dass du *fr mi ganz öppis bsungers bisch.* *We das fr di ou stimmt de legi d charte hüt no ufe tisch.* *Und i has immer gwüsst, i has immer gwüsst ds du das bisch.* *D stärne säge dases ganz u gar unmöglech isch* *Doch sogar d stärne chöisech tüsche, wenn du mir di chance gisch.* *Mit dir cha mi ewigi suechi es ändi ha, *flüg* midmer weg u la üs nöi ahfah.* Und i has immer gwüsst, dass du fr mi ganz *öppis bsungers bisch.* We das *fr di ou stimmt de legi d charte hüt no ufe tisch.* *Chum es isch nie maus z spät fürne nöie weg mir chöi nä ga.* *Viu besser jetz aus nie mir chöi dem nüm entflie üsi zyt isch da.* *Und i gloube dra* Und i has immer gwüsst, i has immer gwüsst, dass du das bisch Na na na nanana na na na naa Chum i has immer gwüsst, dass du fr mi ganz öppis bsungers bisch. We das fr di ou stimmt de legi d charte hüt no ufe tisch. Chum es isch nie maus z spät fürne nöie weg mir chöi nä ga. Viu besser jetz aus nie mir chöin dem nüm entflie üsi zyt isch da. Und i gloube dra Und i has immer gwüsst, dass du das bisch ================================================ FILE: datasets/zuerich/lena.ipynb ================================================ Lueg mal s'füürwerk, *du heschs nöd gern* Für dich machts, eifach *nume lärm* Scho wieder de *gin usgleert*, s halbe glas *verteilt* ufs chleid Sie hend ja scho *abgruumt*, chum mir *schaukled* zeme hei Lena, ich wart *ade kronestrass ufs tram* Oh Lena, *wär lieber bliibe tagelang* Oh Lena, dini blaue auge sind mer z'viel Wil lena, ich *verlür* mich immer drin I dim zimmer, *schmöckschs* mittelmeer *Sither* immer, wenn ich dich gseh Wett ich wieder i dis zimmer *zrugg*, din herzschlag a mim ohr Au wenn du über dini *witz* lachsch, fiir en glich din humor Lena, ich wart ade *kronestrass* ufs tram Oh Lena, wär lieber bliibe tagelang Oh Lena, dini blaue auge sind mer z'viel Will Lena, ich *verlür* mich immer drin Sie kenned mich, kenned mich, bi dir im quartier Jede weiss genau, weiss genau will ich bin so viel bi dir Sie kenned mich, kenned mich, bi dir im quartier Jede weiss genau, weiss genau will ich bin so viel bi dir Oh Lena Lena, ich wart ade kronestrass ufs tram Oh Lena, wär lieber bliibe tagelang Oh Lena, dini blaue auge sind mer z'viel Will Lena, ich verlür mich immer drin Oh Lena, ich wart ade kronestrass ufs tram Oh Lena, wär lieber bliibe tagelang Oh Lena, dini blaue auge sind mer z'viel Will Lena, ich verlür mich immer drin ================================================ FILE: datasets/zuerich/malleuge.txt ================================================ Det staht sie Die schönst ade bar En grosse schluck muet Und denn frög ich sie mal "Wie wers, mit ois zwei Mal chille nur mir beid Macheds ois gmüetlich Was meinsch?" Mal luege het sie gseit, villicht morn oder so Mal luege het sie gseit und isch uf und devo Mal luege het sie gseit, villicht morn oder so Und siither isch vo ihre gar nüt meh cho Sie het mich Uf tinder wieder catcht Han d'bilder uscheckt Swipe nach rechts für de match Ich schrieb "weisch du no Am fritig ide bar Do hesch du gmeint gha Mir chönted mal Du bisch nice Du bisch geil Mal chille nur mir beid Du und ich Es glas wii Chönted im schmale verbi Seg was meinsch?" Mal luege het sie gseit, villicht morn oder so Mal luege het sie gseit und isch uf und devo Mal luege het sie gseit, villicht morn oder so Und siither isch vo ihre gar nüt meh cho Anere wg party z'seebach bim kolleg Ohni scheiss, usem nüt staht sie au det Sie seg so glangwillt, und da ine segs viel z'luut Chum gömer use, tönt für mich nach rendez-vous D'wimpere klimperet, sie frögt ja was ich mein? Und ich han würklich kein momentlang überleit Mal luege han ich gseit, villicht morn oder so Mal luege han ich gseit und bin uf und devo Mal luege han ich gseit, villicht morn oder so Und siither isch vo mir gar nüt meh cho ================================================ FILE: datasets/zuerich/panama.txt ================================================ Es bitzli träume dörf mer doch immer Dass es andersch isch, wie wers mit eme lebe ufere insle? Mängisch hani eifach vo dere dumme schnurre gnueg Denn denki schön wärs, wärs chli ruhig Chum mir gönd uf panama Und fanged vo vorne ah Oh wie schön isch panama Für ois bliibt d'welt stah Mach ein schritt füre und zrugg Eimal z'viel isch de chopf im sand gsi Immer uf de zehe stah, het mit de ziit au nöd so guet tha Mängisch wetti eifach das chlöne nöd ghöre Pack de rucksack und ich bin weg da Chum mir gönd uf panama Und fanged vo vorne ah Oh wie schön isch panama Für ois bliibt d'welt stah Mach ein schritt füre und zrugg Din riise lätsch isch mir denn so egal En caipirinha schlürfe Will ich bin furt, weg, nüme da Du wirsch vom bus agfahre Dir hends s'velo klaut Du hesch din stutz verzockt Isch mir egal, will ich bin furt, weg, nüme da Chum mir gönd uf panama Fanged es neus lebe ah Oh wie schön isch panama Für ois bliibt d'welt stah Chum mir gönd uf panama Und fanged vo vorne ah Oh wie schön isch panama Für ois bliibt d'welt stah Mach ein schritt füre und zrugg ================================================ FILE: datasets/zuerich/teleskop.py ================================================ Ich lueg a *decki voller neonplastiksterne* S′fenster off, s einzig wo *chüehlt i dere summerwärmi* *S'strasseliecht vo duss wirft schattespiel ad zimmerwänd* Ein *gedankekreis*, *ich wüsst gern öbd au a mich denksch* Kei chance zum schlafe *Wo triebsch dich umenand?* Han gnueg vom warte *Hend viel z′viel abstand vonenand* I mim zimmer, *a mim fenster staht es teleskop vo dir* Lueg ich use, usem fenster, wünscht ich du *wärsch bi mir* *Du fehlsch da, suech überall, zoom mit em teleskop durs all* *I mim zimmer, a mim fenster staht es teleskop vo dir* *Uf mim iphone sind no alli *oisi* bilder* D'welt het *gstrahlet* dur de *rosaroti* filter S'wird nüt mit schlafe Min screen hilft dur d′nacht Han gnueg vom warte Seg mer wenn, *wenn hanis gschafft*? I mim zimmer, a mim fenster *staht* es teleskop vo dir Lueg ich use, usem fenster, *wünscht* ich du wärsch bi mir Du fehlsch da, *suech überall*, zoom mit em teleskop durs all I mim zimmer, a mim fenster staht es teleskop vo dir *Ich lueg a decki voller neonplastiksterne* *Irgendwo det uss starrsch du au id ferni* Mini auge *kämpfed* Langsam *werdeds* schwer Ich het mir *gwünscht* gha Dass du bi mir blibe *wärsch* ================================================ FILE: datasets/zuerich/valencia.ipynb ================================================ *Mir sitzed duss vorem paddys Und du rauchsch und du rauchsch A dir isch immer no alles So vertraut, so vertraut* *Aber wenn ich jetzt ehrlich bin* *Gits da ei sach, won ich dir nie gseit han* Ich han immer nume nur a dich denkt Bin dir *pauselos nahgrennt* *Ich han immer nume dich im chopf gha* Siit do in valencia Ich han immer *nume* dich im chopf gha Hend *s’vorstadtlebe* scho zeme *Duregmacht, duregmacht* *Sind dur dick und dünn* *Hend viel erlebt was ois weh macht* Scheiss egal, hends *wegglacht* *Oisi weg hend sich viel z’früеh trennt* Bevor ich, bevor ich dir gsеit han Ich han immer nume nur a dich denkt Bin dir pauselos *nahgrennt* Ich han immer nume dich im chopf gha Siit do in valencia Ich han immer nume dich im chopf gha Siit do in valencia Ich weiss no bis zu dem moment Wo mir *z’spanie* gha hend Han ich i dir nüt gseh nei Genau wie *umgekehrt* au Doch det isch es passiert I mim chopf hets plötzlich *klirrt* S’het sich alles *dreht* und D’welt staht plötzlich *schräg* und Ich han immer nume nur a dich denkt Bin dir pauselos *nahgrennt* Ich han immer nume dich im chopf gha Siit do in valencia Ich han immer nume dich im chopf gha Ich han immer nume nur a dich denkt Bin dir pauselos nahgrennt Ich han immer nume dich im chopf gha Siit do in valencia Ich han immer nume dich im chopf gha ================================================ FILE: datasets/zuerich/zuerich.txt ================================================ das Haus -> sHuus die Strasse -> dStrass das Fahrad -> sVelo das Auto -> sAuto das Mülleimer -> der Chübel (Abfalleimer) der Gehweg -> sTrottuar (pavement) Wie gehts? -> Wie gahts? Ich rede Hochdetusch -> Ich rede Schwiizerdütsch. Wohin gehst du? -> Wohi gasch? Ich gehe nach unten. -> Ich gang abä. Ich nehme den Zug. -> Ich gang uf de Zug. Ruf mich an, wenn du angekommen bist. -> Lüt mir a, wennd acho bisch. Hast du dein Portmonee dabei? -> Häsch du dis Portmonee däbi? Ich muss zur Arbeit. -> Ich muess go schaffä. Ich habe heute ein Brötchen mit Marmelade gefrühstückt. -> Ich han hüt es Weggli mit Confi zum Zmorgä gha. Woher kommen Sie? -> Vo wo chömet Sie? Komm! Wir müssen uns beeilen! -> Chum, mir müend pressirä! Schau mal! -> Lueg mal! Hör mir zu! -> Los mär zue! Ich hätte gern einen Kaffee. -> Ich het gern än Kafi. Was guckst du mich so an? Hast du ein Problem? -> Was luegsch so doof? Häsch es Problem? Wir müssen noch Kartoffeln kaufen. -> Mir müend no Herdöpfel kaufe. Wir ziehen um(haus). -> Mir zuglät. Was hast du heute Abend gegessen? -> Was häsch hüt abig gesse? / Was häsch hüt zum Znach gha? Gehen wir raus? -> Chunsch in Usgang? Ich trink Leitunswasser. -> Ich trinke Letigswasser / Ich trinkä Hahnäwasser. Treffen wir uns in der Kneipe? -> Träffät mir eus i de Beiz? Ich trinke ein Bier in einer Kneipe. -> Ich trink sBier i de Beiz. Wo ist mein Handy? -> Wo isch mis Natel / Handy? Ich gehe schlafen. -> Ich gang go schalfä. Ich gehe schwimmen. -> Ich gang go schwimme. Ich gehe 'n Bier trinken. -> Ich gang go es Bier trinken i de Beiz. Ich habe Schluckauf. -> Ich han de Hidsgi. ================================================ FILE: datasets/zuerich/zuerichII.txt ================================================ (1) öper muess id Post ga und isch denn aber in es ichaufszentrum gange, und denn isch er wieder dihei gsi, het sini sache packt und isch is Training und später het er no i de Beiz es Bier trunke Jemand muss in die Post gehen, und dann ist er aber in ein Einkaufzentrum gegangen, und dann war er wieder zu Hause, hat seine Sachen gepackt, ist trainieren gegangen. und später hat er noch in der Kneipe ein Bier getrunken. (2) Der Janus und ich, Mir hand abgmacht, öpper am drü, nacher hammer öppis welle luege ga, sind am Kasse gange, und der billet sind scho usverkauft gsi. Wir haben ca um 15 uhr haben wir abgemacht, dann sind wir zur Kasse gegangen und wollten Tickets kaufen, aber die waren schon ausverkauft. ================================================ FILE: datasets/zuerich/zuerichbs.txt ================================================ Abwart = Haumeister Estrich = Dachboden Falle = Türklinke Kabis = Weißkohl wischen = fegen fegen = wischen räss = würzig pendent = unerledigt Nachtessen/Znacht = Abendessen Morgenessen/Zmorge = Frühstück ================================================ FILE: datasets/zug/zug.txt ================================================ «Der Zuger Dialekt ist nicht so markant und eigenständig wie zum Beispiel der Urner oder Obwaldner Dialekt», erklärt Markus Gasser. «Zugerdeutsch ist ein sogenannter Übergangs-Dialekt, denn Zug liegt in der Mitte der grossen Sprachgrenze zwischen verschiedenen Deutschschweizer Dialekten.» Es schneit auf die Strasse Zug : Es schnyyt uf d Stròòs (also «schnyye» wie in den alpinen Mundarten, aber «Stròòs» wie westlich im Aargauischen oder Luzernischen.) ================================================ FILE: dist/siriwave.esm.js ================================================ /*! ***************************************************************************** Copyright (c) Microsoft Corporation. Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ***************************************************************************** */ function __rest(s, e) { var t = {}; for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0) t[p] = s[p]; if (s != null && typeof Object.getOwnPropertySymbols === "function") for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) { if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i])) t[p[i]] = s[p[i]]; } return t; } class ClassicCurve { constructor(ctrl, definition) { this.ATT_FACTOR = 4; this.GRAPH_X = 2; this.AMPLITUDE_FACTOR = 0.6; this.ctrl = ctrl; this.definition = definition; } globalAttFn(x) { return Math.pow(this.ATT_FACTOR / (this.ATT_FACTOR + Math.pow(x, this.ATT_FACTOR)), this.ATT_FACTOR); } xPos(i) { return this.ctrl.width * ((i + this.GRAPH_X) / (this.GRAPH_X * 2)); } yPos(i) { return (this.AMPLITUDE_FACTOR * (this.globalAttFn(i) * (this.ctrl.heightMax * this.ctrl.amplitude) * (1 / this.definition.attenuation) * Math.sin(this.ctrl.opt.frequency * i - this.ctrl.phase))); } draw() { const { ctx } = this.ctrl; ctx.moveTo(0, 0); ctx.beginPath(); const finalColor = this.definition.color || this.ctrl.color; const colorHex = finalColor.replace(/rgb\(/g, "").replace(/\)/g, ""); ctx.strokeStyle = `rgba(${colorHex},${this.definition.opacity})`; ctx.lineWidth = this.definition.lineWidth; // Cycle the graph from -X to +X every PX_DEPTH and draw the line for (let i = -this.GRAPH_X; i <= this.GRAPH_X; i += this.ctrl.opt.pixelDepth) { ctx.lineTo(this.xPos(i), this.ctrl.heightMax + this.yPos(i)); } ctx.stroke(); } static getDefinition() { return [ { attenuation: -2, lineWidth: 1, opacity: 0.1, }, { attenuation: -6, lineWidth: 1, opacity: 0.2, }, { attenuation: 4, lineWidth: 1, opacity: 0.4, }, { attenuation: 2, lineWidth: 1, opacity: 0.6, }, { attenuation: 1, lineWidth: 1.5, opacity: 1, }, ]; } } class iOS9Curve { constructor(ctrl, definition) { this.GRAPH_X = 25; this.AMPLITUDE_FACTOR = 0.8; this.SPEED_FACTOR = 1; this.DEAD_PX = 2; this.ATT_FACTOR = 4; this.DESPAWN_FACTOR = 0.02; this.NOOFCURVES_RANGES = [2, 5]; this.AMPLITUDE_RANGES = [0.3, 1]; this.OFFSET_RANGES = [-3, 3]; this.WIDTH_RANGES = [1, 3]; this.SPEED_RANGES = [0.5, 1]; this.DESPAWN_TIMEOUT_RANGES = [500, 2000]; this.ctrl = ctrl; this.definition = definition; this.noOfCurves = 0; this.spawnAt = 0; this.prevMaxY = 0; this.phases = []; this.offsets = []; this.speeds = []; this.finalAmplitudes = []; this.widths = []; this.amplitudes = []; this.despawnTimeouts = []; this.verses = []; } getRandomRange(e) { return e[0] + Math.random() * (e[1] - e[0]); } spawnSingle(ci) { this.phases[ci] = 0; this.amplitudes[ci] = 0; this.despawnTimeouts[ci] = this.getRandomRange(this.DESPAWN_TIMEOUT_RANGES); this.offsets[ci] = this.getRandomRange(this.OFFSET_RANGES); this.speeds[ci] = this.getRandomRange(this.SPEED_RANGES); this.finalAmplitudes[ci] = this.getRandomRange(this.AMPLITUDE_RANGES); this.widths[ci] = this.getRandomRange(this.WIDTH_RANGES); this.verses[ci] = this.getRandomRange([-1, 1]); } getEmptyArray(count) { return new Array(count); } spawn() { this.spawnAt = Date.now(); this.noOfCurves = Math.floor(this.getRandomRange(this.NOOFCURVES_RANGES)); this.phases = this.getEmptyArray(this.noOfCurves); this.offsets = this.getEmptyArray(this.noOfCurves); this.speeds = this.getEmptyArray(this.noOfCurves); this.finalAmplitudes = this.getEmptyArray(this.noOfCurves); this.widths = this.getEmptyArray(this.noOfCurves); this.amplitudes = this.getEmptyArray(this.noOfCurves); this.despawnTimeouts = this.getEmptyArray(this.noOfCurves); this.verses = this.getEmptyArray(this.noOfCurves); for (let ci = 0; ci < this.noOfCurves; ci++) { this.spawnSingle(ci); } } globalAttFn(x) { return Math.pow(this.ATT_FACTOR / (this.ATT_FACTOR + Math.pow(x, 2)), this.ATT_FACTOR); } sin(x, phase) { return Math.sin(x - phase); } yRelativePos(i) { let y = 0; for (let ci = 0; ci < this.noOfCurves; ci++) { // Generate a static T so that each curve is distant from each oterh let t = 4 * (-1 + (ci / (this.noOfCurves - 1)) * 2); // but add a dynamic offset t += this.offsets[ci]; const k = 1 / this.widths[ci]; const x = i * k - t; y += Math.abs(this.amplitudes[ci] * this.sin(this.verses[ci] * x, this.phases[ci]) * this.globalAttFn(x)); } // Divide for NoOfCurves so that y <= 1 return y / this.noOfCurves; } yPos(i) { return (this.AMPLITUDE_FACTOR * this.ctrl.heightMax * this.ctrl.amplitude * this.yRelativePos(i) * this.globalAttFn((i / this.GRAPH_X) * 2)); } xPos(i) { return this.ctrl.width * ((i + this.GRAPH_X) / (this.GRAPH_X * 2)); } drawSupportLine() { const { ctx } = this.ctrl; const coo = [0, this.ctrl.heightMax, this.ctrl.width, 1]; const gradient = ctx.createLinearGradient.apply(ctx, coo); gradient.addColorStop(0, "transparent"); gradient.addColorStop(0.1, "rgba(255,255,255,.5)"); gradient.addColorStop(1 - 0.1 - 0.1, "rgba(255,255,255,.5)"); gradient.addColorStop(1, "transparent"); ctx.fillStyle = gradient; ctx.fillRect.apply(ctx, coo); } draw() { const { ctx } = this.ctrl; ctx.globalAlpha = 0.7; ctx.globalCompositeOperation = "lighter"; if (this.spawnAt === 0) { this.spawn(); } if (this.definition.supportLine) { // Draw the support line return this.drawSupportLine(); } for (let ci = 0; ci < this.noOfCurves; ci++) { if (this.spawnAt + this.despawnTimeouts[ci] <= Date.now()) { this.amplitudes[ci] -= this.DESPAWN_FACTOR; } else { this.amplitudes[ci] += this.DESPAWN_FACTOR; } this.amplitudes[ci] = Math.min(Math.max(this.amplitudes[ci], 0), this.finalAmplitudes[ci]); this.phases[ci] = (this.phases[ci] + this.ctrl.speed * this.speeds[ci] * this.SPEED_FACTOR) % (2 * Math.PI); } let maxY = -Infinity; // Write two opposite waves for (const sign of [1, -1]) { ctx.beginPath(); for (let i = -this.GRAPH_X; i <= this.GRAPH_X; i += this.ctrl.opt.pixelDepth) { const x = this.xPos(i); const y = this.yPos(i); ctx.lineTo(x, this.ctrl.heightMax - sign * y); maxY = Math.max(maxY, y); } ctx.closePath(); ctx.fillStyle = `rgba(${this.definition.color}, 1)`; ctx.strokeStyle = `rgba(${this.definition.color}, 1)`; ctx.fill(); } if (maxY < this.DEAD_PX && this.prevMaxY > maxY) { this.spawnAt = 0; } this.prevMaxY = maxY; return null; } static getDefinition() { return [ { color: "255,255,255", supportLine: true, }, { // blue color: "15, 82, 169", }, { // red color: "173, 57, 76", }, { // green color: "48, 220, 155", }, ]; } } class SiriWave { constructor(_a) { var { container } = _a, rest = __rest(_a, ["container"]); // Phase of the wave (passed to Math.sin function) this.phase = 0; // Boolean value indicating the the animation is running this.run = false; // Curves objects to animate this.curves = []; const csStyle = window.getComputedStyle(container); this.opt = Object.assign({ container, style: "ios", ratio: window.devicePixelRatio || 1, speed: 0.2, amplitude: 1, frequency: 6, color: "#fff", cover: false, width: parseInt(csStyle.width.replace("px", ""), 10), height: parseInt(csStyle.height.replace("px", ""), 10), autostart: true, pixelDepth: 0.02, lerpSpeed: 0.1 }, rest); /** * Actual speed of the animation. Is not safe to change this value directly, use `setSpeed` instead. */ this.speed = Number(this.opt.speed); /** * Actual amplitude of the animation. Is not safe to change this value directly, use `setAmplitude` instead. */ this.amplitude = Number(this.opt.amplitude); /** * Width of the canvas multiplied by pixel ratio */ this.width = Number(this.opt.ratio * this.opt.width); /** * Height of the canvas multiplied by pixel ratio */ this.height = Number(this.opt.ratio * this.opt.height); /** * Maximum height for a single wave */ this.heightMax = Number(this.height / 2) - 6; /** * Color of the wave (used in Classic iOS) */ this.color = `rgb(${this.hex2rgb(this.opt.color)})`; /** * An object containing controller variables that need to be interpolated * to an another value before to be actually changed */ this.interpolation = { speed: this.speed, amplitude: this.amplitude, }; /** * Canvas DOM Element where curves will be drawn */ this.canvas = document.createElement("canvas"); /** * 2D Context from Canvas */ const ctx = this.canvas.getContext("2d"); if (ctx === null) { throw new Error("Unable to create 2D Context"); } this.ctx = ctx; // Set dimensions this.canvas.width = this.width; this.canvas.height = this.height; // By covering, we ensure the canvas is in the same size of the parent if (this.opt.cover === true) { this.canvas.style.width = this.canvas.style.height = "100%"; } else { this.canvas.style.width = `${this.width / this.opt.ratio}px`; this.canvas.style.height = `${this.height / this.opt.ratio}px`; } // Instantiate all curves based on the style switch (this.opt.style) { case "ios9": this.curves = (this.opt.curveDefinition || iOS9Curve.getDefinition()).map((def) => new iOS9Curve(this, def)); break; case "ios": default: this.curves = (this.opt.curveDefinition || ClassicCurve.getDefinition()).map((def) => new ClassicCurve(this, def)); break; } // Attach to the container this.opt.container.appendChild(this.canvas); // Start the animation if (this.opt.autostart) { this.start(); } } /** * Convert an HEX color to RGB */ hex2rgb(hex) { const shorthandRegex = /^#?([a-f\d])([a-f\d])([a-f\d])$/i; hex = hex.replace(shorthandRegex, (m, r, g, b) => r + r + g + g + b + b); const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex); return result ? `${parseInt(result[1], 16).toString()},${parseInt(result[2], 16).toString()},${parseInt(result[3], 16).toString()}` : null; } intLerp(v0, v1, t) { return v0 * (1 - t) + v1 * t; } /** * Interpolate a property to the value found in this.interpolation */ lerp(propertyStr) { const prop = this.interpolation[propertyStr]; if (prop !== null) { this[propertyStr] = this.intLerp(this[propertyStr], prop, this.opt.lerpSpeed); if (this[propertyStr] - prop === 0) { this.interpolation[propertyStr] = null; } } return this[propertyStr]; } /** * Clear the canvas */ clear() { this.ctx.globalCompositeOperation = "destination-out"; this.ctx.fillRect(0, 0, this.width, this.height); this.ctx.globalCompositeOperation = "source-over"; } /** * Draw all curves */ draw() { this.curves.forEach((curve) => curve.draw()); } /** * Clear the space, interpolate values, calculate new steps and draws * @returns */ startDrawCycle() { this.clear(); // Interpolate values this.lerp("amplitude"); this.lerp("speed"); this.draw(); this.phase = (this.phase + (Math.PI / 2) * this.speed) % (2 * Math.PI); if (window.requestAnimationFrame) { this.animationFrameId = window.requestAnimationFrame(this.startDrawCycle.bind(this)); } else { this.timeoutId = setTimeout(this.startDrawCycle.bind(this), 20); } } /* API */ /** * Start the animation */ start() { if (!this.canvas) { throw new Error("This instance of SiriWave has been disposed, please create a new instance"); } this.phase = 0; // Ensure we don't re-launch the draw cycle if (!this.run) { this.run = true; this.startDrawCycle(); } } /** * Stop the animation */ stop() { this.phase = 0; this.run = false; // Clear old draw cycle on stop if (this.animationFrameId) { window.cancelAnimationFrame(this.animationFrameId); } if (this.timeoutId) { clearTimeout(this.timeoutId); } } /** * Dispose */ dispose() { this.stop(); if (this.canvas) { this.canvas.remove(); this.canvas = null; } } /** * Set a new value for a property (interpolated) */ set(propertyStr, value) { this.interpolation[propertyStr] = value; } /** * Set a new value for the speed property (interpolated) */ setSpeed(value) { this.set("speed", value); } /** * Set a new value for the amplitude property (interpolated) */ setAmplitude(value) { this.set("amplitude", value); } } export { SiriWave as default }; ================================================ FILE: dist/siriwave.umd.js ================================================ (function (global, factory) { typeof exports === 'object' && typeof module !== 'undefined' ? module.exports = factory() : typeof define === 'function' && define.amd ? define(factory) : (global = typeof globalThis !== 'undefined' ? globalThis : global || self, global.SiriWave = factory()); })(this, (function () { 'use strict'; /*! ***************************************************************************** Copyright (c) Microsoft Corporation. Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ***************************************************************************** */ var __assign = function() { __assign = Object.assign || function __assign(t) { for (var s, i = 1, n = arguments.length; i < n; i++) { s = arguments[i]; for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) t[p] = s[p]; } return t; }; return __assign.apply(this, arguments); }; function __rest(s, e) { var t = {}; for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0) t[p] = s[p]; if (s != null && typeof Object.getOwnPropertySymbols === "function") for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) { if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i])) t[p[i]] = s[p[i]]; } return t; } var ClassicCurve = /** @class */ (function () { function ClassicCurve(ctrl, definition) { this.ATT_FACTOR = 4; this.GRAPH_X = 2; this.AMPLITUDE_FACTOR = 0.6; this.ctrl = ctrl; this.definition = definition; } ClassicCurve.prototype.globalAttFn = function (x) { return Math.pow(this.ATT_FACTOR / (this.ATT_FACTOR + Math.pow(x, this.ATT_FACTOR)), this.ATT_FACTOR); }; ClassicCurve.prototype.xPos = function (i) { return this.ctrl.width * ((i + this.GRAPH_X) / (this.GRAPH_X * 2)); }; ClassicCurve.prototype.yPos = function (i) { return (this.AMPLITUDE_FACTOR * (this.globalAttFn(i) * (this.ctrl.heightMax * this.ctrl.amplitude) * (1 / this.definition.attenuation) * Math.sin(this.ctrl.opt.frequency * i - this.ctrl.phase))); }; ClassicCurve.prototype.draw = function () { var ctx = this.ctrl.ctx; ctx.moveTo(0, 0); ctx.beginPath(); var finalColor = this.definition.color || this.ctrl.color; var colorHex = finalColor.replace(/rgb\(/g, "").replace(/\)/g, ""); ctx.strokeStyle = "rgba(".concat(colorHex, ",").concat(this.definition.opacity, ")"); ctx.lineWidth = this.definition.lineWidth; // Cycle the graph from -X to +X every PX_DEPTH and draw the line for (var i = -this.GRAPH_X; i <= this.GRAPH_X; i += this.ctrl.opt.pixelDepth) { ctx.lineTo(this.xPos(i), this.ctrl.heightMax + this.yPos(i)); } ctx.stroke(); }; ClassicCurve.getDefinition = function () { return [ { attenuation: -2, lineWidth: 1, opacity: 0.1, }, { attenuation: -6, lineWidth: 1, opacity: 0.2, }, { attenuation: 4, lineWidth: 1, opacity: 0.4, }, { attenuation: 2, lineWidth: 1, opacity: 0.6, }, { attenuation: 1, lineWidth: 1.5, opacity: 1, }, ]; }; return ClassicCurve; }()); var iOS9Curve = /** @class */ (function () { function iOS9Curve(ctrl, definition) { this.GRAPH_X = 25; this.AMPLITUDE_FACTOR = 0.8; this.SPEED_FACTOR = 1; this.DEAD_PX = 2; this.ATT_FACTOR = 4; this.DESPAWN_FACTOR = 0.02; this.NOOFCURVES_RANGES = [2, 5]; this.AMPLITUDE_RANGES = [0.3, 1]; this.OFFSET_RANGES = [-3, 3]; this.WIDTH_RANGES = [1, 3]; this.SPEED_RANGES = [0.5, 1]; this.DESPAWN_TIMEOUT_RANGES = [500, 2000]; this.ctrl = ctrl; this.definition = definition; this.noOfCurves = 0; this.spawnAt = 0; this.prevMaxY = 0; this.phases = []; this.offsets = []; this.speeds = []; this.finalAmplitudes = []; this.widths = []; this.amplitudes = []; this.despawnTimeouts = []; this.verses = []; } iOS9Curve.prototype.getRandomRange = function (e) { return e[0] + Math.random() * (e[1] - e[0]); }; iOS9Curve.prototype.spawnSingle = function (ci) { this.phases[ci] = 0; this.amplitudes[ci] = 0; this.despawnTimeouts[ci] = this.getRandomRange(this.DESPAWN_TIMEOUT_RANGES); this.offsets[ci] = this.getRandomRange(this.OFFSET_RANGES); this.speeds[ci] = this.getRandomRange(this.SPEED_RANGES); this.finalAmplitudes[ci] = this.getRandomRange(this.AMPLITUDE_RANGES); this.widths[ci] = this.getRandomRange(this.WIDTH_RANGES); this.verses[ci] = this.getRandomRange([-1, 1]); }; iOS9Curve.prototype.getEmptyArray = function (count) { return new Array(count); }; iOS9Curve.prototype.spawn = function () { this.spawnAt = Date.now(); this.noOfCurves = Math.floor(this.getRandomRange(this.NOOFCURVES_RANGES)); this.phases = this.getEmptyArray(this.noOfCurves); this.offsets = this.getEmptyArray(this.noOfCurves); this.speeds = this.getEmptyArray(this.noOfCurves); this.finalAmplitudes = this.getEmptyArray(this.noOfCurves); this.widths = this.getEmptyArray(this.noOfCurves); this.amplitudes = this.getEmptyArray(this.noOfCurves); this.despawnTimeouts = this.getEmptyArray(this.noOfCurves); this.verses = this.getEmptyArray(this.noOfCurves); for (var ci = 0; ci < this.noOfCurves; ci++) { this.spawnSingle(ci); } }; iOS9Curve.prototype.globalAttFn = function (x) { return Math.pow(this.ATT_FACTOR / (this.ATT_FACTOR + Math.pow(x, 2)), this.ATT_FACTOR); }; iOS9Curve.prototype.sin = function (x, phase) { return Math.sin(x - phase); }; iOS9Curve.prototype.yRelativePos = function (i) { var y = 0; for (var ci = 0; ci < this.noOfCurves; ci++) { // Generate a static T so that each curve is distant from each oterh var t = 4 * (-1 + (ci / (this.noOfCurves - 1)) * 2); // but add a dynamic offset t += this.offsets[ci]; var k = 1 / this.widths[ci]; var x = i * k - t; y += Math.abs(this.amplitudes[ci] * this.sin(this.verses[ci] * x, this.phases[ci]) * this.globalAttFn(x)); } // Divide for NoOfCurves so that y <= 1 return y / this.noOfCurves; }; iOS9Curve.prototype.yPos = function (i) { return (this.AMPLITUDE_FACTOR * this.ctrl.heightMax * this.ctrl.amplitude * this.yRelativePos(i) * this.globalAttFn((i / this.GRAPH_X) * 2)); }; iOS9Curve.prototype.xPos = function (i) { return this.ctrl.width * ((i + this.GRAPH_X) / (this.GRAPH_X * 2)); }; iOS9Curve.prototype.drawSupportLine = function () { var ctx = this.ctrl.ctx; var coo = [0, this.ctrl.heightMax, this.ctrl.width, 1]; var gradient = ctx.createLinearGradient.apply(ctx, coo); gradient.addColorStop(0, "transparent"); gradient.addColorStop(0.1, "rgba(255,255,255,.5)"); gradient.addColorStop(1 - 0.1 - 0.1, "rgba(255,255,255,.5)"); gradient.addColorStop(1, "transparent"); ctx.fillStyle = gradient; ctx.fillRect.apply(ctx, coo); }; iOS9Curve.prototype.draw = function () { var ctx = this.ctrl.ctx; ctx.globalAlpha = 0.7; ctx.globalCompositeOperation = "lighter"; if (this.spawnAt === 0) { this.spawn(); } if (this.definition.supportLine) { // Draw the support line return this.drawSupportLine(); } for (var ci = 0; ci < this.noOfCurves; ci++) { if (this.spawnAt + this.despawnTimeouts[ci] <= Date.now()) { this.amplitudes[ci] -= this.DESPAWN_FACTOR; } else { this.amplitudes[ci] += this.DESPAWN_FACTOR; } this.amplitudes[ci] = Math.min(Math.max(this.amplitudes[ci], 0), this.finalAmplitudes[ci]); this.phases[ci] = (this.phases[ci] + this.ctrl.speed * this.speeds[ci] * this.SPEED_FACTOR) % (2 * Math.PI); } var maxY = -Infinity; // Write two opposite waves for (var _i = 0, _a = [1, -1]; _i < _a.length; _i++) { var sign = _a[_i]; ctx.beginPath(); for (var i = -this.GRAPH_X; i <= this.GRAPH_X; i += this.ctrl.opt.pixelDepth) { var x = this.xPos(i); var y = this.yPos(i); ctx.lineTo(x, this.ctrl.heightMax - sign * y); maxY = Math.max(maxY, y); } ctx.closePath(); ctx.fillStyle = "rgba(".concat(this.definition.color, ", 1)"); ctx.strokeStyle = "rgba(".concat(this.definition.color, ", 1)"); ctx.fill(); } if (maxY < this.DEAD_PX && this.prevMaxY > maxY) { this.spawnAt = 0; } this.prevMaxY = maxY; return null; }; iOS9Curve.getDefinition = function () { return [ { color: "255,255,255", supportLine: true, }, { // blue color: "15, 82, 169", }, { // red color: "173, 57, 76", }, { // green color: "48, 220, 155", }, ]; }; return iOS9Curve; }()); var SiriWave = /** @class */ (function () { function SiriWave(_a) { var _this = this; var container = _a.container, rest = __rest(_a, ["container"]); // Phase of the wave (passed to Math.sin function) this.phase = 0; // Boolean value indicating the the animation is running this.run = false; // Curves objects to animate this.curves = []; var csStyle = window.getComputedStyle(container); this.opt = __assign({ container: container, style: "ios", ratio: window.devicePixelRatio || 1, speed: 0.2, amplitude: 1, frequency: 6, color: "#fff", cover: false, width: parseInt(csStyle.width.replace("px", ""), 10), height: parseInt(csStyle.height.replace("px", ""), 10), autostart: true, pixelDepth: 0.02, lerpSpeed: 0.1 }, rest); /** * Actual speed of the animation. Is not safe to change this value directly, use `setSpeed` instead. */ this.speed = Number(this.opt.speed); /** * Actual amplitude of the animation. Is not safe to change this value directly, use `setAmplitude` instead. */ this.amplitude = Number(this.opt.amplitude); /** * Width of the canvas multiplied by pixel ratio */ this.width = Number(this.opt.ratio * this.opt.width); /** * Height of the canvas multiplied by pixel ratio */ this.height = Number(this.opt.ratio * this.opt.height); /** * Maximum height for a single wave */ this.heightMax = Number(this.height / 2) - 6; /** * Color of the wave (used in Classic iOS) */ this.color = "rgb(".concat(this.hex2rgb(this.opt.color), ")"); /** * An object containing controller variables that need to be interpolated * to an another value before to be actually changed */ this.interpolation = { speed: this.speed, amplitude: this.amplitude, }; /** * Canvas DOM Element where curves will be drawn */ this.canvas = document.createElement("canvas"); /** * 2D Context from Canvas */ var ctx = this.canvas.getContext("2d"); if (ctx === null) { throw new Error("Unable to create 2D Context"); } this.ctx = ctx; // Set dimensions this.canvas.width = this.width; this.canvas.height = this.height; // By covering, we ensure the canvas is in the same size of the parent if (this.opt.cover === true) { this.canvas.style.width = this.canvas.style.height = "100%"; } else { this.canvas.style.width = "".concat(this.width / this.opt.ratio, "px"); this.canvas.style.height = "".concat(this.height / this.opt.ratio, "px"); } // Instantiate all curves based on the style switch (this.opt.style) { case "ios9": this.curves = (this.opt.curveDefinition || iOS9Curve.getDefinition()).map(function (def) { return new iOS9Curve(_this, def); }); break; case "ios": default: this.curves = (this.opt.curveDefinition || ClassicCurve.getDefinition()).map(function (def) { return new ClassicCurve(_this, def); }); break; } // Attach to the container this.opt.container.appendChild(this.canvas); // Start the animation if (this.opt.autostart) { this.start(); } } /** * Convert an HEX color to RGB */ SiriWave.prototype.hex2rgb = function (hex) { var shorthandRegex = /^#?([a-f\d])([a-f\d])([a-f\d])$/i; hex = hex.replace(shorthandRegex, function (m, r, g, b) { return r + r + g + g + b + b; }); var result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex); return result ? "".concat(parseInt(result[1], 16).toString(), ",").concat(parseInt(result[2], 16).toString(), ",").concat(parseInt(result[3], 16).toString()) : null; }; SiriWave.prototype.intLerp = function (v0, v1, t) { return v0 * (1 - t) + v1 * t; }; /** * Interpolate a property to the value found in this.interpolation */ SiriWave.prototype.lerp = function (propertyStr) { var prop = this.interpolation[propertyStr]; if (prop !== null) { this[propertyStr] = this.intLerp(this[propertyStr], prop, this.opt.lerpSpeed); if (this[propertyStr] - prop === 0) { this.interpolation[propertyStr] = null; } } return this[propertyStr]; }; /** * Clear the canvas */ SiriWave.prototype.clear = function () { this.ctx.globalCompositeOperation = "destination-out"; this.ctx.fillRect(0, 0, this.width, this.height); this.ctx.globalCompositeOperation = "source-over"; }; /** * Draw all curves */ SiriWave.prototype.draw = function () { this.curves.forEach(function (curve) { return curve.draw(); }); }; /** * Clear the space, interpolate values, calculate new steps and draws * @returns */ SiriWave.prototype.startDrawCycle = function () { this.clear(); // Interpolate values this.lerp("amplitude"); this.lerp("speed"); this.draw(); this.phase = (this.phase + (Math.PI / 2) * this.speed) % (2 * Math.PI); if (window.requestAnimationFrame) { this.animationFrameId = window.requestAnimationFrame(this.startDrawCycle.bind(this)); } else { this.timeoutId = setTimeout(this.startDrawCycle.bind(this), 20); } }; /* API */ /** * Start the animation */ SiriWave.prototype.start = function () { if (!this.canvas) { throw new Error("This instance of SiriWave has been disposed, please create a new instance"); } this.phase = 0; // Ensure we don't re-launch the draw cycle if (!this.run) { this.run = true; this.startDrawCycle(); } }; /** * Stop the animation */ SiriWave.prototype.stop = function () { this.phase = 0; this.run = false; // Clear old draw cycle on stop if (this.animationFrameId) { window.cancelAnimationFrame(this.animationFrameId); } if (this.timeoutId) { clearTimeout(this.timeoutId); } }; /** * Dispose */ SiriWave.prototype.dispose = function () { this.stop(); if (this.canvas) { this.canvas.remove(); this.canvas = null; } }; /** * Set a new value for a property (interpolated) */ SiriWave.prototype.set = function (propertyStr, value) { this.interpolation[propertyStr] = value; }; /** * Set a new value for the speed property (interpolated) */ SiriWave.prototype.setSpeed = function (value) { this.set("speed", value); }; /** * Set a new value for the amplitude property (interpolated) */ SiriWave.prototype.setAmplitude = function (value) { this.set("amplitude", value); }; return SiriWave; }()); return SiriWave; })); ================================================ FILE: dist/typ/curve.d.ts ================================================ import SiriWave, { IClassicCurveDefinition, ICurve } from "./index"; export declare class ClassicCurve implements ICurve { ctrl: SiriWave; definition: IClassicCurveDefinition; ATT_FACTOR: number; GRAPH_X: number; AMPLITUDE_FACTOR: number; constructor(ctrl: SiriWave, definition: IClassicCurveDefinition); private globalAttFn; private xPos; private yPos; draw(): void; static getDefinition(): IClassicCurveDefinition[]; } ================================================ FILE: dist/typ/index.d.ts ================================================ #das funtioniert auf IOS declare type CurveStyle = "ios" | "ios9"; export declare type Options = { container: HTMLElement; style?: CurveStyle; ratio?: number; speed?: number; amplitude?: number; frequency?: number; color?: string; cover?: boolean; width?: number; height?: number; autostart?: boolean; pixelDepth?: number; lerpSpeed?: number; curveDefinition?: ICurveDefinition[]; }; export declare type IiOS9CurveDefinition = { supportLine?: boolean; color: string; }; export declare type IClassicCurveDefinition = { attenuation: number; lineWidth: number; opacity: number; color?: string; }; export declare type ICurveDefinition = IiOS9CurveDefinition | IClassicCurveDefinition; export interface ICurve { draw: () => void; } export default class SiriWave { opt: Options; phase: number; run: boolean; curves: ICurve[]; speed: number; amplitude: number; width: number; height: number; heightMax: number; color: string; interpolation: { speed: number | null; amplitude: number | null; }; canvas: HTMLCanvasElement | null; ctx: CanvasRenderingContext2D; animationFrameId: number | undefined; timeoutId: ReturnType | undefined; constructor({ container, ...rest }: Options); /** * Convert an HEX color to RGB */ private hex2rgb; private intLerp; /** * Interpolate a property to the value found in this.interpolation */ private lerp; /** * Clear the canvas */ private clear; /** * Draw all curves */ private draw; /** * Clear the space, interpolate values, calculate new steps and draws * @returns */ private startDrawCycle; /** * Start the animation */ start(): void; /** * Stop the animation */ stop(): void; /** * Dispose */ dispose(): void; /** * Set a new value for a property (interpolated) */ set(propertyStr: "amplitude" | "speed", value: number): void; /** * Set a new value for the speed property (interpolated) */ setSpeed(value: number): void; /** * Set a new value for the amplitude property (interpolated) */ setAmplitude(value: number): void; } export {}; ================================================ FILE: dist/typ/ios.d.ts ================================================ import SiriWave, { ICurve, IiOS9CurveDefinition } from "./index"; export declare class iOS9Curve implements ICurve { ctrl: SiriWave; definition: IiOS9CurveDefinition; spawnAt: number; noOfCurves: number; prevMaxY: number; phases: number[]; amplitudes: number[]; despawnTimeouts: number[]; offsets: number[]; speeds: number[]; finalAmplitudes: number[]; widths: number[]; verses: number[]; GRAPH_X: number; AMPLITUDE_FACTOR: number; SPEED_FACTOR: number; DEAD_PX: number; ATT_FACTOR: number; DESPAWN_FACTOR: number; NOOFCURVES_RANGES: [number, number]; AMPLITUDE_RANGES: [number, number]; OFFSET_RANGES: [number, number]; WIDTH_RANGES: [number, number]; SPEED_RANGES: [number, number]; DESPAWN_TIMEOUT_RANGES: [number, number]; constructor(ctrl: SiriWave, definition: IiOS9CurveDefinition); private getRandomRange; private spawnSingle; private getEmptyArray; private spawn; private globalAttFn; private sin; private yRelativePos; private yPos; private xPos; private drawSupportLine; draw(): void | null; static getDefinition(): IiOS9CurveDefinition[]; } ================================================ FILE: facts/ASR/DeepSpeech.md ================================================ # Automatic Speech Recognition (ASR) - DeepSpeech Swiss German _This is the project for the paper [LTL-UDE at Low-Resource Speech-to-Text Shared Task : Investigating Mozilla DeepSpeech in a low-resource setting](https://www.researchgate.net/publication/342338332_LTL-UDE_at_Low-Resource_Speech-to-Text_Shared_Task_Investigating_Mozilla_DeepSpeech_in_a_low-resource_setting) published at [SWISSTEXT 5th and KONVENS 2020](https://swisstext-and-konvens-2020.org/)._ This project aims to develop a working Speech to Text module using [Mozilla DeepSpeech](https://github.com/mozilla/DeepSpeech), which can be used for any Audio processing pipeline. [Mozillla DeepSpeech](https://github.com/mozilla/DeepSpeech) is a state-of-the-art open-source automatic speech recognition (ASR) toolkit. DeepSpeech is using a model trained by deep learning techniques based on [Baidu's Deep Speech](https://gigaom2.files.wordpress.com/2014/12/deep_speech3_12_17.pdf) research paper. Project DeepSpeech uses Google's TensorFlow to make the implementation easier.

## Important Links: **Paper:** https://www.researchgate.net/publication/342338332_LTL-UDE_at_Low-Resource_Speech-to-Text_Shared_Task_Investigating_Mozilla_DeepSpeech_in_a_low-resource_setting **DeepSpeech-API:** https://github.com/AASHISHAG/DeepSpeech-API This Readme is written for [DeepSpeech v0.6.0](https://github.com/mozilla/DeepSpeech/releases/tag/v0.6.0). Refer to [Mozillla DeepSpeech](https://github.com/mozilla/DeepSpeech) for lastest updates. ## Contents 1. [Requirements](#requirements) 2. [Speech Corpus](#speech-corpus) 3. [Language Model](#language-model) 4. [Training](#training) 5. [Results](#results) 6. [Acknowledgments](#acknowledgments) 7. [References](#references) ### Requirements #### Installing Python bindings ``` virtualenv -p python3 deepspeech-swiss-german source deepspeech-swiss-german/bin/activate pip3 install -r python_requirements.txt ``` #### Mozilla DeepSpeech ``` $ git clone https://github.com/mozilla/DeepSpeech.git $ cd DeepSpeech $ git checkout v0.6.0 $ docker build -t deepspeech_v0.6.0 . $ docker run -d -it --name deepspeech_v0.6.0 --mount type=bind,source="$(pwd)",target=/root deepspeech_v0.6.0 $ docker exec -it deepspeech_v0.6.0 /bin/bash ``` Note: Set the locale to en_US.UTF-8 if required: ``` $ dpkg-reconfigure locales $ https://perlgeek.de/en/article/set-up-a-clean-utf8-environment ``` ### Speech Corpus **1. _English_** * [Mozilla Common Voice](https://voice.mozilla.org/) ~1488h * [LibriSpeech](http://www.openslr.org/12) ~1000h **2. _German_** * [Mozilla Common Voice](https://voice.mozilla.org/) ~454h * [Mailabs](https://www.caito.de/2019/01/the-m-ailabs-speech-dataset/) ~233h * [German Distant Speech Corpus (TUDA-De)](https://www.inf.uni-hamburg.de/en/inst/ab/lt/resources/data/acoustic-models.html) ~184h * [Voxforge](http://www.voxforge.org/home/forums/other-languages/german/open-speech-data-corpus-for-german) ~57h **3. _Swiss-German_** * [SwissText](https://swisstext-and-konvens-2020.org/low-resource-speech-to-text/) ~70h * [ArchiMob](https://www.spur.uzh.ch/en/departments/research/textgroup/ArchiMob.html) ~57h ### Download and Prepare the Audio Data **1. _Mozilla_EN_** ``` $ mkdir mozilla_en $ cd mozilla_en $ wget https://voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com/cv-corpus-4-2019-12-10/en.tar.gz $ tar -xzvf en.tar.gz $ python3 DeepSpeech/bin/import_cv2.py --audio_dir path --filter_alphabet deepspeech-swiss-german/data/en_alphabet.txt export_path ``` **2. _LibriSpeech_EN_** ``` $ mkdir librispeech $ cd librispeech $ python3 DeepSpeech/bin/import_librivox.py export_path ``` **3. _Mozilla_DE_** ``` $ mkdir mozilla_de $ cd mozilla_de $ wget https://voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com/cv-corpus-4-2019-12-10/de.tar.gz $ tar -xzvf de.tar.gz $ python3 DeepSpeech/bin/import_cv2.py --audio_dir path --filter_alphabet deepspeech-swiss-german/data/alphabet.txt export_path ``` **4. _Mailabs_DE_** ``` $ mkdir mailabs $ cd mailabs $ python3 DeepSpeech/bin/import_m-ailabs.py --language de_DE --filter_alphabet deepspeech-swiss-german/data/alphabet.txt export_path ``` **5. _Tuda_DE_** ``` $ mkdir tuda $ cd tuda $ wget http://www.repository.voxforge1.org/downloads/de/german-speechdata-package-v2.tar.gz $ tar -xzvf german-speechdata-package-v2.tar.gz $ deepspeech-swiss-german/pre-processing/prepare_data.py --tuda corpus_path export_path ``` **6. _Voxforge_DE_** ``` $ mkdir voxforge $ cd voxforge ``` ```python python3 $ from audiomate.corpus import io $ dl = io.VoxforgeDownloader(lang='de') $ dl.download(voxforge_corpus_path) ``` ``` $ deepspeech-swiss-german/pre-processing/run_to_utf_8.sh $ python3 deepspeech-swiss-german/prepare_data.py --voxforge corpus_path export_path ``` _NOTE: Change the path accordingly in run_to_utf_8.sh_ **7. _SwissText_DE_** ``` $ mkdir swisstext $ cd swisstext $ https://drive.switch.ch/index.php/s/PpUArRmN5Ba5C8J $ unzip train.zip $ python3 deepspeech-swiss-german/prepare_data_swiss_german.py $ python3 deepspeech-swiss-german/shuffle_and_split.py ``` **8. _ArchiMob_DE_** Follow steps [here](https://github.com/AASHISHAG/archimob-swissgerman-deepspeech-importer): ``` $ https://github.com/AASHISHAG/archimob-swissgerman-deepspeech-importer ``` ### Language Model We used [KenLM](https://github.com/kpu/kenlm.git) toolkit to train a 3-gram language model. It is Language Model inference code by [Kenneth Heafield](https://kheafield.com/) - **Installation** ``` $ git clone https://github.com/kpu/kenlm.git $ cd kenlm $ mkdir -p build $ cd build $ cmake .. $ make -j `nproc` ``` - **Corpus** We used an open-source [German Speech Corpus](http://ltdata1.informatik.uni-hamburg.de/kaldi_tuda_de/German_sentences_8mil_filtered_maryfied.txt.gz) released by [University of Hamburg](https://www.inf.uni-hamburg.de/en/inst/ab/lt/resources/data/acoustic-models.html) and [European Parliament Proceedings Parallel Corpus 1996-2011](https://www.statmt.org/europarl/) 1. Download the data (EN, DE) ``` ##EN $ using Mozilla default LM and Trie ## DE $ wget http://ltdata1.informatik.uni-hamburg.de/kaldi_tuda_de/German_sentences_8mil_filtered_maryfied.txt.gz $ gzip -d German_sentences_8mil_filtered_maryfied.txt.gz $ wget https://www.statmt.org/europarl/v7/de-en.tgz $ tar -xzvf de-en.tgz $ cat German_sentences_8mil_filtered_maryfied.txt >> europarl-v7.de-en.de ``` 2. Pre-process the data (DE) ``` $ deepspeech-swiss-german/pre-processing/prepare_vocab.py europarl-v7.de-en.de exp_path/clean_vocab.txt ``` 3. Build the Language Model (DE) ``` $ kenlm/build/bin/lmplz --text exp_path/clean_vocab.txt --arpa exp_path/words.arpa --o 3 $ kenlm/build/bin/build_binary -T -s exp_path/words.arpa exp_path/de_lm.binary ``` #### NOTE: use [-S](https://kheafield.com/code/kenlm/estimation/) memoryuse_in_%, if malloc expection occurs Example: ``` $ kenlm/build/bin/lmplz --text exp_path/clean_vocab.txt --arpa exp_path/words.arpa --o 3 -S 50% ``` 4. Build Trie (DE) ``` $ DeepSpeech/native_client/generate_trie deepspeech-swiss-german/data/alphabet.txt path/de_lm.binary export_path/de_trie ``` ### Training Change the path accordingly. ``` $ ./DeepSpeech.py --train_files train.csv --dev_files dev.csv --test_files test.csv --alphabet_config_path alphabet.txt --lm_trie_path trie --lm_binary_path lm.binary --test_batch_size 36 --train_batch_size 24 --dev_batch_size 36 --epochs 75 --learning_rate 0.0001 --dropout_rate 0.25 --export_dir ../models ``` ### Training with Augmentation Change the path accordingly. ``` $ ./DeepSpeech.py --train_files train.csv --dev_files dev.csv --test_files test.csv --alphabet_config_path alphabet.txt --lm_trie_path trie --lm_binary_path lm.binary --test_batch_size 36 --train_batch_size 24 --dev_batch_size 36 --epochs 75 --learning_rate 0.0001 --dropout_rate 0.25 --export_dir ../models AUG_AUDIO="--data_aug_features_additive 0.2 --data_aug_features_multiplicative 0.2 --augmentation_speed_up_std 0.2" AUG_FREQ_TIME="--augmentation_freq_and_time_masking --augmentation_freq_and_time_masking_freq_mask_range 5 --augmentation_freq_and_time_masking_number_freq_masks 3 --augmentation_freq_and_time_masking_time_mask_range 2 --augmentation_freq_and_time_masking_number_time_masks 3" AUG_PITCH_TEMPO="--augmentation_pitch_and_tempo_scaling --augmentation_pitch_and_tempo_scaling_min_pitch 0.95 --augmentation_pitch_and_tempo_scaling_max_pitch 1.2 --augmentation_pitch_and_tempo_scaling_max_tempo 1.2" AUG_SPEC_DROP="--augmentation_spec_dropout_keeprate 0.2" ``` ### Results Some results from our findings. - English -> German -> Swiss : 56.6 _NOTE: Refer our paper for more information._ ## Acknowledgments * [Prof. Dr.-Ing. Torsten Zesch](https://www.ltl.uni-due.de/team/torsten-zesch) - Co-Author ## References If you use our findings/scripts in your academic work, please cite: ``` ``` ================================================ FILE: facts/ASR/data/alphabet.txt ================================================ # Each line in this file represents the Unicode codepoint (UTF-8 encoded) # associated with a numeric label. # A line that starts with # is a comment. You can escape it with \# if you wish # to use '#' as a label. a b c d e f g h i j k l m n o p q r s t u v w x y z ä ö ü 0 1 2 3 4 5 6 7 8 9 # The last (non-comment) line needs to end with a newline. ================================================ FILE: facts/ASR/pre-processing/audio.py ================================================ # Author: Mozilla DeepSpeech v0.4.0 import numpy as np import scipy.io.wavfile as wav from python_speech_features import mfcc def audiofile_to_input_vector(audio_filename, numcep, numcontext): """ Given a WAV audio file at ``audio_filename``, calculates ``numcep`` MFCC features at every 0.01s time step with a window length of 0.025s. Appends ``numcontext`` context frames to the left and right of each time step, and returns this data in a numpy array. """ # Load wav files fs, audio = wav.read(audio_filename) # Get mfcc coefficients features = mfcc(audio, samplerate=fs, numcep=numcep, winlen=0.032, winstep=0.02, winfunc=np.hamming) # Add empty initial and final contexts empty_context = np.zeros((numcontext, numcep), dtype=features.dtype) features = np.concatenate((empty_context, features, empty_context)) return features ================================================ FILE: facts/ASR/pre-processing/ch_De_prep.py ================================================ import os import re import sys import codecs import pandas as pd from sox import Transformer files = [] SAMPLE_RATE = 16000 source_dir = os.getcwd() trans_filename = sys.argv[1] ALLOWED_CHARS = { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'ä', 'ö', 'ü', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ' ', #',', ';', ':', '.', '?', '!', } WHITESPACE_REGEX = re.compile(r'[ \t]+') def preprocess_transcript_for_corpus(transcript): transcript = transcript.lower() transcript = transcript.replace('á', 'a') transcript = transcript.replace('à', 'a') transcript = transcript.replace('â', 'a') transcript = transcript.replace('ç', 'c') transcript = transcript.replace('é', 'e') transcript = transcript.replace('è', 'e') transcript = transcript.replace('ê', 'e') transcript = transcript.replace('í', 'i') transcript = transcript.replace('ì', 'i') transcript = transcript.replace('î', 'i') transcript = transcript.replace('ñ', 'n') transcript = transcript.replace('ó', 'o') transcript = transcript.replace('ò', 'o') transcript = transcript.replace('ô', 'o') transcript = transcript.replace('ú', 'u') transcript = transcript.replace('ù', 'u') transcript = transcript.replace('û', 'u') transcript = transcript.replace('ș', 's') transcript = transcript.replace('ş', 's') transcript = transcript.replace('ß', 'ss') transcript = transcript.replace('-', ' ') # Not used consistently, better to replace with space as well transcript = transcript.replace('–', ' ') transcript = transcript.replace('/', ' ') transcript = WHITESPACE_REGEX.sub(' ', transcript) transcript = ''.join([char for char in transcript if char in ALLOWED_CHARS]) transcript = WHITESPACE_REGEX.sub(' ', transcript) transcript = transcript.strip() return transcript with codecs.open(trans_filename, "r", "utf-8") as fin: next(fin) for line in fin: # Parse each segment line columns = line.strip().split('\t') client_id = columns[0] path = columns[1] transcript = columns[2] transcript = preprocess_transcript_for_corpus(transcript) # Convert corresponding FLAC to a WAV flac_file = os.path.join(source_dir, 'clips', path) wav_file = os.path.join(source_dir, 'clips', path.split('.')[0] + '.wav') if not os.path.exists(wav_file): tfm = Transformer() tfm.set_output_format(rate=SAMPLE_RATE) tfm.build(flac_file, wav_file) print('Converting flac file {} to wav format {}.'.format(flac_file, wav_file)) wav_filesize = os.path.getsize(wav_file) files.append((client_id, os.path.abspath(wav_file), wav_filesize, transcript)) df = pd.DataFrame(data=files, columns=["client id", "wav_filename", "wav_filesize", "transcript"]) # Write sets to disk as CSV files df.to_csv(os.path.join(source_dir, trans_filename.split('.')[0]+".csv"), index=False) ================================================ FILE: facts/ASR/pre-processing/corpus_cleaning.py ================================================ # -*- coding: utf-8 -*- import string import collections import re import num2words # # Number patterns # int_pattern = re.compile(r'[0-9]+') float_pattern = re.compile(r'[0-9]+[,\.][0-9]+') # # Allowed characters a-zA-Z'äüö # allowed = list(string.ascii_lowercase) allowed.append("'") allowed.append(' ') allowed.extend(list('äöü')) # # Replacement characters # replacer = { 'àáâãåāăąǟǡǻȁȃȧ': 'a', 'æǣǽ': 'ä', 'çćĉċč': 'c', 'ďđ': 'd', 'èéêëēĕėęěȅȇȩε': 'e', 'ĝğġģǥǧǵ': 'g', 'ĥħȟ': 'h', 'ìíîïĩīĭįıȉȋ': 'i', 'ĵǰ': 'j', 'ķĸǩǩκ': 'k', 'ĺļľŀł': 'l', 'м': 'm', 'ñńņňʼnŋǹ': 'n', 'òóôõøōŏőǫǭǿȍȏðο': 'o', 'œ': 'ö', 'ŕŗřȑȓ': 'r', 'śŝşšș': 's', 'ţťŧț': 't', 'ùúûũūŭůűųȕȗ': 'u', 'ŵ': 'w', 'ýÿŷ': 'y', 'źżžȥ': 'z', 'ß': 'ss', '-­': ' ' } # # Various replacement rules # special_replacers = { ' $ ': 'dollar', ' £ ': 'pfund', 'm³': 'kubikmeter', 'km²': 'quadratkilometer', 'm²': 'quadratmeter' } replacements = {} replacements.update(special_replacers) for all, replacement in replacer.items(): for to_replace in all: replacements[to_replace] = replacement # # Utils # def replace_symbols(word): """ Apply all replacement characters/rules to the given word. """ result = word for to_replace, replacement in replacements.items(): result = result.replace(to_replace, replacement) return result def remove_symbols(word): """ Remove all symbols that are not allowed. """ result = word bad_characters = [] for c in result: if c not in allowed: bad_characters.append(c) for c in bad_characters: result = result.replace(c, '') return result def word_to_num(word): """ Replace numbers with their written representation. """ result = word match = float_pattern.search(result) while match is not None: num_word = num2words.num2words(float(match.group().replace(',', '.')), lang='de').lower() before = result[:match.start()] after = result[match.end():] result = ' '.join([before, num_word, after]) match = float_pattern.search(result) match = int_pattern.search(result) while match is not None: num_word = num2words.num2words(int(match.group()), lang='de') before = result[:match.start()] after = result[match.end():] result = ' '.join([before, num_word, after]) match = int_pattern.search(result) return result def get_bad_character(text): """ Return all characters in the text that are not allowed. """ bad_characters = set() for c in text: if c not in allowed: bad_characters.add(c) return bad_characters def clean_word(word): """ Clean the given word. 1. numbers to words 2. character/rule replacements 3. delete disallowed symbols """ word = word.lower() word = word_to_num(word) word = replace_symbols(word) word = remove_symbols(word) bad_chars = get_bad_character(word) if len(bad_chars) > 0: print('Bad characters in "{}"'.format(word)) print('--> {}'.format(', '.join(bad_chars))) return word def clean_sentence(sentence): """ Clean the given sentence. 1. split into words by spaces 2. numbers to words 3. character/rule replacements 4. delete disallowed symbols 4. join with spaces """ words = sentence.strip().split(' ') cleaned_words = [] for word in words: cleaned_word = clean_word(word) cleaned_words.append(cleaned_word) return ' '.join(cleaned_words) ================================================ FILE: facts/ASR/pre-processing/finden_files.py ================================================ import pandas as pd from audio import audiofile_to_input_vector N_CONTEXT = 9 df = pd.read_csv('tuda-de.csv') def aftiv_length(row): return audiofile_to_input_vector(row['wav_filename'], 26, N_CONTEXT).shape[0] - 2*N_CONTEXT def trans_length(row): return len(row['transcript']) df['aftiv_len'] = df.apply(aftiv_length, axis=1) df['trans_len'] = df.apply(trans_length, axis=1) df['good_flag'] = df.aftiv_len > df.trans_len df.to_csv("final.csv") ================================================ FILE: facts/ASR/pre-processing/folien.py ================================================ #checking if the file exists # Contributor: Aashish Agarwal import os import csv with open('/home/LTLab.lan/agarwal/german-speech-corpus/') as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') line_count = 0 for row in csv_reader: print(row[0]) exists = os.path.isfile(row[0]) if exists: temp=1 else: print("Does not exist:" + row[0]) ================================================ FILE: facts/ASR/pre-processing/prep_vocab.py ================================================ #! /usr/bin/env python """ 1. Load text corpus 2. Clean text 3. Extend with transcriptions from training data 4. Save """ import os import sys sys.path.append(os.path.abspath(os.path.join(__file__, os.path.pardir))) import argparse from audiomate.utils import textfile import text_cleaning def read_training_transcripts(path): transcripts = [] for entry in textfile.read_separated_lines_generator(path, separator=',', max_columns=3, ignore_lines_starting_with=['wav_filename']): transcripts.append(entry[2]) return transcripts parser = argparse.ArgumentParser(description='Clean text corpus.') parser.add_argument('source_path', type=str) parser.add_argument('target_path', type=str) parser.add_argument('--training_csv', type=str) args = parser.parse_args() index = 0 with open(args.source_path, 'r') as source_file, open(args.target_path, 'w') as target_file: for index, line in enumerate(source_file): cleaned_sentence = text_cleaning.clean_sentence(line) target_file.write('{}\n'.format(cleaned_sentence)) if index % 1000 == 0: print(index) print('Cleaned {} lines!'.format(index)) if args.training_csv is not None: training_transcripts = read_training_transcripts(args.training_csv) target_file.write('\n'.join(training_transcripts)) print('Added {} transcripts from training data!'.format(len(training_transcripts))) ================================================ FILE: facts/ASR/pre-processing/prepare_data.py ================================================ #! /usr/bin/env python """ 1. Load all corpora where a path is given. 2. Clean transcriptions. 3. Merge all corpora 4. Create Train/Dev/Test splits 5. Export for DeepSpeech """ import os import sys sys.path.append(os.path.abspath(os.path.join(__file__, os.path.pardir))) import argparse import audiomate from audiomate.corpus import io from audiomate.corpus import subset import text_cleaning def clean_transcriptions(corpus): for utterance in corpus.utterances.values(): ll = utterance.label_lists[audiomate.corpus.LL_WORD_TRANSCRIPT] for label in ll: label.value = text_cleaning.clean_sentence(label.value) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Prepare data for training.') parser.add_argument('target_path', type=str) parser.add_argument('--tuda', type=str) parser.add_argument('--voxforge', type=str) parser.add_argument('--swc', type=str) parser.add_argument('--mailabs', type=str) parser.add_argument('--cv', type=str) args = parser.parse_args() tuda_path = args.tuda voxforge_path = args.voxforge swc_path = args.swc mailabs_path = args.mailabs cv_path = args.cv corpora = [] if tuda_path is not None: tuda_corpus = audiomate.Corpus.load(tuda_path, reader='tuda') corpora.append(tuda_corpus) if voxforge_path is not None: voxforge_corpus = audiomate.Corpus.load( voxforge_path, reader='voxforge') corpora.append(voxforge_corpus) if swc_path is not None: swc_corpus = audiomate.Corpus.load(swc_path, reader='kaldi') corpora.append(swc_corpus) if mailabs_path is not None: mailabs_corpus = audiomate.Corpus.load(mailabs_path, reader='mailabs') corpora.append(mailabs_corpus) if cv_path is not None: cv_corpus = audiomate.Corpus.load(cv_path, reader='common-voice') corpora.append(cv_corpus) if len(corpora) <= 0: raise ValueError('No Corpus given!') merged_corpus = audiomate.Corpus.merge_corpora(corpora) clean_transcriptions(merged_corpus) splitter = subset.Splitter(merged_corpus, random_seed=38) splits = splitter.split_by_length_of_utterances( {'train': 0.7, 'dev': 0.15, 'test': 0.15}, separate_issuers=True) merged_corpus.import_subview('train', splits['train']) merged_corpus.import_subview('dev', splits['dev']) merged_corpus.import_subview('test', splits['test']) deepspeech_writer = io.MozillaDeepSpeechWriter() deepspeech_writer.save(merged_corpus, args.target_path) ================================================ FILE: facts/ASR/pre-processing/shuffle_split.py ================================================ import sys import pandas as pd from sklearn.model_selection import train_test_split df = pd.read_csv(sys.argv[1]) train, dev_test = train_test_split(df, test_size=0.3, shuffle=True, random_state=42) dev, test = train_test_split(dev_test, test_size=0.5, shuffle=True, random_state=42) train.to_csv('train.csv', index=False) dev.to_csv('dev.csv', index=False) test.to_csv('test.csv', index=False) ================================================ FILE: facts/ASR/pre-processing/utf_8.sh ================================================ #!/bin/bash # file name: to_utf8 # Coder: Aashish Agarwal FILES="voxforge/*/etc/prompts-original" #FILES="voxforge/*/etc/PROMPTS" for f in $FILES do #filename="${f%.*}" #echo -n "$f" #file -I $f encoding=$(file -i "$f" | sed "s/.*charset=\(.*\)$/\1/") #if file -I $f | grep -wq "iso-8859-1" if [ "${encoding}" = "iso-8859-1" ] then #mkdir -p converted #cp $f ./converted iconv -f ISO-8859-1 -t UTF-8 $f > "${f}_utf8.tex" mv "${f}_utf8.tex" $f echo ": CONVERTED TO UTF-8." else echo ": UTF-8 ALREADY." fi done ================================================ FILE: facts/GAN.md ================================================ Speical Contribution from Feng Yao who wrote the codes about classical GANs were implemented with simple network structure and tested by MNIST dataset. learn how others implement the GANs, and finally, enjoy the magic of GANs. *************** All have been tested with python2.7+ and tensorflow1.0+ in linux. * Datas: save training data. * Samples: save generated data, each folder contains several figs to **show the results**. * utils: contains 2 files - data.py: prepreocessing data. - nets.py: Generator and Discriminator are saved here. Note: > The final layer can be sigmoid(data: [0,1]) or tanh(data:[-1,1]), my codes all use sigmoid. > Using weights_initializer=tf.random_normal_initializer(0, 0.02) will converge faster. **DCGAN** - [x] conv **Conditional GAN** - [x] condition + mlp `D:G = 1:1` - [x] condition + conv(dcgan) `D:G=1:1 faster than mlp` - [x] dcgan + classifier `D:G:C = 1:1:1 very fast` - [x] wgan + classifier `D:G:C = 5:1:1 fast` Note: > a. The step ratio of G and D is important and it takes some time to reach the balance. Condition+mlp with D:G = 1:1 works better than 2:1. > b. Adding a classfier to trained with conditions and constraint G works faster and better than appending conditions to images for D training. **Wasserstein GAN** - [x] wgan + mlp `D:G = 5: 1 not good, need to *modify*` - [x] wgan + conv(dcgan) `D:G = 5:1 clip = 0.01` **infoGAN** - [x] infogan + mlp + D and Q not share `Q loss to update G(as feedback) not good ` - [x] infogan + mlp + D and Q share `not good. lacking numbers` - [x] infogan + conv + D and Q not share `clear and have 10 number` - [x] infogan + conv + D and Q share `the same with not share, not faster? ` - [ ] infogan + wgan + D and Q not share `to be done` Results are shown in the end of this page. # Adversarial Nets :sparkles: GAN -------- **The beginning.** The first paper. Two main research directions: 1. stabilize the training 2. apply GAN ### paper [[Generative Adversarial Nets]](https://arxiv.org/pdf/1406.2661.pdf) - **Loss** : ![GAN loss](README/images/gan.png) ### blog [[openai/generative-models]](https://blog.openai.com/generative-models/#contributions) (Motivation, Game Theory) [[wiseodd/gan-tensorflow]](http://wiseodd.github.io/techblog/2016/09/17/gan-tensorflow/) (Introduction, Implementation) *************** :sparkles:DCGAN -------- **stabilize the training with some architectural constraints.** GAN is hard to train. Stabilize Generative Adversarial networks with some architectural constraints. Popular used in cv. Most used architecture. ### paper [[Unsupervised Representation Learning with Deep Convolutional Generative Adversarial Networks]](https://arxiv.org/pdf/1511.06434.pdf) **Architecture guidelines for stable Deep Convolutional GANs** * Replace any pooling layers with strided convolutions (discriminator) and fractional-strided convolutions (generator). * Use batchnorm in both the generator and the discriminator * Remove fully connected hidden layers for deeper architectures. Just use average pooling at the end. * Use ReLU activation in generator for all layers except for the output, which uses Tanh. * Use LeakyReLU activation in the discriminator for all layers. ### blog [[bamos/deep-completion]](http://bamos.github.io/2016/08/09/deep-completion/) (Introduction, Implementation) ### code [[carpedm20/DCGAN-tensorflow]](https://github.com/carpedm20/DCGAN-tensorflow)( star 1.6k+, many files, easy to run, hard to read and modify) > G: fc-->reshape--> deconv bn relu (4) --> tanh > D: conv bn lrelu[leak=0.2] (4) --> reshape--> fc(opn=1)-->sigmoid > G Loss: > tf.reduce_mean(sigmoid_cross_entropy_with_logits(self.D_logits_,tf.ones_like(self.D_))) > D Loss: > tf.reduce_mean(sigmoid_cross_entropy_with_logits(self.D_logits, tf.ones_like(self.D))) + tf.reduce_mean(sigmoid_cross_entropy_with_logits(self.D_logits_, tf.zeros_like(self.D_))) > Solver: Adam lr=0.0002 stepG:stepD=1:1 > Data: mnist celebA __normalized to [-1,1] for tanh and [0,1] for sigmoid__ [[sugyan/tf-dcgan]](https://github.com/sugyan/tf-dcgan)( star 20, easy to read, just 1 dcgan file. dcgan, generator, discriminator all class. not tf.contrib.layers. ) > G: fc-->reshape--> deconv bn relu (4) --> tanh > D: conv bn lrelu[leak=0.2] (4) --> reshape--> fc(opn=2 one-hot?) > Losses: softmax_cross_entropy_with_logits > Solver: Adam lr=0.0002 stepG:stepD=1:1 *************** :sparkles:Conditional GAN -------- **Apply GAN by adding condition(supervised)** Add conditions to GAN by feeding y to G. G(z)-->G(z,y) D(X)-->D(X,y) - z: the same as in GAN. unconstrained noise to generate a image. - y: the condition to constraint the network. supervised. A very important structure that used in image applications (data augmentation, image transfer, etc.) Make GAN useful. ### paper [[Conditional Generative Adversarial Nets]](https://arxiv.org/pdf/1411.1784.pdf) - **Loss** : ![CGAN loss](README/images/cgan.png) ### blog [[wiseodd/conditional-gan-tensorflow]](http://wiseodd.github.io/techblog/2016/12/24/conditional-gan-tensorflow/) (Fomulation, Architecture, Implementation) ### code [[wiseodd/conditional_gan]](https://github.com/wiseodd/generative-models/blob/master/GAN/conditional_gan/cgan_tensorflow.py)(star 500+, very simple, 1 file, easy to read and run, not conv, inconvinient to extend) > G: concat(z,y)-->fc-->sigmoid > D: concat(z,y)-->fc-->sigmoid loss > Solver: Adam lr=0.001 stepG:stepD=1:1 > Data: mnist __[0,1]__ [[zhangqianhui/Conditional-Gans]](https://github.com/zhangqianhui/Conditional-Gans)(star 16, easy to extend) > G: concat(z,y)-->fc-->conv-->sigmoid > D: **conv_concat(x,y)**-->conv-->fc-->sigmoid loss > Solver: Adam lr=0.0002 stepG:stepD=2:1 > Data: mnist __[0,1]__ [[fairytale0011/Conditional-WassersteinGAN]](https://github.com/fairytale0011/Conditional-WassersteinGAN/blob/master/WGAN_AC.py) (star 6. use wgan to train GAN, use separate classifier to enforce the condition. very clear, easy to read and modify) > G: concat(z,y)-->fc-->conv-->tanh > D: X-->conv-->fc-->sigmoid loss > **classifier**: X-->conv-->fc-->softmax loss (real label to train classifier, fake label to train G) > clip D var > Solver: RMS lr=0.0002 stepG:stepD:stepC_real:stepC_fake=1:10:0.5:0.5 > Data: mnist __[-1,1]__ *************** :sparkles:Wasserstein GAN -------- **stabilize the training by using Wasserstein-1 distance** GAN before using JS divergence has the problem of non-overlapping, leading to mode collapse and convergence difficulty. Use EM distance or Wasserstein-1 distance, so GAN solve the two problems above without particular architecture (like dcgan). ### paper [[Wasserstein GAN]](https://arxiv.org/pdf/1701.07875.pdf) **Algorithm guidelines for stable GANs** * No log in the loss. The output of D is no longer a probability, hence we do not apply sigmoid at the output of D * Clip the weight of D (0.01) * Train D more than G (5:1) * Use RMSProp instead of ADAM * Lower learning rate (0.00005) ### blog [[AidenN/WassersteinGAN]](https://paper.dropbox.com/doc/Wasserstein-GAN-GvU0p2V9ThzdwY3BbhoP7) (Theory) [[wiseodd/wasserstein-gan]](http://wiseodd.github.io/techblog/2017/02/04/wasserstein-gan/) (Introduction, Implementation) [[zhihu/Wassertein GAN]](https://zhuanlan.zhihu.com/p/25071913) (Introduction, Analysis) ### code [[wiseodd/wgan_tensorflow]](https://github.com/wiseodd/generative-models/blob/master/GAN/wasserstein_gan/wgan_tensorflow.py)(very simple, use mlp) > G: fc-->sigmoid > D: fc clip D > G Loss: > G_loss = -tf.reduce_mean(D_fake) > D Loss: > D_loss = tf.reduce_mean(D_fake) - tf.reduce_mean(D_real) > Solver: RMSProp lr=0.0001 stepG:stepD=1:5 **************** :sparkles:InfoGAN -------- **Apply GAN by learning conditions(unsupervised)** Attempt to make conditional learned automatically. Find and control some useful information in the images. - z: the same as in GAN. unconstrainted noise to generate a image. - c: like c in conditional GAN, but learned by Q instead of given what that is, unsupervised. ### paper [[InfoGAN: Interpretable Representation Learning by Information Maximizing Generative Adversarial Nets]](https://arxiv.org/pdf/1606.03657.pdf) - **Loss** : ![infoGAN loss 1](README/images/infogan1.png) Define: Q(c|x) to approximate P(c|x)(which is the conditional distribution) ![infoGAN loss 2](README/images/infogan2.png) ### blog [[wiseodd/infogan]](http://wiseodd.github.io/techblog/2017/01/29/infogan/) (Introduction Implementation) ### code [[openai/infogan]](https://github.com/openai/InfoGAN)( star 300+, hard to read and modify, too much files) > G: fc(1024 bn relu)-->fc (bn relu) reshape--> deconv bn relu --> deconv flatten--> activate > D and Q: > shared: reshape-->conv lrelu --> conv bn lrelu --> fc bn lrelu > D: fc(1) --> activate > Q: fc(128) bn lrelu --> fc (c_dim) --> activate > activate: softmax(Categorical) / mean stddev:sqrt(e^x) (Gaussian) / sigmoid(Bernoulli) > Losses: > D and G: sigmoid as DCGAN > Q: cross entropy loss (softmax for discrete) ** add Q loss to D and G loss ** > Solver: > Adam beta1=0.5 stepG:stepD=1:1 *my understanding*: > Adding Q loss to D and G loss, then updating D var and G var by 1:1 **equal to** Q loss to update Q(D) var and G var by Q:D:G=2:1:1 [[wiseodd/infogan-tensorflow]](https://github.com/wiseodd/generative-models/blob/master/GAN/infogan/infogan_tensorflow.py)(also simple, use mlp) > Q: fc --> softmax with c (not share with D) update vars: G and Q > Solver: Adam G:D:Q = 1:1:1 # Results All images shown here are from `./Samples`, for more information about results, please go to the folder. Then first number of image name is the trianing epoches. For better observing, I keep generated images in the beginning, middle and final training process for each algorithm. **Conditional GAN** - [x] condition + mlp `each pic has the same condition` ![cgan](README/results/cgan_mlp.png) - [x] condition + conv(dcgan) `each row has the same condition: [0 0 0 0; 1 1 1 1; 2 2 2 2; 3 3 3 3]` ![cgan](https://raw.githubusercontent.com/YadiraF/GAN/master/Samples/mnist_cgan_conv/039_9.png) - [x] dcgan + classifier `condition: [0 1 2 3; 4 5 6 7; 0 1 2 3; 4 5 6 7] ` ![cgan](https://raw.githubusercontent.com/YadiraF/GAN/master/Samples/mnist_cgan_classifier/348_8.png) - [x] wgan + classifier `condition: [0 1 2 3; 4 5 6 7; 0 1 2 3; 4 5 6 7] ` ![cgan](https://raw.githubusercontent.com/YadiraF/GAN/master/Samples/mnist_cgan_wgan_classifier/030_0.png) **InfoGAN** The generated images with the same condition belong to the same category. - [x] infogan + conv + D and Q not share `condition: [0 1 2 3; 4 5 6 7; 8 9 0 1; 2 3 4 5] ` ![cgan](https://raw.githubusercontent.com/YadiraF/GAN/master/Samples/mnist_infogan_conv_without_share/058_8.png) - [x] infogan + conv + D and Q share `condition: [0 1 2 3; 4 5 6 7; 8 9 0 1; 2 3 4 5] ` ![cgan](https://raw.githubusercontent.com/YadiraF/GAN/master/Samples/mnist_infogan_conv/095_5.png) **DCGAN** * 3D face result (dcgan) ![3d face](README/results/face3D_dcgan.png) # Others Tensorflow style: https://www.tensorflow.org/community/style_guide tf.concat(1,[x,y]) in tf 0.12- ---> tf.concat([x,y],1) in tf 1.0+. use tf.get_Variable or tf.contrib.layers to reuse variables. A good website to convert latex equation to img(then insert into README): http://www.sciweavers.org/free-online-latex-equation-editor ================================================ FILE: facts/chrütertee.java ================================================ // 1. kau·der·welsch // 2. gruezi - // 3. chrütertee - tea with herbs dataType[] Wort; // declare an array double[] chrütertee; // allocate memory chrütertee = new double[10]; double[] chrütertee = new double[10]; //declare and initialize and array int[] chrütertee = {12, 4, 5, 2, 5}; class Main { public static void main(String[] args) { // create an array int[] age = {12, 4, 5, 2, 5}; // access each array elements System.out.println("Accessing Elements of Array:"); System.out.println("First Element: " + age[0]); System.out.println("Second Element: " + age[1]); System.out.println("Third Element: " + age[2]); System.out.println("Fourth Element: " + age[3]); System.out.println("Fifth Element: " + age[4]); } } //for loop in Java using chrütertee for (chrütertee 1; chrütertee 2; chrütertee 3) { // code block to be executed } for (int chrütertee = 0; chrütertee < 5; chrütertee++) { System.out.println(i); } for (int schnäbichetsche = 0; i <= 10; i = i + 2) { System.out.println(i); } //syntax for (type variableName : arrayName) { // code block to be executed } // String[] habibi = {"schnäbichetsche ", "chrütertee", "wallah", "habibi"}; for (String i : wallah) { System.out.println(i); } //ETH: Eidgenössische Technische Hochschule Zürich wallah: ================================================ FILE: facts/semantiks.md ================================================ # Semantic-Segmentation - NLP technologies deconstruct words, sentences, paragraphs, and entire documents expressed in human language and map them onto a semantic structure that can be used by a computer. A list of all papers and resoureces on Semantic Segmentation. # Dataset importance ![Dataset importance plot](http://www.it-caesar.com/github/Dataset_importance.png?raw=true "Dataset importance plot") # SemanticSegmentation_DL Some implementation of semantic segmantation for DL model
## Dataset + [voc2012](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/) + [CitySpaces](https://www.cityscapes-dataset.com/) + [Mapillary](https://www.mapillary.com/dataset/vistas) + [ADE20K](http://groups.csail.mit.edu/vision/datasets/ADE20K/) + [PASCAL Context](http://www.cs.stanford.edu/~roozbeh/pascal-context/) + [COCO-Stuff 10K dataset v1.1](https://github.com/nightrome/cocostuff) + [2D-3D-S dataset](http://buildingparser.stanford.edu/dataset.html) + [Mapillary Vistas](https://www.youtube.com/karolmajek) + [Stanford Background Dataset](http://dags.stanford.edu/projects/scenedataset.html) + [Sift Flow Dataset](http://people.csail.mit.edu/celiu/SIFTflow/) + [Barcelona Dataset](http://www.cs.unc.edu/~jtighe/Papers/ECCV10/) + [Microsoft COCO dataset](http://mscoco.org/) + [MSRC Dataset](http://research.microsoft.com/en-us/projects/objectclassrecognition/) + [LITS Liver Tumor Segmentation Dataset](https://competitions.codalab.org/competitions/15595) + [KITTI](http://www.cvlibs.net/datasets/kitti/eval_road.php) + [Pascal Context](http://www.cs.stanford.edu/~roozbeh/pascal-context/) + [Data from Games dataset](https://download.visinf.tu-darmstadt.de/data/from_games/) + [Human parsing dataset](https://github.com/lemondan/HumanParsing-Dataset) + [Mapillary Vistas Dataset](https://www.mapillary.com/dataset/vistas) + [Microsoft AirSim](https://github.com/Microsoft/AirSim) + [MIT Scene Parsing Benchmark](http://sceneparsing.csail.mit.edu/) + [COCO 2017 Stuff Segmentation Challenge](http://cocodataset.org/#stuff-challenge2017) + [ADE20K Dataset](http://groups.csail.mit.edu/vision/datasets/ADE20K/) + [INRIA Annotations for Graz-02](http://lear.inrialpes.fr/people/marszalek/data/ig02/) + [Daimler dataset](http://www.gavrila.net/Datasets/Daimler_Pedestrian_Benchmark_D/daimler_pedestrian_benchmark_d.html) + [ISBI Challenge: Segmentation of neuronal structures in EM stacks](http://brainiac2.mit.edu/isbi_challenge/) + [INRIA Annotations for Graz-02 (IG02)](https://lear.inrialpes.fr/people/marszalek/data/ig02/) + [Pratheepan Dataset](http://cs-chan.com/downloads_skin_dataset.html) + [Clothing Co-Parsing (CCP) Dataset](https://github.com/bearpaw/clothing-co-parsing) ## Resources ## Survey papers - A 2017 Guide to Semantic Segmentation with Deep Learning by Qure AI [[Blog about different sem. segm. methods]](http://blog.qure.ai/notes/semantic-segmentation-deep-learning-review) - A Review on Deep Learning Techniques Applied to Semantic Segmentation [[Survey paper with a special focus on datasets and the highest performing methods]](https://arxiv.org/abs/1704.06857) - Computer Vision for Autonomous Vehicles: Problems, Datasets and State-of-the-Art [[Survey paper about all aspects of autonomous vehicles, including sem. segm.]](https://arxiv.org/abs/1704.05519) [[Webpage with a summary of all relevant publications]](http://www.cvlibs.net/projects/autonomous_vision_survey/) - A Survey on Deep Learning in Medical Image Analysis [[Paper]](https://arxiv.org/pdf/1702.05747） ## Online demos - [CRF as RNN](http://www.robots.ox.ac.uk/~szheng/crfasrnndemo) - [SegNet](http://mi.eng.cam.ac.uk/projects/segnet/demo.php#demo) ## 2D Semantic Segmentation ### Papers: - [2019-CVPR oral] CLAN: Category-level Adversaries for Semantics Consistent [[`paper`]](https://arxiv.org/abs/1809.09478?context=cs) [[`code`]](https://github.com/RoyalVane/CLAN) - [2019-CVPR] BRS: Interactive Image Segmentation via Backpropagating Refinement Scheme(***) [[`paper`]](https://vcg.seas.harvard.edu/publications/interactive-image-segmentation-via-backpropagating-refinement-scheme/paper) [[`code`]](https://github.com/wdjang/BRS-Interactive_segmentation) - [2019-CVPR] DFANet：Deep Feature Aggregation for Real-Time Semantic Segmentation(used in camera) [[`paper`]](https://share.weiyun.com/5NgHbWH) [[`code`]](https://github.com/j-a-lin/DFANet_PyTorch) - [2019-CVPR] DeepCO3: Deep Instance Co-segmentation by Co-peak Search and Co-saliency [[`paper`]](http://cvlab.citi.sinica.edu.tw/images/paper/cvpr-hsu19.pdf) [[`code`]](https://github.com/KuangJuiHsu/DeepCO3) - [2019-CVPR] Domain Adaptation(reducing the domain shif) [[`paper`]](http://openaccess.thecvf.com/content_CVPR_2019/papers/Luo_Taking_a_Closer_Look_at_Domain_Shift_Category-Level_Adversaries_for_CVPR_2019_paper.pdf) - [2019-CVPR] ELKPPNet: An Edge-aware Neural Network with Large Kernel Pyramid Pooling for Learning Discriminative Features in Semantic- Segmentation [[`paper`]](https://arxiv.org/abs/1906.11428) [[`code`]](https://github.com/XianweiZheng104/ELKPPNet) - [2019-CVPR oral] GLNet: Collaborative Global-Local Networks for Memory-Efficient Segmentation of Ultra-High Resolution Images[[`paper`]](https://arxiv.org/abs/1905.06368) [[`code`]](https://github.com/chenwydj/ultra_high_resolution_segmentation) - [2019-CVPR] Instance Segmentation by Jointly Optimizing Spatial Embeddings and Clustering Bandwidth(***SOTA) [[`paper`]](https://arxiv.org/abs/1906.11109) [[`code`]](https://github.com/davyneven/SpatialEmbeddings) - [2019-ECCV] ICNet: Real-Time Semantic Segmentation on High-Resolution Images [[`paper`]](https://arxiv.org/abs/1704.08545) [[`code`]](https://github.com/oandrienko/fast-semantic-segmentation) - [2019-CVPR] LEDNet: A Lightweight Encoder-Decoder Network for Real-Time Semantic Segmentation(***SOTA) [[`paper`]](https://arxiv.org/abs/1905.02423) [[`code`]](https://github.com/xiaoyufenfei/LEDNet) - [2019-arXiv] LightNet++: Boosted Light-weighted Networks for Real-time Semantic Segmentation [[`paper`]](http://arxiv.org/abs/1605.02766) [[`code`]](https://github.com/ansleliu/LightNetPlusPlus) - [2019-CVPR] PTSNet: A Cascaded Network for Video Object Segmentation [[`paper`]](https://arxiv.org/abs/1907.01203) [[`code`]](https://github.com/sydney0zq/PTSNet) - [2019-CVPR] PPGNet: Learning Point-Pair Graph for Line Segment Detection [[`paper`]](https://www.aiyoggle.me/publication/ppgnet-cvpr19/ppgnet-cvpr19.pdf) [[`code`]](https://github.com/svip-lab/PPGNet) - [2019-CVPR] Show, Match and Segment: Joint Learning of Semantic Matching and Object Co-segmentation [[`paper`]](https://arxiv.org/abs/1906.05857) [[`code`]](https://github.com/YunChunChen/MaCoSNet-pytorch) - [2019-CVPR] Video Instance Segmentation [[`paper`]](https://arxiv.org/abs/1905.04804) [[`code`]](https://github.com/youtubevos/MaskTrackRCNN) + Arxiv-2018 ExFuse: Enhancing Feature Fusion for Semantic Segmentation 87.9% mean Iou->voc2012 [[Paper]](https://arxiv.org/pdf/1804.03821.pdf) + CVPR-2018 spotlight Learning to Adapt Structured Output Space for Semantic Segmentation [[Paper]](https://arxiv.org/abs/1802.10349) [[Code]](https://github.com/wasidennis/AdaptSegNet) + Arfix-2018 Adversarial Learning for Semi-supervised Semantic Segmentation [[Paper]](https://arxiv.org/abs/1802.07934) [[Code]](https://github.com/hfslyc/AdvSemiSeg) + Arxiv-2018 Context Encoding for Semantic Segmentation [[Paper]](https://arxiv.org/pdf/1803.08904.pdf) [[Code]](https://github.com/zhanghang1989/MXNet-Gluon-SyncBN) + CVPR-2018 Learning to Adapt Structured Output Space for Semantic Segmentation [[Paper]](https://arxiv.org/abs/1802.10349)[[Code]](https://github.com/wasidennis/AdaptSegNet) + CVPR-2018 Dynamic-structured Semantic Propagation Network [[Paper]](https://arxiv.org/abs/1803.06067) + Deeplab v4: Encoder-Decoder with Atrous Separable Convolution for Semantic Image Segmentation [[Paper]](https://arxiv.org/pdf/1802.02611.pdf) [[Code]](https://github.com/tensorflow/models/tree/master/research/deeplab) + Deep Value Networks Learn to Evaluate and Iteratively Refine Structured Outputs [[Paper]](https://arxiv.org/pdf/1703.04363.pdf)[[Code]](https://github.com/gyglim/dvn) + ICCV-2017 Semantic Line Detection and Its Applications [[Paper]](http://openaccess.thecvf.com/content_ICCV_2017/papers/Lee_Semantic_Line_Detection_ICCV_2017_paper.pdf) + ICCV-2017 Attentive Semantic Video Generation Using Captions [[Paper]](http://openaccess.thecvf.com/content_ICCV_2017/papers/Marwah_Attentive_Semantic_Video_ICCV_2017_paper.pdf) + ICCV-2017 BlitzNet: A Real-Time Deep Network for Scene Understanding [[Paper]](https://arxiv.org/pdf/1708.02813.pdf) [[Code]](https://github.com/dvornikita/blitznet) + ICCV-2017 SCNet: Learning Semantic Correspondence [[Code]](https://github.com/k-han/SCNet) + CVPR-2017 End-to-End Instance Segmentation with Recurrent Attention [[Code]](https://github.com/renmengye/rec-attend-public) + CVPR-2017 Deep Watershed Transform for Instance Segmentation [[Code]](https://github.com/min2209/dwt) + Piecewise Flat Embedding for Image Segmentation [[Paper]](https://pdfs.semanticscholar.org/4690/3c0ca5540e312b8f4c20c012f586e5071914.pdf) + ICCV-2017 Curriculum Domain Adaptation for Semantic Segmentation of Urban Scenes [[Paper]](https://arxiv.org/abs/1707.09465)[[Code]](https://github.com/YangZhang4065/AdaptationSeg) + CVPR-2017 Not All Pixels Are Equal: Difficulty-Aware Semantic Segmentation via Deep Layer Cascade-2017 [[Paper]](https://arxiv.org/abs/1704.01344) + CVPR-2017 Annotating Object Instances with a Polygon-RNN-2017 [[Project]](http://www.cs.toronto.edu/polyrnn/) [[Paper]](https://arxiv.org/abs/1704.05548) + CVPR-2017 Loss maxpooling for semantic image segmentation [[Paper]](http://openaccess.thecvf.com/content_cvpr_2017/papers/Bulo_Loss_Max-Pooling_for_CVPR_2017_paper.pdf) + ICCV-2017 Scale-adaptive convolutions for scene parsing [[Paper]](https://pdfs.semanticscholar.org/f984/781ccef66e6a6114707271b8bb29148ad45d.pdf) + Towards End-to-End Lane Detection: an Instance Segmentation Approach [[Paper]](https://arxiv.org/pdf/1802.05591.pdf)arxiv-1802 + AAAI-2018 Mix-and-Match Tuning for Self-Supervised Semantic Segmentation [[Paper]](https://arxiv.org/pdf/1712.00661.pdf) arxiv-1712 + NIPS-2017-Learning Affinity via Spatial Propagation Networks [[Paper]](https://papers.nips.cc/paper/6750-learning-affinity-via-spatial-propagation-networks.pdf) + AAAI-2018-Spatial As Deep: Spatial CNN for Traffic Scene Understanding [[Paper]](https://arxiv.org/pdf/1712.06080.pdf) + Stacked Deconvolutional Network for Semantic Segmentation-2017 [[Paper]](https://arxiv.org/pdf/1708.04943.pdf)
+ Deeplab v3: Rethinking Atrous Convolution for Semantic Image Segmentation-2017(DeeplabV3) [[Paper]](https://arxiv.org/pdf/1706.05587.pdf)
+ CVPR-2017 Learning Object Interactions and Descriptions for Semantic Image Segmentation-2017 [[Paper]](http://personal.ie.cuhk.edu.hk/~pluo/pdf/wangLLWcvpr17.pdf)
+ Pixel Deconvolutional Networks-2017 [[Code-Tensorflow]](https://github.com/HongyangGao/PixelDCN) [[Paper]](https://arxiv.org/abs/1705.06820)
+ Dilated Residual Networks-2017 [[Paper]](http://vladlen.info/papers/DRN.pdf)
+ A Review on Deep Learning Techniques Applied to Semantic Segmentation-2017 [[Paper]](https://arxiv.org/abs/1704.06857)
+ BiSeg: Simultaneous Instance Segmentation and Semantic Segmentation with Fully Convolutional Networks [[Paper]](https://arxiv.org/abs/1706.02135)
+ ICNet for Real-Time Semantic Segmentation on High-Resolution Images-2017 [[Project]](https://hszhao.github.io/projects/icnet/) [[Code]](https://github.com/hszhao/ICNet) [[Paper]](https://arxiv.org/abs/1704.08545) [[Video]](https://www.youtube.com/watch?v=qWl9idsCuLQ)
- Feature Forwarding: Exploiting Encoder Representations for Efficient Semantic Segmentation-2017 [[Project]](https://codeac29.github.io/projects/linknet/) [[Code-Torch7]](https://github.com/e-lab/LinkNet)
- Reformulating Level Sets as Deep Recurrent Neural Network Approach to Semantic Segmentation-2017 [[Paper]](https://arxiv.org/abs/1704.03593)
- Adversarial Examples for Semantic Image Segmentation-2017 [[Paper]](https://arxiv.org/abs/1703.01101)
- Large Kernel Matters - Improve Semantic Segmentation by Global Convolutional Network-2017 [[Paper]](https://arxiv.org/abs/1703.02719)
- HyperNet: Towards Accurate Region Proposal Generation and Joint Object Detection [[Paper]](https://arxiv.org/pdf/1604.00600) - Hypercolumns for Object Segmentation and Fine-grained Localization [[Paper]](https://arxiv.org/pdf/1411.5752) - Matching-CNN meets KNN: Quasi-parametric human parsing[[Paper]](https://www.cv-foundation.org/openaccess/content_cvpr_2015/app/1B_034_ext.pdf) - Deep Human Parsing with Active Template Regression [[Paper]](https://arxiv.org/pdf/1503.02391.pdf) - TPAMI-2012 **Learning Hierarchical Features for Scene Labeling** The first paper for applying dl on semantic segmentation !!! [[Paper]](http://yann.lecun.com/exdb/publis/pdf/farabet-pami-13.pdf) - Label Refinement Network for Coarse-to-Fine Semantic Segmentation-2017 [[Paper]](https://www.arxiv.org/abs/1703.00551) - Laplacian Pyramid Reconstruction and Refinement for Semantic Segmentation [[Paper]](https://arxiv.org/pdf/1605.02264.pdf) - ParseNet: Looking Wider to See Better [[Paper]](https://www.cs.unc.edu/~wliu/papers/parsenet.pdf) - CVPR-2016 Recombinator Networks: Learning Coarse-to-Fine Feature Aggregation [[Paper]](http://openaccess.thecvf.com/content_cvpr_2016/papers/Honari_Recombinator_Networks_Learning_CVPR_2016_paper.pdf) - **PixelNet: Representation of the pixels, by the pixels, and for the pixels-2017** [[Project]](http://www.cs.cmu.edu/~aayushb/pixelNet/) [[Code-Caffe]](https://github.com/aayushbansal/PixelNet) [[Paper]](https://arxiv.org/abs/1702.06506)
- LabelBank: Revisiting Global Perspectives for Semantic Segmentation-2017 [[Paper]](https://arxiv.org/abs/1703.09891)
- Progressively Diffused Networks for Semantic Image Segmentation-2017 [[Paper]](https://arxiv.org/abs/1702.05839)
- Understanding Convolution for Semantic Segmentation-2017 [[Model-Mxnet]](https://drive.google.com/drive/folders/0B72xLTlRb0SoREhISlhibFZTRmM) [[Paper]](https://arxiv.org/abs/1702.08502) [[Code]](https://github.com/TuSimple/TuSimple-DUC)
- ICCV-2017 Predicting Deeper into the Future of Semantic Segmentation-2017 [[Paper]](https://arxiv.org/abs/1703.07684)
- CVPR-2017 **Pyramid Scene Parsing Network-2017** [[Project]](https://hszhao.github.io/projects/pspnet/) [[Code-Caffe]](https://github.com/hszhao/PSPNet) [[Paper]](https://arxiv.org/abs/1612.01105) [[Slides]](http://image-net.org/challenges/talks/2016/SenseCUSceneParsing.pdf)
- FCNs in the Wild: Pixel-level Adversarial and Constraint-based Adaptation-2016 [[Paper]](https://arxiv.org/abs/1612.02649)
- FusionNet: A deep fully residual convolutional neural network for image segmentation in connectomics-2016 [[Code-PyTorch]](https://github.com/GunhoChoi/FusionNet_Pytorch) [[Paper]](https://arxiv.org/abs/1612.05360)
- RefineNet: Multi-Path Refinement Networks for High-Resolution Semantic Segmentation-2016 [[Code-MatConvNet]](https://github.com/guosheng/refinenet) [[Paper]](https://arxiv.org/abs/1611.06612)
- CVPRW-2017 The One Hundred Layers Tiramisu: Fully Convolutional DenseNets for Semantic Segmentation [[Code-Theano]](https://github.com/SimJeg/FC-DenseNet) [[Code-Keras1]](https://github.com/titu1994/Fully-Connected-DenseNets-Semantic-Segmentation) [[Code-Keras2]](https://github.com/0bserver07/One-Hundred-Layers-Tiramisu) [[Paper]](https://arxiv.org/abs/1611.09326)
- CVPR-2017 Full-Resolution Residual Networks for Semantic Segmentation in Street Scenes [[Code-Theano]](https://github.com/TobyPDE/FRRN) [[Paper]](https://arxiv.org/abs/1611.08323)
- PixelNet: Towards a General Pixel-level Architecture-2016 [[Paper]](http://arxiv.org/abs/1609.06694)
- Recalling Holistic Information for Semantic Segmentation-2016 [[Paper]](https://arxiv.org/abs/1611.08061)
- Semantic Segmentation using Adversarial Networks-2016 [[Paper]](https://arxiv.org/abs/1611.08408) [[Code-Chainer]](https://github.com/oyam/Semantic-Segmentation-using-Adversarial-Networks)
- Region-based semantic segmentation with end-to-end training-2016 [[Paper]](http://arxiv.org/abs/1607.07671)
- Exploring Context with Deep Structured models for Semantic Segmentation-2016 [[Paper]](https://arxiv.org/abs/1603.03183) - Boundary-aware Instance Segmentation-2016 [[Paper]](https://infoscience.epfl.ch/record/227439/files/HayderHeSalzmannCVPR17.pdf)
- Improving Fully Convolution Network for Semantic Segmentation-2016 [[Paper]](https://arxiv.org/abs/1611.08986)
- Deep Structured Features for Semantic Segmentation-2016 [[Paper]](https://arxiv.org/abs/1609.07916)
- DeepLab v2:Semantic Image Segmentation with Deep Convolutional Nets, Atrous Convolution, and Fully Connected CRFs-2016** [[Project]](http://liangchiehchen.com/projects/DeepLab.html) [[Code-Caffe]](https://bitbucket.org/deeplab/deeplab-public/) [[Code-Tensorflow]](https://github.com/DrSleep/tensorflow-deeplab-resnet) [[Code-PyTorch]](https://github.com/isht7/pytorch-deeplab-resnet) [[Paper]](https://arxiv.org/abs/1606.00915)
- DeepLab v1: Semantic Image Segmentation With Deep Convolutional Nets and Fully Connected CRFs-2014** [[Code-Caffe1]](https://bitbucket.org/deeplab/deeplab-public/) [[Code-Caffe2]](https://github.com/TheLegendAli/DeepLab-Context) [[Paper]](http://arxiv.org/abs/1412.7062)
- Deep Learning Markov Random Field for Semantic Segmentation-2016 [[Project]](http://personal.ie.cuhk.edu.hk/~lz013/projects/DPN.html) [[Paper]](https://arxiv.org/abs/1606.07230)
- ECCV2016 Salient Deconvolutional Networks [[Code]](https://github.com/aravindhm/deconvnet_analysis) - Convolutional Random Walk Networks for Semantic Image Segmentation-2016 [[Paper]](https://arxiv.org/abs/1605.07681)
- ENet: A Deep Neural Network Architecture for Real-Time Semantic Segmentation-2016 [[Code-Caffe1]](https://github.com/e-lab/ENet-training)[[Code-Caffe2]](https://github.com/TimoSaemann/ENet) [[Paper]](https://arxiv.org/abs/1606.02147) [[Blog]](https://culurciello.github.io/tech/2016/06/20/training-enet.html)
- High-performance Semantic Segmentation Using Very Deep Fully Convolutional Networks-2016 [[Paper]](https://arxiv.org/abs/1604.04339)
- CVPR-2016-oral ScribbleSup: Scribble-Supervised Convolutional Networks for Semantic Segmentation-2016 [[Paper]](http://arxiv.org/abs/1604.05144)
- Object Boundary Guided Semantic Segmentation-2016 [[Code-Caffe]](https://github.com/twtygqyy/obg_fcn) [[Paper]](http://arxiv.org/abs/1603.09742)
- Segmentation from Natural Language Expressions-2016 [[Project]](http://ronghanghu.com/text_objseg/) [[Code-Tensorflow]](https://github.com/ronghanghu/text_objseg) [[Code-Caffe]](https://github.com/Seth-Park/text_objseg_caffe) [[Paper]](http://arxiv.org/abs/1603.06180)
- Seed, Expand and Constrain: Three Principles for Weakly-Supervised Image Segmentation-2016 [[Code-Caffe]](https://github.com/kolesman/SEC) [[Paper]](https://arxiv.org/abs/1603.06098)
- Global Deconvolutional Networks for Semantic Segmentation-2016 [[Paper]](https://arxiv.org/abs/1602.03930)
- Learning Transferrable Knowledge for Semantic Segmentation with Deep Convolutional Neural Network-2015 [[Project]](http://cvlab.postech.ac.kr/research/transfernet/) [[Code-Caffe]](https://github.com/maga33/TransferNet) [[Paper]](http://arxiv.org/abs/1512.07928)
- Learning Dense Convolutional Embeddings for Semantic Segmentation-2015 [[Paper]](https://arxiv.org/abs/1511.04377)
- ParseNet: Looking Wider to See Better-2015 [[Code-Caffe]](https://github.com/weiliu89/caffe/tree/fcn) [[Model-Caffe]](https://github.com/BVLC/caffe/wiki/Model-Zoo#parsenet-looking-wider-to-see-better) [[Paper]](http://arxiv.org/abs/1506.04579)
- Decoupled Deep Neural Network for Semi-supervised Semantic Segmentation-2015 [[Project]](http://cvlab.postech.ac.kr/research/decouplednet/) [[Code-Caffe]](https://github.com/HyeonwooNoh/DecoupledNet) [[Paper]](http://arxiv.org/abs/1506.04924)
- Bayesian segnet: Model uncertainty in deep convolutional encoder-decoder architectures for scene understanding [[Paper]](https://arxiv.org/pdf/1511.02680.pdf) - **SegNet: A Deep Convolutional Encoder-Decoder Architecture for Image Segmentation-2015** [[Project]](http://mi.eng.cam.ac.uk/projects/segnet/) [[Code-Caffe]](https://github.com/alexgkendall/caffe-segnet) [[Paper]](http://arxiv.org/abs/1511.00561) [[Tutorial1]](http://mi.eng.cam.ac.uk/projects/segnet/tutorial.html) [[Tutorial2]](https://github.com/alexgkendall/SegNet-Tutorial)
- Semantic Image Segmentation with Task-Specific Edge Detection Using CNNs and a Discriminatively Trained Domain Transform-2015 [[Paper]](https://arxiv.org/abs/1511.03328)
- Semantic Segmentation with Boundary Neural Fields-2015 [[Code]](https://github.com/gberta/BNF_globalization) [[Paper]](https://arxiv.org/abs/1511.02674)
- Semantic Image Segmentation via Deep Parsing Network-2015 [[Project]](http://personal.ie.cuhk.edu.hk/~lz013/projects/DPN.html) [[Paper1]](http://arxiv.org/abs/1509.02634) [[Paper2]](http://www.cv-foundation.org/openaccess/content_iccv_2015/papers/Liu_Semantic_Image_Segmentation_ICCV_2015_paper.pdf) [[Slides]](http://personal.ie.cuhk.edu.hk/~pluo/pdf/presentation_dpn.pdf)
- What’s the Point: Semantic Segmentation with Point Supervision-2015 [[Project]](http://vision.stanford.edu/whats_the_point/) [[Code-Caffe]](https://github.com/abearman/whats-the-point1) [[Model-Caffe]](http://vision.stanford.edu/whats_the_point/models.html) [[Paper]](https://arxiv.org/abs/1506.02106)
- U-Net: Convolutional Networks for Biomedical Image Segmentation-2015 [[Project]](http://lmb.informatik.uni-freiburg.de/people/ronneber/u-net/) [[Code+Data]](http://lmb.informatik.uni-freiburg.de/people/ronneber/u-net/u-net-release-2015-10-02.tar.gz) [[Code-Keras]](https://github.com/orobix/retina-unet) [[Code-Tensorflow]](https://github.com/jakeret/tf_unet) [[Paper]](http://arxiv.org/abs/1505.04597) [[Notes]](http://zongwei.leanote.com/post/Pa)
- Learning Deconvolution Network for Semantic Segmentation(DeconvNet)-2015 [[Project]](http://cvlab.postech.ac.kr/research/deconvnet/) [[Code-Caffe]](https://github.com/HyeonwooNoh/DeconvNet) [[Paper]](http://arxiv.org/abs/1505.04366) [[Slides]](http://web.cs.hacettepe.edu.tr/~aykut/classes/spring2016/bil722/slides/w06-deconvnet.pdf)
- Multi-scale Context Aggregation by Dilated Convolutions-2015 [[Project]](http://vladlen.info/publications/multi-scale-context-aggregation-by-dilated-convolutions/) [[Code-Caffe]](https://github.com/fyu/dilation) [[Code-Keras]](https://github.com/nicolov/segmentation_keras) [[Paper]](http://arxiv.org/abs/1511.07122) [[Notes]](http://www.inference.vc/dilated-convolutions-and-kronecker-factorisation/)
- ReSeg: A Recurrent Neural Network-based Model for Semantic Segmentation-2015 [[Code-Theano]](https://github.com/fvisin/reseg) [[Paper]](https://arxiv.org/abs/1511.07053)
- ICCV-2015 BoxSup: Exploiting Bounding Boxes to Supervise Convolutional Networks for Semantic Segmentation-2015 [[Paper]](https://arxiv.org/abs/1503.01640)
- Feedforward semantic segmentation with zoom-out features-2015 [[Code]](https://bitbucket.org/m_mostajabi/zoom-out-release) [[Paper]](http://www.cv-foundation.org/openaccess/content_cvpr_2015/papers/Mostajabi_Feedforward_Semantic_Segmentation_2015_CVPR_paper.pdf) [[Video]](https://www.youtube.com/watch?v=HvgvX1LXQa8)
- Conditional Random Fields as Recurrent Neural Networks-2015 [[Project]](http://www.robots.ox.ac.uk/~szheng/CRFasRNN.html) [[Code-Caffe1]](https://github.com/torrvision/crfasrnn) [[Code-Caffe2]](https://github.com/martinkersner/train-CRF-RNN) [[Demo]](http://www.robots.ox.ac.uk/~szheng/crfasrnndemo) [[Paper1]](http://www.robots.ox.ac.uk/~szheng/papers/CRFasRNN.pdf) [[Paper2]](http://arxiv.org/abs/1502.03240)
- Efficient Piecewise Training of Deep Structured Models for Semantic Segmentation-2015 [[Paper]](https://arxiv.org/abs/1504.01013)
- **Fully Convolutional Networks for Semantic Segmentation-2015** [[Code-Caffe]](https://github.com/shelhamer/fcn.berkeleyvision.org) [[Model-Caffe]](https://github.com/BVLC/caffe/wiki/Model-Zoo#fcn) [[Code-Tensorflow1]](https://github.com/MarvinTeichmann/tensorflow-fcn) [[Code-Tensorflow2]](https://github.com/shekkizh/FCN.tensorflow) [[Code-Chainer]](https://github.com/wkentaro/fcn) [[Code-PyTorch]](https://github.com/wkentaro/pytorch-fcn) [[Paper1]](http://arxiv.org/abs/1411.4038) [[Paper2]](http://arxiv.org/abs/1605.06211) [[Slides1]](https://docs.google.com/presentation/d/1VeWFMpZ8XN7OC3URZP4WdXvOGYckoFWGVN7hApoXVnc) [[Slides2]](http://tutorial.caffe.berkeleyvision.org/caffe-cvpr15-pixels.pdf)
- Deep Joint Task Learning for Generic Object Extraction-2014 [[Project]](http://vision.sysu.edu.cn/projects/deep-joint-task-learning/) [[Code-Caffe]](https://github.com/xiaolonw/nips14_loc_seg_testonly) [[Dataset]](http://objectextraction.github.io/) [[Paper]](http://ss.sysu.edu.cn/~ll/files/NIPS2014_JointTask.pdf)
- Highly Efficient Forward and Backward Propagation of Convolutional Neural Networks for Pixelwise Classification-2014 [[Code-Caffe]](https://dl.dropboxusercontent.com/u/6448899/caffe.zip) [[Paper]](https://arxiv.org/abs/1412.4526)
- **Wider or deeper: Revisiting the resnet model for visual recognition** [[Paper]](https://arxiv.org/abs/1611.10080)
- Describing the Scene as a Whole: Joint Object Detection, Scene Classification and Semantic Segmentation[[Paper]](https://ttic.uchicago.edu/~yaojian/Paper_Holistic.pdf)
- Analyzing Semantic Segmentation Using Hybrid Human-Machine CRFs[[Paper]](http://www.cv-foundation.org/openaccess/content_cvpr_2013/papers/Mottaghi_Analyzing_Semantic_Segmentation_2013_CVPR_paper.pdf)
- Convolutional Patch Networks with Spatial Prior for Road Detection and Urban Scene Understanding[[Paper]](https://arxiv.org/abs/1502.06344.pdf)
- Deep Deconvolutional Networks for Scene Parsing[[Paper]](https://arxiv.org/abs/1411.4101)
- FusionSeg: Learning to combine motion and appearance for fully automatic segmention of generic objects in videos[[Paper]](https://arxiv.org/pdf/1701.05384.pdf)[[Poject]](http://vision.cs.utexas.edu/projects/fusionseg/)
- ICCV-2017 Deep Dual Learning for Semantic Image Segmentation [[Paper]](http://personal.ie.cuhk.edu.hk/~pluo/pdf/luoWLWiccv17.pdf)
- From image-level to pixel level labeling with convolutional networks [[Paper]]()
- Scene Segmentation with DAG-Recurrent Neural Networks [[Paper]](http://ieeexplore.ieee.org/abstract/document/7940028/)
- Learning to Segment Every Thing [[Paper]](https://arxiv.org/pdf/1711.10370.pdf)
- Panoptic Segmentation [[Paper]](https://arxiv.org/pdf/1801.00868.pdf)
- The Devil is in the Decoder [[Paper]](https://arxiv.org/pdf/1707.05847.pdf)
- Attention to Scale: Scale-aware Semantic Image Segmentation [[Paper]](http://arxiv.org/pdf/1511.03339)[[Project]](http://liangchiehchen.com/projects/DeepLab.html)
- Convolutional Oriented Boundaries: From Image Segmentation to High-Level Tasks [[Paper]](https://arxiv.org/pdf/1701.04658.pdf) [[Project]](http://www.vision.ee.ethz.ch/~cvlsegmentation/)
- Scale-Aware Alignment of Hierarchical Image Segmentation [[Paper]](https://www.vision.ee.ethz.ch/en/publications/papers/proceedings/eth_biwi_01271.pdf) [[Project]](http://www.vision.ee.ethz.ch/~cvlsegmentation/)
- ICCV-2017 Semi Supervised Semantic Segmentation Using Generative Adversarial Network[[Paper]](https://arxiv.org/abs/1703.09695)
- Object Region Mining with Adversarial Erasing: A Simple Classification to Semantic Segmentation Approach [[Paper]](https://arxiv.org/pdf/1703.08448.pdf)
+ CVPR-2016 Convolutional Feature Masking for Joint Object and Stuff Segmentation [[Paper]](http://arxiv.org/abs/1412.1283) + ECCV-2016 Laplacian Pyramid Reconstruction and Refinement for Semantic Segmentation [[Paper]](https://arxiv.org/pdf/1411.5752.pdf) - FastMask: Segment Object Multi-scale Candidates in One Shot-2016 [[Code-Caffe]](https://github.com/voidrank/FastMask) [[Paper]](https://arxiv.org/abs/1612.08843)
- **Pixel Objectness-2017** [[Project]](http://vision.cs.utexas.edu/projects/pixelobjectness/) [[Code-Caffe]](https://github.com/suyogduttjain/pixelobjectness) [[Paper]](https://arxiv.org/abs/1701.05349)
## 3D Semantic Segmentation ### Papers - PointNet: Deep Learning on Point Sets for 3D Classification and Segmentation [[Paper]](http://stanford.edu/%7Erqi/pointnet/) - PointNet++: Deep Hierarchical Feature Learning on Point Sets in a Metric Space (2017) [[Paper]](https://arxiv.org/pdf/1706.02413.pdf) - Learning 3D Mesh Segmentation and Labeling (2010) [[Paper]](https://people.cs.umass.edu/~kalo/papers/LabelMeshes/LabelMeshes.pdf) - Unsupervised Co-Segmentation of a Set of Shapes via Descriptor-Space Spectral Clustering (2011) [[Paper]](https://www.cs.sfu.ca/~haoz/pubs/sidi_siga11_coseg.pdf) - Single-View Reconstruction via Joint Analysis of Image and Shape Collections (2015) [[Paper]](https://www.cs.utexas.edu/~huangqx/modeling_sig15.pdf) - 3D Shape Segmentation with Projective Convolutional Networks (2017) [[Paper]](http://people.cs.umass.edu/~kalo/papers/shapepfcn/) - Learning Hierarchical Shape Segmentation and Labeling from Online Repositories (2017) [[Paper]](http://cs.stanford.edu/~ericyi/project_page/hier_seg/index.html) - 3D Graph Neural Networks for RGBD Semantic Segmentation (2017) [[Paper]](http://www.cs.toronto.edu/~rjliao/papers/iccv_2017_3DGNN.pdf) - 3DCNN-DQN-RNN: A Deep Reinforcement Learning Framework for Semantic Parsing of Large-scale 3D Point Clouds (2017)[[Paper]](https://arxiv.org/pdf/1707.06783.pdf) - Multi-view deep learning for consistent semantic mapping with rgb-d cameras [[Paper]](https://arxiv.org/pdf/1703.08866.pdf) + ICCV-2017 Large-scale 3D Shape Reconstruction and Segmentation from ShapeNet Core55 [[Paper]]()[[Project]](https://shapenet.cs.stanford.edu/iccv17/) ## Instance Segmentation + Mask Scoring R-CNN (MS R-CNN) [[Code]](https://github.com/zjhuang22/maskscoring_rcnn)[[Paper]](https://arxiv.org/pdf/1903.00241.pdf) + Predicting Future Instance Segmentations by Forecasting Convolutional Features [[Paper]](https://hal.inria.fr/hal-01757669/document) + CVPR-2018 Path Aggregation Network for Instance Segmentation [[Paper]](https://arxiv.org/pdf/1803.01534.pdf) better than Mask-rcnn!！COCO-2017 1st! + Pixelwise Instance Segmentation with a Dynamically Instantiated Network-2017 [[Paper]](https://arxiv.org/abs/1704.02386)
+ Semantic Instance Segmentation via Deep Metric Learning-2017 [[Paper]](https://arxiv.org/abs/1703.10277)
+ CVPR-2017 FastMask: Segment Multi-scale Object Candidates in One Shot [[Code-Tensorflow]](https://github.com/CharlesShang/FastMaskRCNN) [[Paper]](https://arxiv.org/abs/1703.06870)
+ Pose2Instance: Harnessing Keypoints for Person Instance Segmentation-2017 [[Paper]](https://arxiv.org/abs/1704.01152)
+ Pixelwise Instance Segmentation with a Dynamically Instantiated Network-2017 [[Paper]](https://arxiv.org/abs/1704.02386)
+ CVPR-2017-spotlight Fully Convolutional Instance-aware Semantic Segmentation-2016 [[Code]](https://github.com/msracver/FCIS) [[Paper]](https://arxiv.org/abs/1611.07709)
+ CVPR-2016-oral **Instance-aware Semantic Segmentation via Multi-task Network Cascades-2015** [[Code]](https://github.com/daijifeng001/MNC) [[Paper]](http://arxiv.org/abs/1512.04412)
+ Recurrent Instance Segmentation-2015 [[Project]](http://romera-paredes.com/ris) [[Code-Torch7]](https://github.com/bernard24/ris) [[Paper]](http://arxiv.org/abs/1511.08250) [[Poster]](http://www.eccv2016.org/files/posters/P-4B-46.pdf) [[Video]](https://www.youtube.com/watch?v=l_WD2OWOqBk)
+ Annotating Object Instances with a Polygon-RNN [[Paper]](https://arxiv.org/abs/1704.05548) + MaskLab: Instance Segmentation by Refining Object Detection with Semantic and Direction Features [[Paper]](https://arxiv.org/pdf/1712.04837.pdf) + FCIS:Fully Convolutional Instance-aware Semantic Segmentation [[Paper]](https://arxiv.org/pdf/1611.07709.pdf)[Code](https://github.com/msracver/FCIS) + MNC:Instance-aware Semantic Segmentation via Multi-task Network Cascades [[Paper]](https://arxiv.org/pdf/1512.04412.pdf)[Code](https://github.com/daijifeng001/MNC) + DeepMask:Learning to Segment Object Candidates [[Paper]](https://arxiv.org/pdf/1506.06204.pdf) [Code](https://github.com/facebookresearch/deepmask) + SharpMask:Learning to Refine Object Segments [[Paper]](https://arxiv.org/pdf/1603.08695.pdf)[Code](https://github.com/facebookresearch/deepmask) + RIS:Recurrent Instance Segmentation [[Paper]](https://arxiv.org/pdf/1511.08250.pdf)[Code](https://github.com/bernard24/RIS) + FastMask: Segment Multi-scale Object Candidates in One Shot [[Paper]](https://arxiv.org/pdf/1612.08843.pdf)[Code](https://github.com/voidrank/FastMask) + Proposal-free network for instance-level object segmentation [[Paper]](https://arxiv.org/pdf/1509.02636) + ECCV-2016 Instance-sensitive Fully Convolutional Networks [[Paper]](http://arxiv.org/abs/1603.08678) + Pixel-level encoding and depth layering for instance-level semantic labeling [[Paper]](https://arxiv.org/pdf/1604.05096) ## Robotics - Virtual-to-Real: Learning to Control in Visual Semantic Segmentation [[Paper]](https://arxiv.org/pdf/1802.00285.pdf) - End-to-End Tracking and Semantic Segmentation Using Recurrent Neural Networks [[Paper]](https://arxiv.org/pdf/1604.05091.pdf) - Semantic Segmentation using Adversarial Networks [[Paper]](https://arxiv.org/pdf/1611.08408.pdf) ## Adversarial Training - CVPR-2017-Image-to-Image Translation with Conditional Adversarial Networks [[Paper]](http://openaccess.thecvf.com/content_cvpr_2017/papers/Isola_Image-To-Image_Translation_With_CVPR_2017_paper.pdf) - ICCV-2017-Adversarial Examples for Semantic Segmentation and Object Detection [[Paper]](http://openaccess.thecvf.com/content_ICCV_2017/papers/Xie_Adversarial_Examples_for_ICCV_2017_paper.pdf) ## Scene Understanding ### Papers 1.Spatial As Deep: Spatial CNN for Traffic Scene Understanding [[Paper]](https://arxiv.org/pdf/1712.06080.pdf)
### Dataset & Resources + SUNRGB-D 3D Object Detection Challenge [[Link]](http://rgbd.cs.princeton.edu/challenge.html) 19 object categories for predicting a 3D bounding box in real world dimension Training set: 10,355 RGB-D scene images, Testing set: 2860 RGB-D images + SceneNN (2016) [[Link]](http://people.sutd.edu.sg/~saikit/projects/sceneNN/) 100+ indoor scene meshes with per-vertex and per-pixel annotation. + ScanNet (2017) [[Link]](http://www.scan-net.org/) An RGB-D video dataset containing 2.5 million views in more than 1500 scans, annotated with 3D camera poses, surface reconstructions, and instance-level semantic segmentations. + Matterport3D: Learning from RGB-D Data in Indoor Environments (2017) [[Link]](https://niessner.github.io/Matterport/)
10,800 panoramic views (in both RGB and depth) from 194,400 RGB-D images of 90 building-scale scenes of private rooms. Instance-level semantic segmentations are provided for region (living room, kitchen) and object (sofa, TV) categories. + SUNCG: A Large 3D Model Repository for Indoor Scenes (2017) [[Link]](http://suncg.cs.princeton.edu/)
The dataset contains over 45K different scenes with manually created realistic room and furniture layouts. All of the scenes are semantically annotated at the object level. + MINOS: Multimodal Indoor Simulator (2017) [[Link]](https://github.com/minosworld/minos) MINOS is a simulator designed to support the development of multisensory models for goal-directed navigation in complex indoor environments. MINOS leverages large datasets of complex 3D environments and supports flexible configuration of multimodal sensor suites. MINOS supports SUNCG and Matterport3D scenes. + Facebook House3D: A Rich and Realistic 3D Environment (2017) [[Link]](https://github.com/facebookresearch/House3D)
House3D is a virtual 3D environment which consists of 45K indoor scenes equipped with a diverse set of scene types, layouts and objects sourced from the SUNCG dataset. All 3D objects are fully annotated with category labels. Agents in the environment have access to observations of multiple modalities, including RGB images, depth, segmentation masks and top-down 2D map views. + HoME: a Household Multimodal Environment (2017) [[Link]](https://home-platform.github.io/)
HoME integrates over 45,000 diverse 3D house layouts based on the SUNCG dataset, a scale which may facilitate learning, generalization, and transfer. HoME is an open-source, OpenAI Gym-compatible platform extensible to tasks in reinforcement learning, language grounding, sound-based navigation, robotics, multi-agent learning. + AI2-THOR: Photorealistic Interactive Environments for AI Agents [[Link]](http://ai2thor.allenai.org/)
AI2-THOR is a photo-realistic interactable framework for AI agents. There are a total 120 scenes in version 1.0 of the THOR environment covering four different room categories: kitchens, living rooms, bedrooms, and bathrooms. Each room has a number of actionable objects. ## Weakly-Supervised-Segmentation && Interactive Segmentation && Transferable Semantic Segmentation - arxiv-2018 WebSeg: Learning Semantic Segmentation from Web Searches [[Paper]](https://arxiv.org/pdf/1803.09859.pdf) - Weakly Supervised Object Localization Using Things and Stuff Transfer [[Paper]](https://arxiv.org/pdf/1703.08000.pdf) - Semi and Weakly Supervised Semantic Segmentation Using Generative Adversarial Network [[Paper]](http://crcv.ucf.edu/papers/1703.09695.pdf) - Weakly- and Semi-Supervised Learning of a Deep Convolutional Network for Semantic Image Segmentation [[Paper]](https://arxiv.org/pdf/1502.02734.pdf) + Weakly Supervised Structured Output Learning for Semantic Segmentation [[Paper]](http://groups.inf.ed.ac.uk/calvin/hp_avezhnev/Pubs/VezhnevetsCVPR2012b.pdf) + ICCV-2011 Weakly supervised semantic segmentation with a multi-image model [[Paper]](http://ieeexplore.ieee.org/document/6126299/) + ScribbleSup: Scribble-Supervised Convolutional Networks for Semantic Segmentation. IEEE Conference on Computer Vision and Pattern Recognition (CVPR), 2016[[Paper]](https://arxiv.org/abs/1604.05144)
+ Constrained convolutional neural networks for weakly supervised segmentation. Proceedings of the IEEE International Conference on Computer Vision. 2015.[[Paper]](https://arxiv.org/abs/1506.03648) + Weakly-and semi-supervised learning of a DCNN for semantic image segmentation. arXiv preprint arXiv:1502.02734 (2015).[[Paper]](https://arxiv.org/pdf/1502.02734.pdf) + Learning to segment under various forms of weak supervision. Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2015.[[Paper]](http://www.cv-foundation.org/openaccess/content_cvpr_2015/app/2B_048.pdf) + STC: A Simple to Complex Framework for Weakly-supervised Semantic Segmentation 2017 TPAMI [[Paper]](https://arxiv.org/pdf/1509.03150.pdf) [[Project]]() + [[Paper]](https://www.robots.ox.ac.uk/~vgg/rg/papers/semanticsegmentation.pdf) + CVPR-2017-Simple Does It: Weakly Supervised Instance and Semantic Segmentation [[Paper]](http://openaccess.thecvf.com/content_cvpr_2017/papers/Khoreva_Simple_Does_It_CVPR_2017_paper.pdf) [[tensorflow]](https://github.com/philferriere/tfwss) + CVPR-2017-Weakly Supervised Semantic Segmentation using Web-Crawled Videos [[Paper]](http://openaccess.thecvf.com/content_cvpr_2017/papers/Hong_Weakly_Supervised_Semantic_CVPR_2017_paper.pdf) + AAAI-2017-Weakly Supervised Semantic Segmentation Using Superpixel Pooling Network [[Paper]](https://aaai.org/ocs/index.php/AAAI/AAAI17/paper/view/14445/14288) + ICCV-2015-Weakly supervised graph based semantic segmentation by learning communities of image-parts [[Paper]](https://www.cv-foundation.org/openaccess/content_iccv_2015/papers/Pourian_Weakly_Supervised_Graph_ICCV_2015_paper.pdf) + Towards Weakly Supervised Semantic Segmentation by Means of Multiple Instance and Multitask Learning [[Paper]](https://pdfs.semanticscholar.org/168f/7c5fda213447fb215b57bf00c1e27bbeac7a.pdf) + Weakly-Supervised Semantic Segmentation using Motion Cues [[Paper]](https://arxiv.org/pdf/1603.07188) [[Project]](http://thoth.inrialpes.fr/research/weakseg/) + Weakly Supervised Semantic Segmentation Based on Web Image Co-segmentation [[Paper]](https://arxiv.org/pdf/1705.09052) + Learning to Rene Object Segments [[Paper]](https://arxiv.org/pdf/1603.08695.pdf) - Weakly-Supervised Dual Clustering for Image Semantic Segmentation [[Paper]](https://www.cv-foundation.org/openaccess/content_cvpr_2013/papers/Liu_Weakly-Supervised_Dual_Clustering_2013_CVPR_paper.pdf) + Interactive Video Object Segmentation in the Wild [[Paper]](https://arxiv.org/pdf/1801.00269) ## Video Semantic Segmentation + CVPR-2017 Video Object Segmentation Without Temporal Information One-Shot Video Object Segmentation [[Project]](http://www.vision.ee.ethz.ch/~cvlsegmentation/osvos/) + Feature Space Optimization for Semantic Video Segmentation[[Paper]](http://www.cvlibs.net/projects/autonomous_vision_survey/literature/Kundu2016CVPR.pdf)[[Slides]](http://www.cvlibs.net/projects/autonomous_vision_survey/slides/Kundu2016CVPR/top.pdf)
+ The Basics of Video Object Segmentation [[Blog]](https://techburst.io/video-object-segmentation-the-basics-758e77321914) + ICCV2017----SegFlow_Joint Learning for Video Object Segmentation and Optical Flow
+ OSVOS:One-Shot Video Object Segmentation
+ Surveillance Video Parsing with Single Frame Supervision
+ The 2017 DAVIS Challenge on Video Object Segmentation
+ Video Propagation Networks
+ OnAVOS: Online Adaptation of Convolutional Neural Networks for Video Object Segmentation. P. Voigtlaender, B. Leibe, BMVC 2017. [Project Page] [Precomputed results] + MSK: Learning Video Object Segmentation from Static Images. F. Perazzi*, A. Khoreva*, R. Benenson, B. Schiele, A. Sorkine-Hornung, CVPR 2017. [Project Page] [Precomputed results] + SFL: SegFlow: Joint Learning for Video Object Segmentation and Optical Flow. J. Cheng, Y.-H. Tsai, S. Wang, M.-H. Yang, ICCV 2017. [Project Page] [Precomputed results] + CTN: Online Video Object Segmentation via Convolutional Trident Network. W.-D. Jang, C.-S. Kim, CVPR 2017. [Project Page] [Precomputed results] + VPN: Video Propagation Networks. V. Jampani, R. Gadde, P. V. Gehler, CVPR 2017. [Project Page] [Precomputed results] + PLM: Pixel-level Matching for Video Object Segmentation using Convolutional Neural Networks. J. Shin Yoon, F. Rameau, J. Kim, S. Lee, S. Shin, I. So Kweon, ICCV 2017. [Project Page] [Precomputed results] + OFL: Video Segmentation via Object Flow. Y.-H. Tsai, M.-H. Yang, M. Black, CVPR 2016. [Project Page] [Precomputed results] + BVS: Bilateral Space Video Segmentation. N. Marki, F. Perazzi, O. Wang, A. Sorkine-Hornung, CVPR 2016. [Project Page] [Precomputed results] + FCP: Fully Connected Object Proposals for Video Segmentation. F. Perazzi, O. Wang, M. Gross, A. Sorkine-Hornung, ICCV 2015. [Project Page] [Precomputed results] + JMP: JumpCut: Non-Successive Mask Transfer and Interpolation for Video Cutout. Q. Fan, F. Zhong, D. Lischinski, D. Cohen-Or, B. Chen, SIGGRAPH 2015. [Project Page] [Precomputed results] + HVS: Efficient hierarchical graph-based video segmentation. M. Grundmann, V. Kwatra, M. Han, I. A. Essa, CVPR 2010. [Project Page] [Precomputed results] + SEA: SeamSeg: Video Object Segmentation Using Patch Seams. S. Avinash Ramakanth, R. Venkatesh Babu, CVPR 2014. [Project Page] [Precomputed results] + ARP: Primary Object Segmentation in Videos Based on Region Augmentation and Reduction. Y.J. Koh, C.-S. Kim, CVPR 2017. [Project Page] [Precomputed results] + LVO: Learning Video Object Segmentation with Visual Memory. P. Tokmakov, K. Alahari, C. Schmid, ICCV 2017. [Project Page] [Precomputed results] + FSEG: FusionSeg: Learning to combine motion and appearance for fully automatic segmentation of generic objects in videos. S. Jain, B. Xiong, K. Grauman, CVPR 2017. [Project Page] [Precomputed results] + LMP: Learning Motion Patterns in Videos. P. Tokmakov, K. Alahari, C. Schmid, CVPR 2017. [Project Page] [Precomputed results] + SFL: SegFlow: Joint Learning for Video Object Segmentation and Optical Flow. J. Cheng, Y.-H. Tsai, S. Wang, M.-H. Yang, ICCV 2017. [Project Page] [Precomputed results] FST: Fast Object Segmentation in Unconstrained Video. A. Papazoglou, V. Ferrari, ICCV 2013. [Project Page] [Precomputed results] + CUT: Motion Trajectory Segmentation via Minimum Cost Multicuts. M. Keuper, B. Andres, T. Brox, ICCV 2015. [Project Page] [Precomputed results] + NLC: Video Segmentation by Non-Local Consensus voting. A. Faktor, M. Irani, BMVC 2014. [Project Page] [Precomputed results] + MSG: Object segmentation in video: A hierarchical variational approach for turning point trajectories into dense regions. P. Ochs, T. Brox, ICCV 2011. [Project Page] [Precomputed results] + KEY: Key-segments for video object segmentation. Y. Lee, J. Kim, K. Grauman, ICCV 2011. [Project Page] [Precomputed results] + CVOS: Causal Video Object Segmentation from Persistence of Occlusions. B. Taylor, V. Karasev, S. Soatto, CVPR 2015. [Project Page] [Precomputed results] + TRC: Video segmentation by tracing discontinuities in a trajectory embedding. K. Fragkiadaki, G. Zhang, J. Shi, CVPR 2012. [Project Page] [Precomputed results] + Instance Embedding Transfer to Unsupervised Video Object Segmentation [[Paper]](https://arxiv.org/abs/1801.00908) - [Result of DAVIS-Challenge 2017](http://davischallenge.org/challenge2017/index.html) - [Benchmark](http://davischallenge.org/soa_compare.html) 2016----A Benchmark Dataset and Evaluation Methodology for Video Object Segmentation
2016----Clockwork Convnets for Video Semantic Segmentation
2016----MaskTrack ----Learning Video Object Segmentation from Static Images
2017----DAVIS-Challenge-1st----[Video Object Segmentation with Re-identification](https://arxiv.org/pdf/1708.00197.pdf)
2017----DAVIS-Challenge-2nd----Lucid Data Dreaming for Multiple Object Tracking
2017----DAVIS-Challenge-3rd----Instance Re-Identification Flow for Video Object Segmentation
2017----DAVIS-Challenge-4th----Multiple-Instance Video Segmentation with Sequence-Specific Object Proposals
2017----DAVIS-Challenge-5th Online Adaptation of Convolutional Neural Networks for the 2017 DAVIS Challenge on Video Object Segmentation
2017----DAVIS-Challenge-6th ----Learning to Segment Instances in Videos with Spatial Propagation Network
2017----DAVIS-Challenge-7th----Some Promising Ideas about Multi-instance Video Segmentation
2017----DAVIS-Challenge-8th----[One-Shot Video Object Segmentation with Iterative Online Fine-Tuning](https://arxiv.org/pdf/1611.05198.pdf)
2017----DAVIS-Challenge-9th----Video Object Segmentation using Tracked Object Proposals
## Multi-Task Learning ### Papers: - Multi-Task Learning Using Uncertainty to Weigh Losses for Scene Geometry and Semantics [[Paper]]() - Multi-task Learning using Multi-modal Encoder-Decoder Networks with Shared Skip Connections [[Paper]]() ## Road Segmentation && Real Time Segmentation ### Papers: - Deep Semantic Segmentation for Automated Driving: Taxonomy, Roadmap and Challenges [[Paper]](https://arxiv.org/pdf/1707.02432.pdf) - 2018-arxiv Real-time Semantic Segmentation Comparative Study[[Paper]](https://arxiv.org/pdf/1803.02758.pdf)[[Code]](https://github.com/MSiam/TFSegmentation) - MultiNet: Real-time Joint Semantic Reasoning for Autonomous Driving [[Paper]](https://arxiv.org/pdf/1612.07695.pdf)
- self-driving-car-road-segmentation [[Link]](https://medium.com/@karol_majek/self-driving-car-road-segmentation-514ae80e103a)
- Efficient Deep Models for Monocular Road Segmentation[[Paper]](http://ais.informatik.uni-freiburg.de/publications/papers/oliveira16iros.pdf)
- Semantic Road Segmentation via Multi-scale Ensembles of Learned Features [[Paper]]()
- Distantly Supervised Road Segmentation [[Paper]](https://arxiv.org/pdf/1708.06118)
- Deep Fully Convolutional Networks with Random Data Augmentation for Enhanced Generalization in Road Detection [[Paper]](http://isislab.es/llorca/publications_files/IEEEITSC17_RoadAugmentedDataFCNN.pdf)
- ICCV-2017 Real-time category-based and general obstacle detection for autonomous driving [[Paper]](http://openaccess.thecvf.com/content_ICCV_2017_workshops/papers/w3/Garnett_Real-Time_Category-Based_and_ICCV_2017_paper.pdf)
- ICCV-2017 FoveaNet: Perspective-aware Urban Scene Parsing [[Paper]](http://users.eecs.northwestern.edu/~xsh835/assets/iccv2017_foveanet.pdf)
- CVPR-2017 UberNet: Training a universal convolutional neural network for low-, mid-, and high-level vision using diverse datasets and limited memory [[Paper]](http://openaccess.thecvf.com/content_cvpr_2017/papers/Kokkinos_Ubernet_Training_a_CVPR_2017_paper.pdf) + LinkNet: Exploiting Encoder Representations for Efficient Semantic Segmentation [[Paper]](https://arxiv.org/abs/1707.03718.pdf) + ENet: A Deep Neural Network Architecture for Real-Time Semantic Segmentation-2016 [[Code-Caffe1]](https://github.com/e-lab/ENet-training)[[Code-Caffe2]](https://github.com/TimoSaemann/ENet) [[Paper]](https://arxiv.org/abs/1606.02147) [[Blog]](https://culurciello.github.io/tech/2016/06/20/training-enet.html) + Efficient Deep Models for Monocular Road Segmentation[[Paper]](http://ais.informatik.uni-freiburg.de/publications/papers/oliveira16iros.pdf) + Real-Time Coarse-to-fine Topologically Preserving Segmentation[[Paper]](http://www.cv-foundation.org/openaccess/content_cvpr_2015/papers/Yao_Real-Time_Coarse-to-Fine_Topologically_2015_CVPR_paper.pdf) + ICNet for Real-Time Semantic Segmentation on High-Resolution Images [[Paper]](https://arxiv.org/pdf/1704.08545.pdf) + Efficient and robust deep networks for semantic segmentation [[Paper]](http://journals.sagepub.com/doi/abs/10.1177/0278364917710542) + NIPSW-2017 Speeding up semantic segmentation for autonomous driving [[Paper]](https://openreview.net/pdf?id=S1uHiFyyg) - ECCV-2012 Road Scene Segmentation from a Single Image [[Paper]](http://yann.lecun.com/exdb/publis/pdf/alvarez-eccv-12.pdf) ### Codes + https://github.com/vxy10/p5_VehicleDetection_Unet [Keras] + https://github.com/ndrplz/self-driving-car ## Medical Image Semantic Segmentation ### Papers + Arxiv-2018 Deep learning and its application to medical image segmentation [[Paper]](https://arxiv.org/pdf/1803.08691) - Deep neural networks segment neuronal membranes in electron microscopy images - Semantic Image Segmentation with Deep Learning [[Paper]](http://www.robots.ox.ac.uk/~sadeep/files/crfasrnn_presentation.pdf)
- Automatic Liver and Tumor Segmentation of CT and MRI Volumes Using Cascaded Fully Convolutional Neural Networks [[Paper]](https://arxiv.org/pdf/1702.05970.pdf)
- DeepNAT: Deep Convolutional Neural Network for Segmenting Neuroanatomy [[Paper]](https://arxiv.org/pdf/1702.08192.pdf)
- CNN-based Segmentation of Medical Imaging Data [[Paper]](https://arxiv.org/pdf/1701.03056.pdf)
- Deep Retinal Image Understanding (http://www.vision.ee.ethz.ch/~cvlsegmentation/driu/data/paper/DRIU_MICCAI2016.pdf) - Model-based segmentation of vertebral bodies from MR images with 3D CNNs - Efficient multi-scale 3D CNN with fully connected CRF for accurate brain lesion segmentation - U-net: Convolutional networks for biomedical image segmentation - 3D U-Net: Learning dense volumetric segmentation from sparse annotation. - V-Net: Fully convolutional neural networks for volumetric medical image segmentation.arXiv:1606.04797 - The importance of skip connections in biomedical image segmentation Spatial clockwork recurrent neural network for muscle perimysium segmentation - NPIS-2015 Parallel multi-dimensional LSTM, with application to fast biomedical volumetric image segmentation - Multi-dimensional gated recurrent units for the segmentation of biomedical 3D-data - Combining fully convolutional and recurrent neural networks for 3D biomedical image segmentation - Recurrent fully convolutional neural networks for multi-slice MRI cardiac segmentation. arXiv:1608.03974 - Automatic detection and classification of colorectal polyps by transferring low-level CNN features from nonmedical domain - Deep learning for multi-task medical image segmentation in multiple modalities - Sub-cortical brain structure segmentation using F-CNNs - Segmentation label propagation using deep convolutional neural networks and dense conditional random field - Fast fully automatic segmentation of the human placenta from motion corrupted MRI - Automatic detection of cerebral microbleeds from MR images via 3D convolutional neural networks - Non-uniform patch sampling with deep convolutional neural networks for white matter hyperintensity segmentation - A unified framework for automatic wound segmentation and analysis with deep convolutional neural networks - Deep 3D convolutional encoder networks with shortcuts for multiscale feature integration applied to Multiple Sclerosis lesion segmentation - Brain tumor segmentation using convolutional neural networks in MRI images - Deep feature learning for knee cartilage segmentation using a triplanar convolutional neural network - Automatic Coronary Calcium Scoring in Cardiac CT Angiography Using Convolutional Neural Networks [[Paper]](https://pdfs.semanticscholar.org/4490/792b673bd3b7aead9095777ad611a99f0d64.pdf) - Improving computer-aided detection using convolutional neural networks and random view aggregation [[Paper]](https://arxiv.org/pdf/1505.03046.pdf) - Pulmonary nodule detection in CT images: false positive reduction using multi-view convolutional networks [[Paper]](http://ieeexplore.ieee.org/document/7422783/) ### Codes ## Part Semantic Segmentation - Look into Person: Self-supervised Structure-sensitive Learning and A New Benchmark for Human Parsing-2017 [[Project]](http://hcp.sysu.edu.cn/lip/) [[Code-Caffe]](https://github.com/Engineering-Course/LIP_SSL) [[Paper]](https://arxiv.org/abs/1703.05446)
- **Deep Learning for Human Part Discovery in Images-2016** [[Code-Chainer]](https://github.com/shiba24/deep-learning-for-human-part-discovery-in-images) [[Paper]](http://lmb.informatik.uni-freiburg.de/Publications/2016/OB16a/oliveira16icra.pdf)
- **A CNN Cascade for Landmark Guided Semantic Part Segmentation-2016** [[Project]](http://aaronsplace.co.uk/papers/jackson2016guided/) [[Paper]](https://arxiv.org/abs/1609.09642)
- Deep Learning for Semantic Part Segmentation With High-level Guidance-2015 [[Paper]](https://arxiv.org/abs/1505.02438)
- Neural Activation Constellations-Unsupervised Part Model Discovery with Convolutional Networks-2015 [[Paper]](https://arxiv.org/abs/1504.08289)
- Human Parsing with Contextualized Convolutional Neural Network-2015 [[Paper]](http://www.cv-foundation.org/openaccess/content_iccv_2015/papers/Liang_Human_Parsing_With_ICCV_2015_paper.pdf)
- Part detector discovery in deep convolutional neural networks-2014 [[Code]](https://github.com/cvjena/PartDetectorDisovery) [[Paper]](https://arxiv.org/abs/1411.3159)
- Hypercolumns for object segmentation and fine-grained localization [[Paper]](http://www.eecs.berkeley.edu/~bharath2/pubs/pdfs/BharathCVPR2015.pdf)
## Clothes Parsing - Looking at Outfit to Parse Clothing-2017 [[Paper]](https://arxiv.org/abs/1703.01386)
- Semantic Object Parsing with Local-Global Long Short-Term Memory-2015 [[Paper]](https://arxiv.org/abs/1511.04510)
-A High Performance CRF Model for Clothes Parsing-2014 [[Project]](http://hi.cs.waseda.ac.jp/~esimo/en/code/clothes_parsing/) [[Code]](https://github.com/bobbens/clothes_parsing) [[Dataset]](http://vision.is.tohoku.ac.jp/~kyamagu/ja/research/clothing_parsing/) [[Paper]](https://pdfs.semanticscholar.org/0416/f5d1564d1f2a597acac04e81b02b2eff67d2.pdf)
- Clothing co-parsing by joint image segmentation and labeling-2013 [[Project]](http://hcp.sysu.edu.cn/clothing-co-parsing-by-joint-image-segmentation-and-labeling/) [[Dataset]](https://github.com/bearpaw/clothing-co-parsing) [[Paper]](https://arxiv.org/abs/1502.00739)
- Parsing clothing in fashion photographs-2012 [[Project]](http://vision.is.tohoku.ac.jp/~kyamagu/research/clothing_parsing/) [[Paper]](http://vision.is.tohoku.ac.jp/~kyamagu/papers/yamaguchi_cvpr2012.pdf)
## Popular Methods and Implementations - U-Net [https://arxiv.org/pdf/1505.04597.pdf][Pytorch](https://github.com/tangzhenyu/SemanticSegmentation_DL/tree/master/U-net) - SegNet [https://arxiv.org/pdf/1511.00561.pdf][Caffe](https://github.com/alexgkendall/caffe-segnet) - DeepLab [https://arxiv.org/pdf/1606.00915.pdf][Caffe](https://bitbucket.org/deeplab/deeplab-public/) - FCN [https://arxiv.org/pdf/1605.06211.pdf][tensorflow](https://github.com/tangzhenyu/SemanticSegmentation_DL/tree/master/FCN) - ENet [https://arxiv.org/pdf/1606.02147.pdf][Caffe](https://github.com/TimoSaemann/ENet) - LinkNet [https://arxiv.org/pdf/1707.03718.pdf][Torch](https://github.com/e-lab/LinkNet) - DenseNet [https://arxiv.org/pdf/1608.06993.pdf] - Tiramisu [https://arxiv.org/pdf/1611.09326.pdf] - DilatedNet [https://arxiv.org/pdf/1511.07122.pdf] - PixelNet [https://arxiv.org/pdf/1609.06694.pdf][Caffe](https://github.com/aayushbansal/PixelNet) - ICNet [https://arxiv.org/pdf/1704.08545.pdf][Caffe](https://github.com/hszhao/ICNet ) - ERFNet [http://www.robesafe.uah.es/personal/eduardo.romera/pdfs/Romera17iv.pdf][Torch](https://github.com/Eromera/erfnet ) - RefineNet [https://arxiv.org/pdf/1611.06612.pdf][tensorflow](https://github.com/tangzhenyu/SemanticSegmentation_DL/tree/master/RefineNet) - PSPNet [https://arxiv.org/pdf/1612.01105.pdf,https://hszhao.github.io/projects/pspnet/][Caffe](https://github.com/hszhao/PSPNet ) - Dilated convolution [https://arxiv.org/pdf/1511.07122.pdf][Caffe](https://github.com/fyu/dilation ) - DeconvNet [https://arxiv.org/pdf/1505.04366.pdf][Caffe](http://cvlab.postech.ac.kr/research/deconvnet/ ) - FRRN [https://arxiv.org/pdf/1611.08323.pdf][Lasagne](https://github.com/TobyPDE/FRRN ) - GCN [https://arxiv.org/pdf/1703.02719.pdf][PyTorch](https://github.com/ZijunDeng/pytorch-semantic-segmentation ) - LRR [https://arxiv.org/pdf/1605.02264.pdf][Matconvnet](https://github.com/golnazghiasi/LRR ) - DUC, HDC [https://arxiv.org/pdf/1702.08502.pdf][PyTorch](https://github.com/ZijunDeng/pytorch-semantic-segmentation ) - MultiNet [https://arxiv.org/pdf/1612.07695.pdf] [tensorflow1](https://github.com/MarvinTeichmann/MultiNet)[tensorflow2](https://github.com/MarvinTeichmann/KittiSeg) - Segaware [https://arxiv.org/pdf/1708.04607.pdf][Caffe](https://github.com/aharley/segaware ) - Semantic Segmentation using Adversarial Networks [https://arxiv.org/pdf/1611.08408.pdf] [Chainer](+ https://github.com/oyam/Semantic-Segmentation-using-Adversarial-Networks ) - In-Place Activated BatchNorm:obtain #1 positions [https://arxiv.org/abs/1712.02616] [Pytorch](https://github.com/mapillary/inplace_abn) ## Annotation Tools: + https://github.com/AKSHAYUBHAT/ImageSegmentation + https://github.com/kyamagu/js-segment-annotator + https://github.com/CSAILVision/LabelMeAnnotationTool + https://github.com/seanbell/opensurfaces-segmentation-ui + https://github.com/lzx1413/labelImgPlus + https://github.com/wkentaro/labelme ## Results: + [MSRC-21](http://rodrigob.github.io/are_we_there_yet/build/semantic_labeling_datasets_results.html) + [Cityscapes](https://www.cityscapes-dataset.com/benchmarks/) + [VOC2012](http://host.robots.ox.ac.uk:8080/leaderboard/displaylb.php?challengeid=11&compid=6) # Reference [https://github.com/nightrome/really-awesome-semantic-segmentation](https://github.com/nightrome/really-awesome-semantic-segmentation) [https://github.com/mrgloom/awesome-semantic-segmentation](https://github.com/mrgloom/awesome-semantic-segmentation) ================================================ FILE: facts/word2vec.md ================================================ # Röstigraben ...is an expression used for the border between the German-speaking and the French-speaking Switzerland. In English it would be "the Rösti ditch", and it has probably got something with the different (food) cultures to do... # Merci vilmal ...is a nice language mixture. An explanation that I have got, which probably isn't completely true, but funny anyway, is that the German-speaking Swiss wanted to say "merci" instead of "danke", to prove that they weren't German. The problem was only that then they sounded like they were trying to speak French without being very successful (the Swiss German pronunciation of "merci" is rather special). The solution was to add the typical Swiss German ending "vilmal". Then there would be no doubt of where they were coming from. ....and this is how the expression "merci vilmal" - thanks a lot - may have been created... ================================================ FILE: src/context.ts ================================================ import { ClassicCurve } from "./classic-curve"; import { iOS9Curve } from "./ios9-curve"; type CurveStyle = "ios" | "ios9"; export type Options = { // The DOM container where the DOM canvas element will be added container: HTMLElement; // The style of the wave: `ios` or `ios9` style?: CurveStyle; // Ratio of the display to use. Calculated by default. ratio?: number; // The speed of the animation. speed?: number; // The amplitude of the complete wave. amplitude?: number; // The frequency for the complete wave (how many waves). - Not available in iOS9 Style frequency?: number; // The color of the wave, in hexadecimal form (`#336699`, `#FF0`). - Not available in iOS9 Style color?: string; // The `canvas` covers the entire width or height of the container. cover?: boolean; // Width of the canvas. Calculated by default. width?: number; // Height of the canvas. Calculated by default. height?: number; // Decide wether start the animation on boot. autostart?: boolean; // Number of step (in pixels) used when drawed on canvas. pixelDepth?: number; // Lerp speed to interpolate properties. lerpSpeed?: number; // Curve definition override curveDefinition?: ICurveDefinition[]; }; export type IiOS9CurveDefinition = { supportLine?: boolean; color: string; }; export type IClassicCurveDefinition = { attenuation: number; lineWidth: number; opacity: number; color?: string; }; export type ICurveDefinition = IiOS9CurveDefinition | IClassicCurveDefinition; export interface ICurve { draw: () => void; } export default class SiriWave { opt: Options; // Phase of the wave (passed to Math.sin function) phase: number = 0; // Boolean value indicating the the animation is running run: boolean = false; // Curves objects to animate curves: ICurve[] = []; speed: number; amplitude: number; width: number; height: number; heightMax: number; color: string; interpolation: { speed: number | null; amplitude: number | null; }; canvas: HTMLCanvasElement | null; ctx: CanvasRenderingContext2D; animationFrameId: number | undefined; timeoutId: ReturnType | undefined; constructor({ container, ...rest }: Options) { const csStyle = window.getComputedStyle(container); this.opt = { container, style: "ios", ratio: window.devicePixelRatio || 1, speed: 0.2, amplitude: 1, frequency: 6, color: "#fff", cover: false, width: parseInt(csStyle.width.replace("px", ""), 10), height: parseInt(csStyle.height.replace("px", ""), 10), autostart: true, pixelDepth: 0.02, lerpSpeed: 0.1, ...rest, }; /** * Actual speed of the animation. Is not safe to change this value directly, use `setSpeed` instead. */ this.speed = Number(this.opt.speed); /** * Actual amplitude of the animation. Is not safe to change this value directly, use `setAmplitude` instead. */ this.amplitude = Number(this.opt.amplitude); /** * Width of the canvas multiplied by pixel ratio */ this.width = Number(this.opt.ratio! * this.opt.width!); /** * Height of the canvas multiplied by pixel ratio */ this.height = Number(this.opt.ratio! * this.opt.height!); /** * Maximum height for a single wave */ this.heightMax = Number(this.height / 2) - 6; /** * Color of the wave (used in Classic iOS) */ this.color = `rgb(${this.hex2rgb(this.opt.color!)})`; /** * An object containing controller variables that need to be interpolated * to an another value before to be actually changed */ this.interpolation = { speed: this.speed, amplitude: this.amplitude, }; /** * Canvas DOM Element where curves will be drawn */ this.canvas = document.createElement("canvas"); /** * 2D Context from Canvas */ const ctx = this.canvas.getContext("2d"); if (ctx === null) { throw new Error("Unable to create 2D Context"); } this.ctx = ctx; // Set dimensions this.canvas.width = this.width; this.canvas.height = this.height; // By covering, we ensure the canvas is in the same size of the parent if (this.opt.cover === true) { this.canvas.style.width = this.canvas.style.height = "100%"; } else { this.canvas.style.width = `${this.width / this.opt.ratio!}px`; this.canvas.style.height = `${this.height / this.opt.ratio!}px`; } // Instantiate all curves based on the style switch (this.opt.style) { case "ios9": this.curves = ((this.opt.curveDefinition || iOS9Curve.getDefinition()) as IiOS9CurveDefinition[]).map( (def) => new iOS9Curve(this, def), ); break; case "ios": default: this.curves = ((this.opt.curveDefinition || ClassicCurve.getDefinition()) as IClassicCurveDefinition[]).map( (def) => new ClassicCurve(this, def), ); break; } // Attach to the container this.opt.container.appendChild(this.canvas); // Start the animation if (this.opt.autostart) { this.start(); } } /** * Convert an HEX color to RGB */ private hex2rgb(hex: string): string | null { const shorthandRegex = /^#?([a-f\d])([a-f\d])([a-f\d])$/i; hex = hex.replace(shorthandRegex, (m, r, g, b) => r + r + g + g + b + b); const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex); return result ? `${parseInt(result[1], 16).toString()},${parseInt(result[2], 16).toString()},${parseInt( result[3], 16, ).toString()}` : null; } private intLerp(v0: number, v1: number, t: number): number { return v0 * (1 - t) + v1 * t; } /** * Interpolate a property to the value found in this.interpolation */ private lerp(propertyStr: "amplitude" | "speed"): number | null { const prop = this.interpolation[propertyStr]; if (prop !== null) { this[propertyStr] = this.intLerp(this[propertyStr], prop, this.opt.lerpSpeed!); if (this[propertyStr] - prop === 0) { this.interpolation[propertyStr] = null; } } return this[propertyStr]; } /** * Clear the canvas */ private clear() { this.ctx.globalCompositeOperation = "destination-out"; this.ctx.fillRect(0, 0, this.width, this.height); this.ctx.globalCompositeOperation = "source-over"; } /** * Draw all curves */ private draw() { this.curves.forEach((curve) => curve.draw()); } /** * Clear the space, interpolate values, calculate new steps and draws * @returns */ private startDrawCycle() { this.clear(); // Interpolate values this.lerp("amplitude"); this.lerp("speed"); this.draw(); this.phase = (this.phase + (Math.PI / 2) * this.speed) % (2 * Math.PI); if (window.requestAnimationFrame) { this.animationFrameId = window.requestAnimationFrame(this.startDrawCycle.bind(this)); } else { this.timeoutId = setTimeout(this.startDrawCycle.bind(this), 20); } } /* API */ /** * Start the animation */ start() { if (!this.canvas) { throw new Error("This instance of SiriWave has been disposed, please create a new instance"); } this.phase = 0; // Ensure we don't re-launch the draw cycle if (!this.run) { this.run = true; this.startDrawCycle(); } } /** * Stop the animation */ stop() { this.phase = 0; this.run = false; // Clear old draw cycle on stop if (this.animationFrameId) { window.cancelAnimationFrame(this.animationFrameId); } if (this.timeoutId) { clearTimeout(this.timeoutId); } } /** * Dispose */ dispose() { this.stop(); if (this.canvas) { this.canvas.remove(); this.canvas = null; } } /** * Setzen eines neuen Wertes für eine Eigenschaft (interpoliert) */ set(propertyStr: "amplitude" | "speed", value: number) { this.interpolation[propertyStr] = value; } /** * Set a new value for the speed property (interpolated) */ setSpeed(value: number) { this.set("speed", value); } /** * Set a new value for the amplitude property (interpolated) */ setAmplitude(value: number) { this.set("amplitude", value); } } ================================================ FILE: src/ios9_curve.ts ================================================ import SiriWave, { ICurve, IiOS9CurveDefinition } from "./index"; export class iOS9Curve implements ICurve { ctrl: SiriWave; definition: IiOS9CurveDefinition; spawnAt: number; noOfCurves: number; prevMaxY: number; phases: number[]; amplitudes: number[]; despawnTimeouts: number[]; offsets: number[]; speeds: number[]; finalAmplitudes: number[]; widths: number[]; verses: number[]; GRAPH_X = 25; AMPLITUDE_FACTOR = 0.8; SPEED_FACTOR = 1; DEAD_PX = 2; ATT_FACTOR = 4; DESPAWN_FACTOR = 0.02; NOOFCURVES_RANGES: [number, number] = [2, 5]; AMPLITUDE_RANGES: [number, number] = [0.3, 1]; OFFSET_RANGES: [number, number] = [-3, 3]; WIDTH_RANGES: [number, number] = [1, 3]; SPEED_RANGES: [number, number] = [0.5, 1]; DESPAWN_TIMEOUT_RANGES: [number, number] = [500, 2000]; constructor(ctrl: SiriWave, definition: IiOS9CurveDefinition) { this.ctrl = ctrl; this.definition = definition; this.noOfCurves = 0; this.spawnAt = 0; this.prevMaxY = 0; this.phases = []; this.offsets = []; this.speeds = []; this.finalAmplitudes = []; this.widths = []; this.amplitudes = []; this.despawnTimeouts = []; this.verses = []; } private getRandomRange(e: [number, number]): number { return e[0] + Math.random() * (e[1] - e[0]); } private spawnSingle(ci: number): void { this.phases[ci] = 0; this.amplitudes[ci] = 0; this.despawnTimeouts[ci] = this.getRandomRange(this.DESPAWN_TIMEOUT_RANGES); this.offsets[ci] = this.getRandomRange(this.OFFSET_RANGES); this.speeds[ci] = this.getRandomRange(this.SPEED_RANGES); this.finalAmplitudes[ci] = this.getRandomRange(this.AMPLITUDE_RANGES); this.widths[ci] = this.getRandomRange(this.WIDTH_RANGES); this.verses[ci] = this.getRandomRange([-1, 1]); } private getEmptyArray(count: number): number[] { return new Array(count); } private spawn(): void { this.spawnAt = Date.now(); this.noOfCurves = Math.floor(this.getRandomRange(this.NOOFCURVES_RANGES)); this.phases = this.getEmptyArray(this.noOfCurves); this.offsets = this.getEmptyArray(this.noOfCurves); this.speeds = this.getEmptyArray(this.noOfCurves); this.finalAmplitudes = this.getEmptyArray(this.noOfCurves); this.widths = this.getEmptyArray(this.noOfCurves); this.amplitudes = this.getEmptyArray(this.noOfCurves); this.despawnTimeouts = this.getEmptyArray(this.noOfCurves); this.verses = this.getEmptyArray(this.noOfCurves); for (let ci = 0; ci < this.noOfCurves; ci++) { this.spawnSingle(ci); } } private globalAttFn(x: number): number { return Math.pow(this.ATT_FACTOR / (this.ATT_FACTOR + Math.pow(x, 2)), this.ATT_FACTOR); } private sin(x: number, phase: number): number { return Math.sin(x - phase); } private yRelativePos(i: number): number { let y = 0; for (let ci = 0; ci < this.noOfCurves; ci++) { // Generate a static T so that each curve is distant from each oterh let t = 4 * (-1 + (ci / (this.noOfCurves - 1)) * 2); // but add a dynamic offset t += this.offsets![ci]; const k = 1 / this.widths![ci]; const x = i * k - t; y += Math.abs(this.amplitudes[ci] * this.sin(this.verses[ci] * x, this.phases[ci]) * this.globalAttFn(x)); } // Divide for NoOfCurves so that y <= 1 return y / this.noOfCurves; } private yPos(i: number): number { return ( this.AMPLITUDE_FACTOR * this.ctrl.heightMax * this.ctrl.amplitude * this.yRelativePos(i) * this.globalAttFn((i / this.GRAPH_X) * 2) ); } private xPos(i: number): number { return this.ctrl.width * ((i + this.GRAPH_X) / (this.GRAPH_X * 2)); } private drawSupportLine() { const { ctx } = this.ctrl; const coo: [number, number, number, number] = [0, this.ctrl.heightMax, this.ctrl.width, 1]; const gradient = ctx.createLinearGradient.apply(ctx, coo); gradient.addColorStop(0, "transparent"); gradient.addColorStop(0.1, "rgba(255,255,255,.5)"); gradient.addColorStop(1 - 0.1 - 0.1, "rgba(255,255,255,.5)"); gradient.addColorStop(1, "transparent"); ctx.fillStyle = gradient; ctx.fillRect.apply(ctx, coo); } draw() { const { ctx } = this.ctrl; ctx.globalAlpha = 0.7; ctx.globalCompositeOperation = "lighter"; if (this.spawnAt === 0) { this.spawn(); } if (this.definition.supportLine) { // Draw the support line return this.drawSupportLine(); } for (let ci = 0; ci < this.noOfCurves; ci++) { if (this.spawnAt + this.despawnTimeouts[ci] <= Date.now()) { this.amplitudes[ci] -= this.DESPAWN_FACTOR; } else { this.amplitudes[ci] += this.DESPAWN_FACTOR; } this.amplitudes[ci] = Math.min(Math.max(this.amplitudes[ci], 0), this.finalAmplitudes[ci]); this.phases[ci] = (this.phases[ci] + this.ctrl.speed * this.speeds[ci] * this.SPEED_FACTOR) % (2 * Math.PI); } let maxY = -Infinity; let minX = +Infinity; // Write two opposite waves for (const sign of [1, -1]) { ctx.beginPath(); for (let i = -this.GRAPH_X; i <= this.GRAPH_X; i += this.ctrl.opt.pixelDepth!) { const x = this.xPos(i); const y = this.yPos(i); ctx.lineTo(x, this.ctrl.heightMax - sign * y); minX = Math.min(minX, x); maxY = Math.max(maxY, y); } ctx.closePath(); ctx.fillStyle = `rgba(${this.definition.color}, 1)`; ctx.strokeStyle = `rgba(${this.definition.color}, 1)`; ctx.fill(); } if (maxY < this.DEAD_PX && this.prevMaxY > maxY) { this.spawnAt = 0; } this.prevMaxY = maxY; return null; } static getDefinition(): IiOS9CurveDefinition[] { return [ { color: "255,255,255", supportLine: true, }, { // blue color: "15, 82, 169", }, { // red color: "173, 57, 76", }, { // green color: "48, 220, 155", }, ]; } } ================================================ FILE: src/klassich.ts ================================================ import SiriWave, { IClassicCurveDefinition, ICurve } from "./index"; export class ClassicCurve implements ICurve { ctrl: SiriWave; definition: IClassicCurveDefinition; ATT_FACTOR = 4; GRAPH_X = 2; AMPLITUDE_FACTOR = 0.6; constructor(ctrl: SiriWave, definition: IClassicCurveDefinition) { this.ctrl = ctrl; this.definition = definition; } private globalAttFn(x: number): number { return Math.pow(this.ATT_FACTOR / (this.ATT_FACTOR + Math.pow(x, this.ATT_FACTOR)), this.ATT_FACTOR); } private xPos(i: number): number { return this.ctrl.width * ((i + this.GRAPH_X) / (this.GRAPH_X * 2)); } private yPos(i: number): number { return ( this.AMPLITUDE_FACTOR * (this.globalAttFn(i) * (this.ctrl.heightMax * this.ctrl.amplitude) * (1 / this.definition.attenuation) * Math.sin(this.ctrl.opt.frequency! * i - this.ctrl.phase)) ); } draw(): void { const { ctx } = this.ctrl; ctx.moveTo(0, 0); ctx.beginPath(); const finalColor = this.definition.color || this.ctrl.color; const colorHex = finalColor.replace(/rgb\(/g, "").replace(/\)/g, ""); ctx.strokeStyle = `rgba(${colorHex},${this.definition.opacity})`; ctx.lineWidth = this.definition.lineWidth; // Cycle the graph from -X to +X every PX_DEPTH and draw the line for (let i = -this.GRAPH_X; i <= this.GRAPH_X; i += this.ctrl.opt.pixelDepth!) { ctx.lineTo(this.xPos(i), this.ctrl.heightMax + this.yPos(i)); } ctx.stroke(); } static getDefinition(): IClassicCurveDefinition[] { return [ { attenuation: -2, lineWidth: 1, opacity: 0.1, }, { attenuation: -6, lineWidth: 1, opacity: 0.2, }, { attenuation: 4, lineWidth: 1, opacity: 0.4, }, { attenuation: 2, lineWidth: 1, opacity: 0.6, }, { attenuation: 1, lineWidth: 1.5, opacity: 1, }, ]; } } ================================================ FILE: src/pramp/algodaten/DecodeVariations/answer.java ================================================ import java.io.*; import java.util.*; class Solution { static int decodeVariations(String S) { int[] dp = new int[S.length() + 1]; dp[0] = 1; if (S.charAt(0) != '0'){ dp[1] = 1; } else { dp[1] = 0; } for (int i = 2; i <= S.length(); i++){ int oneD = Integer.parseInt(S.substring(i-1, i)); int twoD = Integer.parseInt(S.substring(i-2, i)); if ( 1 <= oneD && oneD <= 9){ dp[i] += dp[i-1]; } if ( 10 <= twoD && twoD <= 26){ dp[i] += dp[i-2]; } } return dp[S.length()]; } public static void main(String[] args) { System.out.println(decodeVariations("1262") + " Should be 3"); } } ================================================ FILE: src/pramp/algodaten/DecodeVariations/answer.txt ================================================ Solution: Dynamic Programming Let dp(i) be the answer for the string S[i:]. We can calculate dp(i) in terms of dp(i+1) and dp(i+2). If S[i] == 0, then dp(i) = 0. There are no ways, since no encoded letter starts with 0. If S[i] == 1, then we have dp(i) = dp(i+1) + dp(i+2), since either we write A plus any way to write S[i+1:], or we write a letter that encodes somewhere between 10 and 19, plus any way to write S[i+2:]. If S[i] == 2, then we have dp(i) = dp(i+1) + (S[i+1] <= 6 ? dp(i+2) : 0). Either we write B plus any way to write S[i+1:], or we write a letter that encodes somewhere between 20 and 26, plus any way to write S[i+2:]. If S[i] > 2, then we have dp(i) = dp(i+1). For example, if S[i] = 5, then we write E plus any way to write S[i+1:]. We can’t write any other letters because the only letter which has encoding that starts with 5 is E. Putting this all together, we have the following code: function decodeVariations(S): N = S.length dp = new Array(N) dp[N] = 1 for i from N-1 to 0: if S[i] == '0': dp[i] = 0 else if S[i] == '1': dp[i] = dp[i+1] + dp[i+2] else if S[i] == '2': dp[i] = dp[i+1] if i+1 < S.length && S[i+1] <= '6': dp[i] += dp[i+2] else: dp[i] = dp[i+1] return dp[0] Of course, since at each step we only reference dp[i+1] and dp[i+2], we could store these as variables first and second. This means we do not need to store the entire array: function decodeVariations(S): pre, cur = 27, 0 first, second = 1, 1 for i from S.length - 1 to 0: d = int(S[i]) if d == 0: cur = 0 else: cur = first # pre represents the previously seen number S[i+1] # If d*10 + pre < 26 (and d != 0), it is valid to # write a letter that uses two digits of encoding length, # so we count 'second = dp[i+2]' in our current answer. if d * 10 + pre < 27: cur += second pre = d first, second = cur, first return cur Time Complexity: O(N) where N is the length of S. Space Complexity: O(N) to store the array dp. In our space saving variation, the complexity is O(1). ================================================ FILE: src/pramp/algodaten/DecodeVariations/answerII.py ================================================ import unittest def decodeVariations(S): """ @param S: str @return: int Approach 1: Decode Variations -> DP 1262 12 6 2 LFB 1 2 6 2 ABFB 1 26 2 AZB 3 options DP input: i is the index of the current letter in word output: total combinations or options Base Case: DP[len(word)] = 1 Start Case: DP(0) Recursion: firstDigit = int(DP[i]) twoDigit = int(DP[i:i+2]) if 1 <= firstLetter <= 9: total += dp(i+1) if 10 <= twoDigit <= 26: total += dp(i+2) return total DP linear or memo DP R- O(N ) with memo (2^N) without memo S-O(N) """ return VariationsDecoder().decode_variations(S) class VariationsDecoder: def decode_variations(self,S): ''' ''' #hashmap memo cache self.dp_memo_cache = {} self.dp_memo_cache[len(S)] = 1 #store S self.S = S #start case with 0 return self._dp(0) def _dp(self, letter_index): # letter_index is the index of the letter in the word S #memo if letter_index in self.dp_memo_cache: return self.dp_memo_cache[letter_index] #base case if letter_index == len(self.S): # if we reached the end, we found 1 way to decode return 1 #recursion total = 0 firstDigit = int(self.S[letter_index]) if 1 <= firstDigit <= 9: total += self._dp(letter_index+1) if letter_index +1 < len(self.S): twoDigit = int(self.S[letter_index:letter_index+2]) if 10 <= twoDigit <= 26: total += self._dp(letter_index+2) self.dp_memo_cache[letter_index] = total return total class TestVariationsDecoder(unittest.TestCase): def test_1(self): #Arrange S = '1' #Act ans= VariationsDecoder().decode_variations(S) #Assert print(ans) self.assertEqual(ans,1) def test_1262(self): #Arrange S = '1262' #Act ans= VariationsDecoder().decode_variations(S) #Assert print(ans) self.assertEqual(ans,3) #unittest.main(verbosity=2) ================================================ FILE: src/pramp/algodaten/DecodeVariations/hints.txt ================================================ Try to write the number of ways to write S[i:] in terms of the number of ways to write S[i+1:] and S[i+2:]. For example, if there are 2 ways to write S[1:] = '262' ('ZB' and 'BFB'), and one way to write S[2:] = '62' ('FB'), then we could say there are 3 ways to write S as we either write 'A' then a decoding of S[1:], or write 'L' then a decoding of S[2:]. We can store the answer for S[i:] into an integer array dp. The recursion is given below. You can help the candidate with the specific part of the recursion they are having difficulty with: If S[i] == 0, then dp(i) = 0. If S[i] == 1, then we have dp(i) = dp(i+1) + dp(i+2). If S[i] == 2, then we have dp(i) = dp(i+1) + (S[i+1] <= 6 ? dp(i+2) : 0). If S[i] > 2, then we have dp(i) = dp(i+1). ================================================ FILE: src/pramp/algodaten/DecodeVariations/question.txt ================================================ decode-variations A letter can be encoded to a number in the following way: 'A' -> '1', 'B' -> '2', 'C' -> '3', ..., 'Z' -> '26' A message is a string of uppercase letters, and it is encoded first using this scheme. For example, 'AZB' -> '1262' Given a string of digits S from 0-9 representing an encoded message, return the number of ways to decode it. Examples: input: S = '1262' output: 3 explanation: There are 3 messages that encode to '1262': 'AZB', 'ABFB', and 'LFB'. Constraints: [time limit] 5000ms [input] string S 1 ≤ S.length ≤ 12 [output] integer ================================================ FILE: src/pramp/algodaten/Diff Between Two Strings/answer.py ================================================ def diffBetweenTwoStrings(source, target): m, n = len(target), len(source) dp = [[0 for j in range(n + 1)] for i in range(m + 1)] # Build dp for i in range(m + 1): for j in range(n + 1): if i == 0: dp[i][j] = j elif j == 0: dp[i][j] = i else: if target[i - 1] == source[j - 1]: dp[i][j] = dp[i - 1][j - 1] else: dp[i][j] = min(dp[i - 1][j], dp[i][j - 1]) + 1 # Reconstruct path path = [] i, j = m, n while i > 0 and j > 0: if target[i - 1] == source[j - 1]: # Write char with no edits path.append(source[j - 1]) i -= 1 j -= 1 else: # We must either subtract source[j - 1] or add target[i - 1] if dp[i][j - 1] < dp[i - 1][j]: path.append("-" + source[j - 1]) j -= 1 else: path.append("+" + target[i - 1]) i -= 1 while i > 0: path.append("+" + target[i - 1]) i -= 1 while j > 0: path.append("-" + source[j - 1]) j -= 1 path.reverse() return path ================================================ FILE: src/pramp/algodaten/Diff Between Two Strings/question.txt ================================================ Diff Between Two Strings Given two strings of uppercase letters source and target, list (in string form) a sequence of edits to convert from source to target that uses the least edits possible. For example, with strings source = "ABCDEFG", and target = "ABDFFGH" we might return: ["A", "B", "-C", "D", "-E", "F", "+F", "G", "+H" More formally, for each character C in source, we will either write the token C, which does not count as an edit; or write the token -C, which counts as an edit. Additionally, between any token that we write, we may write +D where D is any letter, which counts as an edit. At the end, when reading the tokens from left to right, and not including tokens prefixed with a minus-sign, the letters should spell out target (when ignoring plus-signs.) In the example, the answer of A B -C D -E F +F G +H has total number of edits 4 (the minimum possible), and ignoring subtraction-tokens, spells out A, B, D, F, +F, G, +H which represents the string target. If there are multiple answers, use the answer that favors removing from the source first. Constraints: [time limit] 5000ms [input] string source 2 ≤ source.length ≤ 12 [input] string target 2 ≤ target.length ≤ 12 [output] array.string ================================================ FILE: src/pramp/algodaten/Island Count/answer.c ================================================ #include #include int r_size; int c_size; //void mark_island(int (*mat)[c_size], int r, int c) { void mark_island(int mat[][c_size], int r, int c) { if (r < 0 || r >= r_size || c < 0 || c >= c_size) return; if (mat[r][c] == 0) return; mat[r][c] = 0; // marked mark_island(mat, r - 1, c); mark_island(mat, r, c - 1); mark_island(mat, r + 1, c); mark_island(mat, r, c + 1); } int getNumberOfIslands(size_t numRows, size_t numCols, int binaryMatrix[numRows][numCols]) { int islands = 0; r_size = numRows; c_size = numCols; for (int r = 0; r < numRows; r++) { for (int c = 0; c < numCols; c++) { if (binaryMatrix[r][c] == 1) { islands++; mark_island(binaryMatrix, r, c); } } } return islands; } int main() { int m[2][2] = {{1, 0}, {0, 1}}; int result = getNumberOfIslands(2,2,m); printf("%d", result); return 0; } ================================================ FILE: src/pramp/algodaten/Island Count/hints.txt ================================================ Island Count If your peer is stuck, ask them if they know how to traverse a undirected graph. At this point, you peer may come up with a recursive solution by using a Breadth-First Search (BFS) or a Depth-First Search (DFS) algorithm. While this works, see if you can nudge them toward an iterative solution. This is not a must, but preferable. Make sure that your peer’s code does not access out of bound indices, especially when trying to traverse adjacent cells in binary Matrix. Watch out for duplicate island counting in your peer’s code. It’s important that a visited cell of 1 is marked properly to avoid redundant counting. Any solution that takes more than O(N⋅M) time isn’t optimal. ================================================ FILE: src/pramp/algodaten/Island Count/question.txt ================================================ Island Count Given a 2D array binaryMatrix of 0s and 1s, implement a function getNumberOfIslands that returns the number of islands of 1s in binaryMatrix. An island is defined as a group of adjacent values that are all 1s. A cell in binaryMatrix is considered adjacent to another cell if they are next to each either on the same row or column. Note that two values of 1 are not part of the same island if they’re sharing only a mutual “corner” (i.e. they are diagonally neighbors). Explain and code the most efficient solution possible and analyze its time and space complexities. Example: input: binaryMatrix = [ [0, 1, 0, 1, 0], [0, 0, 1, 1, 1], [1, 0, 0, 1, 0], [0, 1, 1, 0, 0], [1, 0, 1, 0, 1] ] output: 6 # since this is the number of islands in binaryMatrix. # See all 6 islands color-coded below. alt Constraints: [time limit] 5000ms [input] array.array.int binaryMatrix 1 ≤ binaryMatrix.length ≤ 100 1 ≤ binaryMatrix[i].length ≤ 100 [output] integer ================================================ FILE: src/pramp/algodaten/Number of Paths/ans.py ================================================ def num_of_paths_to_dest_v2(n): paths = [[0 for col in range(n)] for row in range(n)] for row in range(n): for col in range(row, n): if row == 0 and col == 0: paths[row][col] = 1 if row > 0: paths[row][col] += paths[row-1][col] if col > 0: paths[row][col] += paths[row][col -1] return paths[-1][-1] ================================================ FILE: src/pramp/algodaten/Number of Paths/answer.txt ================================================ Number of Paths The recursive approach The simplest way to create this function is to write a recursive function, that calculates the possible paths to each square on the grid, using the squares already calculated. Our output is the paths from (0,0) to (n-1,n-1). Notice the following recursive relation: for every path from (0,0) to (i,j), either: The last step is N, in which case it is generated by a legitimate path to (i,j-1), and then one step north. The last step is E, in which case it is generated by a legitimate path to (i-1,j), and then one step east. For example, take all paths to (4,3). The paths that end with an N, such as “EEENN”, induce paths to (4,2) by removing the last N (“EEEN”). On the other hand, the paths that end with an E, such as “ENENE”, induce paths to (3,3) by removing the last E (“ENEN”). This indicates the number of ways to square (i,j) is equal to the number of ways to square (i-1,j) plus the number of ways to (i,j-1). We build the function that uses this recursive relation, to calculate the number of paths. Notice that the function may call the same square multiple times, so we use the memoization technique to reduce the number of calls significantly: Pseudocode: # input: n - a positive integer representing the grid size. # output: number of valid paths from (0,0) to (n-1,n-1). function numOfPathsToDest(n): # allocate a 2D array for memoization memo = [][] # the memoization array is initialized with -1 # to indicate uncalculated squares. for i from 0 to n-1: for j from 0 to n-1: memo[i][j] = -1 return numOfPathsToSquare(n-1, n-1, memo) # input: # i, j - a pair of non-negative integer coordinates # memo - a 2D memoization array. # output: # number of paths from (0,0) to the square represented in (i,j), function numOfPathsToSquare(i, j, memo): if (i < 0 OR j < 0): return 0 else if (i < j): memo[i][j] = 0 else if (memo[i][j] != -1): return memo[i][j] else if (i == 0 AND j == 0): memo[i][j] = 1 else: memo[i][j] = numOfPathsToSquare(i, j -1, memo) + numOfPathsToSquare(i - 1, j, memo) return memo[i][j] Time Complexity: first, notice that in order to calculate the number of paths to a specific square, we need all the square south and west to it. This implies that all squares beneath the diagonal are calculated. In addition, almost every square value is used twice - for the square north to it and east to it (except for the border squares, which are used once). This means that our time complexity is O(n^2), since the recursive function is called once or twice for about half of the squares, and each call takes O(1) time. Space Complexity: the memoization requires the space complexity to be also O(n^2), since we save values for all squares. The iterative approach The space complexity can be improved to O(n) if we choose an iterative solution, which uses the same recursive relation. Notice that for calculating the paths for square (i,j) we only need squares (i-1,j) and (i,j-1). Thus, by calculating the number of paths row by row from south to north, and west to east in the rows, the function needs to save only the last two rows. This may be implemented as follows: Pseudocode: # input: n - a positive integer representing the grid size. # output: number of valid paths from (0,0) to (n-1,n-1). function numOfPathsToDest(n): if (n == 1): return 1 lastRow = [] for i from 1 to n-1: lastRow[i] = 1 # base case - the first row is all ones currentRow = [] for j from 1 to n-1: for i from j to n-1: if (i == j): currentRow[i] = lastRow[i] else: currentRow[i] = currentRow[i-1] + lastRow[i] lastRow = currentRow return currentRow[n-1] Time Complexity: as we see, the function still calculates every square south-east to the diagonal, leaving the time complexity to be O(n^2), Space Complexity: the space complexity is reduced to O(n) since we are memoizing only the last two rows. Combinatorial Remark (optional read): There is a closed formula that answers this question, called the Catalan Number Formula. This formula is based on the fact that every path from (0,0) to (n-1,n-1) is parallel to a sequence of n-1 pairs of parentheses which are correctly matched: Take a path sequence of N’s and E’s. since the car begins at (0,0) and ends at (n-1,n-1), it needs to go exactly n-1 times East, and n-1 times North. Put differently, all sequences consist of n-1 E’s and n-1 N’s. Just like every balanced parenthesis string has the same number of “(“ and “)” signs. Furthermore, in every pair (i,j), the first coordinate is the number of times the car went east so far, and the second coordinate is the number of times the car went north. This indicates that the diagonal restriction means the number of E’s, in every prefix of the string is equal or greater than the number of N’s. Just as in every balanced parenthesis string, the number of “(“ is is equal or greater than the number of “)”. The number of the balanced parenthesis strings of n-1 pairs, is given by the Catalan Number Formula: . The proof for its correctness is beyond our scope, but is located in the Catalan Number Wikipedia page. Note that although this formula is mathematically closed, calculating the Binomial Coefficient , is done in O(n) runtime complexity, so calculating the number directly doesn’t improve the asymptotic runtime complexity. But direct calculation spares the need for saving previous values, so the space complexity may be reduced to O(1) this way. ================================================ FILE: src/pramp/algodaten/Number of Paths/hints.txt ================================================ While this question may seem hard to comprehend at first sight, it is easily solved in the “divide and conquer” recursive method. Encourage your peer to implement first any solution, even if it is not optimal. Once done, ask them to optimize it. If your peer doesn’t have any intuition to the question, guide them towards answering a more general question - given (i,j), calculate all possible ways to travel from (0,0) to (i,j). Also, suggest your peer to draw a grid and try to calculate it manually for a few squares. Another good hint is to direct to first think about the obvious base cases, e.g. if the square (i,j) is on the other side of the border or if it is out of the grid (0 ways). If your peer still doesn’t have a clue, explain the recursive relation provided in the solution. Your peer should get a full score only if the function achieves O(n^2) time complexity, and O(n) space complexity (or better), and doing so without any hints. Obviously, we do not expect your peer to be familiar with the combinatorial solution, or discover it by themselves. Moreover, it’s OK if your peer discovers the combinatorial approach for this question, but since this is a coding interview, if your peer is familiar with it beforehand, encourage them to find another recursive solution to the question. ================================================ FILE: src/pramp/algodaten/Number of Paths/question.txt ================================================ Number of Paths You’re testing a new driverless car that is located at the Southwest (bottom-left) corner of an n×n grid. The car is supposed to get to the opposite, Northeast (top-right), corner of the grid. Given n, the size of the grid’s axes, write a function numOfPathsToDest that returns the number of the possible paths the driverless car can take. alt the car may move only in the white squares For convenience, let’s represent every square in the grid as a pair (i,j). The first coordinate in the pair denotes the east-to-west axis, and the second coordinate denotes the south-to-north axis. The initial state of the car is (0,0), and the destination is (n-1,n-1). The car must abide by the following two rules: it cannot cross the diagonal border. In other words, in every step the position (i,j) needs to maintain i >= j. See the illustration above for n = 5. In every step, it may go one square North (up), or one square East (right), but not both. E.g. if the car is at (3,1), it may go to (3,2) or (4,1). Explain the correctness of your function, and analyze its time and space complexities. Example: input: n = 4 output: 5 # since there are five possibilities: # “EEENNN”, “EENENN”, “ENEENN”, “ENENEN”, “EENNEN”, # where the 'E' character stands for moving one step # East, and the 'N' character stands for moving one step # North (so, for instance, the path sequence “EEENNN” # stands for the following steps that the car took: # East, East, East, North, North, North) Constraints: [time limit] 5000ms [input] integer n 1 ≤ n ≤ 100 [output] integer ================================================ FILE: src/pramp/algodaten/Number_Of_Paths/answer.txt ================================================ Number of Paths The recursive approach The simplest way to create this function is to write a recursive function, that calculates the possible paths to each square on the grid, using the squares already calculated. Our output is the paths from (0,0) to (n-1,n-1). Notice the following recursive relation: for every path from (0,0) to (i,j), either: The last step is N, in which case it is generated by a legitimate path to (i,j-1), and then one step north. The last step is E, in which case it is generated by a legitimate path to (i-1,j), and then one step east. For example, take all paths to (4,3). The paths that end with an N, such as “EEENN”, induce paths to (4,2) by removing the last N (“EEEN”). On the other hand, the paths that end with an E, such as “ENENE”, induce paths to (3,3) by removing the last E (“ENEN”). This indicates the number of ways to square (i,j) is equal to the number of ways to square (i-1,j) plus the number of ways to (i,j-1). We build the function that uses this recursive relation, to calculate the number of paths. Notice that the function may call the same square multiple times, so we use the memoization technique to reduce the number of calls significantly: Pseudocode: # input: n - a positive integer representing the grid size. # output: number of valid paths from (0,0) to (n-1,n-1). function numOfPathsToDest(n): # allocate a 2D array for memoization memo = [][] # the memoization array is initialized with -1 # to indicate uncalculated squares. for i from 0 to n-1: for j from 0 to n-1: memo[i][j] = -1 return numOfPathsToSquare(n-1, n-1, memo) # input: # i, j - a pair of non-negative integer coordinates # memo - a 2D memoization array. # output: # number of paths from (0,0) to the square represented in (i,j), function numOfPathsToSquare(i, j, memo): if (i < 0 OR j < 0): return 0 else if (i < j): memo[i][j] = 0 else if (memo[i][j] != -1): return memo[i][j] else if (i == 0 AND j == 0): memo[i][j] = 1 else: memo[i][j] = numOfPathsToSquare(i, j -1, memo) + numOfPathsToSquare(i - 1, j, memo) return memo[i][j] Time Complexity: first, notice that in order to calculate the number of paths to a specific square, we need all the square south and west to it. This implies that all squares beneath the diagonal are calculated. In addition, almost every square value is used twice - for the square north to it and east to it (except for the border squares, which are used once). This means that our time complexity is O(n^2), since the recursive function is called once or twice for about half of the squares, and each call takes O(1) time. Space Complexity: the memoization requires the space complexity to be also O(n^2), since we save values for all squares. The iterative approach The space complexity can be improved to O(n) if we choose an iterative solution, which uses the same recursive relation. Notice that for calculating the paths for square (i,j) we only need squares (i-1,j) and (i,j-1). Thus, by calculating the number of paths row by row from south to north, and west to east in the rows, the function needs to save only the last two rows. This may be implemented as follows: Pseudocode: # input: n - a positive integer representing the grid size. # output: number of valid paths from (0,0) to (n-1,n-1). function numOfPathsToDest(n): if (n == 1): return 1 lastRow = [] for i from 1 to n-1: lastRow[i] = 1 # base case - the first row is all ones currentRow = [] for j from 1 to n-1: for i from j to n-1: if (i == j): currentRow[i] = lastRow[i] else: currentRow[i] = currentRow[i-1] + lastRow[i] lastRow = currentRow return currentRow[n-1] Time Complexity: as we see, the function still calculates every square south-east to the diagonal, leaving the time complexity to be O(n^2), Space Complexity: the space complexity is reduced to O(n) since we are memoizing only the last two rows. Combinatorial Remark (optional read): There is a closed formula that answers this question, called the Catalan Number Formula. This formula is based on the fact that every path from (0,0) to (n-1,n-1) is parallel to a sequence of n-1 pairs of parentheses which are correctly matched: Take a path sequence of N’s and E’s. since the car begins at (0,0) and ends at (n-1,n-1), it needs to go exactly n-1 times East, and n-1 times North. Put differently, all sequences consist of n-1 E’s and n-1 N’s. Just like every balanced parenthesis string has the same number of “(“ and “)” signs. Furthermore, in every pair (i,j), the first coordinate is the number of times the car went east so far, and the second coordinate is the number of times the car went north. This indicates that the diagonal restriction means the number of E’s, in every prefix of the string is equal or greater than the number of N’s. Just as in every balanced parenthesis string, the number of “(“ is is equal or greater than the number of “)”. The number of the balanced parenthesis strings of n-1 pairs, is given by the Catalan Number Formula: . The proof for its correctness is beyond our scope, but is located in the Catalan Number Wikipedia page. Note that although this formula is mathematically closed, calculating the Binomial Coefficient , is done in O(n) runtime complexity, so calculating the number directly doesn’t improve the asymptotic runtime complexity. But direct calculation spares the need for saving previous values, so the space complexity may be reduced to O(1) this way. ================================================ FILE: src/pramp/algodaten/Number_Of_Paths/hints.txt ================================================ Number of Paths While this question may seem hard to comprehend at first sight, it is easily solved in the “divide and conquer” recursive method. Encourage your peer to implement first any solution, even if it is not optimal. Once done, ask them to optimize it. If your peer doesn’t have any intuition to the question, guide them towards answering a more general question - given (i,j), calculate all possible ways to travel from (0,0) to (i,j). Also, suggest your peer to draw a grid and try to calculate it manually for a few squares. Another good hint is to direct to first think about the obvious base cases, e.g. if the square (i,j) is on the other side of the border or if it is out of the grid (0 ways). If your peer still doesn’t have a clue, explain the recursive relation provided in the solution. Your peer should get a full score only if the function achieves O(n^2) time complexity, and O(n) space complexity (or better), and doing so without any hints. Obviously, we do not expect your peer to be familiar with the combinatorial solution, or discover it by themselves. Moreover, it’s OK if your peer discovers the combinatorial approach for this question, but since this is a coding interview, if your peer is familiar with it beforehand, encourage them to find another recursive solution to the question. ================================================ FILE: src/pramp/algodaten/Number_Of_Paths/question.txt ================================================ You’re testing a new driverless car that is located at the Southwest (bottom-left) corner of an n×n grid. The car is supposed to get to the opposite, Northeast (top-right), corner of the grid. Given n, the size of the grid’s axes, write a function numOfPathsToDest that returns the number of the possible paths the driverless car can take. alt the car may move only in the white squares For convenience, let’s represent every square in the grid as a pair (i,j). The first coordinate in the pair denotes the east-to-west axis, and the second coordinate denotes the south-to-north axis. The initial state of the car is (0,0), and the destination is (n-1,n-1). The car must abide by the following two rules: it cannot cross the diagonal border. In other words, in every step the position (i,j) needs to maintain i >= j. See the illustration above for n = 5. In every step, it may go one square North (up), or one square East (right), but not both. E.g. if the car is at (3,1), it may go to (3,2) or (4,1). Explain the correctness of your function, and analyze its time and space complexities. Example: input: n = 4 output: 5 # since there are five possibilities: # “EEENNN”, “EENENN”, “ENEENN”, “ENENEN”, “EENNEN”, # where the 'E' character stands for moving one step # East, and the 'N' character stands for moving one step # North (so, for instance, the path sequence “EEENNN” # stands for the following steps that the car took: # East, East, East, North, North, North) Constraints: [time limit] 5000ms [input] integer n 1 ≤ n ≤ 100 [output] integer ================================================ FILE: src/pramp/algodaten/Root of Number/ans.c ================================================ #include #include void root(double x, unsigned int n, double *out) { } int main() { return 0; } ================================================ FILE: src/pramp/algodaten/Root of Number/ans.java ================================================ import java.io.*; import java.util.*; class Solution { static double root(double x, int n) { double output=1; while(Math.pow(output, n) < x ) { output++; } if(Math.pow(output, n) == x) { return output; } double start = output-1; double result = 1; for(double i=start; i < output; i += 0.01) { result = Math.pow(i, n); if(result == x) { output = i; break; } if(result > x) { output = i-0.01; break; } } return output; } public static void main(String[] args) { } } // https://github.com/esthicodes ================================================ FILE: src/pramp/algodaten/Root of Number/answer.txt ================================================ Root of Number The solution presented here is much like a binary search in a sorted array. Pseudocode: function root(x, n): if (x == 0): return 0 lowerBound = 0 upperBound = max(1, x) approxRoot = (upperBound + lowerBound) / 2 while (approxRoot - lowerBound >= 0.001): if (power(approxRoot, n) > x): upperBound = approxRoot else if (power(approxRoot, n) < x): lowerBound = approxRoot else break approxRoot = (upperBound + lowerBound) / 2 return approxRoot As we can see, the function finds an initial lower and upper bound for the answer, and sets the initial guess for the answer to be the average. Then the iteration begins, and the we notice that the following facts are true in every step of the iteration: (approxRoot - lowerBound) = ½(upperBound - lowerBound) - Since approxRoot is always the average value between upperBound and lowerBound. At the end of every iteration, the real root always satisfies the inequality lowerBound < root < upperBound: this is true since in the beginning of the iteration, we check if approxRoot to the power of n is greater than or lesser than x. The power function is monotonically increasing (i.e. a < b dictates that a^n < b^n) meaning this indicates whether the approximation is too high or too low. The value of (upperBound - lowerBound) is cut by half in every iteration - since in every step we replace one of the values of upperBound or lowerBound by their current average. Obviously it means that eventually the distance between the bounds is lower than 0.001. Since the real root is between the bounds, this promises the algorithm stops. Thus, since lowerBound< root < upperBound, then the true error - |root-approxRoot|, satisfies |root-approxRoot| < (approxRoot - lowerBound) - so it is indeed enough to check when the value on the right side is lower than 0.001. Thus the algorithm always stops, and output of the algorithm is correct. The best way to explain the solution algorithm is by using it on an example. Let’s try calculating the 3rd root of 7: We mark the real root as y ≅ 1.912931182772389. Our method shall iterate in a main loop, and try to produce a sequence of approximations that converge to y, until reaching a number that is close enough. Iteration 1: Initially, we recognized that 0 < y, since 03 = 0 < 7. Also y < 7. Thus lowerBound = 0 and upperBound = 7. Our first member of the sequence is chosen to be the average between 0 and 7 - approxRoot = 3.5. Since 3.53 ≅ 42.8 > y, then we conclude that 0 < y < 3.5, so we conclude upperBound = 3.5. Note that the error is bounded: |y - approxRoot| < |3.5 - 0| = 3.5. Iteration 2: Since we concluded that 0 < y < 3.5, our second member of the sequence should be, again, the average between our bounds 0 and 3.5 - approxRoot= 1.75. This time 1.753 ≅ 5.3 < 7, which means 1.75 < y < 3.5, so we update lowerBound = 1.75. The error is bounded by: |y - approxRoot| < |upperBound - lowerBound|= |3.5- 1.75| = 1.75. We continue the sequence this way, and notice that the error bound is reduced by half by every iteration. Here is a summary of the first few iterations in this method: # Iteration LowerBound UpperBound approxRoot value |y- approxRoot| - True Error Error bound 1 0 7 3.5 ≅ 1.58 < 3.5 (7×2-1) 2 0 3.5 1.75 ≅0.16 < 1.75 (7×2-2) 3 1.75 3.5 2.625 ≅0.71 < 0.875 (7×2-3) 4 1.75 2.625 2.1875 ≅0.27 < 0.4375 (7×2-4) As we can see, by choosing the average value between By the 13th iteration, we’ll reach an error bound of 7×2^(-13), which is less than 0.001 (notice that the error may be much smaller, but since we don’t actually know the real value of y in the process of computation, we cannot promise a better bound). Time Complexity: notice that every loop iteration is done in O(1), under the assumption that the power function takes a constant time. The initial error is x, and the error is multiplied by 0.5 in every iteration. Thus the number of iterations is the minimal k such that: 2^(-k) x<0.001 i.e. 2^(-k)<(0.001 / x) k >log(x) + 3log(10) = O(log(x)) The number of iterations is therefore O(log(x)), meaning the total runtime is O(log(x)) if we refer to the value stored in x, or O(x) if we refer to the number of bits required to represent x (since it takes Log(x) in average bits to represent a number x). Space Complexity: O(1), since we only need a constant number of variables for the algorithm. Mathematical Note: our explanation used the fact that power function x^n is a monotonically increasing continuous function on the positive numbers. Although it is true, the Intermediate value theorem in calculus, states that this method will work for every continuous function - but the proof for this requires more mathematical tools. ================================================ FILE: src/pramp/algodaten/Root of Number/hints.txt ================================================ If your peer doesn’t know how to tackle this question, advise them to try it first on concrete examples. Encourage them to make an intelligent guess, and then bound the error. If you peer is still stuck, ask them how they can use binary search to solve the problem. Also, it is recommended for your peer to look at the graph of the function xn. And while the function may not use any kind of built-in power function, it is OK for your peer to use such functions in the programming and debugging process. Your peer should receive full score for this question only if they manage to find a method that is at least as efficient as the method below, along with explaining how it is correct and analyzing its time and space complexities. ================================================ FILE: src/pramp/algodaten/Root of Number/question.txt ================================================ Root of Number Many times, we need to re-implement basic functions without using any standard library functions already implemented. For example, when designing a chip that requires very little memory space. In this question we’ll implement a function root that calculates the n’th root of a number. The function takes a nonnegative number x and a positive integer n, and returns the positive n’th root of x within an error of 0.001 (i.e. suppose the real root is y, then the error is: |y-root(x,n)| and must satisfy |y-root(x,n)| < 0.001). Don’t be intimidated by the question. While there are many algorithms to calculate roots that require prior knowledge in numerical analysis (some of them are mentioned here), there is also an elementary method which doesn’t require more than guessing-and-checking. Try to think more in terms of the latter. Make sure your algorithm is efficient, and analyze its time and space complexities. Examples: input: x = 7, n = 3 output: 1.913 input: x = 9, n = 2 output: 3 Constraints: [time limit] 5000ms [input] float x 0 ≤ x [input] integer n 0 < n [output] float ================================================ FILE: src/pramp/algodaten/Root_of_Number/hints.txt ================================================ Root of Number If your peer doesn’t know how to tackle this question, advise them to try it first on concrete examples. Encourage them to make an intelligent guess, and then bound the error. If you peer is still stuck, ask them how they can use binary search to solve the problem. Also, it is recommended for your peer to look at the graph of the function xn. And while the function may not use any kind of built-in power function, it is OK for your peer to use such functions in the programming and debugging process. Your peer should receive full score for this question only if they manage to find a method that is at least as efficient as the method below, along with explaining how it is correct and analyzing its time and space complexities. ================================================ FILE: src/pramp/algodaten/Root_of_Number/problem.txt ================================================ Root of Number Many times, we need to re-implement basic functions without using any standard library functions already implemented. For example, when designing a chip that requires very little memory space. In this question we’ll implement a function root that calculates the n’th root of a number. The function takes a nonnegative number x and a positive integer n, and returns the positive n’th root of x within an error of 0.001 (i.e. suppose the real root is y, then the error is: |y-root(x,n)| and must satisfy |y-root(x,n)| < 0.001). Don’t be intimidated by the question. While there are many algorithms to calculate roots that require prior knowledge in numerical analysis (some of them are mentioned here), there is also an elementary method which doesn’t require more than guessing-and-checking. Try to think more in terms of the latter. Make sure your algorithm is efficient, and analyze its time and space complexities. Examples: input: x = 7, n = 3 output: 1.913 input: x = 9, n = 2 output: 3 Constraints: [time limit] 5000ms [input] float x 0 ≤ x [input] integer n 0 < n [output] float ================================================ FILE: src/pramp/algodaten/Root_of_Number/pseudocode.py ================================================ function root(x, n): if (x == 0): return 0 lowerBound = 0 upperBound = max(1, x) approxRoot = (upperBound + lowerBound) / 2 while (approxRoot - lowerBound >= 0.001): if (power(approxRoot, n) > x): upperBound = approxRoot else if (power(approxRoot, n) < x): lowerBound = approxRoot else break approxRoot = (upperBound + lowerBound) / 2 return approxRoot /*As we can see, the function finds an initial lower and upper bound for the answer, and sets the initial guess for the answer to be the average. Then the iteration begins, and the we notice that the following facts are true in every step of the iteration: (approxRoot - lowerBound) = ½(upperBound - lowerBound) - Since approxRoot is always the average value between upperBound and lowerBound. At the end of every iteration, the real root always satisfies the inequality lowerBound < root < upperBound: this is true since in the beginning of the iteration, we check if approxRoot to the power of n is greater than or lesser than x. The power function is monotonically increasing (i.e. a < b dictates that a^n < b^n) meaning this indicates whether the approximation is too high or too low. The value of (upperBound - lowerBound) is cut by half in every iteration - since in every step we replace one of the values of upperBound or lowerBound by their current average. Obviously it means that eventually the distance between the bounds is lower than 0.001. Since the real root is between the bounds, this promises the algorithm stops. Thus, since lowerBound< root < upperBound, then the true error - |root-approxRoot|, satisfies |root-approxRoot| < (approxRoot - lowerBound) - so it is indeed enough to check when the value on the right side is lower than 0.001. Thus the algorithm always stops, and output of the algorithm is correct. The best way to explain the solution algorithm is by using it on an example. Let’s try calculating the 3rd root of 7: We mark the real root as y ≅ 1.912931182772389. Our method shall iterate in a main loop, and try to produce a sequence of approximations that converge to y, until reaching a number that is close enough. Iteration 1: Initially, we recognized that 0 < y, since 03 = 0 < 7. Also y < 7. Thus lowerBound = 0 and upperBound = 7. Our first member of the sequence is chosen to be the average between 0 and 7 - approxRoot = 3.5. Since 3.53 ≅ 42.8 > y, then we conclude that 0 < y < 3.5, so we conclude upperBound = 3.5. Note that the error is bounded: |y - approxRoot| < |3.5 - 0| = 3.5. Iteration 2: Since we concluded that 0 < y < 3.5, our second member of the sequence should be, again, the average between our bounds 0 and 3.5 - approxRoot= 1.75. This time 1.753 ≅ 5.3 < 7, which means 1.75 < y < 3.5, so we update lowerBound = 1.75. The error is bounded by: |y - approxRoot| < |upperBound - lowerBound|= |3.5- 1.75| = 1.75. We continue the sequence this way, and notice that the error bound is reduced by half by every iteration. Here is a summary of the first few iterations in this method: # Iteration LowerBound UpperBound approxRoot value |y- approxRoot| - True Error Error bound 1 0 7 3.5 ≅ 1.58 < 3.5 (7×2-1) 2 0 3.5 1.75 ≅0.16 < 1.75 (7×2-2) 3 1.75 3.5 2.625 ≅0.71 < 0.875 (7×2-3) 4 1.75 2.625 2.1875 ≅0.27 < 0.4375 (7×2-4) As we can see, by choosing the average value between By the 13th iteration, we’ll reach an error bound of 7×2^(-13), which is less than 0.001 (notice that the error may be much smaller, but since we don’t actually know the real value of y in the process of computation, we cannot promise a better bound). Time Complexity: notice that every loop iteration is done in O(1), under the assumption that the power function takes a constant time. The initial error is x, and the error is multiplied by 0.5 in every iteration. Thus the number of iterations is the minimal k such that: 2^(-k) x<0.001 i.e. 2^(-k)<(0.001 / x) k >log(x) + 3log(10) = O(log(x)) The number of iterations is therefore O(log(x)), meaning the total runtime is O(log(x)) if we refer to the value stored in x, or O(x) if we refer to the number of bits required to represent x (since it takes Log(x) in average bits to represent a number x). Space Complexity: O(1), since we only need a constant number of variables for the algorithm. Mathematical Note: our explanation used the fact that power function x^n is a monotonically increasing continuous function on the positive numbers. Although it is true, the Intermediate value theorem in calculus, states that this method will work for every continuous function - but the proof for this requires more mathematical tools.*/ ================================================ FILE: src/pramp/algodaten/pairs_with_specific_difference/answer.py ================================================ def find_pairs_with_given_difference(arr, k): arr_set = set(arr) ans = [] for i in arr: if k + i in arr_set: ans.append([k+i, i]) return ans arr = [0, -1, -2, 2, 1] k = 1 print(find_pairs_with_given_difference(arr, k)) ================================================ FILE: src/read-Swiss-German.py ================================================ # mer definiere diese funktionen heisst load_words(): das könnte input gsi def load_words(): with open('words_alpha.txt') as word_file: #text swiss german annotation analysis valid_words = set(word_file.read().split()) return valid_words if __name__ == '__main__': #wenn man die code laufen la, wurede diesse funktionen ausgeführt worde. # hier geht das los und startet d' programm. swissgerman_words = load_words() # demo print # fate sueche in swissgerman_words(DB) print('fate' in swissgerman_words) ================================================ FILE: src/srf/comedy/renatoKaiser.txt ================================================ 0:38 wer sollten ich powere mich einmal am Tag aus indem 0:46 ich auf Youtube ein Yogalehrer April 0:51 und auch eine ruhige und entspannt ja dass man einfach nur ausprobieren 0:57 ganz einfach iPhone auf Youtube so bis Igel wir Yoga ganzkörper Flow und dann 1:04 finden Sie in der 320-jährige Betriebswirtschaft Studentin wo im Leben 1:19 da gonja Grenze was darf Satire oder gerade vor so einer linksversiften 1:31 er will immer in die Meister Link gemacht 1:41 und alle anderen wären corona-kunde gut in der Regel 2:05 im Einklang mit der Natur mit dem Körper mit den Seelen mit dem Universum 2:11 warum 2:18 t [Musik] 2:29 wie groß ist vertraue Kraft für die Gedanken wirklich wenn sogar schwerkraftsfest weh tut dann fühl ich 2:38 aber ich weiß der harte Worte von der Realität ist für alle darum glaube ich habe Yoga schnurren Sport für juppis und 2:46 für Snaps für Richi Welt Arbeiterklasse hat kein Parkett 3:02 aber mit dem würde ich mich nicht zu der Arbeiterklasse 3:26 mögliche Positionen dermaßen 3:32 nicht spontan in den hysterische lachhüllekrampf ausbrechen wenn auf 3:38 Youtube irgend so eine Jessica aus Berlin Brandenburg fahren eine Baggersee 3:44 im schniderspraktikantinnenversion von 3:49 Dalai Lama 3:57 schenk dir diesen Moment 4:07 schenk dir diesen Moment tut mir leid Jessica 4:14 wenn du so Sachen sagst den kann ich mich nicht konzentrieren 4:21 dann wird jedes Jahr zum Lachen und bitte verstehen 4:42 einen Moment schenkt oder den stelle ich mir natürlich sehr viel Fragen 4:49 die große Frage vom Leben und die drei große Frage sozusagen 5:00 Mensch eigentlich einzigartig dann Antwort ist einfach 5:08 fleischvogel 5:13 ich muss das vielleicht kurz erklären dann wurde ich einfach sprach also 5:19 Menschen zu überlegene Wässer und ich weiß dass den komisch aus dem Müll vom 5:26 aber trotzdem jede Sprache macht uns Menschen zu überlegen was er will durch Evolution oder oder durch durchsprochen 5:32 durch Kommunikation und Organisation immer so geschafft alle andere Tiere auf 5:37 der Welt wo größer stärker und schneller sind das mir überlisten und unterwerfen mir in so zeige Krone das macht es aber 5:46 noch nicht einzigartig dass mir kein Handy macht einzigartig dass mir so schlecht die Gewinner sind will 5:55 müssen komplett über ergibt sich Moment immer übertrieben wenn es lange nicht 6:02 dass man alle andere Tiere auf der Welt unterworfen und ruft gestern nein was 6:08 mache mir mir züchtet sie nach und machen 6:18 wie beleidigend kann man gegenüber Sinne Mitwesen sie als indem man fürchte sind 6:25 vom absurdesten überhaupt immer so nach halbespot Wurst in freier Natur 6:34 also wenn ein Löwe wird in freier Wildbahn und weshalb treffen denn würde 6:41 das Töten und esse Witze gemeint 6:47 wenn ein Mensch Ufer den tötet 7:20 beste Zeichen für Einzigartigkeit von Mensch das beste Zeichen ist 7:26 der fleischvogel Deutschland zeigt man ja Rindsroulade 7:31 aber der Typ Schweizer Begriff fleischvogel 7:37 ich so viel besser was ist ein fleischvogel man nehme es Rinds Plätzli 7:43 Hals breit und Hinterschinken also drei verschiedene dir 7:50 e noch der Name vom nettia 7:58 Vogel und dazu Fleisch 8:04 damit sie wirklich vegan 8:12 wenn man im Vergleich Vogel ist so einen ganz normaler Vogel fast schon vegan 8:21 aber das lange dies Menschen 8:38 ein Vogel zu spät sie wird Bremer Stadtmusikanten 8:59 ich mache Sex 9:08 machen würde loswerden 9:13 ist 9:22 wenn man gerade im Thema sind da ist doch mal übers Wort Sextäter 9:33 das ist scheiße 9:46 sind Zitat sechs Täter auf der Flucht deutscher sextäterhauptverdächtiger 9:52 Polizei Aargau sucht Sextäter Mario 10:00 wie so eine degenerierten Nintendo Spiele Super Mario 6 hat auf 64 10:08 und warum tönt das so komisch wenn es Wort falsch ist das heißt nicht 10:16 Mario das heißt Steven 10:24 das Wort wirklich 10:34 was macht das mit mir was macht das aus mir weil ich mein ich Bier auch 10:41 in Sex Täter also ich tue Sex 10:52 also ich säge mami stolz 11:00 stolz und ich meine besonders 11:15 det noch gar nicht so viel wahrscheinlich kein 11:21 Serie Täter aber ja Sextäter oder auch Sex grüßen 11:29 die schaust du das Wort aus dem Blick 11:37 Sex Grüssen, okay 11:47 aber lange ich würklich. 11:56 ein Christel ja früher in die Schuel, meine Lehrerin gesagt 12:20 Was ufem Body ghört mir. 12:57 Schuldigung 13:06 gseigt klassische 13:16 Welt 13:26 Verschwörungstheorie ja ich hoffe dass es Verhalten und hygien Verständnis von 13:31 einer Impfgegner 13:44 und jetzt lahme ich dir Magerine vor der Türschwellen abschlecken 13:50 es ist überlebe ich die Masse nicht nein ich weiß natürlich Hygiene ist wichtig für Immunsystem gerade auch im 13:57 Messe darum glaube ich aber nicht das Corona ausbrachen ist gestern hat ich 14:02 glaube corona-schuss gestern 14:17 vom Wort Herr nimmt anders als in Brüssel offen Gebiet sechs sechs und wir 14:23 sind doch Ali es bizli Grüße 14:35 [Musik] Fifty Shades of christm 14:42 a etwas Problem immer noch ist Wort Sex so 14:49 verrückt so moralisch falsch das sind Journalist wenn es einfach nur Vergewaltigung Gott findet das landet 14:54 wenn er schreibt oh ein Sextape da hat Sex getätigt das ist ein Fall für die 15:00 sechs Polizei und glaub mir ich bin überhaupt nicht erwachsen wenn es darum 15:05 geht ganz im Gegenteil ich sage zum Beispiel mega gern das Wort 15:29 macht auf Instagram 15:44 wahrscheinlich gemeint ist 600 oder was wirklich irgendwie Empfang 15:51 suchen 16:00 Angst wurde ihres Schlafzimmern 16:12 und dieser und warum 16:20 lustige Geschichte 16:28 wissen Sie ich bin Satiriker und sie so 16:36 sind sie nicht der Kaiser sind doch das Salzburger Stier 16:42 ja voll also 16:49 mit so süß 16:57 oder mir alle hinter mir mit deiner Wörter mit den Sex Wörter fahren mit den sechs werben mit der Sex tunwörter 17:05 sozusagen 17:26 Liebe machen 17:32 Liebe machen also du hast doch gerade einiges vor Knorr 17:41 wir machen Liebe mir produzieren möchte Liebe 17:50 stärkste Gefühl von der Welt diese Verbundenheit Seele verwandtschaftliche 18:07 [Musik] 18:20 ich habe keine Angst schlimm oder 18:26 Sex machen gemeinsamer komplizit 18:35 aber nenne den Vergewaltigung 18:40 Vergewaltigung ich mir Sex Vergewaltigung Vergewaltigung ist schon Vergewaltigung 18:50 ohne zu verharmlosen [Applaus] 18:59 [Musik] 19:20 wir haben hier so als Hobby er macht im Haupt der Welt 19:26 durch dich so gut du merkst 19:33 du Depp sucht dich so gern der Besuch dich sogar wenn du nicht 19:42 und wie oft hämisch wünscht 19:56 freiwillige Sterbehelfer 20:02 hey wenn Mord ist egal Vergewaltigung ich illegal Sex ich nicht 20:11 illegal aber den nicht [Applaus] 20:28 mich jetzt auch nicht so fest dass mir es unentspannt Verhältnis zum Thema Sex 20:34 hin dass du nur schon weggesprochen weil mehrere Wort für 20:41 Orgasmus also nicht Orgasmus also grundsätzlich 20:53 Drogen nein danke Orgasmus jetzt please aber 20:59 orgasmusischer wissenschaftliche begriffen 21:31 zeige mal Liebe machen und am große Höhepunkt [Musik] 21:38 t 21:55 und bescheuert und du 22:07 mich mit [Applaus] 22:20 kein enthaltige das Maul 22:27 ich meine zwischen Orgasmus 22:37 der kleine Tod 22:43 poetisch dramatisch französisch 22:48 aber mir in der Sicherheit 23:01 nehmen 23:46 [Applaus] 23:53 so viel über das Thema tut nothingt muss ganz ehrlich sagen 23:58 relativ jung ich ernähre mich relativ gesund und gefährlichste 24:07 ich mich einfach nur den 24:14 und vergesse dass ihr am Klickpedal leicht gemacht 24:19 bin 24:25 Pedal fixiert damit du deine ganze Kraft hast du dir mega professionell 24:35 bis zu dem Moment wo es Glück Pedal vergessen 24:58 sie stillig ja und damit meine ich nicht 25:05 auf ihren Alter wer wirklich 25:19 gemacht ich halte mich nur dran 25:25 reden wir noch fahren ist entscheidend 25:51 farbige Spitze zu eng vakuumierter 25:57 und ja das wissen wir schon nachher aber das ist nicht das wo man während dem 26:03 Reden unendliche Freiheit in der Endlichkeit 26:11 vom Velo dress und das fühlt sich aber so geiler und in 26:19 meinem Raum 26:24 ich muss ganz ehrlich sagen ich habe mich noch gespürt wie wenn ich so oben 26:30 ohne im innere knallrotes Hause mit zur Schulterträger bin ein sehr unmuskulöser 26:37 Wrestler warum Spiegel gestorben finde 27:29 wurde wie ein gigantische Oberschenkel 27:34 und wenn deine Mutter am Straßenrand ihres Kindern 27:40 seitdem sie immer irgendwen wenn groß bist du immer nein 27:53 und Zeit zu dem Kind jetzt muss Schluss 28:02 das wird lustig wenn sie weiß wie so das Krieg Bedarf funktioniert wie gesagt zu mir 28:09 offensichtlich will ich fahren nichts ahnend da die einen plane wird immer langsamer langsamer langsam und den 28:18 passiert also es passiert der Welt es macht wirklich 28:30 und den Krippe ich so ganz langsam um 28:41 und der Person 28:47 machen 29:03 und süße dass du nötige kaschmache das war schon philosophisch das macht 29:10 dich demütig das heißt du ergischst dich zu dem Schicksal und gibst so ganz 29:17 langsam um so zu dem Musik vor Enya 29:22 gucken [Musik] 29:34 [Musik] alles und dumme wird unwichtig 29:42 du spürst 29:47 konzentrierst dich auf und schenkst dort der Moment 30:09 ich mach neuerdings einfach cool 30:20 der Youtube Yogalehrerin oxited schenk dir diesen Moment und hier 30:29 haben wir wirklich viel Moment geschenkt da haben wir viel Gedanken gemacht viel Fragen stellt und dir fragen Sie was 30:36 soll das denn überhaupt heiße schenk dir diesen Moment was ist das überhaupt für 30:41 das Geschenk ein Moment schenken 30:48 das wünsche ich dir deine schlimmste finden 30:59 Moment wo man sich selber schenkt das beste überhaupt nicht sitzen oder so mit 31:05 sich alleine 31:13 die Geburt von meiner ersten Tochter Sekte schönste Moment vom See aber jetzt 31:19 und nicht da mit mir alleine 31:30 momentan nur 31:40 ein Moment und was kommt noch genau 31:46 du willst einen Moment und nochmal eine und normalerweise und normalen und was 31:51 passiert richtig du wirst süchtig nach Moment und perfide da ist wenn du dir die 31:59 Moment selber schenkst bist du der Konsument unter Dealer Klick 32:05 und sie schauen nur gratis du wirst zum Perpetuum Mobile 32:33 Mittagspause Vaterschaftsurlaub Kurzzeit Yoga schadet 32:40 der Wirtschaft ja will wenn sie und Schnuffel schon 32:49 ganz ehrlich so ein Geschenk der Moment ist nun dann das als die Sticks Droge 32:55 ist früher Pension ja 33:01 und mir ist das passiert ja das Yoga Spiele direkt 33:08 und ja stört 33:14 du ich schenk mir nur noch Moment 33:22 zum Beispiel ist ein einziger verschenkte Moment und 33:29 ich nur Gabe 33:37 für den Schweizer du willst ganz kurz mit alles sieht zum Türchen Moment 33:48 sitze und Schnuffel ist meine Stärke 33:53 ich bin nicht der Denker ich bin der Sitzer der Atmung das Monument des 34:01 Moments die Statue des Status 34:11 und ihr Frau geht da jetzt vielleicht aber Moment 34:17 der sitzt ja gar nicht und 34:23 wenn man 34:31 zum die frohe Botschaft verkünden und ja ihr Hände 34:38 da aber ihr 34:44 ihr sitzt und schnuffed und wenn er für das Schuh ================================================ FILE: src/srf/dok/Kampf gegen die Suchttxt ================================================ Kampf gegen die Sucht – Drogensucht, Spielsucht, Sportsuch * Sanfte schwermütige Klänge * Reto nimmt seit seinem 16. Lebensjahr Drogen. 0:07 Seit Jahren will er davon loskommen. Einfach immer im Hinterkopf, wenn ich heimkomme, etwas reinzupfeifen. 0:16 Samira hat viele Fitnesswettkämpfe gewonnen. Der Erfolg hat aber seinen Preis: Sie hat eine Fitnesssucht, 0:23 schwere Essstörungen und ist unzufrieden mit sich selber. Zu dick. Ich weiss, ich kann anders ussehen. Drum gefällt es mir nicht. 0:33 Cahit verspielt seit Jahren sein ganzes Geld. Verloren. 0:38 Er hat mit Schulden zu kämpfen und ist allein. Sie alle wollen eine Veränderung. 0:44 Weg von der Sucht, rein in ein neues Läbä. 0:51 * Einzelne eindringliche Streicherklänge * Hier in diesem Block wohnt Reto. 0:58 Ein Bier am Morgen ist nicht sein grösstes Problem. Sondern Heroin, Kokain oder Benzodiazepin. 1:05 Es gibt kaum eine Droge, die er nicht schon genommen hat. Seit 25 Jahren habe ich Drogenprobleme. 1:11 Während der ersten zehn Jahre wollte ich gar nichts ändern. Natürlich möchte ich heute etwas ändern, aber wenn du ... 1:18 ... von deinem Leben ... Ich bin jetzt 41. Von meinem Leben ... 1:25 Mit 16 habe ich angefangen zu spritzen, hatte vorher schon mit Drogen zu tun. 1:31 Es ist wahnsinnig, das Leben einfach zu ändern. Oder aber ich weiss nicht, wie es richtig geht. 1:39 Seit seinem letzten Entzug vor einigen Monaten nimmt Reto keine harten Drogen mehr. 1:45 Als gelernter Metallbauer sucht er einen Job. Er möchte eine Struktur in seinem Leben 1:50 und nicht immer alleine zu Hause sein. Aber mit seiner Vergangenheit ist das schwierig. 1:56 Was soll ich denn machen, wenn ich nur Absagen bekomme und ... 2:01 Bei jeder Absage, die du bekommst, bist du auch irgendwie ... ... wirst du auch irgendwie ... 2:09 ... demotiviert. Du fragst dich: "Wozu bewirbst du dich überhaupt?" 2:14 Und dann ... (Seufzend) Ja ... 2:19 * Einzelne Streicherklänge * 2:26 Ohne Arbeit und Tagesstruktur ver- bringt Reto fast jeden Tag alleine. 2:31 Und mit jedem Tag, der verstreicht, merkt er, wie das Verlangen grösser wird, wieder Drogen zu konsumieren. 2:38 So. Es kommt wie eine Welle. 2:45 Es baut sich auf, bis ein riesiger Druck da ist. 2:50 Was könnte diesen Druck lösen? Das weiss ich auch nicht. 2:56 Sonst würde ich es machen. Ich habe schon viel probiert. 3:04 Regelmässig geht Reto in der Klinik Selhofen in Burgdorf in Therapie. 3:09 Ein Weg, den er gut kennt. (Lachend) Mein zweites Zuhause. Nein. 3:20 Du warst auch schon hier? Und nicht nur zur Therapie. Hier? Zehnmal war ich stationär hier. 3:28 Und Reto spürt, dass er bald wieder abstürzen könnte. 3:35 Welche Gedanken haben Sie sich gemacht? Ob ich nochmals irgendwie drei oder vier Wochen hierher kommen soll. 3:43 Okay, erneut über das Stationäre, worüber wir ... - Ja. ... auch schon gesprochen haben. Ja. 3:50 Aber es fehlt irgendwie noch der letzte Dings, der letzte ... 3:55 Ich bin noch nicht 100 ... Ich mache mir Gedanken darüber, habe mich aber noch nicht entschieden. 4:01 Was wäre denn der Nutzen? Was soll ein Aufenthalt bringen? Es wäre wie ein Restart für mich. 4:10 Ich habe das Gefühl, dass ich früher oder später einen Job finde. Davon bin ich überzeugt. 4:16 Stationär zu kommen wäre eine Art Beschäftigung, bis Sie dann vielleicht einen Job bekommen? 4:25 Ein bisschen. - Ja. Wenn Sie hierher kommen und dann wieder rausgehen 4:32 und trotzdem keine Arbeit bekommen, könnte es dann wieder ins Gleiche gehen? 4:42 Ja, natürlich. Wenn er sagen würde: "Ich muss wirklich nochmals inten- siver an meiner Sucht arbeiten." 4:51 "Ich will wirklich davon wegkommen." Oder: "Ich muss mir über den Anschluss Gedanken machen." 4:57 "Ich muss irgendetwas verändern, auch wenn ich keinen Job habe." "Aber es muss etwas anders sein für danach." 5:03 Denn wäre es für mich ganz anders, als wenn er sagt: "Ich will v.a. einen Job haben." 5:09 "Wenn ich einen Job habe, ist alles gut, und ich komme mal in der Hoffnung, schneller einen Job zu bekommen." 5:18 Es gäbe Beschäftigungsprogramme für Menschen in Retos Situation. Schön, oder? 5:25 Er möchte aber einen richtigen Job und so von den Drogen wegkommen. Einfach immer im Hinterkopf, wenn ich heimkomme, etwas reinzupfeifen, 5:33 das ist der Scheiss. Ich drehe mich im Kreis, seit Jahren. 5:39 Selhofen, dann kommst du wieder raus, reisst dich zusammen, dann stürzt du trotzdem wieder ab, zwischendurch gibt es mal einen Job, 5:48 dann hast du einen Unfall. Die nächsten Tage werden entscheidend sein. 5:55 Entweder ändert sich noch etwas oder es droht der nächste Absturz. 6:05 Das Gym ist ihre Welt. Die 25-jährige Samira ist sechsmal pro Woche da 6:10 für ca. 2 Std. Nicht ins Fitness zu gehen ist für sie keine Option. 6:20 Für mich ist es schwierig, einfach mal zu sagen: "Hey, heute bleibe ich besser zu Hause, weil es mir nicht gut geht 6:27 oder weil ich körperlich nicht in guter Verfassung bin." Dann ... ... weiss ich im Vornherein, dass mein Tag grösstenteils gelaufen ist. 6:37 Also, mir ist das Training schon sehr wichtig. 6:47 Es geht um Disziplin, um Kontrolle. Manchmal auch um Zwang. 6:54 Samira will ihren Körper beherrschen. Und sie ist streng mit sich. 7:06 Auf Instagram hat sie über 20'000 Follower. Kein Wunder: Von ihrem Körper könnten viele Frauen nur träumen. 7:15 Und trotzdem ist sie nicht zufrieden mit sich. Was denkst du, wenn du dich im Spiegel siehst? 7:22 Zu dick. 7:30 Weil ich mal Wettkämpfe gemacht habe, ist das Bild anders. Ich weiss, ich kann anders aussehen. 7:36 Jetzt bin ich halt ... ... gut 15-20 kg schwerer, ja, 15. 7:43 Und ... ... dann gefällt es mir halt nicht. Spitzensport gehört schon lange zu Samiras Leben. 7:51 Bereits als Achtjährige geht sie ins Kunstturnen, trainiert viermal pro Woche. Als Teenager beginnt sie mit Fitness. 7:59 Mit Erfolg: Mit 22 ist Samira Miss Bikini, Schweizer Meisterin im Fitness, 8:05 und holt einen vierten Platz an den Weltmeisterschaften. Samira hat ihren Körper im Griff, isst strikt nach Plan. 8:12 Nach den Wettkämpfen kippt es ins Gegenteil. Sie hat regelmässig Fressattacken und nimmt stark zu. 8:19 Ein Teufelskreis. 8:24 Heute leidet Samira an einer schweren Essstörung. Sie wiegt alles und kontrolliert krankhaft, 8:31 was sie isst und was nicht. Trotz oder gar wegen der strengen Ernährung 8:39 hat sie immer wieder Essattacken. Nachts. Wenn sie alleine ist, verliert sie die Kontrolle. 8:51 Bei mir ist es wirklich so extrem, dass ich dann ... ... nicht mehr aufhören kann mit essen. 8:58 Ich holte mir auch schon nachts an der Tankstelle oder am Selecta-Automaten mehr Essen. 9:06 Ähm, ja ... Das ist natürlich wie ein Rausch. 9:12 Man muss sich das vorstellen, wie wenn man andere Drogen konsumiert - 9:17 gut, das weiss ich ehrlich gesagt nicht. Aber ich stelle es mir ähnlich vor, weil man halt ... 9:23 ... immer noch mehr und noch mehr will und es nicht aufhört. 9:28 Samira besucht regelmässig einen Therapeuten, der helfen soll. Seit einigen Monaten ist sie aber 100 % krankgeschrieben. 9:36 Deshalb hat sie ihre Stelle als Hochbauzeichnerin verloren. Es hat überall, in sehr vielen Bereichen meines Lebens 9:44 zu bröckeln begonnen. Es waren nicht nur die Ernährung und der Sport. 9:49 Es gab auch Berufliches und Privates, das zu Bruch ging, was mich aus der Bahn warf. 9:56 Mein psychischer Zustand hat sich dann halt ... ... extrem verschlechtert. 10:02 Das hat sich durch Fressattacken und Schlafstörungen geäussert. Und durch Übertraining, dadurch, dass ich nicht mehr schlafen konnte. 10:11 Mir war das dann alles zu viel. Mit diesem Formular will Samira ihr Leben jetzt von Grund auf ändern. 10:19 In einigen Tagen geht sie für mehrere Monate in eine Klinik. 10:26 Ah, voila. So wäre es super. Ab und zu ruft Reto via Internettelefon an 10:32 und erzählt, wie es ihm geht. Man gibt sich Mühe, oder. 10:39 Aber es gibt immer Phasen. Manchmal geht es besser, manchmal weniger gut. 10:46 Schon? Aber jetzt geht es gerade gut? Ja, jetzt habe ich gerade eine gute Phase. 10:54 Ich habe irgendwie auch nicht so Lust auf Drogen. Ich will wieder mehr Sport machen. Jeden Tag ein bisschen. 11:02 Ins Fitness, sozusagen? Ah, jetzt ist gut, du. Jetzt ist es super. 11:09 Jetzt ist es etwas weniger nervös. Ich habe gerade gesagt, dass ich versuchen will, 11:16 diese Woche möglichst keinen Alkohol mehr zu trinken. Nicht mehr saufen und so. 11:22 Es geht gut, ja. Es geht mir fast besser. Gestern und heute habe ich nicht getrunken. 11:30 Einige Tage später. Die Anrufe von Reto sind ausgeblieben. Trotz der guten Vorsätze hat er den Kampf gegen die Drogen 11:38 einmal mehr verloren. Ja, leider wieder ... 11:43 ... eben, Absturz auf die Benzos gehabt und ... 11:50 Sobald ich merke, dass es aus dem Ruder läuft, 11:56 dass ich es nicht mehr unter Kontrolle habe, wann ich eins nehme und wann nicht, 12:01 fühle ich mich unter Druck gesetzt. Und das macht mich fertig. 12:07 Ich will es wegputzen. Ich will diesen Scheiss nicht mehr. Reto hat sofort reagiert 12:14 und sich für einen stationären Aufenthalt in der Klinik angemeldet. Jetzt muss er warten, bis er eintreten kann. 12:22 Ja, es ist verrückt. Das ist jetzt das elfte Mal. 12:30 Es ist mir selber peinlich. Ich habe jeweils diejenigen ausgelacht - oder nicht ausgelacht, aber gedacht: 12:37 "Wow, der war schon so oft im Entzug?" Als ich selber erst zwei- oder dreimal war. 12:51 Das sind diese Dormicum. Dormicum ist ein verschreibungspflichtiges Medikament, 12:59 das z.B. gegen Depressionen oder zur Einleitung von Narkosen eingesetzt wird. 13:05 Ein stattlicher Haufen. Der Wirkstoff Benzodiazepin macht schläfrig, entspannt - 13:12 und schnell süchtig. Das Problem ist: Wenn ich es nicht nehme, beginne ich zu zittern. 13:18 Und darauf habe ich keine Lust. Normalerweise nimmt man das Dormicum als Tablette. 13:25 Gemahlen durch die Nase wirkt es aber schneller und heftiger. 13:32 Ah. 13:40 So. Spürst du jetzt sofort etwas? Es brennt einfach in den Augen, oder. Ja. 13:49 Aber jetzt nicht, dass ich direkt ... ... einhängen würde oder so. 13:58 Wobei ich finde, dass auch die Qua- lität der Tabletten abgenommen hat. 14:03 Mit der Dosis, die Reto nimmt, würden die meisten Leute flachliegen. Er kann weiterhin einigermassen funktionieren. 14:11 Da rein. Reto will Pilze fürs Abendessen sammeln. 14:18 Schau, da sind schon welche. Ich kenne sie nicht. Ich esse nichts, was ich nicht esse. 14:26 Mir würden zwei oder drei schöne Steinpilze schon reichen, weisst du. 14:32 Hast du gesehen? Plötzlich wimmelt es davon. Maronen .... Maronenröhrling. 14:39 Schon wieder einer. Das ist ein Rotfüsschen. 14:44 Die schlechten Teile weg und dann los - ab in die Pfanne damit. Wenn ich mal irgendetwas angefangen habe, 14:54 habe ich Mühe mit aufhören - egal mit was, fällt mir auf. 15:00 Ob mit saufen, Pilze sammeln, mit Sport, mit ... 15:06 Ähm ... Drogen und was weiss ich. Ich höre einfach nicht mehr auf. 15:13 Das ist das Problem. Was bedeutet dir die Natur? 15:20 Die Natur ist ein Kraftpunkt für mich. Ich war mit meinem Vater oft in der Natur. 15:30 Er hat mich viel über die Natur gelehrt. Irgendwie, wenn ich in der Natur bin, habe ich manchmal das Gefühl, 15:37 dass ich mit ihm zusammen bin, oder so. Wie wichtig war er? 15:43 Er war mein Vater. "Wie wichtig war er?" 15:49 Er war für mich zu der Zeit die wichtigste Person. 15:58 Als Reto elf Jahre alt war, erkrankte sein Vater an Lungenkrebs und starb kurze Zeit später. 16:06 Du sagst immer, das sei ein Wende- punkt gewesen, als er starb. Oder? 16:11 Ja. Wenn dir mit elf Jahren das Liebste entrissen wird 16:17 und du es nie mehr wieder siehst. 16:22 Da möchte ich andere mal sehen. Ich will nicht immer über meinen Vater sprechen. 16:30 Der ist gestorben. * Sanfte gläserne Klänge * 16:40 Es dauert noch ca. drei Wochen, bis Reto endlich in die Klinik eintreten kann. 16:45 Bis dahin muss er alleine mit seiner Sucht irgendwie seinen Alltag bewältigen. 16:54 * Sanfte gläserne Klänge * 17:14 Das ist Cahit. Er ist 46-jährig und türkisch- schweizerischer Doppelbürger. 17:20 Cahit wohnt in der Stadt Zürich in einem Einzimmerstudio. Er ist oft alleine. 17:26 Auch Eltern und Geschwister besuchen ihn nur selten. Der Grund: Cahit ist spielsüchtig 17:31 und hat das Vertrauen seiner Verwandten immer wieder missbraucht. 17:43 Manchmal ... ... habe ich von meinem Bruder oder von Kollegen Geld ausgeliehen 17:50 und habe dann mit diesem Geld gespielt. Und jetzt haben sie ... 17:57 ... irgendwie kein Vertrauen in einen Zocker. 18:03 Weil du ihr Geld nicht mehr zurückbezahlt hast? Genau. 18:09 Seit 15 Jahren ist er spielsüchtig, arbeitslos und einsam. 18:14 Seine Sucht hat ihm alles genommen. Finanziell ist er von der Sozialhilfe abhängig - 18:20 und von Geld von den Eltern. Er kämpft mit Depressionen. Trotz seiner Situation spielt er immer wieder. 18:27 Weil er unterdessen in allen Casinos gesperrt ist, v.a. mit Sportwetten via Handy. 18:36 Wie viel hast du draufgeladen? 10 Fr. 18:42 Das ist ja nicht so viel. - Ja. Bei Sportwetten ... 18:48 ... setze ich weniger als ... Ich sage jetzt mal 10 Fr. 18:54 Alle zwei bis drei Tage. Einer davon ist Galatasaray gegen Fenerbahce. 19:01 Ich bin auch Fenerbahce-Fan. Die sind aus Istanbul. 2:2. 19:07 Ich habe auf zwei für Fenerbahce ge- tippt, jetzt ist 2:2 unentschieden. Das heisst? 19:13 Verloren. Ja. 19:22 Früher spielte er in echten Casinos. Hat auch gewonnen - und das nicht zu wenig. 19:32 Das Geld habe ich auch alles verspielt. Ein paar Hunderttausend? 19:37 Ja. Schon, ja. Das ist viel. - Ja. 19:43 Das ist viel, ja. Regelmässig ist Cahit in der Langstrasse in Zürich im Ausgang. 19:51 Es ist wie ein Zuhause. Aber hier kann man auch überall Geld verspielen. 19:58 3'000-4'000 Fr., die man am einen Ort gewinnt, 20:04 ein paar Tausend Fr. Dann gehst du zum nächsten und verlierst einfach alles wieder. 20:12 An jeder Ecke ein Kiosk, an dem man legale und auch illegale Online-Games spielen kann. 20:18 Mit der Kamera reingehen will Cahit zu seinem eigenen Schutz nirgends. Wann hast du gemerkt, dass du ein Problem hast? 20:28 Vor Kurzem habe ich von meinem Bruder Geld verlangt. 20:33 Ich habe gesagt: "Ich brauche Geld." Er hat mir seine Kreditkarte gegeben. Die ganze Nacht, bis am Morgen ... 20:41 Ich ging 4'000 Fr. ins Minus auf der Kreditkarte meines Bruders. 20:47 Dann habe ich gesagt: "Ich stecke wirklich ... 20:52 wirklich tief in dieser Sucht." Seit diesem Erlebnis will Cahit sein Leben wieder in den Griff bekommen. 21:00 Aber zu Hause in Zürich lauert die Verlockung überall. Und trotzdem: Weg von hier will er nicht. 21:08 Ja. Ja, dann muss ich die Stadt verlassen, ja. 21:13 Ich bin ... ... halt hier aufgewachsen. 21:19 Irgendwie kann ich das auch nicht. Der Kollegenkreis und v.a. die Familie leben ja in dieser Stadt. 21:26 Also ... * Salsaklänge * 21:40 Samira bricht heute auf in die Klinik. 21:45 Wie geht es dir heute Morgen? Wieder etwas besser 21:52 bzw. war es die ganze letzte Woche für mich ziemlich schwierig, mich mit dem Ganzen abzufinden, quasi. 22:00 Aber, ähm ... Ja, ich habe jetzt einfach versucht ... 22:07 ... mir das Ganze schönzureden, quasi. 22:15 Also, ich erhoffe oder wünsche mir, dass ich in dieser Zeit eigentlich wieder zu mir selber finden kann 22:24 und wieder lerne, das Leben zu geniessen, ohne mich irgendwie gezwungen zu fühlen. 22:42 Zwei bis drei Monate will sie in der Klinik bleiben. Sie geht v.a. wegen ihrer Essstörung in Behandlung. 22:49 Trotzdem nimmt sie ihr eigenes Essen mit. * Sanfte dumpfe Klänge * 23:01 Die Klinik Barmelweid in Erlinsbach im Kanton Aargau. Eine Spezialklinik für psychosoma- tische Medizin und Psychotherapie. 23:10 Während der nächsten Monate Samiras neues Zuhause. Ich mache kurz die Tür zu. 23:26 Das wäre Ihr Zimmer. Hier haben wir eine Glocke. 23:32 Wenn Sie etwas von uns brauchen, können Sie sich damit melden. Okay. 23:37 Dann haben Sie ein eigenes Bad. - Okay. Das wäre dort vorne. 23:44 Ah ja, okay. Super. - Ja. 23:50 Noch die Schuhe ausziehen zum Wägen. Auf fünf oder ...? - Ja, ja, vier. 23:55 Abrunden auf vier. - Vier, fünf. Sind Sie 1.63 m? - Ja. 24:01 Gut. Essen muss Samira von nun an auf der Station. Sie wählt täglich zwischen drei Menüs. 24:08 ... was bestellt worden ist. Wir haben mal das Standardessen angewählt. 24:13 Sie können schauen, ob das für Sie passt. Dann esse ich einfach, was mir schmeckt? Ja. 24:19 Schweinefleisch mag ich z.B. nicht. - Mögen Sie nicht, okay. Dann lassen Sie das heute ausnahmsweise mal aus 24:26 und wir werden vermerken, dass Sie kein Schweinefleisch essen. Gut. Das Brot können Sie direkt löschen. 24:33 * Lachen * - Okay, gut. Salat ist gut, aber das Brot muss nicht sein. 24:40 Das Essverhalten von Samira soll sich ändern. Mit ihren Therapeutinnen schaut sie ihre Situation an. 24:46 Es freut mich, dass Sie sich zu einem stationären Aufenthalt entschieden haben. Können Sie das Problem benennen? 24:53 Was war der Grund für den Klinik- eintritt? Aus Ihrer Sicht. Ich habe gemerkt, dass es für mich nicht mehr normal ist 25:02 bzw. ich den Alltag und das Leben nicht mehr so geniessen kann, wie ich das mal konnte. 25:08 Ich habe auch gemerkt, dass ich meinen Selbstwert extrem vom Training abhängig gemacht habe. 25:15 Dass ich mich schlecht gefühlt habe, wenn ich nicht trainiert habe. Es war ein Teufelskreis mit Schlafstörungen, Essen und Training. 25:24 Genau. Wer schlecht schläft, kann nicht richtig trainieren. Dann habe ich wieder gegessen, habe mich wieder schlecht gefühlt. 25:31 Habe wieder das Gefühl gehabt, ich müsse ins Training. Ich geriet immer tiefer rein. 25:37 Ich hatte das Gefühl: "Das kommt wieder, das legt sich wieder." Irgendwann, als ich noch 100 % arbeitete, 25:44 ist das einfach alles irgendwann ... Es begann überall immer mehr und mehr zu bröckeln. 25:51 Und ... Ja. Deshalb. Sie beschreiben das sehr eindrücklich. 25:57 Sie ist sehr ehrgeizig, sehr diszipliniert ... ... und auch sehr leistungsfähig. 26:04 Auch wenn es momentan nicht mehr so ist. Das ist ein bisschen der Klassiker: 26:09 Man kann sagen, dass generell bei Spitzensportlern sehr schnell - 26:15 oder nicht sehr schnell, aber dass bei den psychischen Störungen die Essstörung an erster Stelle steht. 26:22 Meistens sieht man das bei diesen Sportlern nicht wirklich, weil sie natürlich einen athletischen Körperbau haben. 26:29 Eine Essstörung entwickelt sich dann kontinuierlich. Am Schluss hat man dann diese Essattacken. 26:37 Gibt es etwas, das für Sie nicht passieren darf während des stationären Aufenthalts? Für mich wäre ganz schlimm, wenn ich gar keinen Sport mehr machen könnte. 26:47 Aber ich möchte nicht mehr, dass es so wichtig ist, dass ich mich davon abhängig mache. - Ja. 26:52 Aber trotzdem wäre es gerade zum jetzigen Zeitpunkt schlimm, wenn ich hier rauskäme und ... 26:59 ... und 5-10 kg schwerer wäre oder ... ... vielleicht auch 10 kg leichter und ich sehe, 27:06 dass meine ganze Muskelmasse weg ist. Das wäre das Einzige. Aber sonst habe ich eigentlich nichts zu verlieren. 27:14 Heute darf Samira noch ihr eigenes Mittagessen essen. Aber morgen ist damit Schluss. 27:22 * Dumpfe Klänge * 27:28 * Vogelgezwitscher * Auch Cahit will sein Leben endlich in den Griff bekommen. 27:36 Er besucht eine Therpaie. * Ruhige Gitarrenklänge * 27:44 In den letzten 15 Jahren habe ich viel zu viel verloren. Nicht nur Geld. 27:50 Auch psychisch geht es mir schlecht, habe immer Depressionen, ich bin nicht mehr in meinem Kollegenkreis, 27:57 ich stehe meiner Familie nicht mehr nahe. Machen Sie jetzt noch Schulden damit? 28:04 Nein, Schulden mache ich nicht. Nein, nein. Es ist wichtig, dass jetzt der Teil der Therapie kommt, 28:12 in dem wir wirklich konkret die Tagesstruktur, konkret die Partnersuche, 28:19 konkret die Arbeitssuche, all das besprechen. 28:24 Sie sind motiviert, da rauszukommen, ja? Ja. Und ... 28:30 Es ist nicht zu spät, um rauszukommen. - Mhm. Also: 28:37 Dann beginnen wir mit der linken Seite. Den linken Arm anspannen. 28:43 Nichts forcieren. Jetzt können Sie entspannen. Eine Entspannungsübung ist der erste Schritt. 28:49 * Sanfte, beruhigende Klänge * 28:54 Mit frischem Wind will Cahit jetzt vorwärtsmachen. Erster Schritt: Jobsuche. 29:00 Er arbeitet zwar ab und zu als Pizzakurier, mit Hilfe der Sozialdienste will er jetzt aber sein Dossier aufpeppen. 29:08 Was musst du dort machen? Einen Job suchen. Alles, Lebenslauf, 29:14 Arbeitszeugnisse zusammenstellen. Dann Bewerbungen schreiben - vielleicht dann. 29:21 * Sanfte Gitarrenklänge * Ich, äh, ja. 29:26 Da ist der Lebenslauf drauf. Das ganze Leben ist da drauf. 29:38 Ohne Job ist es mir im Moment verdammt langweilig. Bewerbungen schreiben, ein Job, das ist sehr wichtig, ja. 29:47 Willst du wirklich arbeiten? Hast du Lust? Ich habe Lust, ja, doch. 29:53 * Sanfte Gitarrenklänge * Cahits Traum: ein Chauffeurjob. 30:00 2 Std. später ist er schon fertig. 30:16 Nächste Woche habe ich den nächsten Termin, dann schauen wir. Vielleicht ein paar Bewerbungen schreiben, Inserate anschauen, 30:24 Jobinserate, dann ... ... schauen wir weiter, ja. 30:37 Nur wenige Tage später: Cahits Eu- phorie ist bereits wieder verflogen. 30:42 Seine Kiffervergangenheit hat ihn eingeholt. Vor einigen Monaten fuhr er unter Cannabiseinfluss Auto 30:49 und wurde erwischt. Jetzt ist er den Führerschein los. 30:57 Am Anfang hiess es: drei Monate. Aber sie haben ihn mir auf unbestimmte Zeit entzogen. 31:07 Jeder Zweite in der Schweiz kifft. Oder ein Viertel. Das macht sie ja ... 31:14 Ich denke, die fahren auch alle. Aber ... ... ich bin jetzt der Pechvogel. Oder wie nennt man das? 31:24 Ich bekomme jetzt ... Siehst du dich als Pechvogel? - Ja. 31:30 Ohne Auto kann er auch nicht mehr als Pizzakurier arbeiten. Wie gesagt, ich habe abends als Pizzakurier gearbeitet. 31:38 Das war für mich eine Beschäftigung. Und ich habe etwas Geld verdient. 31:44 Jetzt kann ich das auch nicht mehr. Leider, ja. 31:49 Ein Rückschlag? - Genau. Dazu kommt: Ohne Führerschein 31:55 ist für die nächste Zeit auch sein Traum vom Chauffeurjob geplatzt. 32:08 Reto kann heute endlich in die Klinik. Es wäre eigentlich Zeit, aufzubrechen. 32:15 Ai. Weisst du, wenn man dort 5 Min. zu spät ankommt ... 32:26 Wie spät haben wir denn jetzt? Zehn nach. Elf nach. Also, jetzt gebe ich alles, was ich kann. 32:33 Und wenn wir zu wenig haben, sage ich ihnen, ich hätte vergessen, in meiner Scheibe drin zu wenig einzupacken. 32:41 Fertig. Wenn Reto zu spät in die Klinik geht, ist das gar nicht gut für ihn. 32:47 Wenn ich nicht erscheine ... ... könne ich direkt zu Hause bleiben 32:55 und dann hätte ich 30 Tage lang eine Sperre. In seinem Zustand wäre das eine Katastrophe. 33:04 Dazu kommt: Reto will unbedingt noch eine letzte Dosis Dormicum nehmen, 33:09 vor dem Eintritt. Kannst du das bitte irgendwie verstauen? 33:14 Zack, die ganze Fuhre dabei, wie? 33:21 Wenn es gut läuft, sind das die letzten Dormis, die du in deinem Leben nimmst, oder? 33:31 Herr Jesus, ich bitte dich, gib mir die Kraft, dass du mir ... 33:37 ... hilfst, dass das die letzten Dormicum in meinem Leben sind. Amen. 33:43 Ja, wenn es ... 33:49 Äh. Eigentlich gehst du in die Klinik, um genau davon wegzukommen? 33:56 Etwas paradox, dass du jetzt welche nimmst. Nein, nein, das ist überhaupt nicht paradox. 34:05 Denn dein Hirn denkt: "Oh, jetzt machst du das dort." 34:11 "Dann darfst du nie mehr. Also jetzt noch ein letztes Mal, bitte." 34:22 Ihr Leute stellt Fragen und Fragen. 34:27 Dabei wüsstest du es ... Du hättest ... Die Antwort hättest du dir selber geben können. 34:34 * Hektische, helle Klänge * 34:48 Nächstes Mal geht Reto mit letzter Kraft in die Klinik. 34:53 Morphin zeigt an. - Ja. THC zeigt an. - Mhm. Und Kokain zeigt an. 35:00 Ich tue da nur etwas komisch. - Tun Sie etwas komisch? Cahit will sich ins Leben zurückkämpfen, 35:07 verfällt aber wieder seiner Spielsucht. Ich habe Geld bekommen. 35:15 Nach ein paar Tagen oder schon nach ein paar Stunden war es weg. 35:20 Und Samira kämpft in der Klinik mit verschiedenen Problemen. Gestern hatte ich eine Fressattacke. 35:27 Ich ass alles, was ich noch im Zimmer hatte. Ich habe jetzt schon Bammel vor dem Mittagessen. 35:32 Keine Ahnung, was kommt. Keine Ahnung, was ich davon essen kann und werde. 35:37 Ich vermisse das Gym. ================================================ FILE: src/srf/dok/KampfSucht.txt ================================================ Kampf gegen die Sucht – Drogensucht, Spielsucht, Sportsuch * Sanfte schwermütige Klänge * Reto nimmt seit seinem 16. Lebensjahr Drogen. 0:07 Seit Jahren will er davon loskommen. Einfach immer im Hinterkopf, wenn ich heimkomme, etwas reinzupfeifen. 0:16 Samira hat viele Fitnesswettkämpfe gewonnen. Der Erfolg hat aber seinen Preis: Sie hat eine Fitnesssucht, 0:23 schwere Essstörungen und ist unzufrieden mit sich selber. Zu dick. Ich weiss, ich kann anders ussehen. Drum gefällt es mir nicht. 0:33 Cahit verspielt seit Jahren sein ganzes Geld. Verloren. 0:38 Er hat mit Schulden zu kämpfen und ist allein. Sie alle wollen eine Veränderung. 0:44 Weg von der Sucht, rein in ein neues Läbä. 0:51 * Einzelne eindringliche Streicherklänge * Hier in diesem Block wohnt Reto. 0:58 Ein Bier am Morgen ist nicht sein grösstes Problem. Sondern Heroin, Kokain oder Benzodiazepin. 1:05 Es gibt kaum eine Droge, die er nicht schon genommen hat. Seit 25 Jahren habe ich Drogenprobleme. 1:11 Während der ersten zehn Jahre wollte ich gar nichts ändern. Natürlich möchte ich heute etwas ändern, aber wenn du ... 1:18 ... von deinem Leben ... Ich bin jetzt 41. Von meinem Leben ... 1:25 Mit 16 habe ich angefangen zu spritzen, hatte vorher schon mit Drogen zu tun. 1:31 Es ist wahnsinnig, das Leben einfach zu ändern. Oder aber ich weiss nicht, wie es richtig geht. 1:39 Seit seinem letzten Entzug vor einigen Monaten nimmt Reto keine harten Drogen mehr. 1:45 Als gelernter Metallbauer sucht er einen Job. Er möchte eine Struktur in seinem Leben 1:50 und nicht immer alleine zu Hause sein. Aber mit seiner Vergangenheit ist das schwierig. 1:56 Was soll ich denn machen, wenn ich nur Absagen bekomme und ... 2:01 Bei jeder Absage, die du bekommst, bist du auch irgendwie ... ... wirst du auch irgendwie ... 2:09 ... demotiviert. Du fragst dich: "Wozu bewirbst du dich überhaupt?" 2:14 Und dann ... (Seufzend) Ja ... 2:19 * Einzelne Streicherklänge * 2:26 Ohne Arbeit und Tagesstruktur ver- bringt Reto fast jeden Tag alleine. 2:31 Und mit jedem Tag, der verstreicht, merkt er, wie das Verlangen grösser wird, wieder Drogen zu konsumieren. 2:38 So. Es kommt wie eine Welle. 2:45 Es baut sich auf, bis ein riesiger Druck da ist. 2:50 Was könnte diesen Druck lösen? Das weiss ich auch nicht. 2:56 Sonst würde ich es machen. Ich habe schon viel probiert. 3:04 Regelmässig geht Reto in der Klinik Selhofen in Burgdorf in Therapie. 3:09 Ein Weg, den er gut kennt. (Lachend) Mein zweites Zuhause. Nein. 3:20 Du warst auch schon hier? Und nicht nur zur Therapie. Hier? Zehnmal war ich stationär hier. 3:28 Und Reto spürt, dass er bald wieder abstürzen könnte. 3:35 Welche Gedanken haben Sie sich gemacht? Ob ich nochmals irgendwie drei oder vier Wochen hierher kommen soll. 3:43 Okay, erneut über das Stationäre, worüber wir ... - Ja. ... auch schon gesprochen haben. Ja. 3:50 Aber es fehlt irgendwie noch der letzte Dings, der letzte ... 3:55 Ich bin noch nicht 100 ... Ich mache mir Gedanken darüber, habe mich aber noch nicht entschieden. 4:01 Was wäre denn der Nutzen? Was soll ein Aufenthalt bringen? Es wäre wie ein Restart für mich. 4:10 Ich habe das Gefühl, dass ich früher oder später einen Job finde. Davon bin ich überzeugt. 4:16 Stationär zu kommen wäre eine Art Beschäftigung, bis Sie dann vielleicht einen Job bekommen? 4:25 Ein bisschen. - Ja. Wenn Sie hierher kommen und dann wieder rausgehen 4:32 und trotzdem keine Arbeit bekommen, könnte es dann wieder ins Gleiche gehen? 4:42 Ja, natürlich. Wenn er sagen würde: "Ich muss wirklich nochmals inten- siver an meiner Sucht arbeiten." 4:51 "Ich will wirklich davon wegkommen." Oder: "Ich muss mir über den Anschluss Gedanken machen." 4:57 "Ich muss irgendetwas verändern, auch wenn ich keinen Job habe." "Aber es muss etwas anders sein für danach." 5:03 Denn wäre es für mich ganz anders, als wenn er sagt: "Ich will v.a. einen Job haben." 5:09 "Wenn ich einen Job habe, ist alles gut, und ich komme mal in der Hoffnung, schneller einen Job zu bekommen." 5:18 Es gäbe Beschäftigungsprogramme für Menschen in Retos Situation. Schön, oder? 5:25 Er möchte aber einen richtigen Job und so von den Drogen wegkommen. Einfach immer im Hinterkopf, wenn ich heimkomme, etwas reinzupfeifen, 5:33 das ist der Scheiss. Ich drehe mich im Kreis, seit Jahren. 5:39 Selhofen, dann kommst du wieder raus, reisst dich zusammen, dann stürzt du trotzdem wieder ab, zwischendurch gibt es mal einen Job, 5:48 dann hast du einen Unfall. Die nächsten Tage werden entscheidend sein. 5:55 Entweder ändert sich noch etwas oder es droht der nächste Absturz. 6:05 Das Gym ist ihre Welt. Die 25-jährige Samira ist sechsmal pro Woche da 6:10 für ca. 2 Std. Nicht ins Fitness zu gehen ist für sie keine Option. 6:20 Für mich ist es schwierig, einfach mal zu sagen: "Hey, heute bleibe ich besser zu Hause, weil es mir nicht gut geht 6:27 oder weil ich körperlich nicht in guter Verfassung bin." Dann ... ... weiss ich im Vornherein, dass mein Tag grösstenteils gelaufen ist. 6:37 Also, mir ist das Training schon sehr wichtig. 6:47 Es geht um Disziplin, um Kontrolle. Manchmal auch um Zwang. 6:54 Samira will ihren Körper beherrschen. Und sie ist streng mit sich. 7:06 Auf Instagram hat sie über 20'000 Follower. Kein Wunder: Von ihrem Körper könnten viele Frauen nur träumen. 7:15 Und trotzdem ist sie nicht zufrieden mit sich. Was denkst du, wenn du dich im Spiegel siehst? 7:22 Zu dick. 7:30 Weil ich mal Wettkämpfe gemacht habe, ist das Bild anders. Ich weiss, ich kann anders aussehen. 7:36 Jetzt bin ich halt ... ... gut 15-20 kg schwerer, ja, 15. 7:43 Und ... ... dann gefällt es mir halt nicht. Spitzensport gehört schon lange zu Samiras Leben. 7:51 Bereits als Achtjährige geht sie ins Kunstturnen, trainiert viermal pro Woche. Als Teenager beginnt sie mit Fitness. 7:59 Mit Erfolg: Mit 22 ist Samira Miss Bikini, Schweizer Meisterin im Fitness, 8:05 und holt einen vierten Platz an den Weltmeisterschaften. Samira hat ihren Körper im Griff, isst strikt nach Plan. 8:12 Nach den Wettkämpfen kippt es ins Gegenteil. Sie hat regelmässig Fressattacken und nimmt stark zu. 8:19 Ein Teufelskreis. 8:24 Heute leidet Samira an einer schweren Essstörung. Sie wiegt alles und kontrolliert krankhaft, 8:31 was sie isst und was nicht. Trotz oder gar wegen der strengen Ernährung 8:39 hat sie immer wieder Essattacken. Nachts. Wenn sie alleine ist, verliert sie die Kontrolle. 8:51 Bei mir ist es wirklich so extrem, dass ich dann ... ... nicht mehr aufhören kann mit essen. 8:58 Ich holte mir auch schon nachts an der Tankstelle oder am Selecta-Automaten mehr Essen. 9:06 Ähm, ja ... Das ist natürlich wie ein Rausch. 9:12 Man muss sich das vorstellen, wie wenn man andere Drogen konsumiert - 9:17 gut, das weiss ich ehrlich gesagt nicht. Aber ich stelle es mir ähnlich vor, weil man halt ... 9:23 ... immer noch mehr und noch mehr will und es nicht aufhört. 9:28 Samira besucht regelmässig einen Therapeuten, der helfen soll. Seit einigen Monaten ist sie aber 100 % krankgeschrieben. 9:36 Deshalb hat sie ihre Stelle als Hochbauzeichnerin verloren. Es hat überall, in sehr vielen Bereichen meines Lebens 9:44 zu bröckeln begonnen. Es waren nicht nur die Ernährung und der Sport. 9:49 Es gab auch Berufliches und Privates, das zu Bruch ging, was mich aus der Bahn warf. 9:56 Mein psychischer Zustand hat sich dann halt ... ... extrem verschlechtert. 10:02 Das hat sich durch Fressattacken und Schlafstörungen geäussert. Und durch Übertraining, dadurch, dass ich nicht mehr schlafen konnte. 10:11 Mir war das dann alles zu viel. Mit diesem Formular will Samira ihr Leben jetzt von Grund auf ändern. 10:19 In einigen Tagen geht sie für mehrere Monate in eine Klinik. 10:26 Ah, voila. So wäre es super. Ab und zu ruft Reto via Internettelefon an 10:32 und erzählt, wie es ihm geht. Man gibt sich Mühe, oder. 10:39 Aber es gibt immer Phasen. Manchmal geht es besser, manchmal weniger gut. 10:46 Schon? Aber jetzt geht es gerade gut? Ja, jetzt habe ich gerade eine gute Phase. 10:54 Ich habe irgendwie auch nicht so Lust auf Drogen. Ich will wieder mehr Sport machen. Jeden Tag ein bisschen. 11:02 Ins Fitness, sozusagen? Ah, jetzt ist gut, du. Jetzt ist es super. 11:09 Jetzt ist es etwas weniger nervös. Ich habe gerade gesagt, dass ich versuchen will, 11:16 diese Woche möglichst keinen Alkohol mehr zu trinken. Nicht mehr saufen und so. 11:22 Es geht gut, ja. Es geht mir fast besser. Gestern und heute habe ich nicht getrunken. 11:30 Einige Tage später. Die Anrufe von Reto sind ausgeblieben. Trotz der guten Vorsätze hat er den Kampf gegen die Drogen 11:38 einmal mehr verloren. Ja, leider wieder ... 11:43 ... eben, Absturz auf die Benzos gehabt und ... 11:50 Sobald ich merke, dass es aus dem Ruder läuft, 11:56 dass ich es nicht mehr unter Kontrolle habe, wann ich eins nehme und wann nicht, 12:01 fühle ich mich unter Druck gesetzt. Und das macht mich fertig. 12:07 Ich will es wegputzen. Ich will diesen Scheiss nicht mehr. Reto hat sofort reagiert 12:14 und sich für einen stationären Aufenthalt in der Klinik angemeldet. Jetzt muss er warten, bis er eintreten kann. 12:22 Ja, es ist verrückt. Das ist jetzt das elfte Mal. 12:30 Es ist mir selber peinlich. Ich habe jeweils diejenigen ausgelacht - oder nicht ausgelacht, aber gedacht: 12:37 "Wow, der war schon so oft im Entzug?" Als ich selber erst zwei- oder dreimal war. 12:51 Das sind diese Dormicum. Dormicum ist ein verschreibungspflichtiges Medikament, 12:59 das z.B. gegen Depressionen oder zur Einleitung von Narkosen eingesetzt wird. 13:05 Ein stattlicher Haufen. Der Wirkstoff Benzodiazepin macht schläfrig, entspannt - 13:12 und schnell süchtig. Das Problem ist: Wenn ich es nicht nehme, beginne ich zu zittern. 13:18 Und darauf habe ich keine Lust. Normalerweise nimmt man das Dormicum als Tablette. 13:25 Gemahlen durch die Nase wirkt es aber schneller und heftiger. 13:32 Ah. 13:40 So. Spürst du jetzt sofort etwas? Es brennt einfach in den Augen, oder. Ja. 13:49 Aber jetzt nicht, dass ich direkt ... ... einhängen würde oder so. 13:58 Wobei ich finde, dass auch die Qua- lität der Tabletten abgenommen hat. 14:03 Mit der Dosis, die Reto nimmt, würden die meisten Leute flachliegen. Er kann weiterhin einigermassen funktionieren. 14:11 Da rein. Reto will Pilze fürs Abendessen sammeln. 14:18 Schau, da sind schon welche. Ich kenne sie nicht. Ich esse nichts, was ich nicht esse. 14:26 Mir würden zwei oder drei schöne Steinpilze schon reichen, weisst du. 14:32 Hast du gesehen? Plötzlich wimmelt es davon. Maronen .... Maronenröhrling. 14:39 Schon wieder einer. Das ist ein Rotfüsschen. 14:44 Die schlechten Teile weg und dann los - ab in die Pfanne damit. Wenn ich mal irgendetwas angefangen habe, 14:54 habe ich Mühe mit aufhören - egal mit was, fällt mir auf. 15:00 Ob mit saufen, Pilze sammeln, mit Sport, mit ... 15:06 Ähm ... Drogen und was weiss ich. Ich höre einfach nicht mehr auf. 15:13 Das ist das Problem. Was bedeutet dir die Natur? 15:20 Die Natur ist ein Kraftpunkt für mich. Ich war mit meinem Vater oft in der Natur. 15:30 Er hat mich viel über die Natur gelehrt. Irgendwie, wenn ich in der Natur bin, habe ich manchmal das Gefühl, 15:37 dass ich mit ihm zusammen bin, oder so. Wie wichtig war er? 15:43 Er war mein Vater. "Wie wichtig war er?" 15:49 Er war für mich zu der Zeit die wichtigste Person. 15:58 Als Reto elf Jahre alt war, erkrankte sein Vater an Lungenkrebs und starb kurze Zeit später. 16:06 Du sagst immer, das sei ein Wende- punkt gewesen, als er starb. Oder? 16:11 Ja. Wenn dir mit elf Jahren das Liebste entrissen wird 16:17 und du es nie mehr wieder siehst. 16:22 Da möchte ich andere mal sehen. Ich will nicht immer über meinen Vater sprechen. 16:30 Der ist gestorben. * Sanfte gläserne Klänge * 16:40 Es dauert noch ca. drei Wochen, bis Reto endlich in die Klinik eintreten kann. 16:45 Bis dahin muss er alleine mit seiner Sucht irgendwie seinen Alltag bewältigen. 16:54 * Sanfte gläserne Klänge * 17:14 Das ist Cahit. Er ist 46-jährig und türkisch- schweizerischer Doppelbürger. 17:20 Cahit wohnt in der Stadt Zürich in einem Einzimmerstudio. Er ist oft alleine. 17:26 Auch Eltern und Geschwister besuchen ihn nur selten. Der Grund: Cahit ist spielsüchtig 17:31 und hat das Vertrauen seiner Verwandten immer wieder missbraucht. 17:43 Manchmal ... ... habe ich von meinem Bruder oder von Kollegen Geld ausgeliehen 17:50 und habe dann mit diesem Geld gespielt. Und jetzt haben sie ... 17:57 ... irgendwie kein Vertrauen in einen Zocker. 18:03 Weil du ihr Geld nicht mehr zurückbezahlt hast? Genau. 18:09 Seit 15 Jahren ist er spielsüchtig, arbeitslos und einsam. 18:14 Seine Sucht hat ihm alles genommen. Finanziell ist er von der Sozialhilfe abhängig - 18:20 und von Geld von den Eltern. Er kämpft mit Depressionen. Trotz seiner Situation spielt er immer wieder. 18:27 Weil er unterdessen in allen Casinos gesperrt ist, v.a. mit Sportwetten via Handy. 18:36 Wie viel hast du draufgeladen? 10 Fr. 18:42 Das ist ja nicht so viel. - Ja. Bei Sportwetten ... 18:48 ... setze ich weniger als ... Ich sage jetzt mal 10 Fr. 18:54 Alle zwei bis drei Tage. Einer davon ist Galatasaray gegen Fenerbahce. 19:01 Ich bin auch Fenerbahce-Fan. Die sind aus Istanbul. 2:2. 19:07 Ich habe auf zwei für Fenerbahce ge- tippt, jetzt ist 2:2 unentschieden. Das heisst? 19:13 Verloren. Ja. 19:22 Früher spielte er in echten Casinos. Hat auch gewonnen - und das nicht zu wenig. 19:32 Das Geld habe ich auch alles verspielt. Ein paar Hunderttausend? 19:37 Ja. Schon, ja. Das ist viel. - Ja. 19:43 Das ist viel, ja. Regelmässig ist Cahit in der Langstrasse in Zürich im Ausgang. 19:51 Es ist wie ein Zuhause. Aber hier kann man auch überall Geld verspielen. 19:58 3'000-4'000 Fr., die man am einen Ort gewinnt, 20:04 ein paar Tausend Fr. Dann gehst du zum nächsten und verlierst einfach alles wieder. 20:12 An jeder Ecke ein Kiosk, an dem man legale und auch illegale Online-Games spielen kann. 20:18 Mit der Kamera reingehen will Cahit zu seinem eigenen Schutz nirgends. Wann hast du gemerkt, dass du ein Problem hast? 20:28 Vor Kurzem habe ich von meinem Bruder Geld verlangt. 20:33 Ich habe gesagt: "Ich brauche Geld." Er hat mir seine Kreditkarte gegeben. Die ganze Nacht, bis am Morgen ... 20:41 Ich ging 4'000 Fr. ins Minus auf der Kreditkarte meines Bruders. 20:47 Dann habe ich gesagt: "Ich stecke wirklich ... 20:52 wirklich tief in dieser Sucht." Seit diesem Erlebnis will Cahit sein Leben wieder in den Griff bekommen. 21:00 Aber zu Hause in Zürich lauert die Verlockung überall. Und trotzdem: Weg von hier will er nicht. 21:08 Ja. Ja, dann muss ich die Stadt verlassen, ja. 21:13 Ich bin ... ... halt hier aufgewachsen. 21:19 Irgendwie kann ich das auch nicht. Der Kollegenkreis und v.a. die Familie leben ja in dieser Stadt. 21:26 Also ... * Salsaklänge * 21:40 Samira bricht heute auf in die Klinik. 21:45 Wie geht es dir heute Morgen? Wieder etwas besser 21:52 bzw. war es die ganze letzte Woche für mich ziemlich schwierig, mich mit dem Ganzen abzufinden, quasi. 22:00 Aber, ähm ... Ja, ich habe jetzt einfach versucht ... 22:07 ... mir das Ganze schönzureden, quasi. 22:15 Also, ich erhoffe oder wünsche mir, dass ich in dieser Zeit eigentlich wieder zu mir selber finden kann 22:24 und wieder lerne, das Leben zu geniessen, ohne mich irgendwie gezwungen zu fühlen. 22:42 Zwei bis drei Monate will sie in der Klinik bleiben. Sie geht v.a. wegen ihrer Essstörung in Behandlung. 22:49 Trotzdem nimmt sie ihr eigenes Essen mit. * Sanfte dumpfe Klänge * 23:01 Die Klinik Barmelweid in Erlinsbach im Kanton Aargau. Eine Spezialklinik für psychosoma- tische Medizin und Psychotherapie. 23:10 Während der nächsten Monate Samiras neues Zuhause. Ich mache kurz die Tür zu. 23:26 Das wäre Ihr Zimmer. Hier haben wir eine Glocke. 23:32 Wenn Sie etwas von uns brauchen, können Sie sich damit melden. Okay. 23:37 Dann haben Sie ein eigenes Bad. - Okay. Das wäre dort vorne. 23:44 Ah ja, okay. Super. - Ja. 23:50 Noch die Schuhe ausziehen zum Wägen. Auf fünf oder ...? - Ja, ja, vier. 23:55 Abrunden auf vier. - Vier, fünf. Sind Sie 1.63 m? - Ja. 24:01 Gut. Essen muss Samira von nun an auf der Station. Sie wählt täglich zwischen drei Menüs. 24:08 ... was bestellt worden ist. Wir haben mal das Standardessen angewählt. 24:13 Sie können schauen, ob das für Sie passt. Dann esse ich einfach, was mir schmeckt? Ja. 24:19 Schweinefleisch mag ich z.B. nicht. - Mögen Sie nicht, okay. Dann lassen Sie das heute ausnahmsweise mal aus 24:26 und wir werden vermerken, dass Sie kein Schweinefleisch essen. Gut. Das Brot können Sie direkt löschen. 24:33 * Lachen * - Okay, gut. Salat ist gut, aber das Brot muss nicht sein. 24:40 Das Essverhalten von Samira soll sich ändern. Mit ihren Therapeutinnen schaut sie ihre Situation an. 24:46 Es freut mich, dass Sie sich zu einem stationären Aufenthalt entschieden haben. Können Sie das Problem benennen? 24:53 Was war der Grund für den Klinik- eintritt? Aus Ihrer Sicht. Ich habe gemerkt, dass es für mich nicht mehr normal ist 25:02 bzw. ich den Alltag und das Leben nicht mehr so geniessen kann, wie ich das mal konnte. 25:08 Ich habe auch gemerkt, dass ich meinen Selbstwert extrem vom Training abhängig gemacht habe. 25:15 Dass ich mich schlecht gefühlt habe, wenn ich nicht trainiert habe. Es war ein Teufelskreis mit Schlafstörungen, Essen und Training. 25:24 Genau. Wer schlecht schläft, kann nicht richtig trainieren. Dann habe ich wieder gegessen, habe mich wieder schlecht gefühlt. 25:31 Habe wieder das Gefühl gehabt, ich müsse ins Training. Ich geriet immer tiefer rein. 25:37 Ich hatte das Gefühl: "Das kommt wieder, das legt sich wieder." Irgendwann, als ich noch 100 % arbeitete, 25:44 ist das einfach alles irgendwann ... Es begann überall immer mehr und mehr zu bröckeln. 25:51 Und ... Ja. Deshalb. Sie beschreiben das sehr eindrücklich. 25:57 Sie ist sehr ehrgeizig, sehr diszipliniert ... ... und auch sehr leistungsfähig. 26:04 Auch wenn es momentan nicht mehr so ist. Das ist ein bisschen der Klassiker: 26:09 Man kann sagen, dass generell bei Spitzensportlern sehr schnell - 26:15 oder nicht sehr schnell, aber dass bei den psychischen Störungen die Essstörung an erster Stelle steht. 26:22 Meistens sieht man das bei diesen Sportlern nicht wirklich, weil sie natürlich einen athletischen Körperbau haben. 26:29 Eine Essstörung entwickelt sich dann kontinuierlich. Am Schluss hat man dann diese Essattacken. 26:37 Gibt es etwas, das für Sie nicht passieren darf während des stationären Aufenthalts? Für mich wäre ganz schlimm, wenn ich gar keinen Sport mehr machen könnte. 26:47 Aber ich möchte nicht mehr, dass es so wichtig ist, dass ich mich davon abhängig mache. - Ja. 26:52 Aber trotzdem wäre es gerade zum jetzigen Zeitpunkt schlimm, wenn ich hier rauskäme und ... 26:59 ... und 5-10 kg schwerer wäre oder ... ... vielleicht auch 10 kg leichter und ich sehe, 27:06 dass meine ganze Muskelmasse weg ist. Das wäre das Einzige. Aber sonst habe ich eigentlich nichts zu verlieren. 27:14 Heute darf Samira noch ihr eigenes Mittagessen essen. Aber morgen ist damit Schluss. 27:22 * Dumpfe Klänge * 27:28 * Vogelgezwitscher * Auch Cahit will sein Leben endlich in den Griff bekommen. 27:36 Er besucht eine Therpaie. * Ruhige Gitarrenklänge * 27:44 In den letzten 15 Jahren habe ich viel zu viel verloren. Nicht nur Geld. 27:50 Auch psychisch geht es mir schlecht, habe immer Depressionen, ich bin nicht mehr in meinem Kollegenkreis, 27:57 ich stehe meiner Familie nicht mehr nahe. Machen Sie jetzt noch Schulden damit? 28:04 Nein, Schulden mache ich nicht. Nein, nein. Es ist wichtig, dass jetzt der Teil der Therapie kommt, 28:12 in dem wir wirklich konkret die Tagesstruktur, konkret die Partnersuche, 28:19 konkret die Arbeitssuche, all das besprechen. 28:24 Sie sind motiviert, da rauszukommen, ja? Ja. Und ... 28:30 Es ist nicht zu spät, um rauszukommen. - Mhm. Also: 28:37 Dann beginnen wir mit der linken Seite. Den linken Arm anspannen. 28:43 Nichts forcieren. Jetzt können Sie entspannen. Eine Entspannungsübung ist der erste Schritt. 28:49 * Sanfte, beruhigende Klänge * 28:54 Mit frischem Wind will Cahit jetzt vorwärtsmachen. Erster Schritt: Jobsuche. 29:00 Er arbeitet zwar ab und zu als Pizzakurier, mit Hilfe der Sozialdienste will er jetzt aber sein Dossier aufpeppen. 29:08 Was musst du dort machen? Einen Job suchen. Alles, Lebenslauf, 29:14 Arbeitszeugnisse zusammenstellen. Dann Bewerbungen schreiben - vielleicht dann. 29:21 * Sanfte Gitarrenklänge * Ich, äh, ja. 29:26 Da ist der Lebenslauf drauf. Das ganze Leben ist da drauf. 29:38 Ohne Job ist es mir im Moment verdammt langweilig. Bewerbungen schreiben, ein Job, das ist sehr wichtig, ja. 29:47 Willst du wirklich arbeiten? Hast du Lust? Ich habe Lust, ja, doch. 29:53 * Sanfte Gitarrenklänge * Cahits Traum: ein Chauffeurjob. 30:00 2 Std. später ist er schon fertig. 30:16 Nächste Woche habe ich den nächsten Termin, dann schauen wir. Vielleicht ein paar Bewerbungen schreiben, Inserate anschauen, 30:24 Jobinserate, dann ... ... schauen wir weiter, ja. 30:37 Nur wenige Tage später: Cahits Eu- phorie ist bereits wieder verflogen. 30:42 Seine Kiffervergangenheit hat ihn eingeholt. Vor einigen Monaten fuhr er unter Cannabiseinfluss Auto 30:49 und wurde erwischt. Jetzt ist er den Führerschein los. 30:57 Am Anfang hiess es: drei Monate. Aber sie haben ihn mir auf unbestimmte Zeit entzogen. 31:07 Jeder Zweite in der Schweiz kifft. Oder ein Viertel. Das macht sie ja ... 31:14 Ich denke, die fahren auch alle. Aber ... ... ich bin jetzt der Pechvogel. Oder wie nennt man das? 31:24 Ich bekomme jetzt ... Siehst du dich als Pechvogel? - Ja. 31:30 Ohne Auto kann er auch nicht mehr als Pizzakurier arbeiten. Wie gesagt, ich habe abends als Pizzakurier gearbeitet. 31:38 Das war für mich eine Beschäftigung. Und ich habe etwas Geld verdient. 31:44 Jetzt kann ich das auch nicht mehr. Leider, ja. 31:49 Ein Rückschlag? - Genau. Dazu kommt: Ohne Führerschein 31:55 ist für die nächste Zeit auch sein Traum vom Chauffeurjob geplatzt. 32:08 Reto kann heute endlich in die Klinik. Es wäre eigentlich Zeit, aufzubrechen. 32:15 Ai. Weisst du, wenn man dort 5 Min. zu spät ankommt ... 32:26 Wie spät haben wir denn jetzt? Zehn nach. Elf nach. Also, jetzt gebe ich alles, was ich kann. 32:33 Und wenn wir zu wenig haben, sage ich ihnen, ich hätte vergessen, in meiner Scheibe drin zu wenig einzupacken. 32:41 Fertig. Wenn Reto zu spät in die Klinik geht, ist das gar nicht gut für ihn. 32:47 Wenn ich nicht erscheine ... ... könne ich direkt zu Hause bleiben 32:55 und dann hätte ich 30 Tage lang eine Sperre. In seinem Zustand wäre das eine Katastrophe. 33:04 Dazu kommt: Reto will unbedingt noch eine letzte Dosis Dormicum nehmen, 33:09 vor dem Eintritt. Kannst du das bitte irgendwie verstauen? 33:14 Zack, die ganze Fuhre dabei, wie? 33:21 Wenn es gut läuft, sind das die letzten Dormis, die du in deinem Leben nimmst, oder? 33:31 Herr Jesus, ich bitte dich, gib mir die Kraft, dass du mir ... 33:37 ... hilfst, dass das die letzten Dormicum in meinem Leben sind. Amen. 33:43 Ja, wenn es ... 33:49 Äh. Eigentlich gehst du in die Klinik, um genau davon wegzukommen? 33:56 Etwas paradox, dass du jetzt welche nimmst. Nein, nein, das ist überhaupt nicht paradox. 34:05 Denn dein Hirn denkt: "Oh, jetzt machst du das dort." 34:11 "Dann darfst du nie mehr. Also jetzt noch ein letztes Mal, bitte." 34:22 Ihr Leute stellt Fragen und Fragen. 34:27 Dabei wüsstest du es ... Du hättest ... Die Antwort hättest du dir selber geben können. 34:34 * Hektische, helle Klänge * 34:48 Nächstes Mal geht Reto mit letzter Kraft in die Klinik. 34:53 Morphin zeigt an. - Ja. THC zeigt an. - Mhm. Und Kokain zeigt an. 35:00 Ich tue da nur etwas komisch. - Tun Sie etwas komisch? Cahit will sich ins Leben zurückkämpfen, 35:07 verfällt aber wieder seiner Spielsucht. Ich habe Geld bekommen. 35:15 Nach ein paar Tagen oder schon nach ein paar Stunden war es weg. 35:20 Und Samira kämpft in der Klinik mit verschiedenen Problemen. Gestern hatte ich eine Fressattacke. 35:27 Ich ass alles, was ich noch im Zimmer hatte. Ich habe jetzt schon Bammel vor dem Mittagessen. 35:32 Keine Ahnung, was kommt. Keine Ahnung, was ich davon essen kann und werde. 35:37 Ich vermisse das Gym. ================================================ FILE: swissDeutschBot/.gitignore ================================================ # Project exclude paths /venv/ ================================================ FILE: swissDeutschBot/.idee/Inspektion/.gitignore ================================================ # Default ignored files /shelf/ /workspace.xml ================================================ FILE: swissDeutschBot/.idee/Inspektion/GPT3api.iml ================================================ ================================================ FILE: swissDeutschBot/.idee/Inspektion/miscellanious.xml ================================================ ================================================ FILE: swissDeutschBot/.idee/Inspektion/modules.xml ================================================ ================================================ FILE: swissDeutschBot/.idee/Inspektion/profil_einstellungen.xml ================================================ ================================================ FILE: swissDeutschBot/.idee/Inspektion/projeckt_vorgaben.xml ================================================ ================================================ FILE: swissDeutschBot/.idee/Inspektion/requirements.md ================================================ aiohttp==3.7.4.post0 async-timeout==3.0.1 attrs==21.2.0 certifi==2021.10.8 chardet==4.0.0 charset-normalizer==2.0.7 discord==1.7.3 discord.py==1.7.3 et-xmlfile==1.1.0 idna==3.3 multidict==5.2.0 numpy==1.21.2 openai==0.10.5 openpyxl==3.0.9 pandas==1.3.3 pandas-stubs==1.2.0.27 python-dateutil==2.8.2 pytz==2021.3 requests==2.26.0 six==1.16.0 tqdm==4.62.3 typing-extensions==3.10.0.2 urllib3==1.26.7 yarl==1.7.0 ================================================ FILE: swissDeutschBot/.idee/Inspektion/vcs.xml ================================================ ================================================ FILE: swissDeutschBot/Dockerfile ================================================ worker python main.py ================================================ FILE: swissDeutschBot/mini.py ================================================ import os import discord from discord.ext import commands import OpenAI BOT_TOKEN = os.getenv("BOT_TOKEN") OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") PREFIX = "ger " client = commands.Bot(command_prefix=PREFIX, activity=discord.Game(name=f"{PREFIX}help")) client.remove_command("help") def main(): @client.event async def on_ready(): print("Bot has successfully logged in as: {}".format(client.user)) print("Bot ID: {}\n".format(client.user.id)) @client.event async def on_command_error(ctx, error): if isinstance(error, commands.NoPrivateMessage): await ctx.send( embed=discord.Embed( description='**This command cannot be used in private messages.**', color=discord.Color.red()), ) elif isinstance(error, commands.CommandNotFound): await ctx.send( embed=discord.Embed( description='**This command doesnt exists.**', color=discord.Color.red()), ) @client.command() async def help(ctx): embed = discord.Embed( title="List of commands", color=discord.Color.orange() ) embed.add_field( name="**ger ask**", value="Ask any german related question. " \ "Ex. `ger ask can you explain in english 'akkusativ' with an example?`", inline=False ) embed.add_field( name="**ger correct**", value="Corrects a sentence in German if the bot thinks it is incorrect. " \ "Ex. `ger correct Der hamer klein ist`", inline=False ) embed.set_footer( text="Warning: this bot is still being developed and you may encounter errors" ) emoji = "\u2705" await ctx.message.add_reaction(emoji) await ctx.author.send(embed=embed) list_user = [] @client.command() @commands.guild_only() async def ask(ctx, *, question): if ctx.message.channel.type != "dm": words = len(question.split()) print(words) if 2 >= words > 0: OpenAI.ptype = "oneshot" elif 6 >= words > 2: OpenAI.ptype = "simple" elif words > 6: OpenAI.ptype = "complex" async with ctx.typing(): # await message.channel.send('ping') list_user.append(ctx.message.author.id) answer = OpenAI.ask(question) await ctx.send(answer) @client.command() @commands.guild_only() async def correct(ctx, *, sentence): if ctx.message.channel.type != "dm": async with ctx.typing(): correction = OpenAI.correct(sentence) if correction.strip() == sentence.strip(): await ctx.send("I think there is no issue with your sentence") else: await ctx.send(f"Maybe you should try saying: '{correction}'") @client.command() @commands.is_owner() async def shutdown(): exit() OpenAI.openai.api_key = OPENAI_API_KEY client.run(BOT_TOKEN) if __name__ == "__main__": main() ================================================ FILE: swissDeutschBot/openAI.py ================================================ import openai def set_prompt(prompt_type): if prompt_type == "oneshot": session_prompt = "Conversation with a