Repository: sourcerer-io/sourcerer-app
Branch: develop
Commit: af3d7737ba20
Files: 2930
Total size: 21.2 MB
Directory structure:
gitextract_96evfk3n/
├── .gitattributes
├── .gitignore
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── Dockerfile
├── LICENSE.md
├── README.md
├── build.gradle
├── deploy/
│ ├── GradleDockerfile
│ ├── Jenkinsfile
│ ├── default.conf
│ ├── production_env.sh
│ ├── sandbox_env.sh
│ ├── sourcerer-app.yaml
│ └── staging_env.sh
├── do.sh
└── src/
├── install/
│ └── install
├── main/
│ ├── kotlin/
│ │ └── app/
│ │ ├── Analytics.kt
│ │ ├── FactCodes.kt
│ │ ├── Logger.kt
│ │ ├── Main.kt
│ │ ├── api/
│ │ │ ├── Api.kt
│ │ │ ├── ApiError.kt
│ │ │ ├── MockApi.kt
│ │ │ ├── Result.kt
│ │ │ └── ServerApi.kt
│ │ ├── config/
│ │ │ ├── Config.kt
│ │ │ ├── Configurator.kt
│ │ │ ├── FileConfigurator.kt
│ │ │ └── MockConfigurator.kt
│ │ ├── extractors/
│ │ │ ├── CExtractor.kt
│ │ │ ├── CSharpExtractor.kt
│ │ │ ├── ClassifierManager.kt
│ │ │ ├── CommonExtractor.kt
│ │ │ ├── CppExtractor.kt
│ │ │ ├── CrystalExtractor.kt
│ │ │ ├── CssExtractor.kt
│ │ │ ├── DMExtractor.kt
│ │ │ ├── DartExtractor.kt
│ │ │ ├── DevopsExtractor.kt
│ │ │ ├── ElixirExtractor.kt
│ │ │ ├── Extractor.kt
│ │ │ ├── ExtractorBase.kt
│ │ │ ├── ExtractorInterface.kt
│ │ │ ├── FSharpExtractor.kt
│ │ │ ├── GoExtractor.kt
│ │ │ ├── Heuristics.kt
│ │ │ ├── IPythonExtractor.kt
│ │ │ ├── JavaExtractor.kt
│ │ │ ├── JavascriptExtractor.kt
│ │ │ ├── KotlinExtractor.kt
│ │ │ ├── Languages.kt
│ │ │ ├── ObjectiveCExtractor.kt
│ │ │ ├── PerlExtractor.kt
│ │ │ ├── PhpExtractor.kt
│ │ │ ├── PlpgsqlExtractor.kt
│ │ │ ├── PythonExtractor.kt
│ │ │ ├── RubyExtractor.kt
│ │ │ ├── RustExtractor.kt
│ │ │ ├── ScalaExtractor.kt
│ │ │ ├── SwiftExtractor.kt
│ │ │ └── TypescriptExtractor.kt
│ │ ├── hashers/
│ │ │ ├── AuthorDistanceHasher.kt
│ │ │ ├── CodeLongevity.kt
│ │ │ ├── CommitCrawler.kt
│ │ │ ├── CommitHasher.kt
│ │ │ ├── FactHasher.kt
│ │ │ ├── MetaHasher.kt
│ │ │ ├── RepoHasher.kt
│ │ │ └── Vendors.kt
│ │ ├── model/
│ │ │ ├── Author.kt
│ │ │ ├── AuthorDistance.kt
│ │ │ ├── AuthorDistanceGroup.kt
│ │ │ ├── AuthorGroup.kt
│ │ │ ├── Classifier.kt
│ │ │ ├── Commit.kt
│ │ │ ├── CommitGroup.kt
│ │ │ ├── CommitStats.kt
│ │ │ ├── DiffContent.kt
│ │ │ ├── DiffFile.kt
│ │ │ ├── DiffRange.kt
│ │ │ ├── Error.kt
│ │ │ ├── Errors.kt
│ │ │ ├── Fact.kt
│ │ │ ├── FactGroup.kt
│ │ │ ├── LibraryMeta.kt
│ │ │ ├── LocalRepo.kt
│ │ │ ├── Process.kt
│ │ │ ├── ProcessEntry.kt
│ │ │ ├── Repo.kt
│ │ │ ├── RepoMeta.kt
│ │ │ ├── User.kt
│ │ │ └── UserEmail.kt
│ │ ├── ui/
│ │ │ ├── AddRepoState.kt
│ │ │ ├── AuthState.kt
│ │ │ ├── CloseState.kt
│ │ │ ├── ConsoleState.kt
│ │ │ ├── ConsoleUi.kt
│ │ │ ├── Context.kt
│ │ │ ├── EmailState.kt
│ │ │ ├── ListRepoState.kt
│ │ │ ├── OpenState.kt
│ │ │ └── UpdateRepoState.kt
│ │ └── utils/
│ │ ├── CommandAdd.kt
│ │ ├── CommandConfig.kt
│ │ ├── CommandList.kt
│ │ ├── CommandRemove.kt
│ │ ├── FileHelper.kt
│ │ ├── HashingException.kt
│ │ ├── Metrics.kt
│ │ ├── Misc.kt
│ │ ├── Options.kt
│ │ ├── PasswordHelper.kt
│ │ ├── RepoHelper.kt
│ │ ├── RequestException.kt
│ │ └── UiHelper.kt
│ ├── proto/
│ │ ├── classifier.proto
│ │ └── sourcerer.proto
│ └── resources/
│ └── data/
│ ├── imports/
│ │ ├── cpp.txt
│ │ └── python.txt
│ └── libraries/
│ ├── cs_libraries.txt
│ ├── java_libraries.txt
│ ├── js_libraries.txt
│ └── kotlin_libraries.txt
└── test/
├── delete_repo.sh
├── kotlin/
│ └── test/
│ ├── tests/
│ │ ├── extractors/
│ │ │ ├── DirToLangMap.kt
│ │ │ ├── ExtractorTest.kt
│ │ │ ├── HeuristicsTest.kt
│ │ │ └── IgnoredSamplesWildcards.kt
│ │ ├── hashers/
│ │ │ ├── AuthorDistanceHasherTest.kt
│ │ │ ├── CodeLongevityTest.kt
│ │ │ ├── ColleaguesTest.kt
│ │ │ ├── CommitHasherTest.kt
│ │ │ ├── FactHasherTest.kt
│ │ │ ├── IgnorePathsTest.kt
│ │ │ ├── IgnoreVendorsTest.kt
│ │ │ └── MetaHasherTest.kt
│ │ └── utils/
│ │ └── FileHelperTest.kt
│ └── utils/
│ ├── FactHelper.kt
│ └── TestRepo.kt
└── resources/
└── samples/
├── devops/
│ ├── circleci/
│ │ └── .circleci/
│ │ └── config.yml
│ ├── docker/
│ │ ├── Dockerfile
│ │ ├── alpine.Dockerfile
│ │ └── docker-compose.yml
│ ├── drone/
│ │ └── .drone.yml
│ ├── github-actions/
│ │ └── .github/
│ │ └── workflows/
│ │ └── ci.yml
│ ├── gitlab-ci/
│ │ └── .gitlab-ci.yml
│ ├── jenkins/
│ │ └── Jenkinsfile
│ ├── k8s/
│ │ ├── postgres-manifest.yaml
│ │ ├── rss-site.json
│ │ └── rss-site.yml
│ └── travis/
│ └── .travis.yml
└── langs/
├── 1C Enterprise/
│ ├── Catalog.ИсходящиеПисьма.Form.ФормаЭлемента.Form.Module.bsl
│ ├── Catalog.Товары.Command.ПечатьПрайсЛиста.CommandModule.bsl
│ ├── CommonModule.ОбменМобильныеОбщее.Module.bsl
│ ├── Document.РасходТовара.ObjectModule.bsl
│ ├── ci_before_script.os
│ └── test_canCompile.os
├── ABAP/
│ └── cl_csv_parser.abap
├── AGS Script/
│ ├── GlobalScript.asc
│ ├── GlobalScript.ash
│ ├── KeyboardMovement_102.asc
│ └── KeyboardMovement_102.ash
├── AMPL/
│ ├── CT2.mod
│ └── toy.ampl
├── API Blueprint/
│ ├── actions.apib
│ ├── attributes.apib
│ └── simple.apib
├── APL/
│ ├── DeepakChopra.apl
│ ├── UT.dyalog
│ └── hashbang
├── ASN.1/
│ └── example.asn
├── ATS/
│ ├── CoYonedaLemma.dats
│ ├── DiningPhil2.dats
│ ├── DiningPhil2.sats
│ ├── DiningPhil2_fork.dats
│ ├── DiningPhil2_thread.dats
│ ├── YonedaLemma.dats
│ ├── basis_ssntype.sats
│ ├── csv_parse.hats
│ └── intinf_vt.dats
├── ActionScript/
│ ├── FooBar.as
│ └── HelloWorld.as
├── Adobe Font Metrics/
│ ├── OpenSansCondensed-Bold.afm
│ ├── SpecialElite.afm
│ └── lambda.afm
├── Agda/
│ └── NatCat.agda
├── Alloy/
│ ├── file_system.als
│ ├── marksweepgc.als
│ └── views.als
├── Alpine Abuild/
│ └── filenames/
│ └── APKBUILD
├── AngelScript/
│ ├── botmanager.as
│ └── payload.as
├── Ant Build System/
│ └── filenames/
│ ├── ant.xml
│ └── build.xml
├── ApacheConf/
│ ├── apache.vhost
│ └── filenames/
│ ├── .htaccess
│ ├── apache2.conf
│ └── httpd.conf
├── Apex/
│ ├── ArrayUtils.cls
│ ├── BooleanUtils.cls
│ ├── EmailUtils.cls
│ ├── GeoUtils.cls
│ ├── LanguageUtils.cls
│ └── TwilioAPI.cls
├── Apollo Guidance Computer/
│ └── BURN_BABY_BURN--MASTER_IGNITION_ROUTINE.agc
├── AppleScript/
│ ├── Convert To PDF.applescript
│ ├── Convert To PostScript.applescript
│ ├── Count Messages in All Mailboxes.applescript
│ ├── Crazy Message Text.applescript
│ ├── Get User Name.applescript
│ ├── Time Of Day.applescript
│ └── center.applescript
├── AsciiDoc/
│ ├── encoding.asciidoc
│ ├── list.asc
│ └── sample.adoc
├── AspectJ/
│ ├── CacheAspect.aj
│ └── OptimizeRecursionCache.aj
├── Assembly/
│ ├── External Interrupt.a51
│ ├── FASM.asm
│ ├── forth.nasm
│ ├── fp_sqr32_160_comba.inc
│ ├── lib.inc
│ └── macros.inc
├── AutoHotkey/
│ └── hello.ahk
├── Awk/
│ └── test.awk
├── Ballerina/
│ ├── hello-world-service.bal
│ ├── hello-world.bal
│ ├── json.bal
│ ├── var.bal
│ └── xml.bal
├── BitBake/
│ ├── gstreamer-libav.bb
│ └── qtbase-native.bb
├── Blade/
│ ├── hello.blade
│ └── hello.blade.php
├── BlitzBasic/
│ ├── HalfAndDouble.bb
│ ├── LList.bb
│ └── PObj.bb
├── BlitzMax/
│ └── sample.bmx
├── Bluespec/
│ ├── TL.bsv
│ └── TbTL.bsv
├── Brainfuck/
│ ├── factor.b
│ ├── fib100.bf
│ ├── hello.bf
│ ├── helloworld.bf
│ └── rot13.bf
├── Brightscript/
│ └── SimpleGrid.brs
├── C/
│ ├── 2D.C
│ ├── 2D.H
│ ├── Arduino.cats
│ ├── ArrowLeft.h
│ ├── GLKMatrix4.h
│ ├── NWMan.h
│ ├── Nightmare.h
│ ├── array.c
│ ├── array.h
│ ├── asm.h
│ ├── bitmap.h
│ ├── blob.c
│ ├── blob.h
│ ├── bootstrap.h
│ ├── color.h
│ ├── commit.c
│ ├── commit.h
│ ├── cpuid.h
│ ├── custom_extensions.c
│ ├── driver.h
│ ├── elf.h
│ ├── exception.zep.c
│ ├── exception.zep.h
│ ├── filter.h
│ ├── fudge_node.c
│ ├── git.c
│ ├── hello.c
│ ├── hello.h
│ ├── http_parser.c
│ ├── http_parser.h
│ ├── info.h
│ ├── interface.h
│ ├── ip4.h
│ ├── jni_layer.h
│ ├── main.c
│ ├── markdown.c
│ ├── multiboot.h
│ ├── ntru_encrypt.h
│ ├── portio.h
│ ├── pqiv.h
│ ├── process.c
│ ├── rdiscount.c
│ ├── readline.cats
│ ├── redis.c
│ ├── rf_io.c
│ ├── rf_io.h
│ ├── rfc_string.c
│ ├── rfc_string.h
│ ├── scheduler.h
│ ├── script
│ ├── sgd_fast.c
│ ├── syscalldefs.h
│ ├── syscalls.h
│ ├── vfs.h
│ ├── vmem.h
│ ├── wglew.h
│ └── yajl.c
├── C#/
│ ├── AssemblyInfo.cs
│ ├── BsonPropertyValue.cs
│ ├── Index.cshtml
│ ├── MongoExpressionVisitor.cs
│ ├── Program.cs
│ └── build.cake
├── C++/
│ ├── 16F88.h
│ ├── ClasspathVMSystemProperties.inc
│ ├── CsvStreamer.h
│ ├── Entity.h
│ ├── Field.h
│ ├── Math.inl
│ ├── Memory16F88.h
│ ├── PackageInfoParser.cpp
│ ├── ThreadedQueue.h
│ ├── Types.h
│ ├── bar.h
│ ├── bar.hh
│ ├── bar.hpp
│ ├── bug1163046.--skeleton.re
│ ├── cnokw.re
│ ├── crypter.cpp
│ ├── cvsignore.re
│ ├── env.cpp
│ ├── env.h
│ ├── epoll_reactor.ipp
│ ├── gdsdbreader.h
│ ├── graphics.cpp
│ ├── grpc.pb.cc
│ ├── hello.cpp
│ ├── hello.grpc.pb.h
│ ├── hello.ino
│ ├── initClasses.inc
│ ├── instances.inc
│ ├── json_reader.cpp
│ ├── json_writer.cpp
│ ├── key.cpp
│ ├── key.h
│ ├── libcanister.h
│ ├── main.cpp
│ ├── metrics.h
│ ├── octave_changer.ino
│ ├── program.cp
│ ├── protocol-buffer.pb.cc
│ ├── protocol-buffer.pb.h
│ ├── render_adapter.cpp
│ ├── rpc.h
│ ├── scanner.cc
│ ├── scanner.h
│ ├── simple.re
│ ├── srs_app_ingest.cpp
│ ├── utils.h
│ ├── v8.cc
│ ├── v8.h
│ └── wrapper_inner.cpp
├── CLIPS/
│ ├── demo.clp
│ └── sudoku.clp
├── CMake/
│ ├── filenames/
│ │ └── CMakeLists.txt
│ ├── sample1.cmake
│ ├── sample2.cmake
│ ├── sample3.cmake
│ ├── sample4.cmake
│ ├── sample5.cmake
│ └── uninstall.cmake.in
├── COBOL/
│ ├── hello_world.cbl
│ ├── hello_world.ccp
│ ├── hello_world.cob
│ └── simple.cpy
├── CSON/
│ ├── base.cson
│ ├── config.cson
│ ├── ff-sfd.cson
│ └── wercker-status.cson
├── CSS/
│ └── bootstrap.css
├── CSV/
│ └── cars.csv
├── CWeb/
│ └── sat-life.w
├── CartoCSS/
│ └── amenity-points.mss
├── Ceylon/
│ └── Foo.ceylon
├── Chapel/
│ ├── distributions.chpl
│ ├── hello.chpl
│ ├── lulesh.chpl
│ ├── nbody.chpl
│ └── quicksort.chpl
├── Charity/
│ └── example.ch
├── Cirru/
│ ├── array.cirru
│ ├── block.cirru
│ ├── bool.cirru
│ ├── map.cirru
│ ├── number.cirru
│ ├── require.cirru
│ ├── scope.cirru
│ ├── stdio.cirru
│ └── string.cirru
├── Clarion/
│ ├── CStringClass.clw
│ ├── ConsoleSupport.clw
│ ├── HelloWorld.clw
│ └── hello.clw
├── Clean/
│ ├── GenHylo.dcl
│ ├── GenMap.dcl
│ ├── GenMap.icl
│ ├── fsieve.icl
│ ├── sem.icl
│ ├── stack.dcl
│ ├── stack.icl
│ ├── streams.dcl
│ └── streams.icl
├── Click/
│ ├── sr2.click
│ └── thomer-nat.click
├── Clojure/
│ ├── build.boot
│ ├── for.clj
│ ├── hiccup.hic
│ ├── index.cljs.hl
│ ├── into-array.cljc
│ ├── protocol.cljs
│ ├── rand.cljscm
│ ├── svg.cljx
│ └── unit-test.cl2
├── Closure Templates/
│ └── example.soy
├── CoNLL-U/
│ ├── CF1.conllu
│ ├── en-ud-test-abridged.conllu
│ └── ug-ud-test-abridged.conllu
├── CoffeeScript/
│ ├── browser.coffee
│ ├── build.cake
│ ├── classes.coffee
│ ├── coffee-script.coffee
│ ├── example.cjsx
│ ├── hello.coffee
│ ├── intro.coffee
│ ├── lexer.coffee
│ ├── rack_application.coffee
│ └── xipd.coffee
├── ColdFusion/
│ └── example.cfm
├── ColdFusion CFC/
│ ├── exampleScript.cfc
│ └── exampleTag.cfc
├── Common Lisp/
│ ├── array.l
│ ├── common.l
│ ├── config.sexp
│ ├── hello.lisp
│ ├── macros-advanced.cl
│ ├── motor-inferencia.cl
│ ├── rss.sexp
│ ├── sample.lisp
│ └── sample.lsp
├── Common Workflow Language/
│ └── trunk-peak-score.cwl
├── Component Pascal/
│ ├── Example.cp
│ └── Example2.cps
├── Cool/
│ ├── list.cl
│ └── sample.cl
├── Coq/
│ ├── Computation.v
│ ├── Imp.v
│ ├── JsCorrectness.v
│ ├── JsInterpreterExtraction.v
│ ├── JsNumber.v
│ ├── JsPrettyInterm.v
│ ├── Lists.v
│ ├── Main.v
│ ├── Poly.v
│ ├── Rel.v
│ ├── Smallstep.v
│ ├── Spec.v
│ └── Stlc.v
├── Creole/
│ └── creole.creole
├── Crystal/
│ ├── const_spec.cr
│ ├── declare_var_spec.cr
│ └── transformer.cr
├── Csound/
│ ├── allglass.orc
│ ├── interp.orc
│ └── test.orc
├── Csound Document/
│ ├── allglass.csd
│ ├── interp.csd
│ └── test.csd
├── Csound Score/
│ ├── allglass.sco
│ ├── interp.sco
│ └── test.sco
├── Cuda/
│ ├── scalarProd_kernel.cuh
│ └── vectorAdd.cu
├── Cycript/
│ └── utils.cy
├── D/
│ ├── aa.d
│ ├── arrayops.d
│ ├── function.d
│ ├── hello_world.d
│ ├── mpq.d
│ ├── template.d
│ ├── template_function.d
│ ├── unittest1.d
│ └── unittest2.d
├── DIGITAL Command Language/
│ ├── fis_gtm_kitinstal.com
│ ├── ghostpdl_zlib_make_vms.com
│ ├── libxslt_build.com
│ └── vmsbackup_build.com
├── DM/
│ └── example.dm
├── DNS Zone/
│ ├── sample.arpa
│ └── sneaky.net.zone
├── DOS Batch/
│ └── BatchInstall.bat
├── DTrace/
│ ├── counts.d
│ ├── probes.d
│ └── trace_futexes.d
├── Dart/
│ └── point.dart
├── DataWeave/
│ ├── customInterpolator.dwl
│ ├── directives.dwl
│ ├── functions.dwl
│ ├── literals.dwl
│ └── match.dwl
├── Diff/
│ └── dude-thing-okay--001.patch
├── Dockerfile/
│ └── filenames/
│ └── Dockerfile
├── Dogescript/
│ └── example.djs
├── E/
│ ├── Extends.E
│ ├── Functions.E
│ ├── Guards.E
│ ├── IO.E
│ ├── Promises.E
│ ├── atomic-updates.E
│ └── minChat.E
├── EBNF/
│ ├── grammar.ebnf
│ ├── material.ebnf
│ ├── object.ebnf
│ └── types.ebnf
├── ECL/
│ └── sample.ecl
├── ECLiPSe/
│ └── or-constraint.ecl
├── EJS/
│ ├── dash.ejs
│ └── page.ejs
├── EQ/
│ ├── HTTPServerVirtualHostListener.eq
│ ├── SEButtonEntity.eq
│ └── String.eq
├── Eagle/
│ ├── Eagle.brd
│ └── Eagle.sch
├── Easybuild/
│ └── bzip2-1.0.6-GCC-4.9.2.eb
├── Edje Data Collection/
│ └── mild.edc
├── Eiffel/
│ ├── application.e
│ ├── book_collection.e
│ └── git_checkout_command.e
├── Elixir/
│ ├── hello_world.exs
│ └── hello_world_test.exs
├── Elm/
│ ├── Basic.elm
│ ├── QuickSort.elm
│ └── Tree.elm
├── Emacs Lisp/
│ ├── .emacs.desktop
│ ├── dude.el
│ ├── ess-julia.el
│ └── filenames/
│ ├── .abbrev_defs
│ ├── .gnus
│ ├── .spacemacs
│ ├── .viper
│ ├── Cask
│ ├── Project.ede
│ ├── _emacs
│ └── abbrev_defs
├── EmberScript/
│ └── momentComponent.em
├── Erlang/
│ ├── 170-os-daemons.es
│ ├── elixir_parser.yrl
│ ├── factorial
│ ├── filenames/
│ │ ├── Emakefile
│ │ └── rebar.config
│ ├── hello.escript
│ ├── lfe_scan.xrl
│ ├── record_helper.erl
│ ├── record_utils.erl
│ ├── release
│ ├── sample.app.src
│ └── single-context.es
├── F#/
│ ├── Combinators.fs
│ ├── JsonFormat.fs
│ ├── JsonReader.fs
│ ├── JsonSerializer.fs
│ ├── JsonWriter.fs
│ ├── PerformanceTesters.fs
│ ├── PerformanceTests.fs
│ └── sample.fs
├── FLUX/
│ ├── gameserver.fx
│ ├── imageserver.fx
│ ├── mbittorrent.fx
│ └── test.fx
├── Factor/
│ └── hello-world.factor
├── Fantom/
│ ├── sample1.fan
│ └── sample2.fan
├── Filebench WML/
│ └── copyfiles.f
├── Filterscript/
│ ├── colormatrix.fs
│ └── fs_kernel.fs
├── Formatted/
│ ├── NiAlH_jea.eam.fs
│ ├── long_seq.for
│ └── wksst8110.for
├── Forth/
│ ├── KataDiversion.fth
│ ├── asm.fr
│ ├── bitmap.frt
│ ├── block.fth
│ ├── core-ext.fth
│ ├── core.f
│ ├── core.for
│ ├── core.fs
│ ├── core.fth
│ ├── core1.F
│ ├── enum.frt
│ ├── hello-forth.forth
│ ├── hello-forth.fth
│ ├── macros.frt
│ ├── tools.4TH
│ └── tools.fth
├── Fortran/
│ ├── bug-185631.f
│ ├── m_filt2d.F90
│ ├── sample1.f
│ ├── sample1.for
│ ├── sample2.f
│ ├── sample3.F
│ ├── tst.f03
│ ├── tst.f08
│ ├── tst.f15
│ └── tst.f95
├── FreeMarker/
│ ├── example.ftl
│ └── layout.ftl
├── Frege/
│ ├── CommandLineClock.fr
│ ├── Concurrent.fr
│ ├── Sudoku.fr
│ └── SwingExamples.fr
├── G-code/
│ ├── duettest.g
│ └── square.g
├── GAMS/
│ └── transport.gms
├── GAP/
│ ├── Magic.gd
│ ├── Magic.gi
│ ├── PackageInfo.g
│ ├── bugfix.tst
│ ├── example.gd
│ ├── example.gi
│ ├── factor.tst
│ ├── vspc.gd
│ └── vspc.gi
├── GCC Machine Description/
│ └── pdp10.md
├── GDB/
│ ├── as3.gdbinit
│ └── gdb_lpc17xx_program.gdb
├── GDScript/
│ ├── example.gd
│ ├── grid.gd
│ ├── player.gd
│ └── pong.gd
├── GLSL/
│ ├── SimpleLighting.gl2.frag
│ ├── SyLens.glsl
│ ├── SyLens.shader
│ ├── gbuffers_textured_lit.fsh
│ ├── gbuffers_textured_lit.vsh
│ ├── islandScene.glsl
│ ├── islandScene.shader
│ ├── myfragment.frg
│ ├── myvertex.vrx
│ ├── pntriangles.tesc
│ ├── pntriangles.tese
│ ├── recurse1.frag
│ ├── recurse1.fs
│ └── shader.fp
├── GN/
│ ├── BUILD.2.gn
│ ├── BUILD.3.gn
│ ├── BUILD.gn
│ ├── android-rules.gni
│ ├── clang.gni
│ ├── filenames/
│ │ └── .gn
│ ├── gcc_toolchain.gni
│ ├── icu.gn
│ ├── internal_rules.gni
│ ├── ios-rules.gni
│ └── isolate.gni
├── Game Maker Language/
│ ├── GMLmenus.gml
│ ├── _piwikCacheRequest.gml
│ ├── _piwikSendBasicReq.gml
│ ├── _piwikSendReq.gml
│ ├── characterDrawEvent.gml
│ ├── characterStepEvent.gml
│ ├── draw_menu.gml
│ ├── faucet-http.gml
│ ├── jsonion.gml
│ ├── jsonion_test.gml
│ └── scrInitLevel.gml
├── Genie/
│ ├── Class.gs
│ └── Hello.gs
├── Gerber Image/
│ ├── FelinaePurr-B.Cu.gbl
│ ├── FelinaePurr-B.Mask.gbs
│ ├── FelinaePurr-B.SilkS.gbo
│ ├── FelinaePurr-F.Cu.gtl
│ ├── FelinaePurr-F.Mask.gts
│ ├── FelinaePurr-F.Paste.gtp
│ ├── FelinaePurr-F.SilkS.gto
│ ├── GOLMain_RevA-B.Paste.gbp
│ ├── nonaprs_contour.gko
│ ├── simonShield-B.Cu.gbr
│ ├── simonShield-B.Mask.gbr
│ ├── simonShield-Edge.Cuts.gbr
│ ├── simonShield-F.Cu.gbr
│ ├── simonShield-F.Mask.gbr
│ ├── simonShield-F.SilkS.gbr
│ └── simonShield-drl_map.gbr
├── Gnuplot/
│ ├── dashcolor.1.gnu
│ ├── histograms.2.gnu
│ ├── rates.gp
│ ├── surface1.16.gnu
│ ├── surface1.17.gnu
│ └── world2.1.gnu
├── Go/
│ ├── api.pb.go
│ ├── embedded.go
│ └── gen-go-linguist-thrift.go
├── Golo/
│ ├── adapters.golo
│ ├── async.golo
│ ├── augmentations.golo
│ ├── closures.golo
│ ├── coin-change.golo
│ ├── collection-literals.golo
│ ├── context-decorator.golo
│ ├── decorators.golo
│ ├── dynamic-evaluation.golo
│ ├── dynamic-object-person.golo
│ ├── echo-args.golo
│ ├── enums-thread-state.golo
│ ├── fibonacci.golo
│ ├── helloworld.golo
│ ├── http-server.golo
│ ├── logdeco.golo
│ ├── matching-operator.golo
│ ├── max-int.golo
│ ├── memoize.golo
│ ├── null-safety.golo
│ ├── prepost-decorators.golo
│ ├── structs.golo
│ ├── swing-actionlistener.golo
│ ├── swing-helloworld.golo
│ ├── templates-chat-webapp.golo
│ ├── util-containers.golo
│ └── workers.golo
├── Gosu/
│ ├── Hello.gst
│ ├── Hello.gsx
│ ├── Person.gs
│ ├── Ronin.gs
│ └── hello.vark
├── Grace/
│ ├── ackerman_function.grace
│ └── grace_IDE.grace
├── Gradle/
│ ├── build.gradle
│ └── builder.gradle
├── Grammatical Framework/
│ ├── Foods.gf
│ ├── FoodsAfr.gf
│ ├── FoodsAmh.gf
│ ├── FoodsBul.gf
│ ├── FoodsCat.gf
│ ├── FoodsChi.gf
│ ├── FoodsCze.gf
│ ├── FoodsDut.gf
│ ├── FoodsEng.gf
│ ├── FoodsEpo.gf
│ ├── FoodsFin.gf
│ ├── FoodsFre.gf
│ ├── FoodsGer.gf
│ ├── FoodsHeb.gf
│ ├── FoodsHin.gf
│ ├── FoodsI.gf
│ ├── FoodsIce.gf
│ ├── FoodsIta.gf
│ ├── FoodsJpn.gf
│ ├── FoodsLav.gf
│ ├── FoodsMlt.gf
│ ├── FoodsMon.gf
│ ├── FoodsNep.gf
│ ├── FoodsOri.gf
│ ├── FoodsPes.gf
│ ├── FoodsPor.gf
│ ├── FoodsRon.gf
│ ├── FoodsSpa.gf
│ ├── FoodsSwe.gf
│ ├── FoodsTha.gf
│ ├── FoodsTsn.gf
│ ├── FoodsTur.gf
│ ├── FoodsUrd.gf
│ ├── LexFoods.gf
│ ├── LexFoodsCat.gf
│ ├── LexFoodsFin.gf
│ ├── LexFoodsGer.gf
│ ├── LexFoodsIta.gf
│ ├── LexFoodsSwe.gf
│ ├── ResCze.gf
│ └── transFoodsHin.gf
├── Graph Modeling Language/
│ └── sample.gml
├── GraphQL/
│ ├── kitchen-sink.graphql
│ └── schema-kitchen-sink.graphql
├── Graphviz (DOT)/
│ ├── annoying.DOT
│ └── sample.dot
├── Groovy/
│ ├── build.gvy
│ ├── filenames/
│ │ └── Jenkinsfile
│ ├── groovy
│ ├── script.gvy
│ ├── template.grt
│ └── template.gtpl
├── Groovy Server Pages/
│ ├── bar.gsp
│ ├── hello-pagedirective.gsp
│ ├── hello-resources.gsp
│ └── hello-var.gsp
├── HCL/
│ ├── example.hcl
│ ├── example.tf
│ ├── main.tf
│ └── terraform.tfvars
├── HLSL/
│ ├── accelerated_surface_win.hlsl
│ ├── bloom.cginc
│ ├── corridor.fx
│ ├── jellyfish.fx
│ └── noise.fx
├── HTML/
│ ├── ApiOverviewPage.st
│ ├── example.xht
│ ├── index.html.hl
│ ├── pages.html
│ ├── rpanel.inc
│ └── tailDel.inc
├── HTML+Django/
│ ├── _worker.jinja2
│ └── nunjucks.njk
├── HTML+ECR/
│ └── greeting.ecr
├── HTML+EEX/
│ └── index.html.eex
├── HTML+ERB/
│ ├── fishbowl.html.erb.deface
│ └── index.html.erb
├── HXML/
│ ├── checkstyle.hxml
│ └── vshaxe.hxml
├── Hack/
│ ├── Assert.hh
│ ├── AssertRecipe.hh
│ ├── Controller.hh
│ ├── DBResultRecipe.hh
│ ├── Documentation.hh
│ ├── FakeDB.hh
│ ├── GetAndPostRecipe.hh
│ ├── GetController.hh
│ ├── HomeController.hh
│ ├── MySecureRequest.hh
│ ├── Nav.hh
│ ├── NonStrictFile.hh
│ ├── Recipe.hh
│ ├── RecipeWithDemo.hh
│ ├── Request.hh
│ ├── StandardPage.hh
│ ├── StrictFile.hh
│ ├── UnescapedString.hh
│ ├── UnescapedStringRecipe.hh
│ ├── UserID.hh
│ ├── UserIDRecipe.hh
│ ├── UsingUserID.hh
│ ├── error.hh
│ ├── funs.hh
│ ├── funs.php
│ ├── index.hh
│ ├── phpfile.hh
│ └── startup.hh
├── Haml/
│ ├── buttons.html.haml.deface
│ └── hello.haml
├── Handlebars/
│ ├── basic.handlebars
│ └── each.hbs
├── Haskell/
│ ├── Hello.hs
│ ├── HsColour.hs
│ ├── Main.hs
│ ├── Sudoku.hs
│ └── maze-solving.hs
├── Haxe/
│ └── HaxeExamples.hx
├── Hy/
│ ├── fibonacci.hy
│ └── hello-world.hy
├── HyPhy/
│ ├── AAModelComparison.bf
│ ├── CodonModelCompare.bf
│ ├── MFPositiveSelection.bf
│ ├── MatrixIndexing.bf
│ ├── MolecularClock.bf
│ ├── dNdSDistributionComparison.bf
│ ├── hyphy_cmds.bf
│ └── profile_test.bf
├── IDL/
│ ├── mg_acosh.pro
│ ├── mg_analysis.dlm
│ ├── mg_gcd.pro
│ └── mg_trunc.pro
├── IGOR Pro/
│ ├── functions.ipf
│ └── generic.ipf
├── INI/
│ ├── MouseKeyboard.pro
│ ├── filenames/
│ │ ├── .editorconfig
│ │ ├── .gitconfig
│ │ └── buildozer.spec
│ └── ultimate-temp-controller.pro
├── Idris/
│ └── Chars.idr
├── Inform 7/
│ ├── Trivial Extension.i7x
│ └── story.ni
├── Inno Setup/
│ └── expat.iss
├── Ioke/
│ └── hello.ik
├── Isabelle/
│ └── HelloWorld.thy
├── Isabelle ROOT/
│ └── filenames/
│ └── ROOT
├── J/
│ ├── hashbang
│ └── stwij.ijs
├── JFlex/
│ ├── LexScan.flex
│ └── java.jflex
├── JSON/
│ ├── Git Commit.JSON-tmLanguage
│ ├── Material_Alpha_01.gltf
│ ├── filenames/
│ │ ├── .arcconfig
│ │ ├── .htmlhintrc
│ │ ├── .jscsrc
│ │ ├── .tern-config
│ │ ├── .tern-project
│ │ └── mcmod.info
│ ├── geo.geojson
│ ├── http_response.avsc
│ ├── manifest.webapp
│ ├── manifest.webmanifest
│ ├── person.json
│ ├── product.json
│ ├── schema.json
│ ├── small.tfstate
│ ├── switzerland.topojson
│ └── terraform.tfstate.backup
├── JSON5/
│ ├── example.json5
│ ├── filenames/
│ │ ├── .babelrc
│ │ └── .jslintrc
│ └── package.json5
├── JSONLD/
│ └── sample.jsonld
├── JSONiq/
│ ├── detail.jq
│ └── query.jq
├── JSX/
│ └── sample.jsx
├── Jasmin/
│ ├── if1.j
│ ├── if2.j
│ ├── if3.j
│ ├── if4.j
│ ├── op1.j
│ ├── op2.j
│ ├── op3.j
│ └── op4.j
├── Java/
│ ├── GrammarKit.java
│ ├── HtmlDomParserContext.java
│ ├── Hudson.java
│ ├── JFlexLexer.java
│ ├── NokogiriService.java
│ ├── ProtocolBuffer.java
│ ├── clojure-type.java
│ ├── clojure-util.java
│ └── gen-java-linguist-thrift.java
├── JavaScript/
│ ├── axios.es
│ ├── basic.svelte
│ ├── basic.vue
│ ├── bootstrap-modal.js
│ ├── ccalc-lex.js
│ ├── ccalc-parse.js
│ ├── chart_composers.gs
│ ├── classes-old.js
│ ├── classes.js
│ ├── constant_fold.mjs
│ ├── dude.js
│ ├── entry.mjs
│ ├── gen-js-linguist-thrift.js
│ ├── hello.js
│ ├── helloHanaEndpoint.xsjs
│ ├── helloHanaMath.xsjslib
│ ├── http.js
│ ├── index.es
│ ├── intro-old.js
│ ├── intro.js
│ ├── intro.js.frag
│ ├── itau.gs
│ ├── jquery-1.6.1.js
│ ├── jquery-1.7.2.js
│ ├── js
│ ├── js2
│ ├── jsbuild.jsb
│ ├── json2_backbone.js
│ ├── logo.jscad
│ ├── modernizr.js
│ ├── module.mjs
│ ├── namespace.js
│ ├── outro.js.frag
│ ├── parser.js
│ ├── pre-processors.vue
│ ├── proto.js
│ ├── steelseries-min.js
│ └── uglify.js
├── Jison/
│ ├── ansic.jison
│ ├── classy.jison
│ └── lex.jison
├── Jison Lex/
│ ├── classy.jisonlex
│ └── lex_grammar.jisonlex
├── Jolie/
│ ├── common.iol
│ ├── exam.ol
│ ├── examiner.ol
│ ├── hanoi.ol
│ └── student.ol
├── Julia/
│ ├── julia
│ └── stockcorr.jl
├── KRL/
│ └── helloworld.krl
├── KiCad Layout/
│ ├── C_Disc_D3_P2.5.kicad_mod
│ ├── Conn_Poncho_SinBorde.kicad_mod
│ ├── Fiducial_1mm.kicad_mod
│ ├── LED-5MM.kicad_mod
│ ├── Logo_OSHWA.kicad_mod
│ ├── Logo_Poncho.kicad_mod
│ ├── MagneticBuzzer_ProSignal_ABT-410-RC.kicad_mod
│ ├── Pin_Header_Straight_2x02.kicad_mod
│ ├── Pin_Header_Straight_2x20.kicad_mod
│ ├── Resistor_Horizontal_RM7mm.kicad_mod
│ ├── SW_PUSH_SMALL.kicad_mod
│ ├── TO-92_Molded_Narrow.kicad_mod
│ ├── filenames/
│ │ └── fp-lib-table
│ ├── kivicad.kicad_wks
│ ├── nrf-bga.kicad_pcb
│ └── simonShield.kicad_pcb
├── KiCad Legacy Layout/
│ └── tc14badge.brd
├── KiCad Schematic/
│ ├── Volume.sch
│ ├── buttons.sch
│ ├── buzzer.sch
│ ├── ciaaConector.sch
│ ├── gedda-junk.sch
│ └── ultimate-temp-controller.sch
├── Kit/
│ └── demo.kit
├── Kotlin/
│ └── Foo.kt
├── LFE/
│ ├── church.lfe
│ ├── gps1.lfe
│ ├── mnesia_demo.lfe
│ └── object.lfe
├── LOLCODE/
│ └── LOLTracer.lol
├── LSL/
│ ├── LSL.lsl
│ └── LSL.lslp
├── Lasso/
│ ├── json.lasso
│ ├── json.lasso9
│ ├── knop.las
│ └── knop.ldml
├── Latte/
│ ├── layout.latte
│ └── template.latte
├── Lean/
│ ├── binary.lean
│ └── set.hlean
├── Less/
│ └── screen.less
├── Lex/
│ └── zend_ini_scanner.l
├── Limbo/
│ ├── cat.b
│ ├── lock.b
│ └── lock.m
├── Linker Script/
│ ├── filenames/
│ │ └── ld.script
│ ├── inject.x
│ ├── link.ld
│ └── vmlinux.lds
├── Linux Kernel Module/
│ ├── bcm4334x.mod
│ ├── mbcache.mod
│ └── md5.mod
├── Liquid/
│ ├── layout.liquid
│ └── template.liquid
├── Literate Agda/
│ └── NatCat.lagda
├── Literate CoffeeScript/
│ └── scope.litcoffee
├── LiveScript/
│ └── hello.ls
├── Logos/
│ ├── NCHax.x
│ ├── NoCarrier.x
│ ├── Tweak.x
│ ├── example.xm
│ └── string1.x
├── Logtalk/
│ └── foo.lgt
├── LookML/
│ ├── comments.view.lookml
│ ├── example.model.lkml
│ └── example.view.lkml
├── LoomScript/
│ ├── HelloWorld.ls
│ └── SyntaxExercise.ls
├── Lua/
│ ├── h-counter.pd_lua
│ ├── treegen.p8
│ ├── vidya-file-list-parser.pd_lua
│ ├── vidya-file-modder.pd_lua
│ └── wsapi.fcgi
├── M/
│ ├── Comment.m
│ ├── GMRGPNB0.m
│ ├── MDB.m
│ ├── PRCAAPR.m
│ ├── PXAI.m
│ ├── WVBRNOT.m
│ ├── ZDIOUT1.m
│ ├── _zewdAPI.m
│ ├── _zewdDemo.m
│ ├── arrays.m
│ ├── base64.m
│ ├── digest.m
│ ├── dynamicscoping.m
│ ├── fibonacci.m
│ ├── forloop.m
│ ├── functions.m
│ ├── helloworld.m
│ ├── ifelse.m
│ ├── indirectfunctions.m
│ ├── md5.m
│ ├── mileage.m
│ ├── mumtris.m
│ ├── nesting.m
│ ├── pcre.m
│ ├── pcreexamples.m
│ ├── postconditional.m
│ ├── primes.m
│ ├── url.m
│ └── zmwire.m
├── M4/
│ └── htmlgen.m4
├── M4Sugar/
│ ├── ax_ruby_devel.m4
│ ├── filenames/
│ │ └── configure.ac
│ └── list.m4
├── MAXScript/
│ ├── macro-1.mcr
│ ├── macro-2.mcr
│ ├── rolloutCreator.ms
│ ├── svg-renderer.ms
│ └── volume-calc.ms
├── MQL4/
│ ├── header-sample.mqh
│ ├── indicator-sample.mq4
│ └── script-sample.mq4
├── MQL5/
│ ├── Regex.mqh
│ ├── indicator-sample.mq5
│ └── script-sample.mq5
├── MTML/
│ └── categories_to_columns.mtml
├── MUF/
│ ├── 39.m
│ └── cmd-say.muf
├── Makefile/
│ ├── file-icons.make
│ ├── filenames/
│ │ ├── BSDmakefile
│ │ ├── Kbuild
│ │ ├── Makefile
│ │ ├── Makefile.boot
│ │ ├── Makefile.frag
│ │ ├── Makefile.inc
│ │ ├── Makefile.wat
│ │ ├── makefile.sco
│ │ └── mkfile
│ ├── foo.o.d
│ └── makefile
├── Markdown/
│ ├── README.mdown
│ ├── csharp6.workbook
│ ├── minimal.md
│ ├── symlink.md
│ └── tender.md
├── Marko/
│ ├── counter.marko
│ ├── hello.marko
│ └── rgb-sliders.marko
├── Mask/
│ └── view.mask
├── Mathematica/
│ ├── HeyexImport.m
│ ├── Init.m
│ ├── MiscCalculations.nb
│ ├── MiscCalculations2.nb
│ ├── PacletInfo.m
│ ├── Predicates.m
│ ├── Predicates.wl
│ ├── Problem12.m
│ ├── TestArithmetic.mt
│ ├── TestString.mt
│ ├── TestSuite.mt
│ └── UnitTest.wlt
├── Matlab/
│ ├── CVX/
│ │ ├── Contents.m
│ │ ├── builtins/
│ │ │ ├── @cvx/
│ │ │ │ ├── abs.m
│ │ │ │ ├── blkdiag.m
│ │ │ │ ├── builtins.m
│ │ │ │ ├── cat.m
│ │ │ │ ├── conj.m
│ │ │ │ ├── conv.m
│ │ │ │ ├── ctranspose.m
│ │ │ │ ├── cumprod.m
│ │ │ │ ├── cumsum.m
│ │ │ │ ├── diag.m
│ │ │ │ ├── disp.m
│ │ │ │ ├── end.m
│ │ │ │ ├── eq.m
│ │ │ │ ├── exp.m
│ │ │ │ ├── find.m
│ │ │ │ ├── full.m
│ │ │ │ ├── ge.m
│ │ │ │ ├── gt.m
│ │ │ │ ├── hankel.m
│ │ │ │ ├── horzcat.m
│ │ │ │ ├── imag.m
│ │ │ │ ├── ipermute.m
│ │ │ │ ├── isreal.m
│ │ │ │ ├── kron.m
│ │ │ │ ├── ldivide.m
│ │ │ │ ├── le.m
│ │ │ │ ├── log.m
│ │ │ │ ├── lt.m
│ │ │ │ ├── max.m
│ │ │ │ ├── min.m
│ │ │ │ ├── minus.m
│ │ │ │ ├── mldivide.m
│ │ │ │ ├── mpower.m
│ │ │ │ ├── mrdivide.m
│ │ │ │ ├── mtimes.m
│ │ │ │ ├── ne.m
│ │ │ │ ├── nnz.m
│ │ │ │ ├── norm.m
│ │ │ │ ├── permute.m
│ │ │ │ ├── plus.m
│ │ │ │ ├── polyval.m
│ │ │ │ ├── power.m
│ │ │ │ ├── prod.m
│ │ │ │ ├── rdivide.m
│ │ │ │ ├── real.m
│ │ │ │ ├── reshape.m
│ │ │ │ ├── size.m
│ │ │ │ ├── sparse.m
│ │ │ │ ├── spy.m
│ │ │ │ ├── sqrt.m
│ │ │ │ ├── std.m
│ │ │ │ ├── subsasgn.m
│ │ │ │ ├── subsref.m
│ │ │ │ ├── sum.m
│ │ │ │ ├── times.m
│ │ │ │ ├── toeplitz.m
│ │ │ │ ├── transpose.m
│ │ │ │ ├── tril.m
│ │ │ │ ├── triu.m
│ │ │ │ ├── uminus.m
│ │ │ │ ├── uplus.m
│ │ │ │ ├── var.m
│ │ │ │ └── vertcat.m
│ │ │ ├── @cvxcnst/
│ │ │ │ ├── eq.m
│ │ │ │ ├── ge.m
│ │ │ │ ├── gt.m
│ │ │ │ ├── le.m
│ │ │ │ ├── lt.m
│ │ │ │ └── ne.m
│ │ │ └── Contents.m
│ │ ├── commands/
│ │ │ ├── @cvx/
│ │ │ │ └── commands.m
│ │ │ ├── Contents.m
│ │ │ ├── cvx_begin.m
│ │ │ ├── cvx_clear.m
│ │ │ ├── cvx_end.m
│ │ │ ├── cvx_expert.m
│ │ │ ├── cvx_pause.m
│ │ │ ├── cvx_power_warning.m
│ │ │ ├── cvx_precision.m
│ │ │ ├── cvx_profile.m
│ │ │ ├── cvx_quiet.m
│ │ │ ├── cvx_save_prefs.m
│ │ │ ├── cvx_solver.m
│ │ │ ├── cvx_solver_settings.m
│ │ │ ├── cvx_tic.m
│ │ │ ├── cvx_toc.m
│ │ │ └── cvx_where.m
│ │ ├── cvx_error.m
│ │ ├── cvx_setup.m
│ │ ├── cvx_startup.m
│ │ ├── cvx_version.m
│ │ ├── examples/
│ │ │ ├── Contents.m
│ │ │ ├── antenna_array_design/
│ │ │ │ ├── Contents.m
│ │ │ │ ├── ant_array_min_beamwidth.m
│ │ │ │ ├── ant_array_min_sidelobe.m
│ │ │ │ ├── ant_array_min_therm_noise.m
│ │ │ │ ├── broadband_array_min_sidelobe.m
│ │ │ │ ├── line_array_spec_fact.m
│ │ │ │ ├── polar_plot_ant.m
│ │ │ │ └── spectral_fact.m
│ │ │ ├── circuit_design/
│ │ │ │ ├── Contents.m
│ │ │ │ ├── LC_osc_design.m
│ │ │ │ ├── clock_mesh.m
│ │ │ │ ├── dig_ckt_sizing.m
│ │ │ │ ├── elmore_straight_wire.m
│ │ │ │ ├── inverter_chain_sizing.m
│ │ │ │ ├── plot_four_tapers.m
│ │ │ │ ├── simple_NAND2_gate_design.m
│ │ │ │ ├── simple_step.m
│ │ │ │ ├── tristate_bus_sizing.m
│ │ │ │ ├── wire_driver_sizing.m
│ │ │ │ ├── wire_sizing.m
│ │ │ │ ├── wire_sizing_spacing.m
│ │ │ │ └── wire_sizing_topology.m
│ │ │ ├── closest_toeplitz_psd.m
│ │ │ ├── cvxbook/
│ │ │ │ ├── Ch04_cvx_opt_probs/
│ │ │ │ │ ├── Contents.m
│ │ │ │ │ ├── cantilever_beam.m
│ │ │ │ │ ├── cantilever_beam_plot.m
│ │ │ │ │ ├── cantilever_beam_rec.m
│ │ │ │ │ ├── channel_capacity.m
│ │ │ │ │ ├── chebyshev_center.m
│ │ │ │ │ ├── chebyshev_center_2D.m
│ │ │ │ │ ├── ex_4_27.m
│ │ │ │ │ ├── ex_4_3.m
│ │ │ │ │ ├── ex_4_38.m
│ │ │ │ │ ├── ex_4_5.m
│ │ │ │ │ ├── fastest_mixing_MC.m
│ │ │ │ │ ├── frob_norm_diag_scaling.m
│ │ │ │ │ ├── logopt_investment.m
│ │ │ │ │ ├── max_det_psd_completion.m
│ │ │ │ │ └── min_spec_rad_ppl_dynamics.m
│ │ │ │ ├── Ch05_duality/
│ │ │ │ │ ├── Contents.m
│ │ │ │ │ ├── ex_5_1.m
│ │ │ │ │ ├── ex_5_19.m
│ │ │ │ │ ├── ex_5_33.m
│ │ │ │ │ ├── ex_5_39.m
│ │ │ │ │ ├── matrix_games.m
│ │ │ │ │ ├── matrix_games_LP.m
│ │ │ │ │ ├── norm_approx.m
│ │ │ │ │ └── qcqp.m
│ │ │ │ ├── Ch06_approx_fitting/
│ │ │ │ │ ├── Contents.m
│ │ │ │ │ ├── basispursuit.m
│ │ │ │ │ ├── convex_interpolation.m
│ │ │ │ │ ├── deadzone.m
│ │ │ │ │ ├── fig6_15.m
│ │ │ │ │ ├── fig6_19.m
│ │ │ │ │ ├── fig6_20.m
│ │ │ │ │ ├── fig6_5.m
│ │ │ │ │ ├── fig6_6.m
│ │ │ │ │ ├── fig6_9.m
│ │ │ │ │ ├── penalty_comp_cvx.m
│ │ │ │ │ ├── preference_regions.m
│ │ │ │ │ ├── regressor_cvx.m
│ │ │ │ │ ├── smoothrec_cvx.m
│ │ │ │ │ ├── tv_cvx.m
│ │ │ │ │ └── wcrobls.m
│ │ │ │ ├── Ch07_statistical_estim/
│ │ │ │ │ ├── Contents.m
│ │ │ │ │ ├── ML_covariance_est.m
│ │ │ │ │ ├── cheb.m
│ │ │ │ │ ├── cher.m
│ │ │ │ │ ├── counting_problem_poisson.m
│ │ │ │ │ ├── detector2.m
│ │ │ │ │ ├── expdesign.m
│ │ │ │ │ ├── logistics.m
│ │ │ │ │ ├── logistics_gp.m
│ │ │ │ │ ├── maxent.m
│ │ │ │ │ ├── montecarlo.m
│ │ │ │ │ └── probbounds.m
│ │ │ │ ├── Ch08_geometric_probs/
│ │ │ │ │ ├── Contents.m
│ │ │ │ │ ├── analytic_center.m
│ │ │ │ │ ├── data_floorplan_32.mat
│ │ │ │ │ ├── data_floorplan_60.mat
│ │ │ │ │ ├── eucl_dist_poly.m
│ │ │ │ │ ├── eucl_dist_poly_2D.m
│ │ │ │ │ ├── eucl_proj_cone1.m
│ │ │ │ │ ├── eucl_proj_cone2.m
│ │ │ │ │ ├── eucl_proj_hlf.m
│ │ │ │ │ ├── eucl_proj_hyp.m
│ │ │ │ │ ├── eucl_proj_rect.m
│ │ │ │ │ ├── ex_8_3.m
│ │ │ │ │ ├── ex_8_4.m
│ │ │ │ │ ├── ex_8_5.m
│ │ │ │ │ ├── floor_plan.m
│ │ │ │ │ ├── floor_plan_graphs.m
│ │ │ │ │ ├── floorplan.m
│ │ │ │ │ ├── linear_discr.m
│ │ │ │ │ ├── max_vol_ellip_in_polyhedra.m
│ │ │ │ │ ├── min_vol_elp_finite_set.m
│ │ │ │ │ ├── min_vol_union_ellip.m
│ │ │ │ │ ├── placement_lin.m
│ │ │ │ │ ├── placement_quad.m
│ │ │ │ │ ├── placement_quar.m
│ │ │ │ │ ├── poly3_discr.m
│ │ │ │ │ ├── poly4_discr.m
│ │ │ │ │ ├── quad_discr.m
│ │ │ │ │ ├── robust_lin_discr.m
│ │ │ │ │ ├── separate_ell_2D.m
│ │ │ │ │ ├── separate_poly_2D.m
│ │ │ │ │ ├── separate_pt_poly.m
│ │ │ │ │ ├── svm_1.m
│ │ │ │ │ ├── svm_2.m
│ │ │ │ │ └── test_floorplan.m
│ │ │ │ ├── Ch11_intpt_methods/
│ │ │ │ │ ├── Contents.m
│ │ │ │ │ └── log_utility_flow.m
│ │ │ │ └── Contents.m
│ │ │ ├── equality_constr_norm_min.m
│ │ │ ├── filter_design/
│ │ │ │ ├── Contents.m
│ │ │ │ ├── equalizer_design.m
│ │ │ │ ├── fir_chebychev_design.m
│ │ │ │ ├── fir_lin_phase_lowpass_max_atten.m
│ │ │ │ ├── fir_lin_phase_lowpass_min_order.m
│ │ │ │ ├── fir_lin_phase_lowpass_min_ripple.m
│ │ │ │ ├── fir_lin_phase_lowpass_min_trans.m
│ │ │ │ ├── fir_mag_design_lowpass_max_atten.m
│ │ │ │ ├── fir_mag_design_lowpass_min_order.m
│ │ │ │ ├── iir_mag_design_bandpass_max_atten.m
│ │ │ │ ├── iir_mag_design_lowpass_max_atten.m
│ │ │ │ ├── one_over_f_filter.m
│ │ │ │ └── spectral_fact.m
│ │ │ ├── gp_tutorial/
│ │ │ │ ├── Contents.m
│ │ │ │ ├── basic_odp.m
│ │ │ │ ├── beta_min_odp.m
│ │ │ │ ├── elmore_interconnect.m
│ │ │ │ ├── floor_planning.m
│ │ │ │ ├── max_volume_box.m
│ │ │ │ ├── power_control.m
│ │ │ │ ├── simple_dig_ckt_sizing.m
│ │ │ │ └── simple_dig_ckt_sizing_vect.m
│ │ │ ├── graph_laplacian/
│ │ │ │ ├── Contents.m
│ │ │ │ ├── best_const.m
│ │ │ │ ├── cut_grid_data.m
│ │ │ │ ├── cut_grid_example.m
│ │ │ │ ├── fdla.m
│ │ │ │ ├── fmmc.m
│ │ │ │ ├── larger_example.m
│ │ │ │ ├── max_deg.m
│ │ │ │ ├── mh.m
│ │ │ │ ├── plotgraph.m
│ │ │ │ └── small_example.m
│ │ │ ├── log_exp/
│ │ │ │ ├── Contents.m
│ │ │ │ ├── max_entropy.m
│ │ │ │ ├── sparse_covariance_est.m
│ │ │ │ ├── sparse_covariance_est_tradeoff.m
│ │ │ │ └── weighted_analytic_center.m
│ │ │ ├── make.m
│ │ │ ├── min_phase_spectral_fact.m
│ │ │ ├── nonneg_matrix_fact.m
│ │ │ ├── quickstart.m
│ │ │ ├── regularized_norm_tradeoff.m
│ │ │ ├── simple_LP.m
│ │ │ ├── simple_LP2.m
│ │ │ ├── simple_LS.m
│ │ │ ├── sparse_heuristics/
│ │ │ │ ├── Contents.m
│ │ │ │ ├── sparse_infeas.m
│ │ │ │ ├── sparse_infeas_dual.m
│ │ │ │ └── sparse_solution.m
│ │ │ └── time_series_analysis/
│ │ │ ├── Contents.m
│ │ │ └── l1_trend_filter_snp500.m
│ │ ├── functions/
│ │ │ ├── @cvx/
│ │ │ │ ├── avg_abs_dev.m
│ │ │ │ ├── avg_abs_dev_med.m
│ │ │ │ ├── berhu.m
│ │ │ │ ├── cvx_recip.m
│ │ │ │ ├── det_inv.m
│ │ │ │ ├── det_rootn.m
│ │ │ │ ├── functions.m
│ │ │ │ ├── geo_mean.m
│ │ │ │ ├── huber_pos.m
│ │ │ │ ├── inv_pos.m
│ │ │ │ ├── lambda_max.m
│ │ │ │ ├── lambda_sum_largest.m
│ │ │ │ ├── log_normcdf.m
│ │ │ │ ├── log_sum_exp.m
│ │ │ │ ├── matrix_frac.m
│ │ │ │ ├── norm_nuc.m
│ │ │ │ ├── norms.m
│ │ │ │ ├── pow_abs.m
│ │ │ │ ├── pow_cvx.m
│ │ │ │ ├── pow_p.m
│ │ │ │ ├── pow_pos.m
│ │ │ │ ├── prod_inv.m
│ │ │ │ ├── quad_form.m
│ │ │ │ ├── quad_over_lin.m
│ │ │ │ ├── quad_pos_over_lin.m
│ │ │ │ ├── rel_entr.m
│ │ │ │ ├── sigma_max.m
│ │ │ │ ├── square.m
│ │ │ │ ├── square_abs.m
│ │ │ │ ├── square_pos.m
│ │ │ │ ├── sum_largest.m
│ │ │ │ ├── sum_log.m
│ │ │ │ ├── sum_square.m
│ │ │ │ ├── sum_square_abs.m
│ │ │ │ ├── sum_square_pos.m
│ │ │ │ ├── trace_inv.m
│ │ │ │ ├── trace_sqrtm.m
│ │ │ │ └── vec.m
│ │ │ ├── Contents.m
│ │ │ ├── avg_abs_dev.m
│ │ │ ├── avg_abs_dev_med.m
│ │ │ ├── berhu.m
│ │ │ ├── det_inv.m
│ │ │ ├── det_root2n.m
│ │ │ ├── det_rootn.m
│ │ │ ├── entr.m
│ │ │ ├── geo_mean.m
│ │ │ ├── geomean.m
│ │ │ ├── huber.m
│ │ │ ├── huber_circ.m
│ │ │ ├── huber_pos.m
│ │ │ ├── inv_pos.m
│ │ │ ├── kl_div.m
│ │ │ ├── lambda_max.m
│ │ │ ├── lambda_min.m
│ │ │ ├── lambda_sum_largest.m
│ │ │ ├── lambda_sum_smallest.m
│ │ │ ├── log_det.m
│ │ │ ├── log_normcdf.m
│ │ │ ├── log_prod.m
│ │ │ ├── log_sum_exp.m
│ │ │ ├── logsumexp.m
│ │ │ ├── logsumexp_sdp.m
│ │ │ ├── matrix_frac.m
│ │ │ ├── norm_largest.m
│ │ │ ├── norm_nuc.m
│ │ │ ├── norms.m
│ │ │ ├── norms_largest.m
│ │ │ ├── poly_env.m
│ │ │ ├── polyenv.m
│ │ │ ├── polyval_trig.m
│ │ │ ├── pos.m
│ │ │ ├── pow_abs.m
│ │ │ ├── pow_p.m
│ │ │ ├── pow_pos.m
│ │ │ ├── prod_inv.m
│ │ │ ├── quad_form.m
│ │ │ ├── quad_over_lin.m
│ │ │ ├── quad_pos_over_lin.m
│ │ │ ├── rel_entr.m
│ │ │ ├── sigma_max.m
│ │ │ ├── square_/
│ │ │ │ └── square.m
│ │ │ ├── square_abs.m
│ │ │ ├── square_pos.m
│ │ │ ├── sum_largest.m
│ │ │ ├── sum_log.m
│ │ │ ├── sum_smallest.m
│ │ │ ├── sum_square.m
│ │ │ ├── sum_square_abs.m
│ │ │ ├── sum_square_pos.m
│ │ │ ├── sym.m
│ │ │ ├── trace_inv.m
│ │ │ ├── trace_sqrtm.m
│ │ │ └── vec_/
│ │ │ └── vec.m
│ │ ├── keywords/
│ │ │ ├── Contents.m
│ │ │ ├── In.m
│ │ │ ├── binary.m
│ │ │ ├── dual.m
│ │ │ ├── epigraph.m
│ │ │ ├── expression.m
│ │ │ ├── expressions.m
│ │ │ ├── hypograph.m
│ │ │ ├── integer.m
│ │ │ ├── maximise.m
│ │ │ ├── maximize.m
│ │ │ ├── minimise.m
│ │ │ ├── minimize.m
│ │ │ ├── subject.m
│ │ │ ├── variable.m
│ │ │ └── variables.m
│ │ ├── lib/
│ │ │ ├── @cell/
│ │ │ │ ├── cvx_id.m
│ │ │ │ ├── cvx_setdual.m
│ │ │ │ └── cvx_value.m
│ │ │ ├── @cvx/
│ │ │ │ ├── bcompress.m
│ │ │ │ ├── buncompress.m
│ │ │ │ ├── cvx.m
│ │ │ │ ├── cvx_basis.m
│ │ │ │ ├── cvx_classify.m
│ │ │ │ ├── cvx_constant.m
│ │ │ │ ├── cvx_getdual.m
│ │ │ │ ├── cvx_isaffine.m
│ │ │ │ ├── cvx_isconcave.m
│ │ │ │ ├── cvx_isconstant.m
│ │ │ │ ├── cvx_isconvex.m
│ │ │ │ ├── cvx_isnonzero.m
│ │ │ │ ├── cvx_readlevel.m
│ │ │ │ ├── cvx_setdual.m
│ │ │ │ ├── cvx_value.m
│ │ │ │ ├── cvx_vexity.m
│ │ │ │ ├── in.m
│ │ │ │ ├── keywords.m
│ │ │ │ ├── matlab6.m
│ │ │ │ ├── sets.m
│ │ │ │ ├── sparsify.m
│ │ │ │ ├── svec.m
│ │ │ │ ├── type.m
│ │ │ │ └── value.m
│ │ │ ├── @cvxcnst/
│ │ │ │ ├── cvxcnst.m
│ │ │ │ ├── disp.m
│ │ │ │ ├── display.m
│ │ │ │ ├── double.m
│ │ │ │ ├── logical.m
│ │ │ │ └── rhs.m
│ │ │ ├── @cvxdual/
│ │ │ │ ├── colon.m
│ │ │ │ ├── cvx_basis.m
│ │ │ │ ├── cvx_value.m
│ │ │ │ ├── cvxaff.m
│ │ │ │ ├── cvxdual.m
│ │ │ │ ├── disp.m
│ │ │ │ ├── display.m
│ │ │ │ ├── dof.m
│ │ │ │ ├── inuse.m
│ │ │ │ ├── isreal.m
│ │ │ │ ├── name.m
│ │ │ │ ├── problem.m
│ │ │ │ ├── size.m
│ │ │ │ ├── subsref.m
│ │ │ │ ├── type.m
│ │ │ │ └── value.m
│ │ │ ├── @cvxin/
│ │ │ │ ├── cvxin.m
│ │ │ │ ├── cvxtuple.m
│ │ │ │ ├── gt.m
│ │ │ │ └── lt.m
│ │ │ ├── @cvxobj/
│ │ │ │ ├── cvx_id.m
│ │ │ │ ├── cvxobj.m
│ │ │ │ ├── disp.m
│ │ │ │ ├── display.m
│ │ │ │ ├── isempty.m
│ │ │ │ ├── isequal.m
│ │ │ │ ├── length.m
│ │ │ │ ├── ndims.m
│ │ │ │ ├── numel.m
│ │ │ │ ├── subsasgn.m
│ │ │ │ └── subsref.m
│ │ │ ├── @cvxprob/
│ │ │ │ ├── cvx_value.m
│ │ │ │ ├── cvxprob.m
│ │ │ │ ├── disp.m
│ │ │ │ ├── eliminate.m
│ │ │ │ ├── eq.m
│ │ │ │ ├── extract.m
│ │ │ │ ├── index.m
│ │ │ │ ├── ne.m
│ │ │ │ ├── newcnstr.m
│ │ │ │ ├── newdual.m
│ │ │ │ ├── newnonl.m
│ │ │ │ ├── newobj.m
│ │ │ │ ├── newtemp.m
│ │ │ │ ├── newvar.m
│ │ │ │ ├── pop.m
│ │ │ │ ├── solve.m
│ │ │ │ ├── spy.m
│ │ │ │ ├── subsasgn.m
│ │ │ │ ├── subsref.m
│ │ │ │ └── touch.m
│ │ │ ├── @cvxtuple/
│ │ │ │ ├── apply.m
│ │ │ │ ├── cvx_collapse.m
│ │ │ │ ├── cvx_constant.m
│ │ │ │ ├── cvx_getdual.m
│ │ │ │ ├── cvx_id.m
│ │ │ │ ├── cvx_isaffine.m
│ │ │ │ ├── cvx_isconcave.m
│ │ │ │ ├── cvx_isconstant.m
│ │ │ │ ├── cvx_isconvex.m
│ │ │ │ ├── cvx_setdual.m
│ │ │ │ ├── cvx_value.m
│ │ │ │ ├── cvxtuple.m
│ │ │ │ ├── disp.m
│ │ │ │ ├── eq.m
│ │ │ │ ├── ge.m
│ │ │ │ ├── gt.m
│ │ │ │ ├── in.m
│ │ │ │ ├── le.m
│ │ │ │ ├── lt.m
│ │ │ │ ├── ne.m
│ │ │ │ ├── numel.m
│ │ │ │ ├── subsasgn.m
│ │ │ │ ├── subsref.m
│ │ │ │ └── testall.m
│ │ │ ├── Contents.m
│ │ │ ├── cvx_accept_concave.m
│ │ │ ├── cvx_accept_convex.m
│ │ │ ├── cvx_basis.m
│ │ │ ├── cvx_bcompress.m
│ │ │ ├── cvx_blkdiag.m
│ │ │ ├── cvx_c2r.m
│ │ │ ├── cvx_check_dimension.m
│ │ │ ├── cvx_check_dimlist.m
│ │ │ ├── cvx_class.m
│ │ │ ├── cvx_classify.m
│ │ │ ├── cvx_clearpath.m
│ │ │ ├── cvx_clearspath.m
│ │ │ ├── cvx_collapse.m
│ │ │ ├── cvx_constant.m
│ │ │ ├── cvx_default_dimension.m
│ │ │ ├── cvx_expand_dim.m
│ │ │ ├── cvx_expert_check.m
│ │ │ ├── cvx_getdual.m
│ │ │ ├── cvx_global.m
│ │ │ ├── cvx_id.m
│ │ │ ├── cvx_ids.m
│ │ │ ├── cvx_isaffine.m
│ │ │ ├── cvx_isconcave.m
│ │ │ ├── cvx_isconstant.m
│ │ │ ├── cvx_isconvex.m
│ │ │ ├── cvx_isnonzero.m
│ │ │ ├── cvx_r2c.m
│ │ │ ├── cvx_readlevel.m
│ │ │ ├── cvx_reduce_size.m
│ │ │ ├── cvx_remap.m
│ │ │ ├── cvx_reshape.m
│ │ │ ├── cvx_run_solver.m
│ │ │ ├── cvx_setdual.m
│ │ │ ├── cvx_setpath.m
│ │ │ ├── cvx_setspath.m
│ │ │ ├── cvx_size_check.m
│ │ │ ├── cvx_subs2str.m
│ │ │ ├── cvx_subsasgn.m
│ │ │ ├── cvx_subsref.m
│ │ │ ├── cvx_subsref_check.m
│ │ │ ├── cvx_use_sparse.m
│ │ │ ├── cvx_value.m
│ │ │ ├── cvx_values.m
│ │ │ ├── cvx_vexity.m
│ │ │ ├── cvx_zeros.m
│ │ │ └── narginchk_/
│ │ │ └── narginchk.m
│ │ ├── sets/
│ │ │ ├── Contents.m
│ │ │ ├── complex_lorentz.m
│ │ │ ├── convex_poly_coeffs.m
│ │ │ ├── exponential.m
│ │ │ ├── geo_mean_cone.m
│ │ │ ├── hermitian_semidefinite.m
│ │ │ ├── lorentz.m
│ │ │ ├── nonneg_poly_coeffs.m
│ │ │ ├── nonnegative.m
│ │ │ ├── norm_ball.m
│ │ │ ├── rotated_complex_lorentz.m
│ │ │ ├── rotated_lorentz.m
│ │ │ ├── semidefinite.m
│ │ │ └── simplex.m
│ │ ├── shims/
│ │ │ ├── cvx_glpk.m
│ │ │ ├── cvx_sdpt3.m
│ │ │ └── cvx_sedumi.m
│ │ └── structures/
│ │ ├── @cvx/
│ │ │ └── structures.m
│ │ ├── Contents.m
│ │ ├── cvx_create_structure.m
│ │ ├── cvx_invert_structure.m
│ │ ├── cvx_orthog_structure.m
│ │ ├── cvx_replicate_structure.m
│ │ ├── cvx_s_banded.m
│ │ ├── cvx_s_hankel.m
│ │ ├── cvx_s_hermitian.m
│ │ ├── cvx_s_sparse.m
│ │ ├── cvx_s_symmetric.m
│ │ ├── cvx_s_symmetric_ut.m
│ │ └── cvx_s_upper_hankel.m
│ ├── Check_plot.m
│ ├── FTLEH.m
│ ├── FTLE_reg.m
│ ├── Integrate1.m
│ ├── Integrate2.m
│ ├── Lagr.m
│ ├── Lagrangian_points.m
│ ├── Poincare.m
│ ├── RK4.m
│ ├── TFOCS/
│ │ ├── @double/
│ │ │ ├── tfocs_dot.m
│ │ │ ├── tfocs_size.m
│ │ │ └── vec.m
│ │ ├── @single/
│ │ │ ├── tfocs_dot.m
│ │ │ ├── tfocs_size.m
│ │ │ └── vec.m
│ │ ├── @tfocs_tuple/
│ │ │ ├── abs.m
│ │ │ ├── cell.m
│ │ │ ├── disp.m
│ │ │ ├── display.m
│ │ │ ├── get.m
│ │ │ ├── max.m
│ │ │ ├── min.m
│ │ │ ├── minus.m
│ │ │ ├── mtimes.m
│ │ │ ├── nnz.m
│ │ │ ├── numel.m
│ │ │ ├── plus.m
│ │ │ ├── power.m
│ │ │ ├── size.m
│ │ │ ├── subsref.m
│ │ │ ├── tfocs_dot.m
│ │ │ ├── tfocs_normsq.m
│ │ │ ├── tfocs_size.m
│ │ │ ├── tfocs_tuple.m
│ │ │ ├── tfocs_zeros.m
│ │ │ ├── times.m
│ │ │ └── uminus.m
│ │ ├── Contents.m
│ │ ├── continuation.m
│ │ ├── examples/
│ │ │ ├── demos/
│ │ │ │ ├── SIAM_demo.m
│ │ │ │ ├── callandmap.m
│ │ │ │ ├── demo_MatrixCompletion.m
│ │ │ │ ├── demo_SVM.m
│ │ │ │ ├── demo_alternatingProjections.m
│ │ │ │ ├── project2DCone.m
│ │ │ │ └── recordPoints.m
│ │ │ ├── largescale/
│ │ │ │ ├── BugsBunny.mat
│ │ │ │ ├── PsiTransposeWFF.m
│ │ │ │ ├── PsiWFF.m
│ │ │ │ ├── example_quantumTomography.m
│ │ │ │ ├── explicitPauliTensor.m
│ │ │ │ ├── findWeights.m
│ │ │ │ ├── image_denoising_withSPOT.m
│ │ │ │ ├── plotNow.m
│ │ │ │ ├── test_SVM.m
│ │ │ │ ├── test_sBPDN_W_largescale.m
│ │ │ │ ├── test_sTV_Analysis_largescale.m
│ │ │ │ └── test_sTV_largescale.m
│ │ │ ├── private/
│ │ │ │ ├── eig_backup.m
│ │ │ │ ├── linop_identity.m
│ │ │ │ ├── linop_stack.m
│ │ │ │ ├── print_cell_size.m
│ │ │ │ ├── prox_stack.m
│ │ │ │ ├── safe_eig.m
│ │ │ │ ├── size_ambig.m
│ │ │ │ ├── size_compat.m
│ │ │ │ ├── smooth_stack.m
│ │ │ │ ├── solver_apply.m
│ │ │ │ ├── tfocs_backtrack.m
│ │ │ │ ├── tfocs_cleanup.m
│ │ │ │ ├── tfocs_initialize.m
│ │ │ │ ├── tfocs_iterate.m
│ │ │ │ ├── tfocs_prox.m
│ │ │ │ ├── tfocs_round.m
│ │ │ │ ├── tfocs_smooth.m
│ │ │ │ └── tfocs_zeros.m
│ │ │ └── smallscale/
│ │ │ ├── project_WeightedNorm.m
│ │ │ ├── reference_solutions/
│ │ │ │ ├── LMI.mat
│ │ │ │ ├── LMI_complex.mat
│ │ │ │ ├── LP.mat
│ │ │ │ ├── LP_box.mat
│ │ │ │ ├── SDP.mat
│ │ │ │ ├── SDP_complex.mat
│ │ │ │ ├── basispursuit_WW_problem1_smoothed_noisy.mat
│ │ │ │ ├── basispursuit_W_problem1_smoothed_noisy.mat
│ │ │ │ ├── basispursuit_problem1_noisy.mat
│ │ │ │ ├── basispursuit_problem1_smoothed_noiseless.mat
│ │ │ │ ├── basispursuit_problem1_smoothed_noiseless_nonnegative.mat
│ │ │ │ ├── basispursuit_problem1_smoothed_noisy.mat
│ │ │ │ ├── basispursuit_problem1_smoothed_noisy_complex.mat
│ │ │ │ ├── basispursuit_problem1_smoothed_noisy_complex_2.mat
│ │ │ │ ├── basispursuit_problem1_smoothed_noisy_nonnegative.mat
│ │ │ │ ├── blocknorm_smoothed_noisy.mat
│ │ │ │ ├── complicatedProblem1.mat
│ │ │ │ ├── dantzig_problem1_smoothed_noisy.mat
│ │ │ │ ├── lasso_problem1_noisy.mat
│ │ │ │ ├── nuclearNorm_problem1_noiseless.mat
│ │ │ │ ├── ordered_lasso_problem1_noisy.mat
│ │ │ │ ├── traceLS_problem1_noisy.mat
│ │ │ │ ├── traceLS_problem2_noisy.mat
│ │ │ │ └── tv_problem1_smoothed_noisy.mat
│ │ │ ├── test_LASSO.m
│ │ │ ├── test_LMI.m
│ │ │ ├── test_LinearProgram.m
│ │ │ ├── test_SDP.m
│ │ │ ├── test_SLOPE.m
│ │ │ ├── test_TraceLS.m
│ │ │ ├── test_all.m
│ │ │ ├── test_blockNorm.m
│ │ │ ├── test_complicatedUsage.m
│ │ │ ├── test_nuclearNorm.m
│ │ │ ├── test_psdCompletion.m
│ │ │ ├── test_quadratic.m
│ │ │ ├── test_quadratic_constrained.m
│ │ │ ├── test_sBP.m
│ │ │ ├── test_sBPDN.m
│ │ │ ├── test_sBPDN_W.m
│ │ │ ├── test_sBPDN_WW.m
│ │ │ ├── test_sBPDN_complex.m
│ │ │ ├── test_sBPDN_complex_2.m
│ │ │ ├── test_sBPDN_nonnegative.m
│ │ │ ├── test_sBPDN_withContinuation.m
│ │ │ ├── test_sBP_nonnegative.m
│ │ │ ├── test_sDantzig.m
│ │ │ ├── test_sDantzig_3methods.m
│ │ │ ├── test_sTV.m
│ │ │ └── test_variousSolvers.m
│ │ ├── linop_TV.m
│ │ ├── linop_TV3D.m
│ │ ├── linop_adjoint.m
│ │ ├── linop_compose.m
│ │ ├── linop_dot.m
│ │ ├── linop_explicit.m
│ │ ├── linop_fft.m
│ │ ├── linop_handles.m
│ │ ├── linop_horzcat.m
│ │ ├── linop_matrix.m
│ │ ├── linop_normest.m
│ │ ├── linop_reshape.m
│ │ ├── linop_scale.m
│ │ ├── linop_spot.m
│ │ ├── linop_subsample.m
│ │ ├── linop_test.m
│ │ ├── linop_vec.m
│ │ ├── linop_vertcat.m
│ │ ├── proj_0.m
│ │ ├── proj_Rn.m
│ │ ├── proj_Rplus.m
│ │ ├── proj_affine.m
│ │ ├── proj_box.m
│ │ ├── proj_boxAffine.m
│ │ ├── proj_conic.m
│ │ ├── proj_l1.m
│ │ ├── proj_l1l2.m
│ │ ├── proj_l2.m
│ │ ├── proj_l2group.m
│ │ ├── proj_linf.m
│ │ ├── proj_linfl2.m
│ │ ├── proj_max.m
│ │ ├── proj_maxEig.m
│ │ ├── proj_nuclear.m
│ │ ├── proj_psd.m
│ │ ├── proj_psdUTrace.m
│ │ ├── proj_simplex.m
│ │ ├── proj_singleAffine.m
│ │ ├── proj_spectral.m
│ │ ├── prox_0.m
│ │ ├── prox_Sl1.m
│ │ ├── prox_boxDual.m
│ │ ├── prox_diag.m
│ │ ├── prox_dualize.m
│ │ ├── prox_hinge.m
│ │ ├── prox_hingeDual.m
│ │ ├── prox_l1.m
│ │ ├── prox_l1_and_sum.m
│ │ ├── prox_l1_deadzone.m
│ │ ├── prox_l1_mat.m
│ │ ├── prox_l1l2.m
│ │ ├── prox_l1linf.m
│ │ ├── prox_l1pos.m
│ │ ├── prox_l2.m
│ │ ├── prox_linf.m
│ │ ├── prox_max.m
│ │ ├── prox_maxEig.m
│ │ ├── prox_nuclear.m
│ │ ├── prox_scale.m
│ │ ├── prox_shift.m
│ │ ├── prox_spectral.m
│ │ ├── prox_trace.m
│ │ ├── randomizedSVD.m
│ │ ├── smooth_constant.m
│ │ ├── smooth_entropy.m
│ │ ├── smooth_handles.m
│ │ ├── smooth_huber.m
│ │ ├── smooth_linear.m
│ │ ├── smooth_logLLogistic.m
│ │ ├── smooth_logLPoisson.m
│ │ ├── smooth_logdet.m
│ │ ├── smooth_logsumexp.m
│ │ ├── smooth_quad.m
│ │ ├── solver_L1RLS.m
│ │ ├── solver_LASSO.m
│ │ ├── solver_SLOPE.m
│ │ ├── solver_TraceLS.m
│ │ ├── solver_psdComp.m
│ │ ├── solver_psdCompConstrainedTrace.m
│ │ ├── solver_sBP.m
│ │ ├── solver_sBPDN.m
│ │ ├── solver_sBPDN_W.m
│ │ ├── solver_sBPDN_WW.m
│ │ ├── solver_sDantzig.m
│ │ ├── solver_sDantzig_W.m
│ │ ├── solver_sLMI.m
│ │ ├── solver_sLP.m
│ │ ├── solver_sLP_box.m
│ │ ├── solver_sNuclearBP.m
│ │ ├── solver_sNuclearBPDN.m
│ │ ├── solver_sSDP.m
│ │ ├── test_Lipschitz.m
│ │ ├── test_nonsmooth.m
│ │ ├── test_proxPair.m
│ │ ├── test_smooth.m
│ │ ├── tfocs.m
│ │ ├── tfocs_AT.m
│ │ ├── tfocs_GRA.m
│ │ ├── tfocs_LLM.m
│ │ ├── tfocs_N07.m
│ │ ├── tfocs_N83.m
│ │ ├── tfocs_SCD.m
│ │ ├── tfocs_TS.m
│ │ ├── tfocs_normsq.m
│ │ ├── tfocs_version.m
│ │ ├── tfocs_where.m
│ │ ├── tfunc_scale.m
│ │ └── tfunc_sum.m
│ ├── Traj.m
│ ├── adapting_structural_model.m
│ ├── average.m
│ ├── bicycle_state_space.m
│ ├── convert_variable.m
│ ├── create_ieee_paper_plots.m
│ ├── cross_validation.m
│ ├── distance.m
│ ├── double_gyre.m
│ ├── example.m
│ ├── fit_adapt.m
│ ├── fit_adapt_linear.m
│ ├── gpu_RKF45_FILE.m
│ ├── ieee.m
│ ├── lane_change.m
│ ├── load_bikes.m
│ ├── load_data.m
│ ├── make_filter.m
│ ├── matlab_class.m
│ ├── matlab_function.m
│ ├── matlab_script.m
│ ├── matlab_script2.m
│ ├── normalize.m
│ ├── overwrite_settings.m
│ ├── par_text_to_struct.m
│ ├── plant.m
│ ├── test_rk_par.m
│ ├── test_system_state_space.m
│ ├── varargin_to_structure.m
│ └── write_gains.m
├── Maven POM/
│ └── filenames/
│ └── pom.xml
├── Max/
│ ├── Hello.maxhelp
│ ├── Hello.maxpat
│ └── Hello.mxt
├── MediaWiki/
│ ├── README.wiki
│ └── mediawiki.mediawiki
├── Mercury/
│ ├── code_info.m
│ ├── expr.moo
│ ├── hello.m
│ ├── options.m
│ ├── polymorphism.m
│ ├── rot13_concise.m
│ ├── rot13_ralph.m
│ ├── rot13_verbose.m
│ ├── store.m
│ └── switch_detection_bug.m
├── Meson/
│ └── filenames/
│ ├── meson.build
│ └── meson_options.txt
├── Metal/
│ └── ITMVisualisationEngine.metal
├── Modelica/
│ ├── NestedPackages.mo
│ ├── NewtonCooling.mo
│ ├── Pendulum.mo
│ ├── RLC.mo
│ ├── SecondOrderSystem.mo
│ ├── System.mo
│ ├── Translational.mo
│ ├── modelica.mo
│ ├── package.mo
│ ├── package2.mo
│ ├── package3.mo
│ └── package4.mo
├── Modula-2/
│ └── HuffChan.mod
├── Module Management System/
│ ├── descrip.mms
│ ├── openvms.mmk
│ ├── simh_descrip.mms
│ ├── spline_descrip.mms
│ └── xv_makefile.mms
├── Monkey/
│ ├── encodeToPng.monkey2
│ ├── example.monkey
│ ├── example.monkey2
│ ├── gui.monkey2
│ └── sorting.monkey2
├── Moocode/
│ ├── hello.moo
│ ├── moocode_toolkit.moo
│ └── toy.moo
├── MoonScript/
│ └── transform.moon
├── NCL/
│ ├── PrnOscPat_driver.ncl
│ ├── WRF_static_2.ncl
│ ├── WRF_track_1.ncl
│ ├── cru_8.ncl
│ ├── gsn_csm_xy2_time_series_inputs.ncl
│ ├── hdf4sds_7.ncl
│ ├── mask_12.ncl
│ ├── mcsst_1.ncl
│ ├── primero.ncl
│ ├── topo_9.ncl
│ ├── traj_3.ncl
│ ├── tsdiagram_1.ncl
│ ├── unique_9.ncl
│ ├── viewport_4.ncl
│ ├── weather_sym_6.ncl
│ └── xy_29.ncl
├── NL/
│ ├── assign0.nl
│ └── balassign0.nl
├── NSIS/
│ ├── bigtest.nsi
│ └── x64.nsh
├── Nearley/
│ └── nearley-language-bootstrapped.ne
├── Nemerle/
│ └── hello.n
├── NetLinx/
│ ├── projector.axi
│ └── volume-array.axs
├── NetLinx+ERB/
│ ├── sample.axi.erb
│ └── sample.axs.erb
├── NetLogo/
│ └── Life.nlogo
├── NewLisp/
│ ├── irc.lsp
│ ├── log-to-database.lisp
│ └── queens.nl
├── Nextflow/
│ ├── blast.nf
│ ├── callings.nf
│ ├── filenames/
│ │ └── nextflow.config
│ └── rnaseq.nf
├── Nginx/
│ ├── example.com.vhost
│ └── filenames/
│ └── nginx.conf
├── Nim/
│ └── foo.nim
├── Nit/
│ ├── calculator.nit
│ ├── callback_chimpanze.nit
│ ├── callback_monkey.nit
│ ├── circular_list.nit
│ ├── clock.nit
│ ├── clock_more.nit
│ ├── curl_http.nit
│ ├── curl_mail.nit
│ ├── draw_operation.nit
│ ├── drop_privileges.nit
│ ├── extern_methods.nit
│ ├── fibonacci.nit
│ ├── file.nit
│ ├── hello_world.nit
│ ├── html_page.nit
│ ├── int_stack.nit
│ ├── meetup.nit
│ ├── opengles2_hello_triangle.nit
│ ├── print_arguments.nit
│ ├── procedural_array.nit
│ ├── socket_client.nit
│ ├── socket_server.nit
│ ├── tmpl_composer.nit
│ └── websocket_server.nit
├── Nix/
│ └── nginx.nix
├── Nu/
│ ├── RandomApp.nu
│ └── nu
├── OCaml/
│ ├── Foo.ml
│ ├── cmdliner.ml
│ ├── common.ml
│ ├── date.ml
│ ├── example.eliom
│ ├── map.ml
│ ├── mirage.ml
│ ├── reload.ml
│ ├── sigset.ml
│ └── uutf.ml
├── Objective-C/
│ ├── ASIHTTPRequest.h
│ ├── ASIHTTPRequest.m
│ ├── Foo.h
│ ├── Foo.m
│ ├── FooAppDelegate.h
│ ├── FooAppDelegate.m
│ ├── JSONKit.h
│ ├── JSONKit.m
│ ├── MainMenuViewController.h
│ ├── MainMenuViewController.m
│ ├── PlaygroundViewController.h
│ ├── PlaygroundViewController.m
│ ├── SBJsonParser.h
│ ├── SBJsonParser.m
│ ├── Siesta.h
│ ├── StyleViewController.h
│ ├── StyleViewController.m
│ ├── TUITableView.h
│ ├── TUITableView.m
│ ├── cocoa_monitor.m
│ ├── gen-cocoa-linguist-thrift.m
│ └── hello.m
├── Objective-C++/
│ ├── EventHandlerMac.mm
│ └── objsql.mm
├── Objective-J/
│ ├── AppController.j
│ ├── LightsOff.j
│ └── iTunesLayout.j
├── Omgrofl/
│ └── hello.omgrofl
├── Opa/
│ ├── hello_syntax1.opa
│ └── hello_syntax2.opa
├── Opal/
│ └── DeepakChopra.opal
├── OpenCL/
│ ├── fft.cl
│ └── sample.cl
├── OpenEdge ABL/
│ ├── Email.cls
│ ├── SendEmailAlgorithm.cls
│ ├── SocketReader.p
│ ├── Util.cls
│ ├── openedge.p
│ └── test-win.w
├── OpenRC runscript/
│ └── acpid
├── OpenSCAD/
│ ├── not_simple.scad
│ └── simple.scad
├── Org/
│ └── org.org
├── Ox/
│ ├── IJCEmet2009.oxh
│ ├── ParallelObjective.ox
│ └── particle.oxo
├── Oxygene/
│ └── Loops.oxygene
├── Oz/
│ └── example.oz
├── P4/
│ ├── l2.p4
│ └── mirror_acl.p4
├── PAWN/
│ ├── fixes.inc
│ └── timertest.pwn
├── PHP/
│ ├── Application.php
│ ├── Client.php
│ ├── Controller.php
│ ├── Form.php
│ ├── Model.php
│ ├── ThriftGenerated.php
│ ├── drupal.php
│ ├── exception.zep.php
│ ├── file_display.inc
│ ├── filenames/
│ │ ├── .php
│ │ ├── .php_cs
│ │ └── .php_cs.dist
│ ├── mail.phps
│ ├── php
│ ├── php-script
│ ├── php2
│ ├── prefix.fcgi
│ └── root.php
├── PLSQL/
│ ├── myobject.sql
│ ├── packagebody.pkb
│ ├── packageheader.pks
│ ├── plsqlguide.pck
│ ├── prime#.plsql
│ ├── print_bool.prc
│ ├── videodb.ddl
│ └── who_called_me.sql
├── PLpgSQL/
│ ├── plpgsql_lint-8.4.sql
│ ├── plpgsql_lint-9.0.sql
│ ├── plpgsql_lint-9.1.sql
│ ├── plpgsql_lint-9.2.sql
│ ├── plpgsql_lint-9.3.sql
│ └── procedures.sql
├── POV-Ray SDL/
│ ├── balcony.pov
│ ├── bglass.inc
│ ├── building.inc
│ ├── chair.inc
│ ├── cloth.inc
│ ├── gamma_showcase.pov
│ ├── sky.inc
│ ├── table.inc
│ ├── table_cloth.inc
│ ├── table_stuff.inc
│ ├── terrain.inc
│ └── water.inc
├── Pan/
│ ├── ceph-raid.pan
│ ├── cluster-A.pan
│ ├── databases.pan
│ ├── functions.pan
│ ├── infernalis.pan
│ ├── libvirt.pan
│ ├── link.pan
│ ├── mysql.pan
│ ├── nodes_properties.pan
│ ├── onevm.pan
│ ├── osd-fetch.pan
│ ├── pakiti.pan
│ ├── purge_fqan_accounts.pan
│ ├── resources.pan
│ ├── simple.pan
│ ├── test.pan
│ ├── types.pan
│ └── unit.pan
├── Papyrus/
│ ├── CAMTEST_OverShoulderME.psc
│ ├── vMFX_FXPlugin.psc
│ └── vSCM_MetaQuestScript.psc
├── Parrot Assembly/
│ └── hello.pasm
├── Parrot Internal Representation/
│ └── hello.pir
├── Pascal/
│ ├── bulls-and-cows.pascal
│ ├── custforms.pp
│ ├── cwindirs.pp
│ ├── image_url.inc
│ ├── large.pp
│ ├── libc.inc
│ ├── program.dpr
│ ├── read-a-configuration-file.pascal
│ ├── tw27294.pp
│ └── vmops_impl.inc
├── Pep8/
│ ├── div.pep
│ ├── flag.pep
│ ├── linked.pep
│ ├── msq.pep
│ ├── qsort.pep
│ ├── stri_buf.pep
│ └── stristack.pep
├── Perl/
│ ├── Ack.pm
│ ├── Any.pm
│ ├── Request.pm
│ ├── Response.pm
│ ├── example.cgi
│ ├── exception_handler.pl
│ ├── feedgnuplot
│ ├── fib.pl
│ ├── filenames/
│ │ ├── Makefile.PL
│ │ ├── Rexfile
│ │ ├── ack
│ │ └── cpanfile
│ ├── getchar.al
│ ├── index.fcgi
│ ├── oo1.pl
│ ├── oo2.pl
│ ├── oo3.pl
│ ├── perl
│ ├── perl-test.t
│ ├── script.pl
│ ├── strict.t
│ ├── test-perl.pl
│ ├── test-perl2.pl
│ └── use5.pl
├── Perl 6/
│ ├── 01-dash-uppercase-i.t
│ ├── 01-parse.t
│ ├── A.pm
│ ├── ANSIColor.pm
│ ├── Bailador.pm
│ ├── ContainsUnicode.pm
│ ├── Exception.pm
│ ├── List.pm
│ ├── Model.pm
│ ├── RoleQ.pm6
│ ├── Simple.pm
│ ├── Win32.pm
│ ├── advent2009-day16.t
│ ├── basic-open.t
│ ├── calendar.t
│ ├── for.t
│ ├── grammar-test.p6
│ ├── hash.t
│ ├── htmlify.pl
│ ├── listquote-whitespace.t
│ ├── man-or-boy.t
│ └── test.p6
├── Pic/
│ ├── dextroamphetamine.chem
│ ├── graph.pic
│ └── ritalin.chem
├── Pickle/
│ ├── data.pkl
│ ├── neural-network-ce-l2reg-784-10-30.pkl
│ ├── random.pkl
│ └── save.pkl
├── PicoLisp/
│ └── simul.l
├── PigLatin/
│ └── example.pig
├── Pike/
│ ├── Error.pmod
│ ├── FakeFile.pike
│ └── shebang.pike
├── Pod/
│ ├── PSGI.pod
│ ├── Sample.pod
│ └── contents.pod
├── PogoScript/
│ └── squashy.pogo
├── Pony/
│ ├── circle.pony
│ ├── counter.pony
│ ├── gups-opt.pony
│ ├── hello-world.pony
│ ├── mandelbrot.pony
│ └── mixed.pony
├── PostCSS/
│ └── sample.pcss
├── PostScript/
│ ├── lambda.pfa
│ └── sierpinski.ps
├── PowerBuilder/
│ ├── TestPBT.pbt
│ ├── ginpix7.sra
│ ├── myproject.pbt
│ ├── n_cst_buttonlistbar_gradient.sru
│ ├── part1.srw
│ └── w_export.srw
├── PowerShell/
│ ├── ZLocation.psd1
│ ├── ZLocation.psm1
│ ├── history.ps1
│ └── pwsh-shebang.ps1
├── Processing/
│ └── hello.pde
├── Prolog/
│ ├── admin.pl
│ ├── dleak-report
│ ├── ex6.pl
│ ├── format_spec.pl
│ ├── func.pl
│ ├── logic-problem.pro
│ ├── queues.yap
│ ├── test-prolog.prolog
│ └── turing.pl
├── Propeller Spin/
│ ├── 4x4 Keypad Reader.spin
│ ├── Debug_Lcd.spin
│ ├── Graphics.spin
│ ├── Inductor.spin
│ ├── Keyboard.spin
│ ├── TV.spin
│ ├── TV_Terminal.spin
│ ├── TV_Text.spin
│ ├── VGA.spin
│ └── VocalTract.spin
├── Protocol Buffer/
│ └── addressbook.proto
├── Public Key/
│ ├── gpg_key.asc
│ ├── id.pub
│ ├── id_dsa.asc
│ ├── id_rsa.asc
│ ├── id_rsa.pub
│ ├── sshkey1.asc
│ └── sunCert.asc
├── Pug/
│ ├── hello.jade
│ └── hello.pug
├── Puppet/
│ ├── apacheinit.pp
│ ├── expiringhost.pp
│ ├── hiera_include.pp
│ ├── stages-example.pp
│ └── unmanaged-notify-puppet25.pp
├── PureBasic/
│ ├── Example_Sine.pb
│ └── Memory.pbi
├── PureScript/
│ ├── Control.Arrow.purs
│ ├── Data.Foreign.purs
│ ├── Data.Map.purs
│ └── ReactiveJQueryTest.purs
├── Python/
│ ├── AdditiveWave.pyde
│ ├── Cinema4DPythonPlugin.pyp
│ ├── JupyterNotebook.ipynb
│ ├── MoveEye.pyde
│ ├── action.cgi
│ ├── argparse.pyi
│ ├── backstage.fcgi
│ ├── closure_js_binary.bzl
│ ├── django-models-base.py
│ ├── filenames/
│ │ ├── .gclient
│ │ ├── BUCK
│ │ ├── BUILD
│ │ └── WORKSPACE
│ ├── flask-view.py
│ ├── gen-py-linguist-thrift.py
│ ├── protocol_buffer_pb2.py
│ ├── py3.py3
│ ├── python
│ ├── python2
│ ├── python3
│ ├── simpleclient.rpy
│ ├── spec.linux.spec
│ ├── standalone.gypi
│ ├── toolchain.gypi
│ └── tornado-httpserver.py
├── QML/
│ └── common.qbs
├── QMake/
│ ├── complex.pro
│ ├── functions.pri
│ ├── qmake
│ └── simple.pro
├── R/
│ ├── R-qgis-extension.rsx
│ ├── df.residual.r
│ ├── filenames/
│ │ └── expr-dist
│ ├── git-punchcard
│ ├── hello-r.R
│ ├── import.Rd
│ ├── import.r
│ └── scholar.Rd
├── RAML/
│ └── api.raml
├── RDoc/
│ └── rdoc.rdoc
├── REXX/
│ ├── BatchRemapBrushes.pprx
│ ├── ShapesInfo.rexx
│ ├── SkrivShape.rexx
│ └── ag2xml.rexx
├── RMarkdown/
│ └── example.rmd
├── RPC/
│ ├── rpc.x
│ ├── rusers.x
│ └── yp.x
├── RPM Spec/
│ ├── apache.spec
│ ├── erlang-erlydtl.spec
│ └── manos.spec
├── RUNOFF/
│ ├── VMS_ZIP.RNH
│ ├── contributing.rnh
│ ├── longlib.rno
│ └── mcp_help.rnh
├── Racket/
│ ├── 99-bottles-of-beer.scrbl
│ └── scribble.scrbl
├── Ragel/
│ ├── ephemeris_parser.rl
│ ├── simple_scanner.rl
│ └── simple_tokenizer.rl
├── Rascal/
│ ├── Analyze.rsc
│ ├── Compile.rsc
│ ├── Rascal.rsc
│ └── Syntax.rsc
├── Reason/
│ ├── JSX.re
│ ├── Layout.re
│ ├── Machine.re
│ ├── SuperMerlin.re
│ └── Syntax.re
├── Rebol/
│ ├── GCP-datatypes.r
│ ├── booters.r
│ ├── hello-world.r2
│ ├── hello-world.r3
│ ├── hello-world.reb
│ └── hello-world.rebol
├── Red/
│ ├── example.red
│ └── example.reds
├── Regular Expression/
│ ├── modeline-emacs.regexp
│ ├── modeline-vim.regexp
│ ├── ordinal.regex
│ └── url.regex
├── Ren'Py/
│ └── example.rpy
├── RenderScript/
│ ├── convolve3x3.rs
│ └── scenegraph_objects.rsh
├── Ring/
│ ├── hello.ring
│ ├── natural.ring
│ ├── weblib.ring
│ └── weighthistory.ring
├── RobotFramework/
│ ├── data_driven.robot
│ ├── gherkin.robot
│ └── keyword_driven.robot
├── Roff/
│ ├── Tcl.n
│ ├── an-ext.tmac
│ ├── create_view.l
│ ├── fsinterface.ms
│ ├── refs.rno
│ ├── sample.4
│ └── trekmanual.nr
├── Ruby/
│ ├── any.spec
│ ├── filenames/
│ │ ├── .irbrc
│ │ ├── .pryrc
│ │ ├── Appraisals
│ │ ├── Brewfile
│ │ ├── Capfile
│ │ ├── Dangerfile
│ │ ├── Deliverfile
│ │ ├── Fastfile
│ │ ├── Podfile
│ │ ├── Rakefile
│ │ └── Snapfile
│ ├── foo.rb
│ ├── formula.rb
│ ├── gen-rb-linguist-thrift.rb
│ ├── grit.rb
│ ├── index.json.jbuilder
│ ├── inflector.rb
│ ├── jekyll.rb
│ ├── jenkinsci.pluginspec
│ ├── macruby
│ ├── mdata_server.fcgi
│ ├── rabl.rabl
│ ├── racc.rb
│ ├── resque.rb
│ ├── rexpl
│ ├── ruby
│ ├── ruby2
│ ├── script.rake
│ ├── shoes-swt
│ └── sinatra.rb
├── Rust/
│ ├── hashmap.rs
│ ├── main.rs
│ └── task.rs
├── SAS/
│ ├── data.sas
│ ├── detect_phi.sas
│ └── proc.sas
├── SCSS/
│ └── screen.scss
├── SMT/
│ ├── bignum_lia1.smt2
│ ├── list4.smt2
│ ├── queen10-1.smt2
│ └── shufflevector.smt
├── SPARQL/
│ ├── foaf.sparql
│ └── string-matching.sparql
├── SQF/
│ ├── fn_remoteExecFnc.sqf
│ └── macros.hqf
├── SQL/
│ ├── AvailableInSearchSel.prc
│ ├── create_stuff.sql
│ ├── db.sql
│ ├── drop_stuff.sql
│ ├── dual.sql
│ ├── filial.tab
│ ├── hostcache_set_state.inc
│ ├── object-update.udf
│ ├── suspendedtoday.viw
│ ├── videodb.cql
│ ├── videodb.ddl
│ └── zipcodes.uk.mysql
├── SQLPL/
│ ├── check_reorg.sql
│ ├── comm_amount.db2
│ ├── drop_table.db2
│ ├── runstats.sql
│ ├── sleep.sql
│ └── trigger.sql
├── SRecode Template/
│ └── linguist.srt
├── STON/
│ ├── Array.ston
│ ├── Dictionary.ston
│ ├── Rectangle.ston
│ ├── TestDomainObject.ston
│ ├── ZNResponse.ston
│ ├── methodProperties.ston
│ └── properties.ston
├── Sage/
│ └── polinomios.sagews
├── SaltStack/
│ ├── eval.sls
│ ├── gimp.sls
│ ├── gpg4win-light.sls
│ ├── openoffice.sls
│ ├── top.sls
│ └── truecrypt.sls
├── Sass/
│ └── screen.sass
├── Scala/
│ ├── 99-bottles-of-beer
│ ├── build.sbt
│ ├── car-ride.kojo
│ ├── fib-tree.kojo
│ ├── node11.sc
│ ├── scala
│ └── turtle-controller.kojo
├── Scaml/
│ └── hello.scaml
├── Scheme/
│ ├── asteroids.sps
│ ├── basic.sld
│ ├── lambdastar.sls
│ └── sboyer.sch
├── Scilab/
│ ├── scilab_function.sci
│ ├── scilab_script.sce
│ └── scilab_test.tst
├── ShaderLab/
│ ├── DepthOfField.shader
│ ├── Fog.shader
│ └── Uber.shader
├── Shell/
│ ├── 99-bottles-of-beer
│ ├── bash
│ ├── build.command
│ ├── filenames/
│ │ ├── .bash_logout
│ │ ├── .bash_profile
│ │ ├── .bashrc
│ │ ├── .cshrc
│ │ ├── .login
│ │ ├── .profile
│ │ ├── .zlogin
│ │ ├── .zlogout
│ │ ├── .zprofile
│ │ ├── .zshenv
│ │ ├── .zshrc
│ │ ├── 9fs
│ │ ├── PKGBUILD
│ │ ├── bash_logout
│ │ ├── bash_profile
│ │ ├── bashrc
│ │ ├── cshrc
│ │ ├── gradlew
│ │ ├── login
│ │ ├── man
│ │ ├── profile
│ │ ├── zlogin
│ │ ├── zlogout
│ │ ├── zprofile
│ │ ├── zshenv
│ │ └── zshrc
│ ├── invalid-shebang.sh
│ ├── php.fcgi
│ ├── plugin
│ ├── rbenv-sh-shell.sh
│ ├── rvm.bash
│ ├── sbt
│ ├── script.bash
│ ├── script.sh
│ ├── script.zsh
│ ├── settime.cgi
│ ├── sh
│ ├── string-chopping
│ ├── valid-shebang.tool
│ └── zsh
├── ShellSession/
│ ├── dollar.sh-session
│ ├── gem-install.sh-session
│ └── simple.sh-session
├── Shen/
│ ├── graph.shen
│ ├── html.shen
│ └── json.shen
├── Slash/
│ └── brainfuck.sl
├── Slim/
│ └── sample.slim
├── Smali/
│ ├── ActionBarDrawerToggle.smali
│ ├── DoodleMobileAnaylise.smali
│ ├── ModernAsyncTask.smali
│ ├── PenguinSprite.smali
│ ├── Subject.smali
│ ├── ViewDragHelper.smali
│ └── WbxmlSerializer.smali
├── Smalltalk/
│ ├── Booleans.cs
│ ├── Collections.cs
│ ├── Dinner.st
│ ├── TestBasic.st
│ ├── baselineDependency.st
│ ├── categories.st
│ ├── renderSeasideExampleOn..st
│ ├── scriptWithPragma.st
│ ├── smallMethod.st
│ └── testSimpleChainMatches.st
├── Solidity/
│ └── VotingContract.sol
├── SourcePawn/
│ ├── Check.inc
│ ├── fixed.inc
│ ├── foo.sma
│ ├── foo.sp
│ ├── mfile.inc
│ └── y_testing.inc
├── Squirrel/
│ └── Squirrel.nut
├── Stan/
│ ├── congress.stan
│ ├── dogs.stan
│ └── schools.stan
├── Standard ML/
│ ├── Foo.ML
│ ├── Foo.sig
│ ├── Foo.sml
│ ├── RedBlackTree.fun
│ └── main.fun
├── Stata/
│ ├── common.doh
│ ├── hello.ado
│ ├── include.ihlp
│ ├── limits.matah
│ ├── odkmeta.sthlp
│ ├── regress_example.do
│ └── tanh.mata
├── Stylus/
│ └── demo.styl
├── SubRip Text/
│ └── Adding.NCL.Language.S01E01.1080p.BluRay.x264.srt
├── Sublime Text Config/
│ ├── AMPL.sublime-build
│ ├── CLIPS.sublime-settings
│ ├── Context.sublime-menu
│ ├── Dart.sublime-commands
│ ├── Dart.sublime-project
│ ├── Default (Linux).sublime-mousemap
│ ├── Default.sublime-keymap
│ ├── JavaDoc Add Line.sublime-macro
│ ├── RunBuild.sublime-macro
│ ├── SourcePawn.sublime-build
│ ├── Tubnil.sublime-theme
│ └── jade.sublime-completions
├── SugarSS/
│ └── sample.sss
├── SuperCollider/
│ ├── WarpPreset.sc
│ ├── WarpTate.sc
│ ├── WarpTrack.sc
│ ├── WarpUtil.sc
│ └── example.scd
├── Swift/
│ ├── section-11.swift
│ ├── section-13.swift
│ ├── section-15.swift
│ ├── section-17.swift
│ ├── section-19.swift
│ ├── section-21.swift
│ ├── section-23.swift
│ ├── section-25.swift
│ ├── section-27.swift
│ ├── section-29.swift
│ ├── section-3.swift
│ ├── section-31.swift
│ ├── section-33.swift
│ ├── section-35.swift
│ ├── section-37.swift
│ ├── section-39.swift
│ ├── section-41.swift
│ ├── section-43.swift
│ ├── section-45.swift
│ ├── section-47.swift
│ ├── section-49.swift
│ ├── section-5.swift
│ ├── section-51.swift
│ ├── section-53.swift
│ ├── section-55.swift
│ ├── section-57.swift
│ ├── section-59.swift
│ ├── section-61.swift
│ ├── section-63.swift
│ ├── section-65.swift
│ ├── section-67.swift
│ ├── section-69.swift
│ ├── section-7.swift
│ ├── section-71.swift
│ ├── section-73.swift
│ ├── section-75.swift
│ ├── section-77.swift
│ ├── section-79.swift
│ ├── section-81.swift
│ ├── section-83.swift
│ ├── section-85.swift
│ ├── section-87.swift
│ └── section-9.swift
├── SystemVerilog/
│ ├── endpoint_phy_wrapper.svh
│ ├── fifo.sv
│ ├── priority_encoder.sv
│ └── util.vh
├── TI Program/
│ ├── srcalpha.8xp.txt
│ ├── srcfunc.8xp.txt
│ ├── srcgui.8xp.txt
│ └── srcsort.8xp.txt
├── TLA/
│ ├── AsyncInterface.tla
│ └── fifo.tla
├── TXL/
│ └── Cal.txl
├── Tcl/
│ ├── filenames/
│ │ ├── owh
│ │ └── starfield
│ ├── stream-0.1.tm
│ └── xdgbasedir-0.3.tm
├── TeX/
│ ├── authortitle.cbx
│ ├── beispiel.toc
│ ├── english.lbx
│ ├── perl.toc
│ ├── problemset.cls
│ ├── reedthesis.cls
│ └── verbose.bbx
├── Tea/
│ └── foo.tea
├── Terra/
│ ├── arith.t
│ ├── arrayt.t
│ └── benchmark_nbody.t
├── Text/
│ ├── 01_top.ncl
│ ├── ISO-2022-KR.txt
│ ├── LIDARLite.ncl
│ ├── Site.local.ncl
│ ├── aptitude-defaults.nb
│ ├── filenames/
│ │ ├── COPYING.regex
│ │ ├── LICENSE.mysql
│ │ ├── README.me
│ │ ├── README.mysql
│ │ ├── click.me
│ │ ├── delete.me
│ │ ├── keep.me
│ │ ├── read.me
│ │ ├── readme.1st
│ │ └── test.me
│ ├── foo.txt
│ ├── mac.txt
│ ├── main.ncl
│ ├── messages.fr
│ ├── min-help.ncl
│ ├── readme.txt
│ ├── receiver.ncl
│ ├── rmMonAnnCycLLT-help.ncl
│ ├── tutor.nb
│ ├── tutor.no
│ └── zonalAve-help.ncl
├── Thrift/
│ └── linguist.thrift
├── Turing/
│ ├── simplegame.t
│ └── turing.t
├── Turtle/
│ ├── gnd-record.ttl
│ └── rdf-syntax-grammar.ttl
├── Type Language/
│ ├── builtin.tl
│ └── scheme.tl
├── TypeScript/
│ ├── cache.ts
│ ├── classes.ts
│ ├── hello.ts
│ ├── import.tsx
│ ├── react-native.tsx
│ ├── require.tsx
│ └── triple-slash-reference.tsx
├── Unity3D Asset/
│ ├── GapTile.mat
│ ├── Hover.anim
│ ├── Tiles.meta
│ ├── TimeManager.asset
│ └── canvas_Fullscreen_Fader.prefab
├── Unix Assembly/
│ ├── hello.ms
│ └── hello.s
├── Uno/
│ ├── PlayerPads.uno
│ ├── Pong.uno
│ └── TowerBlock.uno
├── UnrealScript/
│ ├── MutU2Weapons.uc
│ └── US3HelloWorld.uc
├── UrWeb/
│ ├── iso8601.ur
│ └── parse.urs
├── VCL/
│ ├── varnish2_default.vcl
│ └── varnish3_default.vcl
├── VHDL/
│ └── foo.vhd
├── Verilog/
│ ├── button_debounce.v
│ ├── control.v
│ ├── hex_display.v
│ ├── mux.v
│ ├── pipeline_registers.v
│ ├── ps2_mouse.v
│ ├── sha-256-functions.v
│ ├── sign_extender.v
│ ├── sqrt_pipelined.v
│ ├── t_button_debounce.v
│ ├── t_div_pipelined.v
│ ├── t_sqrt_pipelined.v
│ └── vga.v
├── Vim script/
│ ├── filenames/
│ │ ├── .gvimrc
│ │ ├── .nvimrc
│ │ ├── .vimrc
│ │ └── _vimrc
│ └── solarized.vim
├── Visual Basic/
│ ├── Index.vbhtml
│ ├── Module1.vb
│ ├── VBAllInOne.vb
│ ├── cApplication.cls
│ └── vpptype.bas
├── Volt/
│ └── tesla.volt
├── Wavefront Material/
│ ├── dice.mtl
│ ├── ripple.mtl
│ ├── shapes.mtl
│ └── spline.mtl
├── Wavefront Object/
│ ├── dice.obj
│ ├── random.obj
│ ├── ripple.obj
│ ├── shapes.obj
│ └── spline.obj
├── Web Ontology Language/
│ └── sample.owl
├── WebAssembly/
│ ├── add.wat
│ ├── fibonacci.wat
│ ├── imported-min.wast
│ ├── local-cse.wast
│ ├── print.wat
│ └── remove-unused-brs_shrink-level=1_ignore-implicit-traps.wast
├── WebIDL/
│ ├── AnimationEvent.webidl
│ └── Fetch.webidl
├── Wolfram Language/
│ └── mendeley-wolfram.m
├── World of Warcraft Addon Data/
│ ├── addon.toc
│ ├── lingua.toc
│ └── linguist.toc
├── X10/
│ ├── ArraySum.x10
│ ├── Cancellation.x10
│ ├── Fibonacci.x10
│ ├── HeatTransfer_v0.x10
│ ├── HeatTransfer_v1.x10
│ ├── HelloWholeWorld.x10
│ ├── HelloWorld.x10
│ ├── Histogram.x10
│ ├── Integrate.x10
│ ├── KMeans.x10
│ ├── KMeansDist.x10
│ ├── KMeansDistPlh.x10
│ ├── KMeansSPMD.x10
│ ├── MontyPi.x10
│ ├── NQueensDist.x10
│ ├── NQueensPar.x10
│ ├── QSort.x10
│ └── StructSpheres.x10
├── XC/
│ └── main.xc
├── XCompose/
│ └── filenames/
│ └── XCompose
├── XML/
│ ├── Application.xib
│ ├── CSharpVSPackage.vstemplate
│ ├── Default.props
│ ├── Demo.sfproj
│ ├── Example.mdpolicy
│ ├── FXMLSample.fxml
│ ├── HITSP_C32.sch
│ ├── JSBrowser.jsproj
│ ├── MDM.adml
│ ├── MDM.admx
│ ├── MainView.ux
│ ├── MyApp.ux
│ ├── NDepends_Example.ndproj
│ ├── Storyboard.storyboard
│ ├── Strings.resx
│ ├── System.Buffers.pkgproj
│ ├── XmlIO.pluginspec
│ ├── chrome.natvis
│ ├── cloudconfig.cscfg
│ ├── clouddef.csdef
│ ├── configdef.cscfg
│ ├── csproj-sample.csproj
│ ├── dependency-example.depproj
│ ├── example-sharedproj.shproj
│ ├── example.ccproj
│ ├── filenames/
│ │ └── .cproject
│ ├── fsproj-sample.fsproj
│ ├── intellij.iml
│ ├── libsomething.dll.config
│ ├── module.ant
│ ├── module.ivy
│ ├── msbuild-example.proj
│ ├── namespace-strict.sch
│ ├── net_docfile.xml
│ ├── nproj-sample.nproj
│ ├── oasis-table.sch
│ ├── phpunit.xml.dist
│ ├── point-3.1.gml
│ ├── point-3.2.gml
│ ├── pt_BR.ts
│ ├── pt_BR.xml
│ ├── racoon.mjml
│ ├── real-estate.mjml
│ ├── sample.csl
│ ├── sample.nuspec
│ ├── sample.targets
│ ├── some-ideas.mm
│ ├── source.extension.vsixmanifest
│ ├── src.builds
│ ├── tei-odd-sample.odd
│ ├── translation_en3.ts
│ ├── vbproj-sample.vbproj
│ ├── vcxproj-sample.vcxproj
│ ├── vcxproj-sample.vcxproj.filters
│ ├── water.tsx
│ ├── wixdemo.wixproj
│ ├── xhtml-struct-1.mod
│ └── xquery-tutorial.xspec
├── XPM/
│ ├── cc-public_domain_mark_white.pm
│ └── stick-unfocus.xpm
├── XPages/
│ ├── navbar.xsp-config
│ └── navbar.xsp.metadata
├── XProc/
│ └── xproc.xpl
├── XQuery/
│ └── xproc.xqm
├── XS/
│ └── CommonMark.xs
├── XSLT/
│ └── test.xslt
├── Xojo/
│ ├── App.xojo_code
│ ├── BillingReport.xojo_report
│ ├── MainMenuBar.xojo_menu
│ ├── MyToolbar.xojo_toolbar
│ ├── Window1.xojo_window
│ └── database.xojo_script
├── Xtend/
│ ├── BasicExpressions.xtend
│ └── Movies.xtend
├── YAML/
│ ├── Ansible.YAML-tmLanguage
│ ├── HexInspect.sublime-syntax
│ ├── database.yml.mysql
│ ├── filenames/
│ │ ├── .clang-format
│ │ ├── .clang-tidy
│ │ └── .gemrc
│ ├── source.r-console.syntax
│ └── vcr_cassette.yml
├── YANG/
│ └── sfc-lisp-impl.yang
├── YARA/
│ ├── OfExample.yar
│ ├── example.yara
│ └── true.yar
├── Zephir/
│ ├── Cblock.zep
│ └── Router.zep
├── Zimpl/
│ └── sample.zmpl
├── desktop/
│ └── example.desktop
├── eC/
│ └── Designer.ec
├── edn/
│ └── bigger-than-pluto.edn
├── fish/
│ ├── config.fish
│ ├── eval.fish
│ └── funced.fish
├── reStructuredText/
│ └── HACKING.rst.txt
├── sed/
│ └── hanoi.sed
├── wdl/
│ ├── hello.wdl
│ ├── ifs_in_scatters.wdl
│ └── passingfiles.wdl
├── wisp/
│ └── intro.wisp
└── xBase/
├── sample.ch
├── sample.prg
└── sample.prw
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitattributes
================================================
src/test/resources/samples/* linguist-vendored
================================================
FILE: .gitignore
================================================
.DS_Store
.gradle/
.idea/
.sourcerer
build/
sourcerer-app.iml
sourcerer-app.ipr
sourcerer-app.iws
app.iml
app.ipr
app.iws
/confluence/target
/dependencies
/dist
/gh-pages
/ideaSDK
/out
/tmp
*.versionsBackup
/ultimate/dependencies
/ultimate/ideaSDK
/ultimate/out
/ultimate/tmp
tmp_repo
================================================
FILE: CODE_OF_CONDUCT.md
================================================
# Contributor Code of Conduct
As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
This Code of Conduct is adapted from the [Contributor Covenant](http:contributor-covenant.org), version 1.0.0, available at https://www.contributor-covenant.org/version/1/0/0/code-of-conduct.html
================================================
FILE: CONTRIBUTING.md
================================================
# Contribution
We welcome contributions to [Sourcerer App](https://github.com/sourcerer-io/sourcerer-app) by the community.
See [README](https://github.com/sourcerer-io/sourcerer-app) for more information about prerequirements and building project.
## How to report a problem
Please search before creating a new issue. Feel free to add issues related to the app or [sourcerer](https://sourcerer.io) site.
## Submitting сhanges
* Open a new issue in the [issue tracker](https://github.com/sourcerer-io/sourcerer-app/issues).
* Fork and clone the repo with `git clone https://github.com/your-username/sourcerer-app.git`.
* Create a new branch based off the `develop` branch.
* Make changes.
* Make sure all tests pass.
* Submit a pull request, referencing any issues it addresses.
We will review your Pull Request as soon as possible. Thank you for contributing!
## Integration testing
We will work on a special environment for contributors in the nearest future. For now one should use his personal or additional account on site.
## Style guides
### Commit messages
Format:
```type(component): message (jira issue tag, github issue number with #)```
Message types:
* **feat** is used when new feature is provided;
* **wip** is used when making regular commit and changes don't match any other types;
* **fix** is used when you fix a bug;
* **chore** is used when changes are about organization, not about logic;
* **docs** is used when you add/change documentation.
Component is a decomposition unit your commit affected. Write message in present simple.
Examples:
feat(logger): add rotating
chore: remove redundant commas, add copyright
wip: pass models to routers
fix: program exit is prevented when button is pressed (COMP-1, #123)
### Kotlin style guide
* Code has a column limit of 80 characters.
* Inline comments should be indented with 2 spaces from code.
We are using [Kotlin Coding Conventions](https://kotlinlang.org/docs/reference/coding-conventions.html).
## Code of conduct
We value input from each member of the community, however we urge you to abide by [code of conduct](https://github.com/sourcerer-io/sourcerer-app/blob/master/CODE_OF_CONDUCT.md).
================================================
FILE: Dockerfile
================================================
FROM nginx:1.13
COPY deploy/default.conf /etc/nginx/conf.d
RUN mkdir /files
COPY build/libs/sourcerer-app.jar /files/download
COPY src/install/install /files/install
================================================
FILE: LICENSE.md
================================================
Copyright 2017 Sourcerer, Inc. https://sourcerer.io
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
================================================
FILE: README.md
================================================
sourcerer.io
A visual profile for software engineers.
Features
========
* Profile creation with a single click
* Support of 100 languages (even exotic ones like COBOL)
* Detection of more than [1,000 libraries](https://github.com/sourcerer-io/awesome-libraries) in code with per-line statistics
* Visual presentation your development experience
* *Finally!* Summary of all repositories you've contributed to :tada:
* Interesting facts about yourself
Creating your profile is just the first step for us at Sourcerer. Some of the things on our roadmap include:
* Engineers to follow and learn from
* Technology and libraries you should know about
* Projects that could use your help
Get started
===========
The easiest way to get started is with your open source repos. Go to [sourcerer.io/start](https://sourcerer.io/start), and select *Build with GitHub* and watch your profile build.
For closed source repos, you will need to use this app. If you already created an account using GitHub, you would have received an email with credentials for the app. If not, You will need a new account, which you can get at [sourcerer.io/join](https://sourcerer.io/join).
Showcase
========
Requirements
============
* Web browser
or
* Linux or macOS or Windows
* Java 8+ Platform ([JRE](http://www.oracle.com/technetwork/java/javase/downloads/jre8-downloads-2133155.html) for Linux and Windows or [JDK](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) for macOS)
* Bash for Windows (`git bash` should suffice)
Usage
=====
To install sourcerer run the following command in bash:
```
curl -s https://sourcerer.io/app/install | bash
```
To run wizard use `sourcerer` command for macOS and Linux, `java -jar sourcerer.jar` in folder `Users\user\.sourcerer` for Windows.
Use parameter `--help` for additional info.
Internals
=========
The app looks at repos locally on your machine, and then sends stats to sourcerer.io. The best way to verify is to look at the code. Protobuf messages declared in [src/main/proto/sourcerer.proto](https://github.com/sourcerer-io/sourcerer-app/blob/develop/src/main/proto/sourcerer.proto) is a good start as it describes the client-server protocol.
The Sourcerer app does **NOT** upload source code anywhere, and it **NEVER** will.
FAQ
===
### How can I process private repos?
We process only public repos using GitHub OAuth. To process private repos you need to run sourcerer app locally. See [Get started](#get-started) for instructions. Sourcerer app sends only statistical information to our servers and never sends code.
### Why do you need GitHub permissions?
We use emails to identify commit authorship, read orgs access to get list of public repositories that you've contributed to. You also need to grant access to read this public information from an organization.
### Other questions
See [sourcerer.io/faq](https://sourcerer.io/faq).
Contributing
============
We love contributions! Check out the [Contribution guide](https://github.com/sourcerer-io/sourcerer-app/blob/master/CONTRIBUTING.md) for more information. Simplest and really helpful for the community would be contribution meta information to our [supported libraries list](https://github.com/sourcerer-io/awesome-libraries). If you an author of a library you show definitely add yours to the list or you can help to someone whose work you use.
[](https://sourcerer.io/fame/sergey48k/sourcerer-io/sourcerer-app/links/0)
[](https://sourcerer.io/fame/sergey48k/sourcerer-io/sourcerer-app/links/1)
[](https://sourcerer.io/fame/sergey48k/sourcerer-io/sourcerer-app/links/2)
[](https://sourcerer.io/fame/sergey48k/sourcerer-io/sourcerer-app/links/3)
[](https://sourcerer.io/fame/sergey48k/sourcerer-io/sourcerer-app/links/4)
[](https://sourcerer.io/fame/sergey48k/sourcerer-io/sourcerer-app/links/5)
[](https://sourcerer.io/fame/sergey48k/sourcerer-io/sourcerer-app/links/6)
[](https://sourcerer.io/fame/sergey48k/sourcerer-io/sourcerer-app/links/7)
Build
=====
To build and run this application locally, you'll need latest versions of Git, Gradle and JDK installed on your computer. From your command line:
```
# Clone this repository
$ git clone https://github.com/sourcerer-io/sourcerer-app.git
# Go into the repository
$ cd sourcerer-app
# Build
$ gradle build
# Run the app
$ java -jar build/libs/sourcerer-app.jar
```
License
=======
Sourcerer is under the MIT license. See the [LICENSE](https://github.com/sourcerer-io/sourcerer-app/blob/develop/LICENSE.md) for more information.
Links
=====
* [Sourcerer Site](https://sourcerer.io/)
* [Sourcerer Blog](https://blog.sourcerer.io)
* [Follow Sourcerer on Twitter](https://twitter.com/sourcerer_io)
* [Follow Sourcerer on Facebook](https://www.facebook.com/sourcerer.io/)
================================================
FILE: build.gradle
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
buildscript {
ext.kotlin_version = '1.2.31'
repositories {
mavenCentral()
jcenter()
}
dependencies {
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.10"
classpath "org.junit.platform:junit-platform-gradle-plugin:1.0.1"
}
}
plugins {
id 'de.fuerstenau.buildconfig' version '1.1.8'
}
apply plugin: "idea"
apply plugin: "java"
apply plugin: "kotlin"
apply plugin: "application"
apply plugin: "com.google.protobuf"
apply plugin: "org.junit.platform.gradle.plugin"
buildConfig {
clsName = 'BuildConfig'
packageName = 'app'
// API.
def apiBasePath = project.hasProperty('api') ? api : 'https://sourcerer.io/api/commit'
buildConfigField 'String', 'API_BASE_PATH', apiBasePath
// Common.
buildConfigField 'String', 'PROFILE_URL', 'https://sourcerer.io/'
// App version.
buildConfigField 'int', 'VERSION_CODE', '30'
buildConfigField 'String', 'VERSION', '0.3.18'
// Logging.
buildConfigField 'String', 'ENV', project.hasProperty('env') ? env : 'production'
buildConfigField 'String', 'LOG_LEVEL', project.hasProperty('log') ? log : 'info'
buildConfigField 'boolean', 'SILENT_USER_OUTPUT', project.hasProperty('silent') ? silent : 'false'
buildConfigField 'boolean', 'SENTRY_ENABLED', project.hasProperty('sentry') ? sentry : 'false'
buildConfigField 'String', 'SENTRY_DSN', 'https://0263d6473bd24a9ba40e25aa5fb0a242:c5451dc815074bff8ce3fb9f0851f2f5@sentry.io/233260'
buildConfigField 'boolean', 'PRINT_STACK_TRACE', 'false'
// Google Analytics.
buildConfigField 'String', 'GA_BASE_PATH', 'https://www.google-analytics.com'
buildConfigField 'String', 'GA_TRACKING_ID', 'UA-107129190-2'
buildConfigField 'boolean', 'IS_GA_ENABLED', 'true'
// Models storage path.
buildConfigField 'String', 'LIBRARY_MODELS_URL', 'https://storage.googleapis.com/sourcerer-app/library-models/v2/'
// Hashing.
buildConfigField 'boolean', 'COMMIT_HASHER_ENABLED', project.hasProperty('commit-hasher-enabled') ? project.property('commit-hasher-enabled').toString() : 'true'
buildConfigField 'boolean', 'FACT_HASHER_ENABLED', project.hasProperty('fact-hasher-enabled') ? project.property('fact-hasher-enabled').toString() : 'true'
buildConfigField 'boolean', 'LONGEVITY_ENABLED', project.hasProperty('longevity-enabled') ? project.property('longevity-enabled').toString() : 'false'
buildConfigField 'long', 'HEARTBEAT_RATE', project.hasProperty('heartbeat-rate') ? project.property('heartbeat-rate').toString() : '60000'
buildConfigField 'boolean', 'META_HASHER_ENABLED', project.hasProperty('meta-hasher-enabled') ? project.property('meta-hasher-enabled').toString() : 'true'
buildConfigField 'boolean', 'DISTANCES_ENABLED', project.hasProperty('distances-enabled') ? project.property('distances-enabled').toString() : 'true'
buildConfig
}
junitPlatform {
filters {
engines {
include 'spek'
}
}
}
task cleanData {
delete 'build/libs/data'
delete 'build/kotlin/data'
}
test.dependsOn cleanData
mainClassName = "app.MainKt"
repositories {
mavenCentral()
jcenter()
maven { url "https://dl.bintray.com/jetbrains/spek" }
flatDir {
dirs 'libs'
}
}
dependencies {
compile "org.jetbrains.kotlin:kotlin-stdlib-jre8:$kotlin_version"
compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
compile "com.beust:jcommander:1.72"
compile 'com.google.protobuf:protobuf-java:3.5.1'
compile 'commons-codec:commons-codec:1.5'
compile 'com.fasterxml.jackson.core:jackson-databind:2.9.5'
compile 'com.fasterxml.jackson.module:jackson-module-kotlin:2.9.5'
compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.5'
compile 'io.reactivex.rxjava2:rxjava:2.1.12'
compile 'com.github.kittinunf.fuel:fuel:1.12.1'
compile 'com.github.kittinunf.fuel:fuel-rxjava:1.12.1'
compile 'org.eclipse.jgit:org.eclipse.jgit:4.9.0.201710071750-r'
compile 'org.slf4j:slf4j-nop:1.7.2'
compile 'io.sentry:sentry:1.7.3'
compile 'org.json:json:20180813'
testCompile 'commons-io:commons-io:2.6'
testCompile 'org.jetbrains.kotlin:kotlin-test'
testCompile 'org.jetbrains.spek:spek-api:1.1.5'
testCompile 'org.junit.platform:junit-platform-runner:1.0.1'
testRuntime 'org.jetbrains.spek:spek-junit-platform-engine:1.1.5'
}
protobuf {
protoc {
artifact = "com.google.protobuf:protoc:3.5.1"
}
}
sourceSets.main.java.srcDirs += 'build/generated/source/proto/main/java'
sourceSets.test.java.srcDirs += 'src/test/kotlin'
compileKotlin.dependsOn ':generateProto'
// Include dependent libraries in archive.
jar {
manifest {
attributes "Main-Class": "$mainClassName"
}
from {
configurations.compile.collect { it.isDirectory() ? it : zipTree(it) }
} {
exclude "META-INF/*.SF"
exclude "META-INF/*.DSA"
exclude "META-INF/*.RSA"
}
baseName 'sourcerer-app'
}
================================================
FILE: deploy/GradleDockerfile
================================================
FROM gradle:4.2.0
USER root
WORKDIR /root/app
================================================
FILE: deploy/Jenkinsfile
================================================
// Copyright 2017 Sourcerer, Inc. All Rights Reserved.
// Author: Maxim Rusak (maxim@sourcerer.io)
def label = "sourcerer-app-${UUID.randomUUID().toString()}"
def namespace = 'sandbox'
if (env.BRANCH_NAME == 'master') {
namespace = 'production'
} else if (env.BRANCH_NAME == 'develop') {
namespace = 'staging'
}
podTemplate(label: label,
containers: [
containerTemplate(name: 'jnlp', image: 'gcr.io/sourcerer-1377/jenkins-slave:v4', args: '${computer.jnlpmac} ${computer.name}'),
containerTemplate(name: 'gradle', image: 'gcr.io/sourcerer-1377/gradle:4.2.0', ttyEnabled: true, command: 'tail -f /dev/null')
],
envVars: [
envVar(key: 'NAMESPACE', value: namespace),
envVar(key: 'CONTAINER_TAG', value: "${namespace}.${env.BUILD_NUMBER}.${System.currentTimeMillis()}")
],
volumes: [
hostPathVolume(hostPath: '/var/run/docker.sock', mountPath: '/var/run/docker.sock'),
hostPathVolume(hostPath: '/usr/bin/docker', mountPath: '/usr/bin/docker')
]
) {
node(label) {
stage('checkout') {
checkout scm
}
stage('build jar and test') {
container('gradle') {
sh("./do.sh build_jar_inside")
}
}
stage('build nginx') {
container('gradle') {
sh("./do.sh build_prod_inside")
}
}
stage('push') {
sh("./do.sh push")
}
stage('deploy') {
println "Deploying to ${namespace} kubernetes namespace"
sh("./do.sh deploy")
}
}
}
================================================
FILE: deploy/default.conf
================================================
geo $dollar {
default "$";
}
server {
listen 80 default_server;
server_name _;
location /install {
alias /files/install;
sub_filter "${dollar}SERVER_EXT" 'https://$host';
sub_filter_types *;
}
location / {
root /files/;
}
}
================================================
FILE: deploy/production_env.sh
================================================
export REPLICAS="2"
================================================
FILE: deploy/sandbox_env.sh
================================================
export REPLICAS="1"
================================================
FILE: deploy/sourcerer-app.yaml
================================================
apiVersion: v1
kind: Service
metadata:
name: sourcerer-app
labels:
app: sourcerer-app
spec:
type: ClusterIP
ports:
- port: 80
targetPort: 80
selector:
app: sourcerer-app
---
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: sourcerer-app
spec:
replicas: $REPLICAS
template:
metadata:
labels:
app: sourcerer-app
spec:
containers:
- name: sourcerer-app
image: gcr.io/sourcerer-1377/sourcerer-app:$CONTAINER_TAG
imagePullPolicy: Always
resources:
requests:
cpu: 100m
memory: 100Mi
================================================
FILE: deploy/staging_env.sh
================================================
export REPLICAS="0"
================================================
FILE: do.sh
================================================
#!/bin/bash
# Copyright 2017 Sourcerer, Inc. All Rights Reserved.
# Author: Maxim Rusak (maxim@sourcerer.io)
set -x
fn_exists() {
type $1 2>/dev/null | grep -q 'is a function'
}
COMMAND=$1
shift
ARGUMENTS=${@}
CONTAINER_TAG="${CONTAINER_TAG:-latest}"
NAMESPACE="${NAMESPACE:-sandbox}"
LOG="${LOG:-debug}"
VOLUME="${BUILD_VOLUME:-$PWD}"
PROJECT=sourcerer-app
PORT=3182
REPO_NAME=gcr.io/sourcerer-1377/$PROJECT:$CONTAINER_TAG
GRADLE_VERSION=4.2.0
#--------------------#
#----- Commands -----#
#--------------------#
# run only inside build container
build_jar_inside() {
if [ "${NAMESPACE}" == "sandbox" ]; then
API="https://sandbox.sourcerer/api/commit"
LOG="debug"
elif [ "${NAMESPACE}" == "staging" ]; then
API="https://staging.sourcerer/api/commit"
LOG="info"
elif [ "${NAMESPACE}" == "local" ]; then
API="http://localhost:3181"
LOG="debug"
else
API="https://sourcerer.io/api/commit"
LOG="info"
fi
gradle -Penv=${NAMESPACE} -Plog=${LOG} -Papi=${API} build
}
build_jar() {
docker run -i -v ${VOLUME}:/home/gradle/app --workdir=/home/gradle/app \
-e LOG=${LOG} -e NAMESPACE=${NAMESPACE} \
gradle:${GRADLE_VERSION} \
./do.sh build_jar_inside
}
build_prod_inside() {
docker build -t ${REPO_NAME} .
}
deploy() {
source ./deploy/${NAMESPACE}_env.sh
envsubst < ./deploy/sourcerer-app.yaml > /tmp/deploy.yaml
kubectl --namespace=${NAMESPACE} apply -f /tmp/deploy.yaml
}
######################
run_jar() {
docker run -i -v ${VOLUME}:/app --workdir=/app gradle:${GRADLE_VERSION} \
java -jar build/libs/app.jar
}
run_prod() {
docker run -i -p ${PORT}:80 ${REPO_NAME}
}
push() {
gcloud docker -- push ${REPO_NAME}
}
#---------------------#
#----- Execution -----#
#---------------------#
fn_exists ${COMMAND}
if [ $? -eq 0 ]; then
${COMMAND} ${ARGUMENTS}
else
echo "Command not found"
fi
================================================
FILE: src/install/install
================================================
#!/usr/bin/env bash
# Copyright 2017 Sourcerer, Inc. All Rights Reserved.
# License: MIT, https://github.com/sourcerer-io/sourcerer-app/blob/master/LICENSE.md
# Source code: https://github.com/sourcerer-io
echo "Installing sourcerer app.."
SERVER=$SERVER_EXT
DOWNLOAD_URL=$SERVER/app/download
SCRIPT_DIR=$HOME/.sourcerer
JAR_DIR=$SCRIPT_DIR
if [ -f $SCRIPT_DIR/sourcerer ] ; then
read -p "Previous version of sourcerer is detected. Reinstall it? [Y/n] " yesno < /dev/tty
if [ "$yesno" ] && [ "$yesno" = "n" ] ; then
echo "Exiting"
exit
fi
fi
mkdir -p $SCRIPT_DIR
mkdir -p $JAR_DIR
curl -s $DOWNLOAD_URL > $JAR_DIR/sourcerer.jar
cat < $SCRIPT_DIR/sourcerer
#!/usr/bin/env bash
# Copyright 2017 Sourcerer, Inc. All Rights Reserved.
# License: MIT, https://github.com/sourcerer-io/sourcerer-app/blob/master/LICENSE.md
# Source code: https://github.com/sourcerer-io
if [ "\$1" = "--uninstall" ] ; then
read -p "The script will uninstall sourcerer app. Proceed? [Y/n] " yesno < /dev/tty
if [ "$yesno" ] && [ "$yesno" = "n" ] ; then
echo "Exiting"
exit
fi
rm -f /usr/local/bin/sourcerer
rm $SCRIPT_DIR/sourcerer
rm -r $JAR_DIR
echo "Done!"
exit
fi
VERSION=0
if which java > /dev/null ; then
VERSION_STR=\$(java -version 2>&1 | awk -F '"' '/version/ {print \$2}')
MAJOR_VERSION=\$(echo \$VERSION_STR | cut -d. -f1)
MINOR_VERSION=\$(echo \$VERSION_STR | cut -d. -f2)
if [[ "\$MAJOR_VERSION" = "10" || "\$MAJOR_VERSION" > "10" ]]; then # Format: 10
VERSION=10
elif [[ "\$MAJOR_VERSION" = "9" ]]; then # Format: 9.0.1
VERSION=9
elif [[ "\$MAJOR_VERSION" = "1" && "\$MINOR_VERSION" = "8" ]]; then # Format: 1.8.1
VERSION=8
else
echo "Installed $VERSION_STR version of Java is not supported."
fi
fi
if [ \$VERSION = 0 ] ; then
if [ `uname` = "Darwin" ] ; then
echo "Sourcerer requires JDK 8+ installed on the system. You can download it from:"
echo "http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html"
else
echo "Sourcerer requires JRE 8+ installed on the system. You can download it from:"
echo "http://www.oracle.com/technetwork/java/javase/downloads/jre8-downloads-2133155.html"
fi
exit 1
fi
# Java 9 and 10 requires additional parameters.
if [[ \$VERSION = 9 ]] ; then
java --add-modules java.activation --add-opens java.base/java.nio=ALL-UNNAMED -jar $JAR_DIR/sourcerer.jar "\$@"
elif [[ \$VERSION = 10 ]] ; then
java --add-opens java.base/java.nio=ALL-UNNAMED -jar $JAR_DIR/sourcerer.jar "\$@"
else
java -jar $JAR_DIR/sourcerer.jar "\$@"
fi
EOF
chmod +x $SCRIPT_DIR/sourcerer
{
rm -f /usr/local/bin/sourcerer 2> /dev/null &&
ln -s $SCRIPT_DIR/sourcerer /usr/local/bin/sourcerer 2> /dev/null &&
echo 'Done!' &&
echo 'Run sourcerer to start hashing your repos!'
} || {
echo 'We installed app to ~/.sourcerer/sourcerer.'
echo 'You can add it to $PATH or run specified command to finish installation:'
echo "sudo ln -s $SCRIPT_DIR/sourcerer /usr/local/bin/sourcerer"
echo 'Then run sourcerer to start hashing your repos!'
}
================================================
FILE: src/main/kotlin/app/Analytics.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app
import com.github.kittinunf.fuel.core.FuelError
import com.github.kittinunf.fuel.core.FuelManager
import com.github.kittinunf.fuel.core.Method
import com.github.kittinunf.fuel.core.Request
import com.google.protobuf.InvalidProtocolBufferException
import java.security.InvalidParameterException
typealias Param = Pair
/**
* Google Analytics events tracking.
*/
object Analytics {
private val IS_ENABLED = BuildConfig.IS_GA_ENABLED
private val BASE_PATH = BuildConfig.GA_BASE_PATH
private val BASE_URL = "/virtual/app/"
private val PROTOCOL_VERSION = "1"
private val TRACKING_ID = BuildConfig.GA_TRACKING_ID
private val DATA_SOURCE = "app"
private val HIT_PAGEVIEW = "pageview"
private val HIT_EXCEPTION = "exception"
private val fuelManager = FuelManager()
var uuid: String = "" // Should be set on start of the app.
var username: String = "" // Should be set on successful authorization.
init {
fuelManager.basePath = BASE_PATH
}
private fun post(params: List): Request {
return fuelManager.request(Method.POST, "/collect", params)
}
/**
* Google Analytics Measurement Protocol is used to track events.
* User iteration data is sent to GA endpoint via POST request.
* Events (or hits) mapped to virtual urls with "Data Source" parameter.
* Used parameters:
* - v: Protocol Version (Required)
* - tid: Tracking ID - used to specify GA account (Required)
* - cid: Client ID - anonymous client id (UUID type 4)
* - uid: User ID - username
* - t: Hit Type - type of event
* - dp: Document Path - virtual url
*/
fun trackEvent(event: String, params: List = listOf()) {
if (!IS_ENABLED || (username.isEmpty() && uuid.isEmpty())) {
return
}
val idParams = mutableListOf()
if (uuid.isNotEmpty()) {
idParams.add("cid" to uuid)
}
if (username.isNotEmpty()) {
idParams.add("uid" to username)
}
val defaultParams = listOf("v" to PROTOCOL_VERSION,
"tid" to TRACKING_ID,
"ds" to DATA_SOURCE,
"t" to HIT_PAGEVIEW,
"dp" to BASE_URL + event)
try {
// Send event to GA with united params.
val (_, _, result) = post(params +
defaultParams.filter { !params.contains(it) } +
idParams).responseString()
val (_, e) = result
if (e != null) { throw e }
} catch (e: Throwable) {
Logger.error(e, "Error while sending GA report", logOnly = true)
}
}
fun trackError(e: Throwable? = null) {
val url = if (e != null) getErrorUrl(e) else ""
val separator = if (url.isNotEmpty()) "/" else ""
trackEvent("error" + separator + url, listOf("t" to HIT_EXCEPTION))
}
private fun getErrorUrl(e: Throwable): String {
// Mapping for request exceptions.
when (e) {
is FuelError -> return "request"
is InvalidParameterException -> return "request/parsing"
is InvalidProtocolBufferException -> return "request/parsing"
}
// Get concrete class of exception name removing all common parts.
val name = e.javaClass.simpleName.replace("Exception", "")
.replace("Error", "")
.replace("Throwable", "")
if (name.length == 0 || name.length == 1) {
return name
}
// Divide CamelCased words in class name by dashes.
val nameCapitalized = name.toUpperCase()
var url = name[0].toString()
for (i in 1..name.length - 1) {
if (name[i] == nameCapitalized[i]) {
url += "-"
}
url += name[i]
}
return url.toLowerCase()
}
}
================================================
FILE: src/main/kotlin/app/FactCodes.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
package app
object FactCodes {
val COMMIT_DAY_WEEK = 1 // Day of week fun fact and graph.
val COMMIT_DAY_TIME = 2 // Day time fun fact.
val COMMIT_LINE_NUM_AVG = 8 // Average number of lines per commit fun fact.
val COMMIT_NUM = 9 // Used for averaging COMMIT_LINE_NUM_AVG between repos.
// A map of line numbers to commits number. Used in a commit histogram.
val COMMIT_NUM_TO_LINE_NUM = 12
val COMMIT_SHARE = 16 // Used for commit share chart.
val COMMIT_SHARE_REPO_AVG = 17 // Used for commit share chart.
val LINE_LONGEVITY = 3 // Used for longevity graph.
val LINE_LONGEVITY_REPO = 4 // Used for longevity graph.
val LINE_LEN_AVG = 10 // Average length of line fun fact.
val LINE_NUM = 11 // Used for averaging LINE_LEN_AVG between repos.
val REPO_DATE_START = 5 // Repo summary info. Date of first contribution.
val REPO_DATE_END = 6 // Repo summary info. Date of last contribution.
val REPO_TEAM_SIZE = 7 // Repo summary info. Number of contributors.
val VARIABLE_NAMING = 13 // Variables naming fun fact.
val VARIABLE_NAMING_SNAKE_CASE = 0
val VARIABLE_NAMING_CAMEL_CASE = 1
val VARIABLE_NAMING_OTHER = 2
val INDENTATION = 14
val INDENTATION_TABS = 0
val INDENTATION_SPACES = 1
val COLLEAGUES = 15
}
================================================
FILE: src/main/kotlin/app/Logger.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app
import io.sentry.Sentry
import io.sentry.context.Context
import io.sentry.event.Breadcrumb
import io.sentry.event.UserBuilder
import io.sentry.event.BreadcrumbBuilder
import java.util.*
/**
* Singleton class that logs events of different levels.
*/
object Logger {
object Events {
val START = "start"
val AUTH = "auth"
val CONFIG_SETUP = "config/setup"
val CONFIG_CHANGED = "config/changed"
val HASHING_REPO_SUCCESS = "hashing/repo/success"
val HASHING_SUCCESS = "hashing/success"
val EXIT = "exit"
}
/**
* Current log level. All that higher than this level will not be displayed.
*/
@kotlin.PublishedApi
internal val LEVEL : Int
/**
* Error level.
*/
@kotlin.PublishedApi
internal const val ERROR = 0
/**
* Warning level.
*/
@kotlin.PublishedApi
internal const val WARN = 1
/**
* Information level.
*/
@kotlin.PublishedApi
internal const val INFO = 2
/**
* Debug level.
*/
@kotlin.PublishedApi
internal const val DEBUG = 3
/**
* Trace level. For extremely detailed and high volume debug logs.
*/
@kotlin.PublishedApi
internal const val TRACE = 4
const val SILENT = BuildConfig.SILENT_USER_OUTPUT
const val SENTRY_ENABLED = BuildConfig.SENTRY_ENABLED
/**
* Print stack trace on error log.
*/
private const val PRINT_STACK_TRACE = BuildConfig.PRINT_STACK_TRACE
/**
* Context of Sentry error reporting software for adding info.
*/
private val sentryContext: Context?
/**
* Username used for error reporting.
*/
var username: String? = null
set(value) {
if (SENTRY_ENABLED) {
sentryContext?.user = UserBuilder().setUsername(value).build()
}
Analytics.username = value ?: ""
}
var uuid: String? = null
set(value) {
Analytics.uuid = value ?: ""
}
init {
if (SENTRY_ENABLED) {
Sentry.init(BuildConfig.SENTRY_DSN)
sentryContext = Sentry.getContext()
addTags()
} else {
sentryContext = null
}
LEVEL = configLevelValue()
}
private fun configLevelValue() : Int {
val a = mapOf("trace" to TRACE, "debug" to DEBUG, "info" to INFO,
"warn" to WARN, "error" to ERROR)
return a.getValue(BuildConfig.LOG_LEVEL)
}
/**
* Utils.
*/
private fun Double.format(digits: Int, digitsFloating: Int) =
java.lang.String.format("%${digits}.${digitsFloating}f", this)
private fun generateIndent(num: Int): String {
return 0.rangeTo(num).fold("") { ind, _ -> ind + " " }
}
/**
* CLI messages and pretty printing.
*/
fun print(message: Any, indentLine: Boolean = false) {
if (!SILENT) {
print(message.toString(), indentLine)
}
}
fun print(message: String, indentLine: Boolean = false) {
if (!SILENT) {
if (indentLine) {
println()
}
println(message)
}
}
fun printCommit(commitMessage: String, commitHash: String,
percents: Double) {
if (!SILENT) {
val percentsStr = percents.format(6, 2)
val hash = commitHash.substring(0, 7)
val messageTrim = if (commitMessage.length > 59) {
commitMessage.substring(0, 56).plus("...")
} else commitMessage
println(" [$percentsStr%] * $hash $messageTrim")
}
}
private val commitDetailIndent = generateIndent(10) + "|" +
generateIndent(8)
fun printCommitDetail(message: String) {
if (!SILENT) {
val messageTrim = if (message.length > 59) {
message.substring(0, 56).plus("...")
} else message
println(commitDetailIndent + messageTrim)
}
}
/**
* Log error message with exception info.
* Don't log private information with this method.
*
* @property e the exception if presented.
* @property message the message for user and logs.
* @property logOnly only log to console, no additional actions.
*/
fun error(e: Throwable, message: String = "", logOnly: Boolean = false) {
val finalMessage = if (message.isNotBlank()) { message + ": " }
else { "" } + e.message
if (LEVEL >= ERROR) {
println("[e] $finalMessage")
if (PRINT_STACK_TRACE) {
e.printStackTrace()
}
}
if (!logOnly) {
Analytics.trackError(e)
capture(e)
}
addBreadcrumb(finalMessage, Breadcrumb.Level.ERROR)
}
/**
* Log warning message. Don't log private information with this method.
*/
inline fun warn(message: () -> String) {
val msg = message()
if (LEVEL >= WARN) {
println("[w] $msg.")
}
addBreadcrumb(msg, Breadcrumb.Level.WARNING)
}
/**
* Log information message. Don't log private information with this method.
*/
inline fun info(event: String = "", message: () -> String) {
val msg = message()
if (LEVEL >= INFO) {
println("[i] $msg.")
}
if (event.isNotBlank()) {
Analytics.trackEvent(event)
}
addBreadcrumb(msg, Breadcrumb.Level.INFO)
}
/**
* Log debug message.
*/
inline fun debug(message: () -> String) {
if (LEVEL >= DEBUG) {
println("[d] ${message()}.")
}
}
/**
* Log trace message.
*/
inline fun trace(message: () -> String) {
if (LEVEL >= TRACE) {
println("[t] ${message()}.")
}
}
val isTrace: Boolean
inline get() = LEVEL >= TRACE
@kotlin.PublishedApi
internal fun addBreadcrumb(message: String, level: Breadcrumb.Level) {
if (SENTRY_ENABLED) {
sentryContext?.recordBreadcrumb(BreadcrumbBuilder()
.setMessage(message)
.setLevel(level)
.setTimestamp(Date())
.build())
}
}
private fun addTags() {
if (SENTRY_ENABLED) {
val default = "unavailable"
val osName = System.getProperty("os.name", default)
val osVersion = System.getProperty("os.version", default)
val javaVendor = System.getProperty("java.vendor", default)
val javaVersion = System.getProperty("java.version", default)
sentryContext?.addTag("environment", BuildConfig.ENV)
sentryContext?.addTag("log-level", BuildConfig.LOG_LEVEL)
sentryContext?.addTag("version", BuildConfig.VERSION)
sentryContext?.addTag("version-code", BuildConfig.VERSION_CODE
.toString())
sentryContext?.addTag("os-name", osName)
sentryContext?.addTag("os-version", osVersion)
sentryContext?.addTag("java-vendor", javaVendor)
sentryContext?.addTag("java-version", javaVersion)
}
}
private fun capture(e: Throwable) {
if (SENTRY_ENABLED) {
Sentry.capture(e)
}
}
}
================================================
FILE: src/main/kotlin/app/Main.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app
import app.api.ServerApi
import app.config.FileConfigurator
import app.model.LocalRepo
import app.ui.ConsoleUi
import app.utils.CommandConfig
import app.utils.CommandAdd
import app.utils.CommandList
import app.utils.CommandRemove
import app.utils.FileHelper.toPath
import app.utils.Options
import app.utils.PasswordHelper
import app.utils.RepoHelper
import app.utils.UiHelper
import com.beust.jcommander.JCommander
import com.beust.jcommander.MissingCommandException
import java.nio.file.Files
import java.nio.file.Path
import java.security.GeneralSecurityException
import javax.net.ssl.TrustManager
import javax.net.ssl.X509TrustManager
import java.security.cert.X509Certificate
import javax.net.ssl.HttpsURLConnection
import javax.net.ssl.SSLContext
fun main(argv : Array) {
Thread.setDefaultUncaughtExceptionHandler { _, e: Throwable ->
Logger.error(e, "Uncaught exception")
}
if (BuildConfig.ENV != "production") {
disableSslChecks()
}
Main(argv)
}
class Main(argv: Array) {
private val configurator = FileConfigurator()
private val api = ServerApi(configurator)
init {
Logger.uuid = configurator.getUuidPersistent()
Logger.info(Logger.Events.START) { "App started" }
val options = Options()
val commandAdd = CommandAdd()
val commandConfig = CommandConfig()
val commandList = CommandList()
val commandRemove = CommandRemove()
val jc: JCommander = JCommander.newBuilder()
.programName("sourcerer") // Used for usage method.
.addObject(options)
.addCommand(commandAdd.name, commandAdd)
.addCommand(commandConfig.name, commandConfig)
.addCommand(commandList.name, commandList)
.addCommand(commandRemove.name, commandRemove)
.build()
try {
jc.parse(*argv)
options.password = PasswordHelper.hashPassword(options.password)
configurator.setOptions(options)
if (options.help) {
showHelp(jc)
} else if (options.setup) {
doSetup()
} else when (jc.parsedCommand) {
commandAdd.name -> doAdd(commandAdd)
commandConfig.name -> doConfig(commandConfig)
commandList.name -> doList()
commandRemove.name -> doRemove(commandRemove)
else -> startUi()
}
} catch (e: MissingCommandException) {
Logger.warn { "No such command: ${e.unknownCommand}" }
}
Logger.info(Logger.Events.EXIT) { "App finished" }
}
private fun startUi() {
ConsoleUi(api, configurator)
}
private fun doAdd(commandAdd: CommandAdd) {
val paths = commandAdd.paths
paths.forEach {
val path = it.toPath()
val hashAll = commandAdd.hashAll
if (commandAdd.recursive) {
Files.walk(path)
.filter { p -> RepoHelper.isValidGitRepo(p) }
.forEach { p -> processPath(p, hashAll) }
} else {
processPath(path, hashAll)
}
}
}
private fun processPath(path: Path, hashAll: Boolean) {
if (RepoHelper.isValidRepo(path)) {
val localRepo = LocalRepo(path.toString())
localRepo.hashAllContributors = hashAll
configurator.addLocalRepoPersistent(localRepo)
configurator.saveToFile()
Logger.print("Added git repository at $path.")
Logger.info(Logger.Events.CONFIG_CHANGED) { "Config changed" }
} else {
Logger.warn { "No valid git repository found at specified path $path" }
}
}
private fun doConfig(commandOptions: CommandConfig) {
val (key, value) = commandOptions.pair
if (!arrayListOf("username", "password").contains(key)) {
Logger.warn { "No such key $key" }
return
}
when (key) {
"username" -> configurator.setUsernamePersistent(value)
"password" -> configurator.setPasswordPersistent(value)
}
configurator.saveToFile()
Logger.info(Logger.Events.CONFIG_CHANGED) { "Config changed" }
}
private fun doList() {
RepoHelper.printRepos(configurator.getLocalRepos(),
"Tracked repositories:",
"No tracked repositories")
}
private fun doRemove(commandRemove: CommandRemove) {
val path = commandRemove.path
// Don't validate because repository may be deleted already.
if (path != null) {
configurator.removeLocalRepoPersistent(LocalRepo(path))
configurator.saveToFile()
Logger.print("Repository removed from tracking list.")
Logger.info(Logger.Events.CONFIG_CHANGED) { "Config changed" }
} else {
Logger.print("Repository not found in tracking list.")
}
}
private fun doSetup() {
if (!configurator.isFirstLaunch()) {
if (UiHelper.confirm("Are you sure that you want to setup "
+ "Sourcerer again?", defaultIsYes = false)) {
configurator.resetAndSave()
}
}
startUi()
}
private fun showHelp(jc: JCommander) {
Logger.print("Sourcerer hashes your git repositories into intelligent "
+ "engineering profiles.")
Logger.print("If you don't have an account, please, proceed to " +
"https://sourcerer.io/join")
Logger.print("More info at https://sourcerer.io and " +
"https://github.com/sourcerer-io")
jc.usage() // Will show detailed info about usage based on annotations.
}
}
fun disableSslChecks() {
// Create a trust manager that does not validate certificate chains.
val trustAllCerts = arrayOf(object : X509TrustManager {
override fun getAcceptedIssuers(): Array {
return arrayOf()
}
override fun checkClientTrusted(certs: Array,
authType: String) {}
override fun checkServerTrusted(certs: Array,
authType: String) {}
})
// Install the all-trusting trust manager.
try {
val sc = SSLContext.getInstance("SSL")
sc.init(null, trustAllCerts, java.security.SecureRandom())
HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory())
} catch (e: GeneralSecurityException) {}
// Skip server name checking.
HttpsURLConnection.setDefaultHostnameVerifier { _, _ -> true }
}
================================================
FILE: src/main/kotlin/app/api/Api.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.api
import app.model.*
interface Api {
companion object {
val OUT_OF_DATE = 1
val STATUS_CLONING = 80
val PROCESS_STATUS_START = 100
val PROCESS_STATUS_COMPLETE = 200
val PROCESS_STATUS_FAIL = 1000
val CODE_SUCCESS = 0
val PROCESS_ERROR_TOO_MUCH_COMMITS = 4
val PROCESS_ERROR_NO_COMMITS = 5
val PROCESS_ERROR_PROCESSOR = 6
val PROCESS_ERROR_EMPTY_REPO = 8
val PROCESS_ERROR_NO_ACCESS = 9
}
fun authorize(): Result
fun getUser(): Result
fun postUser(user: User): Result
fun postRepo(repo: Repo): Result
fun postCommits(commitsList: List): Result
fun deleteCommits(commitsList: List): Result
fun postFacts(factsList: List): Result
fun postAuthors(authorsList: List): Result
fun postProcessCreate(requestNumEntries: Int): Result
fun postProcess(processEntries: List): Result
fun postAuthorDistances(authorDistanceList: List):
Result
}
================================================
FILE: src/main/kotlin/app/api/ApiError.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.api
import app.Logger
import app.model.Error
import app.model.Errors
import com.github.kittinunf.fuel.core.FuelError
import com.google.protobuf.InvalidProtocolBufferException
import java.nio.charset.Charset
import java.security.InvalidParameterException
class ApiError(exception: Exception) : Exception(exception.message) {
companion object {
private val AUTH_ERROR_CODES = listOf(401, 403)
}
// Response content.
var httpStatusCode: Int = 0
var httpResponseMessage: String = ""
var httpBodyMessage: String = ""
// Server errors from response.
var serverErrors = listOf()
// Type of errors.
var isParseError = false
var isAuthError: Boolean = false
get() = AUTH_ERROR_CODES.contains(httpStatusCode)
constructor(fuelError: FuelError) : this(fuelError as Exception) {
httpStatusCode = fuelError.response.statusCode
httpResponseMessage = fuelError.response.responseMessage
if (fuelError.response.headers["Content-Type"]
?.contains("application/octet-stream") == true) {
try {
serverErrors = Errors(fuelError.response.data).errors
} catch (e: Exception) {
Logger.error(e, "Error while parsing errors from server")
}
} else {
httpBodyMessage = fuelError.response.data
.toString(Charset.defaultCharset())
}
}
constructor(parseException: InvalidProtocolBufferException) :
this(parseException as Exception) {
isParseError = true
}
constructor(parseException: InvalidParameterException) :
this(parseException as Exception) {
isParseError = true
}
fun isWithServerCode(serverErrorCode: Int): Boolean {
return serverErrors.find { error ->
error.code == serverErrorCode } != null
}
}
fun ApiError?.ifNotNullThrow() {
if (this != null) {
throw this
}
}
fun ApiError?.isWithServerCode(serverErrorCode: Int): Boolean {
if (this != null) {
return this.isWithServerCode(serverErrorCode)
}
return false
}
================================================
FILE: src/main/kotlin/app/api/MockApi.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.api
import app.Logger
import app.model.*
class MockApi( // GET requests.
var mockUser: User = User(),
var mockRepo: Repo = Repo(),
var mockProcessEntries: List = listOf()) : Api {
// POST requests.
// In case of multiple requests.
var receivedRepos: MutableList = mutableListOf()
var receivedAddedCommits: MutableList = mutableListOf()
var receivedFacts: MutableList = mutableListOf()
var receivedAuthors: MutableList = mutableListOf()
var receivedUsers: MutableList = mutableListOf()
var receivedProcessCreate: MutableList = mutableListOf()
var receivedProcess: MutableList = mutableListOf()
var receivedDistances: MutableList = mutableListOf()
// DELETE requests.
var receivedDeletedCommits: MutableList = mutableListOf()
override fun authorize(): Result {
Logger.debug { "MockApi: authorize request" }
return Result()
}
override fun getUser(): Result {
Logger.debug { "MockApi: getUser request" }
return Result(mockUser)
}
override fun postUser(user: User): Result {
Logger.debug { "MockApi: postUser request" }
receivedUsers.add(user)
return Result()
}
override fun postRepo(repo: Repo): Result {
Logger.debug { "MockApi: postRepo request" }
receivedRepos.add(repo)
return Result(mockRepo)
}
override fun postCommits(commitsList: List): Result {
Logger.debug {
"MockApi: postCommits request (${commitsList.size} commits)"
}
receivedAddedCommits.addAll(commitsList)
return Result()
}
override fun deleteCommits(commitsList: List): Result {
Logger.debug {
"MockApi: deleteCommits request (${commitsList.size} commits)" }
receivedDeletedCommits.addAll(commitsList)
return Result()
}
override fun postFacts(factsList: List): Result {
Logger.debug { "MockApi: postFacts request (${factsList.size} facts)" }
receivedFacts.addAll(factsList)
return Result()
}
override fun postAuthors(authorsList: List): Result {
Logger.debug { "MockApi: postAuthors request (${authorsList.size} " +
"stats)" }
receivedAuthors.addAll(authorsList)
return Result()
}
override fun postProcessCreate(requestNumEntries: Int): Result {
Logger.debug { "MockApi: postProcessCreate request " +
"($requestNumEntries entries requested)" }
receivedProcessCreate.add(
Process(requestNumEntries = requestNumEntries))
return Result(Process(entries = mockProcessEntries))
}
override fun postProcess(processEntries: List): Result {
Logger.debug { "MockApi: postProcess request (${processEntries.size} " +
"entries updated)" }
receivedProcess.add(Process(entries = processEntries))
return Result()
}
override fun postAuthorDistances(authorDistanceList:
List): Result {
Logger.debug { "MockApi: postAuthorDistances request (${authorDistanceList
.size} distances)" }
receivedDistances.addAll(authorDistanceList)
return Result()
}
}
================================================
FILE: src/main/kotlin/app/api/Result.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.api
data class Result (val data: T? = null, val error: ApiError? = null) {
fun getOrThrow(): T {
if (error == null) {
return data!!
}
throw error
}
fun onErrorThrow() {
if (error != null) {
throw error
}
}
}
================================================
FILE: src/main/kotlin/app/api/ServerApi.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.api
import app.BuildConfig
import app.Logger
import app.config.Configurator
import app.model.*
import com.github.kittinunf.fuel.core.FuelManager
import com.github.kittinunf.fuel.core.Method
import com.github.kittinunf.fuel.core.Request
import com.github.kittinunf.fuel.core.Response
import com.google.protobuf.InvalidProtocolBufferException
import java.security.InvalidParameterException
class ServerApi (private val configurator: Configurator) : Api {
companion object {
private val HEADER_VERSION_CODE = "app-version-code"
private val HEADER_CONTENT_TYPE = "Content-Type"
private val HEADER_CONTENT_TYPE_PROTO = "application/octet-stream"
private val HEADER_COOKIE = "Cookie"
private val HEADER_SET_COOKIE = "Set-Cookie"
private val KEY_TOKEN = "Token="
}
val fuelManager = FuelManager()
private var token = ""
private fun cookieRequestInterceptor() = { req: Request ->
if (token.isNotEmpty()) {
req.header(Pair(HEADER_COOKIE, KEY_TOKEN + token))
}
req
}
private fun cookieResponseInterceptor() = { _: Request, res: Response ->
val newToken = res.headers[HEADER_SET_COOKIE]
?.find { it.startsWith(KEY_TOKEN) }
if (newToken != null && newToken.isNotBlank()) {
token = newToken.substringAfter(KEY_TOKEN).substringBefore(';')
}
res
}
init {
fuelManager.basePath = BuildConfig.API_BASE_PATH
fuelManager.addRequestInterceptor { cookieRequestInterceptor() }
fuelManager.addResponseInterceptor { cookieResponseInterceptor() }
}
private val username
get() = configurator.getUsername()
private val password
get() = configurator.getPassword()
private fun post(path: String): Request {
return fuelManager.request(Method.POST, path)
}
private fun get(path: String): Request {
return fuelManager.request(Method.GET, path)
}
private fun delete(path: String): Request {
return fuelManager.request(Method.DELETE, path)
}
private fun createRequestGetToken(): Request {
return post("/auth").authenticate(username, password)
.header(getVersionCodeHeader())
}
private fun createRequestGetUser(): Request {
return get("/user")
}
private fun createRequestPostUser(user: User): Request {
return post("/user").header(getContentTypeHeader())
.body(user.serialize())
}
private fun createRequestPostRepo(repo: Repo): Request {
return post("/repo").header(getContentTypeHeader())
.body(repo.serialize())
}
private fun createRequestPostCommits(commits: CommitGroup): Request {
return post("/commits").header(getContentTypeHeader())
.body(commits.serialize())
}
private fun createRequestDeleteCommits(commits: CommitGroup): Request {
return delete("/commits").header(getContentTypeHeader())
.body(commits.serialize())
}
private fun createRequestPostFacts(facts: FactGroup): Request {
return post("/facts").header(getContentTypeHeader())
.body(facts.serialize())
}
private fun createRequestPostAuthors(authors: AuthorGroup): Request {
return post("/authors").header(getContentTypeHeader())
.body(authors.serialize())
}
private fun createRequestPostProcessCreate(process: Process): Request {
return post("/process/create").header(getContentTypeHeader())
.body(process.serialize())
}
private fun createRequestPostProcess(process: Process): Request {
return post("/process").header(getContentTypeHeader())
.body(process.serialize())
}
private fun createRequestPostAuthorDistances(distances:
AuthorDistanceGroup): Request {
return post("/distances").header(getContentTypeHeader())
.body(distances.serialize())
}
private fun makeRequest(request: Request,
requestName: String,
parser: (ByteArray) -> T): Result {
var error: ApiError? = null
var data: T? = null
try {
Logger.debug { "Request $requestName initialized" }
val (_, res, result) = request.responseString()
val (_, e) = result
if (e == null) {
Logger.debug { "Request $requestName success" }
data = parser(res.data)
} else {
error = ApiError(e)
}
} catch (e: InvalidProtocolBufferException) {
error = ApiError(e)
} catch (e: InvalidParameterException) {
error = ApiError(e)
}
return Result(data, error)
}
private fun getVersionCodeHeader(): Pair {
return Pair(HEADER_VERSION_CODE, BuildConfig.VERSION_CODE.toString())
}
private fun getContentTypeHeader(): Pair {
return Pair(HEADER_CONTENT_TYPE, HEADER_CONTENT_TYPE_PROTO)
}
override fun authorize(): Result {
return makeRequest(createRequestGetToken(), "getToken", {})
}
override fun getUser(): Result {
return makeRequest(createRequestGetUser(), "getUser",
{ body -> User(body) })
}
override fun postUser(user: User): Result {
return makeRequest(createRequestPostUser(user), "postUser", {})
}
override fun postRepo(repo: Repo): Result {
if (repo.rehash.isBlank()) {
throw IllegalArgumentException()
}
return makeRequest(createRequestPostRepo(repo), "getRepo",
{ body -> Repo(body) })
}
override fun postCommits(commitsList: List): Result {
val commits = CommitGroup(commitsList)
return makeRequest(createRequestPostCommits(commits),
"postCommits", {})
}
override fun deleteCommits(commitsList: List): Result {
val commits = CommitGroup(commitsList)
return makeRequest(createRequestDeleteCommits(commits),
"deleteCommits", {})
}
override fun postFacts(factsList: List): Result {
val facts = FactGroup(factsList)
return makeRequest(createRequestPostFacts(facts), "postFacts", {})
}
override fun postAuthors(authorsList: List): Result {
val authors = AuthorGroup(authorsList)
return makeRequest(createRequestPostAuthors(authors), "postAuthors", {})
}
override fun postProcessCreate(requestNumEntries: Int): Result {
val process = Process(requestNumEntries = requestNumEntries)
return makeRequest(createRequestPostProcessCreate(process),
"postProcessCreate", { body -> Process(body) })
}
override fun postProcess(processEntries: List): Result {
// TODO(anatoly): Restrict possible status and error codes on CS.
val process = Process(entries = processEntries)
return makeRequest(createRequestPostProcess(process), "postProcess", {})
}
override fun postAuthorDistances(authorDistanceList: List):
Result {
val distances = AuthorDistanceGroup(authorDistanceList)
return makeRequest(createRequestPostAuthorDistances(distances),
"postDistances", {})
}
}
================================================
FILE: src/main/kotlin/app/config/Config.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.config
import app.model.LocalRepo
import app.utils.Options
/**
* Config data class.
*/
class Config (
var uuid: String = "",
var username: String = "",
var password: String = "",
var localRepos: MutableSet = mutableSetOf()
) {
fun addRepo(repo: LocalRepo) {
localRepos.remove(repo) // Fields may be updated.
localRepos.add(repo)
}
fun removeRepo(repo: LocalRepo) {
localRepos.remove(repo)
}
fun merge(config: Config): Config {
if (config.username.isNotEmpty()) {
username = config.username
}
if (config.password.isNotEmpty()) {
password = config.password
}
if (config.localRepos.isNotEmpty()) {
localRepos = config.localRepos
}
return this
}
fun merge(options: Options): Config {
if (options.username.isNotEmpty()) {
username = options.username
}
if (options.password.isNotEmpty()) {
password = options.password
}
return this
}
}
================================================
FILE: src/main/kotlin/app/config/Configurator.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.config
import app.model.LocalRepo
import app.model.User
import app.utils.Options
interface Configurator {
fun setOptions(options: Options)
fun getUsername(): String
fun getPassword(): String
fun isValidCredentials(): Boolean
fun getLocalRepos(): List
fun getUser(): User
fun setUsernameCurrent(username: String)
fun setPasswordCurrent(password: String)
fun getUuidPersistent(): String
fun setUsernamePersistent(username: String)
fun setPasswordPersistent(password: String)
fun addLocalRepoPersistent(localRepo: LocalRepo)
fun removeLocalRepoPersistent(localRepo: LocalRepo)
fun setUser(user: User)
fun isFirstLaunch(): Boolean
fun loadFromFile()
fun saveToFile()
fun resetAndSave()
}
================================================
FILE: src/main/kotlin/app/config/FileConfigurator.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.config
import app.Logger
import app.model.LocalRepo
import app.model.User
import app.utils.FileHelper
import app.utils.Options
import app.utils.PasswordHelper
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility
import com.fasterxml.jackson.annotation.PropertyAccessor
import com.fasterxml.jackson.core.JsonParseException
import com.fasterxml.jackson.databind.JsonMappingException
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import com.fasterxml.jackson.module.kotlin.KotlinModule
import java.io.IOException
import java.nio.file.Files
import java.nio.file.InvalidPathException
import java.nio.file.NoSuchFileException
import java.util.UUID
/**
* Singleton class that manage configs and CLI options.
*/
class FileConfigurator : Configurator {
/**
* Persistent configuration file name.
*/
private val CONFIG_FILE_NAME = "config.yaml"
// Config levels are presented in priority decreasing order.
/**
* Configuration based on CLI arguments or temporary user entered data.
*/
private var current: Config = Config()
/**
* Persistent configuration saved in config file in YAML format.
*/
private var persistent: Config = Config()
/**
* Default configuration.
*/
private val default: Config = Config()
/**
* Merger of all configuration levels. Is used to get properties.
*/
private val config: Config
get() = default.merge(persistent).merge(current)
/**
* Command-line arguments. Updates [current] on set.
*/
private var options: Options = Options()
/**
* Used to temporarily save list of repos that known by server.
*/
private var user: User = User()
/**
* Jackson's ObjectMapper.
*/
private val mapper = createMapper()
/**
* Initializer that loads persistent config.
*/
init {
loadFromFile()
assignUuidIfMissing()
}
/**
* Generates UUID for analytics on install.
*/
private fun assignUuidIfMissing() {
if (persistent.uuid.isNotEmpty()) {
return
}
persistent.uuid = UUID.randomUUID().toString()
}
/**
* Creates and setups Jackson's ObjectMapper.
*/
private fun createMapper(): ObjectMapper {
return ObjectMapper(YAMLFactory()) // Enable YAML parsing.
// Map only fields (not getters, etc).
.setVisibility(PropertyAccessor.ALL, Visibility.NONE)
.setVisibility(PropertyAccessor.FIELD, Visibility.ANY)
.registerModule(KotlinModule()) // Enable Kotlin support.
}
override fun setOptions(options: Options) {
current.merge(options)
this.options = options
}
/**
* Gets username from merger of all configuration levels.
*/
override fun getUsername(): String {
return config.username
}
/**
* Gets hashed password from merger of all configuration levels.
*/
override fun getPassword(): String {
return config.password
}
/**
* Checks for non empty credentials from merger of all configuration levels.
*/
override fun isValidCredentials(): Boolean {
return config.username.isNotEmpty() && config.password.isNotEmpty()
}
/**
* Gets list of repos from merger of all configuration levels.
*/
override fun getLocalRepos(): List {
return config.localRepos.toList()
}
/**
* Gets temprorary saved user information.
*/
override fun getUser(): User {
return user
}
/**
* Sets username to current launch temprorary config.
*/
override fun setUsernameCurrent(username: String) {
current.username = username
}
/**
* Sets and hashes password to current launch temprorary config.
*/
override fun setPasswordCurrent(password: String) {
current.password = PasswordHelper.hashPassword(password)
}
/**
* Gets UUID.
*/
override fun getUuidPersistent(): String {
return persistent.uuid
}
/**
* Sets username to persistent config. Use [saveToFile] to save.
*/
override fun setUsernamePersistent(username: String) {
persistent.username = username
}
/**
* Sets and hashes password to persistent config. Use [saveToFile] to save.
*/
override fun setPasswordPersistent(password: String) {
persistent.password = PasswordHelper.hashPassword(password)
}
/**
* Add repo to persistent config. Use [saveToFile] to save.
*/
override fun addLocalRepoPersistent(localRepo: LocalRepo) {
persistent.addRepo(localRepo)
}
/**
* Remove repo from persistent config. Use [saveToFile] to save.
*/
override fun removeLocalRepoPersistent(localRepo: LocalRepo) {
persistent.removeRepo(localRepo)
}
/**
* Temporarily sets info about user.
*/
override fun setUser(user: User) {
this.user = user
}
/**
* Defines whether this is the first run. If any fields are defined then no.
*/
override fun isFirstLaunch(): Boolean {
return persistent.password.isEmpty()
&& persistent.username.isEmpty()
&& persistent.localRepos.isEmpty()
}
/**
* Loads [persistent] configuration from config file.
*/
override fun loadFromFile() {
// Сonfig initialization in case an exception is thrown.
var loadConfig = Config()
try {
loadConfig = Files.newBufferedReader(FileHelper
.getPath(CONFIG_FILE_NAME)).use {
mapper.readValue(it, Config::class.java)
}
} catch (e: IOException) {
if (e is NoSuchFileException){
Logger.warn { "No config file found" }
} else {
Logger.error(e, "Cannot access config file")
}
} catch (e: SecurityException) {
Logger.error(e, "Cannot access config file")
} catch (e: InvalidPathException) {
Logger.error(e, "Cannot access config file")
} catch (e: JsonParseException) {
Logger.error(e, "Cannot parse config file")
} catch (e: JsonMappingException) {
Logger.error(e, "Cannot parse config file")
} catch (e: IllegalStateException) {
Logger.error(e, "Cannot parse config file")
}
persistent = loadConfig
}
/**
* Saves [persistent] configuration to config file.
*/
override fun saveToFile() {
try {
Files.newBufferedWriter(FileHelper.getPath(CONFIG_FILE_NAME)).use {
mapper.writeValue(it, persistent)
}
} catch (e: IOException) {
Logger.error(e, "Cannot save config file")
} catch (e: SecurityException) {
Logger.error(e, "Cannot save config file")
} catch (e: InvalidPathException) {
Logger.error(e, "Cannot save config file")
} catch (e: JsonParseException) {
Logger.error(e, "Cannot parse config file")
} catch (e: JsonMappingException) {
Logger.error(e, "Cannot parse config file")
} catch (e: IllegalStateException) {
Logger.error(e, "Cannot parse config file")
}
}
/**
* Resets all configurations, CLI options and config file.
*/
override fun resetAndSave() {
options = Options()
persistent = Config()
saveToFile()
}
}
================================================
FILE: src/main/kotlin/app/config/MockConfigurator.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.config
import app.model.LocalRepo
import app.model.User
import app.utils.Options
class MockConfigurator(var mockUsername: String = "",
var mockPassword: String = "",
var mockIsValidCredentials: Boolean = true,
var mockIsFirstLaunch: Boolean = true,
var mockUser: User = User(),
var mockLocalRepos: MutableList =
mutableListOf(),
var uuid: String = "") : Configurator {
var mockCurrent: Config = Config()
var mockPersistent: Config = Config()
var mockOptions: Options = Options()
override fun setOptions(options: Options) {
mockOptions = options
}
override fun getUsername(): String {
return mockUsername
}
override fun getPassword(): String {
return mockPassword
}
override fun isValidCredentials(): Boolean {
return mockIsValidCredentials
}
override fun getLocalRepos(): List {
return mockLocalRepos
}
override fun getUser(): User {
return mockUser
}
override fun setUsernameCurrent(username: String) {
mockCurrent.username = username
}
override fun setPasswordCurrent(password: String) {
mockCurrent.password = password
}
override fun getUuidPersistent(): String {
return uuid
}
override fun setUsernamePersistent(username: String) {
mockPersistent.username = username
}
override fun setPasswordPersistent(password: String) {
mockPersistent.password = password
}
override fun addLocalRepoPersistent(localRepo: LocalRepo) {
mockPersistent.localRepos.remove(localRepo)
mockPersistent.localRepos.add(localRepo)
}
override fun removeLocalRepoPersistent(localRepo: LocalRepo) {
mockPersistent.localRepos.remove(localRepo)
}
override fun setUser(user: User) {
mockUser = user
}
override fun isFirstLaunch(): Boolean {
return mockIsFirstLaunch
}
override fun loadFromFile() {}
override fun saveToFile() {}
override fun resetAndSave() {}
}
================================================
FILE: src/main/kotlin/app/extractors/CExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
class CExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.C
val importRegex = Regex("""^([^\n]*#include)\s[^\n]*""")
val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
val extractImportRegex =
Regex("""#include\s+["<](\w+)[/\w+]*\.\w+[">]""")
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = extractImportRegex.find(it)
if (res != null) {
val lineLib = res.groupValues.last()
imports.add(lineLib)
}
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
// TODO(lyaronskaya): Add C to libraries.
return super.mapImportToIndex(import, Lang.CPP, startsWith = true)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/CSharpExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
class CSharpExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.CSHARP
val importRegex = Regex("""^.*using\s+(\w+[.\w+]*)""")
val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
val extractImportRegex = Regex("""using\s+(\w+[.\w+]*)""")
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = extractImportRegex.find(it)
if (res != null) {
imports.add(res.groupValues[1])
}
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
return super.mapImportToIndex(import, lang, startsWith = true)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/ClassifierManager.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
import app.BuildConfig
import app.Logger
import app.model.Classifier
import app.model.LibraryMeta
import app.utils.FileHelper
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClientBuilder
import java.io.FileOutputStream
class ClassifierManager {
companion object {
private const val CLASSIFIERS_DIR = "classifiers"
private const val DATA_EXT = ".pb"
private const val LIBS_META_DIR = ClassifierManager.CLASSIFIERS_DIR
private const val LIBS_META_FILENAME = "libraries_meta.pb"
}
val cache = hashMapOf()
val libsMeta = getLibraryMeta()
/**
* Returns libraries used in a line.
*/
fun estimate(line: List, libraries: List): List {
return libraries.filter { libId ->
if (!cache.containsKey(libId)) {
// Library not downloaded from cloud storage.
if (FileHelper.notExists(libId + DATA_EXT, CLASSIFIERS_DIR)) {
Logger.info { "Downloading $libId classifier" }
downloadClassifier(libId)
Logger.info { "Finished downloading $libId classifier" }
}
// Library not loaded from local storage.
Logger.info { "Loading $libId evaluator" }
loadClassifier(libId)
Logger.info { "$libId evaluator ready" }
}
// Check line for usage of a library.
val prediction = cache[libId]!!.evaluate(line)
// Prediction based on two classes.
val prob = prediction[cache[libId]!!.libraries.indexOf(libId)]
// Define lower bound of classifier output
// that depends on data used to create the model.
// TODO(lyaronskaya): move thresholds to protobuf.
if (libId == "rb.rails") {
prob > 0.91
} else if (libId == "py.flask") {
prob > 0.95
} else if (libId.startsWith(Lang.KOTLIN)) {
prob > 0.8
} else if (libId.startsWith(Lang.PLPGSQL)) {
prob > 0.7
} else if (libId.startsWith(Lang.PHP)) {
prob > 0.75
} else if (libId.startsWith(Lang.PYTHON)) {
prob > 0.75
} else if (libId.startsWith(Lang.SCALA)) {
prob > 0.85
} else if (libId.startsWith(Lang.GO)) {
prob > 0.75
} else if (libId.startsWith(Lang.JAVA)) {
prob > 0.9
} else if (libId.startsWith(Lang.CRYSTAL)) {
prob > 0.7
} else if (libId.startsWith(Lang.PERL)) {
prob > 0.8
} else if (libId.startsWith("rb.dry-")) {
prob > 0.85
} else if (libId == "js.q") {
prob > 0.9
} else if (libId == "cpp.gflags") {
prob > 0.9
} else if (libId == "dart.flutter") {
prob > 0.85
}
else if (libId.startsWith(Lang.CPP)) {
prob > 0.8
}
else if (libId.startsWith(Lang.JAVASCRIPT)) {
prob > 0.7
}
else {
prob > 0.5
}
}
}
/**
* Downloads libraries from cloud.
*/
private fun downloadClassifier(libId: String) {
val file = FileHelper.getFile(libId + DATA_EXT, CLASSIFIERS_DIR)
val langId = libId.split('.')[0]
val url = "${BuildConfig.LIBRARY_MODELS_URL}$langId/$libId$DATA_EXT"
val builder = HttpClientBuilder.create()
val client = builder.build()
try {
client.execute(HttpGet(url)).use { response ->
val entity = response.entity
if (entity != null) {
FileOutputStream(file).use { outstream ->
entity.writeTo(outstream)
outstream.flush()
outstream.close()
}
}
}
} catch (e: Exception) {
Logger.error(e, "Failed to download $libId classifier")
}
}
/**
* Loads libraries from local storage to cache.
*/
private fun loadClassifier(libId: String) {
val bytesArray = FileHelper.getFile(libId + DATA_EXT, CLASSIFIERS_DIR)
.readBytes()
cache[libId] = Classifier(bytesArray)
}
/**
* Downloads libraries meta data from cloud.
*/
private fun downloadLibrariesMeta() {
val file = FileHelper.getFile(LIBS_META_FILENAME, LIBS_META_DIR)
val url = BuildConfig.LIBRARY_MODELS_URL + LIBS_META_FILENAME
val builder = HttpClientBuilder.create()
val client = builder.build()
try {
client.execute(HttpGet(url)).use { response ->
val entity = response.entity
if (entity != null) {
FileOutputStream(file).use { outstream ->
entity.writeTo(outstream)
outstream.flush()
outstream.close()
}
}
}
} catch (e: Exception) {
Logger.error(e, "Failed to download $LIBS_META_FILENAME")
}
}
/**
* Loads libraries meta data from local storage.
*/
private fun getLibraryMeta(): LibraryMeta {
Logger.info { "Downloading $LIBS_META_FILENAME" }
downloadLibrariesMeta()
Logger.info { "Finished downloading $LIBS_META_FILENAME" }
val bytesArray = FileHelper.getFile(LIBS_META_FILENAME,
LIBS_META_DIR).readBytes()
return LibraryMeta(bytesArray)
}
}
================================================
FILE: src/main/kotlin/app/extractors/CommonExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
import app.model.CommitStats
import app.model.DiffFile
class CommonExtractor(private val langName: String) : ExtractorInterface {
override fun extract(files: List): List {
files.map { file -> file.lang = langName }
return super.extract(files)
}
}
================================================
FILE: src/main/kotlin/app/extractors/CppExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
import app.model.CommitStats
import app.model.DiffFile
class CppExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.CPP
const val TEMPLATE = "template"
val importRegex = Regex("""^([^\n]*#include)\s[^\n]*""")
val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
val extractImportRegex = Regex("""#include\s+["<](\w+[/\w+]*(\.\w+)?)[">]""")
val templateRegex = Regex("""template\s*<""")
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = extractImportRegex.find(it)
if (res != null) {
val lineLib = res.groupValues
.last { !it.startsWith(".") && it != ""}
imports.add(lineLib)
}
}
return imports.toSet().toList()
}
override fun extract(files: List): List {
val stats = super.extract(files).toMutableList()
// Templates fun fact.
val allAdded = files.map{ file -> file.getAllAdded() }.flatten()
val allDeleted = files.map{ file -> file.getAllDeleted() }.flatten()
val templateAllAdded = allAdded.filter { isTemplate(it) }.size
val templateAllDeleted = allDeleted.filter { isTemplate(it) }.size
if (templateAllAdded > 0 || templateAllDeleted > 0) {
stats.add(CommitStats(
templateAllAdded, templateAllDeleted, ExtractorInterface.TYPE_SYNTAX,
tech = LANGUAGE_NAME + ExtractorInterface.SEPARATOR + TEMPLATE
))
}
return stats
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
return super.mapImportToIndex(import, lang, startsWith = true)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
private fun isTemplate(line: String): Boolean {
return line.contains(templateRegex)
}
}
================================================
FILE: src/main/kotlin/app/extractors/CrystalExtractor.kt
================================================
// Copyright 2019 Sourcerer Inc. All Rights Reserved.
// Author: Anton Maminov (anton.maminov@gmail.com)
package app.extractors
class CrystalExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.CRYSTAL
val importRegex = Regex("""require\s+\"(\w+)\"""")
val commentRegex = Regex("""^([^\n]*#)[^\n]*""")
val extractImportRegex = Regex("""require\s+\"(.+)\"""")
val includeRegex = Regex("""include\s+(\w+)::.+""")
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = extractImportRegex.find(it)
if (res != null) {
val lineLib = res.groupValues.last { it != "" }
imports.add(lineLib)
}
}
if (imports.isEmpty()) {
fileContent.forEach {
val res = includeRegex.find(it)
if (res != null) {
imports.add(res.groupValues.last().toLowerCase())
}
}
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
return super.mapImportToIndex(import, lang, startsWith = true)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/CssExtractor.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
import app.model.CommitStats
import app.model.DiffFile
class CssExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.CSS
val FILE_EXTS = listOf("css", "scss", "less", "sass")
}
override fun extract(files: List): List {
files.map { file -> file.lang = LANGUAGE_NAME }
val stats = FILE_EXTS.filter { it != "css" }.map { extension ->
val result = files.filter { it.extension == extension }
.fold(Pair(0, 0)) { total, file ->
val currentNumAdded = file.getAllAdded()
.filter { it.isNotBlank() }.size
val currentNumDeleted = file.getAllDeleted()
.filter { it.isNotBlank() }.size
Pair(total.first + currentNumAdded,
total.second + currentNumDeleted)}.toList()
CommitStats(numLinesAdded = result[0],
numLinesDeleted = result[1],
type = ExtractorInterface.TYPE_LIBRARY,
tech = extension)
}.filter { it.numLinesAdded > 0 || it.numLinesDeleted > 0 }
return stats + super.extract(files)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/DMExtractor.kt
================================================
// Copyright 2019 Sourcerer Inc. All Rights Reserved.
// Author: Nickolas Gupton (nickolas@gupton.xyz)
package app.extractors
import app.model.CommitStats
import app.model.DiffFile
class DMExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.DM
}
override fun extractLibStats(files: List): List {
val mapExtension = ".dmm"
val spriteExtension = ".dmi"
val mapFiles = files.filter { it.path.endsWith(mapExtension) }
val spriteFiles = files.filter { it.path.endsWith(spriteExtension) }
// Add stats from *.dmm files.
val mapStats = listOf(CommitStats(
numLinesAdded = mapFiles.map { it.getAllAdded().size }.sum(),
numLinesDeleted = mapFiles.map { it.getAllDeleted().size }.sum(),
type = ExtractorInterface.TYPE_LIBRARY,
tech = "dm.byond-mapping"
)).filter { it.numLinesAdded > 0 || it.numLinesDeleted > 0 }
// Add stats from *.dmi files.
val spriteStats = listOf(CommitStats(
numLinesAdded = spriteFiles.map { it.getAllAdded().size }.sum(),
numLinesDeleted = spriteFiles.map { it.getAllDeleted().size }.sum(),
type = ExtractorInterface.TYPE_LIBRARY,
tech = "dm.byond-sprites"
)).filter { it.numLinesAdded > 0 || it.numLinesDeleted > 0 }
return mapStats + spriteStats;
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/DartExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Alexander Surkov (alex@sourcerer.io)
package app.extractors
object DartExtractor : ExtractorBase(
language = Lang.DART,
importRegex = Regex("""^import ['"](.+)['"];$"""),
commentRegex = Regex("(//.+$)|(/[*].*?[*]/)"))
================================================
FILE: src/main/kotlin/app/extractors/DevopsExtractor.kt
================================================
// Copyright 2019 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
import app.model.CommitStats
import app.model.DiffFile
class DevopsExtractor(private val techName: String) : ExtractorInterface {
companion object {
const val DEVOPS = "devops."
const val JENKINS = "jenkins"
const val CIRCLECI = "circleci"
const val GITLAB_CI = "gitlab-ci"
const val GITHUB_ACTIONS = "github-actions"
const val TRAVIS = "travis"
const val K8S = "k8s"
const val DOCKER = "docker"
const val DRONE = "drone"
}
override fun extract(files: List): List {
return listOf(CommitStats(
numLinesAdded = files.map { it.getAllAdded().size }.sum(),
numLinesDeleted = files.map { it.getAllDeleted().size }.sum(),
type = ExtractorInterface.TYPE_LIBRARY,
tech = DEVOPS + techName
)).filter { it.numLinesAdded > 0 || it.numLinesDeleted > 0 }
}
}
================================================
FILE: src/main/kotlin/app/extractors/ElixirExtractor.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Alexander Surkov (alex@sourcerer.io)
package app.extractors
object ElixirExtractor : ExtractorBase(
language = Lang.ELIXIR,
importRegex = Regex("""^\s+(?:use|import|require) ([a-zA-Z_][a-zA-Z0-9_]*)"""),
commentRegex = Regex("""#.*$"""))
================================================
FILE: src/main/kotlin/app/extractors/Extractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
package app.extractors
import app.model.CommitStats
import app.model.DiffFile
class Extractor : ExtractorInterface {
companion object {
val RESTRICTED_EXTS = listOf(".min.js")
}
override fun extract(files: List): List {
return files
.filter { file -> !RESTRICTED_EXTS.contains(file.extension) }
.mapNotNull { file ->
Heuristics.analyze(file)
}
.fold(mutableListOf()) { accStats, stats ->
accStats.addAll(stats)
accStats
}
}
}
================================================
FILE: src/main/kotlin/app/extractors/ExtractorBase.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Alexander Surkov (alex@sourcerer.io)
package app.extractors
open class ExtractorBase(private val language: String,
private val importRegex: Regex,
private val commentRegex: Regex,
private val importStartsWith: Boolean = false) :
ExtractorInterface {
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val line = commentRegex.replace(it, "")
val res = importRegex.find(line)
if (res != null) {
val lineLib = res.groupValues[1]
imports.add(lineLib)
}
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun mapImportToIndex(import: String, lang: String, startsWith: Boolean): String? {
return super.mapImportToIndex(import, lang, startsWith = importStartsWith)
}
override fun getLanguageName(): String? {
return language
}
}
================================================
FILE: src/main/kotlin/app/extractors/ExtractorInterface.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
package app.extractors
import app.model.*
interface ExtractorInterface {
companion object {
const val TYPE_LANGUAGE = 1
const val TYPE_LIBRARY = 2
const val TYPE_KEYWORD = 3
const val TYPE_SYNTAX = 4
const val SEPARATOR = ">"
private val classifierManager = ClassifierManager()
val stringRegex = Regex("""(".+?"|'.+?')""")
val splitRegex = Regex("""\s|,|;|\*|\n|\(|\)|\[|]|\{|}|\+|=|&|\$|""" +
"""!=|\.|>|<|#|@|:|\?|!""")
}
// Identify libs used in a line with classifiers.
fun determineLibs(line: String, importedLibs: List): List {
val lang = getLanguageName()
if (lang != null) {
return classifierManager.estimate(tokenize(line), importedLibs)
}
return listOf()
}
// Should be defined for each language otherwise libs extraction disabled.
fun extractImports(fileContent: List): List {
return listOf()
}
// Should be defined for additional statistics like keywords.
fun extract(files: List): List {
val lang = getLanguageName()
if (lang != null) {
files.forEach { file -> file.lang = lang }
}
return extractLangStats(files) + extractLibStats(files)
}
fun extractLangStats(files: List): List {
return files.filter { file -> file.lang.isNotBlank() }
.groupBy { file -> file.lang }
.map { (language, files) -> CommitStats(
numLinesAdded = files.fold(0) { total, file ->
total + file.getAllAdded().size },
numLinesDeleted = files.fold(0) { total, file ->
total + file.getAllDeleted().size },
type = TYPE_LANGUAGE,
tech = language)
}
}
fun extractLibStats(files: List): List {
val oldLibs = extractLibsOfDiffs(files.map { Pair(it.lang, it.old) })
val newLibs = extractLibsOfDiffs(files.map { Pair(it.lang, it.new) })
val allLibsIds = oldLibs.keys + newLibs.keys
return allLibsIds.map { libId -> CommitStats(
numLinesAdded = newLibs.getOrDefault(libId, 0),
numLinesDeleted = oldLibs.getOrDefault(libId, 0),
type = TYPE_LIBRARY,
tech = libId
) }.filter { it.numLinesAdded > 0 || it.numLinesDeleted > 0 }
}
fun extractLibsOfDiffs(diffs: List>):
Map {
val libsCount = mutableMapOf()
// Extract imports from files.
diffs.forEach { (_, diff) ->
diff.imports = extractImports(diff.content)
}
// Skip library stats calculation if no imports found.
if (!diffs.any({ (_, diff) -> diff.imports.isNotEmpty() })) {
return mapOf()
}
// Determine libraries used in each line.
diffs.filter { (lang, _) -> lang.isNotBlank() }
.forEach { (lang, diff) ->
val importedLibs = diff.imports.mapNotNull { import ->
mapImportToIndex(import, lang)
}
diff.getAllDiffs().forEach { line ->
determineLibs(line, importedLibs).forEach { libId ->
libsCount[libId] = libsCount.getOrDefault(libId, 0) + 1
}
}
}
return libsCount
}
fun tokenize(line: String): List {
// TODO(lyaronskaya): Multiline comment regex.
// TODO(anatoly): Optimize this regex, better to get rid of it.
val newLine = stringRegex.replace(line, "")
val tokens = newLine.split(' ', '[', ',', ';', '*', '\n', ')', '(',
'[', ']', '}', '{', '+', '-', '=', '&', '$', '!', '.', '>', '<',
'#', '@', ':', '?', ']')
.filter {
it.isNotBlank() && !it.contains('"') && !it.contains('\'') &&
it != "-" && it != "@"
}
return tokens
}
fun getLanguageName(): String? {
return null
}
fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean = false): String? {
val libsMeta = classifierManager.libsMeta
if (!libsMeta.importToIndexMap.contains(lang)) return null
if (startsWith) {
val map = libsMeta.importToIndexMap[lang]
val baseImports = map!!.keys.filter { import.startsWith(it) }
if (baseImports.isEmpty()) {
return null
}
val baseImport = baseImports.reduce { acc, s ->
if (s.length > acc.length) s else acc
}
return map[baseImport]
}
return libsMeta.importToIndexMap[lang]!![import]
}
}
================================================
FILE: src/main/kotlin/app/extractors/FSharpExtractor.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Tuomas Hietanen
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
class FSharpExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.FSHARP
val importRegex = Regex("""^.*open\s+(\w+[.\w+]*)""")
val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
val extractImportRegex = Regex("""open\s+(\w+[.\w+]*)""")
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = extractImportRegex.find(it)
if (res != null) {
imports.add(res.groupValues[1])
}
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
// The behaviour of fsharp library classifier is the same as for csharp.
return super.mapImportToIndex(import, Lang.CSHARP, startsWith = true)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/GoExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
class GoExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.GO
val importRegex = Regex("""^(.*import)\s[^\n]*""")
val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
val singleImportRegex = Regex("""import\s+"(.+?)"""")
val multipleImportRegex = Regex("""import[\s\t\n]+\((.+?)\)""",
RegexOption.DOT_MATCHES_ALL)
val separatorsRegex = Regex("""(\t+|\n+|\s+|")""")
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = singleImportRegex.find(it)
if (res != null) {
val lineLib = res.groupValues.last()
imports.add(lineLib)
}
}
val contentJoined = fileContent.joinToString(separator = "")
multipleImportRegex.findAll(contentJoined).forEach { matchResult ->
imports.addAll(matchResult.groupValues.last()
.split(separatorsRegex)
.filter { it.isNotEmpty() }
.map { it.replace("\"", "") })
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
return super.mapImportToIndex(import, lang, startsWith = true)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/Heuristics.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Alexander Surkov (alex@sourcerer.io)
package app.extractors
import app.model.DiffFile
import app.model.CommitStats
val ActionscriptRegex = Regex(
"^\\s*(package\\s+[a-z0-9_\\.]+|import\\s+[a-zA-Z0-9_\\.]+;|class\\s+[A-Za-z0-9_]+\\s+extends\\s+[A-Za-z0-9_]+)",
RegexOption.MULTILINE
)
val CoqRegex = Regex(
"""^Require\s""",
RegexOption.MULTILINE
)
val CommonLispRegex = Regex(
"^\\s*\\((defun|in-package|defpackage) ",
setOf(RegexOption.MULTILINE, RegexOption.IGNORE_CASE)
)
val CppRegex = Regex(
"(template |class |namespace |#include )",
RegexOption.MULTILINE
)
val DRegex = Regex(
"^module\\s+[\\w.]*\\s*;|import\\s+[\\w\\s,.:]*;|\\w+\\s+\\w+\\s*\\(.*\\)(?:\\(.*\\))?\\s*\\{[^}]*\\}|unittest\\s*(?:\\(.*\\))?\\s*\\{[^}]*\\}",
RegexOption.MULTILINE
)
val DtraceRegex = Regex(
"^(\\w+:\\w*:\\w*:\\w*|BEGIN|END|provider\\s+|(tick|profile)-\\w+\\s+\\{[^}]*\\}|#pragma\\s+D\\s+(option|attributes|depends_on)\\s|#pragma\\s+ident\\s)",
RegexOption.MULTILINE
)
val FilterscriptRegex = Regex(
"#include|#pragma\\s+(rs|version)|__attribute__"
)
val FSharpRegex = Regex(
"^\\s*(#light|import|let|module|namespace|open|type)",
RegexOption.MULTILINE
)
val ForthRegex = Regex(
"^: "
)
val ForthFsRegex = Regex(
"^(: |new-device)"
)
val FortranRegex = Regex(
"^([c*][^abd-z]| (subroutine|program|end|data)\\s|\\s*!)",
RegexOption.IGNORE_CASE
)
val GlslRegex = Regex(
"^\\s*(#version|precision|uniform|varying|vec[234])",
RegexOption.IGNORE_CASE
)
val IdlRegex = Regex(
"^\\s*function[ \\w,]+$",
RegexOption.MULTILINE
)
val IniPropsRegex = Regex(
"\\w+\\s*=\\s*",
RegexOption.IGNORE_CASE
)
val LexRegex = Regex(
"^(%[%{}]xs|<.*>)",
RegexOption.MULTILINE
)
val LimboRegex = Regex(
"^\\w+\\s*:\\s*module\\s*\\{",
RegexOption.MULTILINE
)
val MathematicaRegex = Regex(
"\\*\\)$",
RegexOption.MULTILINE
)
val MatlabRegex = Regex(
"""(^\s*%)|(^end$)""",
RegexOption.MULTILINE
)
val MRegexs = setOf(
Regex(
"^\\s*;",
RegexOption.MULTILINE
),
Regex(
"^\\w+\\s;",
RegexOption.MULTILINE
)
)
val MakefileRegex = Regex(
"([\\/\\\\].*:\\s+.*\\s\\\\$|: \\\\$|^ : |^[\\w\\s\\/\\\\.]+\\w+\\.\\w+\\s*:\\s+[\\w\\s\\/\\\\.]+\\w+\\.\\w+)"
)
val MufRegex =Regex(
"^: ",
RegexOption.MULTILINE
)
val NewLispRegex = Regex(
"^\\s*\\(define ",
RegexOption.MULTILINE
)
val NotSqlRegex = Regex(
"begin|boolean|package|exception",
RegexOption.IGNORE_CASE
)
val ObjectiveCRegex = Regex(
"^\\s*(@(interface|class|protocol|property|end|synchronised|selector|implementation)\\b|#import\\s+.+\\.h[\">])",
RegexOption.MULTILINE
)
val OcamlRegex = Regex(
"(^\\s*module)|let rec |match\\s+(\\S+\\s)+with",
RegexOption.MULTILINE
)
val PascalRegex = Regex(
"(^\\s*uses)|(function)|(program)",
setOf(RegexOption.MULTILINE, RegexOption.IGNORE_CASE)
)
val Perl5Regex = Regex(
"\\buse\\s+(?:strict\\b|v?5\\.)"
)
val Perl6Regex = Regex(
"^\\s*(?:use\\s+v6\\b|\\bmodule\\b|\\b(?:my\\s+)?class\\b)",
RegexOption.MULTILINE
)
val PhpRegex = Regex(
"^<\\?(?:php)?"
)
val PhpIlluminateRegex = Regex(
"(Auth|Bootstrap|Bus|Console|Events|Exceptions|Http|Providers" +
"|Support|Testing|Validation)")
val PicoLispRegex = Regex(
"^\\((de|class|rel|code|data|must)\\s",
RegexOption.MULTILINE
)
val PlpgsqlRegexs = setOf(
Regex(
"^\\\\i\\b|AS \\$\\$|LANGUAGE '?plpgsql'?",
setOf(RegexOption.MULTILINE, RegexOption.IGNORE_CASE)
),
Regex(
"SECURITY (DEFINER|INVOKER)",
RegexOption.IGNORE_CASE
),
Regex(
"BEGIN( WORK| TRANSACTION)?;",
RegexOption.IGNORE_CASE
)
)
val PlsqlRegexs = setOf(
Regex(
"\\\$\\\$PLSQL_|XMLTYPE|sysdate|systimestamp|\\.nextval|connect by|AUTHID (DEFINER|CURRENT_USER)",
RegexOption.IGNORE_CASE
),
Regex(
"constructor\\W+function",
RegexOption.IGNORE_CASE
)
)
val PovRaySdlRegex = Regex(
"^\\s*#(declare|local|macro|while)\\s", RegexOption.MULTILINE
)
val PrologRegex = Regex(
"^[^#]*:-",
RegexOption.MULTILINE
)
val PythonRegex = Regex(
"(^(import|from|class|def)\\s)",
RegexOption.MULTILINE
)
val RRegex = Regex(
"<-|^\\s*#"
)
val RebolRegex = Regex(
"\\bRebol\\b",
RegexOption.IGNORE_CASE
)
val RoffRegex = Regex(
"^\\.[a-z][a-z](\\s|$)",
setOf(RegexOption.MULTILINE, RegexOption.IGNORE_CASE)
)
val RustRegex = Regex(
"^(use |fn |mod |pub |macro_rules|impl|#!?\\[)",
RegexOption.MULTILINE
)
val RenderscriptRegex = Regex(
"#include|#pragma\\s+(rs|version)|__attribute__"
)
val ScalaRegex = Regex(
"^\\s*import (scala|java)\\./.match(data) || /^\\s*val\\s+\\w+\\s*=/.match(data) || /^\\s*class\\b",
RegexOption.MULTILINE
)
val SmalltalkRegex = Regex(
"![\\w\\s]+methodsFor: "
)
val SqlplRegexs = setOf(
Regex(
"(alter module)|(language sql)|(begin( NOT)+ atomic)",
RegexOption.IGNORE_CASE
),
Regex(
"signal SQLSTATE '[0-9]+'",
RegexOption.IGNORE_CASE
)
)
val StandardMlRegex = Regex(
"=> |case\\s+(\\S+\\s)+of"
)
val SupercolliderRegexs = setOf(
Regex("\\^(this|super)\\."),
Regex("^\\s*(\\+|\\*)\\s*\\w+\\s*\\{", RegexOption.MULTILINE),
Regex("^\\s*~\\w+\\s*=\\.", RegexOption.MULTILINE)
)
val TexRegex = Regex(
"\\\\\\w+\\{"
)
val TypescriptRegex = Regex(
"^\\s*(import.+(from\\s+|require\\()['\"]react|\\/\\/\\/\\s*
// Required fields in k8s config: apiVersion, kind, metadata.
buf.contains("apiVersion") && buf.contains("kind")
&& buf.contains("metadata")
}
val bootstrapWebpackExp = { buf: String ->
// Handling php projects with bootstrap Webpack.
buf.contains("/******/ (function(modules) { // webpackBootstrap")
}
/**
* Heuristics to detect a programming language by file extension and content.
* Inspired by GitHub Liguist heuristics (https://github.com/github/linguist).
*/
object Heuristics
{
/**
* Returns a list of language commit stats extracted from the given file.
*/
fun analyze(file: DiffFile) : List? {
val buf = toBuf(file.new.content)
var extractor: ExtractorInterface? = null
// Look for an extractor by a file extension. If failed, then fallback
// to generic content analysis.
val extractorFactory = HeuristicsMap[file.extension]
if (extractorFactory != null) {
extractor = extractorFactory(buf, file.path.toLowerCase())
} else {
if (XmlRegex.containsMatchIn(buf)) {
extractor = CommonExtractor(Lang.XML)
}
}
return extractor?.extract(listOf(file))
}
/**
* Returns a portion of the file content not exceeding the limit.
*/
private const val HEURISTICS_CONSIDER_BYTES = 50 * 1024
private fun toBuf(lines: List) : String {
var buf = ""
for (line in lines) {
buf += "$line\n"
if (buf.length > HEURISTICS_CONSIDER_BYTES) {
break
}
}
return buf
}
}
/**
* A map of file extensions to language extracters.
*/
val HeuristicsMap = mapOf ExtractorInterface?>(
"4" to { _, _ ->
CommonExtractor(Lang.ROFF)
},
"4th" to { _, _ ->
CommonExtractor(Lang.FORTH)
},
"a51" to { _, _ ->
CommonExtractor(Lang.ASSEMBLY)
},
"al" to { _, _ ->
PerlExtractor(Lang.PERL)
},
"as" to { buf, _ ->
if (ActionscriptRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.ACTIONSCRIPT)
} else CommonExtractor(Lang.ANGELSCRIPT)
},
"asm" to { _, _ ->
CommonExtractor(Lang.ASSEMBLY)
},
"b" to { _, _ ->
CommonExtractor(Lang.LIMBO)
},
"bas" to { _, _ ->
CommonExtractor(Lang.VISUALBASIC)
},
"bat" to { _, _ ->
CommonExtractor(Lang.DOSBATCH)
},
"bbx" to { _, _ ->
CommonExtractor(Lang.TEX)
},
"bdy" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"boot" to { _, _ ->
CommonExtractor(Lang.CLOJURE)
},
"c" to { _, _ ->
CExtractor()
},
"cake" to { _, _ ->
CSharpExtractor()
},
"cbl" to { _, _ ->
CommonExtractor(Lang.COBOL)
},
"cbx" to { _, _ ->
CommonExtractor(Lang.TEX)
},
"cc" to { _, _ ->
CppExtractor()
},
"cgi" to { buf, _ ->
if (Perl5Regex.containsMatchIn(buf)) {
PerlExtractor(Lang.PERL)
} else null
},
"cl" to { _, _ ->
CommonExtractor(Lang.COMMONLISP)
},
"cl2" to { _, _ ->
CommonExtractor(Lang.CLOJURE)
},
"clj" to { _, _ ->
CommonExtractor(Lang.CLOJURE)
},
"cljc" to { _, _ ->
CommonExtractor(Lang.CLOJURE)
},
"cljscm" to { _, _ ->
CommonExtractor(Lang.CLOJURE)
},
"cljs" to { _, _ ->
CommonExtractor(Lang.CLOJURE)
},
"cljx" to { _, _ ->
CommonExtractor(Lang.CLOJURE)
},
"cls" to { buf, _ ->
if (TexRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.TEX)
} else {
CommonExtractor(Lang.VISUALBASIC)
}
},
"cob" to { _, _ ->
CommonExtractor(Lang.COBOL)
},
"coffee" to { _, _ ->
CommonExtractor(Lang.COFFEESCRIPT)
},
"cp" to { _, _ ->
CppExtractor()
},
"cpp" to { _, _ ->
CppExtractor()
},
"cr" to { _, _ ->
CrystalExtractor()
},
"cpy" to { _, _ ->
CommonExtractor(Lang.COBOL)
},
"cql" to { _, _ ->
CommonExtractor(Lang.SQL)
},
"cs" to { buf, _ ->
if (SmalltalkRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.SMALLTALK)
} else {
CSharpExtractor()
}
},
"cshtml" to { _, _ ->
CSharpExtractor()
},
"css" to { _, _ ->
CssExtractor()
},
"csx" to { _, _ ->
CSharpExtractor()
},
"cu" to { _, _ ->
CommonExtractor(Lang.CUDA)
},
"cuh" to { _, _ ->
CommonExtractor(Lang.CUDA)
},
"cxx" to { _, _ ->
CppExtractor()
},
"c++" to { _, _ ->
CppExtractor()
},
"d" to { buf, _ ->
if (DRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.D)
} else if (DtraceRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.DTRACE)
} else if (MakefileRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.MAKEFILE)
} else null
},
"dart" to { _, _ ->
DartExtractor
},
"db2" to { _, _ ->
CommonExtractor(Lang.SQLPL)
},
"ddl" to { buf, _ ->
if (PlsqlRegexs.any { re -> re.containsMatchIn(buf)}) {
CommonExtractor(Lang.PLSQL) // Oracle
} else if (!NotSqlRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.SQL) // Generic SQL
} else null
},
"dlm" to { _, _ ->
CommonExtractor(Lang.IDL)
},
"dm" to { _, _ ->
DMExtractor()
},
"dmm" to { _, _ ->
DMExtractor()
},
"dmi" to { _, _ ->
DMExtractor()
},
"dpr" to { _, _ ->
CommonExtractor(Lang.PASCAL)
},
"edn" to { _, _ ->
CommonExtractor(Lang.CLOJURE)
},
"el" to { _, _ ->
CommonExtractor(Lang.EMACSLISP)
},
"elc" to { _, _ ->
CommonExtractor(Lang.EMACSLISP)
},
"eliom" to { _, _ ->
CommonExtractor(Lang.OCAML)
},
"elm" to { _, _ ->
CommonExtractor(Lang.ELM)
},
"erl" to { _, _ ->
CommonExtractor(Lang.ERLANG)
},
"ex" to { _, _ ->
ElixirExtractor
},
"exs" to { _, _ ->
ElixirExtractor
},
"f" to { buf, _ ->
if (ForthRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.FORTH)
} else if (buf.contains("flowop")) {
CommonExtractor(Lang.FILEBENCHWML)
} else if (FortranRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.FORTRAN)
} else null
},
"f03" to { _, _ ->
CommonExtractor(Lang.FORTRAN)
},
"f08" to { _, _ ->
CommonExtractor(Lang.FORTRAN)
},
"f15" to { _, _ ->
CommonExtractor(Lang.FORTRAN)
},
"f90" to { _, _ ->
CommonExtractor(Lang.FORTRAN)
},
"f95" to { _, _ ->
CommonExtractor(Lang.FORTRAN)
},
"factor" to { _, _ ->
CommonExtractor(Lang.FACTOR)
},
"fcgi" to { buf, _ ->
if (Perl5Regex.containsMatchIn(buf)) {
PerlExtractor(Lang.PERL)
} else {
CommonExtractor(Lang.LUA)
}
},
"fnc" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"for" to { buf, _ ->
if (ForthRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.FORTH)
} else if (FortranRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.FORTRAN)
} else null
},
"forth" to { _, _ ->
CommonExtractor(Lang.FORTH)
},
"fp" to { _, _ ->
CommonExtractor(Lang.GLSL)
},
"fr" to { _, _ ->
CommonExtractor(Lang.FORTH)
},
"frag" to { _, _ ->
CommonExtractor(Lang.GLSL)
},
"frg" to { _, _ ->
CommonExtractor(Lang.GLSL)
},
"frt" to { _, _ ->
CommonExtractor(Lang.FORTH)
},
"fs" to { buf, _ ->
if (ForthFsRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.FORTH)
} else if (FSharpRegex.containsMatchIn(buf)) {
FSharpExtractor()
} else if (GlslRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.GLSL)
} else if (FilterscriptRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.FILTERSCRIPT)
} else null
},
"fsh" to { _, _ ->
CommonExtractor(Lang.GLSL)
},
"fsx" to { _, _ ->
FSharpExtractor()
},
"fth" to { _, _ ->
CommonExtractor(Lang.FORTH)
},
"fxml" to { _, _ ->
CommonExtractor(Lang.XML)
},
"glsl" to { _, _ ->
CommonExtractor(Lang.GLSL)
},
"go" to { _, _ ->
GoExtractor()
},
"gradle" to { _, _ ->
CommonExtractor(Lang.GRADLE)
},
"groovy" to { _, _ ->
CommonExtractor(Lang.GROOVY)
},
"h" to { buf, _ ->
if (ObjectiveCRegex.containsMatchIn(buf)) {
ObjectiveCExtractor()
} else if (CppRegex.containsMatchIn(buf)) {
CppExtractor()
} else CExtractor()
},
"h++" to { _, _ ->
CppExtractor()
},
"hh" to { _, _ ->
CppExtractor()
},
"hic" to { _, _ ->
CommonExtractor(Lang.CLOJURE)
},
"hl" to { _, _ ->
CommonExtractor(Lang.CLOJURE)
},
"hpp" to { _, _ ->
CppExtractor()
},
"htm" to { _, _ ->
CommonExtractor(Lang.HTML)
},
"html" to { _, _ ->
CommonExtractor(Lang.HTML)
},
"hs" to { _, _ ->
CommonExtractor(Lang.HASKELL)
},
"hrl" to { _, _ ->
CommonExtractor(Lang.ERLANG)
},
"hx" to { _, _ ->
CommonExtractor(Lang.HAXE)
},
"hxx" to { _, _ ->
CppExtractor()
},
"hy" to { _, _ ->
CommonExtractor(Lang.HY)
},
"ijs" to { _, _ ->
CommonExtractor(Lang.J)
},
"inc" to { buf, _ ->
if (PhpRegex.containsMatchIn(buf)) {
PhpExtractor()
} else if (PovRaySdlRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.POVRAYSDL)
} else if (PascalRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.PASCAL)
} else {
CommonExtractor(Lang.ASSEMBLY)
}
},
"inl" to { _, _ ->
CppExtractor()
},
"ino" to { _, _ ->
CommonExtractor(Lang.ARDUINO)
},
"ipynb" to { _, _ ->
IPythonExtractor()
},
"java" to { _, _ ->
JavaExtractor()
},
"jl" to { _, _ ->
CommonExtractor(Lang.JULIA)
},
"js" to { buf, _ ->
if (bootstrapWebpackExp(buf)) {
null
}
else JavascriptExtractor()
},
"jsx" to { _, _ ->
JavascriptExtractor()
},
"kt" to { _, _ ->
KotlinExtractor()
},
"kojo" to { _, _ ->
ScalaExtractor
},
"l" to { buf, _ ->
if (CommonLispRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.COMMONLISP)
} else if (LexRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.LEX)
} else if (RoffRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.ROFF)
} else if (PicoLispRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.PICOLISP)
} else null
},
"lbx" to { _, _ ->
CommonExtractor(Lang.TEX)
},
"less" to { _, _ ->
CssExtractor()
},
"lhs" to { _, _ ->
CommonExtractor(Lang.HASKELL)
},
"lisp" to { buf, _ ->
if (CommonLispRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.COMMONLISP)
} else if (NewLispRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.NEWLISP)
} else null
},
"litcoffee" to { _, _ ->
CommonExtractor(Lang.COFFEESCRIPT)
},
"lsp" to { buf, _ ->
if (CommonLispRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.COMMONLISP)
} else if (NewLispRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.NEWLISP)
} else null
},
"lua" to { _, _ ->
CommonExtractor(Lang.LUA)
},
"m" to { buf, _ ->
if (ObjectiveCRegex.containsMatchIn(buf)) {
ObjectiveCExtractor()
} else if (buf.contains(":- module")) {
CommonExtractor(Lang.MERCURY)
} else if (MufRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.MUF)
} else if (MRegexs.any { re -> re.containsMatchIn(buf)}) {
CommonExtractor(Lang.M)
} else if (MathematicaRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.MATHEMATICA)
} else if (MatlabRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.MATLAB)
} else if (LimboRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.LIMBO)
} else {
CommonExtractor(Lang.WOLFRAM)
}
},
"make" to { _, _ ->
CommonExtractor(Lang.MAKEFILE)
},
"makefile" to { _, _ ->
CommonExtractor(Lang.MAKEFILE)
},
"mat" to { _, _ ->
CommonExtractor(Lang.MATLAB)
},
"mjml" to { _, _ ->
CommonExtractor(Lang.XML)
},
"ml" to { buf, _ ->
if (OcamlRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.OCAML)
} else if (StandardMlRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.STANDARDML)
} else null
},
"mli" to { _, _ ->
CommonExtractor(Lang.OCAML)
},
"mlx" to { _, _ ->
CommonExtractor(Lang.MATLAB)
},
"mm" to { _, _ ->
ObjectiveCExtractor()
},
"ms" to { _, _ ->
CommonExtractor(Lang.ROFF)
},
"mt" to { _, _ ->
CommonExtractor(Lang.MATHEMATICA)
},
"muf" to { _, _ ->
CommonExtractor(Lang.MUF)
},
"mysql" to { _, _ ->
CommonExtractor(Lang.SQL)
},
"n" to { _, _ ->
CommonExtractor(Lang.ROFF)
},
"nasm" to { _, _ ->
CommonExtractor(Lang.ASSEMBLY)
},
"nb" to { buf, _ ->
if (MathematicaRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.MATHEMATICA)
} else CommonExtractor(Lang.WOLFRAM)
},
"nl" to { _, _ ->
CommonExtractor(Lang.NEWLISP)
},
"nr" to { _, _ ->
CommonExtractor(Lang.ROFF)
},
"oxygene" to { _, _ ->
CommonExtractor(Lang.OXYGENE)
},
"P" to { _, _ ->
CommonExtractor(Lang.PROLOG)
},
"p6" to { _, _ ->
PerlExtractor(Lang.PERL6)
},
"p8" to { _, _ ->
CommonExtractor(Lang.LUA)
},
"pas" to { _, _ ->
CommonExtractor(Lang.PASCAL)
},
"pascal" to { _, _ ->
CommonExtractor(Lang.PASCAL)
},
"pck" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"pd_lua" to { _, _ ->
CommonExtractor(Lang.LUA)
},
"pde" to { _, _ ->
CommonExtractor(Lang.PROCESSING)
},
"php" to { buf, path ->
if (buf.contains("
PhpExtractor()
},
"php3" to { _, _ ->
PhpExtractor()
},
"php4" to { _, _ ->
PhpExtractor()
},
"php5" to { _, _ ->
PhpExtractor()
},
"phps" to { _, _ ->
PhpExtractor()
},
"pkb" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"pks" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"pl" to { buf, _ ->
if (PrologRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.PROLOG)
} else if (Perl6Regex.containsMatchIn(buf)) {
PerlExtractor(Lang.PERL6)
} else {
PerlExtractor(Lang.PERL)
}
},
"plb" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"pls" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"plsql" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"pm" to { buf, _ ->
if (Perl5Regex.containsMatchIn(buf)) {
PerlExtractor(Lang.PERL)
} else if (Perl6Regex.containsMatchIn(buf)) {
PerlExtractor(Lang.PERL6)
} else if (XpmRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.XPM)
} else null
},
"pm6" to { _, _ ->
PerlExtractor(Lang.PERL6)
},
"pom" to { _, _ ->
CommonExtractor(Lang.MAVENPOM)
},
"pov" to { _, _ ->
CommonExtractor(Lang.POVRAYSDL)
},
"pp" to { buf, _ ->
if (PascalRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.PASCAL)
} else {
CommonExtractor(Lang.PUPPET)
}
},
"prc" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"pro" to { buf, _ ->
if (PrologRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.PROLOG)
} else if (buf.contains("last_client=")) {
CommonExtractor(Lang.INI)
} else if (buf.contains("HEADERS") || buf.contains("SOURCES")) {
CommonExtractor(Lang.QMAKE)
} else if (IdlRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.IDL)
} else null
},
"prolog" to { _, _ ->
CommonExtractor(Lang.PROLOG)
},
"props" to { buf, _ ->
if (XmlPropsRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.XML)
} else if (IniPropsRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.INI)
} else null
},
"ps1" to { _, _ ->
CommonExtractor(Lang.POWERSHELL)
},
"psd1" to { _, _ ->
CommonExtractor(Lang.POWERSHELL)
},
"psm1" to { _, _ ->
CommonExtractor(Lang.POWERSHELL)
},
"py" to { _, _ ->
PythonExtractor()
},
"py3" to { _, _ ->
PythonExtractor()
},
"qml" to { _, _ ->
CommonExtractor(Lang.QML)
},
"r" to { buf, _ ->
if (RebolRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.REBOL)
} else if (RRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.R)
} else null
},
"r2" to { _, _ ->
CommonExtractor(Lang.REBOL)
},
"r3" to { _, _ ->
CommonExtractor(Lang.REBOL)
},
"rb" to { _, _ ->
RubyExtractor()
},
"rbw" to { _, _ ->
RubyExtractor()
},
"rd" to { _, _ ->
CommonExtractor(Lang.R)
},
"re" to { _, _ ->
CommonExtractor(Lang.OCAML)
},
"reb" to { _, _ ->
CommonExtractor(Lang.REBOL)
},
"rebol" to { _, _ ->
CommonExtractor(Lang.REBOL)
},
"rno" to { _, _ ->
CommonExtractor(Lang.ROFF)
},
"rpy" to { buf, _ ->
if (PythonRegex.containsMatchIn(buf)) {
PythonExtractor()
} else CommonExtractor(Lang.RENPY)
},
"rs" to { buf, _ ->
if (RustRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.RUST)
} else if (RenderscriptRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.RENDERSCRIPT)
} else null
},
"rsh" to { _, _ ->
CommonExtractor(Lang.RENDERSCRIPT)
},
"rsx" to { _, _ ->
CommonExtractor(Lang.R)
},
"s" to { _, _ ->
CommonExtractor(Lang.ASSEMBLY)
},
"sas" to { _, _ ->
CommonExtractor(Lang.SAS)
},
"sass" to { _, _ ->
CssExtractor()
},
"sbt" to { _, _ ->
ScalaExtractor
},
"sc" to { buf, _ ->
if (SupercolliderRegexs.any { re -> re.containsMatchIn(buf) }) {
CommonExtractor(Lang.SUPERCOLLIDER)
} else if (ScalaRegex.containsMatchIn(buf)) {
ScalaExtractor
} else null
},
"scala" to { _, _ ->
ScalaExtractor
},
"scd" to { _, _ ->
CommonExtractor(Lang.SUPERCOLLIDER)
},
"sch" to { _, _ ->
CommonExtractor(Lang.SCHEME)
},
"scm" to { _, _ ->
CommonExtractor(Lang.SCHEME)
},
"scss" to { _, _ ->
CssExtractor()
},
"sexp" to { _, _ ->
CommonExtractor(Lang.COMMONLISP)
},
"sh" to { _, _ ->
CommonExtractor(Lang.SHELL)
},
"shader" to { _, _ ->
CommonExtractor(Lang.GLSL)
},
"sld" to { _, _ ->
CommonExtractor(Lang.SCHEME)
},
"sls" to { _, _ ->
CommonExtractor(Lang.SCHEME)
},
"sol" to { _, _ ->
CommonExtractor(Lang.SOLIDITY)
},
"spc" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"sps" to { _, _ ->
CommonExtractor(Lang.SCHEME)
},
"sql" to { buf, _ ->
if (PlpgsqlRegexs.any { re -> re.containsMatchIn(buf)}) {
PlpgsqlExtractor // PostgreSQL.
} else if (SqlplRegexs.any { re -> re.containsMatchIn(buf)}) {
CommonExtractor(Lang.SQLPL) // IBM DB2.
} else if (PlsqlRegexs.any { re -> re.containsMatchIn(buf)}) {
CommonExtractor(Lang.PLSQL) // Oracle.
} else {
CommonExtractor(Lang.SQL) // Generic SQL.
}
},
"ss" to { _, _ ->
CommonExtractor(Lang.SCHEME)
},
"st" to { _, _ ->
CommonExtractor(Lang.SMALLTALK)
},
"swift" to { _, _ ->
SwiftExtractor
},
"t" to { buf, _ ->
if (Perl6Regex.containsMatchIn(buf)) {
PerlExtractor(Lang.PERL6)
} else {
PerlExtractor(Lang.PERL)
}
},
"tab" to { _, _ ->
CommonExtractor(Lang.SQL)
},
"tcl" to { _, _ ->
CommonExtractor(Lang.TCL)
},
"tesc" to { _, _ ->
CommonExtractor(Lang.GLSL)
},
"tese" to { _, _ ->
CommonExtractor(Lang.GLSL)
},
"tex" to { _, _ ->
CommonExtractor(Lang.TEX)
},
"tmac" to { _, _ ->
CommonExtractor(Lang.ROFF)
},
"toc" to { _, _ ->
CommonExtractor(Lang.TEX)
},
"tpb" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"tps" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"trg" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"ts" to { buf, _ ->
if (XmltsRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.XML)
} else {
TypescriptExtractor()
}
},
"tsx" to { buf, _ ->
if (TypescriptRegex.containsMatchIn(buf)) {
TypescriptExtractor()
} else if (XmlRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.XML)
} else null
},
"udf" to { _, _ ->
CommonExtractor(Lang.SQL)
},
"ux" to { _, _ ->
CommonExtractor(Lang.XML)
},
"v" to { buf, _ ->
if (CoqRegex.containsMatchIn(buf)) {
CommonExtractor(Lang.COQ)
} else {
CommonExtractor(Lang.VERILOG)
}
},
"vb" to { _, _ ->
CommonExtractor(Lang.VISUALBASIC)
},
"vba" to { _, _ ->
CommonExtractor(Lang.VBA)
},
"vhdl" to { _, _ ->
CommonExtractor(Lang.VHDL)
},
"vbhtml" to { _, _ ->
CommonExtractor(Lang.VISUALBASIC)
},
"vim" to { _, _ ->
CommonExtractor(Lang.VIML)
},
"viw" to { _, _ ->
CommonExtractor(Lang.SQL)
},
"vrx" to { _, _ ->
CommonExtractor(Lang.GLSL)
},
"vsh" to { _, _ ->
CommonExtractor(Lang.GLSL)
},
"vue" to { _, _ ->
JavascriptExtractor()
},
"svelte" to { _, _ ->
JavascriptExtractor()
},
"vw" to { _, _ ->
CommonExtractor(Lang.PLSQL)
},
"wl" to { _, _ ->
CommonExtractor(Lang.MATHEMATICA)
},
"wlt" to { _, _ ->
CommonExtractor(Lang.MATHEMATICA)
},
"xml" to { _, _ ->
CommonExtractor(Lang.XML)
},
"xpm" to { _, _ ->
CommonExtractor(Lang.XPM)
},
"xtend" to { _, _ ->
CommonExtractor(Lang.XTEND)
},
"yap" to { _, _ ->
CommonExtractor(Lang.PROLOG)
},
// DevOps.
"yaml" to { buf, _ ->
when {
k8sExp(buf) -> {
DevopsExtractor(DevopsExtractor.K8S)
}
else -> null
}
},
"yml" to { buf, path ->
when {
path.endsWith("docker-compose.yml") -> {
DevopsExtractor(DevopsExtractor.DOCKER)
}
path.endsWith(".gitlab-ci.yml") -> {
DevopsExtractor(DevopsExtractor.GITLAB_CI)
}
path.endsWith(".travis.yml") -> {
DevopsExtractor(DevopsExtractor.TRAVIS)
}
path.endsWith(".circleci/config.yml") -> {
DevopsExtractor(DevopsExtractor.CIRCLECI)
}
path.endsWith(".drone.yml") -> {
DevopsExtractor(DevopsExtractor.DRONE)
}
path.contains(".github/workflows/") -> {
DevopsExtractor(DevopsExtractor.GITHUB_ACTIONS)
}
k8sExp(buf) -> {
DevopsExtractor(DevopsExtractor.K8S)
}
else -> null
}
},
"json" to { buf, _ ->
when {
k8sExp(buf) -> {
DevopsExtractor(DevopsExtractor.K8S)
}
else -> null
}
},
"dockerfile" to { _, _ ->
DevopsExtractor(DevopsExtractor.DOCKER)
},
"jenkinsfile" to { _, _ ->
DevopsExtractor(DevopsExtractor.JENKINS)
},
"" to { _, path ->
when {
path.endsWith("dockerfile") -> {
DevopsExtractor(DevopsExtractor.DOCKER)
}
path.endsWith("jenkinsfile") -> {
DevopsExtractor(DevopsExtractor.JENKINS)
}
else -> null
}
}
)
================================================
FILE: src/main/kotlin/app/extractors/IPythonExtractor.kt
================================================
// Copyright 2019 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
package app.extractors
import app.model.CommitStats
import app.model.DiffContent
import app.model.DiffFile
import app.model.DiffRange
import org.json.JSONObject
class IPythonExtractor : ExtractorInterface{
companion object {
const val LANGUAGE_NAME = Lang.PYTHON
private val pythonExtractor = PythonExtractor()
}
private fun getCodeContent(diffContent: DiffContent): DiffContent {
if (diffContent.content.isEmpty()) {
return diffContent
}
val code = JSONObject(
diffContent.content.joinToString("\n"))
.optJSONArray("cells")?.let {
0.until(it.length()).map { i -> it.optJSONObject(i) }
}?.filter {
it.optString("cell_type") == "code"
}?.filter {
it.optJSONArray("source") != null
}?.map { it.optJSONArray("source").map {line -> line.toString()}}
val content = code?.fold(mutableListOf()) {
acc: MutableList, x: List ->
acc.addAll(x)
acc
} ?: listOf()
return DiffContent(content = content,
ranges = listOf(DiffRange(0, content.size)),
imports = diffContent.imports)
}
override fun extract(files: List): List {
files.forEach { file ->
file.old = getCodeContent(file.old)
file.new = getCodeContent(file.new)
file.lang = LANGUAGE_NAME
}
return pythonExtractor.extract(files)
}
override fun extractImports(fileContent: List): List {
return pythonExtractor.extractImports(fileContent)
}
override fun tokenize(line: String): List {
return pythonExtractor.tokenize(line)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/JavaExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
package app.extractors
import app.model.CommitStats
import app.model.DiffFile
class JavaExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.JAVA
val KEYWORDS = listOf("abstract", "continue", "for", "new", "switch",
"assert", "default", "goto", "package", "synchronized", "boolean",
"do", "if", "private", "this", "break", "double", "implements",
"protected", "throw", "byte", "else", "import", "public", "throws",
"case", "enum", "instanceof", "return", "transient", "catch",
"extends", "int", "short", "try", "char", "final", "interface",
"static", "void", "class", "finally", "long", "strictfp",
"volatile", "const", "float", "native", "super", "while")
val importRegex = Regex("""^(.*import)\s[^\n]*""")
val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
val packageRegex = Regex("""^(.*package)\s[^\n]*""")
val extractImportRegex = Regex("""import\s+(\w+[.\w+]*)""")
}
override fun extract(files: List): List {
val stats = super.extract(files).toMutableList()
// Keywords stats.
val added = files.fold(mutableListOf(), { total, file ->
total.addAll(file.getAllAdded())
total
})
val deleted = files.fold(mutableListOf(), { total, file ->
total.addAll(file.getAllDeleted())
total
})
KEYWORDS.forEach { keyword ->
val totalAdded = added.count { line -> line.contains(keyword)}
val totalDeleted = deleted.count { line -> line.contains(keyword)}
if (totalAdded > 0 || totalDeleted > 0) {
stats.add(CommitStats(
numLinesAdded = totalAdded,
numLinesDeleted = totalDeleted,
type = ExtractorInterface.TYPE_KEYWORD,
tech = LANGUAGE_NAME + ExtractorInterface.SEPARATOR +
keyword
))
}
}
return stats
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = extractImportRegex.find(it)
if (res != null) {
val importedName = res.groupValues[1]
imports.add(importedName)
}
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
newLine = packageRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
return super.mapImportToIndex(import, lang, startsWith = true)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/JavascriptExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
import app.model.CommitStats
import app.model.DiffFile
class JavascriptExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.JAVASCRIPT
val splitRegex = Regex("""\s+|,|;|:|\*|\n|\(|\)|\[|]|\{|}|\+|=|\.|>|<|#|@|\$""")
val multilineCommentRegex = Regex("""/\*.+?\*/""")
val twoOrMoreWordsRegex = Regex("""(".+?\s.+?"|'.+?\s.+?')""")
val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
}
override fun extractImports(fileContent: List): List {
val line = fileContent.map { line -> commentRegex.replace(line, "") }
.joinToString(separator = " ").toLowerCase()
val fileTokens = multilineCommentRegex.replace(
twoOrMoreWordsRegex.replace(line, ""), "").split(splitRegex)
return fileTokens.distinct()
}
override fun extractLibStats(files: List): List {
val vueExtension = ".vue"
val svelteExtension = ".svelte"
val quasarConf = "quasar.conf.js"
val vueFiles = files.filter { it.path.endsWith(vueExtension) }
val svelteFiles = files.filter { it.path.endsWith(svelteExtension) }
val quasarFile = files.find { it.path.endsWith(quasarConf) }
val otherFiles = files.filter {
!it.path.endsWith(vueExtension) &&
!it.path.endsWith(svelteExtension) &&
!it.path.endsWith(quasarConf)
}
// Add stats from *.vue files.
val vueStats = listOf(CommitStats(
numLinesAdded = vueFiles.map { it.getAllAdded().size }.sum(),
numLinesDeleted = vueFiles.map { it.getAllDeleted().size }.sum(),
type = ExtractorInterface.TYPE_LIBRARY,
tech = "js.vue"
)).filter { it.numLinesAdded > 0 || it.numLinesDeleted > 0 }
// Add stats from *.svelte files.
val svelteStats = listOf(CommitStats(
numLinesAdded = svelteFiles.map { it.getAllAdded().size }.sum(),
numLinesDeleted = svelteFiles.map { it.getAllDeleted().size }.sum(),
type = ExtractorInterface.TYPE_LIBRARY,
tech = "js.svelte"
)).filter { it.numLinesAdded > 0 || it.numLinesDeleted > 0 }
var stats = vueStats + svelteStats + super.extractLibStats(otherFiles)
if (quasarFile == null) {
return stats;
}
val quasarStats = listOf(CommitStats(
numLinesAdded = quasarFile.getAllAdded().size,
numLinesDeleted = quasarFile.getAllDeleted().size,
type = ExtractorInterface.TYPE_LIBRARY,
tech = "js.quasar"
)).filter { it.numLinesAdded > 0 || it.numLinesDeleted > 0 }
return quasarStats + stats;
}
override fun tokenize(line: String): List {
return super.tokenize(commentRegex.replace(line, ""))
}
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
return super.mapImportToIndex(import, lang, startsWith = true)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/KotlinExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
class KotlinExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.KOTLIN
val importRegex = Regex("""^(.*import)\s[^\n]*""")
val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
val packageRegex = Regex("""^(.*package)\s[^\n]*""")
val extractImportRegex = Regex("""import\s+(\w+[.\w+]*)""")
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = extractImportRegex.find(it)
if (res != null) {
val importedName = res.groupValues[1]
imports.add(importedName)
}
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
newLine = packageRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
return super.mapImportToIndex(import, lang, startsWith = true)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/Languages.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Alexander Surkov (alex@sourcerer.io)
package app.extractors
object Lang {
const val ACTIONSCRIPT = "actionscript" // ActionScript
const val ANGELSCRIPT = "angelscript" // AngelScript
const val ASSEMBLY = "assembly" // Assembly
const val ARDUINO = "arduino" // Arduino
const val C = "c" // C
const val COQ = "coq" // Coq
const val CPP = "cpp" // C++
const val CRYSTAL = "crystal" // Crystal
const val CSHARP = "csharp" // C#
const val CSS = "css" // CSS
const val CLOJURE = "clojure" // Clojure
const val COBOL = "cobol" // COBOL
const val COFFEESCRIPT = "coffeescript" // CoffeeScript
const val COMMONLISP = "lisp" // Common Lisp
const val CUDA = "cuda" // Cuda
const val D = "d" // D
const val DART = "dart" // Dart
const val DM = "dm" // DM Language
const val DOSBATCH = "dosbatch" // DOS Batch
const val DTRACE = "dtrace" // DTrace
const val ELIXIR = "elixir" // Elixir
const val ELM = "elm" // Elm
const val EMACSLISP = "emacslisp" // Emacs Lisp
const val ERLANG = "erlang" // Erlang
const val FACTOR = "factor" // Factor
const val FILEBENCHWML = "filebench_wml" // Filebench WML
const val FILTERSCRIPT = "filterscript" // Filterscript
const val FORTH = "forth" // Forth
const val FORTRAN = "fortran" // Fortran
const val FSHARP = "fsharp" // F#
const val GLSL = "glsl" // GLSL
const val GO = "go" // Go
const val GRADLE = "gradle" // Gradle
const val GROOVY = "groovy" // Groovy
const val HACK = "hack" // Hack
const val HASKELL = "haskell" // Haskell
const val HAXE = "haxe" // Haxe
const val HTML = "html" // HTML
const val HY = "hy" // Hy
const val IDL = "idl" // IDL
const val INI = "ini" // INI
const val J = "j" // J
const val JAVA = "java" // Java
const val JAVASCRIPT = "javascript" // JavaScript
const val JULIA = "julia" // Julia
const val KOTLIN = "kotlin" // Kotlin
const val LIMBO = "limbo" // Limbo
const val LEX = "lex" // Lex
const val LUA = "lua" // Lua
const val M = "m" // M
const val MAKEFILE = "makefile" // Makefile
const val MATHEMATICA = "mathematica" // Mathematica
const val MATLAB = "matlab" // Matlab
const val MAVENPOM = "maven" // Maven POM
const val MUF = "muf" // MUF
const val MERCURY = "mercury" // Mercury
const val NEWLISP = "newlisp" // NewLisp
const val OBJECTIVEC = "objectivec" // Objective-C
const val OCAML = "ocaml" // OCaml
const val OXYGENE = "oxygene" // Oxygene
const val PASCAL = "pascal" // Pascal
const val PERL = "perl" // Perl
const val PERL6 = "perl6" // Perl 6
const val PHP = "php" // PHP
const val PICOLISP = "picolisp" // PicoLisp
const val PLPGSQL = "plpgsql" // PLpgSQL
const val PLSQL = "plsql" // PLSQL
const val POVRAYSDL = "pov-ray_sdl" // POV-Ray SDL
const val POWERSHELL = "powershell" // PowerShell
const val PUPPET = "puppet" // Puppet
const val PROCESSING = "processing" // Processing
const val PROLOG = "prolog" // Prolog
const val PYTHON = "python" // Python
const val QMAKE = "qmake" // QMake
const val QML = "qml" // QML
const val R = "r" // R
const val REBOL = "rebol" // Rebol
const val RENDERSCRIPT = "renderscript" // RenderScript
const val RENPY = "renpy" // Ren'Py
const val ROFF = "roff" // Roff
const val RUBY = "ruby" // Ruby
const val RUST = "rust" // Rust
const val SAS = "sas" // SAS
const val SCALA = "scala" // Scala
const val SCHEME = "scheme" // Scheme
const val SHELL = "shell" // Shell
const val SMALLTALK = "smalltalk" // Smalltalk
const val SOLIDITY = "solidity" // Solidity
const val SQL = "sql" // SQL
const val SQLPL = "sqlpl" // SQLPL
const val STANDARDML = "standard_ml" // Standard ML
const val SUPERCOLLIDER = "supercollider" // SuperCollider
const val SWIFT = "swift" // Swift
const val TCL = "tcl" // Tcl
const val TEX = "tex" // TeX
const val TYPESCRIPT = "typescript" // TypeScript
const val VERILOG = "verilog" // Verilog
const val VISUALBASIC = "visualbasic" // Visual Basic
const val VBA = "visualbasicforapps" // Visual Basic for Applications
const val VHDL = "vhdl" // VHDL
const val VIML = "viml" // Vim L
const val VUE = "vue" // Vue
const val SVELTE = "svelte" // Svelte
const val WOLFRAM = "wolframlanguage" // Wolfram Language
const val XML = "xml" // XML
const val XPM = "xpm" // XPM
const val XTEND = "xtend" // Xtend
}
================================================
FILE: src/main/kotlin/app/extractors/ObjectiveCExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
class ObjectiveCExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.OBJECTIVEC
val importRegex = Regex("""^([^\n]*[#@](import|include))\s[^\n]*""")
val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
val sharpImportIncludeRegex =
Regex("""[#@](import|include)\s+["<](\w+)[/\w+]*\.\w+[">]""")
val atImportRegex = Regex("""@import\s+(\w+)""")
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = sharpImportIncludeRegex.findAll(it) +
atImportRegex.findAll(it)
if (res.toList().isNotEmpty()) {
val lineLib = res.toList().map { it.groupValues }.last().last()
imports.add(lineLib)
}
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/PerlExtractor.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Alexander Surkov (alex@sourcerer.io)
package app.extractors
class PerlExtractor(private val language: String) : ExtractorBase(
language,
importRegex = Regex("""^use (.+);"""),
commentRegex = Regex("""([^\n]*#.*$)"""),
importStartsWith = true) {
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
// Perl and Perl6 share libraries
return super.mapImportToIndex(import, Lang.PERL, startsWith)
}
}
================================================
FILE: src/main/kotlin/app/extractors/PhpExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
class PhpExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.PHP
val importRegex = Regex("""^(.*require|require_once|include|""" +
"""include_once|use)\s[^\n]*""")
val commentRegex = Regex("""^([^\n]*//)[^\n]*""")
val useRegex = Regex("""use\s+(\w+)[\\\w+]*""")
val requireIncludeRegex = Regex("""(require|require_once|include|""" +
""""include_once)\s*[(]?'(\w+)[.\w+]*'[)]?""")
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = useRegex.findAll(it) + requireIncludeRegex.findAll(it)
if (res.toList().isNotEmpty()) {
val lineLib = res.toList().map { it.groupValues }.last().last()
imports.add(lineLib)
}
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/PlpgsqlExtractor.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Alexander Surkov (alex@sourcerer.io)
package app.extractors
object PlpgsqlExtractor : ExtractorBase(
language = Lang.PLPGSQL,
importRegex = Regex(""".+CREATE (?:EXTENSION|LANGUAGE) ([a-zA-Z_][a-zA-Z0-9_]*)"""),
commentRegex = Regex("""(--.*$)|(/[*].*?[*]/)"""))
================================================
FILE: src/main/kotlin/app/extractors/PythonExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
import app.model.CommitStats
import app.model.DiffFile
class PythonExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.PYTHON
const val COMPREHENSION_MAP = "map"
const val COMPREHENSION_LIST = "list"
val docImportRegex = Regex("""^([^\n]*#|\s*\"\"\"|\s*import|""" +
"""\s*from)[^\n]*""")
val commentRegex = Regex("""^(.*#).*""")
val extractImportRegex = Regex("""(from\s+(\w+)[.\w+]*\s+import|""" +
"""import\s+(\w+(,\s*\w+)*))(as\s+)*""")
val mapRegex = Regex("""(map\([^,]+?,)""")
val listRegex = Regex("""\[.+? for .+? in .+?]""")
val lineEndRegex = Regex(""",\s*""")
}
override fun extract(files: List): List {
val stats = super.extract(files).toMutableList()
// List comprehension fun fact.
val allAdded = files.map{ file -> file.getAllAdded() }.flatten()
val allDeleted = files.map{ file -> file.getAllDeleted() }.flatten()
val mapAllAdded = allAdded.fold(0) { total, line ->
total + mapRegex.findAll(line).toList().size }
val mapAllDeleted = allDeleted.fold(0) { total, line ->
total + mapRegex.findAll(line).toList().size }
val listAllAdded = allAdded.fold(0) { total, line ->
total + listRegex.findAll(line).toList().size }
val listAllDeleted = allDeleted.fold(0) { total, line ->
total + listRegex.findAll(line).toList().size }
if (mapAllAdded > 0 || mapAllDeleted > 0) {
stats.add(CommitStats(
mapAllAdded, mapAllDeleted, ExtractorInterface.TYPE_SYNTAX,
tech = LANGUAGE_NAME + ExtractorInterface.SEPARATOR +
COMPREHENSION_MAP
))
}
if (listAllAdded > 0 || listAllDeleted > 0) {
stats.add(CommitStats(
listAllAdded, listAllDeleted, ExtractorInterface.TYPE_SYNTAX,
tech = LANGUAGE_NAME + ExtractorInterface.SEPARATOR +
COMPREHENSION_LIST
))
}
return stats
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = extractImportRegex.find(it)
if (res != null) {
val lineLibs = res.groupValues.last {
it != "" && !it.startsWith(',')
}.split(lineEndRegex)
imports.addAll(lineLibs)
}
}
val filteredImports = imports.filter { import ->
!import.endsWith("_pb") && !import.endsWith("_pb2")
}.toMutableList()
if (filteredImports.size < imports.size) {
filteredImports.add("pb")
}
return filteredImports
}
override fun tokenize(line: String): List {
var newLine = docImportRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/RubyExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
class RubyExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.RUBY
val importRegex = Regex("""(require\s+'(\w+)'|load\s+'(\w+)\.\w+')""")
val commentRegex = Regex("""^([^\n]*#)[^\n]*""")
val extractImportRegex =
Regex("""(require\s+'(.+)'|load\s+'(\w+)\.\w+')""")
val includeRegex = Regex("""include\s+(\w+)::.+""")
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = extractImportRegex.find(it)
if (res != null) {
val lineLib = res.groupValues.last { it != "" }
imports.add(lineLib)
}
}
// Try to parse `include ` when imports are in external file.
if (imports.isEmpty()) {
fileContent.forEach {
val res = includeRegex.find(it)
if (res != null) {
imports.add(res.groupValues.last().toLowerCase())
}
}
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
return super.mapImportToIndex(import, lang, startsWith = true)
}
override fun determineLibs(line: String,
importedLibs: List): List {
// TODO(lyaronskaya): Case with no imports.
val libraries = importedLibs + "rb.rails"
return super.determineLibs(line, libraries)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/RustExtractor.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Alexander Surkov (alex@sourcerer.io)
package app.extractors
class RustExtractor : ExtractorInterface {
companion object {
const val LANGUAGE_NAME = Lang.RUST
val importRegex = Regex("""^extern crate \w+;$""")
val commentRegex = Regex("(//.+$)|(/[*].*?[*]/)")
val extractImportRegex = Regex("""^extern crate (\w+);$""")
}
override fun extractImports(fileContent: List): List {
val imports = mutableSetOf()
fileContent.forEach {
val res = extractImportRegex.find(it)
if (res != null) {
val lineLib = res.groupValues[1]
imports.add(lineLib)
}
}
return imports.toList()
}
override fun tokenize(line: String): List {
var newLine = importRegex.replace(line, "")
newLine = commentRegex.replace(newLine, "")
return super.tokenize(newLine)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/extractors/ScalaExtractor.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Alexander Surkov (alex@sourcerer.io)
package app.extractors
object ScalaExtractor : ExtractorBase(
language = Lang.SCALA,
importRegex = Regex("""^import (?:_root_\.)?((?:\.?[a-z]+)+\.)"""),
commentRegex = Regex("(//.+$)|(/[*].*?[*]/)"),
importStartsWith = true)
================================================
FILE: src/main/kotlin/app/extractors/SwiftExtractor.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.extractors
object SwiftExtractor : ExtractorBase(
language = Lang.SWIFT,
importRegex = Regex("""import\s+(\w+)"""),
commentRegex = Regex("""^([^\n]*//)[^\n]*"""))
================================================
FILE: src/main/kotlin/app/extractors/TypescriptExtractor.kt
================================================
// Copyright 2019 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
package app.extractors
import app.model.CommitStats
import app.model.DiffFile
class TypescriptExtractor : ExtractorInterface{
companion object {
const val LANGUAGE_NAME = Lang.TYPESCRIPT
private val javascriptExtractor = JavascriptExtractor()
}
override fun extract(files: List): List {
files.forEach { file ->
file.lang = Lang.JAVASCRIPT
}
val libStats = javascriptExtractor.extractLibStats(files)
files.forEach { file ->
file.lang = LANGUAGE_NAME
}
return super.extractLangStats(files) + libStats
}
override fun extractImports(fileContent: List): List {
return javascriptExtractor.extractImports(fileContent)
}
override fun tokenize(line: String): List {
return javascriptExtractor.tokenize(line)
}
override fun mapImportToIndex(import: String, lang: String,
startsWith: Boolean): String? {
return super.mapImportToIndex(import, Lang.JAVASCRIPT, startsWith = true)
}
override fun getLanguageName(): String? {
return LANGUAGE_NAME
}
}
================================================
FILE: src/main/kotlin/app/hashers/AuthorDistanceHasher.kt
================================================
// Copyright 2018 Sourcerer Inc. All Rights Reserved.
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
package app.hashers
import app.api.Api
import app.model.AuthorDistance
import app.model.Repo
import io.reactivex.Observable
import java.util.concurrent.TimeUnit
class AuthorDistanceHasher(
private val serverRepo: Repo,
private val api: Api,
private val emails: HashSet,
private val userEmails: HashSet) {
fun updateFromObservable(observable: Observable,
onError: (Throwable) -> Unit) {
val authorScores = hashMapOf()
emails.forEach { authorScores[it] = 0.0 }
// Store the time of the earliest commit for a path by user.
val authorPathLastContribution = hashMapOf()
observable.subscribe({
val email = it.email!!
val paths = it.paths!!
val time = it.date!!
if (email in userEmails) {
paths.forEach { path ->
authorPathLastContribution[path] = time
}
}
else {
val score = paths
.filter { path -> path in authorPathLastContribution }
.filter { path ->
val authorTime = authorPathLastContribution[path]!!
val timeDelta = TimeUnit.DAYS.convert(
authorTime - time, TimeUnit.SECONDS)
timeDelta < 365
}.size
authorScores[email] = authorScores[email]!! + score
}
}, onError, {
val stats = mutableListOf()
authorScores.forEach { email, value ->
if (email !in userEmails) {
stats.add(AuthorDistance(serverRepo, email, value))
}
}
postDistancesToServer(stats)
})
}
private fun postDistancesToServer(stats: List) {
if (stats.isNotEmpty()) {
api.postAuthorDistances(stats).onErrorThrow()
}
}
}
================================================
FILE: src/main/kotlin/app/hashers/CodeLongevity.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Alexander Surkov (alex@sourcerer.io)
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.hashers
import app.FactCodes
import app.Logger
import app.api.Api
import app.model.Author
import app.model.Repo
import app.model.Fact
import app.utils.FileHelper
import io.reactivex.Observable
import org.eclipse.jgit.diff.DiffEntry
import org.eclipse.jgit.diff.RawText
import org.eclipse.jgit.api.Git
import org.eclipse.jgit.lib.AnyObjectId
import org.eclipse.jgit.lib.Repository
import org.eclipse.jgit.revwalk.RevCommit
import org.eclipse.jgit.revwalk.RevWalk
import org.eclipse.jgit.treewalk.TreeWalk
import java.io.FileInputStream
import java.io.FileNotFoundException
import java.io.FileOutputStream
import java.io.ObjectOutputStream
import java.io.ObjectInputStream
import java.io.Serializable
import java.lang.Exception
import java.text.SimpleDateFormat
import java.util.Date
/**
* Represents a code line in a file revision.
*/
class RevCommitLine(val commit: RevCommit, val fileId: AnyObjectId,
val file: String, val line: Int,
val isDeleted: Boolean) {
val id : String
get() = "${fileId.name}:$line"
}
/**
* Represents a code line in repo's history.
*
* TODO(Alex): the text arg is solely for testing proposes (remove it)
*/
class CodeLine(val repo: Repository,
val from: RevCommitLine, val to: RevCommitLine) {
// TODO(alex): oldId and newId may be computed as a hash built from commit,
// file name and line number, if we are going to send the data outside a
// local machine.
/**
* Id of the code line in a revision when the line was added. Used to
* identify a line and update its lifetime computed at the previous
* iteration.
*/
val oldId : String
get() = from.id
/**
* Id of the code line in a revision, where the line was deleted, or a head
* revision, if the line is alive.
*/
val newId : String
get() = to.id
/**
* The code line's age in seconds.
*/
var age : Long = 0
get() {
if (field == 0L) {
field = (to.commit.commitTime - from.commit.commitTime).toLong()
}
return field
}
/**
* The code line text.
*/
val text : String
get() = RawText(repo.open(from.fileId).bytes).getString(from.line)
/**
* Email address of the line's author.
*/
val authorEmail : String
get() = from.commit.authorIdent.emailAddress
/**
* Email address of the line's changer.
*/
val editorEmail : String?
get() = if (isDeleted) to.commit.authorIdent.emailAddress else null
/**
* A date when the line was changed.
*/
val editDate : Date
get() = Date(to.commit.commitTime.toLong() * 1000)
/**
* True if the line is deleted.
*/
val isDeleted : Boolean
get() = to.isDeleted
/**
* A pretty print of a code line; debugging.
*/
override fun toString() : String {
val df = SimpleDateFormat("yyyy-MM-dd HH:mm z")
val fd = df.format(Date(from.commit.commitTime.toLong() * 1000))
val td = df.format(Date(to.commit.commitTime.toLong() * 1000))
val fc = "${from.commit.name} '${from.commit.shortMessage}'"
val tc = "${to.commit.name} '${to.commit.shortMessage}'"
val revState = if (isDeleted) "deleted in" else "last known as"
val state = if (isDeleted) "deleted" else "alive"
return "Line '$text' - '${from.file}:${from.line}' added in $fc $fd\n" +
" $revState '${to.file}:${to.line}' in $tc $td,\n" +
" age: $age s - $state"
}
}
/**
* Detects colleagues and their 'work vicinity' from commits.
*/
class Colleagues(private val serverRepo: Repo) {
// A map of pairs to pairs of
// , which indicates to a minimum time in ms between all line
// changes for these two colleagues in a given month (yyyy-mm).
private val map: HashMap,
HashMap> = hashMapOf()
fun collect(line: CodeLine) {
// TODO(alex): ignore same user emails
val authorEmail = line.authorEmail
val editorEmail = line.editorEmail
if (editorEmail == null || authorEmail == editorEmail) {
return
}
val emails = Pair(authorEmail, editorEmail)
val dates = map.getOrPut(emails, { hashMapOf() })
val month = SimpleDateFormat("yyyy-MM").format(line.editDate)
Logger.trace { "collected colleague, age: ${line.age}" }
val vicinity = dates.getOrPut(month, { line.age })
if (vicinity > line.age) {
dates[month] = line.age
}
}
fun calculateAndSendFacts(api: Api) {
// Expose colleagues iff colleague1 edited colleague2 code and
// colleague2 edited colleauge1 code.
val auxHash = hashSetOf>()
for ((pair, dates) in map) {
val email1 = pair.first
val email2 = pair.second
if (auxHash.contains(Pair(email2, email1))) {
continue
}
val min1 = dates.minBy { (_, vicinity) -> vicinity }!!
val dates2 = map[Pair(email2, email1)]
if (dates2 != null) {
auxHash.add(Pair(email1, email2))
val min2 = dates2.minBy { (_, vicinity) -> vicinity }!!
val min: Long =
if (min1.value < min2.value) { min1.value }
else { min2.value }
val stats = mutableListOf()
stats.add(Fact(serverRepo,
FactCodes.COLLEAGUES,
value = email1,
value2 = email2,
value3 = min.toString()))
api.postFacts(stats).onErrorThrow()
}
}
}
/**
* Return colleagues in a form of for the given
* email, where time indicates a minimal time in ms between all line edits
* by these colleagues in a given month (yyyy-mm).
*/
fun get(email: String) : List> {
return map
.filter { (pair, _) -> pair.first == email || pair.second == email }
.flatMap { (pair, dates) ->
val colleagueEmail =
if (email == pair.first) pair.second else pair.first
val list = mutableListOf>()
dates.forEach { month, vicinity ->
list.add(Triple(colleagueEmail, month, vicinity))
}
return list
}
}
}
/**
* A data class used to store line age information.
*/
class CodeLineAges : Serializable, Cloneable {
/**
* A pair of (line age sum, line count) representing an aggregated line
* ages.
*/
data class AggrAge(var sum: Long = 0L, var count: Int = 0) : Serializable
/**
* A code line info: an (age, email) pair.
*/
data class LineInfo(var age: Long, var email: String) : Serializable
/**
* Aggregated code line ages for user emails, collected from all deleted
* lines.
*/
var aggrAges: HashMap = hashMapOf()
/**
* A map of existing code lines ids to their ages at the revision.
*/
var lastingLines: HashMap = hashMapOf()
public override fun clone(): CodeLineAges {
val clone = CodeLineAges()
aggrAges.forEach { (email, age) ->
clone.aggrAges[email] = age.copy() }
lastingLines.forEach { (email, line) ->
clone.lastingLines[email] = line.copy() }
return clone
}
}
/**
* Used to compute age of code lines in the repo.
*/
class CodeLongevity(
private val serverRepo: Repo,
private val emails: HashSet,
private val git: Git) {
val repo: Repository = git.repository
val revWalk = RevWalk(repo)
val head: RevCommit =
try { revWalk.parseCommit(CommitCrawler.getDefaultBranchHead(git)) }
catch(e: Exception) { throw Exception("No branch") }
val dataPath = FileHelper.getPath(serverRepo.rehash, "longevity")
val colleagues = Colleagues(serverRepo)
/**
* Updates code line age statistics on the server.
*/
private fun calculateAndSendFacts(ages: CodeLineAges, api: Api) {
var repoTotal = 0
var repoSum: Long = 0
val aggrAges : HashMap = hashMapOf()
ages.aggrAges.forEach { (email, aggrAge) ->
repoSum += aggrAge.sum
repoTotal += aggrAge.count
if (emails.contains(email)) {
aggrAges[email] = aggrAge
}
}
ages.lastingLines.forEach { (_, info) ->
val aggrAge =
aggrAges.getOrPut(info.email, { CodeLineAges.AggrAge() })
aggrAge.sum += info.age
aggrAge.count += 1
repoSum += info.age
repoTotal += 1
}
val secondsInDay = 86400
val repoAvg = if (repoTotal > 0) { repoSum / repoTotal } else 0
val stats = mutableListOf()
stats.add(Fact(repo = serverRepo,
code = FactCodes.LINE_LONGEVITY_REPO,
value = repoAvg.toString()))
val repoAvgDays = repoAvg / secondsInDay
Logger.info { "Repo average code line age is $repoAvgDays days, " +
"lines total: $repoTotal" }
for (email in emails) {
val aggrAge = aggrAges[email] ?: CodeLineAges.AggrAge()
val avg = if (aggrAge.count > 0) { aggrAge.sum / aggrAge.count }
else 0
stats.add(Fact(repo = serverRepo,
code = FactCodes.LINE_LONGEVITY,
value = avg.toString(),
author = Author(email = email)))
}
if (stats.size > 0) {
api.postFacts(stats).onErrorThrow()
Logger.info { "Sent ${stats.size} facts to server" }
}
colleagues.calculateAndSendFacts(api)
}
/**
* Scans the repo to extract code line ages.
*/
fun updateFromObservable(diffObservable: Observable =
CommitCrawler.getJGitObservable(git),
onError: (Throwable) -> Unit = {},
api: Api,
onDataComplete: (CodeLineAges) -> Unit = {}) {
var storedHead: RevCommit? = null
var ageData = CodeLineAges()
// Load existing age data if any. Expected format: commit id and
// CodeLineAges structure following it.
try {
val file = dataPath.toFile()
val iStream = ObjectInputStream(FileInputStream(file))
val storedHeadId = iStream.readUTF()
Logger.debug { "Stored repo head: $storedHeadId" }
storedHead = revWalk.parseCommit(repo.resolve(storedHeadId))
if (storedHead == head) {
return // TODO(anatoly): Send saved stats in such case.
}
ageData = (iStream.readObject() ?: CodeLineAges()) as CodeLineAges
}
catch(e: FileNotFoundException) { }
catch(e: Exception) { Logger.error(e, "Failed to read longevity " +
"data. CAUTION: data will be recomputed.") }
// Update ages.
getLinesObservable(storedHead, diffObservable, onError).subscribe({
line ->
Logger.trace { "Scanning: $line" }
if (line.isDeleted) {
if (ageData.lastingLines.contains(line.oldId)) {
line.age += ageData.lastingLines.remove(line.oldId)!!.age
}
val aggrAge = ageData.aggrAges.getOrPut(line.authorEmail,
{ CodeLineAges.AggrAge() } )
aggrAge.sum += line.age
aggrAge.count += 1
colleagues.collect(line)
} else {
var age = line.age
if (ageData.lastingLines.contains(line.oldId)) {
age += ageData.lastingLines.remove(line.oldId)!!.age
}
ageData.lastingLines[line.newId] = CodeLineAges.LineInfo(age,
line.authorEmail)
}
}, onError, {
// Store ages for subsequent runs.
try {
val file = dataPath.toFile()
val oStream = ObjectOutputStream(FileOutputStream(file))
oStream.writeUTF(head.name)
oStream.writeObject(ageData)
}
catch(e: Exception) {
Logger.error(e, "Failed to save longevity data. " +
"CAUTION: data will be recomputed on a next run.")
}
onDataComplete(ageData)
calculateAndSendFacts(ageData, api)
})
}
/**
* Clears the stored age data if any.
*/
fun dropSavedData() {
dataPath.toFile().delete()
}
/**
* Returns a list of code lines, both alive and deleted, between
* the revisions of the repo.
*/
fun getLinesList(tail : RevCommit? = null,
diffObservable: Observable =
CommitCrawler.getJGitObservable(git),
onError: (Throwable) -> Unit = {}) : List {
val codeLines: MutableList = mutableListOf()
getLinesObservable(tail, diffObservable, onError).blockingSubscribe {
line -> codeLines.add(line)
}
return codeLines
}
/**
* Returns an observable for for code lines, both alive and deleted, between
* the revisions of the repo.
*/
fun getLinesObservable(tail : RevCommit? = null,
diffObservable: Observable,
onError: (Throwable) -> Unit)
: Observable =
Observable.create { subscriber ->
val headWalk = TreeWalk(repo)
headWalk.isRecursive = true
headWalk.addTree(head.tree)
val files: MutableMap> = mutableMapOf()
// Build a map of file names and their code lines.
while (headWalk.next()) {
try {
val path = headWalk.pathString
val fileId = headWalk.getObjectId(0)
val fileLoader = repo.open(fileId)
if (!RawText.isBinary(fileLoader.openStream())) {
val fileText = RawText(fileLoader.bytes)
val lines = ArrayList(fileText.size())
for (idx in 0 until fileText.size()) {
lines.add(RevCommitLine(head, fileId, path, idx, false))
}
files[path] = lines
}
} catch (e: Exception) {
// TODO(anatoly): better fix of exceptions.
}
}
diffObservable
.takeWhile { (commit, _) -> commit != tail }
.subscribe( { (commit, diffs) ->
// A step back in commits history. Update the files map according
// to the diff. Traverse the diffs backwards to handle double
// renames properly.
// TODO(alex): cover file renames by tests (see APP-132 issue).
for ((diff, editList) in diffs!!.asReversed()) {
val oldPath = diff.oldPath
val oldId = diff.oldId.toObjectId()
val newPath = diff.newPath
val newId = diff.newId.toObjectId()
Logger.trace { "old: '$oldPath', new: '$newPath'" }
// File was deleted, initialize the line array in the files map.
if (diff.changeType == DiffEntry.ChangeType.DELETE) {
val fileLoader = repo.open(oldId)
val fileText = RawText(fileLoader.bytes)
files[oldPath] = ArrayList(fileText.size())
}
// If a file was deleted, then the new path is /dev/null.
val path = if (newPath != DiffEntry.DEV_NULL) {
newPath
} else {
oldPath
}
val lines = files[path]!!
// Update the lines array according to diff insertions.
// Traverse the edit list backwards to keep indices of
// the edit list and the lines array in sync.
for (edit in editList.asReversed()) {
// Insertion case: track the lines.
val insCount = edit.lengthB
if (insCount > 0) {
val insStart = edit.beginB
val insEnd = edit.endB
Logger.trace { "ins ($insStart, $insEnd)" }
for (idx in insStart until insEnd) {
val from = RevCommitLine(commit!!, newId,
newPath, idx, false)
try {
val to = lines[idx]
val cl = CodeLine(repo, from, to)
Logger.trace { "Collected: $cl" }
subscriber.onNext(cl)
}
catch(e: IndexOutOfBoundsException) {
Logger.error(e, "No line at $idx; commit: " +
"${commit.name}; " +
"'${commit.shortMessage}'")
throw e
}
}
lines.subList(insStart, insEnd).clear()
}
}
// Update the lines array according to diff deletions.
for (edit in editList) {
// Deletion case. Chase down the deleted lines through the
// history.
val delCount = edit.lengthA
if (delCount > 0) {
val delStart = edit.beginA
val delEnd = edit.endA
Logger.trace { "del ($delStart, $delEnd)" }
val tmpLines = ArrayList(delCount)
for (idx in delStart until delEnd) {
tmpLines.add(RevCommitLine(commit!!, oldId,
oldPath, idx, true))
}
lines.addAll(delStart, tmpLines)
}
}
// File was renamed, tweak the files map.
if (diff.changeType == DiffEntry.ChangeType.RENAME) {
files[oldPath] = files.remove(newPath)!!
}
}
}, onError, {
// If a tail revision was given then the map has to contain
// unclaimed code lines, i.e. the lines added before the tail
// revision. Push them all into the result lines list, so the
// caller can update their ages properly.
if (tail != null) {
val tailWalk = TreeWalk(repo)
tailWalk.isRecursive = true
tailWalk.addTree(tail.tree)
while (tailWalk.next()) {
val filePath = tailWalk.pathString
val lines = files[filePath]
if (lines != null) {
val fileId = tailWalk.getObjectId(0)
for (idx in 0 until lines.size) {
val from = RevCommitLine(tail, fileId,
filePath, idx, false)
val cl = CodeLine(repo, from, lines[idx])
Logger.trace { "Collected (tail): $cl" }
subscriber.onNext(cl)
}
}
}
}
subscriber.onComplete()
})
}
}
================================================
FILE: src/main/kotlin/app/hashers/CommitCrawler.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
// Author: Liubov Yaronskaya (lyaronskaya@sourcerer.io)
package app.hashers
import app.Logger
import app.model.Author
import app.model.Commit
import app.model.DiffContent
import app.model.DiffFile
import app.model.DiffRange
import app.model.Repo
import app.utils.EmptyRepoException
import io.reactivex.Observable
import java.io.BufferedReader
import java.io.InputStreamReader
import org.apache.commons.codec.digest.DigestUtils
import org.eclipse.jgit.api.Git
import org.eclipse.jgit.diff.DiffEntry
import org.eclipse.jgit.diff.DiffFormatter
import org.eclipse.jgit.diff.EditList
import org.eclipse.jgit.diff.RawText
import org.eclipse.jgit.lib.ObjectId
import org.eclipse.jgit.lib.Repository
import org.eclipse.jgit.revwalk.RevCommit
import org.eclipse.jgit.revwalk.RevWalk
import org.eclipse.jgit.treewalk.filter.PathFilter
import org.eclipse.jgit.treewalk.TreeWalk
import org.eclipse.jgit.util.io.DisabledOutputStream
import java.util.LinkedList
data class JgitData(var commit: RevCommit? = null,
var list: List? = null,
var paths: List? = null,
var date: Long? = null,
var email: String? = null,
var coauthors: List? = null)
data class JgitDiff(val diffEntry: DiffEntry, val editList: EditList)
/**
* Iterates over the diffs between commits in the repo's history.
*/
object CommitCrawler {
private const val REMOTE_HEAD = "refs/remotes/origin/HEAD"
private const val REMOTE_MASTER_BRANCH = "refs/remotes/origin/master"
private const val LOCAL_MASTER_BRANCH = "refs/heads/master"
private const val LOCAL_HEAD = "HEAD"
private val REFS = listOf(REMOTE_HEAD, REMOTE_MASTER_BRANCH,
LOCAL_MASTER_BRANCH, LOCAL_HEAD)
private val CONF_FILE_PATH = ".sourcerer-conf"
private val MAX_DIFF_SIZE = 600000
private val coauthoredRegex = Regex("""Co-authored-by: (.+) <(.+)>""")
fun getDefaultBranchHead(git: Git): ObjectId {
for (ref in REFS) {
val branch = git.repository.resolve(ref) ?: continue
Logger.debug { "Hashing from $ref" }
return branch
}
throw EmptyRepoException("No remote default, master or HEAD found")
}
fun fetchRehashesAndAuthors(git: Git):
Triple, HashSet, HashMap> {
val head: RevCommit = RevWalk(git.repository)
.parseCommit(getDefaultBranchHead(git))
val revWalk = RevWalk(git.repository)
revWalk.markStart(head)
val commitsRehashes = LinkedList()
val emails = hashSetOf()
val names = hashMapOf()
val commitsCount = hashMapOf()
val coauthorsList = mutableListOf()
var commit: RevCommit? = revWalk.next()
while (commit != null) {
commitsRehashes.add(DigestUtils.sha256Hex(commit.name))
val email = commit.authorIdent.emailAddress.toLowerCase()
val name = commit.authorIdent.name
if (!emails.contains(email)) {
emails.add(email)
names[email] = name
} else {
if (name.length > names[email]!!.length) {
names[email] = name
}
}
val coauthors = getCoauthors(commit.fullMessage)
coauthorsList.addAll(coauthors)
commitsCount[email] = commitsCount.getOrDefault(email, 0) + 1
commit.disposeBody()
commit = revWalk.next()
}
revWalk.dispose()
val authors = emails.map { email -> Author(names[email]!!, email) }
.toHashSet()
authors.addAll(coauthorsList)
return Triple(commitsRehashes, authors, commitsCount)
}
fun getJGitObservable(git: Git,
totalCommitCount: Int = 0,
extractCommit: Boolean = true,
extractDiffs: Boolean = true,
extractPaths: Boolean = false,
extractDate: Boolean = false,
extractEmail: Boolean = false,
extractCoauthors: Boolean = false,
filteredEmails: HashSet? = null,
tail : RevCommit? = null) :
Observable = Observable.create { subscriber ->
val repo: Repository = git.repository
val revWalk = RevWalk(repo)
val head: RevCommit =
try { revWalk.parseCommit(getDefaultBranchHead(git)) }
catch(e: Exception) { throw Exception("No head was found!") }
val df = DiffFormatter(DisabledOutputStream.INSTANCE)
df.setRepository(repo)
df.isDetectRenames = true
val confTreeWalk = TreeWalk(repo)
confTreeWalk.addTree(head.getTree())
confTreeWalk.setFilter(PathFilter.create(CONF_FILE_PATH))
var ignoredPaths =
if (confTreeWalk.next()) {
getIgnoredPaths(repo, confTreeWalk.getObjectId(0))
}
else {
listOf()
}
var commitCount = 0
revWalk.markStart(head)
var commit: RevCommit? = revWalk.next() // Move the walker to the head.
while (commit != null && commit != tail) {
commitCount++
val parentCommit: RevCommit? = revWalk.next()
// Smart casts are not yet supported for a mutable variable captured
// in an inline lambda, see
// https://youtrack.jetbrains.com/issue/KT-7186.
if (Logger.isTrace) {
val commitName = commit.name
val commitMsg = commit.shortMessage
Logger.trace { "commit: $commitName; '$commitMsg'" }
if (parentCommit != null) {
val parentCommitName = parentCommit.name
val parentCommitMsg = parentCommit.shortMessage
Logger.trace { "parent commit: $parentCommitName; " +
"'$parentCommitMsg'" }
}
else {
Logger.trace { "parent commit: null" }
}
}
val perc = if (totalCommitCount != 0) {
(commitCount.toDouble() / totalCommitCount) * 100
} else 0.0
Logger.printCommit(commit.shortMessage, commit.name, perc)
val email = commit.authorIdent.emailAddress.toLowerCase()
if (filteredEmails != null && !filteredEmails.contains(email)) {
commit = parentCommit
continue
}
val paths = mutableListOf()
val diffEntries = df.scan(parentCommit, commit)
.filter { diff ->
diff.changeType != DiffEntry.ChangeType.COPY
}
.filter { diff ->
val path = diff.newPath
for (cnv in VendorConventions) {
if (cnv.containsMatchIn(path) ||
cnv.containsMatchIn(diff.oldPath)) {
return@filter false
}
}
val fileId =
if (path != DiffEntry.DEV_NULL) {
diff.newId.toObjectId()
} else {
diff.oldId.toObjectId()
}
val stream = try {
repo.open(fileId).openStream()
} catch (e: Exception) {
null
}
stream != null && !RawText.isBinary(stream)
}
.filter { diff ->
val filePath =
if (diff.getNewPath() != DiffEntry.DEV_NULL) {
diff.getNewPath()
} else {
diff.getOldPath()
}
// Update ignored paths list. The config file has retroactive
// force, i.e. if it was added at this commit, then we presume
// it is applied to all commits, preceding this commit.
if (diff.getOldPath() == CONF_FILE_PATH) {
ignoredPaths =
getIgnoredPaths(repo, diff.getNewId().toObjectId())
}
if (!ignoredPaths.any { path ->
if (path.endsWith("/")) {
filePath.startsWith(path)
}
else {
path == filePath
}
}) {
paths.add(filePath)
true
} else false
}
val jgitData = JgitData()
if (extractCommit) {
jgitData.commit = commit
}
if (extractDiffs) {
val diffEdits = diffEntries
.map { diff ->
JgitDiff(diff, df.toFileHeader(diff).toEditList())
}
.filter { diff ->
diff.editList.fold(0) { acc, edit ->
acc + edit.lengthA + edit.lengthB
} < MAX_DIFF_SIZE
}
jgitData.list = diffEdits
}
if (extractPaths) {
jgitData.paths = paths
}
if (extractDate) {
jgitData.date = commit.authorIdent.getWhen().time / 1000
}
if (extractEmail) {
jgitData.email = email
}
if (extractCoauthors) {
jgitData.coauthors = getCoauthors(commit.fullMessage)
}
subscriber.onNext(jgitData)
commit = parentCommit
}
subscriber.onComplete()
}
fun getObservable(git: Git,
repo: Repo): Observable {
return getObservable(git, getJGitObservable(git), repo)
}
fun getObservable(git: Git,
jgitObservable: Observable,
repo: Repo): Observable {
return jgitObservable.map( { jgitData ->
// Mapping and stats extraction.
val commit = Commit(jgitData.commit!!, jgitData.coauthors)
commit.diffs = getDiffFiles(git.repository, jgitData.list!!)
// Count lines on all non-binary files. This is additional
// statistics to CommitStats because not all file extensions
// may be supported.
commit.numLinesAdded = commit.diffs.fold(0) { total, file ->
total + file.getAllAdded().size
}
commit.numLinesDeleted = commit.diffs.fold(0) { total, file ->
total + file.getAllDeleted().size
}
commit.repo = repo
commit
})
}
private fun getDiffFiles(jgitRepo: Repository,
jgitDiffs: List) : List {
return jgitDiffs
.map { (diff, edits) ->
// TODO(anatoly): Can produce exception for large object.
// Investigate for size.
val new = try {
getContentByObjectId(jgitRepo, diff.newId.toObjectId())
} catch (e: Exception) {
Logger.error(e)
null
}
val old = try {
getContentByObjectId(jgitRepo, diff.oldId.toObjectId())
} catch (e: Exception) {
Logger.error(e)
null
}
val diffFiles = mutableListOf()
if (new != null && old != null) {
val path = when (diff.changeType) {
DiffEntry.ChangeType.DELETE -> diff.oldPath
else -> diff.newPath
}
diffFiles.add(DiffFile(path = path,
changeType = diff.changeType,
old = DiffContent(old, edits.map { edit ->
DiffRange(edit.beginA, edit.endA)
}),
new = DiffContent(new, edits.map { edit ->
DiffRange(edit.beginB, edit.endB)
})
))
}
diffFiles
}
.flatten()
}
private fun getContentByObjectId(repo: Repository,
objectId: ObjectId): List {
return try {
val obj = repo.open(objectId)
val rawText = RawText(obj.bytes)
val content = ArrayList(rawText.size())
for (i in 0..(rawText.size() - 1)) {
content.add(rawText.getString(i))
}
return content
} catch (e: Exception) {
listOf()
}
}
/**
* Return a list of paths that should be ignored in commit analysis.
*/
private fun getIgnoredPaths(repo: Repository, objectId: ObjectId?): List {
return try {
if (objectId == null) {
return listOf()
}
val list = mutableListOf()
val fileLoader = repo.open(objectId)
val reader =
BufferedReader(InputStreamReader(fileLoader.openStream()))
var collectIgnored = false
for (line in reader.lines()) {
if (line == "" || line.startsWith("#")) {
continue
}
if (line.startsWith("[")) {
collectIgnored = (line == "[ignore]")
continue
}
if (collectIgnored) {
list.add(line)
}
}
list
}
catch(e: Exception) {
listOf()
}
}
private fun getCoauthors(message: String): List {
val coauthorsResult = coauthoredRegex.findAll(message)
val coauthors = mutableListOf()
if (coauthorsResult.toList().isNotEmpty()) {
coauthorsResult.toList().map { result ->
val coauthorName = result.groupValues[1]
val coauthorEmail = result.groupValues[2].toLowerCase()
coauthors.add(Author(coauthorName, coauthorEmail))
}
}
return coauthors
}
}
================================================
FILE: src/main/kotlin/app/hashers/CommitHasher.kt
================================================
// Copyright 2017 Sourcerer Inc. All Rights Reserved.
// Author: Anatoly Kislov (anatoly@sourcerer.io)
package app.hashers
import app.Logger
import app.api.Api
import app.extractors.Extractor
import app.model.Commit
import app.model.Repo
import io.reactivex.Observable
import java.util.concurrent.TimeUnit
/**
* CommitHasher hashes repository and uploads stats to server.
*/
class CommitHasher(private val serverRepo: Repo = Repo(),
private val api: Api,
private val rehashes: List
