[ { "path": ".githooks/pre-commit", "content": "#!/bin/sh\n\n# git config core.hooksPath .githooks\n\necho \"Running pre-commit checks at `pwd`\"\n\n{\n echo \"golangci-lint run ./...\"\n\tgolangci-lint run ./...\n} || {\n\texitStatus=$?\n\n\tif [ $exitStatus ]; then\n\t\tprintf \"\\nLint errors in your code, please fix them and try again.\"\n\t\texit 1\n\tfi\n}\n\n{\n echo \"go test ./...\"\n\tgo test ./...\n} || {\n\texitStatus=$?\n\n\tif [ $exitStatus ]; then\n\t\tprintf \"\\nTest errors in your code, please fix them and try again.\"\n\t\texit 1\n\tfi\n}\n\n" }, { "path": ".github/workflows/test-lint-release.yml", "content": "name: Test, Lint & Release\n\non: [ push, pull_request ]\n\njobs:\n go-test:\n strategy:\n fail-fast: false\n matrix:\n go: ['1.21.1']\n platform: [ubuntu-latest, windows-latest]\n runs-on: ${{ matrix.platform }}\n steps:\n - if: github.actor == 'nektos/act'\n name: act workaround\n run: apt update && apt install -y zstd gcc git\n - name: Set up Go\n uses: actions/setup-go@v2\n with:\n go-version: ${{ matrix.go }}\n - name: Show go version\n run: go version\n - name: Checkout\n uses: actions/checkout@v2\n - name: go mod package cache\n uses: actions/cache@v2\n with:\n # In order:\n # * Module download cache\n # * Build cache (Linux)\n # * Build cache (Mac)\n # * Build cache (Windows)\n path: |\n ~/go/pkg/mod\n ~/.cache/go-build\n ~/Library/Caches/go-build\n %LocalAppData%\\go-build\n key: ${{ runner.os }}-go-${{ matrix.go }}-${{ hashFiles('**/go.mod') }}\n restore-keys: |\n ${{ runner.os }}-go-${{ matrix.go }}\n - name: Run tests\n run: go test -short -cover ./...\n\n go-lint:\n name: lint\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v2\n - name: golangci-lint\n uses: golangci/golangci-lint-action@v2\n with:\n version: latest\n\n xgo:\n if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')\n needs: [go-test, go-lint]\n \n strategy:\n fail-fast: false\n matrix:\n go_version: [ 1.21.x ]\n\n runs-on: ubuntu-latest\n\n steps:\n - name: Checkout\n uses: actions/checkout@v2\n - name: Get current date\n id: date\n run: echo \"::set-output name=date::$(date +'%F')\"\n - name: Get current git tag or commit\n id: tag\n run: echo \"::set-output name=tag::$(git describe --tags --always)\"\n - name: Build with xgo\n uses: crazy-max/ghaction-xgo@v1\n with:\n xgo_version: latest\n go_version: ${{ matrix.go_version }}\n pkg: cmd/rapina\n dest: build\n prefix: rapina-${{ steps.tag.outputs.tag }}\n targets: windows/386,windows/amd64,linux/386,linux/amd64,darwin/386,darwin/amd64\n v: false\n x: false\n race: false\n ldflags: -s -w -X main.build=${{ steps.date.outputs.date }} -X main.version=${{ steps.tag.outputs.tag }}\n buildmode: default\n - name: Run UPX\n uses: gacts/upx@master\n with:\n dir: 'build'\n upx_args: '-9'\n - name: Checksum\n run: |\n cd build\n sha1sum rapina* > sha1sum.txt\n - name: Generate changelog\n id: changelog\n uses: metcalfc/changelog-generator@v1.0.0\n with:\n myToken: ${{ secrets.GITHUB_TOKEN }}\n - name: Release\n uses: softprops/action-gh-release@v1\n with:\n files: build/*\n body: ${{ steps.changelog.outputs.changelog }}\n draft: false\n prerelease: false\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".gitignore", "content": "# Binaries for programs and plugins\n*.exe\n*.exe~\n*.dll\n*.so\n*.dylib\ndebug\n*.db-journal\n.vscode\nbin/**\n\n# Temporary and data files\n*.zip\n**/.data\n*.csv\n*.xls*\n*.db\n*.old\n.vscode/*\n*.sql\n*_string.go\n*.yaml\n*.yml\n!.travis.yml\nwiki/**\n.DS_Store\n\n# Test binary, build with `go test -c`\n*.test\n\n# Output of the go coverage tool, specifically when used with LiteIDE\n*.out\n\n# Skip these (keep this at the end)\n!.github/**\n\n# Dependency Analytics\ntarget/**\n" }, { "path": "LICENSE", "content": "The MIT License (MIT)\n\nCopyright © 2018 Adriano P\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE." }, { "path": "Makefile", "content": "BUILDDIR=./cmd/...\nSOURCEDIR=.\nSOURCES := $(shell find $(SOURCEDIR) -name '*.go')\n\nBINARYDIR=./bin/\nBINARY=bin/rapina\nWINBINARY=bin/rapina.exe\nOSXBINARY=bin/rapina-osx\n\nVERSION=`git describe --tags --always`\nBUILD_TIME=`date +%F`\n\nexport GO111MODULE=on\n\n# Setup the -ldflags option for go build here, interpolate the variable values\nLDFLAGS=-ldflags \"-w -s -X main.version=${VERSION} -X main.build=${BUILD_TIME}\"\n\n.DEFAULT_GOAL: $(BINARY)\n\n$(BINARY): $(SOURCES) $(wildcard ../*.go) $(wildcard ../parsers/*.go) $(wildcard ../reports/*.go)\n\tCGO_CFLAGS=\"-O2 -Wno-return-local-addr\" go build ${LDFLAGS} -o $(BINARYDIR) $(BUILDDIR)\n\nwin: $(SOURCES)\n\t# go get -v -d ../...\n\tGOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc CXX=i686-w64-mingw32-g++ CGO_CFLAGS=\"-O2 -Wno-return-local-addr\" CGO_LDFLAGS=\"-lssp -w\" go build ${LDFLAGS} -o ${BINARYDIR} $(BUILDDIR)\n\nosx: $(SOURCES)\n\t# go get -v -d ../...\n\tGOOS=darwin GOARCH=amd64 CGO_ENABLED=1 CC=o64-clang CXX=o64-clang++ CGO_CFLAGS=\"-O2 -Wno-return-local-addr\" CGO_LDFLAGS=\"-w\" go build ${LDFLAGS} -o ${BINARYDIR} $(BUILDDIR)\n\n.PHONY: install\ninstall:\n\tgo install ${LDFLAGS} ./...\n\n.PHONY: clean\nclean:\n\tif [ -f ${BINARY} ] ; then rm ${BINARY} ; fi\n\n.PHONY: list\nlist:\n\tcd .. && go list -f '{{ join .Imports \"\\n\" }}'\n" }, { "path": "NOTICE", "content": "================================================================================\n| Open Database License (ODbL) |\n\nContains information from \"Portal Dados Abertos CVM\", which is made available\nhere under the Open Database License (ODbL).\n\n https://www.opendatacommons.org/licenses/odbl/1.0/\n\n\n================================================================================\n| BSD-2-Clause |\n\n==> github.com/pkg/errors: Copyright (c) 2015, Dave Cheney .\nAll rights reserved.\n\n https://opensource.org/licenses/BSD-2-Clause\n\n\n================================================================================\n| BSD 3-Clause License |\n\n==> github.com/360EntSecGroup-Skylar/excelize:\nCopyright (c) 2016 - 2018 360 Enterprise Security Group, Endpoint Security, inc.\nAll rights reserved.\n\n==> github.com/manifoldco/promptui: Copyright (c) 2017, Arigato Machine Inc.\nAll rights reserved.\n\n https://opensource.org/licenses/BSD-3-Clause\n\n\n================================================================================\n| Apache License, Version 2.0 |\n\n==> github.com/spf13/cobra: Copyright © 2013 Steve Francia .\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n\n================================================================================\n| The MIT License (MIT) |\n\n==> github.com/mattn/go-sqlite3: Copyright (c) 2014 Yasuhiro Matsumoto.\n\n https://opensource.org/licenses/MIT" }, { "path": "README.md", "content": "# 𝚛𝚊𝚙𝚒𝚗𝚊\n\nDownload e processamento de dados[1](#disclaimer) financeiros de empresas brasileiras diretamente da [CVM](http://dados.cvm.gov.br/dados/CIA_ABERTA/DOC/DFP/).\n\n[![GitHub release](https://img.shields.io/github/tag/dude333/rapina.svg?label=latest)](https://github.com/dude333/rapina/releases)\n[![Travis](https://img.shields.io/travis/dude333/rapina/master.svg)](https://travis-ci.org/dude333/rapina)\n[![License](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)\n\nEste programa baixa e processa os arquivos CSV do site da CVM e os armazena em um banco de dados local (sqlite), onde são extraídos os dados **consolidados** do balanço patrimonial, fluxo de caixa, DRE (demonstração de resultado), DVA (demonstração de valor adicionado).\n\nSão coletados vários arquivos CSV desde 2010. Cada um destes arquivos contém informações do ano corrente e também do ano anterior, dessa forma foi possível extrair também os dados de 2009.\n\nCom base nestes dados, são criados os relatórios por empresa, com um comparativo de outras empresas do mesmo setor. A classificação dos setores é baixada do site da Bovespa e armazenada no arquivo setores.yml (no formato [YAML](https://medium.com/@akio.miyake/introdu%C3%A7%C3%A3o-b%C3%A1sica-ao-yaml-para-ansiosos-2ac4f91a4443)), que pode ser editado para se adequar aos seus critérios, caso necessário.\n\nA partir do release v0.11.0, passou-se a usar os dados trimestrais para compor os valores do ano corrente, usando-se para isso os últimos 4 trimestre ([TTM](#ttm-calc)), ou seja, a soma dos dados trimestrais do ano corrente com alguns do ano anterior, mantendo-se assim uma mesma base de comparação com os anos anteriores. \n\n| :memo: | **NOTA**: Desenvolvi a [versão 2 do rapina](https://github.com/dude333/rapinav2) com o intuito de criar relatórios trimestrais. Pretendo integrar essa nova funcionalidade a este repositório no futuro. |\n|---------------|:------------------------|\n\n# 1. Instalação\n\nNão é necessário instalar, basta baixar o executável da [página de release](https://github.com/dude333/rapina/releases) e renomeie o executável para `rapina.exe` (no caso do Windows) ou `rapina` (para o Linux ou macOS).\n\nAbra o terminal ([CMD](https://superuser.com/a/340051/61616) no Windows) e rode os comandos listados abaixo.\n\n# 2. Uso\n\nNa primeira vez, rodar o seguinte comando para baixar e processar os arquivos do site da CVM:\n\n ./rapina update\n\nDepois, para obter o relatório de uma determinada empresa, com o resumo das empresas do mesmo setor:\n\n ./rapina report \n\n_Eventualmente, as empresas corrigem algum dado e enviam um novo arquivo à CVM, então é recomendável rodar o `rapina update` periodicamente._\n\n# 3. Detalhe dos Comandos\n\n## 3.1. update\n\n**Download e armazenamento de dados financeiros no banco de dados local.**\n\n ./rapina update [-s]\n\nBaixa todos os arquivos disponíveis no servidor da CVM, processa o conteúdo e o armazena num banco de dados sqlite em `.data/rapina.db`.\n\nEste comando deve ser executado **pelo menos uma vez** antes dos outros comandos.\n\n### 3.1.1 Opção\n\n```\n -s, --sectors Baixa a classificação setorial das empresas e fundos negociados na B3\n```\n\nUsado para obter apenas o arquivo de classificação setorial atualizado.\n\n## 3.2. list\n\n**Listas**\n\n ./rapina list\n\n### 3.2.1 Lista todas as empresas disponíveis\n\n```\n -e, --empresas Lista todas as empresas disponíveis\n```\n\n### 3.2.2 Lista as empresas do mesmo setor\n\n```\n -s, --setor string Lista todas as empresas do mesmo setor\n```\n\nPor exemplo, para listar todas as empras do mesmo setor do Itaú: `./rapina lista -s itau`\n\nO resultado mostra a lista das empresas do mesmo setor contidos no banco de dados e no arquivo **setores.yml**, que você pode editar caso queira realocar os setores das empresas.\n\n### 3.2.3 Lista empresas com critério de lucro líquido\n\n```\n -l, --lucroLiquido número Lista empresas com lucros lucros positivos e com a taxa de crescimento definida\n```\n\nLista as empresas com lucros líquidos positivos e com uma taxa de crescimento definida em relação ao mês anterior. \nPor exemplo:\n* Para listar as empresas com crescimento mínimo de 10% em relação ao ano anterior: `./rapina list -l 0.1`\n* Para listar as empresas com variação no lucro de maiores que -5% em relação ao ano anterior: `./rapina list -l -0.05`\n\n\n## 3.3. report\n\n**Cria uma planilha com os dados financeiros de uma empresa.**\n\n ./rapina report [opções] empresa\n\nSerá criada uma planilha com os dados financeiros (BP, DRE, DFC) e, em outra aba, o resumo de todas as empresas do mesmo setor.\n\nA lista setorial é obtida da B3 e salva no arquivo `setor.yml` (via comando `update -s`). Caso deseje alterar o agrupamento setorial, basta editar este arquivo. Mas lembre-se que ao rodar o `update -s` o arquivo será sobrescrito.\n\nNo **Linux** ou **macOS**, use as setas para navegar na lista das empresas. No **Windows**, use j e k.\n\n### 3.3.1. Opções\n\n```\n -a, --all Mostra todos os indicadores\n -x, --extraRatios Reporte de índices extras\n -F, --fleuriet Capital de giro no modelo Fleuriet\n -o, --omitSector Omite o relatório das empresas do mesmo setor\n -d, --outputDir string Diretório onde o relatório será salvo (default \"reports\")\n -s, --scriptMode Para modo script (escolhe a empresa com nome mais próximo)\n -f, --showShares Mostra o número de ações e free float\n\n```\n\n\n### 3.3.2. Exemplos\n\n ./rapina report WEG\n\nA planilha será salva em `./reports`\n\n ./rapina report \"TEC TOY\" -s -d /tmp/output\n\nA planilha será salva em `/tmp/output`\n\n# 4. Nova funções\n\n## 4.1. fii\n\n**Relatórios relacionados aos Fundos de Investimento Imobiliários**\n\n### 4.1.1. rendimentos\n\n ./rapina fii rendimentos [-n] ABCD11 EFGH11...\n\nOnde `-n` é o número de meses a serem apresentados.\n\nE como parâmetros, passe uma lista de FIIs separados por espaço.\n\n#### 4.1.1.1 Exemplo\n\n ./rapina fii rendimentos -n 2 knip11 hfof11\n\n```\n-------------------------------------------------------------------\nKNIP11\n-------------------------------------------------------------------\n DATA COM RENDIMENTO COTAÇÃO YELD YELD a.a.\n ---------- ---------- ---------- ------ ---------\n 2021-04-30 R$ 1,00 R$ 113,00 0,88% 11,15%\n 2021-03-31 R$ 1,02 R$ 115,95 0,88% 11,08%\n-------------------------------------------------------------------\nHFOF11\n-------------------------------------------------------------------\n DATA COM RENDIMENTO COTAÇÃO YELD YELD a.a.\n ---------- ---------- ---------- ------ ---------\n 2021-04-30 R$ 0,60 R$ 99,75 0,60% 7,46%\n 2021-03-31 R$ 0,56 R$ 100,70 0,56% 6,88%\n-------------------------------------------------------------------\n\n```\n\n# 4.2. server\n\n**Web server para visualização dos relatórios no browser**\n\n## 4.2.1. Exemplo\n\n ./rapina server\n\n 2021/05/11 19:23:15 Listening on :3000...\n\nPara visualizar a página, abrir o link http://localhost:3000\n\n**NOTA:** Por hora só está disponível o relatório de rendimentos de FIIs.\n\n\n# 5. Possíveis problemas\n\nAlgumas distribuições Linux (Fedora 34, por exemplo) podem encontrar problemas com as autoridades certificadores (Global Sign) presentes nos certificados SSL dos websites da B3. Em caso de erro `x509: certificate signed by unknown authority`, deve-se importar manualmente o Root CA para o trusted database do sistemas operacional:\n\n**Fedora 34 / CentOS** \n\n1. Realizar o download do Issuer Root Cert\n\n `curl http://secure.globalsign.com/cacert/gsrsaovsslca2018.crt > /tmp/global-signer.der`\n\n2. Converter de .der para .pem\n\n `openssl x509 -inform der -in /tmp/global-signer.der -out /tmp/globalsignroot.pem`\n\n3. Importar .pem arquivo para pasta de anchors\n\n `sudo cp /tmp/globalsignroot.pem /usr/share/pki/ca-trust-source/anchors/`\n\n4. Atualizar base de trusted certificates\n\n `sudo update-ca-trust`\n\n**Ubuntu** \n\n1. Realizar o download do Issuer Root Cert\n\n `curl https://secure.globalsign.net/cacert/Root-R1.crt > /tmp/GlobalSign_Root_CA.crt`\n `curl https://secure.globalsign.net/cacert/Root-R2.crt > /tmp/GlobalSign_Root_CA_R2.crt`\n\n2. Importar .crt arquivos para pasta de certificados\n\n `sudo cp /tmp/GlobalSign_Root_CA.crt /usr/local/share/ca-certificates/`\n `sudo cp /tmp/GlobalSign_Root_CA_R2.crt /usr/local/share/ca-certificates/`\n\n3. Atualizar base de trusted certificates\n\n `sudo update-ca-trust`\n\n# 6. Como compilar\n\nSe quiser compilar seu próprio executável, primeiro [baixe e instale](https://golang.org/dl/) o compilador Go (v1.16 ou maior). Depois execute estes passos:\n\n1. `git clone github.com/dude333/rapina`\n2. `cd rapina`\n3. `make`\n\nO executável será criado na pasta `bin`. Você pode movê-lo para outro local. Ao rodar a primeira vez, apenar o executável é necessário, mas após rodá-lo, será criado um diretório `.data` que deverá ser movido junto com o executável, caso queira trazer o dados.\n\nIMPORTANTE: para compilar a biblioteca do sqlite, é necessário ter um compilador C instalado na máquina (para o Windows, mais detalhes [aqui](https://github.com/mattn/go-sqlite3#windows)).\n\n# 7. Contribua\n\n1. Faça um fork deste projeto no [github.com](github.com/dude333/rapina)\n2. `git clone https://github.com/`*your_username*`/rapina && cd rapina`\n3. `git checkout -b `*my-new-feature*\n4. Faça as modificações\n5. `git add .`\n6. `git commit -m 'Add some feature'`\n7. `git push origin my-new-feature`\n8. Crie um _pull request_\n\n# 8. Screenshot\n\n![WEG](https://i.imgur.com/czPhPkH.png)\n\n\n# 9. License\n\nMIT\n\n\n\n\n
\n
\n
\n1: *Os dados são fornecidos \"no estado em que se encontram\" e somente para fins informativos, não para fins comerciais ou de consultoria.*\n" }, { "path": "README_en.md", "content": "# 𝚛𝚊𝚙𝚒𝚗𝚊\n\nDownload and process Brazilian companies' financial data directly from [CVM](http://dados.cvm.gov.br/dados/CIA_ABERTA/DOC/DFP/). [[Em português](./README.md)]\n\n[![GitHub release](https://img.shields.io/github/tag/dude333/rapina.svg?label=latest)](https://github.com/dude333/rapina/releases)\n[![Travis](https://img.shields.io/travis/dude333/rapina/master.svg)](https://travis-ci.org/dude333/rapina)\n[![License](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)\n\n# 1. Installation\n\nNo installation required, just download the [latest released executable](https://github.com/dude333/rapina/releases). Then open a terminal ([CMD](https://superuser.com/a/340051/61616) on Windows) and run the commands shown below.\n\n# 2. Commands\n\nFor the first time, run the following command:\n\n ./rapina get\n\nThen, to get a company report, together with a summary for the companies from the same sector:\n\n ./rapina report \n\n## 2.1. `get`| Download and store financial data into the local database\n\n ./rapina get [-s]\n\nIt downloads all files from CVM web server, parses their contents and stores on a sqlite database at `.data/rapina.db`.\n\nThis command must be run **at least once** before you run the other commands.\n\n### 2.1.1 Option\n\n```\n -s, --sectors Download and sector classification for companies listed at B3\n```\n\nUsed to get only a summary for the other companies from the same sector.\n\n[![asciicast](https://asciinema.org/a/656x2hrtCFFZLVLa9fGGcetw7.svg)](https://asciinema.org/a/656x2hrtCFFZLVLa9fGGcetw7?speed=4&autoplay=1&loop=1)\n\n## 2.2. `list`| List all companies\n\n ./rapina list\n\n[![asciicast](https://asciinema.org/a/TbJyGaOodJUxEzjDySQu3MaEW.svg)](https://asciinema.org/a/TbJyGaOodJUxEzjDySQu3MaEW?autoplay=1&loop=1)\n\n## 2.3. `report`| Create a spreadsheet with a company financial data\n\n ./rapina report [flags] company_name\n\nA spreadsheet with the financial data will be created and, on another sheet, the summary of all companies in the same sector.\n\nThe sector list is obtained from B3 and saved in the `sector.yml` file (via `get -s` command). If you want to change the sector grouping, just edit this file.\n\n### 2.3.1. Options\n\n```\n -d, --outputDir string Output directory [default: ./reports]\n -s, --scriptMode Does not show companies list; uses the most similar\n company name\n```\n\nOn **Linux** or **macOS**, use the arrow keys to navigate through the companies list. On **Windows**, use j and k.\n\n[![asciicast](https://asciinema.org/a/jhmHxzgROtc8EBh3tkSwYTaa9.svg)](https://asciinema.org/a/jhmHxzgROtc8EBh3tkSwYTaa9?autoplay=1&loop=1)\n\n### 2.3.2. Examples\n\n ./rapina report WEG\n\nThe spreadsheet will be saved at `./reports`\n\n ./rapina report \"TEC TOY\" -s -d /tmp/output\n\nThe spreadsheet will be saved at `/tmp/output`\n\n# 3. Troubleshooting\n\nSome Linux distributions (e.g. Fedora 34) might face some issues regarding the signer authority (Global Sign) that B3 is using on its SSL certificates. In case of `x509: certificate signed by unknown authority` error, one should manually import the Root CA certificate into the O.S. trusted database:\n\n**Fedora 34 / CentOS** \n\n1. Download the Issuer Root Cert\n\n `curl http://secure.globalsign.com/cacert/gsrsaovsslca2018.crt > /tmp/global-signer.der`\n\n2. Convert from .der to .pem\n\n `openssl x509 -inform der -in /tmp/global-signer.der -out /tmp/globalsignroot.pem`\n\n3. Move the .pem file to the anchors folder\n\n `sudo cp /tmp/globalsignroot.pem /usr/share/pki/ca-trust-source/anchors/`\n\n4. Update the trusted certificates database\n\n `sudo update-ca-trust`\n\n**Ubuntu** \n\n1. Download the Issuer Root Cert\n\n `curl https://secure.globalsign.net/cacert/Root-R1.crt > /tmp/GlobalSign_Root_CA.crt`\n `curl https://secure.globalsign.net/cacert/Root-R2.crt > /tmp/GlobalSign_Root_CA_R2.crt`\n\n2. Move the .crt files to the certificates folder\n\n `sudo cp /tmp/GlobalSign_Root_CA.crt /usr/local/share/ca-certificates/`\n `sudo cp /tmp/GlobalSign_Root_CA_R2.crt /usr/local/share/ca-certificates/`\n\n3. Update the trusted certificates database\n\n `sudo update-ca-trust`\n\n\n# 4. How to compile\n\nIf you want to compile your own executable, you need first to [download and install](https://golang.org/dl/) the Go compiler. Then follow these steps:\n\n1. `go get github.com/dude333/rapina`\n2. `cd $GOPATH/src/github.com/dude333/rapina`\n3. Change to the cli directory (`cd cli`)\n4. Compile using the Makefile (`make`). _To cross compile for Windows on Linux, use `make win`_.\n\n# 5. Contributing\n\n1. Fork it\n2. `cd $GOPATH/src/github.com/your_username`\n3. Download your fork to your PC (`git clone https://github.com/your_username/rapina && cd rapina`)\n4. Create your feature branch (`git checkout -b my-new-feature`)\n5. Make changes and add them (`git add .`)\n6. Commit your changes (`git commit -m 'Add some feature'`)\n7. Push to the branch (`git push origin my-new-feature`)\n8. Create new pull request\n\n# 6. Screenshot\n\n![WEG](https://i.imgur.com/czPhPkH.png)\n\n# 7. License\n\nMIT\n" }, { "path": "cmd/rapina/cmdutils.go", "content": "package main\n\nimport (\n\t\"database/sql\"\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/manifoldco/promptui\"\n\t\"github.com/pkg/errors\"\n)\n\n// Directory where the DB and downloaded files are stored\nconst dataDir = \".data\"\nconst yamlFile = \"./setores.yml\"\n\n// Parms holds the input parameters\ntype Parms struct {\n\t// Company name to be processed\n\tCompany string\n\t// SpcfctnCd to indentify the ticker\n\tSpcfctnCd string\n\t// Report format (xlsx/stdout)\n\tFormat string\n\t// OutputDir: path of the output xlsx\n\tOutputDir string\n\t// YamlFile: file with the companies' sectors\n\tYamlFile string\n\t// Reports is a map with the reports and reports items to be printed\n\tReports map[string]bool\n}\n\n//\n// openDatabase to be used by parsers and reporting\n//\nfunc openDatabase() (db *sql.DB, err error) {\n\tif err := os.MkdirAll(dataDir, os.ModePerm); err != nil {\n\t\treturn nil, err\n\t}\n\tconnStr := \"file:\" + dataDir + \"/rapina.db?cache=shared&mode=rwc&_journal_mode=WAL&_busy_timeout=5000\"\n\tdb, err = sql.Open(\"sqlite3\", connStr)\n\tif err != nil {\n\t\treturn db, errors.Wrap(err, \"database open failed\")\n\t}\n\tdb.SetMaxOpenConns(1)\n\n\treturn\n}\n\n//\n// promptUser presents a navigable list to be selected on CLI\n//\nfunc promptUser(list []string, label string) (result string) {\n\tif label == \"\" {\n\t\tlabel = \"Selecione a Empresa\"\n\t}\n\ttemplates := &promptui.SelectTemplates{\n\t\tHelp: `{{ \"Use estas teclas para navegar:\" | faint }} {{ .NextKey | faint }} ` +\n\t\t\t`{{ .PrevKey | faint }} {{ .PageDownKey | faint }} {{ .PageUpKey | faint }} ` +\n\t\t\t`{{ if .Search }} {{ \"and\" | faint }} {{ .SearchKey | faint }} {{ \"toggles search\" | faint }}{{ end }}`,\n\t}\n\n\tprompt := promptui.Select{\n\t\tLabel: label,\n\t\tItems: list,\n\t\tTemplates: templates,\n\t}\n\n\t_, result, err := prompt.Run()\n\n\tif err != nil {\n\t\tfmt.Printf(\"Prompt failed %v\\n\", err)\n\t\treturn\n\t}\n\n\treturn\n}\n\n//\n// filename cleans up the filename and returns the path/filename\nfunc filename(path, name string) (fpath string, err error) {\n\tclean := func(r rune) rune {\n\t\tswitch r {\n\t\tcase ' ', ',', '/', '\\\\':\n\t\t\treturn '_'\n\t\t}\n\t\treturn r\n\t}\n\tpath = strings.TrimSuffix(path, \"/\")\n\tname = strings.TrimSuffix(name, \".\")\n\tname = strings.Map(clean, name)\n\tfpath = filepath.FromSlash(path + \"/\" + name + \".xlsx\")\n\n\tconst max = 50\n\tvar x int\n\tfor x = 1; x <= max; x++ {\n\t\t_, err = os.Stat(fpath)\n\t\tif err == nil {\n\t\t\t// File exists, try again with another name\n\t\t\tfpath = fmt.Sprintf(\"%s/%s(%d).xlsx\", path, name, x)\n\t\t} else if os.IsNotExist(err) {\n\t\t\terr = nil // reset error\n\t\t\tbreak\n\t\t} else {\n\t\t\terr = fmt.Errorf(\"file %s stat error: %v\", fpath, err)\n\t\t\treturn\n\t\t}\n\t}\n\n\tif x > max {\n\t\terr = fmt.Errorf(\"remova o arquivo %s/%s.xlsx antes de continuar\", path, name)\n\t\treturn\n\t}\n\n\t// Create directory\n\t_ = os.Mkdir(path, os.ModePerm)\n\n\t// Check if the directory was created\n\tif _, err := os.Stat(path); os.IsNotExist(err) {\n\t\treturn \"\", errors.Wrap(err, \"diretório não pode ser criado\")\n\t}\n\n\treturn\n}\n" }, { "path": "cmd/rapina/cmdutils_test.go", "content": "package main\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n)\n\nfunc TestFilename(t *testing.T) {\n\ttempDir, _ := os.MkdirTemp(\"\", \"rapina-test\")\n\n\ttable := []struct {\n\t\tpath string\n\t\tname string\n\t\texpected string\n\t}{\n\t\t{tempDir + \"/test\", \"sample\", tempDir + \"/test/sample.xlsx\"},\n\t\t{tempDir, \"File 100\", tempDir + \"/File_100.xlsx\"},\n\t\t{tempDir, \"An,odd/file\\\\name\", tempDir + \"/An_odd_file_name.xlsx\"},\n\t}\n\n\tfor _, x := range table {\n\t\treturned, err := filename(x.path, x.name)\n\t\texpected := filepath.FromSlash(x.expected)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"filename returned an error %v.\", err)\n\t\t} else if returned != expected {\n\t\t\tt.Errorf(\"filename got: %s, want: %s.\", returned, expected)\n\t\t}\n\t}\n\n}\n" }, { "path": "cmd/rapina/fii.go", "content": "/*\nCopyright © 2021 Adriano P \nDistributed under the MIT License.\n*/\npackage main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"path/filepath\"\n\n\t\"github.com/spf13/cobra\"\n)\n\ntype fiiFlags struct {\n\tnum int // number of months since current\n\tdividends fiiDividendsFlags\n\tmonthly fiiMonthlyFlags\n}\n\n// fiiCmd represents the fii command\nvar fiiCmd = &cobra.Command{\n\tUse: \"fii\",\n\tShort: \"Comando relacionados aos FIIs\",\n\tLong: `Comando relacionado aos Fundos de Investiment Imobiliários (FII).`,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\t_ = cmd.Help()\n\t},\n\tExample: func() string {\n\t\treturn fmt.Sprintf(\"%s fii rendimentos KNIP11 KNCR11 HGLG11 -n 4\", filepath.Base(os.Args[0]))\n\t}(),\n}\n\nfunc init() {\n\trootCmd.AddCommand(fiiCmd)\n\tfiiCmd.PersistentFlags().IntVarP(&flags.fii.num,\n\t\tFnum, \"n\", 1, \"número de meses desde o último disponível\")\n}\n" }, { "path": "cmd/rapina/fii_dividends.go", "content": "/*\nCopyright © 2021 Adriano P \nDistributed under the MIT License.\n*/\npackage main\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/dude333/rapina/reports\"\n\t\"github.com/spf13/cobra\"\n\t\"github.com/spf13/viper\"\n)\n\ntype fiiDividendsFlags struct {\n\tformat string // output format of the report\n}\n\n// fiiDividendsCmd represents the rendimentos command\nvar fiiDividendsCmd = &cobra.Command{\n\tUse: \"rendimentos\",\n\tAliases: []string{\"rend\", \"dividendos\", \"dividends\", \"div\"},\n\tArgs: cobra.MinimumNArgs(1),\n\tShort: \"Lista os rendimentos de um FII\",\n\tLong: `Lista os rendimentos de um Fundos de Investiment Imobiliários (FII).`,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\t// Number of reports\n\t\tn := flags.fii.num\n\t\tif n <= 0 {\n\t\t\tn = 1\n\t\t}\n\n\t\tparms := make(map[string]string)\n\t\t// Verbose\n\t\tif flags.verbose {\n\t\t\tparms[Fverbose] = \"true\"\n\t\t}\n\t\t// Report format\n\t\tparms[Fformat] = flags.fii.dividends.format\n\n\t\tif err := FIIDividends(parms, args, n); err != nil {\n\t\t\tlog.Println(err)\n\t\t}\n\n\t},\n\tExample: func() string {\n\t\treturn fmt.Sprintf(\"%s fii rendimentos KNIP11 KNCR11 HGLG11 -n 4\", filepath.Base(os.Args[0]))\n\t}(),\n}\n\nfunc init() {\n\tfiiCmd.AddCommand(fiiDividendsCmd)\n\tfiiDividendsCmd.Flags().StringVarP(&flags.fii.dividends.format, Fformat,\n\t\t\"f\", \"tabela\", \"formato do relatório: tabela|csv|csvrend\")\n}\n\n// FIIDividends prints the dividends from 'code' for 'n' months,\n// starting from latest.\nfunc FIIDividends(parms map[string]string, codes []string, n int) error {\n\tfor i := 0; i < len(codes); i++ {\n\t\tcodes[i] = strings.ToUpper(codes[i])\n\t}\n\n\tdb, err := openDatabase()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\topts := reports.FIITerminalOptions{\n\t\tAPIKey: viper.GetString(\"apikey\"),\n\t\tDataDir: dataDir,\n\t}\n\n\tr, err := reports.NewFIITerminal(db, opts)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tr.SetParms(parms)\n\n\terr = r.Dividends(codes, n)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n" }, { "path": "cmd/rapina/fii_monthly.go", "content": "/*\nCopyright © 2021 Adriano P \nDistributed under the MIT License.\n*/\npackage main\n\nimport (\n\t\"log\"\n\t\"strings\"\n\n\t\"github.com/dude333/rapina/reports\"\n\t\"github.com/spf13/cobra\"\n\t\"github.com/spf13/viper\"\n)\n\ntype fiiMonthlyFlags struct {\n\tformat string // output format of the report\n}\n\n// fiiMonthlyCmd represents the rendimentos command\nvar fiiMonthlyCmd = &cobra.Command{\n\tHidden: true,\n\tUse: \"mensal\",\n\tAliases: []string{\"monthly\"},\n\tArgs: cobra.MinimumNArgs(1),\n\tShort: \"Lista os informes mensais de um FII\",\n\tLong: `Lista os informes mensais de um Fundos de Investiment Imobiliários (FII).`,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\t// Number of reports\n\t\tn := flags.fii.num\n\n\t\tparms := make(map[string]string)\n\t\t// Verbose\n\t\tif flags.verbose {\n\t\t\tparms[Fverbose] = \"true\"\n\t\t}\n\t\t// Report format\n\t\tparms[Fformat] = flags.fii.monthly.format\n\n\t\tif err := FIIMonthly(parms, args, n); err != nil {\n\t\t\tlog.Println(err)\n\t\t}\n\n\t},\n}\n\nfunc init() {\n\tfiiCmd.AddCommand(fiiMonthlyCmd)\n\tfiiMonthlyCmd.Flags().StringVarP(&flags.fii.monthly.format, Fformat,\n\t\t\"f\", \"tabela\", \"formato do relatório: tabela|csv|csvrend\")\n}\n\n// FIIMonthly prints the monthly reports from 'code' for 'n' months,\n// starting from latest.\nfunc FIIMonthly(parms map[string]string, codes []string, n int) error {\n\tfor i := 0; i < len(codes); i++ {\n\t\tcodes[i] = strings.ToUpper(codes[i])\n\t}\n\n\tdb, err := openDatabase()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\topts := reports.FIITerminalOptions{\n\t\tAPIKey: viper.GetString(\"apikey\"),\n\t\tDataDir: dataDir,\n\t}\n\n\tr, err := reports.NewFIITerminal(db, opts)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tr.SetParms(parms)\n\n\terr = r.Monthly(codes, n)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n" }, { "path": "cmd/rapina/flags.go", "content": "package main\n\n// Flags constants\nconst (\n\t// Root persistent\n\tFverbose = \"verbose\"\n\n\t// fiiCmd persistent\n\tFnum = \"num\"\n\n\t// fiiDividendsCmd\n\tFformat = \"format\"\n)\n" }, { "path": "cmd/rapina/list.go", "content": "// Copyright © 2018 Adriano P\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy\n// of this software and associated documentation files (the \"Software\"), to deal\n// in the Software without restriction, including without limitation the rights\n// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n// copies of the Software, and to permit persons to whom the Software is\n// furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in\n// all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n// THE SOFTWARE.\n\npackage main\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/dude333/rapina/reports\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/spf13/cobra\"\n)\n\n// listCmd represents the list command\nvar listCmd = &cobra.Command{\n\tUse: \"list\",\n\tShort: \"Lista informações armazenadas no banco de dados\",\n}\n\nfunc init() {\n\tvar (\n\t\tlistCompanies bool\n\t\tsector string\n\t\tnetProfitRate float32\n\t)\n\n\trootCmd.AddCommand(listCmd)\n\n\tlistCmd.Flags().BoolVarP(&listCompanies, \"empresas\", \"e\", false, \"Lista todas as empresas disponíveis\")\n\tlistCmd.Flags().StringVarP(§or, \"setor\", \"s\", \"\", \"Lista todas as empresas do mesmo setor\")\n\tlistCmd.Flags().Float32VarP(&netProfitRate, \"lucroLiquido\", \"l\", -0.8, \"Lista empresas com lucros lucros positivos e com a taxa de crescimento definida\")\n\n\tlistCmd.Run = func(cmd *cobra.Command, args []string) {\n\t\tvar err error\n\n\t\tif listCmd.Flags().NFlag() == 0 {\n\t\t\t_ = listCmd.Help()\n\t\t\treturn\n\t\t}\n\n\t\tif listCompanies {\n\t\t\terr = ListCompanies()\n\t\t} else if sector != \"\" {\n\t\t\terr = ListSector(sector, yamlFile)\n\t\t} else if listCmd.Flags().Changed(\"lucroLiquido\") {\n\t\t\terr = ListCompaniesProfits(netProfitRate)\n\t\t}\n\t\tif err != nil {\n\t\t\tfmt.Println(\"[x]\", err)\n\t\t}\n\t}\n\n}\n\n//\n// ListCompanies a company from DB to Excel\n//\nfunc ListCompanies() (err error) {\n\tdb, err := openDatabase()\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"fail to open db\")\n\t}\n\n\tcom, err := reports.ListCompanies(db)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"erro ao listar empresas\")\n\t}\n\tfor _, c := range com {\n\t\tfmt.Println(c)\n\t}\n\n\treturn\n}\n\n//\n// ListSector shows all companies from the same sector as 'company'\n//\nfunc ListSector(company, yamlFile string) (err error) {\n\tdb, err := openDatabase()\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"fail to open db\")\n\t}\n\n\terr = reports.ListSector(db, company, yamlFile)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"erro ao listar empresas\")\n\t}\n\n\treturn\n}\n\n//\n// ListCompaniesProfits lists companies profits\n//\nfunc ListCompaniesProfits(rate float32) (err error) {\n\tdb, err := openDatabase()\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"fail to open db\")\n\t}\n\n\terr = reports.ListCompaniesProfits(db, rate)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"erro ao listar lucros\")\n\t}\n\n\treturn\n}\n" }, { "path": "cmd/rapina/main.go", "content": "// Copyright © 2018 Adriano P\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy\n// of this software and associated documentation files (the \"Software\"), to deal\n// in the Software without restriction, including without limitation the rights\n// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n// copies of the Software, and to permit persons to whom the Software is\n// furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in\n// all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n// THE SOFTWARE.\n\npackage main\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"os/signal\"\n\n\t\"github.com/dude333/rapina/progress\"\n\thomedir \"github.com/mitchellh/go-homedir\"\n\t\"github.com/spf13/cobra\"\n\t\"github.com/spf13/viper\"\n)\n\nvar flags = struct {\n\tverbose bool\n\tfii fiiFlags\n\tserver serverFlags\n}{}\n\nvar cfgFile string\n\n// rootCmd represents the base command when called without any subcommands\nvar rootCmd = &cobra.Command{\n\tUse: \"rapina\",\n\tShort: \"Dados Financeiros de Empresas via CVM.\",\n\tLong: `\nEste programa coleta informações sobre os dados financeiros do\nsite da CVM e os exporta para uma planilha. Dados usados: balanço\npatrimonial ativo e passivo, e também o demonstrativo de resultado\ndo exercício (DRE).`,\n\t// Uncomment the following line if your bare application\n\t// has an action associated with it:\n\t//\tRun: func(cmd *cobra.Command, args []string) { },\n}\n\n// Execute adds all child commands to the root command and sets flags appropriately.\n// This is called by main.main(). It only needs to happen once to the rootCmd.\nfunc Execute() int {\n\tif err := rootCmd.Execute(); err != nil {\n\t\tfmt.Println(err)\n\t\treturn 1\n\t}\n\treturn 0\n}\n\nfunc init() {\n\tcobra.OnInitialize(initConfig)\n\n\t// Here you will define your flags and configuration settings.\n\t// Cobra supports persistent flags, which, if defined here,\n\t// will be global for your application.\n\t// rootCmd.PersistentFlags().StringVar(&cfgFile, \"config\", \"\", \"config file (default is $HOME/.cli.yaml)\")\n\n\t// Cobra also supports local flags, which will only run\n\t// when this action is called directly.\n\trootCmd.PersistentFlags().BoolVarP(&flags.verbose, Fverbose, \"v\", false, \"Mostrar mensagens de execução\")\n\n\tstr := `Uso:{{if .Runnable}}\n {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}}\n {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}}\n\nAliases:\n {{.NameAndAliases}}{{end}}{{if .HasExample}}\n\nExemplos:\n {{.Example}}{{end}}{{if .HasAvailableSubCommands}}\n\nComandos Disponíveis:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name \"help\"))}}\n {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}}\n\nFlags:\n{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}}\n\nGlobal Flags:\n{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}}\n\nTópicos de ajuda opcionais:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}}\n {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}}\n\nUse \"{{.CommandPath}} [command] --help\" para mais informações sobre um comando.{{end}}\n`\n\trootCmd.SetUsageTemplate(str)\n}\n\n// initConfig reads in config file and ENV variables if set.\nfunc initConfig() {\n\tif cfgFile != \"\" {\n\t\t// Use config file from the flag.\n\t\tviper.SetConfigFile(cfgFile)\n\t} else {\n\t\t// Find home directory.\n\t\thome, err := homedir.Dir()\n\t\tif err != nil {\n\t\t\tfmt.Println(err)\n\t\t\tos.Exit(1)\n\t\t}\n\n\t\t// Search config in home directory with name \".rapina\" (without extension).\n\t\tviper.AddConfigPath(home)\n\t\tviper.AddConfigPath(\".\")\n\t\tviper.SetConfigName(\"config\")\n\t}\n\n\tviper.AutomaticEnv() // read in environment variables that match\n\n\t// If a config file is found, read it in.\n\tif err := viper.ReadInConfig(); err == nil {\n\t\tfmt.Fprintf(os.Stderr, \"[INFO] Usando arquivo de configuração %s\\n\\n\", viper.ConfigFileUsed())\n\t}\n}\n\nvar (\n\tversion string\n\tbuild string\n)\n\nfunc main() {\n\tfmt.Fprint(os.Stderr, \"Rapina - Dados Financeiros de Empresas Brasileiras - \")\n\tfmt.Fprintf(os.Stderr, \"%s-%s\\n\", version, build)\n\tfmt.Fprint(os.Stderr, \"(2018-2020) github.com/dude333/rapina\\n\\n\")\n\n\tprogress.Cursor(false)\n\tdefer func() {\n\t\tprogress.Cursor(true)\n\t\tif err := recover(); err != nil { //catch\n\t\t\tfmt.Fprintf(os.Stderr, \"Exception: %v\\n\", err)\n\t\t\tos.Exit(1)\n\t\t}\n\t}()\n\n\t// Handle Ctrl+C\n\tc := make(chan os.Signal, 1)\n\tsignal.Notify(c, os.Interrupt)\n\tgo func() {\n\t\t<-c\n\t\tprogress.Cursor(true)\n\t\tos.Exit(0)\n\t}()\n\n\tret := Execute()\n\tprogress.Cursor(true)\n\tos.Exit(ret)\n}\n" }, { "path": "cmd/rapina/report.go", "content": "/// Copyright © 2018 Adriano P\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy\n// of this software and associated documentation files (the \"Software\"), to deal\n// in the Software without restriction, including without limitation the rights\n// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n// copies of the Software, and to permit persons to whom the Software is\n// furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in\n// all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n// THE SOFTWARE.\n\npackage main\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strings\"\n\n\t\"github.com/dude333/rapina/reports\"\n\t\"github.com/lithammer/fuzzysearch/fuzzy\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/spf13/cobra\"\n)\n\n// Flags\nvar scriptMode bool\nvar all bool\nvar showShares bool\nvar extraRatios bool\nvar fleuriet bool\nvar omitSector bool\nvar outputDir = \"reports\"\nvar format string // output format of the report\n\n// reportCmd represents the report command\nvar reportCmd = &cobra.Command{\n\tUse: \"report [-s] nome_empresa\",\n\tShort: \"Cria planilha com dados da companhia escolhida\",\n\tLong: \"Cria planilha com dados da companhia escolhida\",\n\tArgs: cobra.MinimumNArgs(1),\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\treport(args[0])\n\t},\n}\n\nfunc init() {\n\trootCmd.AddCommand(reportCmd)\n\n\treportCmd.Flags().BoolVarP(&scriptMode, \"scriptMode\", \"s\", false, \"Para modo script (escolhe a empresa com nome mais próximo)\")\n\treportCmd.Flags().BoolVarP(&all, \"all\", \"a\", false, \"Mostra todos os indicadores\")\n\treportCmd.Flags().BoolVarP(&showShares, \"showShares\", \"f\", false, \"Mostra o número de ações e free float\")\n\treportCmd.Flags().BoolVarP(&extraRatios, \"extraRatios\", \"x\", false, \"Reporte de índices extras\")\n\treportCmd.Flags().BoolVarP(&fleuriet, \"fleuriet\", \"F\", false, \"Capital de giro no modelo Fleuriet\")\n\treportCmd.Flags().BoolVarP(&omitSector, \"omitSector\", \"o\", false, \"Omite o relatório das empresas do mesmo setor\")\n\treportCmd.Flags().StringVarP(&outputDir, \"outputDir\", \"d\", \"reports\", \"Diretório onde o relatório será salvo\")\n\treportCmd.Flags().StringVarP(&format, \"format\", \"r\", \"xlsx\", \"Formato do relatório: xlsx|stdout\")\n}\n\nfunc report(company string) {\n\tvar spcfctnCd string = \"ON\"\n\tcompany = SelectCompany(company, scriptMode)\n\tif company == \"\" {\n\t\tfmt.Println(\"[x] Empresa não encontrada\")\n\t\treturn\n\t}\n\tif strings.Contains(company, \"@#\") {\n\t\tcompanyWithTicker := strings.Split(company, \"@#\")\n\t\tcompany = companyWithTicker[0]\n\t\tspcfctnCd = companyWithTicker[1]\n\t}\n\tfmt.Println()\n\tfmt.Printf(\"[√] Criando relatório para %s ========\\n\", company)\n\n\tif all {\n\t\textraRatios = true\n\t\tshowShares = true\n\t\tfleuriet = true\n\t}\n\n\tr := make(map[string]bool)\n\tr[\"ExtraRatios\"] = extraRatios\n\tr[\"ShowShares\"] = showShares\n\tr[\"Fleuriet\"] = fleuriet\n\tr[\"PrintSector\"] = !omitSector\n\n\tparms := Parms{\n\t\tCompany: company,\n\t\tSpcfctnCd: spcfctnCd,\n\t\tFormat: format,\n\t\tOutputDir: outputDir,\n\t\tYamlFile: yamlFile,\n\t\tReports: r,\n\t}\n\terr := Report(parms)\n\tif err != nil {\n\t\tfmt.Println(\"[x]\", err)\n\t}\n}\n\n//\n// SelectCompany returns the company name compared to the names\n// stored in the DB\n//\nfunc SelectCompany(company string, scriptMode bool) string {\n\tdb, err := openDatabase()\n\tif err != nil {\n\t\tfmt.Println(\"[x]\", err)\n\t\treturn \"\"\n\t}\n\n\tcompanies, err := reports.ListCompanies(db)\n\tif err != nil {\n\t\tfmt.Println(\"[x]\", err)\n\t\treturn \"\"\n\t}\n\n\t// Do a fuzzy match on the company name against\n\t// all companies listed on the DB\n\tmatches := make([]string, 0, 10)\n\tfor _, c := range companies {\n\t\tif fuzzy.MatchNormalizedFold(company, c) {\n\t\t\tmatches = append(matches, c)\n\t\t}\n\t}\n\n\t// Script mode\n\tif len(matches) >= 1 && scriptMode {\n\t\trank := fuzzy.RankFindNormalizedFold(company, matches)\n\t\tif len(rank) <= 0 {\n\t\t\treturn \"\"\n\t\t}\n\t\tsort.Sort(rank)\n\t\treturn rank[0].Target\n\t}\n\n\t// Interactive menu\n\tif len(matches) >= 1 {\n\t\tresult := promptUser(matches, \"Selecione a Empresa\")\n\n\t\ttickers, err := reports.ListTickers(db, result)\n\t\tif err != nil {\n\t\t\tfmt.Println(\"[x] Recuperando lista de tickers \", err)\n\t\t\treturn result\n\t\t}\n\n\t\t// Interactive menu\n\t\tif len(tickers) > 0 {\n\t\t\tticker := promptUser(tickers, \"Selecione o ticker\")\n\t\t\tresultWithTicker := fmt.Sprintf(\"%s@#%s\", result, reports.GetSpcfctnCd(db, result, ticker))\n\t\t\treturn resultWithTicker\n\t\t}\n\n\t\treturn result\n\t}\n\n\treturn \"\"\n}\n\n//\n// Report a company from DB to Excel\n//\nfunc Report(p Parms) (err error) {\n\n\tdb, err := openDatabase()\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"fail to open db\")\n\t}\n\n\tif p.OutputDir == \"\" {\n\t\tp.OutputDir = outputDir\n\t}\n\n\tfile, err := filename(p.OutputDir, p.Company)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tparms := map[string]interface{}{\n\t\t\"db\": db,\n\t\t\"dataDir\": dataDir,\n\t\t\"company\": p.Company,\n\t\t\"SpcfctnCd\": p.SpcfctnCd,\n\t\t\"format\": p.Format,\n\t\t\"filename\": file,\n\t\t\"yamlFile\": p.YamlFile,\n\t\t\"reports\": p.Reports,\n\t}\n\n\tif p.Format == \"stdout\" {\n\t\treturn reports.ReportToStdout(parms)\n\t}\n\n\treturn reports.ReportToXlsx(parms)\n}\n" }, { "path": "cmd/rapina/server.go", "content": "/*\nCopyright © 2021 Adriano P \nDistributed under the MIT License.\n*/\npackage main\n\nimport (\n\t\"log\"\n\n\t\"github.com/dude333/rapina/server\"\n\t\"github.com/spf13/cobra\"\n\t\"github.com/spf13/viper\"\n)\n\ntype serverFlags struct {\n}\n\n// serverCmd represents the server command\nvar serverCmd = &cobra.Command{\n\tUse: \"server\",\n\tShort: \"Inicia o servidor web\",\n\tLong: `Comando para iniciar o servidor para a exibição dos dados via web browser.`,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tparms := make(map[string]string)\n\t\t// Verbose\n\t\tif flags.verbose {\n\t\t\tparms[Fverbose] = \"true\"\n\t\t}\n\n\t\terr := serve(parms)\n\t\tif err != nil {\n\t\t\tlog.Println(err)\n\t\t}\n\t},\n}\n\nfunc init() {\n\trootCmd.AddCommand(serverCmd)\n\t// serverCmd.Flags().IntVarP(&flags.server.num,\n\t// \tFnum, \"n\", 1, \"número de meses desde o último disponível\")\n}\n\nfunc serve(parms map[string]string) error {\n\n\tdb, err := openDatabase()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tv := parms[Fverbose] == \"true\"\n\n\tserver.HTML(\n\t\tserver.WithDB(db),\n\t\tserver.WithAPIKey(viper.GetString(\"apikey\")),\n\t\tserver.WithDataDir(dataDir),\n\t\tserver.Verbose(v))\n\n\treturn nil\n}\n" }, { "path": "cmd/rapina/update.go", "content": "// Copyright © 2018 Adriano P\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy\n// of this software and associated documentation files (the \"Software\"), to deal\n// in the Software without restriction, including without limitation the rights\n// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n// copies of the Software, and to permit persons to whom the Software is\n// furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in\n// all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n// THE SOFTWARE.\n\npackage main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/dude333/rapina\"\n\t\"github.com/dude333/rapina/fetch\"\n\t\"github.com/dude333/rapina/reports\"\n\t\"github.com/spf13/cobra\"\n\t\"github.com/spf13/viper\"\n)\n\nvar sectors bool\n\n// getUpdate represents the get command\nvar getUpdate = &cobra.Command{\n\tUse: \"update\",\n\tAliases: []string{\"get\"},\n\tShort: \"Baixa os arquivos da CVM e atualiza o bando de dados\",\n\tLong: `Baixa os arquivos do site da CVM, processa e os armazena no bando de dados.`,\n\tRun: func(cmd *cobra.Command, args []string) {\n\t\tdb, err := openDatabase()\n\t\tif err != nil {\n\t\t\tfmt.Println(\"[x]\", err)\n\t\t\treturn\n\t\t}\n\n\t\tfmt.Println(\"[√] Coletando dados ===========\")\n\t\terr = fetch.Sectors(yamlFile)\n\t\tif err != nil && !errors.Is(err, rapina.ErrFileNotUpdated) {\n\t\t\tfmt.Println(\"[x]\", err)\n\t\t\treturn\n\t\t}\n\t\tif err == nil {\n\t\t\tfmt.Println(\"[√] Arquivo salvo:\", yamlFile)\n\t\t}\n\t\t//\n\t\tfmt.Println()\n\t\t//\n\t\tif sectors { // skip if -s flag is selected (dowload only the sectors)\n\t\t\treturn\n\t\t}\n\n\t\terr = fetch.CVM(db, dataDir)\n\t\tif err != nil {\n\t\t\tfmt.Println(\"[x]\", err)\n\t\t\treturn\n\t\t}\n\n\t\t// Stock codes\n\t\tlog := reports.NewLogger(os.Stderr)\n\t\tstock, err := fetch.NewStock(db, log, viper.GetString(\"apikey\"), dataDir)\n\t\tif err != nil {\n\t\t\tlog.Error(err.Error())\n\t\t\treturn\n\t\t}\n\t\t_ = stock.UpdateStockCodes()\n\t},\n}\n\nfunc init() {\n\trootCmd.AddCommand(getUpdate)\n\n\tgetUpdate.Flags().BoolVarP(§ors, \"sectors\", \"s\", false, \"Baixa a classificação setorial das empresas e fundos negociados na B3\")\n}\n" }, { "path": "common.go", "content": "package rapina\n\nimport (\n\t\"fmt\"\n\t\"net/url\"\n\t\"path\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\n// IsDate checks if date is in format YYYY-MM-DD.\nfunc IsDate(date string) bool {\n\tif len(date) != len(\"2021-04-26\") || strings.Count(date, \"-\") != 2 {\n\t\treturn false\n\t}\n\n\ty, errY := strconv.Atoi(date[0:4])\n\tm, errM := strconv.Atoi(date[5:7])\n\td, errD := strconv.Atoi(date[8:10])\n\tif errY != nil || errM != nil || errD != nil {\n\t\treturn false\n\t}\n\n\t// Ok, we'll still be using this in 2200 :)\n\tif y < 1970 || y > 2200 {\n\t\treturn false\n\t}\n\tif m < 1 || m > 12 {\n\t\treturn false\n\t}\n\tnDays := [13]int{0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}\n\tif d < 1 || d > nDays[m] {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// IsURL returns true if 'str' is a valid URL.\nfunc IsURL(str string) bool {\n\tu, err := url.Parse(str)\n\treturn err == nil && u.Scheme != \"\" && u.Host != \"\"\n}\n\n// JoinURL joins strings as URL paths\nfunc JoinURL(base string, paths ...string) string {\n\tp := path.Join(paths...)\n\treturn fmt.Sprintf(\"%s/%s\", strings.TrimRight(base, \"/\"), strings.TrimLeft(p, \"/\"))\n}\n\nvar _timeNow = time.Now\n\n// MonthsFromToday returns a list of months including the current.\n// Date formatted as YYYY-MM.\nfunc MonthsFromToday(n int) []string {\n\tif n < 1 {\n\t\tn = 1\n\t}\n\tif n > 100 {\n\t\tn = 100\n\t}\n\n\tnow := _timeNow()\n\tnow = time.Date(now.Year(), now.Month(), 15, 12, 0, 0, 0, time.UTC)\n\n\tvar monthYears []string\n\tfor ; n > 0; n-- {\n\t\tmonthYears = append(monthYears, now.Format(\"2006-01\"))\n\t\tnow = now.AddDate(0, -1, 0)\n\t}\n\n\treturn monthYears\n}\n\n// LastBusinessDayOfYear returns the last business day of the 'year' (the business\n// day before Dec 30). If current year, returns last business day before today.\n// Returns date as YYYY-MM-DD.\nfunc LastBusinessDayOfYear(year int) string {\n\ttoday := time.Now()\n\tif year == today.Year() {\n\t\treturn LastBusinessDay(1)\n\t}\n\n\tdate := time.Date(year, time.December, 29, 12, 0, 0, 0, time.UTC)\n\n\tif date.Weekday() == time.Saturday {\n\t\tdate = date.AddDate(0, 0, -1)\n\t}\n\tif date.Weekday() == time.Sunday {\n\t\tdate = date.AddDate(0, 0, -2)\n\t}\n\n\treturn date.Format(\"2006-01-02\")\n}\n\n// LastBusinessDay returns the most recent business day 'n' days before today.\n// Returns date as YYYY-MM-DD.\nfunc LastBusinessDay(n int) string {\n\tdate := time.Now()\n\tif n > 0 {\n\t\tdate = date.AddDate(0, 0, -n)\n\t}\n\n\tif date.Weekday() == time.Saturday {\n\t\tdate = date.AddDate(0, 0, -1)\n\t}\n\tif date.Weekday() == time.Sunday {\n\t\tdate = date.AddDate(0, 0, -2)\n\t}\n\n\treturn date.Format(\"2006-01-02\")\n}\n" }, { "path": "common_test.go", "content": "package rapina\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestIsDate(t *testing.T) {\n\ttype args struct {\n\t\tdate string\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twant bool\n\t}{\n\t\t{\n\t\t\tname: \"should be true\",\n\t\t\targs: args{date: \"2021-04-26\"},\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"should be true too\",\n\t\t\targs: args{date: \"2030-12-31\"},\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"should be false\",\n\t\t\targs: args{date: \"2021-04-31\"},\n\t\t\twant: false,\n\t\t},\n\t\t{\n\t\t\tname: \"should be false too\",\n\t\t\targs: args{date: \"20/12/2000\"},\n\t\t\twant: false,\n\t\t},\n\t\t{\n\t\t\tname: \"should be false three\",\n\t\t\targs: args{date: \"2021-07-32\"},\n\t\t\twant: false,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif got := IsDate(tt.args.date); got != tt.want {\n\t\t\t\tt.Errorf(\"IsDate() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestIsUrl(t *testing.T) {\n\ttype args struct {\n\t\tstr string\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twant bool\n\t}{\n\t\t{\n\t\t\tname: \"should be true\",\n\t\t\targs: args{str: \"http://example.com/path\"},\n\t\t\twant: true,\n\t\t},\n\t\t{\n\t\t\tname: \"should be false\",\n\t\t\targs: args{str: \"example.com/path\"},\n\t\t\twant: false,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif got := IsURL(tt.args.str); got != tt.want {\n\t\t\t\tt.Errorf(\"IsUrl() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMonthsFromToday(t *testing.T) {\n\ttimeNow1 := func() time.Time {\n\t\treturn time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)\n\t}\n\ttimeNow2 := func() time.Time {\n\t\treturn time.Date(2009, time.March, 31, 23, 0, 0, 0, time.UTC)\n\t}\n\n\ttype args struct {\n\t\tn int\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\ttimeNow func() time.Time\n\t\twant []string\n\t}{\n\t\t{\n\t\t\tname: \"should show 3 months\",\n\t\t\targs: args{n: 3},\n\t\t\ttimeNow: timeNow1,\n\t\t\twant: []string{\"2009-11\", \"2009-10\", \"2009-09\"},\n\t\t},\n\t\t{\n\t\t\tname: \"should show 2 months\",\n\t\t\targs: args{n: 2},\n\t\t\ttimeNow: timeNow2,\n\t\t\twant: []string{\"2009-03\", \"2009-02\"},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_timeNow = tt.timeNow\n\t\t\tif got := MonthsFromToday(tt.args.n); !reflect.DeepEqual(got, tt.want) {\n\t\t\t\tt.Errorf(\"MonthsFromToday() = %#v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLastBusinessDayOfYear(t *testing.T) {\n\ttype args struct {\n\t\tyear int\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twant string\n\t}{\n\t\t{\n\t\t\tname: \"2022\",\n\t\t\targs: args{2022},\n\t\t\twant: \"2022-12-29\",\n\t\t},\n\t\t{\n\t\t\tname: \"2020\",\n\t\t\targs: args{2020},\n\t\t\twant: \"2020-12-29\",\n\t\t},\n\t\t{\n\t\t\tname: \"2017\",\n\t\t\targs: args{2017},\n\t\t\twant: \"2017-12-29\",\n\t\t},\n\t\t{\n\t\t\tname: \"2016\",\n\t\t\targs: args{2016},\n\t\t\twant: \"2016-12-29\",\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif got := LastBusinessDayOfYear(tt.args.year); got != tt.want {\n\t\t\t\tt.Errorf(\"LastBusinessDayOfYear() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "errors.go", "content": "package rapina\n\nimport \"errors\"\n\n// Error codes\nvar (\n\tErrRecordExists = errors.New(\"insert ignored, register already exists\")\n\tErrFileNotUpdated = errors.New(\"file not updated\")\n\tErrInvalidAPIKey = errors.New(\"apiKey inválida, configure uma chave em\" +\n\t\t\" https://www.alphavantage.co/support/#api-key e adicione no arquivo\" +\n\t\t\" config.yml\")\n\tErrInvalidDate = errors.New(\"invalid date format\")\n)\n" }, { "path": "fetch/fetch.go", "content": "// Copyright © 2018 Adriano P\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy\n// of this software and associated documentation files (the \"Software\"), to deal\n// in the Software without restriction, including without limitation the rights\n// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n// copies of the Software, and to permit persons to whom the Software is\n// furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in\n// all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\n// THE SOFTWARE.\n\npackage fetch\n\nimport (\n\t\"database/sql\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/dude333/rapina/parsers\"\n\t\"github.com/dustin/go-humanize\"\n\t_ \"github.com/mattn/go-sqlite3\" // requires CGO_ENABLED=1 and gcc\n\t\"github.com/pkg/errors\"\n)\n\nvar (\n\t// ErrFileNotFound error\n\tErrFileNotFound = errors.New(\"file not found\")\n\t// ErrItemNotFound for string not found on []string\n\tErrItemNotFound = errors.New(\"item not found\")\n)\n\n//\n// CVM fetches all statements from a range\n// of years\n//\nfunc CVM(db *sql.DB, dataDir string) error {\n\tnow := time.Now().Year()\n\ttry(processQuarterlyReport, db, dataDir, \"Arquivo ITR não encontrado\", now, now-1, 2)\n\ttry(processAnnualReport, db, dataDir, \"Arquivo DFP não encontrado\", now-1, 2010, 2)\n\ttry(processFREReport, db, dataDir, \"Arquivo FRE não encontrado\", now-1, 2010, 2)\n\n\treturn nil\n}\n\ntype fn func(*sql.DB, string, int) error\n\n// try to run the function 'f' 'n' times, in case there are network errors.\nfunc try(f fn, db *sql.DB, dataDir, errMsg string, now, limit, n int) {\n\ttries := n\n\tvar err error\n\n\tfor year := now; tries > 0 && year >= limit; year-- {\n\t\tfmt.Printf(\"[>] %d ---------------------\\n\", year)\n\t\terr = f(db, dataDir, year)\n\t\tif err == ErrFileNotFound {\n\t\t\tfmt.Printf(\"[x] %s\\n\", errMsg)\n\t\t\ttries--\n\t\t\tcontinue\n\t\t} else if err != nil {\n\t\t\tfmt.Printf(\"[x] Erro ao processar arquivo de %d: %v\\n\", year, err)\n\t\t\ttries--\n\t\t} else {\n\t\t\ttries = n\n\t\t}\n\t}\n}\n\n// processAnnualReport will get data from .zip files downloaded\n// directly from CVM and insert its data into the DB\nfunc processAnnualReport(db *sql.DB, dataDir string, year int) error {\n\n\turl := fmt.Sprintf(\"http://dados.cvm.gov.br/dados/CIA_ABERTA/DOC/DFP/DADOS/dfp_cia_aberta_%d.zip\", year)\n\tzipfile := fmt.Sprintf(\"%s/dfp_%d.zip\", dataDir, year)\n\n\t// Download files from CVM server\n\tfmt.Print(\"[ ] Download do arquivo DFP\")\n\tfiles, err := fetchFiles(url, dataDir, zipfile)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdataTypes := []string{\"BPA\", \"BPP\", \"DRE\", \"DFC_MD\", \"DFC_MI\", \"DVA\"}\n\n\tfor _, dt := range dataTypes {\n\t\tpattern := fmt.Sprintf(\"dfp_cia_aberta_%s_con_%d.csv\", dt, year)\n\t\treqFile, err := findFile(files, pattern)\n\t\tif err == ErrItemNotFound {\n\t\t\tfilesCleanup(files)\n\t\t\treturn fmt.Errorf(\"arquivo %s não encontrado\", reqFile)\n\t\t}\n\n\t\t// Import file into DB\n\t\tif err = parsers.ImportCsv(db, dt, reqFile); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tfilesCleanup(files) // remove remaining (unused) files\n\n\treturn nil\n}\n\n//\n// processQuarterlyReport download quarter files from CVM and store them on DB\n//\nfunc processQuarterlyReport(db *sql.DB, dataDir string, year int) error {\n\n\turl := fmt.Sprintf(\"http://dados.cvm.gov.br/dados/CIA_ABERTA/DOC/ITR/DADOS/ITR_CIA_ABERTA_%d.zip\", year)\n\tzipfile := fmt.Sprintf(\"%s/itr_%d.zip\", dataDir, year)\n\n\t// Download files from CVM server\n\tfmt.Print(\"[ ] Download do arquivo ITR\")\n\tfiles, err := fetchFiles(url, dataDir, zipfile)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdataTypes := []string{\"BPA\", \"BPP\", \"DRE\", \"DFC_MD\", \"DFC_MI\", \"DVA\"}\n\n\tfor _, dt := range dataTypes {\n\t\tpattern := fmt.Sprintf(\"ITR_CIA_ABERTA_%s_con_%d.csv\", dt, year)\n\t\treqFile, err := findFile(files, pattern)\n\t\tif err == ErrItemNotFound {\n\t\t\tfilesCleanup(files)\n\t\t\treturn fmt.Errorf(\"arquivo %s não encontrado\", reqFile)\n\t\t}\n\n\t\t// Import file into DB (the trick is to add ITR to the data type so the\n\t\t// ImportCSV loads that into the ITR table)\n\t\tif err = parsers.ImportCsv(db, dt+\"_ITR\", reqFile); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tfilesCleanup(files) // remove remaining (unused) files\n\n\treturn nil\n}\n\n//\n// processFREReport download FRE (Reference Form) files from CVM and store\n// them on DB.\n//\nfunc processFREReport(db *sql.DB, dataDir string, year int) error {\n\turl := fmt.Sprintf(\"http://dados.cvm.gov.br/dados/CIA_ABERTA/DOC/FRE/DADOS/fre_cia_aberta_%d.zip\", year)\n\tzipfile := fmt.Sprintf(\"%s/fre_%d.zip\", dataDir, year)\n\n\t// Download files from CVM server\n\tfmt.Print(\"[ ] Download do arquivo FRE\")\n\tfiles, err := fetchFiles(url, dataDir, zipfile)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tpatterns := []string{\"fre_cia_aberta_distribuicao_capital_%d.csv\"}\n\n\tfor _, p := range patterns {\n\t\tpattern := fmt.Sprintf(p, year)\n\t\treqFile, err := findFile(files, pattern)\n\t\tif err == ErrItemNotFound {\n\t\t\tfilesCleanup(files)\n\t\t\treturn fmt.Errorf(\"arquivo %s não encontrado\", reqFile)\n\t\t}\n\n\t\tif err = parsers.ImportCsv(db, \"FRE\", reqFile); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t}\n\n\tfilesCleanup(files) // remove remaining (unused) files\n\n\treturn nil\n}\n\n//\n// fetchFiles from web verbosely.\n//\nfunc fetchFiles(url, dataDir string, zipfile string) ([]string, error) {\n\treturn fetchFilesVerbosity(url, dataDir, zipfile, true)\n}\n\n//\n// fetchFilesVerbosity from web.\n//\nfunc fetchFilesVerbosity(url, dataDir string, zipfile string, verbose bool) ([]string, error) {\n\n\t// Download file from web\n\terr := downloadFile(url, zipfile, verbose)\n\tif verbose {\n\t\tfmt.Println()\n\t}\n\tif err != nil {\n\t\treturn nil, ErrFileNotFound\n\t}\n\n\t// Unzip and list files\n\tfiles, err := Unzip(zipfile, dataDir, verbose)\n\tos.Remove(zipfile)\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"could not unzip file\")\n\t}\n\n\treturn files, nil\n}\n\n// WriteCounter counts the number of bytes written the io.Writer.\n// source: https://golangcode.com/download-a-file-with-progress/\ntype WriteCounter struct {\n\tTotal uint64\n}\n\n// Write implements the io.Writer interface and will be passed to io.TeeReader().\nfunc (wc *WriteCounter) Write(p []byte) (int, error) {\n\tn := len(p)\n\twc.Total += uint64(n)\n\twc.printProgress()\n\treturn n, nil\n}\n\nfunc (wc WriteCounter) printProgress() {\n\tfmt.Printf(\"\\r[ %7s\", humanize.Bytes(wc.Total))\n}\n\n//\n// downloadFile source: https://stackoverflow.com/a/33853856/276311\n//\nfunc downloadFile(url, filepath string, verbose bool) (err error) {\n\t// Create dir if necessary\n\tbasepath := path.Dir(filepath)\n\tif err = os.MkdirAll(basepath, os.ModePerm); err != nil {\n\t\treturn err\n\t}\n\n\t// Create the file\n\tout, err := os.Create(filepath)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// https://www.joeshaw.org/dont-defer-close-on-writable-files/\n\tdefer func() {\n\t\tcerr := out.Close()\n\t\tif err == nil {\n\t\t\terr = cerr\n\t\t}\n\t}()\n\n\t// Get the data\n\tresp, err := http.Get(url)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer resp.Body.Close()\n\n\t// Check server response\n\tif resp.StatusCode != http.StatusOK {\n\t\treturn fmt.Errorf(\"bad status: %s\", resp.Status)\n\t}\n\n\t// Write the body to file\n\tcounter := io.Discard\n\tif verbose {\n\t\tcounter = &WriteCounter{}\n\t}\n\t_, err = io.Copy(out, io.TeeReader(resp.Body, counter))\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn\n}\n\n//\n// Sectors checks if the configuration file is already populated.\n// If 'force' is set or if the config is empty, it retrieves data from B3,\n// unzip and extract a spreadsheet containing a list of companies divided by\n// sector, subsector, and segment; then this info is set into the config file.\n//\nfunc Sectors(yamlFile string) (err error) {\n\terr = parsers.SectorsToYaml(yamlFile)\n\n\treturn\n}\n\n//\n// filesCleanup\n//\nfunc filesCleanup(files []string) {\n\t// Clean up\n\tfor _, f := range files {\n\t\tif err := os.Remove(f); err != nil {\n\t\t\tfmt.Println(\"could not delete file\", f)\n\t\t}\n\t}\n}\n\n//\n// findFile finds an item on list that matches pattern (case insensitive)\n//\nfunc findFile(list []string, pattern string) (string, error) {\n\n\tfor i := range list {\n\t\tf := filepath.Base(list[i])\n\t\tif strings.EqualFold(f, pattern) {\n\t\t\treturn list[i], nil\n\t\t}\n\t}\n\n\treturn \"\", ErrItemNotFound\n}\n" }, { "path": "fetch/fetch_fii.go", "content": "package fetch\n\n/*\n\tURL List:\n\n\tFundos.NET: where the report IDs are obtained.\n\t=> https://fnet.bmfbovespa.com.br/fnet/publico/pesquisarGerenciadorDocumentosCVM?paginaCertificados=false&tipoFundo=1\n\t=> GET\n\thttps://fnet.bmfbovespa.com.br/fnet/publico/pesquisarGerenciadorDocumentosDados?d=3&s=0&l=10&o[0][dataEntrega]=desc&tipoFundo=1&idCategoriaDocumento=14&idTipoDocumento=41&idEspecieDocumento=0&situacao=A&cnpj=28737771000185&dataInicial=01/02/2021&dataFinal=28/02/2021&_=1619467786288\n*/\n\nimport (\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"database/sql\"\n\t\"encoding/base64\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/dude333/rapina\"\n\t\"github.com/dude333/rapina/parsers\"\n\t\"github.com/dude333/rapina/progress\"\n\t\"github.com/gocolly/colly/v2\"\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/net/html\"\n)\n\nconst MAX_N = 100\n\n// FII holds the infrastructure data.\ntype FII struct {\n\tstorage rapina.FIIStorage\n}\n\n// NewFII creates a new instace of FII.\nfunc NewFII(db *sql.DB, log rapina.Logger) (*FII, error) {\n\tstorage, err := parsers.NewFII(db, log)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfii := &FII{\n\t\tstorage: storage,\n\t}\n\treturn fii, nil\n}\n\ntype id int\n\n// Report holds the result of all documents filtered by a criteria defined by a\n// http.Get on the B3 server.\ntype Report struct {\n\tData []docID `json:\"data\"`\n}\ntype docID struct {\n\tID id `json:\"id\"`\n\tDescription string `json:\"descricaoFundo\"`\n\tDocType string `json:\"tipoDocumento\"`\n\tStatus string `json:\"situacaoDocumento\"`\n}\n\n// Dividends gets the report IDs for one company ('cnpj') and then the\n// yeld montlhy report for 'n' months, starting from the latest released.\nfunc (fii FII) Dividends(code string, n int) (*[]rapina.Dividend, error) {\n\tdividends, months, err := fii.dividendsFromDB(code, n)\n\tif err == nil {\n\t\tif months >= n {\n\t\t\treturn dividends, err\n\t\t}\n\t}\n\n\tdividends, err = fii.dividendsFromServer(code, n)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfor _, d := range *dividends {\n\t\terr := fii.storage.SaveDividend(d) // Save dividends to DB\n\t\tif err != nil {\n\t\t\tprogress.ErrorMsg(\"Erro ao salvar dividendos no banco de dados: %s - %v\", err, d)\n\t\t}\n\t}\n\n\t// Load dividends from DB to filter results\n\tdividends, _, err = fii.dividendsFromDB(code, n)\n\treturn dividends, err\n}\n\nfunc (fii FII) dividendsFromDB(code string, n int) (*[]rapina.Dividend, int, error) {\n\tvar dividends []rapina.Dividend\n\tvar months int\n\tfor _, monthYear := range rapina.MonthsFromToday(n + 2) {\n\t\td, err := fii.storage.Dividends(code, monthYear)\n\t\tif err == nil { // ignore errors\n\t\t\tdividends = append(dividends, *d...)\n\t\t\tmonths++\n\t\t}\n\t\tif months == n {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif len(dividends) == 0 {\n\t\treturn nil, 0, errors.New(\"dividendos não encontrados\")\n\t}\n\n\treturn ÷nds, months, nil\n}\n\n// Dividends gets the report IDs for one company ('cnpj') and then the\n// yeld montlhy report for 'n' months, starting from the latest released.\n//\n// If the number of reports does not match n, it'll retry with a bigger n as\n// sometimes reports from follow-on offerings (FPO).\nfunc (fii *FII) dividendsFromServer(code string, n int) (*[]rapina.Dividend, error) {\n\tn = int(float64(n) * 1.25)\n\tif n > MAX_N {\n\t\tn = MAX_N\n\t}\n\n\tids, err := fii.reportIDs(repDividends, code, n)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tprogress.Debug(\"Report IDs: %v\", ids)\n\n\tprogress.Status(\"Relatórios de dividendos: %s\", code)\n\tdividends, err := fii.dividendReport(code, ids)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn dividends, nil\n}\n\n// dividendReport parses the dividend reports and returns their dividends.\nfunc (fii *FII) dividendReport(code string, ids []id) (*[]rapina.Dividend, error) {\n\tvar dividends []rapina.Dividend\n\n\t// HTTP client setup\n\tclient := &http.Client{\n\t\tTransport: &http.Transport{\n\t\t\tTLSClientConfig: &tls.Config{\n\t\t\t\tInsecureSkipVerify: true,\n\t\t\t},\n\t\t\tMaxIdleConnsPerHost: 10,\n\t\t},\n\t}\n\n\tfor _, id := range ids {\n\t\turl := fmt.Sprintf(\"https://fnet.bmfbovespa.com.br/fnet/publico/exibirDocumento?id=%d&cvm=true\", id)\n\t\tprogress.Debug(\"GET %s\", url)\n\n\t\t// Make HTTP request\n\t\treq, err := http.NewRequest(\"GET\", url, nil)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// Reuse the same client for subsequent requests\n\t\tresp, err := client.Do(req)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif resp.StatusCode != http.StatusOK {\n\t\t\tresp.Body.Close()\n\t\t\treturn nil, errors.Wrapf(err, \"unexpected status code: %d\", resp.StatusCode)\n\t\t}\n\n\t\t// Read response body\n\t\tbody, err := io.ReadAll(resp.Body)\n\t\tresp.Body.Close()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// Decode base64 encoded body\n\t\tdecodedBody, err := base64.StdEncoding.DecodeString(strings.Trim(string(body), `\"`))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tdoc, err := html.Parse(bytes.NewReader(decodedBody))\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"error parsing HTML: %s\")\n\t\t}\n\n\t\tvar data []string\n\t\tvar extractData func(*html.Node)\n\t\textractData = func(n *html.Node) {\n\t\t\tif n.Type == html.ElementNode && n.Data == \"td\" {\n\t\t\t\ttext := getTextContent(n)\n\t\t\t\tdata = append(data, text)\n\t\t\t}\n\t\t\tfor c := n.FirstChild; c != nil; c = c.NextSibling {\n\t\t\t\textractData(c)\n\t\t\t}\n\t\t}\n\t\textractData(doc)\n\n\t\t// Store dividend\n\t\tif d, ok := parseData(data); ok {\n\t\t\tdividends = append(dividends, d)\n\t\t}\n\t}\n\n\treturn ÷nds, nil\n}\n\nfunc parseData(data []string) (rapina.Dividend, bool) {\n\tdividend := rapina.Dividend{}\n\tfieldName := \"\"\n\tcount := 0\n\tfor _, str := range data {\n\t\tif fieldName == \"\" {\n\t\t\tif str != \"\" {\n\t\t\t\tfieldName = str\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif strings.Contains(fieldName, \"Código de negociação\") {\n\t\t\tdividend.Code = str\n\t\t\tcount++\n\t\t} else if strings.Contains(fieldName, \"Data-base\") {\n\t\t\tdividend.Date = fixDate(str)\n\t\t\tcount++\n\t\t} else if strings.Contains(fieldName, \"Data do pagamento\") {\n\t\t\tdividend.PaymentDate = fixDate(str)\n\t\t\tcount++\n\t\t} else if strings.Contains(fieldName, \"Valor do provento\") {\n\t\t\tdividend.Val = comma2dot(str)\n\t\t\tcount++\n\t\t}\n\t\tfieldName = \"\"\n\t}\n\n\treturn dividend, count == 4 // false if not all fields are filled\n}\n\nfunc comma2dot(val string) float64 {\n\ta := strings.ReplaceAll(val, \".\", \"\")\n\tb := strings.ReplaceAll(a, \",\", \".\")\n\tn, _ := strconv.ParseFloat(b, 64)\n\treturn n\n}\n\n// fixDate converts dates from DD/MM/YYYY to YYYY-MM-DD.\nfunc fixDate(date string) string {\n\tif len(date) != len(\"26/04/2021\") || strings.Count(date, \"/\") != 2 {\n\t\treturn date\n\t}\n\n\treturn date[6:10] + \"-\" + date[3:5] + \"-\" + date[0:2]\n}\n\nfunc getTextContent(n *html.Node) string {\n\ttextContent := \"\"\n\tif n == nil {\n\t\treturn \"\"\n\t}\n\tif n.Type == html.TextNode {\n\t\treturn n.Data\n\t}\n\tfor c := n.FirstChild; c != nil; c = c.NextSibling {\n\t\ttextContent += getTextContent(c)\n\t}\n\treturn strings.TrimSpace(textContent)\n}\n\nfunc (fii *FII) MonthlyReportIDs(code string, n int) ([]id, error) {\n\tids, err := fii.reportIDs(repMonthly, code, n)\n\tif err != nil {\n\t\treturn []id{}, err\n\t}\n\t_, err = fii.monthlyReport(code, ids)\n\tif err != nil {\n\t\treturn []id{}, err\n\t}\n\n\treturn ids, nil\n}\n\n// monthlyReport parses the FII monthly reports.\nfunc (fii *FII) monthlyReport(code string, ids []id) (*[]rapina.Monthly, error) {\n\tyeld := make(map[string]string, len(ids))\n\n\tc := colly.NewCollector()\n\tc.WithTransport(&http.Transport{\n\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t})\n\n\tc.OnRequest(func(r *colly.Request) {\n\t\tr.Headers.Set(\"Accept\", \"text/html\")\n\t})\n\n\tc.OnError(func(r *colly.Response, err error) {\n\t\tprogress.ErrorMsg(\"Request URL: %v failed with response: %v\\nError: %v\", r.Request.URL, string(r.Body), err)\n\t})\n\n\t// Handles the html report\n\tc.OnHTML(\"tr\", func(e *colly.HTMLElement) {\n\t\tvar fieldName string\n\t\te.ForEach(\"td\", func(_ int, el *colly.HTMLElement) {\n\t\t\tv := strings.Trim(el.Text, \" \\r\\n\")\n\t\t\tprogress.Debug(\"%q\", v)\n\t\t\tif v != \"\" {\n\t\t\t\tif fieldName == \"\" {\n\t\t\t\t\tif v[0] < '0' || v[0] > '9' { // Ignore fields starting with number\n\t\t\t\t\t\tfieldName = v\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tfmt.Printf(\"%-30s => %s\\n\", fieldName, v)\n\t\t\t\t\tyeld[fieldName] = v\n\t\t\t\t\tfieldName = \"\"\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t\tprogress.Status(\"----------------------\")\n\t})\n\n\t// Get the yeld monthly report given the list of 'report IDs' -- returns HTML\n\tmonthly := make([]rapina.Monthly, 0, len(ids))\n\tfor _, id := range ids {\n\t\tu := fmt.Sprintf(\"https://fnet.bmfbovespa.com.br/fnet/publico/exibirDocumento?id=%d&cvm=true\", id)\n\t\tprogress.Debug(u)\n\t\tif err := c.Visit(u); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\t// d, err := fii.storage.SaveDividend(yeld)\n\t\t// if err != nil {\n\t\t// \tfii.log.Error(\"%v\", err)\n\t\t// \tcontinue\n\t\t// }\n\t\t// // fmt.Println(\"from server\", d.Code, d.Date, d.Val)\n\t\t// if d.Code == code {\n\t\t// \tmonthly = append(monthly, *d)\n\t\t// }\n\t}\n\n\treturn &monthly, nil\n}\n\n// Details returns the FII Details from DB. If not found:\n// fetches from server, stores it in the DB and returns the Details.\nfunc (fii *FII) Details(fiiCode string) (*rapina.FIIDetails, error) {\n\tif len(fiiCode) != 4 && len(fiiCode) != 6 {\n\t\treturn nil, fmt.Errorf(\"wrong code '%s'\", fiiCode)\n\t}\n\n\tdetails, err := fii.storage.Details(fiiCode)\n\tif err == nil && details.DetailFund.CNPJ != \"\" {\n\t\treturn details, nil\n\t}\n\n\tprogress.Warning(\"Detalhes do %s não encontrado no bd. Consultando web...\", fiiCode)\n\n\t// Fetch from server if not found in the database\n\tdata := fmt.Sprintf(`{\"typeFund\":7,\"cnpj\":\"0\",\"identifierFund\":\"%s\"}`, fiiCode[0:4])\n\tenc := base64.URLEncoding.EncodeToString([]byte(data))\n\tfundDetailURL := rapina.JoinURL(\n\t\t`https://sistemaswebb3-listados.b3.com.br/fundsProxy/fundsCall/GetDetailFundSIG/`,\n\t\tenc,\n\t)\n\n\ttr := &http.Transport{\n\t\tDisableCompression: true,\n\t\tIdleConnTimeout: _http_timeout,\n\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t}\n\tclient := &http.Client{Transport: tr}\n\n\tresp, err := client.Get(fundDetailURL)\n\tif err != nil {\n\t\treturn details, err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode != 200 {\n\t\treturn details, fmt.Errorf(\"%s: %s\", resp.Status, fundDetailURL)\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn nil, errors.Wrapf(err, \"FII Details(%s): reading body\", fiiCode)\n\t}\n\n\terr = fii.storage.SaveDetails(body)\n\tif err != nil {\n\t\treturn details, errors.Wrap(err, \"armazenando detalhes do FII\")\n\t}\n\n\treturn fii.storage.Details(fiiCode)\n}\n\n// Report type\ntype repType int\n\nconst (\n\trepMonthly repType = iota + 1\n\trepDividends\n)\n\nfunc (fii *FII) reportIDs(rt repType, code string, n int) ([]id, error) {\n\tn = minmax(n, 1, MAX_N)\n\n\t// Parameters to list the report IDs for the last 'n' dividend reports\n\ttimestamp := strconv.FormatInt(int64(time.Now().UnixNano()/1e6), 10)\n\tnMonthAgo := time.Now()\n\tnMonthAgo = nMonthAgo.AddDate(0, -n, -nMonthAgo.Day()+1)\n\tdet, err := fii.Details(code)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcnpj := det.DetailFund.CNPJ\n\n\tvar idTipoDocumento, idCategoriaDocumento, d string\n\tif rt == repMonthly {\n\t\tidTipoDocumento = \"40\"\n\t\tidCategoriaDocumento = \"6\"\n\t\td = \"0\"\n\t} else if rt == repDividends {\n\t\tidTipoDocumento = \"41\"\n\t\tidCategoriaDocumento = \"14\"\n\t\td = \"2\"\n\t} else {\n\t\treturn []id{}, errors.New(\"invalid report type\")\n\t}\n\n\tv := url.Values{\n\t\t\"tipoFundo\": []string{\"1\"},\n\t\t\"cnpjFundo\": []string{cnpj},\n\t\t\"idTipoDocumento\": []string{idTipoDocumento},\n\t\t\"idCategoriaDocumento\": []string{idCategoriaDocumento},\n\t\t\"d\": []string{d},\n\t\t\"idEspecieDocumento\": []string{\"0\"},\n\t\t\"situacao\": []string{\"A\"},\n\t\t\"s\": []string{\"0\"},\n\t\t\"l\": []string{\"200\"}, // 'n*2' latest reports as other codes may appear (e.g.:ABCD11, ABCD12, ABCD13...)\n\t\t\"dataFinal\": []string{time.Now().Format(\"02/01/2006\")},\n\t\t\"dataInicial\": []string{nMonthAgo.Format(\"02/01/2006\")},\n\t\t\"o[0][dataReferencia]\": []string{\"asc\"},\n\t\t\"_\": []string{timestamp},\n\t}\n\n\t// Get the 'report IDs' for a given company (CNPJ) -- returns JSON\n\tvar report Report\n\tu := \"https://fnet.bmfbovespa.com.br/fnet/publico/pesquisarGerenciadorDocumentosDados?\" +\n\t\tv.Encode()\n\tprogress.Debug(\"* Report IDs: %s\", u)\n\tif err := getJSON(u, &report); err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar ids []id\n\tfor _, d := range report.Data {\n\t\tif d.Status == \"A\" {\n\t\t\tids = append(ids, d.ID)\n\t\t}\n\t}\n\n\treturn ids, nil\n}\n\n// minmax returns n limited to [min, max]\nfunc minmax(n, min, max int) int {\n\tif n < min {\n\t\tn = min\n\t}\n\tif n > max {\n\t\tn = MAX_N\n\t}\n\treturn n\n}\n" }, { "path": "fetch/fetch_fii_test.go", "content": "package fetch\n\nimport (\n\t\"testing\"\n)\n\nfunc Test_comma2dot(t *testing.T) {\n\ttype args struct {\n\t\tval string\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twant float64\n\t}{\n\t\t{\n\t\t\tname: \"should work\",\n\t\t\targs: args{val: \"1.230,56\"},\n\t\t\twant: 1230.56,\n\t\t},\n\t\t{\n\t\t\tname: \"should return 0\",\n\t\t\targs: args{val: \"shouldbeanum\"},\n\t\t\twant: 0,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif got := comma2dot(tt.args.val); got != tt.want {\n\t\t\t\tt.Errorf(\"comma2dot() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc Test_FixDate(t *testing.T) {\n\ttype args struct {\n\t\tdate string\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twant string\n\t}{\n\t\t{\n\t\t\tname: \"should work\",\n\t\t\targs: args{date: \"01/02/2021\"},\n\t\t\twant: \"2021-02-01\",\n\t\t},\n\t\t{\n\t\t\tname: \"should return the input\",\n\t\t\targs: args{date: \"wrong/date\"},\n\t\t\twant: \"wrong/date\",\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif got := fixDate(tt.args.date); got != tt.want {\n\t\t\t\tt.Errorf(\"fixDate() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "fetch/fetch_http.go", "content": "package fetch\n\nimport (\n\t\"crypto/tls\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/dude333/rapina/progress\"\n)\n\nconst _http_timeout = 30 * time.Second\n\n// HTTPFetch implements a generic HTTP fetcher.\ntype HTTPFetch struct {\n\tclient *http.Client\n}\n\n// NewHTTP creates a new HTTPFetch instance.\nfunc NewHTTP() *HTTPFetch {\n\tc := &http.Client{Timeout: _http_timeout}\n\treturn &HTTPFetch{client: c}\n}\n\n// JSON handles json responses.\nfunc (h HTTPFetch) JSON(url string, target interface{}) error {\n\tr, err := h.client.Get(url)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer r.Body.Close()\n\n\t// for _, c := range r.Cookies() {\n\t// \tfmt.Printf(\"COOKIE: %+v\\n\", c)\n\t// }\n\n\treturn json.NewDecoder(r.Body).Decode(target)\n}\n\nfunc getJSON(url string, target interface{}) error {\n\tc := &http.Client{\n\t\tTimeout: _http_timeout,\n\t\tTransport: &http.Transport{\n\t\t\tDisableCompression: true,\n\t\t\tIdleConnTimeout: _http_timeout,\n\t\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t\t},\n\t}\n\n\tr, err := c.Get(url)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif r.StatusCode < 200 || r.StatusCode >= 300 {\n\t\treturn fmt.Errorf(\"unexpected status code: %d\", r.StatusCode)\n\t}\n\n\tdefer func() {\n\t\tif err := r.Body.Close(); err != nil {\n\t\t\tprogress.ErrorMsg(\"Failed to close response body: %v\", err)\n\t\t}\n\t}()\n\n\treturn json.NewDecoder(r.Body).Decode(target)\n}\n" }, { "path": "fetch/fetch_http_test.go", "content": "package fetch\n\nimport (\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar ts *httptest.Server\n\nfunc init() {\n\thandler := http.NewServeMux()\n\thandler.HandleFunc(\"/server/api/v1/json\", jsonsMock)\n\n\tts = httptest.NewServer(handler)\n}\n\nfunc jsonsMock(w http.ResponseWriter, r *http.Request) {\n\t_, _ = w.Write([]byte(`{\"text\": \"mock\"}`))\n}\n\ntype jsonData struct {\n\tText string `json:\"text\"`\n}\n\nfunc TestHTTPFetch_JSON(t *testing.T) {\n\th := NewHTTP()\n\n\tvar got jsonData\n\n\terr := h.JSON(ts.URL+\"/server/api/v1/json\", &got)\n\n\tassert.Equal(t, jsonData{Text: \"mock\"}, got)\n\tassert.Nil(t, err)\n}\n" }, { "path": "fetch/fetch_stock.go", "content": "package fetch\n\nimport (\n\t\"crypto/tls\"\n\t\"database/sql\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/dude333/rapina\"\n\t\"github.com/dude333/rapina/parsers\"\n\t\"github.com/dude333/rapina/progress\"\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/text/encoding/charmap\"\n\t\"golang.org/x/text/transform\"\n)\n\n// API providers\nconst (\n\tAPInone = iota\n\tAPIalphavantage\n\tAPIyahoo\n)\n\n// Stock implements a fetcher for stock info.\ntype Stock struct {\n\tapiKey string // API key for Alpha Vantage API server\n\tstore rapina.StockStorage\n\tcache map[string]int // Cache to avoid duplicated fetch on Alpha Vantage server\n\tdataDir string // working directory where files will be stored to be parsed\n\tlog rapina.Logger\n}\n\n//\n// NewStock returns a new instance of *Stock\n//\nfunc NewStock(db *sql.DB, log rapina.Logger, apiKey, dataDir string) (*Stock, error) {\n\tstore, err := parsers.NewStock(db, log)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &Stock{\n\t\tapiKey: apiKey,\n\t\tstore: store,\n\t\tcache: make(map[string]int),\n\t\tdataDir: dataDir,\n\t\tlog: log,\n\t}, nil\n}\n\n// Quote returns the quote for 'code' on 'date'.\n// Date format: YYYY-MM-DD.\nfunc (s *Stock) Quote(code, date string) (float64, error) {\n\tif len(code) < len(\"CODE3\") {\n\t\treturn 0, fmt.Errorf(\"código inválido: %q\", code)\n\t}\n\tif !rapina.IsDate(date) {\n\t\treturn 0, fmt.Errorf(\"data inválida: %q\", date)\n\t}\n\n\tval, err := s.store.Quote(code, date)\n\tif err == nil {\n\t\treturn val, nil // returning data found on db\n\t}\n\n\t// Load quotes from B3\n\tif err := s.stockQuoteFromB3(date); ifNot(err) {\n\t\tif val, err = s.store.Quote(code, date); ifNot(err) {\n\t\t\treturn val, nil // returning data found on B3\n\t\t}\n\t}\n\n\t// Fallback to Yahoo Finance if not found on B3\n\tif err := s.stockQuoteFromAPIServer(code, date, APIyahoo); ifNot(err) {\n\t\tif val, err = s.store.Quote(code, date); ifNot(err) {\n\t\t\treturn val, nil // returning data found on Yahoo\n\t\t}\n\t}\n\n\terrNoProvider := errors.New(\"cotação não encontrada em nenhum provedor (B3, Yahoo e Alpha Vantage)\")\n\tif s.apiKey == \"\" {\n\t\terrNoProvider = errors.New(\"cotação não encontrada em nenhum provedor (B3 e Yahoo)\")\n\t}\n\n\t// Fallback to Alpha Vantage if not found on B3 and Yahoo\n\tif s.apiKey == \"\" {\n\t\treturn 0, errNoProvider\n\t}\n\tif err := s.stockQuoteFromAPIServer(code, date, APIalphavantage); err != nil {\n\t\treturn 0, errNoProvider\n\t}\n\t// Last try: return quote loaded by Alpha Vantage\n\tval, err = s.store.Quote(code, date)\n\tif err != nil {\n\t\treturn 0, errNoProvider\n\t}\n\n\treturn val, nil\n}\n\n//\n// stockQuoteFromB3 downloads the quotes for all companies for the given date,\n// where 'date' format is YYYY-MM-DD.\n//\nfunc (s *Stock) stockQuoteFromB3(date string) error {\n\t// Convert date string from YYYY-MM-DD to DDMMYYYY\n\tif len(date) != len(\"2021-05-03\") {\n\t\treturn fmt.Errorf(\"data com formato inválido: %s\", date)\n\t}\n\tconv := date[8:10] + date[5:7] + date[0:4]\n\turl := fmt.Sprintf(`http://bvmf.bmfbovespa.com.br/InstDados/SerHist/COTAHIST_D%s.ZIP`,\n\t\tconv)\n\t// Download ZIP file and unzips its files\n\tzip := fmt.Sprintf(\"%s/COTAHIST_D%s.ZIP\", s.dataDir, conv)\n\tfiles, err := fetchFilesVerbosity(url, s.dataDir, zip, false)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Delete files on return\n\tdefer filesCleanup(files)\n\n\t// Parse and store files content\n\tfor _, f := range files {\n\t\tfh, err := os.Open(f)\n\t\tif err != nil {\n\t\t\treturn errors.Wrapf(err, \"abrindo arquivo %s\", f)\n\t\t}\n\t\tdefer fh.Close()\n\n\t\tdec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder())\n\t\tif _, err := s.store.Save(dec, \"\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n//\n// stockQuoteFromAPIServer fetches the daily time series (date, daily open, daily high,\n// daily low, daily close, daily volume) of the global equity specified,\n// covering 20+ years of historical data.\n//\nfunc (s *Stock) stockQuoteFromAPIServer(code, date string, apiProvider int) error {\n\tif v := s.cache[code]; v == APIalphavantage && apiProvider == APIalphavantage {\n\t\treturn nil // silent return if this fetch has been run already\n\t}\n\n\t// Download quote for 'code'\n\ttr := &http.Transport{\n\t\tDisableCompression: true,\n\t\tIdleConnTimeout: _http_timeout,\n\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t}\n\tclient := &http.Client{Transport: tr}\n\tu := apiURL(apiProvider, s.apiKey, code, date)\n\tif u == \"\" {\n\t\treturn errors.New(\"URL do API server\")\n\t}\n\tresp, err := client.Get(u)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer resp.Body.Close()\n\n\tif resp.StatusCode != http.StatusOK {\n\t\treturn fmt.Errorf(\"%s\", resp.Status)\n\t}\n\n\ts.cache[code] = apiProvider // mark map to avoid unnecessary downloads\n\n\t// JSON means error response\n\tif resp.Header.Get(\"Content-Type\") == \"application/json\" {\n\t\tjsonMap := make(map[string]interface{})\n\t\terr := json.NewDecoder(resp.Body).Decode(&jsonMap)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn errors.New(map2str(jsonMap))\n\t}\n\n\tprogress.Running(\"Armazendo cotações no banco de dados...\")\n\t_, err = s.store.Save(resp.Body, code)\n\tif err != nil {\n\t\tprogress.RunFail()\n\t\treturn errors.Wrapf(err, \"armazenando cotações de %s\", code)\n\t}\n\tprogress.RunOK()\n\n\treturn err\n}\n\nfunc (s *Stock) Code(companyName, stockType string) (string, error) {\n\tif val, err := s.store.Code(companyName, stockType); err == nil {\n\t\treturn val, nil // returning data found on db\n\t}\n\n\tif err := s.UpdateStockCodes(); err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn s.store.Code(companyName, stockType)\n}\n\ntype b3CodesFile struct {\n\tRedirectURL string `json:\"redirectUrl\"`\n\tToken string `json:\"token\"`\n\tFile struct {\n\t\tName string `json:\"name\"`\n\t\tExtension string `json:\"extension\"`\n\t} `json:\"file\"`\n}\n\n//\n// UpdateStockCodes get the most recent file from B3.com.br with the stock trading code and\n// saves them on the storage.\n//\nfunc (s *Stock) UpdateStockCodes() error {\n\t// Get file url\n\tvar f b3CodesFile\n\turl := `https://arquivos.b3.com.br/api/download/requestname?fileName=InstrumentsConsolidated&date=`\n\turl += rapina.LastBusinessDay(2)\n\th := NewHTTP()\n\terr := h.JSON(url, &f)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Download file\n\tfp := fmt.Sprintf(\"%s/codes.csv\", s.dataDir)\n\ttries := 3\n\tfor {\n\t\turl = fmt.Sprintf(`https://arquivos.b3.com.br/api/download/?token=%s`, f.Token)\n\t\tprogress.Download(\"Download do arquivo de códigos\")\n\t\terr = downloadFile(url, fp, false)\n\t\tif err != nil {\n\t\t\ttries--\n\t\t\tif tries <= 0 {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ttime.Sleep(2 * time.Second)\n\t\t\tcontinue\n\t\t}\n\t\t// Delete files on return\n\t\tdefer filesCleanup([]string{fp})\n\t\tbreak\n\t}\n\n\t// Parse and store files content\n\tfh, err := os.Open(fp)\n\tif err != nil {\n\t\treturn errors.Wrapf(err, \"abrindo arquivo %s\", fp)\n\t}\n\tdefer fh.Close()\n\n\tdec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder())\n\t_, err = s.store.Save(dec, \"\")\n\n\treturn err\n}\n\n/* --- UTILS --- */\n\nfunc apiURL(provider int, apiKey, code, date string) string {\n\tv := url.Values{}\n\tswitch provider {\n\tcase APIalphavantage:\n\t\tv.Set(\"function\", \"TIME_SERIES_DAILY\")\n\t\tv.Add(\"symbol\", code+\".SA\")\n\t\tv.Add(\"apikey\", apiKey)\n\t\tv.Add(\"outputsize\", \"full\")\n\t\tv.Add(\"datatype\", \"csv\")\n\t\treturn \"https://www.alphavantage.co/query?\" + v.Encode()\n\n\tcase APIyahoo:\n\t\tconst layout = \"2006-01-02 15:04:05 -0700 MST\"\n\t\tt1, err1 := time.Parse(layout, date+\" 00:00:00 -0300 GMT\")\n\t\tt2, err2 := time.Parse(layout, date+\" 23:59:59 -0300 GMT\")\n\t\tif err1 != nil || err2 != nil {\n\t\t\treturn \"\"\n\t\t}\n\t\tv.Set(\"period1\", fmt.Sprint(t1.Unix()))\n\t\tv.Add(\"period2\", fmt.Sprint(t2.Unix()))\n\t\tv.Add(\"interval\", \"1d\")\n\t\tv.Add(\"events\", \"history\")\n\t\tv.Add(\"includeAdjustedClose\", \"true\")\n\t\treturn fmt.Sprintf(\"https://query1.finance.yahoo.com/v7/finance/download/%s.SA?%s\",\n\t\t\tcode, v.Encode())\n\t}\n\n\treturn \"\"\n}\n\nfunc map2str(data map[string]interface{}) string {\n\tvar buf string\n\tfor k, v := range data {\n\t\tbuf += fmt.Sprintln(k+\":\", v)\n\t}\n\treturn buf\n}\n\n// ifNot returns true if no error is found.\nfunc ifNot(err error) bool {\n\treturn err == nil\n}\n" }, { "path": "fetch/fetch_test.go", "content": "package fetch\n\nimport (\n\t\"testing\"\n\n\t_ \"github.com/mattn/go-sqlite3\"\n)\n\nfunc Test_findFile(t *testing.T) {\n\ttype args struct {\n\t\tlist []string\n\t\tpattern string\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twant string\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"should find item\",\n\t\t\targs: args{[]string{\"aaa\", \"aaa bbb CCC ddd\"}, \"aaa bbb CCC ddd\"},\n\t\t\twant: \"aaa bbb CCC ddd\",\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"should not find item\",\n\t\t\targs: args{[]string{\"aaa\", \"aaa bbb CCC ddd\"}, \"aaa bbb xCC ddd\"},\n\t\t\twant: \"\",\n\t\t\twantErr: true,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tgot, err := findFile(tt.args.list, tt.args.pattern)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"findFiles() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif got != tt.want {\n\t\t\t\tt.Errorf(\"findFiles() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "fetch/unzip.go", "content": "package fetch\n\nimport (\n\t\"archive/zip\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n)\n\n//\n// UnzipVerbosity will decompress a zip archive, moving all files and folders\n// within the zip file (parameter 1) to an output directory (parameter 2).\n// Source: https://golangcode.com/unzip-files-in-go/\n//\nfunc Unzip(src string, dest string, verbose bool) ([]string, error) {\n\n\tvar filenames []string\n\n\tr, err := zip.OpenReader(src)\n\tif err != nil {\n\t\treturn filenames, err\n\t}\n\tdefer r.Close()\n\n\tfor _, f := range r.File {\n\n\t\tif !valid(f.Name) {\n\t\t\tcontinue\n\t\t}\n\n\t\trc, err := f.Open()\n\t\tif err != nil {\n\t\t\treturn filenames, err\n\t\t}\n\t\tdefer rc.Close()\n\n\t\t// Store filename/path for returning and using later on\n\t\tfpath := filepath.Join(dest, f.Name)\n\n\t\t// Check for ZipSlip. More Info: http://bit.ly/2MsjAWE\n\t\tif !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) {\n\t\t\treturn filenames, fmt.Errorf(\"%s: illegal file path\", fpath)\n\t\t}\n\n\t\tfilenames = append(filenames, fpath)\n\n\t\tif f.FileInfo().IsDir() {\n\n\t\t\t// Make Folder\n\t\t\tif err = os.MkdirAll(fpath, os.ModePerm); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t} else {\n\n\t\t\t// Make File\n\t\t\tif err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {\n\t\t\t\treturn filenames, err\n\t\t\t}\n\n\t\t\toutFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())\n\t\t\tif err != nil {\n\t\t\t\treturn filenames, err\n\t\t\t}\n\n\t\t\tcounter := io.Discard\n\t\t\tif verbose {\n\t\t\t\tfmt.Printf(\"[ ] Unziping %s\", fpath)\n\t\t\t\tcounter = &WriteCounter{}\n\t\t\t}\n\t\t\t_, err = io.Copy(outFile, io.TeeReader(rc, counter))\n\t\t\tif verbose {\n\t\t\t\tfmt.Println()\n\t\t\t}\n\n\t\t\t// Close the file without defer to close before next iteration of loop\n\t\t\toutFile.Close()\n\n\t\t\tif err != nil {\n\t\t\t\treturn filenames, err\n\t\t\t}\n\n\t\t}\n\t}\n\treturn filenames, nil\n}\n\nfunc valid(filename string) bool {\n\tn := strings.ToLower(filename)\n\n\tif strings.Contains(n, \"_ind_\") {\n\t\treturn false\n\t}\n\n\tlist := []string{\"_bpa_\", \"_bpp_\", \"_dfc_\", \"_dre_\", \"_dva_\", \"fre_\", \"cotahist_\"}\n\n\tfor _, item := range list {\n\t\tif strings.Contains(n, item) {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n" }, { "path": "fii.go", "content": "package rapina\n\n// Dividend contains the stock 'Code', and the 'Date' for the stock dividend 'Val'.\ntype Dividend struct {\n\tCode string\n\tDate string\n\tPaymentDate string\n\tVal float64\n}\n\n// Monthly contains the FII monthly report fields\ntype Monthly struct {\n}\n\n// FIIDetails details (ID field: DetailFund.CNPJ)\ntype FIIDetails struct {\n\tDetailFund struct {\n\t\tAcronym string `json:\"acronym\"`\n\t\tTradingName string `json:\"tradingName\"`\n\t\tTradingCode string `json:\"tradingCode\"`\n\t\tTradingCodeOthers string `json:\"tradingCodeOthers\"`\n\t\tCNPJ string `json:\"cnpj\"`\n\t\tClassification string `json:\"classification\"`\n\t\tWebSite string `json:\"webSite\"`\n\t\tFundAddress string `json:\"fundAddress\"`\n\t\tFundPhoneNumberDDD string `json:\"fundPhoneNumberDDD\"`\n\t\tFundPhoneNumber string `json:\"fundPhoneNumber\"`\n\t\tFundPhoneNumberFax string `json:\"fundPhoneNumberFax\"`\n\t\tPositionManager string `json:\"positionManager\"`\n\t\tManagerName string `json:\"managerName\"`\n\t\tCompanyAddress string `json:\"companyAddress\"`\n\t\tCompanyPhoneNumberDDD string `json:\"companyPhoneNumberDDD\"`\n\t\tCompanyPhoneNumber string `json:\"companyPhoneNumber\"`\n\t\tCompanyPhoneNumberFax string `json:\"companyPhoneNumberFax\"`\n\t\tCompanyEmail string `json:\"companyEmail\"`\n\t\tCompanyName string `json:\"companyName\"`\n\t\tQuotaCount string `json:\"quotaCount\"`\n\t\tQuotaDateApproved string `json:\"quotaDateApproved\"`\n\t\tCodes []string `json:\"codes\"`\n\t\tCodesOther interface{} `json:\"codesOther\"`\n\t\tSegment interface{} `json:\"segment\"`\n\t} `json:\"detailFund\"`\n\tShareHolder struct {\n\t\tShareHolderName string `json:\"shareHolderName\"`\n\t\tShareHolderAddress string `json:\"shareHolderAddress\"`\n\t\tShareHolderPhoneNumberDDD string `json:\"shareHolderPhoneNumberDDD\"`\n\t\tShareHolderPhoneNumber string `json:\"shareHolderPhoneNumber\"`\n\t\tShareHolderFaxNumber string `json:\"shareHolderFaxNumber\"`\n\t\tShareHolderEmail string `json:\"shareHolderEmail\"`\n\t} `json:\"shareHolder\"`\n}\n\n// FIIStorage is the interface that contains the methods needed to parse, save and\n// retrieve FII data to/from a storage.\ntype FIIStorage interface {\n\tDetails(code string) (*FIIDetails, error)\n\tSaveDetails(stream []byte) error\n\n\tDividends(code, monthYear string) (*[]Dividend, error)\n\tSaveDividend(dividend Dividend) error\n}\n" }, { "path": "go.mod", "content": "module github.com/dude333/rapina\n\nrequire (\n\tgithub.com/360EntSecGroup-Skylar/excelize v1.4.1\n\tgithub.com/PuerkitoBio/goquery v1.8.1\n\tgithub.com/andybalholm/cascadia v1.3.2 // indirect\n\tgithub.com/antchfx/htmlquery v1.3.0 // indirect\n\tgithub.com/antchfx/xmlquery v1.3.18 // indirect\n\tgithub.com/antchfx/xpath v1.2.5 // indirect\n\tgithub.com/dustin/go-humanize v1.0.0\n\tgithub.com/gocolly/colly/v2 v2.1.0\n\tgithub.com/golang/protobuf v1.5.3 // indirect\n\tgithub.com/lithammer/fuzzysearch v1.1.0\n\tgithub.com/manifoldco/promptui v0.6.0\n\tgithub.com/mattn/go-sqlite3 v2.0.1+incompatible\n\tgithub.com/mitchellh/go-homedir v1.1.0\n\tgithub.com/pkg/errors v0.8.1\n\tgithub.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect\n\tgithub.com/spf13/cobra v0.0.5\n\tgithub.com/spf13/viper v1.6.1\n\tgithub.com/stretchr/testify v1.4.0\n\tgithub.com/temoto/robotstxt v1.1.2 // indirect\n\tgolang.org/x/net v0.20.0\n\tgolang.org/x/text v0.14.0\n\tgolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect\n\tgoogle.golang.org/appengine v1.6.8 // indirect\n\tgoogle.golang.org/protobuf v1.32.0 // indirect\n\tgopkg.in/yaml.v2 v2.4.0\n)\n\ngo 1.16\n" }, { "path": "go.sum", "content": "cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ngithub.com/360EntSecGroup-Skylar/excelize v1.4.1 h1:l55mJb6rkkaUzOpSsgEeKYtS6/0gHwBYyfo5Jcjv/Ks=\ngithub.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE=\ngithub.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=\ngithub.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=\ngithub.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=\ngithub.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=\ngithub.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=\ngithub.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=\ngithub.com/alecthomas/gometalinter v3.0.0+incompatible h1:e9Zfvfytsw/e6Kd/PYd75wggK+/kX5Xn8IYDUKyc5fU=\ngithub.com/alecthomas/gometalinter v3.0.0+incompatible/go.mod h1:qfIpQGGz3d+NmgyPBqv+LSh50emm1pt72EtcX2vKYQk=\ngithub.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=\ngithub.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=\ngithub.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=\ngithub.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=\ngithub.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=\ngithub.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=\ngithub.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=\ngithub.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=\ngithub.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E=\ngithub.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8=\ngithub.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM=\ngithub.com/antchfx/xmlquery v1.3.18 h1:FSQ3wMuphnPPGJOFhvc+cRQ2CT/rUj4cyQXkJcjOwz0=\ngithub.com/antchfx/xmlquery v1.3.18/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA=\ngithub.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=\ngithub.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=\ngithub.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=\ngithub.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=\ngithub.com/antchfx/xpath v1.2.5 h1:hqZ+wtQ+KIOV/S3bGZcIhpgYC26um2bZYP2KVGcR7VY=\ngithub.com/antchfx/xpath v1.2.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=\ngithub.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=\ngithub.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=\ngithub.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=\ngithub.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=\ngithub.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=\ngithub.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=\ngithub.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=\ngithub.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=\ngithub.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=\ngithub.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=\ngithub.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=\ngithub.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=\ngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=\ngithub.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=\ngithub.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=\ngithub.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=\ngithub.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=\ngithub.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=\ngithub.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=\ngithub.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=\ngithub.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=\ngithub.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=\ngithub.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=\ngithub.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=\ngithub.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=\ngithub.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=\ngithub.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=\ngithub.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=\ngithub.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=\ngithub.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=\ngithub.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=\ngithub.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=\ngithub.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=\ngithub.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=\ngithub.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs=\ngithub.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0=\ngithub.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=\ngithub.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=\ngithub.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=\ngithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=\ngithub.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=\ngithub.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=\ngithub.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=\ngithub.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=\ngithub.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=\ngithub.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=\ngithub.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=\ngithub.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=\ngithub.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=\ngithub.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg=\ngithub.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=\ngithub.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=\ngithub.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=\ngithub.com/gordonklaus/ineffassign v0.0.0-20180909121442-1003c8bd00dc h1:cJlkeAx1QYgO5N80aF5xRGstVsRQwgLR7uA2FnP1ZjY=\ngithub.com/gordonklaus/ineffassign v0.0.0-20180909121442-1003c8bd00dc/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU=\ngithub.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=\ngithub.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=\ngithub.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=\ngithub.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=\ngithub.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=\ngithub.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=\ngithub.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=\ngithub.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=\ngithub.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=\ngithub.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=\ngithub.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=\ngithub.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=\ngithub.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU=\ngithub.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU=\ngithub.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=\ngithub.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=\ngithub.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=\ngithub.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=\ngithub.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=\ngithub.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=\ngithub.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/lithammer/fuzzysearch v1.1.0 h1:go9v8tLCrNTTlH42OAaq4eHFe81TDHEnlrMEb6R4f+A=\ngithub.com/lithammer/fuzzysearch v1.1.0/go.mod h1:Bqx4wo8lTOFcJr3ckpY6HA9lEIOO0H5HrkJ5CsN56HQ=\ngithub.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw=\ngithub.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=\ngithub.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=\ngithub.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=\ngithub.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=\ngithub.com/manifoldco/promptui v0.6.0 h1:GuXmIdl5lhlamnWf3NbsKWYlaWyHABeStbD1LLsQMuA=\ngithub.com/manifoldco/promptui v0.6.0/go.mod h1:o9/C5VV8IPXxjxpl9au84MtQGIi5dwn7eldAgEdePPs=\ngithub.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=\ngithub.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=\ngithub.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=\ngithub.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=\ngithub.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=\ngithub.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=\ngithub.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=\ngithub.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=\ngithub.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=\ngithub.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=\ngithub.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=\ngithub.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=\ngithub.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/nicksnyder/go-i18n v1.10.1 h1:isfg77E/aCD7+0lD/D00ebR2MV5vgeQ276WYyDaCRQc=\ngithub.com/nicksnyder/go-i18n v1.10.1/go.mod h1:e4Di5xjP9oTVrC6y3C7C0HoSYXjSbhh/dU0eUV32nB4=\ngithub.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=\ngithub.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=\ngithub.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=\ngithub.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=\ngithub.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=\ngithub.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=\ngithub.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=\ngithub.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=\ngithub.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=\ngithub.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=\ngithub.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=\ngithub.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=\ngithub.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=\ngithub.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=\ngithub.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=\ngithub.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=\ngithub.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=\ngithub.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=\ngithub.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=\ngithub.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=\ngithub.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=\ngithub.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=\ngithub.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=\ngithub.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=\ngithub.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=\ngithub.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=\ngithub.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=\ngithub.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=\ngithub.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=\ngithub.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=\ngithub.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=\ngithub.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=\ngithub.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=\ngithub.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=\ngithub.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=\ngithub.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk=\ngithub.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=\ngithub.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=\ngithub.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=\ngithub.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=\ngithub.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=\ngithub.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=\ngithub.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=\ngithub.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=\ngithub.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=\ngithub.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=\ngithub.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9 h1:vY5WqiEon0ZSTGM3ayVVi+twaHKHDFUVloaQ/wug9/c=\ngithub.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9/go.mod h1:q+QjxYvZ+fpjMXqs+XEriussHjSYqeXVnAdSV1tkMYk=\ngithub.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=\ngithub.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=\ngithub.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=\ngithub.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=\ngithub.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=\ngo.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=\ngo.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=\ngo.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=\ngo.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=\ngolang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=\ngolang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=\ngolang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=\ngolang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=\ngolang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=\ngolang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=\ngolang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=\ngolang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=\ngolang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=\ngolang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=\ngolang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=\ngolang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=\ngolang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=\ngolang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=\ngolang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=\ngolang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=\ngolang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=\ngolang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=\ngolang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=\ngolang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=\ngolang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=\ngolang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=\ngolang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=\ngolang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=\ngolang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=\ngolang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=\ngolang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=\ngolang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=\ngolang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=\ngolang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=\ngolang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=\ngolang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=\ngolang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20181122213734-04b5d21e00f1/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=\ngolang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=\ngolang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=\ngolang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngoogle.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=\ngoogle.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=\ngoogle.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=\ngoogle.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=\ngoogle.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=\ngoogle.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=\ngoogle.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=\ngoogle.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=\ngoogle.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=\ngoogle.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=\ngoogle.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=\ngoogle.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=\ngoogle.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=\ngoogle.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=\ngoogle.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=\ngoogle.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngoogle.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=\ngoogle.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=\ngopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=\ngopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20171010053543-63abe20a23e2 h1:5zOHKFi4LqGWG+3d+isqpbPrN/2yhDJnlO+BhRiuR6U=\ngopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20171010053543-63abe20a23e2/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=\ngopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=\ngopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=\ngopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=\ngopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=\ngopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=\nhonnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\n" }, { "path": "logger.go", "content": "package rapina\n\nimport \"io\"\n\n// Logger interface contains the methods needed to poperly display log messages.\ntype Logger interface {\n\tRun(format string, v ...interface{})\n\tOk()\n\tNok()\n\tPrintf(format string, v ...interface{})\n\tTrace(format string, v ...interface{})\n\tDebug(format string, v ...interface{})\n\tInfo(format string, v ...interface{})\n\tWarn(format string, v ...interface{})\n\tError(format string, v ...interface{})\n\tSetOut(out io.Writer)\n}\n" }, { "path": "parsers/codeaccounts.go", "content": "package parsers\n\nimport (\n\t\"strings\"\n)\n\n// Bookkeeping account codes\n// If you add new const values, run 'go generate'\n// to update the generated code\nconst (\n\tUNDEF uint32 = iota\n\tSPACE\n\n\t// Balance Sheet\n\tCaixa\n\tAplicFinanceiras\n\tEstoque\n\tEquity\n\tContasARecebCirc\n\tContasARecebNCirc\n\tAtivoCirc\n\tAtivoNCirc\n\tAtivoTotal\n\tPassivoCirc\n\tPassivoNCirc\n\tPassivoTotal\n\tDividaCirc\n\tDividaNCirc\n\tDividendosJCP\n\tDividendosMin\n\n\t// Income Statement\n\tVendas\n\tCustoVendas\n\tDespesasOp\n\tEBIT\n\tResulFinanc\n\tResulOpDescont\n\tLucLiq\n\n\t// DFC\n\tFCO\n\tFCI\n\tFCF\n\n\t// Value Added Statement\n\tDeprec\n\tJurosCapProp\n\tDividendos\n\n\t// Values stored on table 'fre'\n\tShares\n\tFreeFloat\n\n\t// Financial ratios\n\tEstoqueMedio\n\tEquityAvg\n\n\t// Financial scale (unit, thousand)\n\tEscala\n\n\t// Stock quote from last day of year\n\tQuote\n)\n\n// account code, description and bookkeeping code\ntype account struct {\n\tcdAccount string\n\tdsAccount string\n\tcode uint32\n}\n\nvar _accountsTable = []account{\n\t// BPA\n\t{\"1\", \"Ativo Total\", AtivoTotal},\n\t{\"1.01\", \"Ativo Circulante\", AtivoCirc},\n\t{\"1.02\", \"Ativo Não Circulante\", AtivoNCirc},\n\t{\"1.01.01\", \"Caixa e Equivalentes de Caixa\", Caixa},\n\t{\"1.01.02\", \"Aplicações Financeiras\", AplicFinanceiras},\n\t{\"1.01.04\", \"Estoques\", Estoque}, // or \"Títulos e Créditos a Receber\" for security companies\n\t{\"1.01.03\", \"Contas a Receber\", ContasARecebCirc},\n\t{\"1.02.01.03\", \"Contas a Receber\", ContasARecebNCirc},\n\t{\"1.02.01.04\", \"Contas a Receber\", ContasARecebNCirc},\n\n\t// BPP\n\t{\"2\", \"Passivo Total\", PassivoTotal},\n\t{\"2.01\", \"Passivo Circulante\", PassivoCirc},\n\t{\"2.02\", \"Passivo Não Circulante\", PassivoNCirc},\n\t{\"2.*\", \"Patrimônio Líquido Consolidado\", Equity},\n\t{\"2.01.04\", \"Empréstimos e Financiamentos\", DividaCirc},\n\t{\"2.02.01\", \"Empréstimos e Financiamentos\", DividaNCirc},\n\t{\"2.01.05.02.01\", \"Dividendos e JCP a Pagar\", DividendosJCP},\n\t{\"2.01.05.02.02\", \"Dividendo Mínimo Obrigatório a Pagar\", DividendosMin},\n\n\t// DRE\n\t{\"3.01\", \"\", Vendas},\n\t{\"3.02\", \"\", CustoVendas},\n\t{\"3.04\", \"\", DespesasOp},\n\t{\"3.*\", \"Resultado Antes do Resultado Financeiro e dos Tributos\", EBIT},\n\t{\"3.06\", \"Resultado Financeiro\", ResulFinanc},\n\t{\"3.07\", \"Resultado Financeiro\", ResulFinanc},\n\t{\"3.08\", \"Resultado Financeiro\", ResulFinanc},\n\t{\"3.10\", \"Resultado Líquido de Operações Descontinuadas\", ResulOpDescont},\n\t{\"3.11\", \"Resultado Líquido de Operações Descontinuadas\", ResulOpDescont},\n\t{\"3.12\", \"Resultado Líquido de Operações Descontinuadas\", ResulOpDescont},\n\t{\"3.*\", \"Lucro/Prejuízo Consolidado do Período\", LucLiq},\n\t{\"3.*\", \"Lucro/Prejuízo do Período\", LucLiq},\n\n\t// DFC\n\t{\"6.01\", \"\", FCO},\n\t{\"6.02\", \"\", FCI},\n\t{\"6.03\", \"\", FCF},\n\n\t// DVA\n\t{\"7.*\", \"Depreciação, Amortização e Exaustão\", Deprec},\n\t{\"7.*\", \"Juros sobre o Capital Próprio\", JurosCapProp},\n\t{\"7.*\", \"Dividendos\", Dividendos},\n}\n\n// acctCode returns the code based on the account code and\n// account description; if the code is not found in the table\n// returns the hash.\nfunc acctCode(cdAccount, dsAccount string) uint32 {\n\tdsAccount = strings.ToLower(dsAccount)\n\n\tfor _, acc := range _accountsTable {\n\t\tdescr := strings.ToLower(acc.dsAccount)\n\t\tl := len(acc.cdAccount)\n\t\tcode := \"\"\n\t\tif l > 1 && acc.cdAccount[l-1] == '*' {\n\t\t\tcode = acc.cdAccount[:l-1] // remove the '*'\n\t\t}\n\n\t\tif code != \"\" && strings.HasPrefix(cdAccount, code) {\n\t\t\tif descr == \"\" || descr == dsAccount {\n\t\t\t\treturn acc.code\n\t\t\t}\n\t\t} else if acc.cdAccount == \"\" || acc.cdAccount == cdAccount {\n\t\t\tif descr == \"\" || descr == dsAccount {\n\t\t\t\treturn acc.code\n\t\t\t}\n\t\t}\n\t}\n\n\treturn Hash(cdAccount + dsAccount)\n}\n" }, { "path": "parsers/companies.go", "content": "package parsers\n\nimport (\n\t\"database/sql\"\n\n\t\"github.com/pkg/errors\"\n)\n\ntype company struct {\n\tid int\n\tname string\n}\n\nfunc loadCompanies(db *sql.DB) (map[string]company, error) {\n\tcompanies := make(map[string]company)\n\n\tselectCompanies := `SELECT ID, CNPJ, NAME from companies`\n\trows, err := db.Query(selectCompanies)\n\tif err != nil {\n\t\treturn companies, errors.Wrap(err, \"falha ao ler banco de dados\")\n\t}\n\n\tvar id int\n\tvar cnpj, name string\n\n\tdefer rows.Close()\n\tfor rows.Next() {\n\t\terr := rows.Scan(&id, &cnpj, &name)\n\t\tif err != nil && err != sql.ErrNoRows {\n\t\t\treturn nil, err\n\t\t}\n\t\tcompanies[cnpj] = company{id, name}\n\t}\n\n\treturn companies, nil\n}\n\nfunc saveCompanies(db *sql.DB, companies map[string]company) error {\n\tinsert := `INSERT OR IGNORE INTO companies (ID,CNPJ,NAME) VALUES (?,?,?);`\n\tstmt, err := db.Prepare(insert)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"erro ao preparar insert da lista de empresas\")\n\t}\n\tdefer stmt.Close()\n\n\tfor cnpj, value := range companies {\n\t\t_, err := stmt.Exec(value.id, cnpj, value.name)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"falha ao inserir empresa\")\n\t\t}\n\t}\n\n\treturn nil\n}\n\n//\n// updateCompanies inserts a new company to the map\n//\nfunc updateCompanies(companies map[string]company, cnpj, name string) {\n\tif _, exists := companies[cnpj]; !exists {\n\t\tcompanies[cnpj] = company{\n\t\t\tlen(companies) + 100,\n\t\t\tname,\n\t\t}\n\t}\n}\n" }, { "path": "parsers/fii.go", "content": "package parsers\n\n/*\n//\n// FetchFIIs downloads the list of FIIs to get their code (e.g. 'HGLG'),\n// then it uses this code to retrieve its details to get the CNPJ.\n// Original baseURL: https://sistemaswebb3-listados.b3.com.br.\n//\nfunc FetchFIIList(baseURL string) ([]string, error) {\n\tlistFundsURL := JoinURL(baseURL, `/fundsProxy/fundsCall/GetListFundDownload/eyJ0eXBlRnVuZCI6NywicGFnZU51bWJlciI6MSwicGFnZVNpemUiOjIwfQ==`)\n\t// fundsDetailsURL := `https://sistemaswebb3-listados.b3.com.br/fundsProxy/fundsCall/GetDetailFundSIG`\n\n\ttr := &http.Transport{\n\t\tDisableCompression: true,\n\t\tIdleConnTimeout: 30 * time.Second,\n\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t}\n\tclient := &http.Client{Transport: tr}\n\n\tresp, err := client.Get(listFundsURL)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer resp.Body.Close()\n\tif resp.StatusCode != 200 {\n\t\treturn nil, errors.New(resp.Status)\n\t}\n\n\tbody, err := io.ReadAll(resp.Body)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tunq, err := strconv.Unquote(string(body))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttxt, err := base64.StdEncoding.DecodeString(unq)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar codes []string\n\n\tfor _, line := range strings.Split(string(txt), \"\\n\") {\n\t\tp := strings.Split(line, \";\")\n\t\tif len(p) > 3 && len(p[3]) == 4 {\n\t\t\tcodes = append(codes, p[3])\n\t\t}\n\t}\n\n\treturn codes, nil\n}\n\n\n*/\n" }, { "path": "parsers/fiidb.go", "content": "package parsers\n\nimport (\n\t\"database/sql\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/dude333/rapina\"\n\t\"github.com/dude333/rapina/progress\"\n\t\"github.com/pkg/errors\"\n)\n\n// Error codes\nvar (\n\tErrDBUnset = errors.New(\"database not set\")\n\tErrNotFound = errors.New(\"not found\")\n)\n\n// FIIParser implements sqlite storage for a rapina.FIIParser object.\ntype FIIParser struct {\n\tdb *sql.DB\n\tlog rapina.Logger\n\tmu sync.Mutex // ensures atomic writes on db\n}\n\n// NewFII creates a new instace of FII.\nfunc NewFII(db *sql.DB, log rapina.Logger) (*FIIParser, error) {\n\terr := createAllTables(db)\n\treturn &FIIParser{\n\t\tdb: db,\n\t\tlog: log,\n\t}, err\n}\n\n// StoreFIIDetails parses the stream data into FIIDetails and returns\n// the *FIIDetails.\nfunc (fii *FIIParser) SaveDetails(stream []byte) error {\n\tfii.mu.Lock()\n\tdefer fii.mu.Unlock()\n\n\tif !hasTable(fii.db, \"fii_details\") {\n\t\tif err := createTable(fii.db, \"fii_details\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tvar fiiDetails rapina.FIIDetails\n\tif err := json.Unmarshal(stream, &fiiDetails); err != nil {\n\t\treturn errors.Wrap(err, \"json unmarshal\")\n\t}\n\n\ttrimFIIDetails(&fiiDetails)\n\n\tx := fiiDetails.DetailFund\n\tif x.CNPJ == \"\" {\n\t\treturn errors.New(\"CNPJ não encontrado\")\n\t}\n\n\tinsert := `INSERT OR IGNORE INTO fii_details \n\t\t(cnpj, acronym, trading_code, json) \n\t\tVALUES (?,?,?,?);`\n\t_, err := fii.db.Exec(insert,\n\t\tx.CNPJ, x.Acronym, x.TradingCode, stream)\n\n\treturn err\n}\n\n// Details returns the FII Details for the 'code' or\n// an empty string if not found in the db.\nfunc (fii *FIIParser) Details(code string) (*rapina.FIIDetails, error) {\n\tdetails := rapina.FIIDetails{}\n\n\tif fii.db == nil {\n\t\treturn nil, ErrDBUnset\n\t}\n\n\tvar query string\n\tif len(code) == 4 {\n\t\tquery = `SELECT json FROM fii_details WHERE acronym=?`\n\t} else if len(code) == 6 {\n\t\tquery = `SELECT json FROM fii_details WHERE trading_code=?`\n\t} else {\n\t\treturn nil, fmt.Errorf(\"invalid code '%s'\", code)\n\t}\n\n\tvar jsonStr []byte\n\trow := fii.db.QueryRow(query, code)\n\terr := row.Scan(&jsonStr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := json.Unmarshal(jsonStr, &details); err != nil {\n\t\tprogress.ErrorMsg(\"FII details [%v]: %s\\n\", err, string(jsonStr))\n\t\treturn nil, errors.Wrap(err, \"json unmarshal\")\n\t}\n\n\treturn &details, nil\n}\n\n// Dividends returns the dividend from the db.\nfunc (fii *FIIParser) Dividends(code, monthYear string) (*[]rapina.Dividend, error) {\n\tfii.mu.Lock()\n\tdefer fii.mu.Unlock()\n\n\tconst s = `SELECT trading_code, base_date, value\n\tFROM fii_dividends \n\tWHERE trading_code=$1 \n\tAND base_date LIKE $2;`\n\trows, err := fii.db.Query(s, code, monthYear+\"%\")\n\tif err != nil {\n\t\treturn nil, errors.Wrap(err, \"lendo dividendos do bd\")\n\t}\n\tdefer rows.Close()\n\n\tdividends := []rapina.Dividend{}\n\tvar (\n\t\ttradingCode, baseDate string\n\t\tvalue float64\n\t)\n\tfor rows.Next() {\n\t\terr := rows.Scan(&tradingCode, &baseDate, &value)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// fii.log.Debug(\"reading: %v %v %v\", tradingCode, baseDate, value)\n\n\t\tdividends = append(dividends, rapina.Dividend{\n\t\t\tCode: tradingCode,\n\t\t\tDate: baseDate,\n\t\t\tVal: value,\n\t\t})\n\t}\n\n\tif err := rows.Err(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif len(dividends) == 0 {\n\t\treturn nil, errors.New(\"dividendos não encontrados\")\n\t}\n\n\treturn ÷nds, nil\n}\n\n// SaveDividend parses and stores the map in the db. Returns the parsed stream.\nfunc (fii *FIIParser) SaveDividend(dividend rapina.Dividend) error {\n\tfii.mu.Lock()\n\tdefer fii.mu.Unlock()\n\n\tif err := createTable(fii.db, \"fii_dividends\"); err != nil {\n\t\treturn err\n\t}\n\n\tconst insert = `INSERT OR IGNORE INTO fii_dividends \n\t(trading_code, base_date, payment_date, value) VALUES (?,?,?,?)`\n\t_, err := fii.db.Exec(insert, dividend.Code, dividend.Date, dividend.PaymentDate, dividend.Val)\n\n\treturn errors.Wrap(err, \"inserting data on fii_dividends\")\n}\n\nfunc (fii *FIIParser) SelectFIIDetails(code string) (*rapina.FIIDetails, error) {\n\tif fii.db == nil {\n\t\treturn nil, ErrDBUnset\n\t}\n\n\tvar query string\n\tif len(code) == 4 {\n\t\tquery = `SELECT cnpj, acronym, trading_code FROM fii_details WHERE acronym=?`\n\t} else if len(code) == 6 {\n\t\tquery = `SELECT cnpj, acronym, trading_code FROM fii_details WHERE trading_code=?`\n\t} else {\n\t\treturn nil, fmt.Errorf(\"invalid code '%s'\", code)\n\t}\n\n\tvar cnpj, acronym, tradingCode string\n\trow := fii.db.QueryRow(query, code)\n\terr := row.Scan(&cnpj, &acronym, &tradingCode)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar fiiDetail rapina.FIIDetails\n\tfiiDetail.DetailFund.CNPJ = cnpj\n\tfiiDetail.DetailFund.Acronym = acronym\n\tfiiDetail.DetailFund.TradingCode = tradingCode\n\n\treturn &fiiDetail, nil\n}\n\n/* -------- Utils ----------- */\n\nfunc trimFIIDetails(f *rapina.FIIDetails) {\n\tf.DetailFund.CNPJ = strings.TrimSpace(f.DetailFund.CNPJ)\n\tf.DetailFund.Acronym = strings.TrimSpace(f.DetailFund.Acronym)\n\ttradingCodes := strings.Split(\n\t\tstrings.TrimSpace(f.DetailFund.TradingCode), \" \")\n\tf.DetailFund.TradingCode = tradingCodes[0]\n}\n" }, { "path": "parsers/financial.go", "content": "// financial.go\n// Parses data from csv files containing financial statements\n\npackage parsers\n\nimport (\n\t\"bufio\"\n\t\"database/sql\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/text/encoding/charmap\"\n\t\"golang.org/x/text/transform\"\n)\n\nvar (\n\t// ErrAccumITR error for accumulatd quarterly results\n\tErrAccumITR = fmt.Errorf(\"accumulated quarterly results\")\n)\n\n//\n// ImportCsv start the data import process, including the database creation\n// if necessary\n//\nfunc ImportCsv(db *sql.DB, dataType string, file string) (err error) {\n\n\t// Create status table\n\tif err = createTable(db, \"STATUS\"); err != nil {\n\t\treturn err\n\t}\n\n\t// Create companies table\n\tif err = createTable(db, \"COMPANIES\"); err != nil {\n\t\treturn err\n\t}\n\n\t// Check table version, wipe it if version differs from current version, and\n\t// (re)create the table\n\tfor _, t := range []string{dataType, \"MD5\"} {\n\t\tif v, table := dbVersion(db, t); v != currentDbVersion {\n\t\t\tif v > 0 {\n\t\t\t\tfmt.Printf(\"[i] Apagando tabela %s versão %d (versão atual: %d)\\n\", table, v, currentDbVersion)\n\t\t\t}\n\t\t\tif err := wipeDB(db, t); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif err := createTable(db, t); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t}\n\n\tisNew, err := isNewFile(db, file)\n\tif !isNew && err == nil { // if error, process file\n\t\tfmt.Printf(\"[ ] %s já processado anteriormente\\n\", dataType)\n\t\treturn\n\t}\n\n\tvar count int\n\tif dataType == \"FRE\" {\n\t\tcount, err = populateFRE(db, file)\n\t} else {\n\t\tcount, err = populateTable(db, dataType, file)\n\t}\n\tif err == nil {\n\t\tfmt.Printf(\"\\r[√] %-7s %7d linhas processadas\", dataType+\":\", count)\n\t\tstoreFile(db, file)\n\t} else {\n\t\tfmt.Print(\"\\r[x\")\n\t}\n\tfmt.Println()\n\n\treturn err\n}\n\n//\n// populateTable loop thru file and insert its lines into DB\n// and returns the number os lines inserted.\n//\nfunc populateTable(db *sql.DB, dataType, file string) (int, error) {\n\tprogress := []string{\"/\", \"-\", \"\\\\\", \"|\", \"-\", \"\\\\\"}\n\tp := 0\n\n\ttable, err := whatTable(dataType)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tcompanies, _ := loadCompanies(db)\n\n\tfh, err := os.Open(file)\n\tif err != nil {\n\t\treturn 0, errors.Wrapf(err, \"erro ao abrir arquivo %s\", file)\n\t}\n\tdefer fh.Close()\n\n\tdec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder())\n\n\t// BEGIN TRANSACTION\n\ttx, err := db.Begin()\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, \"Failed to begin transaction\")\n\t}\n\n\t// Data used inside loop\n\theader := make(map[string]int) // stores the header item position (e.g., DT_FIM_EXERC:9)\n\tscanner := bufio.NewScanner(dec)\n\tcount := 0\n\tinsert := \"\"\n\tvar stmt *sql.Stmt\n\n\t// Loop thru file, line by line\n\tfmt.Print(\"[ ] Processando arquivo \", dataType)\n\tfor scanner.Scan() {\n\t\tline := scanner.Text()\n\t\tif len(line) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tfields := strings.Split(line, \";\")\n\n\t\tif len(header) == 0 { // HEADER\n\t\t\t// Get header positioning\n\t\t\tfor i, h := range fields {\n\t\t\t\theader[h] = i\n\t\t\t}\n\t\t\t// Prepare insert statement\n\t\t\tinsert = fmt.Sprintf(`INSERT OR IGNORE INTO %s (\n\t\t\t\tID, ID_CIA, CODE, YEAR, DATA_TYPE,\n\t\t\t\tVERSAO,\n\t\t\t\tMOEDA, ESCALA_MOEDA, \n\t\t\t\tDT_FIM_EXERC,\n\t\t\t\tCD_CONTA, DS_CONTA, VL_CONTA\n\t\t\t) VALUES (\n\t\t\t\t?, ?, ?, ?, \"%s\",\n\t\t\t\t?,\n\t\t\t\t?, ?,\n\t\t\t\t?,\n\t\t\t\t?, ?, ?\n\t\t\t\t);`, table, dataType)\n\t\t\tstmt, err = tx.Prepare(insert)\n\t\t\tif err != nil {\n\t\t\t\terr = errors.Wrapf(err, \"erro ao preparar insert (verificar cabeçalho do arquivo %s)\", file)\n\t\t\t\treturn count, err\n\t\t\t}\n\t\t\tdefer stmt.Close()\n\n\t\t} else { // VALUES\n\n\t\t\tif len(fields) <= 12 {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// Only use penultimate for 2010 file, that's the last year published,\n\t\t\t// to get data from 2009\n\t\t\tif fields[header[\"ORDEM_EXERC\"]] == \"PENÚLTIMO\" {\n\t\t\t\tdt := fields[header[\"DT_FIM_EXERC\"]]\n\t\t\t\tif len(dt) < 4 || dt[:4] != \"2009\" {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// UPDATE COMPANIES\n\t\t\tn1, ok1 := header[\"CNPJ_CIA\"]\n\t\t\tn2, ok2 := header[\"DENOM_CIA\"]\n\t\t\tif ok1 && ok2 && n1 >= 0 && n1 < len(fields) && n2 >= 0 && n2 < len(fields) {\n\t\t\t\tupdateCompanies(companies, fields[header[\"CNPJ_CIA\"]], fields[header[\"DENOM_CIA\"]])\n\t\t\t}\n\n\t\t\t// INSERT\n\t\t\tf, err := prepareFields(dataType, header, fields, companies)\n\t\t\tif err == ErrAccumITR {\n\t\t\t\tcontinue // ignore accumulated ITR data\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn count, errors.Wrap(err, \"falha ao preparar registro\")\n\t\t\t}\n\t\t\t_, err = stmt.Exec(f...)\n\t\t\tif err != nil {\n\t\t\t\treturn count, errors.Wrap(err, \"falha ao inserir registro\")\n\t\t\t}\n\t\t}\n\n\t\t// fmt.Println(\"-------------------------------\")\n\t\tif count++; count%1000 == 0 {\n\t\t\tfmt.Printf(\"\\r[%s\", progress[p%6])\n\t\t\tp++\n\t\t}\n\t}\n\n\tfmt.Print(\"\\r[*\")\n\n\t// END TRANSACTION\n\terr = tx.Commit()\n\tif err != nil {\n\t\treturn count, errors.Wrap(err, \"Failed to commit transaction\")\n\t}\n\n\tif err := scanner.Err(); err != nil {\n\t\treturn count, errors.Wrapf(err, \"erro ao ler arquivo %s\", file)\n\t}\n\n\terr = saveCompanies(db, companies)\n\n\treturn count, err\n}\n\n// Cache (optimization)\nvar unixTime = make(map[string]int64)\n\n//\n// prepareFields prepares all fields (columns) to be inserted on the DB.\n//\n// Returns:\n// ID, ID_CIA, CODE, YEAR,\n// VERSAO,\n// MOEDA, ESCALA_MOEDA,\n// DT_FIM_EXERC,\n// CD_CONTA, DS_CONTA, VL_CONTA\n//\n// Tip: to convert Unix timestamp to date on sqlite: strftime('%Y-%m-%d', DT_REFER, 'unixepoch')\n//\nfunc prepareFields(dataType string, header map[string]int, fields []string, companies map[string]company) ([]interface{}, error) {\n\t// AUX FUNCTIONS\n\tval := func(key string) string {\n\t\tv, ok := header[key]\n\t\tif !ok {\n\t\t\treturn \"\"\n\t\t}\n\t\treturn fields[v]\n\t}\n\n\t// Convert date string (YYYY-MM-DD) into Unix timestamp\n\ttim := func(key string) int64 {\n\t\tv, ok := header[key]\n\t\tif !ok {\n\t\t\treturn 0\n\t\t}\n\t\tf := fields[v]\n\t\tif ut, ok := unixTime[f]; ok {\n\t\t\treturn ut\n\t\t}\n\t\tt, err := time.Parse(\"2006-01-02\", f)\n\t\tif err != nil {\n\t\t\treturn 0\n\t\t}\n\t\tunixTime[f] = t.Unix()\n\t\treturn unixTime[f]\n\t}\n\n\t// REFERENCE DATE\n\tv, ok := header[\"DT_FIM_EXERC\"]\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"DT_FIM_EXERC não encontrado\")\n\t}\n\tif len(fields[v]) < 4 || tim(\"DT_FIM_EXERC\") == 0 {\n\t\treturn nil, fmt.Errorf(\"DT_FIM_EXERC incorreto: %v\", fields[v])\n\t}\n\t// Check if quarterly data contains data from 90 days, except for \"BPA_ITR\" and \"BPP_ITR\"\n\tif dataType != \"BPA_ITR\" && dataType != \"BPP_ITR\" && strings.HasSuffix(dataType, \"_ITR\") {\n\t\tt1 := tim(\"DT_INI_EXERC\")\n\t\tt2 := tim(\"DT_FIM_EXERC\")\n\t\tdays := (t2 - t1) / 60 / 60 / 24\n\t\tif days < 80 || days > 100 {\n\t\t\treturn nil, ErrAccumITR\n\t\t}\n\t}\n\tyear := fields[v][:4]\n\n\t// CNPJ_CIA and DENOM_CIA are replaced by company id\n\tcnpj := val(\"CNPJ_CIA\")\n\tc, ok := companies[cnpj]\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"CNPJ %s não encontrado\", cnpj)\n\t}\n\tcompanyID := c.id\n\n\t// Unique value to be used as PRIMARY KEY\n\thash := Hash(cnpj + val(\"GRUPO_DFP\") + val(\"DT_FIM_EXERC\") + val(\"VERSAO\") + val(\"CD_CONTA\") + val(\"VL_CONTA\"))\n\n\t// Output -- need to follow INSERT sequence\n\tf := make([]interface{}, 11)\n\tf[0] = hash // ID\n\tf[1] = companyID // ID_CIA\n\tf[2] = acctCode(fields[header[\"CD_CONTA\"]], fields[header[\"DS_CONTA\"]]) // CODE\n\tf[3] = year // YEAR\n\tf[4] = val(\"VERSAO\")\n\tf[5] = val(\"MOEDA\")\n\tf[6] = val(\"ESCALA_MOEDA\")\n\tf[7] = tim(\"DT_FIM_EXERC\")\n\tf[8] = val(\"CD_CONTA\")\n\tf[9] = val(\"DS_CONTA\")\n\tf[10] = val(\"VL_CONTA\")\n\n\treturn f, nil\n}\n" }, { "path": "parsers/financial_test.go", "content": "package parsers\n\nimport (\n\t\"database/sql\"\n\t\"os\"\n\t\"testing\"\n\n\t_ \"github.com/mattn/go-sqlite3\"\n)\n\nfunc tempFilename(t *testing.T) string {\n\tf, err := os.CreateTemp(\"\", \"rapina-test-\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tf.Close()\n\treturn f.Name()\n}\n\nfunc samples(filename string) error {\n\tbpa := []byte(`\nCNPJ_CIA;DT_REFER;VERSAO;DENOM_CIA;CD_CVM;GRUPO_DFP;MOEDA;ESCALA_MOEDA;ORDEM_EXERC;DT_FIM_EXERC;CD_CONTA;DS_CONTA;VL_CONTA\n00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1;Ativo Total;1162167882.00\n00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.01;Caixa e Equivalentes de Caixa;68841638.00\n00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02;Aplica��es Financeiras;110019404.00\n00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02.01;Aplica��es Financeiras Avaliadas a Valor Justo;109376121.00\n00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02.01.01;T�tulos para Negocia��o;18991047.00\n00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02.01.02;T�tulos Dispon�veis para Venda;90385074.00\n00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02.02;Aplica��es Financeiras Avaliadas ao Custo Amortizado;643283.00\n00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02.02.01;T�tulos Mantidos at� o Vencimento;643283.00\n00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.03;Empr�stimos e Receb�veis;755821983.00\n00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.04;Tributos Diferidos;21954460.00\n\t`)\n\terr := os.WriteFile(filename, bpa, 0600)\n\n\treturn err\n}\n\nfunc TestImportCsv(t *testing.T) {\n\tvar db *sql.DB\n\tvar err error\n\tfileBPA := tempFilename(t)\n\tdefer os.Remove(fileBPA)\n\tfileDB := tempFilename(t)\n\tdefer os.Remove(fileDB)\n\n\tif db, err = sql.Open(\"sqlite3\", fileDB); err != nil {\n\t\tt.Errorf(\"Fail to open db: %v\", err)\n\t}\n\tdefer db.Close()\n\n\tif err = samples(fileBPA); err != nil {\n\t\tt.Errorf(\"Fail to create samples: %v\", err)\n\t}\n\n\tif err = ImportCsv(db, \"BPA\", fileBPA); err != nil {\n\t\tt.Errorf(\"Fail to parse: %v\", err)\n\t}\n\n\tfor _, tp := range []string{\"BPA\", \"MD5\"} {\n\t\tif v, table := dbVersion(db, tp); v != currentDbVersion {\n\t\t\tt.Errorf(\"Expecting table %s on version %d, received %d\", table, currentDbVersion, v)\n\t\t}\n\t}\n\n\tisNew, err := isNewFile(db, fileBPA)\n\tif isNew && err == nil {\n\t\tt.Errorf(\"Expecting processed file, got new file\")\n\t}\n\n}\n\nfunc TestGetHash(t *testing.T) {\n\ttable := []struct {\n\t\ts string\n\t\th uint32\n\t}{\n\t\t{\"test1\", 2569220284},\n\t\t{\"random data\", 1626193638},\n\t\t{\"excel\", 1973829744},\n\t\t{\"One More...12345!\", 2258028052},\n\t}\n\tfor _, x := range table {\n\t\th := Hash(x.s)\n\t\tif h != x.h {\n\t\t\tt.Errorf(\"Hash was incorrect, got: %d, want: %d.\", h, x.h)\n\t\t}\n\t}\n}\n\nfunc TestRemoveDiacritics(t *testing.T) {\n\tlist := []struct {\n\t\tstr string\n\t\texp string\n\t}{\n\t\t{\"ITAÚ\", \"ITAU\"},\n\t\t{\"SÃO\", \"SAO\"},\n\t\t{\"São Paulo\", \"Sao Paulo\"},\n\t\t{\"ÁÉÍÓÚáéíóúÀàÃÕãõÇç\", \"AEIOUaeiouAaAOaoCc\"},\n\t}\n\n\tfor _, l := range list {\n\t\tif RemoveDiacritics(l.str) != l.exp {\n\t\t\tt.Errorf(\"Expecting %s, received %s\", l.exp, RemoveDiacritics(l.str))\n\t\t}\n\t}\n}\n\nfunc Test_prepareFields(t *testing.T) {\n\tcompanies := make(map[string]company)\n\tcompanies[\"54321\"] = company{1, \"A\"}\n\n\ttype args struct {\n\t\tdataType string\n\t\theader map[string]int\n\t\tfields []string\n\t\tcompanies map[string]company\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\t\"dt_refer not found\",\n\t\t\targs{\n\t\t\t\t\"BPA\",\n\t\t\t\tmap[string]int{\"a\": 0, \"b\": 1},\n\t\t\t\t[]string{\"a\", \"b\"},\n\t\t\t\tcompanies,\n\t\t\t},\n\t\t\ttrue,\n\t\t}, {\n\t\t\t\"should work\",\n\t\t\targs{\n\t\t\t\t\"BPA\",\n\t\t\t\tmap[string]int{\"x\": 0, \"y\": 1, \"DT_FIM_EXERC\": 2, \"CNPJ_CIA\": 3},\n\t\t\t\t[]string{\"X\", \"Y\", \"2020-02-25\", \"54321\"},\n\t\t\t\tcompanies,\n\t\t\t},\n\t\t\tfalse,\n\t\t}, {\n\t\t\t\"cnpj not found\",\n\t\t\targs{\n\t\t\t\t\"BPA\",\n\t\t\t\tmap[string]int{\"x\": 0, \"y\": 2, \"DT_FIM_EXERC\": 1},\n\t\t\t\t[]string{\"X\", \"2020-02-25\", \"Y\"},\n\t\t\t\tcompanies,\n\t\t\t},\n\t\t\ttrue,\n\t\t}, {\n\t\t\t\"DT_FIM_EXERC not found\",\n\t\t\targs{\n\t\t\t\t\"BPA\",\n\t\t\t\tmap[string]int{\"x\": 0, \"y\": 2, \"DT_FIM_EXERC\": 1},\n\t\t\t\t[]string{\"X\", \"202\", \"Y\"},\n\t\t\t\tcompanies,\n\t\t\t},\n\t\t\ttrue,\n\t\t}, {\n\t\t\t\"itr should work\",\n\t\t\targs{\n\t\t\t\t\"BPA_ITR\",\n\t\t\t\tmap[string]int{\"x\": 0, \"y\": 1, \"DT_INI_EXERC\": 2, \"DT_FIM_EXERC\": 3, \"CNPJ_CIA\": 4},\n\t\t\t\t[]string{\"X\", \"Y\", \"2020-01-01\", \"2020-06-30\", \"54321\"},\n\t\t\t\tcompanies,\n\t\t\t},\n\t\t\tfalse,\n\t\t}, {\n\t\t\t\"itr should fail\",\n\t\t\targs{\n\t\t\t\t\"DRE_ITR\",\n\t\t\t\tmap[string]int{\"x\": 0, \"y\": 1, \"DT_INI_EXERC\": 2, \"DT_FIM_EXERC\": 3, \"CNPJ_CIA\": 4},\n\t\t\t\t[]string{\"X\", \"Y\", \"2020-01-01\", \"2020-06-30\", \"54321\"},\n\t\t\t\tcompanies,\n\t\t\t},\n\t\t\ttrue,\n\t\t}, {\n\t\t\t\"itr should pass\",\n\t\t\targs{\n\t\t\t\t\"DRE_ITR\",\n\t\t\t\tmap[string]int{\"x\": 0, \"y\": 1, \"DT_INI_EXERC\": 2, \"DT_FIM_EXERC\": 3, \"CNPJ_CIA\": 4},\n\t\t\t\t[]string{\"X\", \"Y\", \"2020-01-01\", \"2020-03-30\", \"54321\"},\n\t\t\t\tcompanies,\n\t\t\t},\n\t\t\tfalse,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t_, err := prepareFields(tt.args.dataType, tt.args.header, tt.args.fields, tt.args.companies)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"prepareFields() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t\treturn\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc BenchmarkPrepareFields(b *testing.B) {\n\tcompanies := make(map[string]company)\n\tcompanies[\"54321\"] = company{1, \"A\"}\n\n\th := map[string]int{\"x\": 0, \"y\": 1, \"DT_FIM_EXERC\": 2, \"CNPJ_CIA\": 3}\n\n\tf := []string{\"X\", \"Y\", \"2020-02-25\", \"54321\"}\n\n\t// run the prepareFields function b.N times\n\tfor n := 0; n < b.N; n++ {\n\t\t_, err := prepareFields(\"BPA\", h, f, companies)\n\t\tif err != nil {\n\t\t\tb.Errorf(\"error: %v\", err)\n\t\t\treturn\n\t\t}\n\t}\n}\n" }, { "path": "parsers/fre.go", "content": "package parsers\n\nimport (\n\t\"bufio\"\n\t\"database/sql\"\n\t\"fmt\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/text/encoding/charmap\"\n\t\"golang.org/x/text/transform\"\n)\n\nvar (\n\t// ErrCNPJNotFound error\n\tErrCNPJNotFound = fmt.Errorf(\"CNPJ not found\")\n)\n\nfunc populateFRE(db *sql.DB, file string) (int, error) {\n\tprogress := []string{\"/\", \"-\", \"\\\\\", \"|\", \"-\", \"\\\\\"}\n\tp := 0\n\tvar err error\n\n\ttable, err := whatTable(\"FRE\")\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tcompanies, _ := loadCompanies(db)\n\n\tfh, err := os.Open(file)\n\tif err != nil {\n\t\treturn 0, errors.Wrapf(err, \"erro ao abrir arquivo %s\", file)\n\t}\n\tdefer fh.Close()\n\n\tdec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder())\n\n\t// BEGIN TRANSACTION\n\ttx, err := db.Begin()\n\tif err != nil {\n\t\treturn 0, errors.Wrap(err, \"Failed to begin transaction\")\n\t}\n\n\t// Data used inside loop\n\tsep := func(r rune) bool {\n\t\treturn r == ';'\n\t}\n\theader := make(map[string]int) // stores the header item position (e.g., DT_FIM_EXERC:9)\n\tscanner := bufio.NewScanner(dec)\n\tcount := 0\n\tinsert := \"\"\n\tvar stmt *sql.Stmt\n\n\t// Loop thru file, line by line\n\tfmt.Print(\"[ ] Processando arquivo FRE\")\n\tfor scanner.Scan() {\n\t\tline := scanner.Text()\n\t\tif len(line) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tfields := strings.FieldsFunc(line, sep)\n\n\t\tif len(header) == 0 { // HEADER\n\t\t\t// Get header positioning\n\t\t\tfor i, h := range fields {\n\t\t\t\theader[h] = i\n\t\t\t}\n\t\t\t// Prepare insert statement\n\t\t\tinsert = fmt.Sprintf(`INSERT OR IGNORE INTO %s (\n\t\t\t\tID, ID_CIA, YEAR, \n\t\t\t\tVersao,\n\t\t\t\tQuantidade_Total_Acoes_Circulacao, \n\t\t\t\tPercentual_Total_Acoes_Circulacao \n\t\t\t) VALUES (\n\t\t\t\t?, ?, ?,\n\t\t\t\t?,\n\t\t\t\t?,\n\t\t\t\t?\n\t\t\t\t);`, table)\n\t\t\tstmt, err = tx.Prepare(insert)\n\t\t\tif err != nil {\n\t\t\t\terr = errors.Wrapf(err, \"erro ao preparar insert (verificar cabeçalho do arquivo %s)\", file)\n\t\t\t\treturn count, err\n\t\t\t}\n\t\t\tdefer stmt.Close()\n\n\t\t} else { // VALUES\n\n\t\t\tif len(fields) <= 12 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t// INSERT\n\t\t\tf, err := prepareFREFields(header, fields, companies)\n\t\t\tif err == ErrCNPJNotFound {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tfmt.Println(line)\n\t\t\t\tfmt.Printf(\"\\r[x] Falha ao preparar registro: %v\\n\", err)\n\t\t\t\tfmt.Print(\"[ ] Processando arquivo FRE\")\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\t_, err = stmt.Exec(f...)\n\t\t\tif err != nil {\n\t\t\t\treturn count, errors.Wrap(err, \"falha ao inserir registro\")\n\t\t\t}\n\t\t}\n\n\t\tif count++; count%60 == 0 {\n\t\t\tfmt.Printf(\"\\r[%s\", progress[p%6])\n\t\t\tp++\n\t\t}\n\t}\n\n\tfmt.Print(\"\\r[*\")\n\n\t// END TRANSACTION\n\terr = tx.Commit()\n\tif err != nil {\n\t\treturn count, errors.Wrap(err, \"Failed to commit transaction\")\n\t}\n\n\tif err := scanner.Err(); err != nil {\n\t\treturn count, errors.Wrapf(err, \"erro ao ler arquivo %s\", file)\n\t}\n\n\treturn count, nil\n}\n\nfunc prepareFREFields(header map[string]int, fields []string, companies map[string]company) ([]interface{}, error) {\n\tif len(fields) < len(header)-1 {\n\t\treturn nil, fmt.Errorf(\"len(fields)=%d != len(header)=%d\", len(fields), len(header))\n\t}\n\n\t// val checks and gets the value from a map\n\tval := func(key string) string {\n\t\tv, ok := header[key]\n\t\tif !ok {\n\t\t\treturn \"\"\n\t\t}\n\t\treturn fields[v]\n\t}\n\n\t// YEAR\n\tv, ok := header[\"Data_Referencia\"]\n\tif !ok {\n\t\treturn nil, fmt.Errorf(\"Data_Referencia não encontrado\")\n\t}\n\tif len(fields[v]) != 10 {\n\t\treturn nil, fmt.Errorf(\"DT_FIM_EXERC incorreto: %v\", fields[v])\n\t}\n\tyear := fields[v][:4]\n\n\t// CNPJ_Companhia is replaced by company id\n\tcnpj := val(\"CNPJ_Companhia\")\n\tc, ok := companies[cnpj]\n\tif !ok {\n\t\treturn nil, ErrCNPJNotFound\n\t}\n\tcompanyID := c.id\n\n\t// Free float\n\tff := val(\"Percentual_Total_Acoes_Circulacao\")\n\tvar freeFloat float32\n\tif ff != \"\" {\n\t\tif f, err := strconv.ParseFloat(ff, 32); err == nil {\n\t\t\tfreeFloat = float32(f / 100)\n\t\t}\n\t}\n\n\t// Total shares considering the free float\n\tshares := val(\"Quantidade_Total_Acoes_Circulacao\")\n\tvar totalShares float32\n\tif shares != \"\" {\n\t\tif f, err := strconv.ParseFloat(shares, 32); err == nil {\n\t\t\tif freeFloat > 0 {\n\t\t\t\ttotalShares = float32(f) / freeFloat\n\t\t\t}\n\t\t}\n\t}\n\n\t// Unique value to be used as PRIMARY KEY\n\thash := Hash(cnpj + val(\"Data_Referencia\") + val(\"Versao\") + val(\"ID_Documento\") + val(\"Quantidade_Total_Acoes_Circulacao\"))\n\n\t// Output -- need to match INSERT sequence\n\tvar f []interface{}\n\tf = append(f, hash) // ID\n\tf = append(f, companyID) // ID_CIA\n\tf = append(f, year) // YEAR\n\n\tf = append(f, val(\"Versao\"))\n\tf = append(f, totalShares)\n\tf = append(f, freeFloat)\n\n\treturn f, nil\n}\n" }, { "path": "parsers/fuzzy.go", "content": "package parsers\n\nimport (\n\t\"strings\"\n\n\t\"github.com/lithammer/fuzzysearch/fuzzy\"\n)\n\n//\n// FuzzyMatch measures the Levenshtein distance between\n// the source and the list, returning true if the distance\n// is less or equal the 'distance'.\n// Diacritics are removed from 'src' and 'list'.\n//\nfunc FuzzyMatch(src string, list []string, distance int) bool {\n\treturn FuzzyFind(src, list, distance) != \"\"\n}\n\n//\n// FuzzyFind returns the most approximate string inside 'list' that\n// matches the 'src' string within a maximum 'distance'.\n//\nfunc FuzzyFind(source string, targets []string, maxDistance int) (found string) {\n\tfor _, target := range targets {\n\t\tsrc := fix(source)\n\t\ttrg := fix(target)\n\t\tif strings.HasPrefix(src, trg) || strings.HasPrefix(trg, src) {\n\t\t\treturn target\n\t\t}\n\t\tdistance := fuzzy.LevenshteinDistance(src, trg)\n\t\tif distance <= maxDistance {\n\t\t\tmaxDistance = distance\n\t\t\tfound = target\n\t\t}\n\t}\n\n\tif found == \"\" {\n\t\tfor _, target := range targets {\n\t\t\tsrc := strings.Split(fix(source), \" \")\n\t\t\ttrg := strings.Split(fix(target), \" \")\n\n\t\t\tif len(src) > 2 && len(trg) > 2 {\n\t\t\t\tif src[0] == trg[0] && src[1] == trg[1] {\n\t\t\t\t\treturn target\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn\n}\n\nfunc fix(txt string) string {\n\ttxt = strings.ToUpper(txt)\n\ttxt = strings.Replace(txt, \"BCO \", \"BANCO \", 1)\n\treturn RemoveDiacritics(txt)\n}\n" }, { "path": "parsers/fuzzy_test.go", "content": "package parsers\n\nimport \"testing\"\n\nfunc TestFuzzyFind(t *testing.T) {\n\tlist := []struct {\n\t\tsrc string\n\t\ttrg []string\n\t\tmaxDist int\n\t\texpected string\n\t}{\n\t\t{\"ABCD\", []string{\"ABC\", \"ACD\"}, 2, \"ABC\"},\n\t\t{\"ABCD\", []string{\"XYZ\", \"ACD\"}, 1, \"ACD\"},\n\t\t{\"ABCDÉ\", []string{\"XYZ\", \"ACD\", \"ABCDE\"}, 0, \"ABCDE\"},\n\t\t{\"ABCDÉ FGH\", []string{\"XYZ\", \"ACD\", \"FGH\"}, 6, \"FGH\"},\n\t\t{\"BCO ABC\", []string{\"XYZ\", \"BANCO ABC\", \"FGH\"}, 0, \"BANCO ABC\"},\n\t}\n\n\tfor _, l := range list {\n\t\tr := FuzzyFind(l.src, l.trg, l.maxDist)\n\t\tif r != l.expected {\n\t\t\tt.Errorf(\"Expected: %s, got: %s\", l.expected, r)\n\t\t}\n\t}\n}\n" }, { "path": "parsers/md5.go", "content": "package parsers\n\nimport (\n\t\"crypto/md5\"\n\t\"database/sql\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n)\n\n//\n// isNewFile checks the database to see if this file has been\n// processed already\n//\nfunc isNewFile(db *sql.DB, filename string) (isNew bool, err error) {\n\tisNew = true\n\n\tmd5, err := md5FromFile(filename)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tsqlStmt := `SELECT md5 FROM md5 WHERE md5 = ?`\n\terr = db.QueryRow(sqlStmt, md5).Scan(&md5)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tisNew = false\n\treturn\n}\n\n//\n// storeFile into md5 table (only successfully processed files)\n//\nfunc storeFile(db *sql.DB, filename string) (md5 string) {\n\tmd5, err := md5FromFile(filename)\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\tinsert := fmt.Sprintf(`INSERT OR IGNORE INTO md5 (md5) VALUES (\"%s\")`, md5)\n\t_, err = db.Exec(insert)\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\treturn md5\n}\n\n//\n// md5FromFile\n//\nfunc md5FromFile(filename string) (string, error) {\n\tf, err := os.Open(filename)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tdefer f.Close()\n\n\th := md5.New()\n\tif _, err = io.Copy(h, f); err != nil {\n\t\treturn \"\", err\n\t}\n\n\treturn fmt.Sprintf(\"%x\", h.Sum(nil)), nil\n}\n" }, { "path": "parsers/md5_test.go", "content": "package parsers\n\nimport (\n\t\"database/sql\"\n\t\"os\"\n\t\"testing\"\n\n\t_ \"github.com/mattn/go-sqlite3\"\n\t\"github.com/pkg/errors\"\n)\n\nfunc TestIsNewFile(t *testing.T) {\n\tif testing.Short() {\n\t\tt.Skip(\"skipping testing in short mode\") // used in CI\n\t}\n\n\tdb, err := openDatabase()\n\tif err != nil {\n\t\tt.Errorf(\"cannot open db: %v\", err)\n\t\treturn\n\t}\n\n\tif err := createTable(db, \"MD5\"); err != nil {\n\t\tt.Errorf(\"could not create table: %v\", err)\n\t}\n\n\tfile := \"../cli/.data/bpa_cia_aberta_con_2017.csv\"\n\tisNew, err := isNewFile(db, file)\n\texpected := false\n\tif _, err := os.Stat(file); !os.IsNotExist(err) {\n\t\texpected = true\n\t}\n\n\tif isNew == expected {\n\t\tt.Errorf(\"isNewFile returned %v. If 'rapina get' has run before it should've returned false.\\nError: [%v]\", expected, err)\n\t}\n}\n\nfunc openDatabase() (db *sql.DB, err error) {\n\n\tdb, err = sql.Open(\"sqlite3\", \"../bin/.data/rapina.db\")\n\tif err != nil {\n\t\treturn db, errors.Wrap(err, \"database open failed\")\n\t}\n\n\treturn\n}\n" }, { "path": "parsers/meta/meta_bpa_cia_aberta.txt", "content": "-----------------------\nCampo: CNPJ_CIA\n-----------------------\n Descrição: CNPJ da companhia\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 20\n Scale: 0\n-----------------------\nCampo: DT_REFER\n-----------------------\n Descrição: Data de referência do documento\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: VERSAO\n-----------------------\n Descrição: Versão do documento\n Domínio: Numérico\n Tipo dados: smallint\n Precisão: 5\n Scale: 0\n-----------------------\nCampo: DENOM_CIA\n-----------------------\n Descrição: Nome empresarial da companhia\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 100\n Scale: 0\n-----------------------\nCampo: CD_CVM\n-----------------------\n Descrição: Código CVM\n Domínio: Numérico\n Tipo dados: numeric\n Precisão: 7\n Scale: 0\n-----------------------\nCampo: GRUPO_DFP\n-----------------------\n Descrição: Nome e nível de agregação da demonstração\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 206\n Scale: 0\n-----------------------\nCampo: MOEDA\n-----------------------\n Descrição: Moeda\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 4\n Scale: 0\n-----------------------\nCampo: ESCALA_MOEDA\n-----------------------\n Descrição: Escala monetária\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 7\n Scale: 0\n-----------------------\nCampo: ORDEM_EXERC\n-----------------------\n Descrição: Ordem do exercício social\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 9\n Scale: 0\n-----------------------\nCampo: DT_FIM_EXERC\n-----------------------\n Descrição: Data fim do exercício social\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: CD_CONTA\n-----------------------\n Descrição: Código da conta\n Domínio: Numérico\n Tipo dados: varchar\n Precisão: 18\n Scale: 0\n-----------------------\nCampo: DS_CONTA\n-----------------------\n Descrição: Descrição da conta\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 100\n Scale: 0\n-----------------------\nCampo: VL_CONTA\n-----------------------\n Descrição: Valor da conta\n Domínio: Numérico\n Tipo dados: numeric\n Precisão: 29\n Scale: 2\n" }, { "path": "parsers/meta/meta_bpp_cia_aberta.txt", "content": "-----------------------\nCampo: CNPJ_CIA\n-----------------------\n Descrição: CNPJ da companhia\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 20\n Scale: 0\n-----------------------\nCampo: DT_REFER\n-----------------------\n Descrição: Data de referência do documento\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: VERSAO\n-----------------------\n Descrição: Versão do documento\n Domínio: Numérico\n Tipo dados: smallint\n Precisão: 5\n Scale: 0\n-----------------------\nCampo: DENOM_CIA\n-----------------------\n Descrição: Nome empresarial da companhia\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 100\n Scale: 0\n-----------------------\nCampo: CD_CVM\n-----------------------\n Descrição: Código CVM\n Domínio: Numérico\n Tipo dados: numeric\n Precisão: 7\n Scale: 0\n-----------------------\nCampo: GRUPO_DFP\n-----------------------\n Descrição: Nome e nível de agregação da demonstração\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 206\n Scale: 0\n-----------------------\nCampo: MOEDA\n-----------------------\n Descrição: Moeda\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 4\n Scale: 0\n-----------------------\nCampo: ESCALA_MOEDA\n-----------------------\n Descrição: Escala monetária\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 7\n Scale: 0\n-----------------------\nCampo: ORDEM_EXERC\n-----------------------\n Descrição: Ordem do exercício social\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 9\n Scale: 0\n-----------------------\nCampo: DT_FIM_EXERC\n-----------------------\n Descrição: Data fim do exercício social\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: CD_CONTA\n-----------------------\n Descrição: Código da conta\n Domínio: Numérico\n Tipo dados: varchar\n Precisão: 18\n Scale: 0\n-----------------------\nCampo: DS_CONTA\n-----------------------\n Descrição: Descrição da conta\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 100\n Scale: 0\n-----------------------\nCampo: VL_CONTA\n-----------------------\n Descrição: Valor da conta\n Domínio: Numérico\n Tipo dados: numeric\n Precisão: 29\n Scale: 2\n" }, { "path": "parsers/meta/meta_dfc_md_cia_aberta.txt", "content": "-----------------------\nCampo: CNPJ_CIA\n-----------------------\n Descrição: CNPJ da companhia\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 20\n Scale: 0\n-----------------------\nCampo: DT_REFER\n-----------------------\n Descrição: Data de referência do documento\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: VERSAO\n-----------------------\n Descrição: Versão do documento\n Domínio: Numérico\n Tipo dados: smallint\n Precisão: 5\n Scale: 0\n-----------------------\nCampo: DENOM_CIA\n-----------------------\n Descrição: Nome empresarial da companhia\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 100\n Scale: 0\n-----------------------\nCampo: CD_CVM\n-----------------------\n Descrição: Código CVM\n Domínio: Numérico\n Tipo dados: numeric\n Precisão: 7\n Scale: 0\n-----------------------\nCampo: GRUPO_DFP\n-----------------------\n Descrição: Nome e nível de agregação da demonstração\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 206\n Scale: 0\n-----------------------\nCampo: MOEDA\n-----------------------\n Descrição: Moeda\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 4\n Scale: 0\n-----------------------\nCampo: ESCALA_MOEDA\n-----------------------\n Descrição: Escala monetária\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 7\n Scale: 0\n-----------------------\nCampo: ORDEM_EXERC\n-----------------------\n Descrição: Ordem do exercício social\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 9\n Scale: 0\n-----------------------\nCampo: DT_INI_EXERC\n-----------------------\n Descrição: Data início do exercício social\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: DT_FIM_EXERC\n-----------------------\n Descrição: Data fim do exercício social\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: CD_CONTA\n-----------------------\n Descrição: Código da conta\n Domínio: Numérico\n Tipo dados: varchar\n Precisão: 18\n Scale: 0\n-----------------------\nCampo: DS_CONTA\n-----------------------\n Descrição: Descrição da conta\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 100\n Scale: 0\n-----------------------\nCampo: VL_CONTA\n-----------------------\n Descrição: Valor da conta\n Domínio: Numérico\n Tipo dados: numeric\n Precisão: 29\n Scale: 2\n" }, { "path": "parsers/meta/meta_dfc_mi_cia_aberta.txt", "content": "-----------------------\nCampo: CNPJ_CIA\n-----------------------\n Descrição: CNPJ da companhia\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 20\n Scale: 0\n-----------------------\nCampo: DT_REFER\n-----------------------\n Descrição: Data de referência do documento\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: VERSAO\n-----------------------\n Descrição: Versão do documento\n Domínio: Numérico\n Tipo dados: smallint\n Precisão: 5\n Scale: 0\n-----------------------\nCampo: DENOM_CIA\n-----------------------\n Descrição: Nome empresarial da companhia\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 100\n Scale: 0\n-----------------------\nCampo: CD_CVM\n-----------------------\n Descrição: Código CVM\n Domínio: Numérico\n Tipo dados: numeric\n Precisão: 7\n Scale: 0\n-----------------------\nCampo: GRUPO_DFP\n-----------------------\n Descrição: Nome e nível de agregação da demonstração\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 206\n Scale: 0\n-----------------------\nCampo: MOEDA\n-----------------------\n Descrição: Moeda\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 4\n Scale: 0\n-----------------------\nCampo: ESCALA_MOEDA\n-----------------------\n Descrição: Escala monetária\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 7\n Scale: 0\n-----------------------\nCampo: ORDEM_EXERC\n-----------------------\n Descrição: Ordem do exercício social\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 9\n Scale: 0\n-----------------------\nCampo: DT_INI_EXERC\n-----------------------\n Descrição: Data início do exercício social\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: DT_FIM_EXERC\n-----------------------\n Descrição: Data fim do exercício social\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: CD_CONTA\n-----------------------\n Descrição: Código da conta\n Domínio: Numérico\n Tipo dados: varchar\n Precisão: 18\n Scale: 0\n-----------------------\nCampo: DS_CONTA\n-----------------------\n Descrição: Descrição da conta\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 100\n Scale: 0\n-----------------------\nCampo: VL_CONTA\n-----------------------\n Descrição: Valor da conta\n Domínio: Numérico\n Tipo dados: numeric\n Precisão: 29\n Scale: 2\n" }, { "path": "parsers/meta/meta_dre_cia_aberta.txt", "content": "-----------------------\nCampo: CNPJ_CIA\n-----------------------\n Descrição: CNPJ da companhia\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 20\n Scale: 0\n-----------------------\nCampo: DT_REFER\n-----------------------\n Descrição: Data de referência do documento\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: VERSAO\n-----------------------\n Descrição: Versão do documento\n Domínio: Numérico\n Tipo dados: smallint\n Precisão: 5\n Scale: 0\n-----------------------\nCampo: DENOM_CIA\n-----------------------\n Descrição: Nome empresarial da companhia\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 100\n Scale: 0\n-----------------------\nCampo: CD_CVM\n-----------------------\n Descrição: Código CVM\n Domínio: Numérico\n Tipo dados: numeric\n Precisão: 7\n Scale: 0\n-----------------------\nCampo: GRUPO_DFP\n-----------------------\n Descrição: Nome e nível de agregação da demonstração\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 206\n Scale: 0\n-----------------------\nCampo: ESCALA_DRE\n-----------------------\n Descrição: Escala monetária\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 7\n Scale: 0\n-----------------------\nCampo: ORDEM_EXERC\n-----------------------\n Descrição: Ordem do exercício social\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 9\n Scale: 0\n-----------------------\nCampo: DT_INI_EXERC\n-----------------------\n Descrição: Data início do exercício social\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: DT_FIM_EXERC\n-----------------------\n Descrição: Data fim do exercício social\n Domínio: AAAA-MM-DD\n Tipo dados: date\n Precisão: 10\n Scale: 0\n-----------------------\nCampo: CD_CONTA\n-----------------------\n Descrição: Código da conta\n Domínio: Numérico\n Tipo dados: varchar\n Precisão: 18\n Scale: 0\n-----------------------\nCampo: DS_CONTA\n-----------------------\n Descrição: Descrição da conta\n Domínio: Alfanumérico\n Tipo dados: varchar\n Precisão: 100\n Scale: 0\n-----------------------\nCampo: VL_CONTA\n-----------------------\n Descrição: Valor da conta\n Domínio: Numérico\n Tipo dados: numeric\n Precisão: 29\n Scale: 2\n" }, { "path": "parsers/sectors.go", "content": "package parsers\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"os\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/PuerkitoBio/goquery\"\n\t\"github.com/dude333/rapina\"\n\t\"github.com/gocolly/colly/v2\"\n\t\"github.com/pkg/errors\"\n\tyaml \"gopkg.in/yaml.v2\"\n)\n\n// SectorsToYaml grab data from B3 website and prints out to a yaml file\n// with all companies grouped by sector, subsector, segment\nfunc SectorsToYaml(yamlFile string) (err error) {\n\tprogress := []string{\"/\", \"-\", \"\\\\\", \"|\", \"-\", \"\\\\\"}\n\tvar p int32\n\n\tif !overwritePrompt(yamlFile) {\n\t\treturn rapina.ErrFileNotUpdated\n\t}\n\tf, err := os.Create(yamlFile)\n\tif err != nil {\n\t\treturn errors.Wrapf(err, \"falha ao criar arquivo %s\", yamlFile)\n\t}\n\tdefer f.Close()\n\n\tw := bufio.NewWriter(f)\n\n\tc := colly.NewCollector(\n\t\t// Restrict crawling to specific domains\n\t\t// colly.AllowedDomains(\"bvmf.bmfbovespa.com.br\"),\n\t\tcolly.AllowURLRevisit(),\n\t\tcolly.Async(false),\n\t\tcolly.CacheDir(\".data/cache\"),\n\t)\n\n\tc.OnHTML(\"tr\", func(e *colly.HTMLElement) {\n\t\tvar sector string\n\t\tvar subsectors []string\n\t\tc := 0\n\t\te.ForEach(\"td\", func(_ int, elem *colly.HTMLElement) {\n\t\t\telem.DOM.Each(func(_ int, s *goquery.Selection) {\n\t\t\t\th, _ := s.Html()\n\t\t\t\tif c == 0 {\n\t\t\t\t\tsector = h\n\t\t\t\t\tfmt.Fprintln(w, \" - Setor:\", sector)\n\t\t\t\t\tfmt.Fprintln(w, \" Subsetores:\")\n\t\t\t\t} else if c == 1 {\n\t\t\t\t\tsubsectors = strings.Split(h, \"
\")\n\t\t\t\t\tlast := subsectors[0]\n\t\t\t\t\tfor i := range subsectors {\n\t\t\t\t\t\tif subsectors[i] == \"\" {\n\t\t\t\t\t\t\tsubsectors[i] = last\n\t\t\t\t\t\t}\n\t\t\t\t\t\tlast = subsectors[i]\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tc++\n\t\t\t})\n\n\t\t\tlastSub := \"\"\n\t\t\telem.ForEach(\"a[href]\", func(i int, elem *colly.HTMLElement) {\n\t\t\t\tif strings.Contains(elem.Attr(\"href\"), \"BuscaEmpresaListada.aspx\") {\n\t\t\t\t\t// fmt.Printf(\"\\n=> %s > %s > %s:\\n\", sector, subsectors[i], elem.Text) //, elem.Attr(\"href\"))\n\t\t\t\t\tif subsectors[i] != lastSub {\n\t\t\t\t\t\tfmt.Fprintln(w, \" - Subsetor:\", subsectors[i])\n\t\t\t\t\t\tfmt.Fprintln(w, \" Segmentos:\")\n\t\t\t\t\t}\n\t\t\t\t\tlastSub = subsectors[i]\n\t\t\t\t\tfmt.Fprintln(w, \" - Segmento:\", removeYamlInvalidChar(elem.Text))\n\t\t\t\t\tfmt.Fprintln(w, \" Empresas:\")\n\t\t\t\t\t_ = companies(w, \"http://bvmf.bmfbovespa.com.br/cias-listadas/empresas-listadas/\"+elem.Attr(\"href\"))\n\t\t\t\t}\n\n\t\t\t\tfmt.Printf(\"\\r[%s]\", progress[p%6])\n\t\t\t\tp++\n\t\t\t})\n\t\t})\n\t})\n\n\tfmt.Print(\"[ ] Lendo informações do site da B3\")\n\tfmt.Fprintln(w, \"Setores:\")\n\n\terr = c.Visit(\"http://bvmf.bmfbovespa.com.br/cias-listadas/empresas-listadas/BuscaEmpresaListada.aspx?opcao=1&indiceAba=1&Idioma=pt-br\")\n\n\tfmt.Println()\n\tw.Flush()\n\n\treturn\n}\n\n// companies lists all companies in the same sector/subsector/segment\nfunc companies(w *bufio.Writer, url string) error {\n\tc := colly.NewCollector(\n\t\t// Restrict crawling to specific domains\n\t\t// colly.AllowedDomains(\"bvmf.bmfbovespa.com.br\"),\n\t\tcolly.AllowURLRevisit(),\n\t\tcolly.Async(false),\n\t\tcolly.CacheDir(\".data/cache\"),\n\t)\n\n\t// Find and visit all links\n\tc.OnHTML(\"tr\", func(e *colly.HTMLElement) {\n\t\t// if e.Attr(\"class\") != \"GridRow_SiteBmfBovespa GridBovespaItemStyle\" {\n\t\t// \treturn\n\t\t// }\n\n\t\te.ForEachWithBreak(\"a\", func(_ int, elem *colly.HTMLElement) bool {\n\t\t\tif strings.Contains(elem.Attr(\"href\"), \"ResumoEmpresaPrincipal.aspx\") {\n\t\t\t\tfmt.Fprintln(w, \" -\", removeYamlInvalidChar(elem.Text))\n\t\t\t}\n\t\t\treturn false // get only the 1st elem\n\t\t})\n\t})\n\n\treturn c.Visit(url)\n}\n\n// overwritePrompt prompts to overwrite file if it exists\nfunc overwritePrompt(filename string) bool {\n\tif _, err := os.Stat(filename); err == nil { // check if file exists\n\t\tfmt.Printf(\"\\n[?] Deseja sobrescrever o arquivo \\\"%s\\\"? (s/N) \", filename)\n\t\treader := bufio.NewReader(os.Stdin)\n\t\tprompt, _ := reader.ReadString('\\n')\n\n\t\tif !strings.EqualFold(prompt, \"s\\n\") && !strings.EqualFold(prompt, \"sim\\n\") &&\n\t\t\t!strings.EqualFold(prompt, \"s\\r\\n\") && !strings.EqualFold(prompt, \"sim\\r\\n\") {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\n// S contains the sectors\ntype S struct {\n\tSectors []Sector `yaml:\"Setores\"`\n}\n\n// Sector is divided into subsectors\ntype Sector struct {\n\tName string `yaml:\"Setor\"`\n\tSubsectors []Subsector `yaml:\"Subsetores\"`\n}\n\n// Subsector is divided into segments\ntype Subsector struct {\n\tName string `yaml:\"Subsetor\"`\n\tSegments []Segment `yaml:\"Segmentos\"`\n}\n\n// Segment contains companies from the same sector/subsector/segment\ntype Segment struct {\n\tName string `yaml:\"Segmento\"`\n\tCompanies []string `yaml:\"Empresas\"`\n}\n\n// FromSector returns all companies from the same sector as the 'company'\nfunc FromSector(company, yamlFile string) (companies []string, sectorName string, err error) {\n\ty, err := os.ReadFile(yamlFile)\n\tif err != nil {\n\t\terr = errors.Wrapf(err, \"ReadFile: %v\", err)\n\t\treturn\n\t}\n\n\ts := S{}\n\tif err := yaml.Unmarshal(y, &s); err != nil {\n\t\treturn nil, \"\", err\n\t}\n\n\tfor _, sector := range s.Sectors {\n\t\tfor _, subsector := range sector.Subsectors {\n\t\t\tfor _, segment := range subsector.Segments {\n\t\t\t\tif FuzzyMatch(company, segment.Companies, 2) {\n\t\t\t\t\tcompanies = segment.Companies\n\t\t\t\t\tsectorName = strings.Join([]string{sector.Name, subsector.Name, segment.Name}, \" > \")\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn\n}\n\n// removeYamlInvalidChar removes yaml invalid characters\nfunc removeYamlInvalidChar(text string) string {\n\tyaml_invalid_chars := regexp.MustCompile(`[^/\\s.A-zÀ-ú0-9&():-]`)\n\treturn yaml_invalid_chars.ReplaceAllString(text, \"\")\n}\n" }, { "path": "parsers/sectors_test.go", "content": "package parsers\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestFromSector(t *testing.T) {\n\ttempDir, _ := os.MkdirTemp(\"\", \"rapina-test\")\n\tfilename := tempDir + \"/test_sectors.yml\"\n\n\tcreateYaml(filename)\n\ts, _, _ := FromSector(\"GRENDENE S.A.\", filename)\n\texpected := [...]string{\"ALPARGATAS S.A.\", \"CAMBUCI S.A.\", \"GRENDENE S.A.\", \"VULCABRAS/AZALEIA S.A.\"}\n\tif len(s) != 4 {\n\t\tt.Errorf(\"\\n- Expected: %v\\n- Got: %v\", expected, s)\n\t}\n\n\tvar arr [4]string\n\tcopy(arr[:], s)\n\tif arr != expected {\n\t\tt.Errorf(\"\\n- Expected: %v\\n- Got: %v\", expected, s)\n\t}\n\n\tos.Remove(filename)\n}\n\nfunc createYaml(filename string) {\n\tyaml := []byte(\n\t\t`Setores:\n- Setor: Bens Industriais\n Subsetores:\n - Subsetor: Comércio\n Segmentos:\n - Segmento: Material de Transporte\n Empresas:\n - MINASMAQUINAS S.A.\n - WLM PART. E COMÉRCIO DE MÁQUINAS E VEÍCULOS S.A.\n- Setor: Consumo Cíclico\n Subsetores:\n - Subsetor: Tecidos. Vestuário e Calçados\n Segmentos:\n - Segmento: Acessórios\n Empresas:\n - MUNDIAL S.A. - PRODUTOS DE CONSUMO\n - TECHNOS S.A.\n - Segmento: Calçados\n Empresas:\n - ALPARGATAS S.A.\n - CAMBUCI S.A.\n - GRENDENE S.A.\n - VULCABRAS/AZALEIA S.A.`)\n\n\t_ = os.WriteFile(filename, yaml, 0644)\n}\n" }, { "path": "parsers/stock.go", "content": "package parsers\n\n/*\n\tTODO:\n\thttps://query1.finance.yahoo.com/v7/finance/download/RBVA11.SA?period1=1588395063&period2=1619931063&interval=1d&events=history&includeAdjustedClose=true\n\thttps://query1.finance.yahoo.com/v7/finance/download/BBPO11.SA?period1=1619654400&period2=1619740800&interval=1d&events=history&includeAdjustedClose=true\n*/\n\nimport (\n\t\"bufio\"\n\t\"database/sql\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/dude333/rapina\"\n\t\"github.com/dude333/rapina/progress\"\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/text/encoding/charmap\"\n\t\"golang.org/x/text/transform\"\n)\n\ntype stockQuote struct {\n\tStock string\n\tDate string\n\tOpen float64\n\tHigh float64\n\tLow float64\n\tClose float64\n\tVolume float64\n}\n\ntype stockCode struct {\n\tTckrSymb string // Code\n\tSgmtNm string // value: CASH\n\tSctyCtgyNm string // values: SHARES, UNIT, FUNDS\n\tCrpnNm string // Company name\n\tSpcfctnCd string // values: ON, ON NM, PN N2, etc.\n\tCorpGovnLvlNm string // values: NOVO MERCADO, NIVEL 2, etc.\n}\n\ntype StockParser struct {\n\tdb *sql.DB\n\tlog rapina.Logger\n}\n\n//\n// NewStock creates the required tables, if necessary, and returns a StockParser instance.\n//\nfunc NewStock(db *sql.DB, log rapina.Logger) (*StockParser, error) {\n\tfor _, t := range []string{\"status\", \"stock_quotes\", \"stock_codes\"} {\n\t\tif err := createTable(db, t); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\ts := &StockParser{db: db, log: log}\n\treturn s, nil\n}\n\n//\n// Quote returns the quote from DB.\n//\nfunc (s *StockParser) Quote(code, date string) (float64, error) {\n\tquery := `SELECT close FROM stock_quotes WHERE stock=$1 AND date=$2;`\n\tvar close float64\n\terr := s.db.QueryRow(query, code, date).Scan(&close)\n\tif err == sql.ErrNoRows {\n\t\treturn 0, errors.New(\"não encontrado no bd\")\n\t}\n\tif err != nil {\n\t\treturn 0, errors.Wrapf(err, \"lendo cotação de %s do bd\", code)\n\t}\n\n\treturn close, nil\n}\n\n//\n// Quote returns the company ON stock code, where stockType is:\n// ON, PN, UNT, CI [CI = FII]\n//\nfunc (s *StockParser) Code(companyName, stockType string) (string, error) {\n\tquery := `SELECT trading_code FROM stock_codes WHERE company_name LIKE ? AND SpcfctnCd LIKE ?;`\n\tst := strings.ToUpper(stockType + \"%\")\n\tvar code string\n\terr := s.db.QueryRow(query, \"%\"+companyName+\"%\", st).Scan(&code)\n\tif err == sql.ErrNoRows {\n\t\treturn \"\", errors.New(\"não encontrado no bd\")\n\t}\n\tif err != nil {\n\t\treturn \"\", errors.Wrapf(err, \"lendo código de %s do bd\", companyName)\n\t}\n\n\treturn code, nil\n}\n\nfunc (s *StockParser) SaveB3Quotes(filename string) error {\n\tisNew, err := isNewFile(s.db, filename)\n\tif !isNew && err == nil { // if error, process file\n\t\tprogress.Warning(\"%s já processado anteriormente\", filename)\n\t\treturn errors.New(\"este arquivo de cotações já foi importado anteriormente\")\n\t}\n\n\tif err := s.populateStockQuotes(filename); err != nil {\n\t\treturn err\n\t}\n\n\tstoreFile(s.db, filename)\n\n\treturn nil\n}\n\nfunc (s *StockParser) populateStockQuotes(filename string) error {\n\tfh, err := os.Open(filename)\n\tif err != nil {\n\t\treturn errors.Wrapf(err, \"abrindo arquivo %s\", filename)\n\t}\n\tdefer fh.Close()\n\n\t// BEGIN TRANSACTION\n\ttx, err := s.db.Begin()\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"Failed to begin transaction\")\n\t}\n\n\tdec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder())\n\tscanner := bufio.NewScanner(dec)\n\tfor scanner.Scan() {\n\t\tline := scanner.Text()\n\t\tif len(line) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tq, err := parseB3Quote(line)\n\t\tif err != nil {\n\t\t\tcontinue // ignore line\n\t\t}\n\t\tfmt.Printf(\"%+v\\n\", q)\n\t}\n\n\t// END TRANSACTION\n\tif err := tx.Commit(); err != nil {\n\t\treturn errors.Wrap(err, \"Failed to commit transaction\")\n\t}\n\n\tif err := scanner.Err(); err != nil {\n\t\treturn errors.Wrapf(err, \"lendo arquivo %s\", filename)\n\t}\n\n\treturn nil\n}\n\n//\n// Save parses the 'stream', get the 'code' stock quotes and\n// store it on 'db'. Returns the number of registers saved.\n//\nfunc (s *StockParser) Save(stream io.Reader, code string) (int, error) {\n\tif s.db == nil {\n\t\treturn 0, errors.New(\"bd inválido\")\n\t}\n\tif stream == nil {\n\t\treturn 0, errors.New(\"sem dados\")\n\t}\n\n\tscanner := bufio.NewScanner(stream)\n\n\t// Read 1st line\n\tscanner.Scan()\n\tprov := provider(scanner.Text())\n\n\tvar r rec\n\tif err := r.open(s.db, prov); err != nil {\n\t\treturn 0, err\n\t}\n\tdefer r.close()\n\n\t// Read stream, line by line\n\tvar count int\n\tfor scanner.Scan() {\n\t\tline := scanner.Text()\n\n\t\tvar q *stockQuote\n\t\tvar c *stockCode\n\t\tvar err error\n\t\tswitch prov {\n\t\tcase b3Quotes:\n\t\t\tq, err = parseB3Quote(line)\n\t\tcase yahoo:\n\t\t\tq, err = parseYahoo(line, code)\n\t\tcase alphaVantage:\n\t\t\tq, err = parseAlphaVantage(line, code)\n\t\tcase b3Codes:\n\t\t\tc, err = parseB3Code(line)\n\t\t}\n\t\tif err != nil {\n\t\t\tcontinue // ignore lines with error\n\t\t}\n\n\t\tif q != nil {\n\t\t\terr = r.storeQuote(q)\n\t\t\tif err == nil {\n\t\t\t\tcount++\n\t\t\t}\n\t\t}\n\t\tif c != nil {\n\t\t\terr = r.storeCode(c)\n\t\t\tif err == nil {\n\t\t\t\tcount++\n\t\t\t}\n\t\t}\n\t}\n\n\tif err := scanner.Err(); err != nil {\n\t\treturn count, err\n\t}\n\n\treturn count, nil\n}\n\n// open prepares the insert statement.\nfunc (s *rec) open(db *sql.DB, provider int) error {\n\tvar err error\n\tinsert := `INSERT OR IGNORE INTO stock_quotes \n\t(stock, date, open, high, low, close, volume) VALUES (?,?,?,?,?,?,?);`\n\n\tif provider == b3Codes {\n\t\tinsert = `INSERT OR IGNORE INTO stock_codes \n\t(trading_code, company_name, SpcfctnCd, CorpGovnLvlNm) VALUES (?,?,?,?);`\n\t}\n\n\ts.stmt, err = db.Prepare(insert)\n\tif err != nil || s.stmt == nil {\n\t\treturn errors.Wrap(err, \"insert on db\")\n\t}\n\n\treturn nil\n}\n\n// storeQuote stores the data using the insert statement.\nfunc (s *rec) storeQuote(q *stockQuote) error {\n\tif s.stmt == nil {\n\t\treturn errors.New(\"sql statement not initalized\")\n\t}\n\n\ts.mu.Lock()\n\n\tres, err := s.stmt.Exec(\n\t\tq.Stock,\n\t\tq.Date,\n\t\tq.Open,\n\t\tq.High,\n\t\tq.Low,\n\t\tq.Close,\n\t\tq.Volume,\n\t)\n\n\ts.mu.Unlock()\n\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"salvando cotação\")\n\t}\n\n\tn, err := res.RowsAffected()\n\tif n == 0 || err != nil {\n\t\treturn errors.New(\"registro não salvo (duplicado)\")\n\t}\n\n\treturn nil\n}\n\n// storeQuote stores the data using the insert statement.\nfunc (s *rec) storeCode(c *stockCode) error {\n\tif s.stmt == nil {\n\t\treturn errors.New(\"sql statement not initalized\")\n\t}\n\n\ts.mu.Lock()\n\n\tres, err := s.stmt.Exec(\n\t\tc.TckrSymb, // trading_code\n\t\tc.CrpnNm, // company_name\n\t\tc.SpcfctnCd,\n\t\tc.CorpGovnLvlNm,\n\t)\n\n\ts.mu.Unlock()\n\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"salvando códigos\")\n\t}\n\n\tn, err := res.RowsAffected()\n\tif n == 0 || err != nil {\n\t\treturn errors.New(\"registro não salvo (duplicado)\")\n\t}\n\n\treturn nil\n}\n\n// close closes the insert statement.\nfunc (s *rec) close() error {\n\tvar err error\n\tif s.stmt != nil {\n\t\terr = s.stmt.Close()\n\t}\n\treturn err\n}\n\n// API providers.\nconst (\n\tnone int = iota\n\talphaVantage\n\tyahoo\n\tb3Quotes\n\tb3Codes\n)\n\n// provider returns stream type based on header\nfunc provider(header string) int {\n\tif header == \"timestamp,open,high,low,close,volume\" {\n\t\treturn alphaVantage\n\t}\n\tif header == \"Date,Open,High,Low,Close,Adj Close,Volume\" {\n\t\treturn yahoo\n\t}\n\tif strings.HasPrefix(header, \"00COTAHIST.\") {\n\t\treturn b3Quotes\n\t}\n\tif strings.HasPrefix(header, \"RptDt;TckrSymb;Asst;AsstDesc;SgmtNm;MktNm;SctyCtgyNm;XprtnDt;\") {\n\t\treturn b3Codes\n\t}\n\treturn none\n}\n\n// parseAlphaVantage parses lines downloaded from Alpha Vantage API server\n// and returns *stockQuote for 'code'.\nfunc parseAlphaVantage(line, code string) (*stockQuote, error) {\n\tfields := strings.Split(line, \",\")\n\tif len(fields) != 6 {\n\t\treturn nil, errors.New(\"linha inválida\") // ignore lines with error\n\t}\n\n\t// Columns: timestamp,open,high,low,close,volume\n\tvar err error\n\tvar floats [5]float64\n\tfor i := 1; i <= 5; i++ {\n\t\tfloats[i-1], err = strconv.ParseFloat(fields[i], 64)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"campo inválido\")\n\t\t}\n\t}\n\n\treturn &stockQuote{\n\t\tStock: code,\n\t\tDate: fields[0],\n\t\tOpen: floats[0],\n\t\tHigh: floats[1],\n\t\tLow: floats[2],\n\t\tClose: floats[3],\n\t\tVolume: floats[4],\n\t}, nil\n}\n\n// parseYahoo parses lines downloaded from Yahoo Finance API server\n// and returns *stockQuote for 'code'.\nfunc parseYahoo(line, code string) (*stockQuote, error) {\n\tfields := strings.Split(line, \",\")\n\tif len(fields) != 7 {\n\t\treturn nil, errors.New(\"linha inválida\") // ignore lines with error\n\t}\n\n\t// Columns: Date,Open,High,Low,Close,Adj Close,Volume\n\tvar err error\n\tvar floats [6]float64\n\tfor i := 1; i <= 6; i++ {\n\t\tfloats[i-1], err = strconv.ParseFloat(fields[i], 64)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrap(err, \"campo inválido\")\n\t\t}\n\t}\n\n\treturn &stockQuote{\n\t\tStock: code,\n\t\tDate: fields[0],\n\t\tOpen: floats[0],\n\t\tHigh: floats[1],\n\t\tLow: floats[2],\n\t\tClose: floats[3],\n\t\tVolume: floats[5],\n\t}, nil\n}\n\n// parseB3Quote parses the line based on this layout:\n// http://www.b3.com.br/data/files/33/67/B9/50/D84057102C784E47AC094EA8/SeriesHistoricas_Layout.pdf\n//\n// CAMPO/CONTEÚDO TIPO E TAMANHO POS. INIC.\t POS. FINAL\n// TIPREG “01” N(02) 01 02\n// DATA “AAAAMMDD” N(08) 03 10\n// CODBDI X(02) 11 12\n// CODNEG X(12) 13 24\n// TPMERC N(03) 25 27\n// PREABE (11)V99 57 69\n// PREMAX (11)V99 70 82\n// PREMIN (11)V99 83 95\n// PREULT (11)V99 109 121\n// QUATOT N18 153 170\n// VOLTOT (16)V99 171 188\n//\n// CODBDI:\n// 02 LOTE PADRÃO\n// 12 FUNDO IMOBILIÁRIO\n//\n// TPMERC:\n// 010 VISTA\n// 020 FRACIONÁRIO\nfunc parseB3Quote(line string) (*stockQuote, error) {\n\tif len(line) != 245 {\n\t\treturn nil, errors.New(\"linha deve conter 245 bytes\")\n\t}\n\n\trecType := line[0:2]\n\tif recType != \"01\" {\n\t\treturn nil, fmt.Errorf(\"registro %s ignorado\", recType)\n\t}\n\n\tcodBDI := line[10:12]\n\tif codBDI != \"02\" && codBDI != \"12\" && codBDI != \"13\" && codBDI != \"14\" {\n\t\treturn nil, fmt.Errorf(\"BDI %s ignorado\", codBDI)\n\t}\n\n\ttpMerc := line[24:27]\n\tif tpMerc != \"010\" && tpMerc != \"020\" {\n\t\treturn nil, fmt.Errorf(\"tipo de mercado %s ignorado\", tpMerc)\n\t}\n\n\tdate := line[2:6] + \"-\" + line[6:8] + \"-\" + line[8:10]\n\tcode := strings.TrimSpace(line[12:24])\n\n\tnumRanges := [5]struct {\n\t\ti, f int\n\t}{\n\t\t{56, 69}, // PREABE = open\n\t\t{69, 82}, // PREMAX = high\n\t\t{82, 95}, // PREMIN = low\n\t\t{108, 121}, // PREULT = close\n\t\t{170, 188}, // VOLTOT = volume\n\t}\n\tvar vals [5]int\n\tfor i, r := range numRanges {\n\t\tnum, err := strconv.Atoi(line[r.i:r.f])\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tvals[i] = num\n\t}\n\n\treturn &stockQuote{\n\t\tStock: code,\n\t\tDate: date,\n\t\tOpen: float64(vals[0]) / 100,\n\t\tHigh: float64(vals[1]) / 100,\n\t\tLow: float64(vals[2]) / 100,\n\t\tClose: float64(vals[3]) / 100,\n\t\tVolume: float64(vals[4]) / 100,\n\t}, nil\n}\n\ntype rec struct {\n\tstmt *sql.Stmt\n\tmu sync.Mutex // ensures atomic writes to db\n}\n\n// parseB3Code parses lines downloaded from B3 server\n// and returns *stockCode.\n//\nfunc parseB3Code(line string) (*stockCode, error) {\n\tfields := strings.Split(line, \";\")\n\n\t// Columns:\n\t// RptDt;TckrSymb(2);Asst;AsstDesc;SgmtNm(5);MktNm;SctyCtgyNm(7);XprtnDt;XprtnCd;\n\t// TradgStartDt;TradgEndDt;BaseCd;ConvsCritNm;MtrtyDtTrgtPt;ReqrdConvsInd;\n\t// ISIN;CFICd;DlvryNtceStartDt;DlvryNtceEndDt;OptnTp;CtrctMltplr;AsstQtnQty;\n\t// AllcnRndLot;TradgCcy;DlvryTpNm;WdrwlDays;WrkgDays;ClnrDays;RlvrBasePricNm;\n\t// OpngFutrPosDay;SdTpCd1;UndrlygTckrSymb1;SdTpCd2;UndrlygTckrSymb2;\n\t// PureGoldWght;ExrcPric;OptnStyle;ValTpNm;PrmUpfrntInd;OpngPosLmtDt;\n\t// DstrbtnId;PricFctr;DaysToSttlm;SrsTpNm;PrtcnFlg;AutomtcExrcInd;SpcfctnCd(47);\n\t// CrpnNm(48);CorpActnStartDt;CtdyTrtmntTpNm;MktCptlstn;CorpGovnLvlNm(52)\n\tif len(fields) != 52 {\n\t\treturn nil, fmt.Errorf(\"linha inválida %d\", len(fields)) // ignore lines with error\n\t}\n\n\ts := stockCode{\n\t\tTckrSymb: fields[1],\n\t\tSgmtNm: fields[4],\n\t\tSctyCtgyNm: fields[6],\n\t\tCrpnNm: fields[47],\n\t\tSpcfctnCd: fields[46],\n\t\tCorpGovnLvlNm: fields[51],\n\t}\n\n\tif s.SgmtNm != \"CASH\" ||\n\t\t(s.SctyCtgyNm != \"SHARES\" &&\n\t\t\ts.SctyCtgyNm != \"FUNDS\" &&\n\t\t\ts.SctyCtgyNm != \"UNIT\") {\n\t\treturn nil, errors.New(\"linha ignorada\")\n\t}\n\n\treturn &s, nil\n}\n" }, { "path": "parsers/stock_test.go", "content": "package parsers\n\nimport (\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc Test_parseB3(t *testing.T) {\n\n\tconst file = `012021010412NSLU11 010FII LOURDES CI ER R$ 000000002840000000000284000000000027700000000002809000000000281900000000028029000000002819000168000000000000001381000000000038793560000000000000009999123100000010000000000000BRNSLUCTF008272\n012021010412NVHO11 010FII NOVOHORICI ER R$ 000000000154000000000015900000000001535000000000153700000000015400000000001536000000000154000092000000000000006200000000000009533490000000000000009999123100000010000000000000BRNVHOCTF003186\n012021010412ONEF11 010FII THE ONE CI R$ 000000001478800000000148000000000014717000000001478900000000147360000000014735000000001478700035000000000000002546000000000037652878000000000000009999123100000010000000000000BRONEFCTF003200`\n\n\twant := []stockQuote{\n\t\t{Stock: \"NSLU11\", Date: \"2021-01-04\", Open: 284, High: 284, Low: 277, Close: 281.9, Volume: 387935.6},\n\t\t{Stock: \"NVHO11\", Date: \"2021-01-04\", Open: 15.4, High: 15.9, Low: 15.35, Close: 15.4, Volume: 95334.9},\n\t\t{Stock: \"ONEF11\", Date: \"2021-01-04\", Open: 147.88, High: 148, Low: 147.17, Close: 147.36, Volume: 376528.78},\n\t}\n\n\tfor i, line := range strings.Split(file, \"\\n\") {\n\t\tgot, err := parseB3Quote(line)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"parseB3() error = %v\", err)\n\t\t\treturn\n\t\t}\n\n\t\tif err == nil && !reflect.DeepEqual(got, &want[i]) {\n\t\t\tt.Errorf(\"parseB3() got %+v, want %+v\", got, &want[i])\n\t\t}\n\n\t}\n\n}\n\nfunc Test_parseB3Code(t *testing.T) {\n\ttype args struct {\n\t\tline string\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twant *stockCode\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"funds\",\n\t\t\targs: args{\n\t\t\t\t`2021-05-13;ALMI11;ALMI;ALMI;CASH;EQUITY-CASH;FUNDS;;;2018-09-24;9999-12-31;;;;;BRALMICTF003;CICIRU;;;;;;1;BRL;;;;;;;;;;;;;;;;;250;1;2;;;;CI;FDO INV IMOB - FII TORRE ALMIRANTE;9999-12-31;FUNGIBLE;111177;`,\n\t\t\t},\n\t\t\twant: &stockCode{TckrSymb: \"ALMI11\", SgmtNm: \"CASH\", SctyCtgyNm: \"FUNDS\", CrpnNm: \"FDO INV IMOB - FII TORRE ALMIRANTE\", SpcfctnCd: \"CI\", CorpGovnLvlNm: \"\"},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"unit\",\n\t\t\targs: args{\n\t\t\t\t`2021-05-13;ALUP11;ALUP;ALUP;CASH;EQUITY-CASH;UNIT;;;2021-04-28;9999-12-31;;;;;BRALUPCDAM15;EMXXXR;;;;;;1;BRL;;;;;;;;;;;;;;;;;112;1;2;;;;UNT N2;ALUPAR INVESTIMENTO S/A;9999-12-31;FUNGIBLE;136606616;NIVEL 2`,\n\t\t\t},\n\t\t\twant: &stockCode{TckrSymb: \"ALUP11\", SgmtNm: \"CASH\", SctyCtgyNm: \"UNIT\", CrpnNm: \"ALUPAR INVESTIMENTO S/A\", SpcfctnCd: \"UNT N2\", CorpGovnLvlNm: \"NIVEL 2\"},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"shares\",\n\t\t\targs: args{\n\t\t\t\t`2021-05-13;ALPA3;ALPA;ALPA;CASH;EQUITY-CASH;SHARES;;;2020-02-17;9999-12-31;;;;;BRALPAACNOR0;ESVUFR;;;;;;1;BRL;;;;;;;;;;;;;;;;;229;1;2;;;;ON N1;ALPARGATAS S.A.;9999-12-31;FUNGIBLE;302010689;NIVEL 1`,\n\t\t\t},\n\t\t\twant: &stockCode{TckrSymb: \"ALPA3\", SgmtNm: \"CASH\", SctyCtgyNm: \"SHARES\", CrpnNm: \"ALPARGATAS S.A.\", SpcfctnCd: \"ON N1\", CorpGovnLvlNm: \"NIVEL 1\"},\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"should fail on odd lot\",\n\t\t\targs: args{\n\t\t\t\t`2021-05-13;ANIM3F;ANIM;ANIM;ODD LOT;EQUITY-CASH;SHARES;;;2021-02-19;9999-12-31;;;;;BRANIMACNOR6;ESVUFR;;;;;;1;BRL;;;;;;;;;;;;;;;;;107;1;2;;;;ON NM;ANIMA HOLDING S.A.;9999-12-31;FUNGIBLE;403868805;NOVO MERCADO`,\n\t\t\t},\n\t\t\twant: &stockCode{},\n\t\t\twantErr: true,\n\t\t},\n\t\t{\n\t\t\tname: \"should fail on bdr\",\n\t\t\targs: args{\n\t\t\t\t`2021-05-13;AMZO34;AMZO;AMZO;CASH;EQUITY-CASH;BDR;;;2020-11-09;9999-12-31;;;;;BRAMZOBDR002;EDSXPR;;;;;;1;BRL;;;;;;;;;;;;;;;;;102;1;2;;;;DRN;AMAZON.COM, INC;9999-12-31;FUNGIBLE;79059664651;`,\n\t\t\t},\n\t\t\twant: &stockCode{},\n\t\t\twantErr: true,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tgot, err := parseB3Code(tt.args.line)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"parseB3Code() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif !tt.wantErr && !reflect.DeepEqual(got, tt.want) {\n\t\t\t\tt.Errorf(\"parseB3Code() = %#v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "parsers/tables.go", "content": "package parsers\n\nimport (\n\t\"database/sql\"\n\t\"fmt\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n)\n\nconst currentDbVersion = 210514\nconst currentFIIDbVersion = 210426\nconst currentStockCodesVersion = 210518\nconst currentStockQuotesVersion = 210305\n\nvar createTableMap = map[string]string{\n\t\"dfp\": `CREATE TABLE IF NOT EXISTS dfp\n\t(\n\t\t\"ID\" PRIMARY KEY,\n\t\t\"ID_CIA\" integer,\n\t\t\"CODE\" integer,\n\t\t\"YEAR\" string,\n\t\t\"DATA_TYPE\" string,\n\n\t\t\"VERSAO\" integer,\n\t\t\"MOEDA\" varchar(4),\n\t\t\"ESCALA_MOEDA\" varchar(7),\n\t\t\"DT_FIM_EXERC\" integer,\n\t\t\"CD_CONTA\" varchar(18),\n\t\t\"DS_CONTA\" varchar(100),\n\t\t\"VL_CONTA\" real\n\t);`,\n\n\t\"itr\": `CREATE TABLE IF NOT EXISTS itr\n\t(\n\t\t\"ID\" PRIMARY KEY,\n\t\t\"ID_CIA\" integer,\n\t\t\"CODE\" integer,\n\t\t\"YEAR\" string,\n\t\t\"DATA_TYPE\" string,\n\n\t\t\"VERSAO\" integer,\n\t\t\"MOEDA\" varchar(4),\n\t\t\"ESCALA_MOEDA\" varchar(7),\n\t\t\"DT_FIM_EXERC\" integer,\n\t\t\"CD_CONTA\" varchar(18),\n\t\t\"DS_CONTA\" varchar(100),\n\t\t\"VL_CONTA\" real\n\t);`,\n\n\t\"fre\": `CREATE TABLE IF NOT EXISTS fre\n\t(\n\t\t\"ID\" PRIMARY KEY,\n\t\t\"ID_CIA\" integer,\n\t\t\"YEAR\" string,\n\n\t\t\"Versao\" integer,\n\t\t\"Quantidade_Total_Acoes_Circulacao\" integer,\n\t\t\"Percentual_Total_Acoes_Circulacao\" real\n\t);`,\n\n\t\"codes\": `CREATE TABLE IF NOT EXISTS codes\n\t(\n\t\t\"CODE\" INTEGER NOT NULL PRIMARY KEY,\n\t\t\"NAME\" varchar(100)\n\t);`,\n\n\t\"companies\": `CREATE TABLE IF NOT EXISTS companies\n\t(\n\t\t\"ID\" INTEGER NOT NULL PRIMARY KEY,\n\t\t\"CNPJ\" varchar(20),\n\t\t\"NAME\" varchar(100)\n\t);`,\n\n\t\"stock_codes\": `CREATE TABLE IF NOT EXISTS stock_codes\n\t(\n\t\t\"trading_code\" VARCHAR NOT NULL PRIMARY KEY,\n\t\t\"company_name\" VARCHAR,\n\t\t\"SpcfctnCd\" VARCHAR,\n\t\t\"CorpGovnLvlNm\" VARCHAR\n\t);`,\n\n\t\"md5\": `CREATE TABLE IF NOT EXISTS md5\n\t(\n\t\tmd5 NOT NULL PRIMARY KEY\n\t);`,\n\n\t\"fii_details\": `CREATE TABLE IF NOT EXISTS fii_details\n\t(\n\t\tcnpj TEXT NOT NULL PRIMARY KEY,\n\t\tacronym varchar(4),\n\t\ttrading_code varchar(6),\n\t\tjson varchar\n\t);`,\n\n\t\"stock_quotes\": `CREATE TABLE IF NOT EXISTS stock_quotes\n\t(\n\t\tstock varchar(12) NOT NULL,\n\t\tdate varchar(10) NOT NULL,\n\t\topen real,\n\t\thigh real,\n\t\tlow real,\n\t\tclose real,\n\t\tvolume real\n\t);`,\n\n\t\"fii_dividends\": `CREATE TABLE IF NOT EXISTS fii_dividends\n\t(\n\t\ttrading_code varchar(12) NOT NULL, \n\t\tbase_date varchar(10) NOT NULL, \n\t\tpayment_date varchar(10), \n\t\tvalue real\n\t);`,\n\n\t\"status\": `CREATE TABLE IF NOT EXISTS status\n\t(\n\t\ttable_name TEXT NOT NULL PRIMARY KEY,\n\t\tversion integer\n\t);`,\n}\n\nfunc allTables() []string {\n\tkeys := make([]string, len(createTableMap))\n\ti := 0\n\tfor k := range createTableMap {\n\t\tkeys[i] = k\n\t\ti++\n\t}\n\treturn keys\n}\n\n//\n// whatTable for the data type\n//\nfunc whatTable(dataType string) (table string, err error) {\n\tswitch dataType {\n\tcase \"dfp\", \"BPA\", \"BPP\", \"DRE\", \"DFC_MD\", \"DFC_MI\", \"DVA\":\n\t\ttable = \"dfp\"\n\tcase \"itr\", \"BPA_ITR\", \"BPP_ITR\", \"DRE_ITR\", \"DFC_MD_ITR\", \"DFC_MI_ITR\", \"DVA_ITR\":\n\t\ttable = \"itr\"\n\tcase \"fre\", \"FRE\":\n\t\ttable = \"fre\"\n\tcase \"codes\", \"CODES\":\n\t\ttable = \"codes\"\n\tcase \"md5\", \"MD5\":\n\t\ttable = \"md5\"\n\tcase \"status\", \"STATUS\":\n\t\ttable = \"status\"\n\tcase \"companies\", \"COMPANIES\":\n\t\ttable = \"companies\"\n\tcase \"fii_details\":\n\t\ttable = dataType\n\tcase \"fii_dividends\":\n\t\ttable = dataType\n\tcase \"stock_codes\":\n\t\ttable = dataType\n\tcase \"stock_quotes\":\n\t\ttable = dataType\n\tdefault:\n\t\treturn \"\", errors.Wrapf(err, \"tipo de informação inexistente: %s\", dataType)\n\t}\n\n\treturn\n}\n\n//\n// createTable creates the table if not created yet\n//\nfunc createTable(db *sql.DB, dataType string) (err error) {\n\n\ttable, err := whatTable(dataType)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t_, err = db.Exec(createTableMap[table])\n\tif err != nil {\n\t\treturn errors.Wrapf(err, \"erro ao criar tabela '%s'\", table)\n\t}\n\n\terr = createIndexes(db, table)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"erro ao criar índice para table \"+table)\n\t}\n\n\tif strings.ToUpper(dataType) == \"STATUS\" {\n\t\treturn nil\n\t}\n\n\tversion := currentDbVersion\n\tswitch dataType {\n\tcase \"fii_details\":\n\t\tversion = currentFIIDbVersion\n\tcase \"fii_dividends\":\n\t\tversion = currentFIIDbVersion\n\tcase \"stock_codes\":\n\t\tversion = currentStockCodesVersion\n\tcase \"stock_quotes\":\n\t\tversion = currentStockQuotesVersion\n\t}\n\n\t_, err = db.Exec(fmt.Sprintf(`INSERT OR REPLACE INTO status (table_name, version) VALUES (\"%s\",%d)`, table, version))\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"erro ao atualizar tabela \"+table)\n\t}\n\n\treturn nil\n}\n\nfunc createAllTables(db *sql.DB) (err error) {\n\tif err := createTable(db, \"status\"); err != nil {\n\t\treturn err\n\t}\n\tfor _, t := range allTables() {\n\t\tif t == \"status\" {\n\t\t\tcontinue\n\t\t}\n\t\tif err := createTable(db, t); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n//\n// dbVersion returns the version stored in DB\n//\nfunc dbVersion(db *sql.DB, dataType string) (v int, table string) {\n\ttable, err := whatTable(dataType)\n\tif err != nil {\n\t\treturn\n\t}\n\n\tsqlStmt := `SELECT version FROM status WHERE table_name = ?`\n\terr = db.QueryRow(sqlStmt, table).Scan(&v)\n\tif err != nil {\n\t\treturn\n\t}\n\n\treturn\n}\n\n//\n// wipeDB drops the table! Use with care\n//\nfunc wipeDB(db *sql.DB, dataType string) (err error) {\n\ttable, err := whatTable(dataType)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t_, err = db.Exec(\"DROP TABLE IF EXISTS \" + table)\n\tif err != nil {\n\t\treturn errors.Wrap(err, \"erro ao apagar tabela\")\n\t}\n\n\treturn\n}\n\n//\n// createIndexes create indexes based on table name\n//\nfunc createIndexes(db *sql.DB, table string) error {\n\tindexes := []string{}\n\n\tswitch table {\n\tcase \"dfp\":\n\t\tindexes = []string{\n\t\t\t\"CREATE INDEX IF NOT EXISTS dfp_metrics ON dfp (CODE, ID_CIA, YEAR, VL_CONTA);\",\n\t\t\t\"CREATE INDEX IF NOT EXISTS dfp_year_ver ON dfp (ID_CIA, YEAR, VERSAO);\",\n\t\t}\n\tcase \"itr\":\n\t\tindexes = []string{\n\t\t\t\"CREATE INDEX IF NOT EXISTS itr_metrics ON itr (CODE, ID_CIA, YEAR, VL_CONTA);\",\n\t\t\t\"CREATE INDEX IF NOT EXISTS itr_quarter_ver ON itr (ID_CIA, DT_FIM_EXERC, VERSAO);\",\n\t\t}\n\tcase \"stock_quotes\":\n\t\tindexes = []string{\n\t\t\t\"CREATE UNIQUE INDEX IF NOT EXISTS stock_quotes_stockdate ON stock_quotes (stock, date);\",\n\t\t}\n\tcase \"fii_dividends\":\n\t\tindexes = []string{\n\t\t\t\"CREATE UNIQUE INDEX IF NOT EXISTS fii_dividends_pk ON fii_dividends (trading_code, base_date);\",\n\t\t}\n\t}\n\n\tfor _, idx := range indexes {\n\t\t_, err := db.Exec(idx)\n\t\tif err != nil {\n\t\t\treturn errors.Wrap(err, \"erro ao criar índice\")\n\t\t}\n\t}\n\n\treturn nil\n}\n\n//\n// hasTable checks if the table exists\n//\nfunc hasTable(db *sql.DB, tableName string) bool {\n\tsqlStmt := `SELECT name FROM sqlite_master WHERE type='table' AND name=?;`\n\tvar n string\n\terr := db.QueryRow(sqlStmt, tableName).Scan(&n)\n\tif err != nil {\n\t\treturn false\n\t}\n\n\treturn n == tableName\n}\n" }, { "path": "parsers/transform.go", "content": "package parsers\n\nimport (\n\t\"hash/fnv\"\n\t\"unicode\"\n\n\t\"golang.org/x/text/runes\"\n\t\"golang.org/x/text/transform\"\n\t\"golang.org/x/text/unicode/norm\"\n)\n\n// fnvHash is a global var set to speed up Hash\nvar fnvHash = fnv.New32a()\n\n//\n// Hash returns the FNV-1 non-cryptographic hash\n//\nfunc Hash(s string) uint32 {\n\tfnvHash.Write([]byte(s))\n\tdefer fnvHash.Reset()\n\n\treturn fnvHash.Sum32()\n}\n\n//\n// RemoveDiacritics transforms, for example, \"žůžo\" into \"zuzo\"\n//\nfunc RemoveDiacritics(original string) (result string) {\n\tt := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)\n\tresult, _, _ = transform.String(t, original)\n\n\treturn\n}\n" }, { "path": "progress/cmd/main.go", "content": "package main\n\nimport (\n\t\"errors\"\n\t\"time\"\n\n\t\"github.com/dude333/rapina/progress\"\n)\n\nfunc main() {\n\n\tprogress.Cursor(false)\n\tdefer progress.Cursor(true)\n\tprogress.Status(\"a status msg\")\n\n\tprogress.Running(\"start process\")\n\tprogress.Error(errors.New(\"some error\"))\n\ttime.Sleep(time.Second)\n\tprogress.RunOK()\n\n\tprogress.Running(\"start another process\")\n\ttime.Sleep(time.Second)\n\tprogress.Status(\"middle\")\n\ttime.Sleep(time.Second)\n\tprogress.RunFail()\n\n\tf1()\n\n\tprogress.Running(\"start spinner\")\n\tfor i := 0; i < 100; i++ {\n\t\ttime.Sleep(10 * time.Millisecond)\n\t\tprogress.Spinner()\n\t\tif i == 20 {\n\t\t\tprogress.Status(\"spinner interrupt\")\n\t\t}\n\t}\n\tprogress.RunOK()\n\n\tprogress.Status(\"end.\")\n}\n\nfunc f1() {\n\tprogress.Running(\"Running *f1*\")\n\ttime.Sleep(time.Second)\n\tprogress.Warning(\"f1 warning\")\n\ttime.Sleep(time.Second)\n\tprogress.RunOK()\n}\n" }, { "path": "progress/progress.go", "content": "// progress prints the program progress on screen. It's similar to a logger, but with\n// better formatting.\npackage progress\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n)\n\n// event stores logs IDs and messages.\ntype event struct {\n\tid int\n\tformat string\n}\n\n// Log messages as new Events\nvar (\n\t// evStatus = event{1, \"[>] %v\"}\n\tevError = event{1, \"[✗] %v\"}\n\tevRunning = event{2, \"[ ] %v\"}\n\tevRunOk = event{3, \"\\r[✓]\"}\n\tevRunFail = event{4, \"\\r[✗]\"}\n)\n\nconst spinners = `/-\\|`\n\nconst (\n\tcolorReset = \"\\033[0m\"\n\n\tcolorRed = \"\\033[31m\"\n\t// colorGreen = \"\\033[32m\"\n\tcolorYellow = \"\\033[33m\"\n\tcolorBlue = \"\\033[34m\"\n\t// colorPurple = \"\\033[35m\"\n\tcolorCyan = \"\\033[36m\"\n\t// colorWhite = \"\\033[37m\"\n)\n\ntype Progress struct {\n\tout io.Writer // destination for output, usually os.Stderr\n\trunning []byte\n\tseq int // sequence of spinners\n\tdebug bool\n}\n\nvar p *Progress\n\nfunc init() {\n\tp = &Progress{out: os.Stderr, debug: false}\n}\n\nfunc SetDebug(on bool) {\n\tp.debug = on\n}\n\nfunc Cursor(show bool) {\n\tif p.out != os.Stdout && p.out != os.Stderr {\n\t\treturn\n\t}\n\tif show {\n\t\toutput([]byte(\"\\033[?25h\")) // Show cursor\n\t} else {\n\t\toutput([]byte(\"\\033[?25l\")) // Hide cursor\n\t}\n}\n\nfunc Status(format string, a ...interface{}) {\n\tif len(p.running) > 0 {\n\t\tclearLine()\n\t\toutput([]byte(colorCyan))\n\t}\n\n\toutputln(\"[>] \" + fmt.Sprintf(format, a...))\n\n\tif len(p.running) > 0 {\n\t\toutput([]byte(colorReset))\n\t\toutput(p.running)\n\t}\n}\n\nfunc Error(err error) {\n\tif len(p.running) > 0 {\n\t\tclearLine()\n\t}\n\n\toutput([]byte(colorRed))\n\toutputln(fmt.Sprintf(evError.format, err))\n\toutput([]byte(colorReset))\n\n\tif len(p.running) > 0 {\n\t\toutput(p.running)\n\t}\n\n}\n\nfunc ErrorMsg(format string, a ...interface{}) {\n\tif len(p.running) > 0 {\n\t\tclearLine()\n\t}\n\n\toutput([]byte(colorRed))\n\toutputln(\"[✗] \" + fmt.Sprintf(format, a...))\n\toutput([]byte(colorReset))\n\n\tif len(p.running) > 0 {\n\t\toutput(p.running)\n\t}\n\n}\n\nfunc Warning(format string, a ...interface{}) {\n\tif len(p.running) > 0 {\n\t\tclearLine()\n\t}\n\n\toutput([]byte(colorYellow))\n\toutputln(\"[!] \" + fmt.Sprintf(format, a...))\n\toutput([]byte(colorReset))\n\n\tif len(p.running) > 0 {\n\t\toutput(p.running)\n\t}\n\n}\n\nfunc Debug(format string, a ...interface{}) {\n\tif !p.debug {\n\t\treturn\n\t}\n\tif len(p.running) > 0 {\n\t\tclearLine()\n\t}\n\n\toutput([]byte(colorBlue))\n\toutputln(\"--- \" + fmt.Sprintf(format, a...))\n\toutput([]byte(colorReset))\n\n\tif len(p.running) > 0 {\n\t\toutput(p.running)\n\t}\n}\n\nfunc Running(msg string) {\n\tp.running = []byte(fmt.Sprintf(evRunning.format, msg))\n\toutput(p.running)\n}\n\nfunc Spinner() {\n\toutput([]byte{'\\r', '[', spinners[p.seq], ']'})\n\tp.seq = (p.seq + 1) % len(spinners)\n}\n\nfunc RunOK() {\n\toutputln(evRunOk.format)\n\tp.running = p.running[:0]\n}\n\nfunc RunFail() {\n\toutput([]byte(colorRed))\n\n\tif len(p.running) > 0 {\n\t\tclearLine()\n\t\toutput(p.running)\n\t}\n\n\toutputln(evRunFail.format)\n\tp.running = p.running[:0]\n\n\toutput([]byte(colorReset))\n}\n\nfunc Download(a string) {\n\toutput([]byte(\"[ ] \" + a))\n}\n\n/* ------ static ---------\nfunc Status(format string, a ...interface{}) {\n\t_progress.Status(format, a...)\n}\n\nfunc Warning(format string, a ...interface{}) {\n\t_progress.Warning(format, a...)\n}\n\nfunc Error(err error) {\n\t_progress.Error(err)\n}\n\nfunc ErrorMsg(format string, a ...interface{}) {\n\t_progress.ErrorMsg(format, a...)\n}\n\nfunc Download(a string) {\n\t_progress.Download(a)\n}\n\n/* ------- output ------- */\n\nfunc clearLine() {\n\tif len(p.running) == 0 {\n\t\treturn\n\t}\n\tbuf := bytes.Repeat([]byte(\" \"), len(p.running)+2)\n\tbuf[0] = byte('\\r')\n\tbuf[len(buf)-1] = byte('\\r')\n\t_, _ = p.out.Write(buf)\n}\n\nfunc output(buf []byte) {\n\t_, _ = p.out.Write(buf)\n}\n\nfunc outputln(s string) {\n\tif p.out == nil {\n\t\treturn\n\t}\n\tvar buf []byte\n\tbuf = append(buf, s...)\n\tif len(s) == 0 || s[len(s)-1] != '\\n' {\n\t\tbuf = append(buf, '\\n')\n\t}\n\toutput(buf)\n}\n" }, { "path": "reports/db.go", "content": "package reports\n\nimport (\n\t\"database/sql\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/dude333/rapina\"\n\t\"github.com/dude333/rapina/parsers\"\n\t\"github.com/pkg/errors\"\n)\n\ntype accItems struct {\n\tcode uint32\n\tcdConta string\n\tdsConta string\n}\n\ntype AccountValue struct {\n\taccItem accItems\n\tvalue float32\n\tyear int\n}\n\n//\n// accountsItems returns all accounts codes and descriptions, e.g.:\n// [1 Ativo Total, 1.01 Ativo Circulante, ...]\n//\nfunc (r Report) accountsItems(cid int) (items []accItems, err error) {\n\tselectItems := fmt.Sprintf(`\n\tSELECT DISTINCT\n\t\tCODE, CD_CONTA, DS_CONTA\n\tFROM\n\t\tdfp a\n\tWHERE\n\t\tID_CIA = \"%d\"\n\t\tAND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR)\n\tORDER BY\n\t\tCD_CONTA, DS_CONTA\n\t;`, cid)\n\n\trows, err := r.db.Query(selectItems)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tdefer rows.Close()\n\n\tvar item accItems\n\tfor rows.Next() {\n\t\terr = rows.Scan(&item.code, &item.cdConta, &item.dsConta)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\titems = append(items, item)\n\t}\n\n\treturn\n}\n\n//\n// accountsValues stores the values for each account into a map using a hash\n// of the account code and description as its key\n//\nfunc (r Report) accountsValues(year int) (map[uint32]float32, error) {\n\tvalues := make(map[uint32]float32)\n\n\tlastYear, isITR, err := r.lastYear(r.cid)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif year == lastYear && isITR {\n\t\terr = r.ttm(r.cid, values)\n\t} else {\n\t\terr = r.dfp(r.cid, year, values)\n\t}\n\tif err != nil {\n\t\treturn values, err\n\t}\n\t// Stop if year has empty values\n\tif sum(values) == 0 {\n\t\treturn values, nil\n\t}\n\n\t// Financial scale\n\ttable := \"dfp\"\n\tif isITR {\n\t\ttable = \"itr\"\n\t}\n\tvalues[parsers.Escala] = r.scale(r.cid, year, table)\n\n\t// Shares and free float\n\t_ = r.shares(r.cid, year, values)\n\n\tvar v float32\n\t// Inventory average\n\tv, err = r.value(r.cid, year-1, parsers.Estoque)\n\tif err == nil {\n\t\tvalues[parsers.EstoqueMedio] = avg(values[parsers.Estoque], v)\n\t}\n\t// Equity average\n\tv, err = r.value(r.cid, year-1, parsers.Equity)\n\tif err == nil {\n\t\tvalues[parsers.EquityAvg] = avg(values[parsers.Equity], v)\n\t}\n\n\t// Stock code\n\tif r.code != \"\" {\n\t\tdate := rapina.LastBusinessDayOfYear(year)\n\t\tq, err := r.fetchStock.Quote(r.code, date)\n\t\tif err == nil {\n\t\t\tvalues[parsers.Quote] = float32(q)\n\t\t}\n\t}\n\n\treturn values, nil\n}\n\n// avg returns the average, ignoring numbers <= 0.\nfunc avg(nums ...float32) float32 {\n\tvar total float32 = 0\n\tvar n float32 = 0\n\tfor _, num := range nums {\n\t\tif num > 0 {\n\t\t\ttotal += num\n\t\t\tn++\n\t\t}\n\t}\n\tif n <= 0 {\n\t\treturn 0\n\t}\n\treturn total / n\n}\n\n//\n// lastYear considers the current year as the latest year recorded on the DB.\n// Returns this lastest year, if it's to use the ITR table (instead of the DFP),\n// and the error, if any.\n//\nfunc (r Report) lastYear(cid int) (int, bool, error) {\n\tif cid == 0 {\n\t\treturn 0, false, fmt.Errorf(\"customer ID not set\")\n\t}\n\n\tnumErr := 0\n\n\tselectDfpLastYear := `SELECT MAX(CAST(YEAR AS INTEGER)) YEAR FROM dfp WHERE ID_CIA = ?;`\n\tdfp := 0\n\terr := r.db.QueryRow(selectDfpLastYear, cid).Scan(&dfp)\n\tif err != nil {\n\t\tnumErr++\n\t}\n\n\tselectItrLastYear := `SELECT MAX(CAST(YEAR AS INTEGER)) YEAR FROM itr WHERE ID_CIA = ?;`\n\titr := 0\n\terr = r.db.QueryRow(selectItrLastYear, cid).Scan(&itr)\n\tif err != nil {\n\t\tnumErr++\n\t}\n\n\tif numErr == 2 {\n\t\treturn 0, false, sql.ErrNoRows\n\t}\n\n\tif itr > dfp {\n\t\treturn itr, true, nil // Use ITR\n\t}\n\n\treturn dfp, false, nil // Use DFP\n}\n\n//\n// LastYearRange returns the 1st and last day from last year stored on the DB\n// for this company id. Return dates in unix epoch format.\n//\nfunc (r Report) LastYearRange(cid int) (int, int, error) {\n\tif cid == 0 {\n\t\treturn 0, 0, fmt.Errorf(\"customer ID not set\")\n\t}\n\n\ts := `\n\t\tSELECT DISTINCT DT_FIM_EXERC\n\t\tFROM dfp\n\t\tWHERE ID_CIA = ?\n\t\tORDER BY DT_FIM_EXERC DESC\n\t\tLIMIT 2;\n `\n\trows, err := r.db.Query(s, cid)\n\tif err != nil {\n\t\treturn 0, 0, err\n\t}\n\tdefer rows.Close()\n\n\tvar dateRange [2]int // [0] = last day, [1] = first day\n\ti := 0\n\n\tfor rows.Next() {\n\t\t_ = rows.Scan(&dateRange[i])\n\t\ti++\n\t\tif i >= len(dateRange) {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tfor _, v := range dateRange {\n\t\tif v == 0 {\n\t\t\treturn 0, 0, fmt.Errorf(\"range not found\")\n\t\t}\n\t}\n\n\t// return the first and last day of the year\n\treturn dateRange[1], dateRange[0], nil\n}\n\nfunc (r Report) dfp(cid, year int, _values map[uint32]float32) error {\n\tselectReport := `\n\tSELECT\n\t\tCODE, VL_CONTA\n\tFROM\n\t\tdfp a\n\tWHERE\n\t\tID_CIA = $1 \n\t\tAND YEAR = $2\n\t\tAND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR)\n\t;`\n\n\trows, err := r.db.Query(selectReport, cid, year)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer rows.Close()\n\n\tfor rows.Next() {\n\t\tvar code uint32\n\t\tvar vlConta float32\n\t\terr := rows.Scan(&code, &vlConta)\n\t\tif err == nil {\n\t\t\t_values[code] = vlConta\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (r Report) RawAccounts(cid, year int) ([]AccountValue, error) {\n\tselectReport := `\n\tSELECT\n\t\tCD_CONTA, DS_CONTA, CODE, YEAR, VL_CONTA\n\tFROM\n\t\tdfp a\n\tWHERE\n\t\tID_CIA = $1 \n\t\tAND YEAR = $2\n\t\tAND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR)\n\t;`\n\n\tvalues := make([]AccountValue, 0, 10)\n\n\trows, err := r.db.Query(selectReport, cid, year)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer rows.Close()\n\n\tfor rows.Next() {\n\t\tvar codeConta string\n\t\tvar descConta string\n\t\tvar code uint32\n\t\tvar vlConta float32\n\t\tvar year int\n\n\t\terr := rows.Scan(&codeConta, &descConta, &code, &year, &vlConta)\n\n\t\tav := AccountValue{\n\t\t\taccItem: accItems{\n\t\t\t\tcode: code,\n\t\t\t\tcdConta: codeConta,\n\t\t\t\tdsConta: descConta,\n\t\t\t},\n\t\t\tyear: year,\n\t\t\tvalue: vlConta,\n\t\t}\n\n\t\tif err == nil {\n\t\t\tvalues = append(values, av)\n\t\t}\n\t}\n\n\treturn values, nil\n}\n\nfunc (r Report) lastDate(cid int) (int, string, error) {\n\trowDfp := r.db.QueryRow(\"SELECT MAX(DT_FIM_EXERC) FROM dfp WHERE ID_CIA = ? LIMIT 1;\", cid)\n\tmaxDfp := 0\n\terr := rowDfp.Scan(&maxDfp)\n\tif err != nil {\n\t\treturn 0, \"\", err\n\t}\n\n\trowItr := r.db.QueryRow(\"SELECT MAX(DT_FIM_EXERC) FROM itr WHERE ID_CIA = ? LIMIT 1;\", cid)\n\tmaxItr := 0\n\terr = rowItr.Scan(&maxItr)\n\tif err != nil {\n\t\treturn 0, \"\", err\n\t}\n\n\tif maxDfp >= maxItr {\n\t\treturn maxDfp, \"dfp\", nil\n\t}\n\n\treturn maxItr, \"itr\", nil\n}\n\n//\n// lastBalance returns a hash with the '[code] = value' from the balance sheet\n// with the newest date available on the dfp or itr tables.\n//\nfunc (r Report) lastBalance(cid int) (map[uint32]float32, error) {\n\td, table, err := r.lastDate(cid)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif table != \"dfp\" && table != \"itr\" {\n\t\treturn nil, fmt.Errorf(\"table %s is not allowed\", table)\n\t}\n\n\tselectBalance := fmt.Sprintf(`\n\t\tSELECT \n\t\t\tdate(DT_FIM_EXERC, 'unixepoch') DT, CODE, SUM(VL_CONTA) TOTAL\n\t\tFROM %s t\n\t\tWHERE \n\t\t\tID_CIA = $1\n\t\t\tAND DT_FIM_EXERC = $2\n\t\t\tAND VERSAO = (SELECT MAX(VERSAO) FROM %s WHERE ID_CIA = t.ID_CIA AND DT_FIM_EXERC = t.DT_FIM_EXERC)\n\t\t\tAND CAST(substr(CD_CONTA, 1, 1) as decimal) <= 2\n\t\tGROUP BY\n\t\t\tDT_FIM_EXERC, CODE, CD_CONTA;\n\t`, table, table)\n\n\trows, err := r.db.Query(selectBalance, cid, d)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer rows.Close()\n\n\tbalance := make(map[uint32]float32)\n\tvar maxDt string\n\tvar code uint32\n\tvar vlConta float32\n\n\tfor rows.Next() {\n\t\terr := rows.Scan(&maxDt, &code, &vlConta)\n\t\tif err == nil {\n\t\t\tbalance[code] = vlConta\n\t\t}\n\t}\n\n\treturn balance, nil\n}\n\n//\n// ttm (twelve trailling months) retrieves the 4 quarters from\n// last year, the quarters from the current year and sums up the last 4\n// quarters for every account, returning a map with '[account_code] = value'.\n//\nfunc (r Report) ttm(cid int, _values map[uint32]float32) error {\n\tlastYear, err := r.lastDFPYear(cid)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tselectQuarters := `SELECT CODE, sum(VAL) FROM (\n\t-- Last quarter from last year\n\tSELECT CODE, sum(VAL) VAL FROM (\n\t\tSELECT CODE, VL_CONTA VAL -- Year total\n\t\tFROM dfp d \n\t\tWHERE \n\t\t\tID_CIA = $1\n\t\t\tAND YEAR = $2\n\t\t\tAND VERSAO = (SELECT max(VERSAO) FROM dfp WHERE ID_CIA = d.ID_CIA AND YEAR = d.YEAR)\t\n\t\t\tAND CAST(substr(CD_CONTA, 1, 1) as decimal) > 2 -- IGNORE BALANCE SHEETS\t\n\n\t\tUNION\n\n\t\tSELECT CODE, -1*VL_CONTA VAL -- Minus 3 semesters\n\t\tFROM itr i\n\t\tWHERE \n\t\t\tID_CIA = $1\n\t\t\tAND YEAR <= $2\n\t\t\tAND CAST(substr(CD_CONTA, 1, 1) as decimal) > 2 -- IGNORE BALANCE SHEETS\t\n\t\t\tAND VERSAO = (SELECT MAX(VERSAO) FROM itr WHERE ID_CIA = i.ID_CIA AND CODE = i.CODE AND DT_FIM_EXERC = i.DT_FIM_EXERC)\n\t\t\tAND ID IN (SELECT ID FROM itr WHERE ID_CIA = i.ID_CIA AND CODE = i.CODE AND DT_FIM_EXERC = i.DT_FIM_EXERC ORDER BY DT_FIM_EXERC desc LIMIT 3)\n\t)\n\tGROUP BY CODE\n\n\tUNION\n\n\t-- Last 3 quarters\n\tSELECT CODE, sum(VL_CONTA) VAL\n\tFROM itr i\n\tWHERE \n\t\tID_CIA = $1\n\t\tAND CAST(substr(CD_CONTA, 1, 1) as decimal) > 2 -- IGNORE BALANCE SHEETS\t\n\t\tAND VERSAO = (SELECT MAX(VERSAO) FROM itr WHERE ID_CIA = i.ID_CIA AND CODE = i.CODE AND DT_FIM_EXERC = i.DT_FIM_EXERC)\n\t\tAND ID IN (SELECT ID FROM itr WHERE ID_CIA = i.ID_CIA AND CODE = i.CODE ORDER BY DT_FIM_EXERC desc LIMIT 3)\n\tGROUP BY CODE\n)\nGROUP BY CODE\t\nORDER BY CODE;`\n\n\trows, err := r.db.Query(selectQuarters, cid, lastYear)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer rows.Close()\n\n\tvar code uint32\n\tvar vlConta float32\n\tfor rows.Next() {\n\t\terr := rows.Scan(&code, &vlConta)\n\t\tif err == nil {\n\t\t\t_values[code] = vlConta\n\t\t}\n\t}\n\n\tbal, err := r.lastBalance(cid)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor k, v := range bal {\n\t\t_values[k] = v\n\t}\n\n\treturn nil\n}\n\nfunc (r Report) lastDFPYear(cid int) (int, error) {\n\tif cid == 0 {\n\t\treturn 0, fmt.Errorf(\"customer ID not set\")\n\t}\n\n\ts := `SELECT MAX(YEAR) FROM dfp\tWHERE ID_CIA = ?;`\n\trow := r.db.QueryRow(s, cid)\n\tvar lastDate int\n\terr := row.Scan(&lastDate)\n\n\treturn lastDate, err\n}\n\n//\n// shares set the 'values' map with the number of shares and the free float of\n// a given conpany in a given year.\n//\nfunc (r Report) shares(cid int, year int, values map[uint32]float32) error {\n\tselectFRE := `\n\t\tSELECT \n\t\t\tQuantidade_Total_Acoes_Circulacao, Percentual_Total_Acoes_Circulacao\n\t\tFROM fre f\n\t\tWHERE\n\t\t\tID_CIA = $1\n\t\t\tAND YEAR = $2\n\t\t\tAND Versao = (SELECT MAX(Versao) FROM fre WHERE ID_CIA = f.ID_CIA AND YEAR = f.YEAR);\n\t`\n\n\trow := r.db.QueryRow(selectFRE, cid, year)\n\tvar shares float32\n\tvar freeFloat float32\n\terr := row.Scan(&shares, &freeFloat)\n\tif err != nil && err != sql.ErrNoRows {\n\t\treturn err\n\t}\n\n\tvalues[parsers.Shares] = shares\n\tvalues[parsers.FreeFloat] = freeFloat\n\n\treturn nil\n}\n\n//\n// sharesAvg set the 'values' map with the average number of shares and\n// the free float of a given conpany in a given year.\n//\nfunc (r Report) sharesAvg(cids []string, year int, values map[uint32]float32) error {\n\tselectFRE := fmt.Sprintf(`\n\t\tSELECT \n\t\t\tAVG(Quantidade_Total_Acoes_Circulacao), AVG(Percentual_Total_Acoes_Circulacao)\n\t\tFROM fre f\n\t\tWHERE\n\t\t\tID_CIA IN (%s)\n\t\t\tAND YEAR = $1\n\t\t\tAND Versao = (SELECT MAX(Versao) FROM fre WHERE ID_CIA = f.ID_CIA AND YEAR = f.YEAR);\n\t`, strings.Join(cids, \",\"))\n\n\trow := r.db.QueryRow(selectFRE, year)\n\tvar sharesAvg float32\n\tvar freeFloatAvg float32\n\terr := row.Scan(&sharesAvg, &freeFloatAvg)\n\tif err != nil && err != sql.ErrNoRows {\n\t\treturn err\n\t}\n\n\tvalues[parsers.Shares] = sharesAvg\n\tvalues[parsers.FreeFloat] = freeFloatAvg\n\n\treturn nil\n}\n\n//\n// value returns the account value for company id 'cid', 'year' and code.\n//\nfunc (r Report) value(cid, year int, code uint32) (float32, error) {\n\tselectInventory := `\n\tSELECT\n\t\tVL_CONTA\n\tFROM\n\t\tdfp a\n\tWHERE\n\t\tID_CIA = $1 \n\t\tAND YEAR = $2\n\t\tAND CODE = $3\n\t\tAND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR)\n\t;`\n\n\tval := float32(0)\n\terr := r.db.QueryRow(selectInventory, cid, year, code).Scan(&val)\n\tif err != nil && err != sql.ErrNoRows {\n\t\treturn 0, err\n\t}\n\n\treturn val, nil\n}\n\n//\n// accountsAverage stores the average of all companies of the same sector\n// for each account into a map using a hash of the account code and\n// description as its key\n//\nfunc (r Report) accountsAverage(company string, year int) (map[uint32]float32, error) {\n\n\tcompanies, _, err := r.fromSector(company)\n\tif len(companies) <= 1 || err != nil {\n\t\terr = errors.Wrap(err, \"erro ao ler arquivo de setores \"+r.yamlFile)\n\t\treturn nil, err\n\t}\n\n\tif len(companies) == 0 {\n\t\terr = errors.Errorf(\"erro ao procurar empresas\")\n\t\treturn nil, err\n\t}\n\n\tcids := make([]string, len(companies))\n\tfor i, co := range companies {\n\t\tif id, err := r.getCid(co); err == nil {\n\t\t\tcids[i] = strconv.Itoa(id)\n\t\t}\n\t}\n\n\tselectReport := fmt.Sprintf(`\n\tSELECT\n\t\tCODE, AVG(VL_CONTA)\n\tFROM\n\t\tdfp a\n\tWHERE\n\t\tID_CIA IN (%s)\n\t\tAND YEAR = \"%d\"\n\t\tAND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR)\n\tGROUP BY\n\t\tCODE;\n\t`, strings.Join(cids, \",\"), year)\n\n\trows, err := r.db.Query(selectReport)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer rows.Close()\n\n\tvalues := make(map[uint32]float32)\n\tfor rows.Next() {\n\t\tvar code uint32\n\t\tvar vlConta float32\n\t\terr := rows.Scan(\n\t\t\t&code,\n\t\t\t&vlConta,\n\t\t)\n\t\tif err == nil {\n\t\t\tvalues[code] = vlConta\n\t\t}\n\t}\n\n\tvar err1, err2 error\n\tvalues[parsers.EstoqueMedio], err1 = r.movingAvg(cids, year, parsers.Estoque)\n\tvalues[parsers.EquityAvg], err2 = r.movingAvg(cids, year, parsers.Equity)\n\n\tif err1 == nil && err2 == nil {\n\t\t_ = r.sharesAvg(cids, year, values)\n\t}\n\n\treturn values, nil\n}\n\n// movingAvg returns the moving average of account 'code' between year and\n// last year for all companies listed on 'cids'.\nfunc (r Report) movingAvg(cids []string, year int, code uint32) (float32, error) {\n\ts := fmt.Sprintf(`\n\t\tSELECT \n\t\t\t(SELECT AVG(VL_CONTA) FROM dfp d2 \n\t\t\tWHERE d1.ID_CIA = d2.ID_CIA AND d2.CODE = d1.CODE \n\t\t\tAND d2.YEAR >= (d1.YEAR - 1) AND d2.YEAR <= d1.YEAR\n\t\t\tAND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = d2.ID_CIA AND YEAR = d2.YEAR)\n\t\t\t) AS MAVG\n\t\tFROM dfp d1\n\t\tWHERE ID_CIA IN (%s) \n\t\tAND YEAR = $1\n\t\tAND CODE = $2\n\t\tAND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = d1.ID_CIA AND YEAR = d1.YEAR)\n\t\tGROUP BY YEAR; \n\t`, strings.Join(cids, \",\"))\n\n\tmavg := float32(0)\n\n\terr := r.db.QueryRow(s, year, code).Scan(&mavg)\n\tif err != nil && err != sql.ErrNoRows {\n\t\treturn 0, err\n\t}\n\n\treturn mavg, nil\n}\n\nfunc (r Report) fromSector(company string) (companies []string, sectorName string, err error) {\n\t// Companies from the same sector\n\tsecCo, secName, err := parsers.FromSector(company, r.yamlFile)\n\tif len(secCo) <= 1 || err != nil {\n\t\terr = errors.Wrap(err, \"erro ao ler arquivo dos setores \"+r.yamlFile)\n\t\treturn\n\t}\n\n\t// All companies stored on db\n\tlist, err := ListCompanies(r.db)\n\tif err != nil {\n\t\terr = errors.Wrap(err, \"erro ao listar empresas\")\n\t\treturn\n\t}\n\n\t// Translate company names to match the name stored on db\n\tfor _, s := range secCo {\n\t\tz := parsers.FuzzyFind(s, list, 3)\n\t\tif len(z) > 0 {\n\t\t\tcompanies = append(companies, z)\n\t\t}\n\t}\n\n\treturn removeDuplicates(companies), secName, nil\n}\n\n// CompanyInfo contains the company name and CNPJ\ntype CompanyInfo struct {\n\tid int\n\tname string\n}\n\n//\n// companies returns available companies in the DB\n//\nfunc companies(db *sql.DB) ([]CompanyInfo, error) {\n\n\tselectCompanies := `\n\t\tSELECT ID, NAME\n\t\tFROM companies\n\t\tORDER BY NAME;`\n\n\trows, err := db.Query(selectCompanies)\n\tif err != nil {\n\t\terr = errors.Wrap(err, \"falha ao ler banco de dados\")\n\t\treturn nil, err\n\t}\n\tdefer rows.Close()\n\n\tvar info CompanyInfo\n\tvar list []CompanyInfo\n\tfor rows.Next() {\n\t\terr := rows.Scan(&info.id, &info.name)\n\t\tif err == nil {\n\t\t\tlist = append(list, info)\n\t\t}\n\t}\n\n\treturn list, nil\n}\n\n// TickerInfo contains the ticker name and SpcfctnCd\ntype TickerInfo struct {\n\tname \t\tstring\n\tSpcfctnCd \tstring\n}\n\n//\n// tickers returns available tickers for a company name in the DB\n//\nfunc tickers(db *sql.DB, companyName string) ([]TickerInfo, error) {\n\n\tselectTickers := `\n\t\tSELECT trading_code, SpcfctnCd\n\t\tFROM stock_codes\n\t\tWHERE company_name LIKE ?\n\t\tORDER BY trading_code;`\n\n\trows, err := db.Query(selectTickers, \"%\"+companyName+\"%\")\n\tif err != nil {\n\t\terr = errors.Wrap(err, \"falha ao ler banco de dados\")\n\t\treturn nil, err\n\t}\n\tdefer rows.Close()\n\n\tvar info TickerInfo\n\tvar list []TickerInfo\n\tfor rows.Next() {\n\t\terr := rows.Scan(&info.name, &info.SpcfctnCd)\n\t\tif err == nil {\n\t\t\tlist = append(list, info)\n\t\t}\n\t}\n\n\treturn list, nil\n}\n\n//\n// setCompany sets the company ID, CNPJ and stock code based on it's name...\n//\n// func (r *Report) setCompany(company string) error {\n// \treturn r.setCompanyAndTicker(company,\"ON\")\n// }\n\n//\n// setCompanyAndTicker sets the company ID, CNPJ and stock code based on it's name.\n//\nfunc (r *Report) setCompanyAndTicker(company string, spcfctnCd string) error {\n\tif company == \"\" {\n\t\treturn errors.New(\"company name not set\")\n\t}\n\tif r.fetchStock == nil {\n\t\treturn errors.New(\"fetchStock not set\")\n\t}\n\n\t// Reset company data\n\tr.cid = 0\n\tr.cnpj = \"\"\n\tr.code = \"\"\n\n\tquery := `SELECT DISTINCT ID, NAME, CNPJ FROM companies WHERE NAME LIKE ?`\n\tvar cid int\n\tvar name, cnpj string\n\terr := r.db.QueryRow(query, \"%\"+company+\"%\").Scan(&cid, &name, &cnpj)\n\tif err != nil {\n\t\treturn err\n\t}\n\tr.cid = cid\n\tr.company = name // reset company name to match the name stored on db\n\tr.cnpj = cnpj\n\n\t// Stock code\n\tr.code, err = r.fetchStock.Code(r.company, spcfctnCd)\n\tif err != nil {\n\t\tfmt.Printf(\"\\n[x] Erro obtendo código negociação: %v\\n\", err)\n\t}\n\n\treturn nil\n}\n\nfunc (r *Report) getCid(companyName string) (int, error) {\n\tselectID := `SELECT DISTINCT ID FROM companies WHERE NAME LIKE ?`\n\tvar cid int\n\terr := r.db.QueryRow(selectID, \"%\"+companyName+\"%\").Scan(&cid)\n\n\treturn cid, err\n}\n\n//\n// scale returns the financial scale used on the values (unit or thousands).\n//\nfunc (r Report) scale(cid, year int, table string) float32 {\n\ts := fmt.Sprintf(\n\t\t`SELECT ESCALA_MOEDA FROM %s WHERE ID_CIA = $1 AND YEAR = $2 limit 1;`,\n\t\ttable,\n\t)\n\tvar scale string\n\terr := r.db.QueryRow(s, cid, year).Scan(&scale)\n\tif err != nil {\n\t\treturn 1000\n\t}\n\n\tswitch scale {\n\tcase \"UNIDADE\":\n\t\treturn 1\n\tcase \"MIL\":\n\t\treturn 1000\n\tcase \"MILHAO\":\n\t\treturn 1000000\n\t}\n\n\treturn 1000\n}\n\n//\n// timeRange returns the begin=min(year) and end=max(year)\n//\nfunc timeRange(db *sql.DB) (int, int, error) {\n\n\tselectYears := `\n\tSELECT\n\t\tMIN(CAST(YEAR AS INTEGER)),\n\t\tMAX(CAST(YEAR AS INTEGER))\n\tFROM dfp;`\n\tbegin := 0\n\tend := 0\n\terr := db.QueryRow(selectYears).Scan(&begin, &end)\n\tif err != nil {\n\t\treturn 0, 0, err\n\t}\n\n\tselectItrYears := `\n\tSELECT\n\t\tMAX(CAST(YEAR AS INTEGER))\n\tFROM itr;`\n\tend2 := 0\n\terr = db.QueryRow(selectItrYears).Scan(&end2)\n\tif err == nil && end2 > end {\n\t\tend = end2\n\t}\n\n\t// Check year\n\tif begin < 1900 || begin > 2100 || end < 1900 || end > 2100 {\n\t\terr = errors.Wrap(err, \"ano inválido\")\n\t\treturn 0, 0, err\n\t}\n\tif begin > end {\n\t\taux := end\n\t\tend = begin\n\t\tbegin = aux\n\t}\n\n\treturn begin, end, nil\n}\n\nfunc removeDuplicates(elements []string) []string { // change string to int here if required\n\t// Use map to record duplicates as we find them.\n\tencountered := map[string]bool{} // change string to int here if required\n\tresult := []string{} // change string to int here if required\n\n\tfor v := range elements {\n\t\tif encountered[elements[v]] {\n\t\t\t// Do not add duplicate.\n\t\t} else {\n\t\t\t// Record this element as an encountered element.\n\t\t\tencountered[elements[v]] = true\n\t\t\t// Append to result slice.\n\t\t\tresult = append(result, elements[v])\n\t\t}\n\t}\n\t// Return the new slice.\n\treturn result\n}\n\ntype profit struct {\n\tyear int\n\tprofit float32\n}\n\nfunc companyProfits(db *sql.DB, companyID int) ([]profit, error) {\n\n\tselectProfits := fmt.Sprintf(`\n\tSELECT\n\t\tYEAR,\n\t\tVL_CONTA\n\tFROM\n\t\tdfp a\n\tWHERE\n\t\tID_CIA = \"%d\"\n\t\tAND CODE = \"%d\"\n\t\tAND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR)\n\tORDER BY\n\t\tYEAR;`, companyID, parsers.LucLiq)\n\n\trows, err := db.Query(selectProfits)\n\tif err != nil {\n\t\terr = errors.Wrap(err, \"falha ao ler banco de dados\")\n\t\treturn nil, err\n\t}\n\tdefer rows.Close()\n\n\tvar profits []profit\n\tfor rows.Next() {\n\t\tvar year int\n\t\tvar val float32\n\t\terr := rows.Scan(&year, &val)\n\t\tif err == nil {\n\t\t\tprofits = append(profits, profit{year, val})\n\t\t}\n\t}\n\n\treturn profits, nil\n}\n" }, { "path": "reports/db_test.go", "content": "package reports\n\nimport \"testing\"\n\nfunc Test_avg(t *testing.T) {\n\ttype args struct {\n\t\tnums []float32\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twant float32\n\t}{\n\t\t{\"average 2\", args{[]float32{1, 2, 3}}, 2},\n\t\t{\"average 10\", args{[]float32{10, 2, 18}}, 10},\n\t\t{\"average 12.705\", args{[]float32{6, 20.4, 18.1, 6.32}}, 12.705},\n\t\t{\"average 5.5\", args{[]float32{5.5, 0}}, 5.5},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif got := avg(tt.args.nums...); got != tt.want {\n\t\t\t\tt.Errorf(\"avg() = %v, want %v\", got, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "reports/excel.go", "content": "package reports\n\nimport (\n\t\"encoding/json\"\n\t\"strconv\"\n\n\t\"github.com/360EntSecGroup-Skylar/excelize\"\n\t\"github.com/pkg/errors\"\n)\n\n// Excel instance reachable data\ntype Excel struct {\n\txlsx *excelize.File\n}\n\n//\n// newExcel creates a new Excel instance\n//\nfunc newExcel() (e *Excel) {\n\te = &Excel{}\n\te.xlsx = excelize.NewFile()\n\treturn\n}\n\n//\n// saveAndCloseExcel saves to filename (need to set the directory as well)\n//\nfunc (e *Excel) saveAndCloseExcel(filename string) (err error) {\n\t// newFilename = time.Now().Format(\"02Jan06_150405.000\") + \".xlsx\" // DDMMMYY\n\te.xlsx.DeleteSheet(\"Sheet1\")\n\te.xlsx.SetActiveSheet(1)\n\terr = e.xlsx.SaveAs(filename)\n\tif err != nil {\n\t\treturn errors.Wrapf(err, \"erro ao salvar planilha\")\n\t}\n\treturn\n}\n\n// Sheet struct\ntype Sheet struct {\n\txlsx *excelize.File\n\tname string\n}\n\nfunc (e *Excel) newSheet(name string) (s *Sheet, err error) {\n\ts = &Sheet{}\n\ts.name = name\n\ts.xlsx = e.xlsx\n\n\t// Create a new sheet.\n\t// Avoid duplicated sheet\n\tif index := e.xlsx.GetSheetIndex(name); index > 0 {\n\t\treturn nil, errors.Wrapf(err, \"erro ao criar planilha %s\", name)\n\t}\n\n\te.xlsx.NewSheet(name)\n\n\treturn\n}\n\nfunc (s Sheet) printCell(row, col int, value interface{}, styleID int) {\n\t// Print value\n\tcell := axis(col, row)\n\ts.xlsx.SetCellValue(s.name, cell, value)\n\n\t// Format cell\n\ts.xlsx.SetCellStyle(s.name, cell, cell, styleID)\n}\n\n//\n// printTitle prints the cols titles in Excel\n//\nfunc (s *Sheet) printTitle(cell string, title string) (err error) {\n\t// Print header\n\ts.xlsx.SetSheetRow(s.name, cell, &[]string{title})\n\n\t// Set styles\n\tstyle, err := s.xlsx.NewStyle(`{\"number_format\": 0,\"font\":{\"bold\":true},\"alignment\":{\"horizontal\":\"center\"},\"border\":[{\"type\":\"bottom\",\"color\":\"333333\",\"style\":3}]}`)\n\tif err == nil {\n\t\ts.xlsx.SetCellStyle(s.name, cell, cell, style)\n\t}\n\n\treturn\n}\n\n//\n// print cols in Excel\n//\nfunc (s *Sheet) print(startingCel string, slice *[]string, format int, bold bool) error {\n\tvar err error\n\tvar style int\n\n\t// Set styles\n\tjson, err := jsonStyle(10, format, bold)\n\tif err != nil {\n\t\treturn err\n\t}\n\tstyle, err = s.xlsx.NewStyle(string(json))\n\tif style > 0 && err == nil {\n\t\tcol, row := cell2axis(startingCel)\n\t\tcol += len(*slice)\n\t\ts.xlsx.SetCellStyle(s.name, startingCel, axis(col, row), style)\n\t}\n\n\t// Print row\n\ts.xlsx.SetSheetRow(s.name, startingCel, slice)\n\n\treturn nil\n}\n\n//\n// printValues prints cols in Excel\n// Values >0 and <= 100 will be printed as %\n//\nfunc (s *Sheet) printValue(cell string, value float32, format int, bold bool) (err error) {\n\n\ts.xlsx.SetSheetRow(s.name, cell, &[]float32{value})\n\n\t// Set styles\n\tjson, err := jsonStyle(10, format, bold)\n\tif err == nil {\n\t\tstyle, err := s.xlsx.NewStyle(string(json))\n\t\tif err == nil {\n\t\t\ts.xlsx.SetCellStyle(s.name, cell, cell, style)\n\t\t}\n\t}\n\n\treturn nil\n}\n\n//\n// printFormula\n//\nfunc (s *Sheet) printFormula(cell string, formula string, format int, bold bool) (err error) {\n\n\ts.xlsx.SetCellFormula(s.name, cell, formula)\n\n\t// Set styles\n\tjson, err := jsonStyle(9, format, bold)\n\tif err == nil {\n\t\tstyle, err := s.xlsx.NewStyle(string(json))\n\t\tif err == nil {\n\t\t\ts.xlsx.SetCellStyle(s.name, cell, cell, style)\n\t\t}\n\t}\n\n\treturn\n}\n\n//\n// jsonStyle\n//\nfunc jsonStyle(size, format int, bold bool) ([]byte, error) {\n\tm := map[string]interface{}{\n\t\t\"font\": map[string]interface{}{\"size\": size, \"bold\": bold},\n\t}\n\n\tswitch format {\n\tcase PERCENT:\n\t\tm[\"custom_number_format\"] = \"0%;-0%;- \"\n\tcase INDEX:\n\t\tm[\"custom_number_format\"] = \"0.00;-0.00;-\"\n\tcase NUMBER:\n\t\tm[\"custom_number_format\"] = \"_-* #,##0,_-;_-* (#,##0,);_-* \\\"-\\\"_-;_-@_-\"\n\tcase RIGHT:\n\t\tm[\"alignment\"] = map[string]interface{}{\"horizontal\": \"right\"}\n\t}\n\n\tj, err := json.Marshal(m)\n\treturn j, err\n}\n\n//\n// mergeCell\n//\nfunc (s *Sheet) mergeCell(a, b string) {\n\ts.xlsx.MergeCell(s.name, a, b)\n}\n\n//\n// autoWidth adjust the cols width\n//\nfunc (s *Sheet) autoWidth() {\n\tconst cols string = \"ABCDEFGHIJKLMNOPQRSTUVWXYZ\"\n\tsetColWidth := s.xlsx.SetColWidth\n\tsetColWidth(s.name, \"A\", \"A\", 16)\n\tsetColWidth(s.name, \"B\", \"B\", 48)\n\n\t// Get the space that separates the account numbers from the\n\t// vertical analysis numbers\n\tvar spaced int\n\tfor col := 2; col < len(cols); col++ {\n\t\tif len(s.xlsx.GetCellValue(s.name, axis(col, 1))) == 0 {\n\t\t\tspaced = col\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// 012345678901234567890\n\t// AB2DE5GHI => 5-2+5 = 8\n\t// AB2DEF6HIJK => 6-2+6 = 10\n\t// AB2DEFG7IJKLM => 7-2+7 = 12\n\tsetColWidth(s.name, \"C\", string(cols[spaced-1]), 9.5) // Account values\n\tsetColWidth(s.name, string(cols[spaced]), \"AC\", 4.64) // Vertical Analysis values\n}\n\nfunc (s *Sheet) setColWidth(col int, width float64) {\n\tc := excelize.ToAlphaString(col)\n\ts.xlsx.SetColWidth(s.name, c, c, width)\n}\n\n//\n// axis transforms (2, 3) into \"B3\"\n//\nfunc axis(col, row int) string {\n\treturn excelize.ToAlphaString(col) + strconv.Itoa(row)\n}\n\n//\n// cell2axis only works from A1 to Z999\n//\nfunc cell2axis(cell string) (col, row int) {\n\tcol = int(cell[0] - 'A')\n\trow, _ = strconv.Atoi(cell[1:])\n\n\treturn\n}\n\n//\n// colLetter transforms '2' into 'B'\n//\nfunc colLetter(col int) string {\n\treturn excelize.ToAlphaString(col)\n}\n" }, { "path": "reports/format.go", "content": "package reports\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/360EntSecGroup-Skylar/excelize\"\n\t\"github.com/dude333/rapina/parsers\"\n)\n\n// Global var to keep track of added styles\nvar stylesMap = make(map[uint32]int)\n\n// Used by style\nconst (\n\tDEFAULT = iota + 1\n\n\t// Number format\n\tGENERAL\n\tNUMBER\n\tINDEX\n\tPERCENT\n\n\tEMPTY\n\n\t// Text position\n\tLEFT\n\tRIGHT\n\tCENTER\n)\n\n// formatFont directly maps the styles settings of the fonts.\ntype formatFont struct {\n\tBold bool `json:\"bold\"`\n\tItalic bool `json:\"italic\"`\n\tUnderline string `json:\"underline\"`\n\tFamily string `json:\"family\"`\n\tSize int `json:\"size\"`\n\tColor string `json:\"color\"`\n}\n\ntype formatAlignment struct {\n\tHorizontal string `json:\"horizontal\"`\n\tIndent int `json:\"indent\"`\n\tJustifyLastLine bool `json:\"justify_last_line\"`\n\tReadingOrder uint64 `json:\"reading_order\"`\n\tRelativeIndent int `json:\"relative_indent\"`\n\tShrinkToFit bool `json:\"shrink_to_fit\"`\n\tTextRotation int `json:\"text_rotation\"`\n\tVertical string `json:\"vertical\"`\n\tWrapText bool `json:\"wrap_text\"`\n}\n\ntype formatBorder struct {\n\tType string `json:\"type\"`\n\tColor string `json:\"color\"`\n\tStyle int `json:\"style\"`\n}\n\ntype formatFill struct {\n\tType string `json:\"type\"`\n\tPattern int `json:\"pattern\"`\n\tColor []string `json:\"color\"`\n\tShading int `json:\"shading\"`\n}\n\n// formatStyle directly maps the styles settings of the cells.\ntype formatStyle struct {\n\tBorder []formatBorder `json:\"border\"`\n\tFill formatFill `json:\"fill\"`\n\tFont *formatFont `json:\"font\"`\n\tAlignment *formatAlignment `json:\"alignment\"`\n\tProtection *struct {\n\t\tHidden bool `json:\"hidden\"`\n\t\tLocked bool `json:\"locked\"`\n\t} `json:\"protection\"`\n\tNumFmt int `json:\"number_format\"`\n\tDecimalPlaces int `json:\"decimal_places\"`\n\tCustomNumFmt *string `json:\"custom_number_format\"`\n\tLang string `json:\"lang\"`\n\tNegRed bool `json:\"negred\"`\n}\n\n//\n// newFormat provides a struct to create style for cells\n//\nfunc newFormat(format int, position int, bold bool) (f *formatStyle) {\n\tf = &formatStyle{}\n\n\tcustom := \"\"\n\tswitch format {\n\tcase PERCENT:\n\t\tcustom = \"0%;-0%;- \"\n\tcase INDEX:\n\t\tcustom = \"0.00;-0.00;-\"\n\tcase NUMBER:\n\t\tcustom = \"_-* #,##0,_-;_-* (#,##0,);_-* \\\"-\\\"_-;_-@_-\"\n\t}\n\n\tif custom != \"\" {\n\t\tf.CustomNumFmt = &custom\n\t}\n\n\tswitch position {\n\tcase RIGHT:\n\t\tf.Alignment = &formatAlignment{Horizontal: \"right\"}\n\tcase CENTER:\n\t\tf.Alignment = &formatAlignment{Horizontal: \"center\"}\n\t}\n\n\tif bold {\n\t\tf.Font = &formatFont{Bold: true}\n\t}\n\n\treturn\n}\n\nfunc (f *formatStyle) size(s int) {\n\tf.Font = &formatFont{Size: s}\n}\n\nfunc (f formatStyle) newStyle(e *excelize.File) (style int) {\n\tj, err := json.Marshal(f)\n\n\tif err == nil {\n\t\ts := string(j)\n\t\tk := parsers.Hash(s)\n\n\t\t// Check if style already exists\n\t\tid, ok := stylesMap[k]\n\t\tif ok {\n\t\t\t// fmt.Printf(\"[i] Reusing style %d [%d]\\n\", id, k)\n\t\t\treturn id\n\t\t}\n\n\t\t// Create new style\n\t\tstyle, err = e.NewStyle(s)\n\t\tstylesMap[k] = style\n\t\tif err != nil {\n\t\t\treturn 0\n\t\t}\n\t}\n\n\t// fmt.Printf(\"[i] New style %d\\n\", style)\n\n\treturn\n}\n" }, { "path": "reports/format_test.go", "content": "package reports\n\nimport (\n\t\"testing\"\n\n\t\"github.com/360EntSecGroup-Skylar/excelize\"\n)\n\nfunc TestFormat(t *testing.T) {\n\tvar f [2]*formatStyle\n\tvar style [2]int\n\txlsx := excelize.NewFile()\n\n\tf[0] = newFormat(NUMBER, LEFT, false)\n\tf[1] = newFormat(INDEX, RIGHT, false)\n\n\tf[0].NumFmt = 10\n\tf[1].Lang = \"en-US\"\n\n\tfor i := range f {\n\t\tstyle[i] = f[i].newStyle(xlsx)\n\t\tif style[i] == 0 {\n\t\t\tt.Error(\"Expecting style > 0, received 0\")\n\t\t}\n\t}\n\n\tvar f2 [2]*formatStyle\n\tvar style2 [2]int\n\n\tf2[0] = newFormat(NUMBER, LEFT, false)\n\tf2[1] = newFormat(INDEX, RIGHT, false)\n\n\tf2[0].NumFmt = 10\n\tf2[1].Lang = \"en-US\"\n\n\tfor i := range f {\n\t\tstyle2[i] = f[i].newStyle(xlsx)\n\t\tif style2[i] != style[i] {\n\t\t\tt.Errorf(\"Expecting style == %d, received %d\", style[i], style2[i])\n\t\t}\n\t}\n}\n" }, { "path": "reports/list.go", "content": "package reports\n\nimport (\n\t\"database/sql\"\n\t\"fmt\"\n\t\"math\"\n\t\"strings\"\n\n\t\"github.com/dude333/rapina/parsers\"\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/text/collate\"\n\t\"golang.org/x/text/language\"\n\t\"golang.org/x/text/message\"\n)\n\n//\n// ListCompanies shows all available companies\n//\nfunc ListCompanies(db *sql.DB) (names []string, err error) {\n\tinfo, err := companies(db)\n\n\tif err != nil {\n\t\tfmt.Println(\"[x] Falha:\", err)\n\t\treturn\n\t}\n\n\tif len(info) == 0 {\n\t\terr = fmt.Errorf(\"lista vazia\")\n\t\treturn\n\t}\n\n\t// Extract companies names\n\tnames = make([]string, len(info))\n\tfor i, co := range info {\n\t\tnames[i] = co.name\n\t}\n\n\t// Sort accents correctly\n\tcl := collate.New(language.BrazilianPortuguese, collate.Loose)\n\tcl.SortStrings(names)\n\n\treturn\n}\n\n//\n// ListTickers shows all available tickers for a companie name\n//\nfunc ListTickers(db *sql.DB, companyName string) (names []string, err error) {\n\tinfo, err := tickers(db, companyName)\n\n\tif err != nil {\n\t\tfmt.Println(\"[x] Falha:\", err)\n\t\treturn\n\t}\n\n\tif len(info) == 0 {\n\t\terr = fmt.Errorf(\"lista vazia\")\n\t\treturn\n\t}\n\n\t// Extract companies names\n\tnames = make([]string, len(info))\n\tfor i, co := range info {\n\t\tnames[i] = co.name\n\t}\n\n\t// Sort accents correctly\n\tcl := collate.New(language.BrazilianPortuguese, collate.Loose)\n\tcl.SortStrings(names)\n\n\treturn\n}\n\n//\n// ListTickers returns SpcfctnCd of a ticker\n//\nfunc GetSpcfctnCd(db *sql.DB, companyName string, ticker string) string {\n\tinfo, err := tickers(db, companyName)\n\n\tif err != nil {\n\t\tfmt.Println(\"[x] Falha:\", err)\n\t\treturn \"\"\n\t}\n\n\tif len(info) == 0 {\n\t\tfmt.Println(\"[x] Lista vazia\")\n\t\treturn \"\"\n\t}\n\n\t// Extract companies names\n\tvar spcfctnCd string = \"\"\n\tfor _, co := range info {\n\t\tif co.name == ticker {\n\t\t\tspcfctnCd = co.SpcfctnCd\n\t\t}\n\t}\n\n\treturn spcfctnCd\n}\n\n//\n// ListSector shows all companies from the same sector as 'company'\n//\nfunc ListSector(db *sql.DB, company, yamlFile string) (err error) {\n\t// Companies from the same sector\n\tsecCo, secName, err := parsers.FromSector(company, yamlFile)\n\tif len(secCo) <= 1 || err != nil {\n\t\terr = errors.Wrap(err, \"erro ao ler arquivo dos setores \"+yamlFile)\n\t\treturn\n\t}\n\n\t// All companies stored on db\n\tlist, err := ListCompanies(db)\n\tif err != nil {\n\t\terr = errors.Wrap(err, \"erro ao listar empresas\")\n\t\treturn\n\t}\n\n\t// Translate company names to match the name stored on db\n\tfmt.Printf(\"%-40s %s\\n\", \"ARQUIVO YAML\", \"BANCO DE DADOS\")\n\tfmt.Printf(\"%-40s %s\\n\", strings.Repeat(\"-\", 40), strings.Repeat(\"-\", 40))\n\tfor _, s := range secCo {\n\t\tz := parsers.FuzzyFind(s, list, 3)\n\t\tif len(z) > 0 {\n\t\t\tfmt.Printf(\"%-40s %s\\n\", s, z)\n\t\t} else {\n\t\t\tfmt.Printf(\"%-40s %s\\n\", s, \"Nao encontrado\")\n\t\t}\n\t}\n\tfmt.Printf(\"\\nSETOR: %s\\n\", secName)\n\n\treturn\n}\n\n//\n// ListCompaniesProfits lists companies by net profit: more sustainable growth\n// listed first\n//\nfunc ListCompaniesProfits(db *sql.DB, rate float32) error {\n\n\tinfo, err := companies(db)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"falha ao obter a lista de empresas (%v)\", err)\n\t}\n\n\tyi, yf, err := timeRange(db)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"falha ao obter a faixa de datas (%v)\", err)\n\t}\n\n\t// Header\n\tvar sep string\n\tfmt.Printf(\"%20s \", \" \") // Space to match company name\n\tfor y := yi; y <= yf; y++ {\n\t\tfmt.Printf(\"%10d \", y)\n\t\tsep += fmt.Sprintf(\"%s \", strings.Repeat(\"-\", 10))\n\t}\n\tfmt.Printf(\"%10s\\n\", \"CAGR\")\n\tsep += fmt.Sprintf(\"%s \", strings.Repeat(\"-\", 10))\n\tfmt.Printf(\"%20s %s\\n\", \" \", sep)\n\n\t// Profits\n\tpt := message.NewPrinter(language.Portuguese)\n\tfor _, co := range info {\n\t\tprofits, err := companyProfits(db, co.id)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"falha ao obter lucros de %s (%v)\", co.name, err)\n\t\t}\n\n\t\t// FILTERS ----------------------------\n\t\t// At least 4 years\n\t\tif len(profits) < 4 {\n\t\t\tcontinue\n\t\t}\n\t\t// Ignore if there is no recent data\n\t\tif profits[len(profits)-1].year < yf-1 {\n\t\t\tcontinue\n\t\t}\n\n\t\tpi := profits[0].profit\n\t\tpf := profits[len(profits)-1].profit\n\n\t\tif pf < pi {\n\t\t\tcontinue\n\t\t}\n\n\t\t// Ignore if next profix < 'rate' * current profit\n\t\tprofitable := true\n\t\tfor i := 1; i < len(profits); i++ {\n\t\t\tif profits[i].profit < 0 || profits[i].profit < (1+rate)*profits[i-1].profit {\n\t\t\t\tprofitable = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !profitable {\n\t\t\tcontinue\n\t\t}\n\n\t\t// COMPANY NAME -----------------------\n\t\tfmt.Printf(\"%-20.20s \", co.name)\n\n\t\t// PROFIT VALUES ----------------------\n\t\ti := 0\n\t\tfor y := yi; y <= yf; y++ {\n\t\t\tif i < len(profits) && profits[i].year == y {\n\t\t\t\tpt.Printf(\"%10.0f \", profits[i].profit)\n\t\t\t\ti++\n\t\t\t} else {\n\t\t\t\tfmt.Printf(\"%10s \", \" \")\n\t\t\t}\n\t\t}\n\n\t\t// CAGR -------------------------------\n\t\tif pi != 0 && pf != 0 && pf*pi >= 0 {\n\t\t\tcagr := math.Pow(float64(pf/pi), 1/float64(yf-yi-1)) - 1\n\t\t\tpt.Printf(\"%10.1f%%\", cagr*100)\n\t\t}\n\t\tfmt.Println()\n\n\t} // next c\n\n\tfmt.Printf(\"\\nEmpresas com lucros crescentes e variação mínima de %0.0f%% de um ano para o outro.\\n\\n\", rate*100)\n\n\treturn nil\n}\n" }, { "path": "reports/logger.go", "content": "package reports\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n)\n\ntype Logger struct {\n\tout io.Writer // destination for output\n\tbuf []byte // for accumulating text to write\n}\n\n// New creates a new Logger\nfunc NewLogger(out io.Writer) *Logger {\n\treturn &Logger{out: out}\n}\n\nfunc (l *Logger) SetOut(out io.Writer) {\n\tl.out = out\n}\n\n// Run prints a message before running a process.\nfunc (l *Logger) Run(format string, v ...interface{}) {\n\ts := fmt.Sprintf(format, v...)\n\tif len(s) > 0 && s[len(s)-1] == '\\n' {\n\t\ts = s[:len(s)-1]\n\t}\n\tl.output(\"[ ] \" + s)\n}\n\n// Ok prints a checkmark after a successful Run()\nfunc (l *Logger) Ok() {\n\tl.outputln(\"\\r[✓]\")\n}\n\n// Nok prints a x mark after a unsuccessful Run()\nfunc (l *Logger) Nok() {\n\tl.outputln(\"\\r[✗]\")\n}\n\n// Printf prints the plain text.\nfunc (l *Logger) Printf(format string, v ...interface{}) {\n\tl.output(fmt.Sprintf(format, v...))\n}\n\n// Trace for very low level logs.\nfunc (l *Logger) Trace(format string, v ...interface{}) {\n\tl.outputln(\"[TRACE] \" + fmt.Sprintf(format, v...))\n}\n\n// Debug for debugging information.\nfunc (l *Logger) Debug(format string, v ...interface{}) {\n\tl.outputln(\"[DEBUG] \" + fmt.Sprintf(format, v...))\n}\n\n// Info for something noteworthy.\nfunc (l *Logger) Info(format string, v ...interface{}) {\n\tl.outputln(\"[INFO] \" + fmt.Sprintf(format, v...))\n}\n\n// Warn for a warning message.\nfunc (l *Logger) Warn(format string, v ...interface{}) {\n\tl.outputln(\"[WARN] \" + fmt.Sprintf(format, v...))\n}\n\n// Error message. Always print to Stderr.\nfunc (l *Logger) Error(format string, v ...interface{}) {\n\thold := l.out\n\tl.out = os.Stderr\n\tl.outputln(\"[ERRO] \" + fmt.Sprintf(format, v...))\n\tl.out = hold\n}\n\nfunc (l *Logger) output(s string) {\n\tif l.out == nil {\n\t\treturn\n\t}\n\tl.buf = l.buf[:0]\n\tl.buf = append(l.buf, s...)\n\t_, _ = l.out.Write(l.buf)\n}\n\nfunc (l *Logger) outputln(s string) {\n\tif l.out == nil {\n\t\treturn\n\t}\n\tl.buf = l.buf[:0]\n\tl.buf = append(l.buf, s...)\n\tif len(s) == 0 || s[len(s)-1] != '\\n' {\n\t\tl.buf = append(l.buf, '\\n')\n\t}\n\t_, _ = l.out.Write(l.buf)\n}\n" }, { "path": "reports/logger_test.go", "content": "package reports\n\nimport (\n\t\"bytes\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestLogger(t *testing.T) {\n\tvar buf bytes.Buffer\n\tlog := NewLogger(&buf)\n\n\ttests := []struct {\n\t\tname string\n\t\tfn func(format string, v ...interface{})\n\t\tmsg string\n\t\twant string\n\t}{\n\t\t{\n\t\t\tname: \"printf\",\n\t\t\tfn: log.Printf,\n\t\t\tmsg: \"this is a normal message\",\n\t\t\twant: \"this is a normal message\",\n\t\t},\n\t\t{\n\t\t\tname: \"trace\",\n\t\t\tfn: log.Trace,\n\t\t\tmsg: \"this is a trace log\\n\",\n\t\t\twant: \"[TRACE] this is a trace log\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"debug\",\n\t\t\tfn: log.Debug,\n\t\t\tmsg: \"this is a debug log\",\n\t\t\twant: \"[DEBUG] this is a debug log\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"info\",\n\t\t\tfn: log.Info,\n\t\t\tmsg: \"you have been informed\",\n\t\t\twant: \"[INFO] you have been informed\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"warn\",\n\t\t\tfn: log.Warn,\n\t\t\tmsg: \"this is a warning\",\n\t\t\twant: \"[WARN] this is a warning\\n\",\n\t\t},\n\t\t// {\n\t\t// \tname: \"error\",\n\t\t// \tfn: log.Error,\n\t\t// \tmsg: \"this is an error message\",\n\t\t// \twant: \"[ERRO] this is an error message\\n\",\n\t\t// },\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\ttt.fn(tt.msg)\n\t\t\tassert.Equal(t, tt.want, buf.String())\n\t\t\tbuf.Reset()\n\t\t})\n\t}\n}\n\nfunc TestLogger_Run(t *testing.T) {\n\tvar buf bytes.Buffer\n\tlog := NewLogger(&buf)\n\n\tlog.Run(\"starting process %d\\n\", 10)\n\tlog.Ok()\n\tassert.Equal(t, \"[ ] starting process 10\\r[✓]\\n\", buf.String())\n\tbuf.Reset()\n\n\tlog.Run(\"starting process %d\", 15)\n\tlog.Nok()\n\tassert.Equal(t, \"[ ] starting process 15\\r[✗]\\n\", buf.String())\n}\n" }, { "path": "reports/reports.go", "content": "package reports\n\nimport (\n\t\"database/sql\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/signal\"\n\t\"path\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/360EntSecGroup-Skylar/excelize\"\n\t\"github.com/dude333/rapina/fetch\"\n\tp \"github.com/dude333/rapina/parsers\"\n\t\"github.com/pkg/errors\"\n\t\"golang.org/x/text/language\"\n\t\"golang.org/x/text/message\"\n)\n\nconst sectorAverage = \"MÉDIA DO SETOR\"\n\nconst (\n\tgrpAccts int = iota + 100\n\tgrpShares\n\tgrpExtra\n\tgrpFleuriet\n)\n\n// metric parameters\ntype metric struct {\n\tdescr string\n\tval float32\n\tformat int // mapped by constants NUMBER, INDEX, PERCENT\n\tgroup int // mapped by constants grpX\n}\n\n// Report parameters used in most functions\ntype Report struct {\n\t// average metric values/year. Index 0: year, index 1: metric\n\taverage [][]float32\n\n\t// groups that will be printed on the output xlsx\n\t// - ExtraRatios: enables some extra financial ratios on report\n\t// - ShowShares: shows the number of shares and free float on report\n\t// - Sector: creates a sheet with the sector report\n\tgroups map[int]bool\n\n\t// if true will print the reports for comanpanies in the same sector\n\tprintSector bool\n\n\t// get the stock quotes\n\tfetchStock *fetch.Stock\n\n\t/* Current company */\n\tcid int // Company ID\n\tcnpj string // Company CNPJ\n\tcode string // Company stock code\n\n\t/* Parameters from caller */\n\tdb *sql.DB // Sqlite3 handler\n\tcompany string // company name to be processed\n\tspcfctnCd \tstring // spcfctnCd used to select the correct ticker\n\tformat string // report format\n\tfilename string // path and filename of the output xlsx\n\tyamlFile string // file with the companies' sectors\n}\n\nfunc New(parms map[string]interface{}) (*Report, error) {\n\tvar r Report\n\n\tif v, ok := parms[\"db\"]; ok {\n\t\tr.db = v.(*sql.DB)\n\t}\n\tif v, ok := parms[\"company\"]; ok {\n\t\tr.company = v.(string)\n\t}\n\tif v, ok := parms[\"SpcfctnCd\"]; ok {\n\t\tr.spcfctnCd = v.(string)\n\t}\n\tif v, ok := parms[\"format\"]; ok {\n\t\tr.format = v.(string)\n\t}\n\tif v, ok := parms[\"filename\"]; ok {\n\t\tr.filename = v.(string)\n\t}\n\tif v, ok := parms[\"yamlFile\"]; ok {\n\t\tr.yamlFile = v.(string)\n\t}\n\tif v, ok := parms[\"reports\"]; ok {\n\t\tp := v.(map[string]bool)\n\t\tr.groups = make(map[int]bool, 4)\n\t\tr.groups[grpAccts] = true\n\t\tr.groups[grpShares] = p[\"ShowShares\"]\n\t\tr.groups[grpExtra] = p[\"ExtraRatios\"]\n\t\tr.groups[grpFleuriet] = p[\"Fleuriet\"]\n\n\t\tr.printSector = true\n\t\tif v, ok := p[\"PrintSector\"]; ok {\n\t\t\tr.printSector = v\n\t\t}\n\t}\n\n\tdataDir := path.Join(\".\", \"data\")\n\tif v, ok := parms[\"dataDir\"]; ok {\n\t\tdataDir = v.(string)\n\t}\n\tapiKey := \"\"\n\tif v, ok := parms[\"apiKey\"]; ok {\n\t\tapiKey = v.(string)\n\t}\n\n\tvar err error\n\tlog := NewLogger(os.Stderr)\n\tr.fetchStock, err = fetch.NewStock(r.db, log, apiKey, dataDir)\n\n\treturn &r, err\n}\n\n//\n// ReportToXlsx reports company financial data from DB to Excel.\n//\nfunc ReportToXlsx(parms map[string]interface{}) error {\n\t// Initialize report object\n\tr, err := New(parms)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = r.setCompanyAndTicker(r.company,r.spcfctnCd)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"empresa '%s' não encontrada no banco de dados\", r.company)\n\t}\n\n\te := newExcel()\n\tsheet, _ := e.newSheet(r.company)\n\n\t// Company name\n\tsheet.mergeCell(\"A1\", \"B1\")\n\tsheet.print(\"A1\", &[]string{r.company}, LEFT, true)\n\n\t// ACCOUNT NUMBERING AND DESCRIPTION (COLS A AND B) ===============\\/\n\taccounts, _ := r.accountsItems(r.cid)\n\tbaseItems, lastStatementsRow, lastMetricsRow := r.printCodesAndDescriptions(sheet, accounts, 'A', 2)\n\n\t// \tVALUES (COLS C, D, E...) / PER YEAR ===========================\\/\n\n\tbegin, end, err := timeRange(r.db)\n\tif err != nil {\n\t\treturn err\n\t}\n\tvar values map[uint32]float32\n\n\t// LOOP THROUGH YEARS =============================================\\/\n\tfor y := begin; y <= end; y++ {\n\t\t// Title\n\t\trow := 2\n\t\tcol := colLetter(2 + y - begin) // start on col 'C'\n\t\tcell := col + \"1\"\n\t\ttitle := \"[\" + strconv.Itoa(y) + \"]\"\n\n\t\tlastYear, isTTM, err := r.lastYear(r.cid)\n\t\tif lastYear == y && isTTM && err == nil {\n\t\t\ttitle = \"[TTM/\" + strconv.Itoa(y) + \"]\"\n\t\t}\n\n\t\t// ACCOUNT VALUES (COLS C, D, E...) / YEAR ====================\\/\n\t\tvalues, err = r.accountsValues(y)\n\t\tif err != nil {\n\t\t\tfmt.Println(\"[x]\", err)\n\t\t\tcontinue\n\t\t}\n\t\t// Skip last year if empty\n\t\tif y == end && sum(values) == 0 {\n\t\t\tend--\n\t\t\tbreak\n\t\t}\n\t\t_ = sheet.printTitle(cell, title) // Print year as title on row 1\n\t\tfor _, acct := range accounts {\n\t\t\tcell := col + strconv.Itoa(row)\n\t\t\t_ = sheet.printValue(cell, values[acct.code], NUMBER, baseItems[row])\n\t\t\trow++\n\t\t}\n\n\t\t// FINANCIAL METRICS (COLS C, D, E...) / YEAR =================\\/\n\t\trow++\n\t\tcell = col + strconv.Itoa(row)\n\t\t_ = sheet.printTitle(cell, title) // Print year as title\n\t\trow++\n\t\t// Print report in the sequence defined on metricsList()\n\t\tfor _, metric := range metricsList(values) {\n\t\t\tif !r.groups[metric.group] {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif metric.format != EMPTY {\n\t\t\t\tcell := col + strconv.Itoa(row)\n\t\t\t\t_ = sheet.printValue(cell, metric.val, metric.format, false)\n\t\t\t}\n\t\t\trow++\n\t\t}\n\n\t} // next year\n\n\t//\n\t// VERTICAL ANALYSIS\n\t//\n\t// CODES | DESCRIPTION | Y1 | Y2 | Yn | sp | v1 | v2 | v3\n\t//\n\twide := (end - begin)\n\tyear := begin\n\ttop := 2\n\tbottom := top\n\tfor col := 2; col <= 2+wide; col++ {\n\t\tvCol := col + wide + 2 // Column where the vertical analysis will be printed\n\t\t_ = sheet.printTitle(axis(vCol, 1), \"'\"+strconv.Itoa(year)) // Print year\n\t\tyear++\n\t\tvar ref string\n\t\tfor row := top; row <= lastStatementsRow; row++ {\n\t\t\tidx := row - top\n\t\t\tif idx < 0 || idx >= len(accounts) {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif len(accounts[idx].cdConta) == 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tn, _ := strconv.Atoi(accounts[idx].cdConta[:1])\n\t\t\tif n > 3 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tswitch accounts[idx].cdConta {\n\t\t\tcase \"1\", \"2\", \"3.01\":\n\t\t\t\tref = axis(col, row)\n\t\t\t}\n\t\t\tval := axis(col, row)\n\t\t\tformula := fmt.Sprintf(`=IfError(%s/%s, \"-\")`, val, ref)\n\n\t\t\t_ = sheet.printFormula(axis(vCol, row), formula, PERCENT, baseItems[row])\n\t\t\tbottom = row\n\t\t}\n\t}\n\n\t// Print VERTICAL ANALYSIS title\n\tsheet.mergeCell(axis(1+wide+2, top), axis(1+wide+2, bottom))\n\tformat := newFormat(DEFAULT, RIGHT, true)\n\tformat.Alignment.Vertical = \"top\"\n\tformat.Alignment.TextRotation = 90\n\trotatedTextStyle := format.newStyle(sheet.xlsx)\n\tsheet.printCell(top, 1+wide+2, \"ANÁLISE VERTICAL\", rotatedTextStyle)\n\n\t//\n\t// HORIZONTAL ANALYSIS\n\t//\n\t// sp | DESCRIPTION | Y1 | Y2 | Yn | sp | h1 | h2 | hn\n\t//\n\twide = (end - begin)\n\tyear = begin\n\ttop = lastStatementsRow + 2\n\tbottom = lastMetricsRow\n\tfor col := 0; col <= wide-1; col++ {\n\t\tyear++\n\t\tvCol := (2 + wide + 2) + col // Column where the horizontal analysis will be printed\n\t\t_ = sheet.printTitle(axis(vCol, top), \"'\"+strconv.Itoa(year)) // Print year\n\t\tfor row := top + 1; row <= bottom; row++ {\n\t\t\tvt0 := axis(col+2, row)\n\t\t\tvtn := axis(col+3, row)\n\t\t\tformula := fmt.Sprintf(`=IF(OR(%s=\"\", %s=\"\"), \"\", IF(MIN(%s, %s)<=0, IF((%s - %s)>0, \" ⇧\", \" ⇩\"), (%s/%s)-1))`,\n\t\t\t\tvtn, vt0, vtn, vt0, vtn, vt0, vtn, vt0)\n\t\t\t_ = sheet.printFormula(axis(vCol, row), formula, PERCENT, false)\n\t\t}\n\t}\n\n\t// Print HORIZONTAL ANALYSIS title\n\tsheet.mergeCell(axis(2+wide+1, top+1), axis(2+wide+1, bottom))\n\tsheet.printCell(top+1, 1+wide+2, \"ANÁLISE HORIZONTAL\", rotatedTextStyle)\n\n\t// CAGR (compound annual growth rate)\n\t// CAGR (t0, tn) = (V(tn)/V(t0))^(1/(tn-t0-1))-1\n\tvCol := (2 + wide + 2) + wide + 1\n\t_ = sheet.printTitle(axis(vCol, top), \"CAGR\")\n\tfor row := top + 1; row <= bottom; row++ {\n\t\tvt0 := axis(2, row)\n\t\tvtn := axis(2+wide, row)\n\t\tformula := fmt.Sprintf(`=IF(OR(%s=\"\", %s=\"\", %s=0, (%s*%s)<0), \"\", (%s/%s)^(1/%d)-1)`,\n\t\t\tvtn, vt0, vt0, vt0, vtn, vtn, vt0, wide)\n\t\t_ = sheet.printFormula(axis(vCol, row), formula, PERCENT, false)\n\t}\n\n\t// ADJUST COLUMNS WIDTH\n\tsheet.autoWidth()\n\n\t// SECTOR REPORT\n\tif r.printSector {\n\t\tsheet2, err := e.newSheet(\"SETOR\")\n\t\tif err == nil {\n\t\t\t_ = sheet2.xlsx.SetSheetViewOptions(sheet2.name, 0,\n\t\t\t\texcelize.ShowGridLines(false),\n\t\t\t\texcelize.ZoomScale(80),\n\t\t\t)\n\t\t\t_ = r.sectorReport(sheet2, r.company)\n\t\t}\n\t}\n\n\terr = e.saveAndCloseExcel(r.filename)\n\tif err == nil {\n\t\tfmt.Printf(\"[√] Dados salvos em %s\\n\", r.filename)\n\t}\n\n\treturn err\n}\n\n//ReportToStdout reports company financial data from DB to Stdout.\nfunc ReportToStdout(parms map[string]interface{}) error {\n\n\tr, err := New(parms)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = r.setCompanyAndTicker(r.company,r.spcfctnCd)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"empresa '%s' não encontrada no banco de dados\", r.company)\n\t}\n\n\tbegin, end, err := timeRange(r.db)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tacc := []AccountValue{}\n\n\tfor y := begin; y <= end; y++ {\n\n\t\td, err := r.RawAccounts(r.cid, y)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tacc = append(acc, d...)\n\t}\n\n\taccBuf, err := buildStdAccountReport(acc)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfmt.Print(accBuf)\n\n\treturn err\n}\n\nfunc buildStdAccountReport(data []AccountValue) (*strings.Builder, error) {\n\n\tbuf := &strings.Builder{}\n\tp := message.NewPrinter(language.BrazilianPortuguese)\n\n\tsort.Slice(data, func(i, j int) bool {\n\t\treturn data[i].year < data[j].year\n\t})\n\n\tfor _, acc := range data {\n\t\tfmt.Fprintf(buf, \"%d;\", acc.year)\n\n\t\tif _, err := p.Fprintf(buf, \"%s;%s;\", acc.accItem.cdConta, acc.accItem.dsConta); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tfmt.Fprintf(buf, \"%d\", int(acc.value))\n\t\tbuf.WriteByte('\\n')\n\t}\n\treturn buf, nil\n}\n\n//\n// sectorReport gets all the companies related to the 'company' and reports\n// their financial summary\n//\nfunc (r Report) sectorReport(sheet *Sheet, company string) (err error) {\n\tvar interrupt bool\n\n\t// Handle Ctrl+C\n\tc := make(chan os.Signal, 1)\n\tsignal.Notify(c, os.Interrupt)\n\tgo func() {\n\t\t<-c\n\t\tfmt.Println(\"\\n[ ] Processamento interrompido\")\n\t\tinterrupt = true\n\t}()\n\n\t// Companies from the same sector\n\tcompanies, secName, err := r.fromSector(company)\n\tif len(companies) <= 1 || err != nil {\n\t\terr = errors.Wrap(err, \"erro ao ler arquivo de setores \"+r.yamlFile)\n\t\treturn\n\t}\n\tcompanies = append([]string{sectorAverage}, companies...)\n\n\tfmt.Println(\"[i] Criando relatório setorial (Ctrl+C para interromper)\")\n\tvar top, row, col int = 2, 0, 0\n\tvar count int\n\tfor _, co := range companies {\n\t\trow = top\n\t\tcol++\n\n\t\tfmt.Printf(\"[ ] - %s\", co)\n\t\tavg := false\n\t\tif co == sectorAverage {\n\t\t\tavg = true\n\t\t\tco = company\n\t\t}\n\t\tempty, err := r.companySummary(sheet, &row, &col, co, secName, count%3 == 0, avg)\n\t\tok := \"√\"\n\t\tif err != nil || empty {\n\t\t\tok = \"x\"\n\t\t\tcol--\n\t\t} else {\n\t\t\tcount++\n\t\t\tif count%3 == 0 {\n\t\t\t\ttop = row + 2\n\t\t\t\tcol = 0\n\t\t\t}\n\t\t}\n\t\tif interrupt {\n\t\t\treturn nil\n\t\t}\n\t\tfmt.Printf(\"\\r[%s\\n\", ok)\n\t}\n\n\tsheet.setColWidth(0, 2)\n\n\treturn\n}\n\n//\n// companySummary reports all companies from the same segment into the\n// 'Setor' sheet.\n//\nfunc (r *Report) companySummary(sheet *Sheet, row, col *int, _company, sectorName string, printDescr, sectorAvg bool) (empty bool, err error) {\n\t// if !sectorAvg && !r.isCompany(company) {\n\t// \treturn true, nil\n\t// }\n\n\terr = r.setCompanyAndTicker(r.company,r.spcfctnCd)\n\tif err != nil {\n\t\terr = errors.Errorf(\"empresa '%s' não encontrada no banco de dados\", _company)\n\t\treturn\n\t}\n\n\tbegin, end, err := timeRange(r.db)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// Formats used in this report\n\tsTitle := newFormat(DEFAULT, RIGHT, true).newStyle(sheet.xlsx)\n\tfCompanyName := newFormat(DEFAULT, CENTER, true)\n\tfCompanyName.size(16)\n\tsCompanyName := fCompanyName.newStyle(sheet.xlsx)\n\tfSectorName := newFormat(DEFAULT, LEFT, false)\n\tfSectorName.size(14)\n\tsSectorName := fSectorName.newStyle(sheet.xlsx)\n\t//\n\tfDescr := newFormat(DEFAULT, RIGHT, false)\n\tfDescr.Border = []formatBorder{{Type: \"left\", Color: \"333333\", Style: 1}}\n\tsDescr := fDescr.newStyle(sheet.xlsx)\n\tfDescr.Border = []formatBorder{\n\t\t{Type: \"top\", Color: \"333333\", Style: 1},\n\t\t{Type: \"left\", Color: \"333333\", Style: 1},\n\t}\n\tsDescrTop := fDescr.newStyle(sheet.xlsx)\n\tfDescr.Border = []formatBorder{\n\t\t{Type: \"top\", Color: \"333333\", Style: 1},\n\t}\n\tsDescrBottom := fDescr.newStyle(sheet.xlsx)\n\n\t// Company name\n\tif printDescr {\n\t\t*col++\n\t}\n\tsheet.mergeCell(axis(*col, *row), axis(*col+end-begin+1, *row))\n\tif sectorAvg {\n\t\tsheet.printCell(*row-1, *col-1, sectorName, sSectorName)\n\t\tsheet.printCell(*row, *col, sectorAverage, sCompanyName)\n\t} else {\n\t\tsheet.printCell(*row, *col, _company, sCompanyName)\n\t}\n\tif printDescr {\n\t\t*col--\n\t}\n\t*row++\n\n\t// Save starting row\n\trw := *row\n\n\t// Set width for the description col\n\tif printDescr {\n\t\tsheet.setColWidth(*col, 18)\n\t\t*col++\n\t}\n\n\t// Print values ONE YEAR PER COLUMN\n\tfor y := begin; y <= end; y++ {\n\t\tvar values map[uint32]float32\n\t\tvar err error\n\t\tif sectorAvg {\n\t\t\tvalues, err = r.accountsAverage(_company, y)\n\t\t\tr.average = append(r.average, []float32{})\n\t\t} else {\n\t\t\tvalues, err = r.accountsValues(y)\n\t\t}\n\t\tif err != nil {\n\t\t\tfmt.Printf(\" -- %v\", err)\n\t\t\treturn false, err\n\t\t}\n\n\t\t// Skip last year if empty\n\t\tif y == end && sum(values) == 0 {\n\t\t\tend--\n\t\t\tbreak\n\t\t}\n\n\t\t*row = rw\n\n\t\t// Print year\n\t\tsheet.printCell(*row, *col, \"[\"+strconv.Itoa(y)+\"]\", sTitle)\n\t\t*row++\n\n\t\t// Print financial metrics\n\t\ti := 0\n\t\tfor _, metric := range metricsList(values) {\n\t\t\tif !r.groups[metric.group] {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif sectorAvg {\n\t\t\t\tr.average[y-begin] = append(r.average[y-begin], metric.val)\n\t\t\t}\n\t\t\t// Description\n\t\t\tif printDescr {\n\t\t\t\tstl := sDescr\n\t\t\t\tif i == 0 {\n\t\t\t\t\tstl = sDescrTop\n\t\t\t\t}\n\t\t\t\tsheet.printCell(*row, *col-1, metric.descr, stl)\n\t\t\t}\n\t\t\t// Values\n\t\t\tif metric.format != EMPTY {\n\t\t\t\tfVal := newFormat(metric.format, DEFAULT, false)\n\t\t\t\tfVal.Border = []formatBorder{\n\t\t\t\t\t{Type: \"top\", Color: \"cccccc\", Style: 1},\n\t\t\t\t\t{Type: \"right\", Color: \"cccccc\", Style: 1},\n\t\t\t\t\t{Type: \"bottom\", Color: \"cccccc\", Style: 1},\n\t\t\t\t\t{Type: \"left\", Color: \"cccccc\", Style: 1},\n\t\t\t\t}\n\t\t\t\t// Color the cell background according to its value compared with the average\n\t\t\t\tif len(r.average) > 0 && len(r.average[y-begin]) > 0 && len(r.average[y-begin]) >= i {\n\t\t\t\t\tf := formatFill{Type: \"pattern\", Pattern: 1}\n\t\t\t\t\tif metric.val > r.average[y-begin][i] {\n\t\t\t\t\t\tf.Color = []string{\"c6efce\"} // green\n\t\t\t\t\t\tfVal.Fill = f\n\t\t\t\t\t} else if metric.val < r.average[y-begin][i] {\n\t\t\t\t\t\tf.Color = []string{\"ffc7ce\"} // red\n\t\t\t\t\t\tfVal.Fill = f\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tstl := fVal.newStyle(sheet.xlsx)\n\t\t\t\tsheet.printCell(*row, *col, metric.val, stl)\n\t\t\t}\n\t\t\t*row++\n\t\t\ti++\n\t\t}\n\n\t\tif printDescr {\n\t\t\tsheet.printCell(*row, *col-1, \"\", sDescrBottom)\n\t\t}\n\n\t\tprintDescr = false\n\t\t*col++\n\t} // next year\n\n\tbottom := *row\n\n\t// CAGR (compound annual growth rate)\n\t// CAGR (t0, tn) = (V(tn)/V(t0))^(1/(tn-t0-1))-1\n\twide := end - begin\n\t_ = sheet.printTitle(axis(*col, rw), \"CAGR\")\n\tfor r := rw + 1; r <= bottom; r++ {\n\t\tvt0 := axis(*col-wide-1, r)\n\t\tvtn := axis(*col-1, r)\n\t\tformula := fmt.Sprintf(`=IF(OR(%s=\"\", %s=\"\", %s=0, (%s*%s)<0), \"\", (%s/%s)^(1/%d)-1)`,\n\t\t\tvtn, vt0, vt0, vt0, vtn, vtn, vt0, wide)\n\t\t_ = sheet.printFormula(axis(*col, r), formula, PERCENT, false)\n\t}\n\t*col++\n\n\treturn\n}\n\nfunc (r *Report) Summary(company string) (map[string]string, error) {\n\tm := make(map[string]string)\n\n\terr := r.setCompanyAndTicker(r.company,r.spcfctnCd)\n\tif err != nil {\n\t\treturn m, err\n\t}\n\tvalues, err := r.accountsValues(2020)\n\tif err != nil {\n\t\treturn m, err\n\t}\n\n\tfor _, metric := range metricsList(values) {\n\t\tif !r.groups[metric.group] {\n\t\t\tcontinue\n\t\t}\n\t}\n\n\treturn m, nil\n}\n\n//\n// metricsList returns the sequence to be printed after the financial statements\n//\nfunc metricsList(v map[uint32]float32) (metrics []metric) {\n\tdividaBruta := v[p.DividaCirc] + v[p.DividaNCirc]\n\tcaixa := v[p.Caixa] + v[p.AplicFinanceiras]\n\tdividaLiquida := dividaBruta - caixa\n\tEBITDA := v[p.EBIT] - v[p.Deprec]\n\tproventos := v[p.Dividendos] + v[p.JurosCapProp]\n\n\tvar roe float32\n\tif v[p.LucLiq] > 0 && v[p.EquityAvg] > 0 {\n\t\troe = zeroIfNeg(safeDiv(v[p.LucLiq], v[p.EquityAvg]))\n\t}\n\tvar cg float32 = v[p.AtivoCirc] - v[p.PassivoCirc]\n\tvar st float32 = v[p.Caixa] + v[p.AplicFinanceiras] - (v[p.DividaCirc] + v[p.DividendosJCP] + v[p.DividendosMin])\n\tvar ncg float32 = cg - st\n\n\tvar lpa float32 = safeDiv(v[p.LucLiq]*v[p.Escala], v[p.Shares])\n\n\treturn []metric{\n\t\t{\"Patrimônio Líquido\", v[p.Equity], NUMBER, grpAccts},\n\t\t{\"\", 0, EMPTY, grpAccts},\n\n\t\t{\"Receita Líquida\", v[p.Vendas], NUMBER, grpAccts},\n\t\t{\"EBITDA\", EBITDA, NUMBER, grpAccts},\n\t\t{\"EBIT\", v[p.EBIT], NUMBER, grpAccts},\n\t\t{\"Resultado Financeiro\", v[p.ResulFinanc], NUMBER, grpAccts},\n\t\t{\"Operações Descontinuadas\", v[p.ResulOpDescont], NUMBER, grpAccts},\n\t\t{\"Lucro Líquido\", v[p.LucLiq], NUMBER, grpAccts},\n\t\t{\"\", 0, EMPTY, grpAccts},\n\n\t\t{\"LPA\", lpa, INDEX, grpAccts},\n\t\t{\"VPA\", safeDiv(v[p.Equity]*v[p.Escala], v[p.Shares]), INDEX, grpAccts},\n\t\t{\"P/L\", safeDiv(v[p.Quote], lpa), INDEX, grpAccts},\n\t\t{\"Cotação\", v[p.Quote], INDEX, grpAccts},\n\t\t{\"\", 0, EMPTY, grpAccts},\n\n\t\t{\"Marg. EBITDA\", zeroIfNeg(safeDiv(EBITDA, v[p.Vendas])), PERCENT, grpAccts},\n\t\t{\"Marg. EBIT\", zeroIfNeg(safeDiv(v[p.EBIT], v[p.Vendas])), PERCENT, grpAccts},\n\t\t{\"Marg. Líq.\", zeroIfNeg(safeDiv(v[p.LucLiq], v[p.Vendas])), PERCENT, grpAccts},\n\t\t{\"ROE\", roe, PERCENT, grpAccts},\n\t\t{\"\", 0, EMPTY, grpAccts},\n\n\t\t{\"Caixa\", caixa, NUMBER, grpAccts},\n\t\t{\"Dívida Bruta\", dividaBruta, NUMBER, grpAccts},\n\t\t{\"Dívida Líq.\", dividaLiquida, NUMBER, grpAccts},\n\t\t{\"Dív. Bru./PL\", zeroIfNeg(safeDiv(dividaBruta, v[p.Equity])), PERCENT, grpAccts},\n\t\t{\"Dív.Líq./EBITDA\", zeroIfNeg(safeDiv(dividaLiquida, EBITDA)), INDEX, grpAccts},\n\t\t{\"\", 0, EMPTY, grpAccts},\n\n\t\t{\"FCO\", v[p.FCO], NUMBER, grpAccts},\n\t\t{\"FCI\", v[p.FCI], NUMBER, grpAccts},\n\t\t{\"FCF\", v[p.FCF], NUMBER, grpAccts},\n\t\t{\"FCT\", v[p.FCO] + v[p.FCI] + v[p.FCF], NUMBER, grpAccts},\n\t\t{\"FCL (FCO+FCI)\", v[p.FCO] + v[p.FCI], NUMBER, grpAccts},\n\t\t{\"\", 0, EMPTY, grpAccts},\n\n\t\t{\"Proventos\", proventos, NUMBER, grpAccts},\n\t\t{\"Payout\", zeroIfNeg(safeDiv(proventos, v[p.LucLiq])), PERCENT, grpAccts},\n\t\t{\"\", 0, EMPTY, grpAccts},\n\n\t\t{\"Total de Ações\", v[p.Shares], GENERAL, grpShares},\n\t\t{\"Free Float\", v[p.FreeFloat], PERCENT, grpShares},\n\t\t{\"\", 0, EMPTY, grpShares},\n\n\t\t{\"Liquidez Corrente (Ativo Circ./Passivo Circ.)\", safeDiv(v[p.AtivoCirc], v[p.PassivoCirc]), INDEX, grpExtra},\n\t\t{\"Liquidez Seco [(Ativo Circ.-Estoque)/Passivo Circ.]\", safeDiv(v[p.AtivoCirc]-v[p.Estoque], v[p.PassivoCirc]), INDEX, grpExtra},\n\t\t{\"Giro dos Ativos (Vendas/Ativo)\", safeDiv(v[p.Vendas], v[p.AtivoTotal]), INDEX, grpExtra},\n\t\t{\"\", 0, EMPTY, grpExtra},\n\t\t{\"Giro de Estoque (dias)\", safeDiv(v[p.EstoqueMedio], -v[p.CustoVendas]/360), INDEX, grpExtra},\n\t\t{\"Prazo Médio de Recebimento (dias)\", safeDiv(v[p.ContasARecebCirc]+v[p.ContasARecebNCirc], v[p.Vendas]/360), INDEX, grpExtra},\n\t\t{\"\", 0, EMPTY, grpExtra},\n\t\t{\"Poder de Ganho Básico (EBITDA/Ativo)\", safeDiv(EBITDA, v[p.AtivoTotal]), PERCENT, grpExtra},\n\t\t{\"ROA\", safeDiv(v[p.LucLiq], v[p.AtivoTotal]), PERCENT, grpExtra},\n\t\t{\"ROE\", roe, PERCENT, grpExtra},\n\t\t{\"\", 0, EMPTY, grpExtra},\n\n\t\t{\"Capital de Giro (CG)\", cg, NUMBER, grpFleuriet},\n\t\t{\"Saldo de Tesouraria (ST)\", st, NUMBER, grpFleuriet},\n\t\t{\"Necessidade de Capital de Giro (NCG=CG-ST)\", ncg, NUMBER, grpFleuriet},\n\t}\n}\n\nfunc zeroIfNeg(n float32) float32 {\n\tif n < 0 {\n\t\treturn 0\n\t}\n\treturn n\n}\n\nfunc safeDiv(n, d float32) float32 {\n\tif d == 0 {\n\t\treturn 0\n\t}\n\treturn n / d\n}\n\n//\n// ident returns the number of spaces according to the code level, e.g.:\n// \"1.1 ABC\" => \" \" (2 spaces)\n// \"1.1.1 ABC\" => \" \" (4 spaces)\n// For items equal or above 3, only returns spaces after 2nd level:\n// \"3.01 ABC\" => \"\"\n// \"3.01.01 ABC\" => \" \"\n//\nfunc ident(str string) (spaces string, baseItem bool) {\n\tnum := strings.SplitN(str, \".\", 2)[0]\n\tc := strings.Count(str, \".\")\n\tif num != \"1\" && num != \"2\" && c > 0 {\n\t\tc--\n\t}\n\tif c > 0 {\n\t\tspaces = strings.Repeat(\" \", c)\n\t}\n\n\tif num == \"1\" || num == \"2\" {\n\t\tbaseItem = c <= 1\n\t} else {\n\t\tbaseItem = c == 0\n\t}\n\n\treturn\n}\n\n// printCodesAndDescription prints 'accounts' codes and descriptions on\n// columns 'col' and 'col+1' (A <= col <= Z), starting on row 2.\n// Adjust space related to the group, e.g.:\n// 3.02 ABC <= print in bold if base item and stores the row position in baseItems[]\n// 3.02.01 ABC\n//\n// Returns:\n// - []bool indicates if a row is a base item,\n// - the row of the last statement,\n// - the row of the last metric item.\nfunc (r Report) printCodesAndDescriptions(sheet *Sheet, accounts []accItems, col rune, row int) ([]bool, int, int) {\n\tbaseItems := make([]bool, len(accounts)+row)\n\tfor _, it := range accounts {\n\t\tvar sp string\n\t\tsp, baseItems[row] = ident(it.cdConta)\n\t\tcell := string(col) + strconv.Itoa(row)\n\t\tsheet.print(cell, &[]string{sp + it.cdConta, sp + it.dsConta}, LEFT, baseItems[row])\n\t\trow++\n\t}\n\tlastStatementsRow := row - 1\n\trow += 2\n\tcol++\n\t// Metrics descriptions\n\tfor _, metric := range metricsList(nil) {\n\t\tif !r.groups[metric.group] {\n\t\t\tcontinue\n\t\t}\n\t\tif metric.descr != \"\" {\n\t\t\tcell := string(col) + strconv.Itoa(row)\n\t\t\tsheet.print(cell, &[]string{metric.descr}, RIGHT, false)\n\t\t}\n\t\trow++\n\t}\n\tlastMetricsRow := row - 1\n\n\treturn baseItems, lastStatementsRow, lastMetricsRow\n}\n\n// sum returns a float32 with the sum of all values from a map\nfunc sum(values map[uint32]float32) float32 {\n\tvar sum float32\n\tfor _, v := range values {\n\t\tsum += v\n\t}\n\treturn sum\n}\n" }, { "path": "reports/reports_fii.go", "content": "package reports\n\nimport (\n\t\"database/sql\"\n\t\"fmt\"\n\t\"math\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/dude333/rapina\"\n\t\"github.com/dude333/rapina/fetch\"\n\t\"github.com/dude333/rapina/progress\"\n\t\"golang.org/x/text/language\"\n\t\"golang.org/x/text/message\"\n)\n\nvar line = strings.Repeat(\"-\", 67)\n\n// Type of report output\nconst (\n\tRtable = iota + 1\n\tRcsv\n\tRcsvrend\n)\n\n// FIITerminal implements reports related to FII funds on the terminal.\ntype FIITerminal struct {\n\tfetchFII *fetch.FII\n\tfetchStock *fetch.Stock\n\treportFormat int\n}\n\ntype FIITerminalOptions struct {\n\tAPIKey, DataDir string\n}\n\n// NewFIITerminal creates a new instace of a FIITerminal\nfunc NewFIITerminal(db *sql.DB, opts FIITerminalOptions) (*FIITerminal, error) {\n\tvar log rapina.Logger\n\n\tfetchStock, err := fetch.NewStock(db, log, opts.APIKey, opts.DataDir)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfetchFII, err := fetch.NewFII(db, log)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &FIITerminal{\n\t\tfetchFII: fetchFII,\n\t\tfetchStock: fetchStock,\n\t\treportFormat: Rtable,\n\t}, nil\n}\n\n// SetParms set the terminal reports parameters.\nfunc (t *FIITerminal) SetParms(parms map[string]string) {\n\tif _, ok := parms[\"verbose\"]; ok {\n\t\tprogress.SetDebug(true)\n\t}\n\tif r, ok := parms[\"format\"]; ok {\n\t\tswitch r {\n\t\tcase \"table\", \"tabela\", \"tab\":\n\t\t\tt.reportFormat = Rtable\n\t\tcase \"csv\":\n\t\t\tt.reportFormat = Rcsv\n\t\tcase \"csvrend\":\n\t\t\tt.reportFormat = Rcsvrend\n\t\t}\n\t}\n}\n\n// Dividends prints the dividends report on terminal.\nfunc (t FIITerminal) Dividends(codes []string, n int) error {\n\t// Header\n\tif t.reportFormat == Rcsv {\n\t\tfmt.Println(\"Código,Data Com,Rendimento,Cotação,Yeld,Yeld a.a.\")\n\t}\n\tif t.reportFormat == Rcsvrend {\n\t\tfmt.Print(`Código/Data-Com`)\n\t\tfor _, date := range revMonthsFromToday(n) {\n\t\t\tfmt.Printf(\",%s\", date)\n\t\t}\n\t\tfmt.Println()\n\t}\n\n\t// Remove codes\n\tc := make([]string, 0, len(codes))\n\tfor _, code := range codes {\n\t\tif len(code) == 6 {\n\t\t\tc = append(c, code)\n\t\t} else {\n\t\t\tprogress.ErrorMsg(\"Código inválido: %s. Padrão esperado: ABCD11.\", code)\n\t\t}\n\t}\n\tcodes = c\n\n\tdividends := sync.Map{}\n\tvar wg sync.WaitGroup\n\tfor i, code := range codes {\n\t\twg.Add(1)\n\t\ti := i\n\t\tgo func(code string, n int) {\n\t\t\tdefer wg.Done()\n\t\t\tdiv, err := t.fetchFII.Dividends(code, n)\n\t\t\tif err != nil {\n\t\t\t\tprogress.ErrorMsg(\"%s: %v\", code, err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tprogress.Debug(\"[go routine %d] dividends (%d): %v\", i, len(*div), div)\n\t\t\tdividends.Store(code, div)\n\t\t}(code, n)\n\t}\n\twg.Wait()\n\n\tfor _, code := range codes {\n\t\tdiv, ok := dividends.Load(code)\n\t\tif !ok {\n\t\t\tcontinue\n\t\t}\n\t\tdividendsForCode := div.(*[]rapina.Dividend)\n\t\tvar buf *strings.Builder\n\t\tvar err error\n\t\tswitch t.reportFormat {\n\t\tcase Rcsv:\n\t\t\tbuf, err = t.csvDividends(code, dividendsForCode)\n\t\tcase Rcsvrend:\n\t\t\tbuf, err = t.csvDividendsOnly(code, n, dividendsForCode)\n\t\tdefault:\n\t\t\tbuf, err = t.printDividends(code, dividendsForCode)\n\t\t}\n\t\tif err != nil {\n\t\t\tprogress.Error(err)\n\t\t} else {\n\t\t\tfmt.Print(buf)\n\t\t}\n\t}\n\n\t// Footer\n\t// if t.reportFormat == Rtable {\n\t// \tfmt.Println(line)\n\t// }\n\n\treturn nil\n}\n\nfunc (t FIITerminal) printDividends(code string, dividends *[]rapina.Dividend) (*strings.Builder, error) {\n\tbuf := &strings.Builder{}\n\tp := message.NewPrinter(language.BrazilianPortuguese)\n\n\tp.Fprintln(buf, line)\n\tp.Fprintln(buf, code)\n\tp.Fprintln(buf, line)\n\tp.Fprintln(buf, \" DATA COM RENDIMENTO COTAÇÃO YELD YELD a.a.\")\n\tp.Fprintln(buf, \" ---------- ---------- ---------- ------ ---------\")\n\n\tfor _, d := range *dividends {\n\t\tp.Fprintf(buf, \" %s R$%8.2f \", d.Date, d.Val)\n\n\t\tq, err := t.fetchStock.Quote(code, d.Date)\n\t\tif err != nil {\n\t\t\tprogress.ErrorMsg(\"Cotação de %s (%s): %v\", code, d.Date, err)\n\t\t}\n\t\tif q > 0 && err == nil {\n\t\t\ti := d.Val / q\n\t\t\tp.Fprintf(buf, \"R$%8.2f %8.2f%% %8.2f%%\", q, 100*i, 100*(math.Pow(1+i, 12)-1))\n\t\t}\n\t\tbuf.WriteByte('\\n')\n\t}\n\tbuf.WriteByte('\\n')\n\n\treturn buf, nil\n}\n\nfunc (t FIITerminal) csvDividends(code string, dividends *[]rapina.Dividend) (*strings.Builder, error) {\n\tbuf := &strings.Builder{}\n\tp := message.NewPrinter(language.BrazilianPortuguese)\n\tfor _, d := range *dividends {\n\t\tp.Fprintf(buf, `%s,%s,\"%f\",`, code, d.Date, d.Val)\n\n\t\tq, err := t.fetchStock.Quote(code, d.Date)\n\t\tif err != nil {\n\t\t\tprogress.ErrorMsg(\"Cotação de %s (%s): %v\", code, d.Date, err)\n\t\t}\n\t\tif q > 0 && err == nil {\n\t\t\ti := d.Val / q\n\t\t\tp.Fprintf(buf, `\"%f\",\"%f%%\",\"%f%%\"`, q, 100*i, 100*(math.Pow(1+i, 12)-1))\n\t\t} else {\n\t\t\tbuf.WriteString(`\"\",\"\",\"\"`)\n\t\t}\n\t\tbuf.WriteByte('\\n')\n\t}\n\n\treturn buf, nil\n}\n\nfunc (t FIITerminal) csvDividendsOnly(code string, n int, dividends *[]rapina.Dividend) (*strings.Builder, error) {\n\tbuf := &strings.Builder{}\n\tp := message.NewPrinter(language.BrazilianPortuguese)\n\tbuf.WriteString(code)\n\n\tfor _, month := range revMonthsFromToday(n) {\n\t\tfound := false\n\t\tfor _, div := range *dividends {\n\t\t\tif div.Date[0:len(\"YYYY-MM\")] == month {\n\t\t\t\tp.Fprintf(buf, `,\"%f\"`, div.Val)\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\tbuf.WriteString(`,\"\"`)\n\t\t}\n\n\t}\n\tbuf.WriteByte('\\n')\n\n\treturn buf, nil\n}\n\nfunc revMonthsFromToday(n int) []string {\n\trev := make([]string, 0, n)\n\tdates := rapina.MonthsFromToday(n)\n\tfor i := len(dates) - 1; i >= 0; i-- {\n\t\trev = append(rev, dates[i][0:len(\"YYYY-MM\")])\n\t}\n\treturn rev\n}\n\n/* ------- MONTHLY REPORTS -------- */\n\nfunc (t FIITerminal) Monthly(codes []string, n int) error {\n\n\tfor _, c := range codes {\n\t\tii, err := t.fetchFII.MonthlyReportIDs(c, n)\n\t\tprogress.Status(\"indexes: %v (err: %v)\", ii, err)\n\t}\n\n\treturn nil\n}\n" }, { "path": "reports/reports_html.go", "content": "package reports\n" }, { "path": "reports/reports_test.go", "content": "package reports\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\tp \"github.com/dude333/rapina/parsers\"\n)\n\n// AssertEqual checks if values are equal\nfunc AssertEqual(t *testing.T, msg string, a interface{}, b interface{}) {\n\tif a == b {\n\t\treturn\n\t}\n\t// debug.PrintStack()\n\tt.Errorf(\"%s was incorrect, received %v, expected %v.\", msg, a, b)\n}\n\nfunc TestIdent(t *testing.T) {\n\ttable := []struct {\n\t\tin string\n\t\texpected string\n\t\tisBaseItem bool\n\t}{\n\t\t{\"1\", \"\", true},\n\t\t{\"1.1\", \" \", true},\n\t\t{\"1.1.2\", \" \", false},\n\t\t{\"2\", \"\", true},\n\t\t{\"2.3.4.5\", \" \", false},\n\t\t{\"2.10\", \" \", true},\n\t\t{\"3\", \"\", true},\n\t\t{\"3.1\", \"\", true},\n\t\t{\"3.1.2\", \" \", false},\n\t}\n\n\tfor _, x := range table {\n\t\tspaces, baseItem := ident(x.in)\n\t\tif spaces != x.expected {\n\t\t\tt.Errorf(\"ident was incorrect for %s, got: '%s', want: '%s'.\", x.in, spaces, x.expected)\n\t\t}\n\t\tif baseItem != x.isBaseItem {\n\t\t\tt.Errorf(\"ident was incorrect for %s, got: %v, want: %v.\", x.in, baseItem, x.isBaseItem)\n\t\t}\n\t}\n}\n\nfunc TestZeroIfNeg(t *testing.T) {\n\tfor x := float32(10); x >= -10; x -= 0.1 {\n\t\ty := zeroIfNeg(x)\n\t\tif (x >= 0 && x != y) || (x < 0 && y != 0) {\n\t\t\tt.Errorf(\"zeroIfNeg was incorrect, got: %f, want: %f\", y, x)\n\t\t}\n\t}\n}\n\nfunc TestSafeDiv(t *testing.T) {\n\tfor x := float32(10); x >= -10; x -= 0.1 {\n\t\ty := safeDiv(2, x)\n\t\tif (x == 0 && y != 0) || (y != 2/x) {\n\t\t\tt.Errorf(\"safeDiv was incorrect, got: %f, want: %f\", y, x)\n\t\t}\n\t}\n}\n\nfunc TestMetricsList(t *testing.T) {\n\tv := make(map[uint32]float32)\n\n\tfor x := uint32(p.Caixa); x <= uint32(p.Dividendos); x++ {\n\t\tv[x] = float32(x) * 123456\n\t}\n\tl := metricsList(v)\n\n\tseq := []float32{\n\t\tv[p.Equity],\n\t\t0,\n\t\tv[p.Vendas],\n\t\tv[p.EBIT] - v[p.Deprec], // EBITDA\n\t\tv[p.EBIT],\n\t\tv[p.ResulFinanc],\n\t\tv[p.ResulOpDescont],\n\t\tv[p.LucLiq],\n\t\t0,\n\t}\n\n\tfor i, val := range seq {\n\t\tAssertEqual(t, \"metricsList [\"+l[i].descr+\"]\", l[i].val, val)\n\t}\n\n}\n\nfunc TestStdBuildReport(t *testing.T) {\n\n\tdata := []AccountValue{\n\t\t{\n\t\t\taccItem: accItems{\n\t\t\t\tcode: 123,\n\t\t\t\tcdConta: \"cdConta Second\",\n\t\t\t\tdsConta: \"desc Second Conta\",\n\t\t\t},\n\t\t\tvalue: 456,\n\t\t\tyear: 2012,\n\t\t},\n\t\t{\n\t\t\taccItem: accItems{\n\t\t\t\tcode: 987,\n\t\t\t\tcdConta: \"cdConta First\",\n\t\t\t\tdsConta: \"desc First Conta\",\n\t\t\t},\n\t\t\tvalue: 654,\n\t\t\tyear: 2011,\n\t\t},\n\t}\n\n\tbuilder, err := buildStdAccountReport(data)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tlines := strings.Split(strings.TrimSuffix(builder.String(), \"\\n\"), \"\\n\")\n\n\tstrings.EqualFold(\"2011;cdConta First;desc First Conta;654\", lines[0])\n\tstrings.EqualFold(\"2012;cdConta Second;desc Second Conta;456\", lines[1])\n}\n" }, { "path": "server/fs_dev.go", "content": "// +build dev\n\npackage server\n\nimport (\n\t\"io/fs\"\n\t\"log\"\n\t\"os\"\n)\n\nvar _fs = os.DirFS(\".\")\nvar _contentFS fs.FS\n\nfunc init() {\n\tvar err error\n\t_contentFS, err = fs.Sub(_fs, \"templates\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n}\n" }, { "path": "server/fs_prod.go", "content": "// +build !dev\n\npackage server\n\nimport (\n\t\"embed\"\n\t\"io/fs\"\n\t\"log\"\n)\n\n//go:embed templates\nvar _fs embed.FS\nvar _contentFS fs.FS\n\nfunc init() {\n\tvar err error\n\t_contentFS, err = fs.Sub(_fs, \"templates\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n}\n" }, { "path": "server/payload.go", "content": "package server\n\nimport (\n\t\"math\"\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/dude333/rapina/progress\"\n)\n\n// fiiDividendsPayload returns the data to be used in the FII template.\nfunc fiiDividendsPayload(srv *Server, fiiCodes []string, months int) interface{} {\n\tvar payload struct {\n\t\tCodes string\n\t\tMonths int\n\t\tData interface{}\n\t}\n\n\tpayload.Codes = strings.Join(fiiCodes, \" \")\n\tpayload.Months = months\n\tpayload.Data = fiiDividends(srv, fiiCodes, months)\n\n\treturn &payload\n}\n\nfunc fiiDividends(srv *Server, codes []string, n int) interface{} {\n\ttype value struct {\n\t\tDate string\n\t\tDividend float64\n\t\tQuote float64\n\t\tYeld float64\n\t\tYeldYear float64\n\t}\n\n\ttype data struct {\n\t\tCode string\n\t\tName string\n\t\tWebsite string\n\t\tValues []value\n\t}\n\n\tvar dataset []data\n\n\t// Fill 'data' for every stock code\n\tfor _, code := range codes {\n\t\tcode = strings.ToUpper(code)\n\t\tvalues := make([]value, 0, n)\n\n\t\t// Dividends from last \"n\" months\n\t\tdiv, err := srv.fetchFII.Dividends(code, n)\n\t\tif err != nil {\n\t\t\tprogress.ErrorMsg(\"%s: %v\", code, err)\n\t\t\tcontinue\n\t\t}\n\n\t\t// Stock quotes from the days when the dividends were received\n\t\tfor _, d := range *div {\n\t\t\tq, err := srv.fetchStock.Quote(code, d.Date)\n\t\t\tif err != nil {\n\t\t\t\tprogress.ErrorMsg(\"Cotação de %s (%s): %v\", code, d.Date, err)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tv := value{\n\t\t\t\tDate: d.Date,\n\t\t\t\tDividend: d.Val,\n\t\t\t\tQuote: q,\n\t\t\t}\n\t\t\tif q > 0 {\n\t\t\t\ti := d.Val / q\n\t\t\t\tv.Yeld = 100 * i\n\t\t\t\tv.YeldYear = 100 * (math.Pow(1+i, 12) - 1)\n\t\t\t}\n\t\t\tvalues = append(values, v)\n\t\t}\n\n\t\t// FII details, if found\n\t\tdetails, err := srv.fetchFII.Details(code)\n\t\tvar name, a string\n\t\tif err == nil {\n\t\t\tname = details.DetailFund.CompanyName\n\t\t\tu, err := url.Parse(details.DetailFund.WebSite)\n\t\t\tif err == nil && u.Scheme == \"\" {\n\t\t\t\tu.Scheme = \"https\"\n\t\t\t\ta = u.String()\n\t\t\t}\n\t\t}\n\n\t\td := data{\n\t\t\tCode: code,\n\t\t\tName: name,\n\t\t\tWebsite: a,\n\t\t\tValues: values,\n\t\t}\n\n\t\tdataset = append(dataset, d)\n\t} // next code\n\n\treturn &dataset\n}\n\n// func financialsPayload(srv *Server, stockCode string) interface{} {\n// \tvar payload struct {\n// \t\tEquity float32\n// \t}\n\n// \treturn &payload\n// }\n" }, { "path": "server/server.go", "content": "package server\n\nimport (\n\t\"database/sql\"\n\t\"errors\"\n\t\"html/template\"\n\t\"log\"\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/dude333/rapina/fetch\"\n\t\"github.com/dude333/rapina/progress\"\n\t\"github.com/dude333/rapina/reports\"\n\t\"golang.org/x/text/language\"\n\t\"golang.org/x/text/message\"\n)\n\ntype Server struct {\n\tdb *sql.DB\n\tfetchFII *fetch.FII\n\tfetchStock *fetch.Stock\n\treport *reports.Report\n\tdataDir string\n\tapiKey string\n\tverbose bool\n}\n\ntype ServerOption func(*Server)\n\nfunc WithDB(db *sql.DB) ServerOption {\n\treturn func(s *Server) {\n\t\ts.db = db\n\t}\n}\nfunc WithAPIKey(apiKey string) ServerOption {\n\treturn func(s *Server) {\n\t\ts.apiKey = apiKey\n\t}\n}\nfunc WithDataDir(dataDir string) ServerOption {\n\treturn func(s *Server) {\n\t\ts.dataDir = dataDir\n\t}\n}\nfunc Verbose(on bool) ServerOption {\n\treturn func(s *Server) {\n\t\ts.verbose = on\n\t}\n}\n\nfunc initServer(opts ...ServerOption) (*Server, error) {\n\tvar srv Server\n\tfor _, opt := range opts {\n\t\topt(&srv)\n\t}\n\tif srv.db == nil {\n\t\treturn nil, errors.New(\"BD inválido\")\n\t}\n\n\tprogress.SetDebug(srv.verbose)\n\n\tsrv.db.SetMaxOpenConns(1)\n\tlog := reports.NewLogger(os.Stderr)\n\tfetchStock, err := fetch.NewStock(srv.db, log, srv.apiKey, srv.dataDir)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfetchFII, err := fetch.NewFII(srv.db, log)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treport, err := reports.New(map[string]interface{}{\"db\": srv.db})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tsrv.fetchFII = fetchFII\n\tsrv.fetchStock = fetchStock\n\tsrv.report = report\n\n\treturn &srv, nil\n}\n\n// HTML is a very basic html server to handle the reports.\nfunc HTML(opts ...ServerOption) {\n\tsrv, err := initServer(opts...)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\thttp.HandleFunc(\"/\", renderTemplate(srv))\n\n\tlog.Println(\"Listening on :3000...\")\n\terr = http.ListenAndServe(\":3000\", nil)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n}\n\n// renderTemplate renders the file related to the URL path inside the layout\n// templates. Template files are locates in _contentFS.\nfunc renderTemplate(srv *Server) http.HandlerFunc {\n\treturn func(w http.ResponseWriter, r *http.Request) {\n\t\tfp := filepath.Clean(r.URL.Path)\n\t\tif strings.HasPrefix(fp, `/`) || strings.HasPrefix(fp, `\\`) {\n\t\t\tfp = fp[1:] // remove starting \"/\" (or \"\\\" on Windows)\n\t\t}\n\t\tif fp == \"\" {\n\t\t\tfp = \"index.html\"\n\t\t}\n\n\t\tlog.Println(\"rendering\", fp)\n\n\t\t// TODO: load all templates outside this funcion\n\t\ttmpl, err := template.New(\"\").Funcs(template.FuncMap{\n\t\t\t\"ptFmtFloat\": ptFmtFloat,\n\t\t}).ParseFS(_contentFS, \"layout.html\", fp)\n\n\t\tif err != nil {\n\t\t\tlog.Println(err)\n\t\t\treturn\n\t\t}\n\n\t\t// Set the payload according to the URL path\n\t\tvar payload interface{}\n\t\tif strings.Contains(fp, \"fii.html\") && r.Method == http.MethodPost {\n\t\t\tcodes := parseCodes(r.FormValue(\"codes\"))\n\t\t\tmonths := parseNumeric(r.FormValue(\"months\"), 1)\n\t\t\tpayload = fiiDividendsPayload(srv, codes, months)\n\t\t}\n\n\t\terr = tmpl.ExecuteTemplate(w, \"layout\", payload)\n\t\tif err != nil {\n\t\t\tlog.Println(err)\n\t\t}\n\t}\n}\n\nfunc ptFmtFloat(f float64) string {\n\tp := message.NewPrinter(language.BrazilianPortuguese)\n\treturn p.Sprintf(\"%.2f\", f)\n}\n\nfunc parseCodes(text string) []string {\n\tvar codes []string\n\tfor _, field := range strings.FieldsFunc(text, split) {\n\t\tfield = strings.TrimSpace(field)\n\t\tif len(field) == len(\"ABCD11\") {\n\t\t\tcodes = append(codes, field)\n\t\t}\n\t}\n\n\treturn codes\n}\n\nfunc split(r rune) bool {\n\treturn r == ' ' || r == ',' || r == ';' || r == '\\n'\n}\n\n// parseNumeric converts \"numeric\" to integer, or returns \"alt\" in case of error.\nfunc parseNumeric(numeric string, alt int) int {\n\tn, err := strconv.Atoi(numeric)\n\tif err != nil {\n\t\tn = alt\n\t}\n\treturn n\n}\n" }, { "path": "server/templates/fii.html", "content": "{{define \"body\"}}\n\n\n\n

Rendimentos dos FII

\n\n
\n \n \n \n \n \n
\n\n{{if not .Data}}\n\n{{else}}\n{{range .Data}}\n

\n ⯆ {{.Code}}\n \n

\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n {{range .Values}}\n \n \n \n \n \n \n \n \n \n {{end}}\n \n
{{.Name}}\n YeldYeld\n \n
Data ComRendimentoCotaçãoa.m.a.a.a.m.a.a.
{{.Date}}R$ {{ptFmtFloat .Dividend}}R$ {{ptFmtFloat .Quote}}{{ptFmtFloat .Yeld}}%{{ptFmtFloat .YeldYear}}%
\n{{end}}\n{{end}}\n\n{{end}}" }, { "path": "server/templates/financials.html", "content": "{{define \"body\"}}\n

Finanças

\n{{end}}" }, { "path": "server/templates/index.html", "content": "{{define \"body\"}}\n\n

Relatórios

\nRendimentos dos FII\n
\nFinanças\n\n{{end}}" }, { "path": "server/templates/layout.html", "content": "{{define \"layout\"}}\n\n\n\n\n \n \n Rapina\n 🦅\n \">\n \n \n \n\n\n\n
\n
\n Rapina\n
\n FII:Rendimentos\n Ações:Finanças\n
\n
\n
\n
\n\n
\n
\n {{template \"body\" .}}\n
\n
\n\n \n\n\n\n{{end}}" }, { "path": "stock.go", "content": "package rapina\n\nimport \"io\"\n\n// StockStorage is the interface that contains the methods needed to parse, save and\n// retrieve stock data to/from a storage.\ntype StockStorage interface {\n\tQuote(code, date string) (float64, error)\n\tCode(companyName, stockType string) (string, error)\n\tSave(stream io.Reader, code string) (int, error)\n}\n" } ]