Repository: dude333/rapina Branch: master Commit: d0d1de8ad991 Files: 79 Total size: 286.7 KB Directory structure: gitextract_5rc_9ybs/ ├── .githooks/ │ └── pre-commit ├── .github/ │ └── workflows/ │ └── test-lint-release.yml ├── .gitignore ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── README_en.md ├── cmd/ │ └── rapina/ │ ├── cmdutils.go │ ├── cmdutils_test.go │ ├── fii.go │ ├── fii_dividends.go │ ├── fii_monthly.go │ ├── flags.go │ ├── list.go │ ├── main.go │ ├── report.go │ ├── server.go │ └── update.go ├── common.go ├── common_test.go ├── errors.go ├── fetch/ │ ├── fetch.go │ ├── fetch_fii.go │ ├── fetch_fii_test.go │ ├── fetch_http.go │ ├── fetch_http_test.go │ ├── fetch_stock.go │ ├── fetch_test.go │ └── unzip.go ├── fii.go ├── go.mod ├── go.sum ├── logger.go ├── parsers/ │ ├── codeaccounts.go │ ├── companies.go │ ├── fii.go │ ├── fiidb.go │ ├── financial.go │ ├── financial_test.go │ ├── fre.go │ ├── fuzzy.go │ ├── fuzzy_test.go │ ├── md5.go │ ├── md5_test.go │ ├── meta/ │ │ ├── meta_bpa_cia_aberta.txt │ │ ├── meta_bpp_cia_aberta.txt │ │ ├── meta_dfc_md_cia_aberta.txt │ │ ├── meta_dfc_mi_cia_aberta.txt │ │ └── meta_dre_cia_aberta.txt │ ├── sectors.go │ ├── sectors_test.go │ ├── stock.go │ ├── stock_test.go │ ├── tables.go │ └── transform.go ├── progress/ │ ├── cmd/ │ │ └── main.go │ └── progress.go ├── reports/ │ ├── db.go │ ├── db_test.go │ ├── excel.go │ ├── format.go │ ├── format_test.go │ ├── list.go │ ├── logger.go │ ├── logger_test.go │ ├── reports.go │ ├── reports_fii.go │ ├── reports_html.go │ └── reports_test.go ├── server/ │ ├── fs_dev.go │ ├── fs_prod.go │ ├── payload.go │ ├── server.go │ └── templates/ │ ├── fii.html │ ├── financials.html │ ├── index.html │ └── layout.html └── stock.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .githooks/pre-commit ================================================ #!/bin/sh # git config core.hooksPath .githooks echo "Running pre-commit checks at `pwd`" { echo "golangci-lint run ./..." golangci-lint run ./... } || { exitStatus=$? if [ $exitStatus ]; then printf "\nLint errors in your code, please fix them and try again." exit 1 fi } { echo "go test ./..." go test ./... } || { exitStatus=$? if [ $exitStatus ]; then printf "\nTest errors in your code, please fix them and try again." exit 1 fi } ================================================ FILE: .github/workflows/test-lint-release.yml ================================================ name: Test, Lint & Release on: [ push, pull_request ] jobs: go-test: strategy: fail-fast: false matrix: go: ['1.21.1'] platform: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: - if: github.actor == 'nektos/act' name: act workaround run: apt update && apt install -y zstd gcc git - name: Set up Go uses: actions/setup-go@v2 with: go-version: ${{ matrix.go }} - name: Show go version run: go version - name: Checkout uses: actions/checkout@v2 - name: go mod package cache uses: actions/cache@v2 with: # In order: # * Module download cache # * Build cache (Linux) # * Build cache (Mac) # * Build cache (Windows) path: | ~/go/pkg/mod ~/.cache/go-build ~/Library/Caches/go-build %LocalAppData%\go-build key: ${{ runner.os }}-go-${{ matrix.go }}-${{ hashFiles('**/go.mod') }} restore-keys: | ${{ runner.os }}-go-${{ matrix.go }} - name: Run tests run: go test -short -cover ./... go-lint: name: lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: golangci-lint uses: golangci/golangci-lint-action@v2 with: version: latest xgo: if: github.event_name == 'push' && contains(github.ref, 'refs/tags/') needs: [go-test, go-lint] strategy: fail-fast: false matrix: go_version: [ 1.21.x ] runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 - name: Get current date id: date run: echo "::set-output name=date::$(date +'%F')" - name: Get current git tag or commit id: tag run: echo "::set-output name=tag::$(git describe --tags --always)" - name: Build with xgo uses: crazy-max/ghaction-xgo@v1 with: xgo_version: latest go_version: ${{ matrix.go_version }} pkg: cmd/rapina dest: build prefix: rapina-${{ steps.tag.outputs.tag }} targets: windows/386,windows/amd64,linux/386,linux/amd64,darwin/386,darwin/amd64 v: false x: false race: false ldflags: -s -w -X main.build=${{ steps.date.outputs.date }} -X main.version=${{ steps.tag.outputs.tag }} buildmode: default - name: Run UPX uses: gacts/upx@master with: dir: 'build' upx_args: '-9' - name: Checksum run: | cd build sha1sum rapina* > sha1sum.txt - name: Generate changelog id: changelog uses: metcalfc/changelog-generator@v1.0.0 with: myToken: ${{ secrets.GITHUB_TOKEN }} - name: Release uses: softprops/action-gh-release@v1 with: files: build/* body: ${{ steps.changelog.outputs.changelog }} draft: false prerelease: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .gitignore ================================================ # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib debug *.db-journal .vscode bin/** # Temporary and data files *.zip **/.data *.csv *.xls* *.db *.old .vscode/* *.sql *_string.go *.yaml *.yml !.travis.yml wiki/** .DS_Store # Test binary, build with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Skip these (keep this at the end) !.github/** # Dependency Analytics target/** ================================================ FILE: LICENSE ================================================ The MIT License (MIT) Copyright © 2018 Adriano P Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ BUILDDIR=./cmd/... SOURCEDIR=. SOURCES := $(shell find $(SOURCEDIR) -name '*.go') BINARYDIR=./bin/ BINARY=bin/rapina WINBINARY=bin/rapina.exe OSXBINARY=bin/rapina-osx VERSION=`git describe --tags --always` BUILD_TIME=`date +%F` export GO111MODULE=on # Setup the -ldflags option for go build here, interpolate the variable values LDFLAGS=-ldflags "-w -s -X main.version=${VERSION} -X main.build=${BUILD_TIME}" .DEFAULT_GOAL: $(BINARY) $(BINARY): $(SOURCES) $(wildcard ../*.go) $(wildcard ../parsers/*.go) $(wildcard ../reports/*.go) CGO_CFLAGS="-O2 -Wno-return-local-addr" go build ${LDFLAGS} -o $(BINARYDIR) $(BUILDDIR) win: $(SOURCES) # go get -v -d ../... GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc CXX=i686-w64-mingw32-g++ CGO_CFLAGS="-O2 -Wno-return-local-addr" CGO_LDFLAGS="-lssp -w" go build ${LDFLAGS} -o ${BINARYDIR} $(BUILDDIR) osx: $(SOURCES) # go get -v -d ../... GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 CC=o64-clang CXX=o64-clang++ CGO_CFLAGS="-O2 -Wno-return-local-addr" CGO_LDFLAGS="-w" go build ${LDFLAGS} -o ${BINARYDIR} $(BUILDDIR) .PHONY: install install: go install ${LDFLAGS} ./... .PHONY: clean clean: if [ -f ${BINARY} ] ; then rm ${BINARY} ; fi .PHONY: list list: cd .. && go list -f '{{ join .Imports "\n" }}' ================================================ FILE: NOTICE ================================================ ================================================================================ | Open Database License (ODbL) | Contains information from "Portal Dados Abertos CVM", which is made available here under the Open Database License (ODbL). https://www.opendatacommons.org/licenses/odbl/1.0/ ================================================================================ | BSD-2-Clause | ==> github.com/pkg/errors: Copyright (c) 2015, Dave Cheney . All rights reserved. https://opensource.org/licenses/BSD-2-Clause ================================================================================ | BSD 3-Clause License | ==> github.com/360EntSecGroup-Skylar/excelize: Copyright (c) 2016 - 2018 360 Enterprise Security Group, Endpoint Security, inc. All rights reserved. ==> github.com/manifoldco/promptui: Copyright (c) 2017, Arigato Machine Inc. All rights reserved. https://opensource.org/licenses/BSD-3-Clause ================================================================================ | Apache License, Version 2.0 | ==> github.com/spf13/cobra: Copyright © 2013 Steve Francia . http://www.apache.org/licenses/LICENSE-2.0 ================================================================================ | The MIT License (MIT) | ==> github.com/mattn/go-sqlite3: Copyright (c) 2014 Yasuhiro Matsumoto. https://opensource.org/licenses/MIT ================================================ FILE: README.md ================================================ # 𝚛𝚊𝚙𝚒𝚗𝚊 Download e processamento de dados[1](#disclaimer) financeiros de empresas brasileiras diretamente da [CVM](http://dados.cvm.gov.br/dados/CIA_ABERTA/DOC/DFP/). [![GitHub release](https://img.shields.io/github/tag/dude333/rapina.svg?label=latest)](https://github.com/dude333/rapina/releases) [![Travis](https://img.shields.io/travis/dude333/rapina/master.svg)](https://travis-ci.org/dude333/rapina) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) Este programa baixa e processa os arquivos CSV do site da CVM e os armazena em um banco de dados local (sqlite), onde são extraídos os dados **consolidados** do balanço patrimonial, fluxo de caixa, DRE (demonstração de resultado), DVA (demonstração de valor adicionado). São coletados vários arquivos CSV desde 2010. Cada um destes arquivos contém informações do ano corrente e também do ano anterior, dessa forma foi possível extrair também os dados de 2009. Com base nestes dados, são criados os relatórios por empresa, com um comparativo de outras empresas do mesmo setor. A classificação dos setores é baixada do site da Bovespa e armazenada no arquivo setores.yml (no formato [YAML](https://medium.com/@akio.miyake/introdu%C3%A7%C3%A3o-b%C3%A1sica-ao-yaml-para-ansiosos-2ac4f91a4443)), que pode ser editado para se adequar aos seus critérios, caso necessário. A partir do release v0.11.0, passou-se a usar os dados trimestrais para compor os valores do ano corrente, usando-se para isso os últimos 4 trimestre ([TTM](#ttm-calc)), ou seja, a soma dos dados trimestrais do ano corrente com alguns do ano anterior, mantendo-se assim uma mesma base de comparação com os anos anteriores. | :memo: | **NOTA**: Desenvolvi a [versão 2 do rapina](https://github.com/dude333/rapinav2) com o intuito de criar relatórios trimestrais. Pretendo integrar essa nova funcionalidade a este repositório no futuro. | |---------------|:------------------------| # 1. Instalação Não é necessário instalar, basta baixar o executável da [página de release](https://github.com/dude333/rapina/releases) e renomeie o executável para `rapina.exe` (no caso do Windows) ou `rapina` (para o Linux ou macOS). Abra o terminal ([CMD](https://superuser.com/a/340051/61616) no Windows) e rode os comandos listados abaixo. # 2. Uso Na primeira vez, rodar o seguinte comando para baixar e processar os arquivos do site da CVM: ./rapina update Depois, para obter o relatório de uma determinada empresa, com o resumo das empresas do mesmo setor: ./rapina report _Eventualmente, as empresas corrigem algum dado e enviam um novo arquivo à CVM, então é recomendável rodar o `rapina update` periodicamente._ # 3. Detalhe dos Comandos ## 3.1. update **Download e armazenamento de dados financeiros no banco de dados local.** ./rapina update [-s] Baixa todos os arquivos disponíveis no servidor da CVM, processa o conteúdo e o armazena num banco de dados sqlite em `.data/rapina.db`. Este comando deve ser executado **pelo menos uma vez** antes dos outros comandos. ### 3.1.1 Opção ``` -s, --sectors Baixa a classificação setorial das empresas e fundos negociados na B3 ``` Usado para obter apenas o arquivo de classificação setorial atualizado. ## 3.2. list **Listas** ./rapina list ### 3.2.1 Lista todas as empresas disponíveis ``` -e, --empresas Lista todas as empresas disponíveis ``` ### 3.2.2 Lista as empresas do mesmo setor ``` -s, --setor string Lista todas as empresas do mesmo setor ``` Por exemplo, para listar todas as empras do mesmo setor do Itaú: `./rapina lista -s itau` O resultado mostra a lista das empresas do mesmo setor contidos no banco de dados e no arquivo **setores.yml**, que você pode editar caso queira realocar os setores das empresas. ### 3.2.3 Lista empresas com critério de lucro líquido ``` -l, --lucroLiquido número Lista empresas com lucros lucros positivos e com a taxa de crescimento definida ``` Lista as empresas com lucros líquidos positivos e com uma taxa de crescimento definida em relação ao mês anterior. Por exemplo: * Para listar as empresas com crescimento mínimo de 10% em relação ao ano anterior: `./rapina list -l 0.1` * Para listar as empresas com variação no lucro de maiores que -5% em relação ao ano anterior: `./rapina list -l -0.05` ## 3.3. report **Cria uma planilha com os dados financeiros de uma empresa.** ./rapina report [opções] empresa Será criada uma planilha com os dados financeiros (BP, DRE, DFC) e, em outra aba, o resumo de todas as empresas do mesmo setor. A lista setorial é obtida da B3 e salva no arquivo `setor.yml` (via comando `update -s`). Caso deseje alterar o agrupamento setorial, basta editar este arquivo. Mas lembre-se que ao rodar o `update -s` o arquivo será sobrescrito. No **Linux** ou **macOS**, use as setas para navegar na lista das empresas. No **Windows**, use j e k. ### 3.3.1. Opções ``` -a, --all Mostra todos os indicadores -x, --extraRatios Reporte de índices extras -F, --fleuriet Capital de giro no modelo Fleuriet -o, --omitSector Omite o relatório das empresas do mesmo setor -d, --outputDir string Diretório onde o relatório será salvo (default "reports") -s, --scriptMode Para modo script (escolhe a empresa com nome mais próximo) -f, --showShares Mostra o número de ações e free float ``` ### 3.3.2. Exemplos ./rapina report WEG A planilha será salva em `./reports` ./rapina report "TEC TOY" -s -d /tmp/output A planilha será salva em `/tmp/output` # 4. Nova funções ## 4.1. fii **Relatórios relacionados aos Fundos de Investimento Imobiliários** ### 4.1.1. rendimentos ./rapina fii rendimentos [-n] ABCD11 EFGH11... Onde `-n` é o número de meses a serem apresentados. E como parâmetros, passe uma lista de FIIs separados por espaço. #### 4.1.1.1 Exemplo ./rapina fii rendimentos -n 2 knip11 hfof11 ``` ------------------------------------------------------------------- KNIP11 ------------------------------------------------------------------- DATA COM RENDIMENTO COTAÇÃO YELD YELD a.a. ---------- ---------- ---------- ------ --------- 2021-04-30 R$ 1,00 R$ 113,00 0,88% 11,15% 2021-03-31 R$ 1,02 R$ 115,95 0,88% 11,08% ------------------------------------------------------------------- HFOF11 ------------------------------------------------------------------- DATA COM RENDIMENTO COTAÇÃO YELD YELD a.a. ---------- ---------- ---------- ------ --------- 2021-04-30 R$ 0,60 R$ 99,75 0,60% 7,46% 2021-03-31 R$ 0,56 R$ 100,70 0,56% 6,88% ------------------------------------------------------------------- ``` # 4.2. server **Web server para visualização dos relatórios no browser** ## 4.2.1. Exemplo ./rapina server 2021/05/11 19:23:15 Listening on :3000... Para visualizar a página, abrir o link http://localhost:3000 **NOTA:** Por hora só está disponível o relatório de rendimentos de FIIs. # 5. Possíveis problemas Algumas distribuições Linux (Fedora 34, por exemplo) podem encontrar problemas com as autoridades certificadores (Global Sign) presentes nos certificados SSL dos websites da B3. Em caso de erro `x509: certificate signed by unknown authority`, deve-se importar manualmente o Root CA para o trusted database do sistemas operacional: **Fedora 34 / CentOS** 1. Realizar o download do Issuer Root Cert `curl http://secure.globalsign.com/cacert/gsrsaovsslca2018.crt > /tmp/global-signer.der` 2. Converter de .der para .pem `openssl x509 -inform der -in /tmp/global-signer.der -out /tmp/globalsignroot.pem` 3. Importar .pem arquivo para pasta de anchors `sudo cp /tmp/globalsignroot.pem /usr/share/pki/ca-trust-source/anchors/` 4. Atualizar base de trusted certificates `sudo update-ca-trust` **Ubuntu** 1. Realizar o download do Issuer Root Cert `curl https://secure.globalsign.net/cacert/Root-R1.crt > /tmp/GlobalSign_Root_CA.crt` `curl https://secure.globalsign.net/cacert/Root-R2.crt > /tmp/GlobalSign_Root_CA_R2.crt` 2. Importar .crt arquivos para pasta de certificados `sudo cp /tmp/GlobalSign_Root_CA.crt /usr/local/share/ca-certificates/` `sudo cp /tmp/GlobalSign_Root_CA_R2.crt /usr/local/share/ca-certificates/` 3. Atualizar base de trusted certificates `sudo update-ca-trust` # 6. Como compilar Se quiser compilar seu próprio executável, primeiro [baixe e instale](https://golang.org/dl/) o compilador Go (v1.16 ou maior). Depois execute estes passos: 1. `git clone github.com/dude333/rapina` 2. `cd rapina` 3. `make` O executável será criado na pasta `bin`. Você pode movê-lo para outro local. Ao rodar a primeira vez, apenar o executável é necessário, mas após rodá-lo, será criado um diretório `.data` que deverá ser movido junto com o executável, caso queira trazer o dados. IMPORTANTE: para compilar a biblioteca do sqlite, é necessário ter um compilador C instalado na máquina (para o Windows, mais detalhes [aqui](https://github.com/mattn/go-sqlite3#windows)). # 7. Contribua 1. Faça um fork deste projeto no [github.com](github.com/dude333/rapina) 2. `git clone https://github.com/`*your_username*`/rapina && cd rapina` 3. `git checkout -b `*my-new-feature* 4. Faça as modificações 5. `git add .` 6. `git commit -m 'Add some feature'` 7. `git push origin my-new-feature` 8. Crie um _pull request_ # 8. Screenshot ![WEG](https://i.imgur.com/czPhPkH.png) # 9. License MIT


1: *Os dados são fornecidos "no estado em que se encontram" e somente para fins informativos, não para fins comerciais ou de consultoria.* ================================================ FILE: README_en.md ================================================ # 𝚛𝚊𝚙𝚒𝚗𝚊 Download and process Brazilian companies' financial data directly from [CVM](http://dados.cvm.gov.br/dados/CIA_ABERTA/DOC/DFP/). [[Em português](./README.md)] [![GitHub release](https://img.shields.io/github/tag/dude333/rapina.svg?label=latest)](https://github.com/dude333/rapina/releases) [![Travis](https://img.shields.io/travis/dude333/rapina/master.svg)](https://travis-ci.org/dude333/rapina) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE) # 1. Installation No installation required, just download the [latest released executable](https://github.com/dude333/rapina/releases). Then open a terminal ([CMD](https://superuser.com/a/340051/61616) on Windows) and run the commands shown below. # 2. Commands For the first time, run the following command: ./rapina get Then, to get a company report, together with a summary for the companies from the same sector: ./rapina report ## 2.1. `get`| Download and store financial data into the local database ./rapina get [-s] It downloads all files from CVM web server, parses their contents and stores on a sqlite database at `.data/rapina.db`. This command must be run **at least once** before you run the other commands. ### 2.1.1 Option ``` -s, --sectors Download and sector classification for companies listed at B3 ``` Used to get only a summary for the other companies from the same sector. [![asciicast](https://asciinema.org/a/656x2hrtCFFZLVLa9fGGcetw7.svg)](https://asciinema.org/a/656x2hrtCFFZLVLa9fGGcetw7?speed=4&autoplay=1&loop=1) ## 2.2. `list`| List all companies ./rapina list [![asciicast](https://asciinema.org/a/TbJyGaOodJUxEzjDySQu3MaEW.svg)](https://asciinema.org/a/TbJyGaOodJUxEzjDySQu3MaEW?autoplay=1&loop=1) ## 2.3. `report`| Create a spreadsheet with a company financial data ./rapina report [flags] company_name A spreadsheet with the financial data will be created and, on another sheet, the summary of all companies in the same sector. The sector list is obtained from B3 and saved in the `sector.yml` file (via `get -s` command). If you want to change the sector grouping, just edit this file. ### 2.3.1. Options ``` -d, --outputDir string Output directory [default: ./reports] -s, --scriptMode Does not show companies list; uses the most similar company name ``` On **Linux** or **macOS**, use the arrow keys to navigate through the companies list. On **Windows**, use j and k. [![asciicast](https://asciinema.org/a/jhmHxzgROtc8EBh3tkSwYTaa9.svg)](https://asciinema.org/a/jhmHxzgROtc8EBh3tkSwYTaa9?autoplay=1&loop=1) ### 2.3.2. Examples ./rapina report WEG The spreadsheet will be saved at `./reports` ./rapina report "TEC TOY" -s -d /tmp/output The spreadsheet will be saved at `/tmp/output` # 3. Troubleshooting Some Linux distributions (e.g. Fedora 34) might face some issues regarding the signer authority (Global Sign) that B3 is using on its SSL certificates. In case of `x509: certificate signed by unknown authority` error, one should manually import the Root CA certificate into the O.S. trusted database: **Fedora 34 / CentOS** 1. Download the Issuer Root Cert `curl http://secure.globalsign.com/cacert/gsrsaovsslca2018.crt > /tmp/global-signer.der` 2. Convert from .der to .pem `openssl x509 -inform der -in /tmp/global-signer.der -out /tmp/globalsignroot.pem` 3. Move the .pem file to the anchors folder `sudo cp /tmp/globalsignroot.pem /usr/share/pki/ca-trust-source/anchors/` 4. Update the trusted certificates database `sudo update-ca-trust` **Ubuntu** 1. Download the Issuer Root Cert `curl https://secure.globalsign.net/cacert/Root-R1.crt > /tmp/GlobalSign_Root_CA.crt` `curl https://secure.globalsign.net/cacert/Root-R2.crt > /tmp/GlobalSign_Root_CA_R2.crt` 2. Move the .crt files to the certificates folder `sudo cp /tmp/GlobalSign_Root_CA.crt /usr/local/share/ca-certificates/` `sudo cp /tmp/GlobalSign_Root_CA_R2.crt /usr/local/share/ca-certificates/` 3. Update the trusted certificates database `sudo update-ca-trust` # 4. How to compile If you want to compile your own executable, you need first to [download and install](https://golang.org/dl/) the Go compiler. Then follow these steps: 1. `go get github.com/dude333/rapina` 2. `cd $GOPATH/src/github.com/dude333/rapina` 3. Change to the cli directory (`cd cli`) 4. Compile using the Makefile (`make`). _To cross compile for Windows on Linux, use `make win`_. # 5. Contributing 1. Fork it 2. `cd $GOPATH/src/github.com/your_username` 3. Download your fork to your PC (`git clone https://github.com/your_username/rapina && cd rapina`) 4. Create your feature branch (`git checkout -b my-new-feature`) 5. Make changes and add them (`git add .`) 6. Commit your changes (`git commit -m 'Add some feature'`) 7. Push to the branch (`git push origin my-new-feature`) 8. Create new pull request # 6. Screenshot ![WEG](https://i.imgur.com/czPhPkH.png) # 7. License MIT ================================================ FILE: cmd/rapina/cmdutils.go ================================================ package main import ( "database/sql" "fmt" "os" "path/filepath" "strings" "github.com/manifoldco/promptui" "github.com/pkg/errors" ) // Directory where the DB and downloaded files are stored const dataDir = ".data" const yamlFile = "./setores.yml" // Parms holds the input parameters type Parms struct { // Company name to be processed Company string // SpcfctnCd to indentify the ticker SpcfctnCd string // Report format (xlsx/stdout) Format string // OutputDir: path of the output xlsx OutputDir string // YamlFile: file with the companies' sectors YamlFile string // Reports is a map with the reports and reports items to be printed Reports map[string]bool } // // openDatabase to be used by parsers and reporting // func openDatabase() (db *sql.DB, err error) { if err := os.MkdirAll(dataDir, os.ModePerm); err != nil { return nil, err } connStr := "file:" + dataDir + "/rapina.db?cache=shared&mode=rwc&_journal_mode=WAL&_busy_timeout=5000" db, err = sql.Open("sqlite3", connStr) if err != nil { return db, errors.Wrap(err, "database open failed") } db.SetMaxOpenConns(1) return } // // promptUser presents a navigable list to be selected on CLI // func promptUser(list []string, label string) (result string) { if label == "" { label = "Selecione a Empresa" } templates := &promptui.SelectTemplates{ Help: `{{ "Use estas teclas para navegar:" | faint }} {{ .NextKey | faint }} ` + `{{ .PrevKey | faint }} {{ .PageDownKey | faint }} {{ .PageUpKey | faint }} ` + `{{ if .Search }} {{ "and" | faint }} {{ .SearchKey | faint }} {{ "toggles search" | faint }}{{ end }}`, } prompt := promptui.Select{ Label: label, Items: list, Templates: templates, } _, result, err := prompt.Run() if err != nil { fmt.Printf("Prompt failed %v\n", err) return } return } // // filename cleans up the filename and returns the path/filename func filename(path, name string) (fpath string, err error) { clean := func(r rune) rune { switch r { case ' ', ',', '/', '\\': return '_' } return r } path = strings.TrimSuffix(path, "/") name = strings.TrimSuffix(name, ".") name = strings.Map(clean, name) fpath = filepath.FromSlash(path + "/" + name + ".xlsx") const max = 50 var x int for x = 1; x <= max; x++ { _, err = os.Stat(fpath) if err == nil { // File exists, try again with another name fpath = fmt.Sprintf("%s/%s(%d).xlsx", path, name, x) } else if os.IsNotExist(err) { err = nil // reset error break } else { err = fmt.Errorf("file %s stat error: %v", fpath, err) return } } if x > max { err = fmt.Errorf("remova o arquivo %s/%s.xlsx antes de continuar", path, name) return } // Create directory _ = os.Mkdir(path, os.ModePerm) // Check if the directory was created if _, err := os.Stat(path); os.IsNotExist(err) { return "", errors.Wrap(err, "diretório não pode ser criado") } return } ================================================ FILE: cmd/rapina/cmdutils_test.go ================================================ package main import ( "os" "path/filepath" "testing" ) func TestFilename(t *testing.T) { tempDir, _ := os.MkdirTemp("", "rapina-test") table := []struct { path string name string expected string }{ {tempDir + "/test", "sample", tempDir + "/test/sample.xlsx"}, {tempDir, "File 100", tempDir + "/File_100.xlsx"}, {tempDir, "An,odd/file\\name", tempDir + "/An_odd_file_name.xlsx"}, } for _, x := range table { returned, err := filename(x.path, x.name) expected := filepath.FromSlash(x.expected) if err != nil { t.Errorf("filename returned an error %v.", err) } else if returned != expected { t.Errorf("filename got: %s, want: %s.", returned, expected) } } } ================================================ FILE: cmd/rapina/fii.go ================================================ /* Copyright © 2021 Adriano P Distributed under the MIT License. */ package main import ( "fmt" "os" "path/filepath" "github.com/spf13/cobra" ) type fiiFlags struct { num int // number of months since current dividends fiiDividendsFlags monthly fiiMonthlyFlags } // fiiCmd represents the fii command var fiiCmd = &cobra.Command{ Use: "fii", Short: "Comando relacionados aos FIIs", Long: `Comando relacionado aos Fundos de Investiment Imobiliários (FII).`, Run: func(cmd *cobra.Command, args []string) { _ = cmd.Help() }, Example: func() string { return fmt.Sprintf("%s fii rendimentos KNIP11 KNCR11 HGLG11 -n 4", filepath.Base(os.Args[0])) }(), } func init() { rootCmd.AddCommand(fiiCmd) fiiCmd.PersistentFlags().IntVarP(&flags.fii.num, Fnum, "n", 1, "número de meses desde o último disponível") } ================================================ FILE: cmd/rapina/fii_dividends.go ================================================ /* Copyright © 2021 Adriano P Distributed under the MIT License. */ package main import ( "fmt" "log" "os" "path/filepath" "strings" "github.com/dude333/rapina/reports" "github.com/spf13/cobra" "github.com/spf13/viper" ) type fiiDividendsFlags struct { format string // output format of the report } // fiiDividendsCmd represents the rendimentos command var fiiDividendsCmd = &cobra.Command{ Use: "rendimentos", Aliases: []string{"rend", "dividendos", "dividends", "div"}, Args: cobra.MinimumNArgs(1), Short: "Lista os rendimentos de um FII", Long: `Lista os rendimentos de um Fundos de Investiment Imobiliários (FII).`, Run: func(cmd *cobra.Command, args []string) { // Number of reports n := flags.fii.num if n <= 0 { n = 1 } parms := make(map[string]string) // Verbose if flags.verbose { parms[Fverbose] = "true" } // Report format parms[Fformat] = flags.fii.dividends.format if err := FIIDividends(parms, args, n); err != nil { log.Println(err) } }, Example: func() string { return fmt.Sprintf("%s fii rendimentos KNIP11 KNCR11 HGLG11 -n 4", filepath.Base(os.Args[0])) }(), } func init() { fiiCmd.AddCommand(fiiDividendsCmd) fiiDividendsCmd.Flags().StringVarP(&flags.fii.dividends.format, Fformat, "f", "tabela", "formato do relatório: tabela|csv|csvrend") } // FIIDividends prints the dividends from 'code' for 'n' months, // starting from latest. func FIIDividends(parms map[string]string, codes []string, n int) error { for i := 0; i < len(codes); i++ { codes[i] = strings.ToUpper(codes[i]) } db, err := openDatabase() if err != nil { return err } opts := reports.FIITerminalOptions{ APIKey: viper.GetString("apikey"), DataDir: dataDir, } r, err := reports.NewFIITerminal(db, opts) if err != nil { return err } r.SetParms(parms) err = r.Dividends(codes, n) if err != nil { return err } return nil } ================================================ FILE: cmd/rapina/fii_monthly.go ================================================ /* Copyright © 2021 Adriano P Distributed under the MIT License. */ package main import ( "log" "strings" "github.com/dude333/rapina/reports" "github.com/spf13/cobra" "github.com/spf13/viper" ) type fiiMonthlyFlags struct { format string // output format of the report } // fiiMonthlyCmd represents the rendimentos command var fiiMonthlyCmd = &cobra.Command{ Hidden: true, Use: "mensal", Aliases: []string{"monthly"}, Args: cobra.MinimumNArgs(1), Short: "Lista os informes mensais de um FII", Long: `Lista os informes mensais de um Fundos de Investiment Imobiliários (FII).`, Run: func(cmd *cobra.Command, args []string) { // Number of reports n := flags.fii.num parms := make(map[string]string) // Verbose if flags.verbose { parms[Fverbose] = "true" } // Report format parms[Fformat] = flags.fii.monthly.format if err := FIIMonthly(parms, args, n); err != nil { log.Println(err) } }, } func init() { fiiCmd.AddCommand(fiiMonthlyCmd) fiiMonthlyCmd.Flags().StringVarP(&flags.fii.monthly.format, Fformat, "f", "tabela", "formato do relatório: tabela|csv|csvrend") } // FIIMonthly prints the monthly reports from 'code' for 'n' months, // starting from latest. func FIIMonthly(parms map[string]string, codes []string, n int) error { for i := 0; i < len(codes); i++ { codes[i] = strings.ToUpper(codes[i]) } db, err := openDatabase() if err != nil { return err } opts := reports.FIITerminalOptions{ APIKey: viper.GetString("apikey"), DataDir: dataDir, } r, err := reports.NewFIITerminal(db, opts) if err != nil { return err } r.SetParms(parms) err = r.Monthly(codes, n) if err != nil { return err } return nil } ================================================ FILE: cmd/rapina/flags.go ================================================ package main // Flags constants const ( // Root persistent Fverbose = "verbose" // fiiCmd persistent Fnum = "num" // fiiDividendsCmd Fformat = "format" ) ================================================ FILE: cmd/rapina/list.go ================================================ // Copyright © 2018 Adriano P // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. package main import ( "fmt" "github.com/dude333/rapina/reports" "github.com/pkg/errors" "github.com/spf13/cobra" ) // listCmd represents the list command var listCmd = &cobra.Command{ Use: "list", Short: "Lista informações armazenadas no banco de dados", } func init() { var ( listCompanies bool sector string netProfitRate float32 ) rootCmd.AddCommand(listCmd) listCmd.Flags().BoolVarP(&listCompanies, "empresas", "e", false, "Lista todas as empresas disponíveis") listCmd.Flags().StringVarP(§or, "setor", "s", "", "Lista todas as empresas do mesmo setor") listCmd.Flags().Float32VarP(&netProfitRate, "lucroLiquido", "l", -0.8, "Lista empresas com lucros lucros positivos e com a taxa de crescimento definida") listCmd.Run = func(cmd *cobra.Command, args []string) { var err error if listCmd.Flags().NFlag() == 0 { _ = listCmd.Help() return } if listCompanies { err = ListCompanies() } else if sector != "" { err = ListSector(sector, yamlFile) } else if listCmd.Flags().Changed("lucroLiquido") { err = ListCompaniesProfits(netProfitRate) } if err != nil { fmt.Println("[x]", err) } } } // // ListCompanies a company from DB to Excel // func ListCompanies() (err error) { db, err := openDatabase() if err != nil { return errors.Wrap(err, "fail to open db") } com, err := reports.ListCompanies(db) if err != nil { return errors.Wrap(err, "erro ao listar empresas") } for _, c := range com { fmt.Println(c) } return } // // ListSector shows all companies from the same sector as 'company' // func ListSector(company, yamlFile string) (err error) { db, err := openDatabase() if err != nil { return errors.Wrap(err, "fail to open db") } err = reports.ListSector(db, company, yamlFile) if err != nil { return errors.Wrap(err, "erro ao listar empresas") } return } // // ListCompaniesProfits lists companies profits // func ListCompaniesProfits(rate float32) (err error) { db, err := openDatabase() if err != nil { return errors.Wrap(err, "fail to open db") } err = reports.ListCompaniesProfits(db, rate) if err != nil { return errors.Wrap(err, "erro ao listar lucros") } return } ================================================ FILE: cmd/rapina/main.go ================================================ // Copyright © 2018 Adriano P // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. package main import ( "fmt" "os" "os/signal" "github.com/dude333/rapina/progress" homedir "github.com/mitchellh/go-homedir" "github.com/spf13/cobra" "github.com/spf13/viper" ) var flags = struct { verbose bool fii fiiFlags server serverFlags }{} var cfgFile string // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ Use: "rapina", Short: "Dados Financeiros de Empresas via CVM.", Long: ` Este programa coleta informações sobre os dados financeiros do site da CVM e os exporta para uma planilha. Dados usados: balanço patrimonial ativo e passivo, e também o demonstrativo de resultado do exercício (DRE).`, // Uncomment the following line if your bare application // has an action associated with it: // Run: func(cmd *cobra.Command, args []string) { }, } // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. func Execute() int { if err := rootCmd.Execute(); err != nil { fmt.Println(err) return 1 } return 0 } func init() { cobra.OnInitialize(initConfig) // Here you will define your flags and configuration settings. // Cobra supports persistent flags, which, if defined here, // will be global for your application. // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.cli.yaml)") // Cobra also supports local flags, which will only run // when this action is called directly. rootCmd.PersistentFlags().BoolVarP(&flags.verbose, Fverbose, "v", false, "Mostrar mensagens de execução") str := `Uso:{{if .Runnable}} {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} {{.CommandPath}} [command]{{end}}{{if gt (len .Aliases) 0}} Aliases: {{.NameAndAliases}}{{end}}{{if .HasExample}} Exemplos: {{.Example}}{{end}}{{if .HasAvailableSubCommands}} Comandos Disponíveis:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} Flags: {{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} Global Flags: {{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}} Tópicos de ajuda opcionais:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}} Use "{{.CommandPath}} [command] --help" para mais informações sobre um comando.{{end}} ` rootCmd.SetUsageTemplate(str) } // initConfig reads in config file and ENV variables if set. func initConfig() { if cfgFile != "" { // Use config file from the flag. viper.SetConfigFile(cfgFile) } else { // Find home directory. home, err := homedir.Dir() if err != nil { fmt.Println(err) os.Exit(1) } // Search config in home directory with name ".rapina" (without extension). viper.AddConfigPath(home) viper.AddConfigPath(".") viper.SetConfigName("config") } viper.AutomaticEnv() // read in environment variables that match // If a config file is found, read it in. if err := viper.ReadInConfig(); err == nil { fmt.Fprintf(os.Stderr, "[INFO] Usando arquivo de configuração %s\n\n", viper.ConfigFileUsed()) } } var ( version string build string ) func main() { fmt.Fprint(os.Stderr, "Rapina - Dados Financeiros de Empresas Brasileiras - ") fmt.Fprintf(os.Stderr, "%s-%s\n", version, build) fmt.Fprint(os.Stderr, "(2018-2020) github.com/dude333/rapina\n\n") progress.Cursor(false) defer func() { progress.Cursor(true) if err := recover(); err != nil { //catch fmt.Fprintf(os.Stderr, "Exception: %v\n", err) os.Exit(1) } }() // Handle Ctrl+C c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) go func() { <-c progress.Cursor(true) os.Exit(0) }() ret := Execute() progress.Cursor(true) os.Exit(ret) } ================================================ FILE: cmd/rapina/report.go ================================================ /// Copyright © 2018 Adriano P // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. package main import ( "fmt" "sort" "strings" "github.com/dude333/rapina/reports" "github.com/lithammer/fuzzysearch/fuzzy" "github.com/pkg/errors" "github.com/spf13/cobra" ) // Flags var scriptMode bool var all bool var showShares bool var extraRatios bool var fleuriet bool var omitSector bool var outputDir = "reports" var format string // output format of the report // reportCmd represents the report command var reportCmd = &cobra.Command{ Use: "report [-s] nome_empresa", Short: "Cria planilha com dados da companhia escolhida", Long: "Cria planilha com dados da companhia escolhida", Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { report(args[0]) }, } func init() { rootCmd.AddCommand(reportCmd) reportCmd.Flags().BoolVarP(&scriptMode, "scriptMode", "s", false, "Para modo script (escolhe a empresa com nome mais próximo)") reportCmd.Flags().BoolVarP(&all, "all", "a", false, "Mostra todos os indicadores") reportCmd.Flags().BoolVarP(&showShares, "showShares", "f", false, "Mostra o número de ações e free float") reportCmd.Flags().BoolVarP(&extraRatios, "extraRatios", "x", false, "Reporte de índices extras") reportCmd.Flags().BoolVarP(&fleuriet, "fleuriet", "F", false, "Capital de giro no modelo Fleuriet") reportCmd.Flags().BoolVarP(&omitSector, "omitSector", "o", false, "Omite o relatório das empresas do mesmo setor") reportCmd.Flags().StringVarP(&outputDir, "outputDir", "d", "reports", "Diretório onde o relatório será salvo") reportCmd.Flags().StringVarP(&format, "format", "r", "xlsx", "Formato do relatório: xlsx|stdout") } func report(company string) { var spcfctnCd string = "ON" company = SelectCompany(company, scriptMode) if company == "" { fmt.Println("[x] Empresa não encontrada") return } if strings.Contains(company, "@#") { companyWithTicker := strings.Split(company, "@#") company = companyWithTicker[0] spcfctnCd = companyWithTicker[1] } fmt.Println() fmt.Printf("[√] Criando relatório para %s ========\n", company) if all { extraRatios = true showShares = true fleuriet = true } r := make(map[string]bool) r["ExtraRatios"] = extraRatios r["ShowShares"] = showShares r["Fleuriet"] = fleuriet r["PrintSector"] = !omitSector parms := Parms{ Company: company, SpcfctnCd: spcfctnCd, Format: format, OutputDir: outputDir, YamlFile: yamlFile, Reports: r, } err := Report(parms) if err != nil { fmt.Println("[x]", err) } } // // SelectCompany returns the company name compared to the names // stored in the DB // func SelectCompany(company string, scriptMode bool) string { db, err := openDatabase() if err != nil { fmt.Println("[x]", err) return "" } companies, err := reports.ListCompanies(db) if err != nil { fmt.Println("[x]", err) return "" } // Do a fuzzy match on the company name against // all companies listed on the DB matches := make([]string, 0, 10) for _, c := range companies { if fuzzy.MatchNormalizedFold(company, c) { matches = append(matches, c) } } // Script mode if len(matches) >= 1 && scriptMode { rank := fuzzy.RankFindNormalizedFold(company, matches) if len(rank) <= 0 { return "" } sort.Sort(rank) return rank[0].Target } // Interactive menu if len(matches) >= 1 { result := promptUser(matches, "Selecione a Empresa") tickers, err := reports.ListTickers(db, result) if err != nil { fmt.Println("[x] Recuperando lista de tickers ", err) return result } // Interactive menu if len(tickers) > 0 { ticker := promptUser(tickers, "Selecione o ticker") resultWithTicker := fmt.Sprintf("%s@#%s", result, reports.GetSpcfctnCd(db, result, ticker)) return resultWithTicker } return result } return "" } // // Report a company from DB to Excel // func Report(p Parms) (err error) { db, err := openDatabase() if err != nil { return errors.Wrap(err, "fail to open db") } if p.OutputDir == "" { p.OutputDir = outputDir } file, err := filename(p.OutputDir, p.Company) if err != nil { return err } parms := map[string]interface{}{ "db": db, "dataDir": dataDir, "company": p.Company, "SpcfctnCd": p.SpcfctnCd, "format": p.Format, "filename": file, "yamlFile": p.YamlFile, "reports": p.Reports, } if p.Format == "stdout" { return reports.ReportToStdout(parms) } return reports.ReportToXlsx(parms) } ================================================ FILE: cmd/rapina/server.go ================================================ /* Copyright © 2021 Adriano P Distributed under the MIT License. */ package main import ( "log" "github.com/dude333/rapina/server" "github.com/spf13/cobra" "github.com/spf13/viper" ) type serverFlags struct { } // serverCmd represents the server command var serverCmd = &cobra.Command{ Use: "server", Short: "Inicia o servidor web", Long: `Comando para iniciar o servidor para a exibição dos dados via web browser.`, Run: func(cmd *cobra.Command, args []string) { parms := make(map[string]string) // Verbose if flags.verbose { parms[Fverbose] = "true" } err := serve(parms) if err != nil { log.Println(err) } }, } func init() { rootCmd.AddCommand(serverCmd) // serverCmd.Flags().IntVarP(&flags.server.num, // Fnum, "n", 1, "número de meses desde o último disponível") } func serve(parms map[string]string) error { db, err := openDatabase() if err != nil { return err } v := parms[Fverbose] == "true" server.HTML( server.WithDB(db), server.WithAPIKey(viper.GetString("apikey")), server.WithDataDir(dataDir), server.Verbose(v)) return nil } ================================================ FILE: cmd/rapina/update.go ================================================ // Copyright © 2018 Adriano P // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. package main import ( "errors" "fmt" "os" "github.com/dude333/rapina" "github.com/dude333/rapina/fetch" "github.com/dude333/rapina/reports" "github.com/spf13/cobra" "github.com/spf13/viper" ) var sectors bool // getUpdate represents the get command var getUpdate = &cobra.Command{ Use: "update", Aliases: []string{"get"}, Short: "Baixa os arquivos da CVM e atualiza o bando de dados", Long: `Baixa os arquivos do site da CVM, processa e os armazena no bando de dados.`, Run: func(cmd *cobra.Command, args []string) { db, err := openDatabase() if err != nil { fmt.Println("[x]", err) return } fmt.Println("[√] Coletando dados ===========") err = fetch.Sectors(yamlFile) if err != nil && !errors.Is(err, rapina.ErrFileNotUpdated) { fmt.Println("[x]", err) return } if err == nil { fmt.Println("[√] Arquivo salvo:", yamlFile) } // fmt.Println() // if sectors { // skip if -s flag is selected (dowload only the sectors) return } err = fetch.CVM(db, dataDir) if err != nil { fmt.Println("[x]", err) return } // Stock codes log := reports.NewLogger(os.Stderr) stock, err := fetch.NewStock(db, log, viper.GetString("apikey"), dataDir) if err != nil { log.Error(err.Error()) return } _ = stock.UpdateStockCodes() }, } func init() { rootCmd.AddCommand(getUpdate) getUpdate.Flags().BoolVarP(§ors, "sectors", "s", false, "Baixa a classificação setorial das empresas e fundos negociados na B3") } ================================================ FILE: common.go ================================================ package rapina import ( "fmt" "net/url" "path" "strconv" "strings" "time" ) // IsDate checks if date is in format YYYY-MM-DD. func IsDate(date string) bool { if len(date) != len("2021-04-26") || strings.Count(date, "-") != 2 { return false } y, errY := strconv.Atoi(date[0:4]) m, errM := strconv.Atoi(date[5:7]) d, errD := strconv.Atoi(date[8:10]) if errY != nil || errM != nil || errD != nil { return false } // Ok, we'll still be using this in 2200 :) if y < 1970 || y > 2200 { return false } if m < 1 || m > 12 { return false } nDays := [13]int{0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} if d < 1 || d > nDays[m] { return false } return true } // IsURL returns true if 'str' is a valid URL. func IsURL(str string) bool { u, err := url.Parse(str) return err == nil && u.Scheme != "" && u.Host != "" } // JoinURL joins strings as URL paths func JoinURL(base string, paths ...string) string { p := path.Join(paths...) return fmt.Sprintf("%s/%s", strings.TrimRight(base, "/"), strings.TrimLeft(p, "/")) } var _timeNow = time.Now // MonthsFromToday returns a list of months including the current. // Date formatted as YYYY-MM. func MonthsFromToday(n int) []string { if n < 1 { n = 1 } if n > 100 { n = 100 } now := _timeNow() now = time.Date(now.Year(), now.Month(), 15, 12, 0, 0, 0, time.UTC) var monthYears []string for ; n > 0; n-- { monthYears = append(monthYears, now.Format("2006-01")) now = now.AddDate(0, -1, 0) } return monthYears } // LastBusinessDayOfYear returns the last business day of the 'year' (the business // day before Dec 30). If current year, returns last business day before today. // Returns date as YYYY-MM-DD. func LastBusinessDayOfYear(year int) string { today := time.Now() if year == today.Year() { return LastBusinessDay(1) } date := time.Date(year, time.December, 29, 12, 0, 0, 0, time.UTC) if date.Weekday() == time.Saturday { date = date.AddDate(0, 0, -1) } if date.Weekday() == time.Sunday { date = date.AddDate(0, 0, -2) } return date.Format("2006-01-02") } // LastBusinessDay returns the most recent business day 'n' days before today. // Returns date as YYYY-MM-DD. func LastBusinessDay(n int) string { date := time.Now() if n > 0 { date = date.AddDate(0, 0, -n) } if date.Weekday() == time.Saturday { date = date.AddDate(0, 0, -1) } if date.Weekday() == time.Sunday { date = date.AddDate(0, 0, -2) } return date.Format("2006-01-02") } ================================================ FILE: common_test.go ================================================ package rapina import ( "reflect" "testing" "time" ) func TestIsDate(t *testing.T) { type args struct { date string } tests := []struct { name string args args want bool }{ { name: "should be true", args: args{date: "2021-04-26"}, want: true, }, { name: "should be true too", args: args{date: "2030-12-31"}, want: true, }, { name: "should be false", args: args{date: "2021-04-31"}, want: false, }, { name: "should be false too", args: args{date: "20/12/2000"}, want: false, }, { name: "should be false three", args: args{date: "2021-07-32"}, want: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := IsDate(tt.args.date); got != tt.want { t.Errorf("IsDate() = %v, want %v", got, tt.want) } }) } } func TestIsUrl(t *testing.T) { type args struct { str string } tests := []struct { name string args args want bool }{ { name: "should be true", args: args{str: "http://example.com/path"}, want: true, }, { name: "should be false", args: args{str: "example.com/path"}, want: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := IsURL(tt.args.str); got != tt.want { t.Errorf("IsUrl() = %v, want %v", got, tt.want) } }) } } func TestMonthsFromToday(t *testing.T) { timeNow1 := func() time.Time { return time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) } timeNow2 := func() time.Time { return time.Date(2009, time.March, 31, 23, 0, 0, 0, time.UTC) } type args struct { n int } tests := []struct { name string args args timeNow func() time.Time want []string }{ { name: "should show 3 months", args: args{n: 3}, timeNow: timeNow1, want: []string{"2009-11", "2009-10", "2009-09"}, }, { name: "should show 2 months", args: args{n: 2}, timeNow: timeNow2, want: []string{"2009-03", "2009-02"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _timeNow = tt.timeNow if got := MonthsFromToday(tt.args.n); !reflect.DeepEqual(got, tt.want) { t.Errorf("MonthsFromToday() = %#v, want %v", got, tt.want) } }) } } func TestLastBusinessDayOfYear(t *testing.T) { type args struct { year int } tests := []struct { name string args args want string }{ { name: "2022", args: args{2022}, want: "2022-12-29", }, { name: "2020", args: args{2020}, want: "2020-12-29", }, { name: "2017", args: args{2017}, want: "2017-12-29", }, { name: "2016", args: args{2016}, want: "2016-12-29", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := LastBusinessDayOfYear(tt.args.year); got != tt.want { t.Errorf("LastBusinessDayOfYear() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: errors.go ================================================ package rapina import "errors" // Error codes var ( ErrRecordExists = errors.New("insert ignored, register already exists") ErrFileNotUpdated = errors.New("file not updated") ErrInvalidAPIKey = errors.New("apiKey inválida, configure uma chave em" + " https://www.alphavantage.co/support/#api-key e adicione no arquivo" + " config.yml") ErrInvalidDate = errors.New("invalid date format") ) ================================================ FILE: fetch/fetch.go ================================================ // Copyright © 2018 Adriano P // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. package fetch import ( "database/sql" "fmt" "io" "net/http" "os" "path" "path/filepath" "strings" "time" "github.com/dude333/rapina/parsers" "github.com/dustin/go-humanize" _ "github.com/mattn/go-sqlite3" // requires CGO_ENABLED=1 and gcc "github.com/pkg/errors" ) var ( // ErrFileNotFound error ErrFileNotFound = errors.New("file not found") // ErrItemNotFound for string not found on []string ErrItemNotFound = errors.New("item not found") ) // // CVM fetches all statements from a range // of years // func CVM(db *sql.DB, dataDir string) error { now := time.Now().Year() try(processQuarterlyReport, db, dataDir, "Arquivo ITR não encontrado", now, now-1, 2) try(processAnnualReport, db, dataDir, "Arquivo DFP não encontrado", now-1, 2010, 2) try(processFREReport, db, dataDir, "Arquivo FRE não encontrado", now-1, 2010, 2) return nil } type fn func(*sql.DB, string, int) error // try to run the function 'f' 'n' times, in case there are network errors. func try(f fn, db *sql.DB, dataDir, errMsg string, now, limit, n int) { tries := n var err error for year := now; tries > 0 && year >= limit; year-- { fmt.Printf("[>] %d ---------------------\n", year) err = f(db, dataDir, year) if err == ErrFileNotFound { fmt.Printf("[x] %s\n", errMsg) tries-- continue } else if err != nil { fmt.Printf("[x] Erro ao processar arquivo de %d: %v\n", year, err) tries-- } else { tries = n } } } // processAnnualReport will get data from .zip files downloaded // directly from CVM and insert its data into the DB func processAnnualReport(db *sql.DB, dataDir string, year int) error { url := fmt.Sprintf("http://dados.cvm.gov.br/dados/CIA_ABERTA/DOC/DFP/DADOS/dfp_cia_aberta_%d.zip", year) zipfile := fmt.Sprintf("%s/dfp_%d.zip", dataDir, year) // Download files from CVM server fmt.Print("[ ] Download do arquivo DFP") files, err := fetchFiles(url, dataDir, zipfile) if err != nil { return err } dataTypes := []string{"BPA", "BPP", "DRE", "DFC_MD", "DFC_MI", "DVA"} for _, dt := range dataTypes { pattern := fmt.Sprintf("dfp_cia_aberta_%s_con_%d.csv", dt, year) reqFile, err := findFile(files, pattern) if err == ErrItemNotFound { filesCleanup(files) return fmt.Errorf("arquivo %s não encontrado", reqFile) } // Import file into DB if err = parsers.ImportCsv(db, dt, reqFile); err != nil { return err } } filesCleanup(files) // remove remaining (unused) files return nil } // // processQuarterlyReport download quarter files from CVM and store them on DB // func processQuarterlyReport(db *sql.DB, dataDir string, year int) error { url := fmt.Sprintf("http://dados.cvm.gov.br/dados/CIA_ABERTA/DOC/ITR/DADOS/ITR_CIA_ABERTA_%d.zip", year) zipfile := fmt.Sprintf("%s/itr_%d.zip", dataDir, year) // Download files from CVM server fmt.Print("[ ] Download do arquivo ITR") files, err := fetchFiles(url, dataDir, zipfile) if err != nil { return err } dataTypes := []string{"BPA", "BPP", "DRE", "DFC_MD", "DFC_MI", "DVA"} for _, dt := range dataTypes { pattern := fmt.Sprintf("ITR_CIA_ABERTA_%s_con_%d.csv", dt, year) reqFile, err := findFile(files, pattern) if err == ErrItemNotFound { filesCleanup(files) return fmt.Errorf("arquivo %s não encontrado", reqFile) } // Import file into DB (the trick is to add ITR to the data type so the // ImportCSV loads that into the ITR table) if err = parsers.ImportCsv(db, dt+"_ITR", reqFile); err != nil { return err } } filesCleanup(files) // remove remaining (unused) files return nil } // // processFREReport download FRE (Reference Form) files from CVM and store // them on DB. // func processFREReport(db *sql.DB, dataDir string, year int) error { url := fmt.Sprintf("http://dados.cvm.gov.br/dados/CIA_ABERTA/DOC/FRE/DADOS/fre_cia_aberta_%d.zip", year) zipfile := fmt.Sprintf("%s/fre_%d.zip", dataDir, year) // Download files from CVM server fmt.Print("[ ] Download do arquivo FRE") files, err := fetchFiles(url, dataDir, zipfile) if err != nil { return err } patterns := []string{"fre_cia_aberta_distribuicao_capital_%d.csv"} for _, p := range patterns { pattern := fmt.Sprintf(p, year) reqFile, err := findFile(files, pattern) if err == ErrItemNotFound { filesCleanup(files) return fmt.Errorf("arquivo %s não encontrado", reqFile) } if err = parsers.ImportCsv(db, "FRE", reqFile); err != nil { return err } } filesCleanup(files) // remove remaining (unused) files return nil } // // fetchFiles from web verbosely. // func fetchFiles(url, dataDir string, zipfile string) ([]string, error) { return fetchFilesVerbosity(url, dataDir, zipfile, true) } // // fetchFilesVerbosity from web. // func fetchFilesVerbosity(url, dataDir string, zipfile string, verbose bool) ([]string, error) { // Download file from web err := downloadFile(url, zipfile, verbose) if verbose { fmt.Println() } if err != nil { return nil, ErrFileNotFound } // Unzip and list files files, err := Unzip(zipfile, dataDir, verbose) os.Remove(zipfile) if err != nil { return nil, errors.Wrap(err, "could not unzip file") } return files, nil } // WriteCounter counts the number of bytes written the io.Writer. // source: https://golangcode.com/download-a-file-with-progress/ type WriteCounter struct { Total uint64 } // Write implements the io.Writer interface and will be passed to io.TeeReader(). func (wc *WriteCounter) Write(p []byte) (int, error) { n := len(p) wc.Total += uint64(n) wc.printProgress() return n, nil } func (wc WriteCounter) printProgress() { fmt.Printf("\r[ %7s", humanize.Bytes(wc.Total)) } // // downloadFile source: https://stackoverflow.com/a/33853856/276311 // func downloadFile(url, filepath string, verbose bool) (err error) { // Create dir if necessary basepath := path.Dir(filepath) if err = os.MkdirAll(basepath, os.ModePerm); err != nil { return err } // Create the file out, err := os.Create(filepath) if err != nil { return err } // https://www.joeshaw.org/dont-defer-close-on-writable-files/ defer func() { cerr := out.Close() if err == nil { err = cerr } }() // Get the data resp, err := http.Get(url) if err != nil { return err } defer resp.Body.Close() // Check server response if resp.StatusCode != http.StatusOK { return fmt.Errorf("bad status: %s", resp.Status) } // Write the body to file counter := io.Discard if verbose { counter = &WriteCounter{} } _, err = io.Copy(out, io.TeeReader(resp.Body, counter)) if err != nil { return err } return } // // Sectors checks if the configuration file is already populated. // If 'force' is set or if the config is empty, it retrieves data from B3, // unzip and extract a spreadsheet containing a list of companies divided by // sector, subsector, and segment; then this info is set into the config file. // func Sectors(yamlFile string) (err error) { err = parsers.SectorsToYaml(yamlFile) return } // // filesCleanup // func filesCleanup(files []string) { // Clean up for _, f := range files { if err := os.Remove(f); err != nil { fmt.Println("could not delete file", f) } } } // // findFile finds an item on list that matches pattern (case insensitive) // func findFile(list []string, pattern string) (string, error) { for i := range list { f := filepath.Base(list[i]) if strings.EqualFold(f, pattern) { return list[i], nil } } return "", ErrItemNotFound } ================================================ FILE: fetch/fetch_fii.go ================================================ package fetch /* URL List: Fundos.NET: where the report IDs are obtained. => https://fnet.bmfbovespa.com.br/fnet/publico/pesquisarGerenciadorDocumentosCVM?paginaCertificados=false&tipoFundo=1 => GET https://fnet.bmfbovespa.com.br/fnet/publico/pesquisarGerenciadorDocumentosDados?d=3&s=0&l=10&o[0][dataEntrega]=desc&tipoFundo=1&idCategoriaDocumento=14&idTipoDocumento=41&idEspecieDocumento=0&situacao=A&cnpj=28737771000185&dataInicial=01/02/2021&dataFinal=28/02/2021&_=1619467786288 */ import ( "bytes" "crypto/tls" "database/sql" "encoding/base64" "fmt" "io" "net/http" "net/url" "strconv" "strings" "time" "github.com/dude333/rapina" "github.com/dude333/rapina/parsers" "github.com/dude333/rapina/progress" "github.com/gocolly/colly/v2" "github.com/pkg/errors" "golang.org/x/net/html" ) const MAX_N = 100 // FII holds the infrastructure data. type FII struct { storage rapina.FIIStorage } // NewFII creates a new instace of FII. func NewFII(db *sql.DB, log rapina.Logger) (*FII, error) { storage, err := parsers.NewFII(db, log) if err != nil { return nil, err } fii := &FII{ storage: storage, } return fii, nil } type id int // Report holds the result of all documents filtered by a criteria defined by a // http.Get on the B3 server. type Report struct { Data []docID `json:"data"` } type docID struct { ID id `json:"id"` Description string `json:"descricaoFundo"` DocType string `json:"tipoDocumento"` Status string `json:"situacaoDocumento"` } // Dividends gets the report IDs for one company ('cnpj') and then the // yeld montlhy report for 'n' months, starting from the latest released. func (fii FII) Dividends(code string, n int) (*[]rapina.Dividend, error) { dividends, months, err := fii.dividendsFromDB(code, n) if err == nil { if months >= n { return dividends, err } } dividends, err = fii.dividendsFromServer(code, n) if err != nil { return nil, err } for _, d := range *dividends { err := fii.storage.SaveDividend(d) // Save dividends to DB if err != nil { progress.ErrorMsg("Erro ao salvar dividendos no banco de dados: %s - %v", err, d) } } // Load dividends from DB to filter results dividends, _, err = fii.dividendsFromDB(code, n) return dividends, err } func (fii FII) dividendsFromDB(code string, n int) (*[]rapina.Dividend, int, error) { var dividends []rapina.Dividend var months int for _, monthYear := range rapina.MonthsFromToday(n + 2) { d, err := fii.storage.Dividends(code, monthYear) if err == nil { // ignore errors dividends = append(dividends, *d...) months++ } if months == n { break } } if len(dividends) == 0 { return nil, 0, errors.New("dividendos não encontrados") } return ÷nds, months, nil } // Dividends gets the report IDs for one company ('cnpj') and then the // yeld montlhy report for 'n' months, starting from the latest released. // // If the number of reports does not match n, it'll retry with a bigger n as // sometimes reports from follow-on offerings (FPO). func (fii *FII) dividendsFromServer(code string, n int) (*[]rapina.Dividend, error) { n = int(float64(n) * 1.25) if n > MAX_N { n = MAX_N } ids, err := fii.reportIDs(repDividends, code, n) if err != nil { return nil, err } progress.Debug("Report IDs: %v", ids) progress.Status("Relatórios de dividendos: %s", code) dividends, err := fii.dividendReport(code, ids) if err != nil { return nil, err } return dividends, nil } // dividendReport parses the dividend reports and returns their dividends. func (fii *FII) dividendReport(code string, ids []id) (*[]rapina.Dividend, error) { var dividends []rapina.Dividend // HTTP client setup client := &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, }, MaxIdleConnsPerHost: 10, }, } for _, id := range ids { url := fmt.Sprintf("https://fnet.bmfbovespa.com.br/fnet/publico/exibirDocumento?id=%d&cvm=true", id) progress.Debug("GET %s", url) // Make HTTP request req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } // Reuse the same client for subsequent requests resp, err := client.Do(req) if err != nil { return nil, err } if resp.StatusCode != http.StatusOK { resp.Body.Close() return nil, errors.Wrapf(err, "unexpected status code: %d", resp.StatusCode) } // Read response body body, err := io.ReadAll(resp.Body) resp.Body.Close() if err != nil { return nil, err } // Decode base64 encoded body decodedBody, err := base64.StdEncoding.DecodeString(strings.Trim(string(body), `"`)) if err != nil { return nil, err } doc, err := html.Parse(bytes.NewReader(decodedBody)) if err != nil { return nil, errors.Wrap(err, "error parsing HTML: %s") } var data []string var extractData func(*html.Node) extractData = func(n *html.Node) { if n.Type == html.ElementNode && n.Data == "td" { text := getTextContent(n) data = append(data, text) } for c := n.FirstChild; c != nil; c = c.NextSibling { extractData(c) } } extractData(doc) // Store dividend if d, ok := parseData(data); ok { dividends = append(dividends, d) } } return ÷nds, nil } func parseData(data []string) (rapina.Dividend, bool) { dividend := rapina.Dividend{} fieldName := "" count := 0 for _, str := range data { if fieldName == "" { if str != "" { fieldName = str } continue } if strings.Contains(fieldName, "Código de negociação") { dividend.Code = str count++ } else if strings.Contains(fieldName, "Data-base") { dividend.Date = fixDate(str) count++ } else if strings.Contains(fieldName, "Data do pagamento") { dividend.PaymentDate = fixDate(str) count++ } else if strings.Contains(fieldName, "Valor do provento") { dividend.Val = comma2dot(str) count++ } fieldName = "" } return dividend, count == 4 // false if not all fields are filled } func comma2dot(val string) float64 { a := strings.ReplaceAll(val, ".", "") b := strings.ReplaceAll(a, ",", ".") n, _ := strconv.ParseFloat(b, 64) return n } // fixDate converts dates from DD/MM/YYYY to YYYY-MM-DD. func fixDate(date string) string { if len(date) != len("26/04/2021") || strings.Count(date, "/") != 2 { return date } return date[6:10] + "-" + date[3:5] + "-" + date[0:2] } func getTextContent(n *html.Node) string { textContent := "" if n == nil { return "" } if n.Type == html.TextNode { return n.Data } for c := n.FirstChild; c != nil; c = c.NextSibling { textContent += getTextContent(c) } return strings.TrimSpace(textContent) } func (fii *FII) MonthlyReportIDs(code string, n int) ([]id, error) { ids, err := fii.reportIDs(repMonthly, code, n) if err != nil { return []id{}, err } _, err = fii.monthlyReport(code, ids) if err != nil { return []id{}, err } return ids, nil } // monthlyReport parses the FII monthly reports. func (fii *FII) monthlyReport(code string, ids []id) (*[]rapina.Monthly, error) { yeld := make(map[string]string, len(ids)) c := colly.NewCollector() c.WithTransport(&http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }) c.OnRequest(func(r *colly.Request) { r.Headers.Set("Accept", "text/html") }) c.OnError(func(r *colly.Response, err error) { progress.ErrorMsg("Request URL: %v failed with response: %v\nError: %v", r.Request.URL, string(r.Body), err) }) // Handles the html report c.OnHTML("tr", func(e *colly.HTMLElement) { var fieldName string e.ForEach("td", func(_ int, el *colly.HTMLElement) { v := strings.Trim(el.Text, " \r\n") progress.Debug("%q", v) if v != "" { if fieldName == "" { if v[0] < '0' || v[0] > '9' { // Ignore fields starting with number fieldName = v } } else { fmt.Printf("%-30s => %s\n", fieldName, v) yeld[fieldName] = v fieldName = "" } } }) progress.Status("----------------------") }) // Get the yeld monthly report given the list of 'report IDs' -- returns HTML monthly := make([]rapina.Monthly, 0, len(ids)) for _, id := range ids { u := fmt.Sprintf("https://fnet.bmfbovespa.com.br/fnet/publico/exibirDocumento?id=%d&cvm=true", id) progress.Debug(u) if err := c.Visit(u); err != nil { return nil, err } // d, err := fii.storage.SaveDividend(yeld) // if err != nil { // fii.log.Error("%v", err) // continue // } // // fmt.Println("from server", d.Code, d.Date, d.Val) // if d.Code == code { // monthly = append(monthly, *d) // } } return &monthly, nil } // Details returns the FII Details from DB. If not found: // fetches from server, stores it in the DB and returns the Details. func (fii *FII) Details(fiiCode string) (*rapina.FIIDetails, error) { if len(fiiCode) != 4 && len(fiiCode) != 6 { return nil, fmt.Errorf("wrong code '%s'", fiiCode) } details, err := fii.storage.Details(fiiCode) if err == nil && details.DetailFund.CNPJ != "" { return details, nil } progress.Warning("Detalhes do %s não encontrado no bd. Consultando web...", fiiCode) // Fetch from server if not found in the database data := fmt.Sprintf(`{"typeFund":7,"cnpj":"0","identifierFund":"%s"}`, fiiCode[0:4]) enc := base64.URLEncoding.EncodeToString([]byte(data)) fundDetailURL := rapina.JoinURL( `https://sistemaswebb3-listados.b3.com.br/fundsProxy/fundsCall/GetDetailFundSIG/`, enc, ) tr := &http.Transport{ DisableCompression: true, IdleConnTimeout: _http_timeout, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client := &http.Client{Transport: tr} resp, err := client.Get(fundDetailURL) if err != nil { return details, err } defer resp.Body.Close() if resp.StatusCode != 200 { return details, fmt.Errorf("%s: %s", resp.Status, fundDetailURL) } body, err := io.ReadAll(resp.Body) if err != nil { return nil, errors.Wrapf(err, "FII Details(%s): reading body", fiiCode) } err = fii.storage.SaveDetails(body) if err != nil { return details, errors.Wrap(err, "armazenando detalhes do FII") } return fii.storage.Details(fiiCode) } // Report type type repType int const ( repMonthly repType = iota + 1 repDividends ) func (fii *FII) reportIDs(rt repType, code string, n int) ([]id, error) { n = minmax(n, 1, MAX_N) // Parameters to list the report IDs for the last 'n' dividend reports timestamp := strconv.FormatInt(int64(time.Now().UnixNano()/1e6), 10) nMonthAgo := time.Now() nMonthAgo = nMonthAgo.AddDate(0, -n, -nMonthAgo.Day()+1) det, err := fii.Details(code) if err != nil { return nil, err } cnpj := det.DetailFund.CNPJ var idTipoDocumento, idCategoriaDocumento, d string if rt == repMonthly { idTipoDocumento = "40" idCategoriaDocumento = "6" d = "0" } else if rt == repDividends { idTipoDocumento = "41" idCategoriaDocumento = "14" d = "2" } else { return []id{}, errors.New("invalid report type") } v := url.Values{ "tipoFundo": []string{"1"}, "cnpjFundo": []string{cnpj}, "idTipoDocumento": []string{idTipoDocumento}, "idCategoriaDocumento": []string{idCategoriaDocumento}, "d": []string{d}, "idEspecieDocumento": []string{"0"}, "situacao": []string{"A"}, "s": []string{"0"}, "l": []string{"200"}, // 'n*2' latest reports as other codes may appear (e.g.:ABCD11, ABCD12, ABCD13...) "dataFinal": []string{time.Now().Format("02/01/2006")}, "dataInicial": []string{nMonthAgo.Format("02/01/2006")}, "o[0][dataReferencia]": []string{"asc"}, "_": []string{timestamp}, } // Get the 'report IDs' for a given company (CNPJ) -- returns JSON var report Report u := "https://fnet.bmfbovespa.com.br/fnet/publico/pesquisarGerenciadorDocumentosDados?" + v.Encode() progress.Debug("* Report IDs: %s", u) if err := getJSON(u, &report); err != nil { return nil, err } var ids []id for _, d := range report.Data { if d.Status == "A" { ids = append(ids, d.ID) } } return ids, nil } // minmax returns n limited to [min, max] func minmax(n, min, max int) int { if n < min { n = min } if n > max { n = MAX_N } return n } ================================================ FILE: fetch/fetch_fii_test.go ================================================ package fetch import ( "testing" ) func Test_comma2dot(t *testing.T) { type args struct { val string } tests := []struct { name string args args want float64 }{ { name: "should work", args: args{val: "1.230,56"}, want: 1230.56, }, { name: "should return 0", args: args{val: "shouldbeanum"}, want: 0, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := comma2dot(tt.args.val); got != tt.want { t.Errorf("comma2dot() = %v, want %v", got, tt.want) } }) } } func Test_FixDate(t *testing.T) { type args struct { date string } tests := []struct { name string args args want string }{ { name: "should work", args: args{date: "01/02/2021"}, want: "2021-02-01", }, { name: "should return the input", args: args{date: "wrong/date"}, want: "wrong/date", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := fixDate(tt.args.date); got != tt.want { t.Errorf("fixDate() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: fetch/fetch_http.go ================================================ package fetch import ( "crypto/tls" "encoding/json" "fmt" "net/http" "time" "github.com/dude333/rapina/progress" ) const _http_timeout = 30 * time.Second // HTTPFetch implements a generic HTTP fetcher. type HTTPFetch struct { client *http.Client } // NewHTTP creates a new HTTPFetch instance. func NewHTTP() *HTTPFetch { c := &http.Client{Timeout: _http_timeout} return &HTTPFetch{client: c} } // JSON handles json responses. func (h HTTPFetch) JSON(url string, target interface{}) error { r, err := h.client.Get(url) if err != nil { return err } defer r.Body.Close() // for _, c := range r.Cookies() { // fmt.Printf("COOKIE: %+v\n", c) // } return json.NewDecoder(r.Body).Decode(target) } func getJSON(url string, target interface{}) error { c := &http.Client{ Timeout: _http_timeout, Transport: &http.Transport{ DisableCompression: true, IdleConnTimeout: _http_timeout, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } r, err := c.Get(url) if err != nil { return err } if r.StatusCode < 200 || r.StatusCode >= 300 { return fmt.Errorf("unexpected status code: %d", r.StatusCode) } defer func() { if err := r.Body.Close(); err != nil { progress.ErrorMsg("Failed to close response body: %v", err) } }() return json.NewDecoder(r.Body).Decode(target) } ================================================ FILE: fetch/fetch_http_test.go ================================================ package fetch import ( "net/http" "net/http/httptest" "testing" "github.com/stretchr/testify/assert" ) var ts *httptest.Server func init() { handler := http.NewServeMux() handler.HandleFunc("/server/api/v1/json", jsonsMock) ts = httptest.NewServer(handler) } func jsonsMock(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(`{"text": "mock"}`)) } type jsonData struct { Text string `json:"text"` } func TestHTTPFetch_JSON(t *testing.T) { h := NewHTTP() var got jsonData err := h.JSON(ts.URL+"/server/api/v1/json", &got) assert.Equal(t, jsonData{Text: "mock"}, got) assert.Nil(t, err) } ================================================ FILE: fetch/fetch_stock.go ================================================ package fetch import ( "crypto/tls" "database/sql" "encoding/json" "fmt" "net/http" "net/url" "os" "time" "github.com/dude333/rapina" "github.com/dude333/rapina/parsers" "github.com/dude333/rapina/progress" "github.com/pkg/errors" "golang.org/x/text/encoding/charmap" "golang.org/x/text/transform" ) // API providers const ( APInone = iota APIalphavantage APIyahoo ) // Stock implements a fetcher for stock info. type Stock struct { apiKey string // API key for Alpha Vantage API server store rapina.StockStorage cache map[string]int // Cache to avoid duplicated fetch on Alpha Vantage server dataDir string // working directory where files will be stored to be parsed log rapina.Logger } // // NewStock returns a new instance of *Stock // func NewStock(db *sql.DB, log rapina.Logger, apiKey, dataDir string) (*Stock, error) { store, err := parsers.NewStock(db, log) if err != nil { return nil, err } return &Stock{ apiKey: apiKey, store: store, cache: make(map[string]int), dataDir: dataDir, log: log, }, nil } // Quote returns the quote for 'code' on 'date'. // Date format: YYYY-MM-DD. func (s *Stock) Quote(code, date string) (float64, error) { if len(code) < len("CODE3") { return 0, fmt.Errorf("código inválido: %q", code) } if !rapina.IsDate(date) { return 0, fmt.Errorf("data inválida: %q", date) } val, err := s.store.Quote(code, date) if err == nil { return val, nil // returning data found on db } // Load quotes from B3 if err := s.stockQuoteFromB3(date); ifNot(err) { if val, err = s.store.Quote(code, date); ifNot(err) { return val, nil // returning data found on B3 } } // Fallback to Yahoo Finance if not found on B3 if err := s.stockQuoteFromAPIServer(code, date, APIyahoo); ifNot(err) { if val, err = s.store.Quote(code, date); ifNot(err) { return val, nil // returning data found on Yahoo } } errNoProvider := errors.New("cotação não encontrada em nenhum provedor (B3, Yahoo e Alpha Vantage)") if s.apiKey == "" { errNoProvider = errors.New("cotação não encontrada em nenhum provedor (B3 e Yahoo)") } // Fallback to Alpha Vantage if not found on B3 and Yahoo if s.apiKey == "" { return 0, errNoProvider } if err := s.stockQuoteFromAPIServer(code, date, APIalphavantage); err != nil { return 0, errNoProvider } // Last try: return quote loaded by Alpha Vantage val, err = s.store.Quote(code, date) if err != nil { return 0, errNoProvider } return val, nil } // // stockQuoteFromB3 downloads the quotes for all companies for the given date, // where 'date' format is YYYY-MM-DD. // func (s *Stock) stockQuoteFromB3(date string) error { // Convert date string from YYYY-MM-DD to DDMMYYYY if len(date) != len("2021-05-03") { return fmt.Errorf("data com formato inválido: %s", date) } conv := date[8:10] + date[5:7] + date[0:4] url := fmt.Sprintf(`http://bvmf.bmfbovespa.com.br/InstDados/SerHist/COTAHIST_D%s.ZIP`, conv) // Download ZIP file and unzips its files zip := fmt.Sprintf("%s/COTAHIST_D%s.ZIP", s.dataDir, conv) files, err := fetchFilesVerbosity(url, s.dataDir, zip, false) if err != nil { return err } // Delete files on return defer filesCleanup(files) // Parse and store files content for _, f := range files { fh, err := os.Open(f) if err != nil { return errors.Wrapf(err, "abrindo arquivo %s", f) } defer fh.Close() dec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder()) if _, err := s.store.Save(dec, ""); err != nil { return err } } return nil } // // stockQuoteFromAPIServer fetches the daily time series (date, daily open, daily high, // daily low, daily close, daily volume) of the global equity specified, // covering 20+ years of historical data. // func (s *Stock) stockQuoteFromAPIServer(code, date string, apiProvider int) error { if v := s.cache[code]; v == APIalphavantage && apiProvider == APIalphavantage { return nil // silent return if this fetch has been run already } // Download quote for 'code' tr := &http.Transport{ DisableCompression: true, IdleConnTimeout: _http_timeout, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client := &http.Client{Transport: tr} u := apiURL(apiProvider, s.apiKey, code, date) if u == "" { return errors.New("URL do API server") } resp, err := client.Get(u) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("%s", resp.Status) } s.cache[code] = apiProvider // mark map to avoid unnecessary downloads // JSON means error response if resp.Header.Get("Content-Type") == "application/json" { jsonMap := make(map[string]interface{}) err := json.NewDecoder(resp.Body).Decode(&jsonMap) if err != nil { return err } return errors.New(map2str(jsonMap)) } progress.Running("Armazendo cotações no banco de dados...") _, err = s.store.Save(resp.Body, code) if err != nil { progress.RunFail() return errors.Wrapf(err, "armazenando cotações de %s", code) } progress.RunOK() return err } func (s *Stock) Code(companyName, stockType string) (string, error) { if val, err := s.store.Code(companyName, stockType); err == nil { return val, nil // returning data found on db } if err := s.UpdateStockCodes(); err != nil { return "", err } return s.store.Code(companyName, stockType) } type b3CodesFile struct { RedirectURL string `json:"redirectUrl"` Token string `json:"token"` File struct { Name string `json:"name"` Extension string `json:"extension"` } `json:"file"` } // // UpdateStockCodes get the most recent file from B3.com.br with the stock trading code and // saves them on the storage. // func (s *Stock) UpdateStockCodes() error { // Get file url var f b3CodesFile url := `https://arquivos.b3.com.br/api/download/requestname?fileName=InstrumentsConsolidated&date=` url += rapina.LastBusinessDay(2) h := NewHTTP() err := h.JSON(url, &f) if err != nil { return err } // Download file fp := fmt.Sprintf("%s/codes.csv", s.dataDir) tries := 3 for { url = fmt.Sprintf(`https://arquivos.b3.com.br/api/download/?token=%s`, f.Token) progress.Download("Download do arquivo de códigos") err = downloadFile(url, fp, false) if err != nil { tries-- if tries <= 0 { return err } time.Sleep(2 * time.Second) continue } // Delete files on return defer filesCleanup([]string{fp}) break } // Parse and store files content fh, err := os.Open(fp) if err != nil { return errors.Wrapf(err, "abrindo arquivo %s", fp) } defer fh.Close() dec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder()) _, err = s.store.Save(dec, "") return err } /* --- UTILS --- */ func apiURL(provider int, apiKey, code, date string) string { v := url.Values{} switch provider { case APIalphavantage: v.Set("function", "TIME_SERIES_DAILY") v.Add("symbol", code+".SA") v.Add("apikey", apiKey) v.Add("outputsize", "full") v.Add("datatype", "csv") return "https://www.alphavantage.co/query?" + v.Encode() case APIyahoo: const layout = "2006-01-02 15:04:05 -0700 MST" t1, err1 := time.Parse(layout, date+" 00:00:00 -0300 GMT") t2, err2 := time.Parse(layout, date+" 23:59:59 -0300 GMT") if err1 != nil || err2 != nil { return "" } v.Set("period1", fmt.Sprint(t1.Unix())) v.Add("period2", fmt.Sprint(t2.Unix())) v.Add("interval", "1d") v.Add("events", "history") v.Add("includeAdjustedClose", "true") return fmt.Sprintf("https://query1.finance.yahoo.com/v7/finance/download/%s.SA?%s", code, v.Encode()) } return "" } func map2str(data map[string]interface{}) string { var buf string for k, v := range data { buf += fmt.Sprintln(k+":", v) } return buf } // ifNot returns true if no error is found. func ifNot(err error) bool { return err == nil } ================================================ FILE: fetch/fetch_test.go ================================================ package fetch import ( "testing" _ "github.com/mattn/go-sqlite3" ) func Test_findFile(t *testing.T) { type args struct { list []string pattern string } tests := []struct { name string args args want string wantErr bool }{ { name: "should find item", args: args{[]string{"aaa", "aaa bbb CCC ddd"}, "aaa bbb CCC ddd"}, want: "aaa bbb CCC ddd", wantErr: false, }, { name: "should not find item", args: args{[]string{"aaa", "aaa bbb CCC ddd"}, "aaa bbb xCC ddd"}, want: "", wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := findFile(tt.args.list, tt.args.pattern) if (err != nil) != tt.wantErr { t.Errorf("findFiles() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { t.Errorf("findFiles() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: fetch/unzip.go ================================================ package fetch import ( "archive/zip" "fmt" "io" "os" "path/filepath" "strings" ) // // UnzipVerbosity will decompress a zip archive, moving all files and folders // within the zip file (parameter 1) to an output directory (parameter 2). // Source: https://golangcode.com/unzip-files-in-go/ // func Unzip(src string, dest string, verbose bool) ([]string, error) { var filenames []string r, err := zip.OpenReader(src) if err != nil { return filenames, err } defer r.Close() for _, f := range r.File { if !valid(f.Name) { continue } rc, err := f.Open() if err != nil { return filenames, err } defer rc.Close() // Store filename/path for returning and using later on fpath := filepath.Join(dest, f.Name) // Check for ZipSlip. More Info: http://bit.ly/2MsjAWE if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) { return filenames, fmt.Errorf("%s: illegal file path", fpath) } filenames = append(filenames, fpath) if f.FileInfo().IsDir() { // Make Folder if err = os.MkdirAll(fpath, os.ModePerm); err != nil { return nil, err } } else { // Make File if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil { return filenames, err } outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) if err != nil { return filenames, err } counter := io.Discard if verbose { fmt.Printf("[ ] Unziping %s", fpath) counter = &WriteCounter{} } _, err = io.Copy(outFile, io.TeeReader(rc, counter)) if verbose { fmt.Println() } // Close the file without defer to close before next iteration of loop outFile.Close() if err != nil { return filenames, err } } } return filenames, nil } func valid(filename string) bool { n := strings.ToLower(filename) if strings.Contains(n, "_ind_") { return false } list := []string{"_bpa_", "_bpp_", "_dfc_", "_dre_", "_dva_", "fre_", "cotahist_"} for _, item := range list { if strings.Contains(n, item) { return true } } return false } ================================================ FILE: fii.go ================================================ package rapina // Dividend contains the stock 'Code', and the 'Date' for the stock dividend 'Val'. type Dividend struct { Code string Date string PaymentDate string Val float64 } // Monthly contains the FII monthly report fields type Monthly struct { } // FIIDetails details (ID field: DetailFund.CNPJ) type FIIDetails struct { DetailFund struct { Acronym string `json:"acronym"` TradingName string `json:"tradingName"` TradingCode string `json:"tradingCode"` TradingCodeOthers string `json:"tradingCodeOthers"` CNPJ string `json:"cnpj"` Classification string `json:"classification"` WebSite string `json:"webSite"` FundAddress string `json:"fundAddress"` FundPhoneNumberDDD string `json:"fundPhoneNumberDDD"` FundPhoneNumber string `json:"fundPhoneNumber"` FundPhoneNumberFax string `json:"fundPhoneNumberFax"` PositionManager string `json:"positionManager"` ManagerName string `json:"managerName"` CompanyAddress string `json:"companyAddress"` CompanyPhoneNumberDDD string `json:"companyPhoneNumberDDD"` CompanyPhoneNumber string `json:"companyPhoneNumber"` CompanyPhoneNumberFax string `json:"companyPhoneNumberFax"` CompanyEmail string `json:"companyEmail"` CompanyName string `json:"companyName"` QuotaCount string `json:"quotaCount"` QuotaDateApproved string `json:"quotaDateApproved"` Codes []string `json:"codes"` CodesOther interface{} `json:"codesOther"` Segment interface{} `json:"segment"` } `json:"detailFund"` ShareHolder struct { ShareHolderName string `json:"shareHolderName"` ShareHolderAddress string `json:"shareHolderAddress"` ShareHolderPhoneNumberDDD string `json:"shareHolderPhoneNumberDDD"` ShareHolderPhoneNumber string `json:"shareHolderPhoneNumber"` ShareHolderFaxNumber string `json:"shareHolderFaxNumber"` ShareHolderEmail string `json:"shareHolderEmail"` } `json:"shareHolder"` } // FIIStorage is the interface that contains the methods needed to parse, save and // retrieve FII data to/from a storage. type FIIStorage interface { Details(code string) (*FIIDetails, error) SaveDetails(stream []byte) error Dividends(code, monthYear string) (*[]Dividend, error) SaveDividend(dividend Dividend) error } ================================================ FILE: go.mod ================================================ module github.com/dude333/rapina require ( github.com/360EntSecGroup-Skylar/excelize v1.4.1 github.com/PuerkitoBio/goquery v1.8.1 github.com/andybalholm/cascadia v1.3.2 // indirect github.com/antchfx/htmlquery v1.3.0 // indirect github.com/antchfx/xmlquery v1.3.18 // indirect github.com/antchfx/xpath v1.2.5 // indirect github.com/dustin/go-humanize v1.0.0 github.com/gocolly/colly/v2 v2.1.0 github.com/golang/protobuf v1.5.3 // indirect github.com/lithammer/fuzzysearch v1.1.0 github.com/manifoldco/promptui v0.6.0 github.com/mattn/go-sqlite3 v2.0.1+incompatible github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.8.1 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/spf13/cobra v0.0.5 github.com/spf13/viper v1.6.1 github.com/stretchr/testify v1.4.0 github.com/temoto/robotstxt v1.1.2 // indirect golang.org/x/net v0.20.0 golang.org/x/text v0.14.0 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/yaml.v2 v2.4.0 ) go 1.16 ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/360EntSecGroup-Skylar/excelize v1.4.1 h1:l55mJb6rkkaUzOpSsgEeKYtS6/0gHwBYyfo5Jcjv/Ks= github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/alecthomas/gometalinter v3.0.0+incompatible h1:e9Zfvfytsw/e6Kd/PYd75wggK+/kX5Xn8IYDUKyc5fU= github.com/alecthomas/gometalinter v3.0.0+incompatible/go.mod h1:qfIpQGGz3d+NmgyPBqv+LSh50emm1pt72EtcX2vKYQk= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM= github.com/antchfx/xmlquery v1.3.18 h1:FSQ3wMuphnPPGJOFhvc+cRQ2CT/rUj4cyQXkJcjOwz0= github.com/antchfx/xmlquery v1.3.18/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/antchfx/xpath v1.2.5 h1:hqZ+wtQ+KIOV/S3bGZcIhpgYC26um2bZYP2KVGcR7VY= github.com/antchfx/xpath v1.2.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs= github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg= github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gordonklaus/ineffassign v0.0.0-20180909121442-1003c8bd00dc h1:cJlkeAx1QYgO5N80aF5xRGstVsRQwgLR7uA2FnP1ZjY= github.com/gordonklaus/ineffassign v0.0.0-20180909121442-1003c8bd00dc/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU= github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lithammer/fuzzysearch v1.1.0 h1:go9v8tLCrNTTlH42OAaq4eHFe81TDHEnlrMEb6R4f+A= github.com/lithammer/fuzzysearch v1.1.0/go.mod h1:Bqx4wo8lTOFcJr3ckpY6HA9lEIOO0H5HrkJ5CsN56HQ= github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw= github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/manifoldco/promptui v0.6.0 h1:GuXmIdl5lhlamnWf3NbsKWYlaWyHABeStbD1LLsQMuA= github.com/manifoldco/promptui v0.6.0/go.mod h1:o9/C5VV8IPXxjxpl9au84MtQGIi5dwn7eldAgEdePPs= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nicksnyder/go-i18n v1.10.1 h1:isfg77E/aCD7+0lD/D00ebR2MV5vgeQ276WYyDaCRQc= github.com/nicksnyder/go-i18n v1.10.1/go.mod h1:e4Di5xjP9oTVrC6y3C7C0HoSYXjSbhh/dU0eUV32nB4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.6.1 h1:VPZzIkznI1YhVMRi6vNFLHSwhnhReBfgTxIPccpfdZk= github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9 h1:vY5WqiEon0ZSTGM3ayVVi+twaHKHDFUVloaQ/wug9/c= github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9/go.mod h1:q+QjxYvZ+fpjMXqs+XEriussHjSYqeXVnAdSV1tkMYk= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181122213734-04b5d21e00f1/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20171010053543-63abe20a23e2 h1:5zOHKFi4LqGWG+3d+isqpbPrN/2yhDJnlO+BhRiuR6U= gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20171010053543-63abe20a23e2/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= ================================================ FILE: logger.go ================================================ package rapina import "io" // Logger interface contains the methods needed to poperly display log messages. type Logger interface { Run(format string, v ...interface{}) Ok() Nok() Printf(format string, v ...interface{}) Trace(format string, v ...interface{}) Debug(format string, v ...interface{}) Info(format string, v ...interface{}) Warn(format string, v ...interface{}) Error(format string, v ...interface{}) SetOut(out io.Writer) } ================================================ FILE: parsers/codeaccounts.go ================================================ package parsers import ( "strings" ) // Bookkeeping account codes // If you add new const values, run 'go generate' // to update the generated code const ( UNDEF uint32 = iota SPACE // Balance Sheet Caixa AplicFinanceiras Estoque Equity ContasARecebCirc ContasARecebNCirc AtivoCirc AtivoNCirc AtivoTotal PassivoCirc PassivoNCirc PassivoTotal DividaCirc DividaNCirc DividendosJCP DividendosMin // Income Statement Vendas CustoVendas DespesasOp EBIT ResulFinanc ResulOpDescont LucLiq // DFC FCO FCI FCF // Value Added Statement Deprec JurosCapProp Dividendos // Values stored on table 'fre' Shares FreeFloat // Financial ratios EstoqueMedio EquityAvg // Financial scale (unit, thousand) Escala // Stock quote from last day of year Quote ) // account code, description and bookkeeping code type account struct { cdAccount string dsAccount string code uint32 } var _accountsTable = []account{ // BPA {"1", "Ativo Total", AtivoTotal}, {"1.01", "Ativo Circulante", AtivoCirc}, {"1.02", "Ativo Não Circulante", AtivoNCirc}, {"1.01.01", "Caixa e Equivalentes de Caixa", Caixa}, {"1.01.02", "Aplicações Financeiras", AplicFinanceiras}, {"1.01.04", "Estoques", Estoque}, // or "Títulos e Créditos a Receber" for security companies {"1.01.03", "Contas a Receber", ContasARecebCirc}, {"1.02.01.03", "Contas a Receber", ContasARecebNCirc}, {"1.02.01.04", "Contas a Receber", ContasARecebNCirc}, // BPP {"2", "Passivo Total", PassivoTotal}, {"2.01", "Passivo Circulante", PassivoCirc}, {"2.02", "Passivo Não Circulante", PassivoNCirc}, {"2.*", "Patrimônio Líquido Consolidado", Equity}, {"2.01.04", "Empréstimos e Financiamentos", DividaCirc}, {"2.02.01", "Empréstimos e Financiamentos", DividaNCirc}, {"2.01.05.02.01", "Dividendos e JCP a Pagar", DividendosJCP}, {"2.01.05.02.02", "Dividendo Mínimo Obrigatório a Pagar", DividendosMin}, // DRE {"3.01", "", Vendas}, {"3.02", "", CustoVendas}, {"3.04", "", DespesasOp}, {"3.*", "Resultado Antes do Resultado Financeiro e dos Tributos", EBIT}, {"3.06", "Resultado Financeiro", ResulFinanc}, {"3.07", "Resultado Financeiro", ResulFinanc}, {"3.08", "Resultado Financeiro", ResulFinanc}, {"3.10", "Resultado Líquido de Operações Descontinuadas", ResulOpDescont}, {"3.11", "Resultado Líquido de Operações Descontinuadas", ResulOpDescont}, {"3.12", "Resultado Líquido de Operações Descontinuadas", ResulOpDescont}, {"3.*", "Lucro/Prejuízo Consolidado do Período", LucLiq}, {"3.*", "Lucro/Prejuízo do Período", LucLiq}, // DFC {"6.01", "", FCO}, {"6.02", "", FCI}, {"6.03", "", FCF}, // DVA {"7.*", "Depreciação, Amortização e Exaustão", Deprec}, {"7.*", "Juros sobre o Capital Próprio", JurosCapProp}, {"7.*", "Dividendos", Dividendos}, } // acctCode returns the code based on the account code and // account description; if the code is not found in the table // returns the hash. func acctCode(cdAccount, dsAccount string) uint32 { dsAccount = strings.ToLower(dsAccount) for _, acc := range _accountsTable { descr := strings.ToLower(acc.dsAccount) l := len(acc.cdAccount) code := "" if l > 1 && acc.cdAccount[l-1] == '*' { code = acc.cdAccount[:l-1] // remove the '*' } if code != "" && strings.HasPrefix(cdAccount, code) { if descr == "" || descr == dsAccount { return acc.code } } else if acc.cdAccount == "" || acc.cdAccount == cdAccount { if descr == "" || descr == dsAccount { return acc.code } } } return Hash(cdAccount + dsAccount) } ================================================ FILE: parsers/companies.go ================================================ package parsers import ( "database/sql" "github.com/pkg/errors" ) type company struct { id int name string } func loadCompanies(db *sql.DB) (map[string]company, error) { companies := make(map[string]company) selectCompanies := `SELECT ID, CNPJ, NAME from companies` rows, err := db.Query(selectCompanies) if err != nil { return companies, errors.Wrap(err, "falha ao ler banco de dados") } var id int var cnpj, name string defer rows.Close() for rows.Next() { err := rows.Scan(&id, &cnpj, &name) if err != nil && err != sql.ErrNoRows { return nil, err } companies[cnpj] = company{id, name} } return companies, nil } func saveCompanies(db *sql.DB, companies map[string]company) error { insert := `INSERT OR IGNORE INTO companies (ID,CNPJ,NAME) VALUES (?,?,?);` stmt, err := db.Prepare(insert) if err != nil { return errors.Wrap(err, "erro ao preparar insert da lista de empresas") } defer stmt.Close() for cnpj, value := range companies { _, err := stmt.Exec(value.id, cnpj, value.name) if err != nil { return errors.Wrap(err, "falha ao inserir empresa") } } return nil } // // updateCompanies inserts a new company to the map // func updateCompanies(companies map[string]company, cnpj, name string) { if _, exists := companies[cnpj]; !exists { companies[cnpj] = company{ len(companies) + 100, name, } } } ================================================ FILE: parsers/fii.go ================================================ package parsers /* // // FetchFIIs downloads the list of FIIs to get their code (e.g. 'HGLG'), // then it uses this code to retrieve its details to get the CNPJ. // Original baseURL: https://sistemaswebb3-listados.b3.com.br. // func FetchFIIList(baseURL string) ([]string, error) { listFundsURL := JoinURL(baseURL, `/fundsProxy/fundsCall/GetListFundDownload/eyJ0eXBlRnVuZCI6NywicGFnZU51bWJlciI6MSwicGFnZVNpemUiOjIwfQ==`) // fundsDetailsURL := `https://sistemaswebb3-listados.b3.com.br/fundsProxy/fundsCall/GetDetailFundSIG` tr := &http.Transport{ DisableCompression: true, IdleConnTimeout: 30 * time.Second, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } client := &http.Client{Transport: tr} resp, err := client.Get(listFundsURL) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != 200 { return nil, errors.New(resp.Status) } body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } unq, err := strconv.Unquote(string(body)) if err != nil { return nil, err } txt, err := base64.StdEncoding.DecodeString(unq) if err != nil { return nil, err } var codes []string for _, line := range strings.Split(string(txt), "\n") { p := strings.Split(line, ";") if len(p) > 3 && len(p[3]) == 4 { codes = append(codes, p[3]) } } return codes, nil } */ ================================================ FILE: parsers/fiidb.go ================================================ package parsers import ( "database/sql" "encoding/json" "fmt" "strings" "sync" "github.com/dude333/rapina" "github.com/dude333/rapina/progress" "github.com/pkg/errors" ) // Error codes var ( ErrDBUnset = errors.New("database not set") ErrNotFound = errors.New("not found") ) // FIIParser implements sqlite storage for a rapina.FIIParser object. type FIIParser struct { db *sql.DB log rapina.Logger mu sync.Mutex // ensures atomic writes on db } // NewFII creates a new instace of FII. func NewFII(db *sql.DB, log rapina.Logger) (*FIIParser, error) { err := createAllTables(db) return &FIIParser{ db: db, log: log, }, err } // StoreFIIDetails parses the stream data into FIIDetails and returns // the *FIIDetails. func (fii *FIIParser) SaveDetails(stream []byte) error { fii.mu.Lock() defer fii.mu.Unlock() if !hasTable(fii.db, "fii_details") { if err := createTable(fii.db, "fii_details"); err != nil { return err } } var fiiDetails rapina.FIIDetails if err := json.Unmarshal(stream, &fiiDetails); err != nil { return errors.Wrap(err, "json unmarshal") } trimFIIDetails(&fiiDetails) x := fiiDetails.DetailFund if x.CNPJ == "" { return errors.New("CNPJ não encontrado") } insert := `INSERT OR IGNORE INTO fii_details (cnpj, acronym, trading_code, json) VALUES (?,?,?,?);` _, err := fii.db.Exec(insert, x.CNPJ, x.Acronym, x.TradingCode, stream) return err } // Details returns the FII Details for the 'code' or // an empty string if not found in the db. func (fii *FIIParser) Details(code string) (*rapina.FIIDetails, error) { details := rapina.FIIDetails{} if fii.db == nil { return nil, ErrDBUnset } var query string if len(code) == 4 { query = `SELECT json FROM fii_details WHERE acronym=?` } else if len(code) == 6 { query = `SELECT json FROM fii_details WHERE trading_code=?` } else { return nil, fmt.Errorf("invalid code '%s'", code) } var jsonStr []byte row := fii.db.QueryRow(query, code) err := row.Scan(&jsonStr) if err != nil { return nil, err } if err := json.Unmarshal(jsonStr, &details); err != nil { progress.ErrorMsg("FII details [%v]: %s\n", err, string(jsonStr)) return nil, errors.Wrap(err, "json unmarshal") } return &details, nil } // Dividends returns the dividend from the db. func (fii *FIIParser) Dividends(code, monthYear string) (*[]rapina.Dividend, error) { fii.mu.Lock() defer fii.mu.Unlock() const s = `SELECT trading_code, base_date, value FROM fii_dividends WHERE trading_code=$1 AND base_date LIKE $2;` rows, err := fii.db.Query(s, code, monthYear+"%") if err != nil { return nil, errors.Wrap(err, "lendo dividendos do bd") } defer rows.Close() dividends := []rapina.Dividend{} var ( tradingCode, baseDate string value float64 ) for rows.Next() { err := rows.Scan(&tradingCode, &baseDate, &value) if err != nil { return nil, err } // fii.log.Debug("reading: %v %v %v", tradingCode, baseDate, value) dividends = append(dividends, rapina.Dividend{ Code: tradingCode, Date: baseDate, Val: value, }) } if err := rows.Err(); err != nil { return nil, err } if len(dividends) == 0 { return nil, errors.New("dividendos não encontrados") } return ÷nds, nil } // SaveDividend parses and stores the map in the db. Returns the parsed stream. func (fii *FIIParser) SaveDividend(dividend rapina.Dividend) error { fii.mu.Lock() defer fii.mu.Unlock() if err := createTable(fii.db, "fii_dividends"); err != nil { return err } const insert = `INSERT OR IGNORE INTO fii_dividends (trading_code, base_date, payment_date, value) VALUES (?,?,?,?)` _, err := fii.db.Exec(insert, dividend.Code, dividend.Date, dividend.PaymentDate, dividend.Val) return errors.Wrap(err, "inserting data on fii_dividends") } func (fii *FIIParser) SelectFIIDetails(code string) (*rapina.FIIDetails, error) { if fii.db == nil { return nil, ErrDBUnset } var query string if len(code) == 4 { query = `SELECT cnpj, acronym, trading_code FROM fii_details WHERE acronym=?` } else if len(code) == 6 { query = `SELECT cnpj, acronym, trading_code FROM fii_details WHERE trading_code=?` } else { return nil, fmt.Errorf("invalid code '%s'", code) } var cnpj, acronym, tradingCode string row := fii.db.QueryRow(query, code) err := row.Scan(&cnpj, &acronym, &tradingCode) if err != nil { return nil, err } var fiiDetail rapina.FIIDetails fiiDetail.DetailFund.CNPJ = cnpj fiiDetail.DetailFund.Acronym = acronym fiiDetail.DetailFund.TradingCode = tradingCode return &fiiDetail, nil } /* -------- Utils ----------- */ func trimFIIDetails(f *rapina.FIIDetails) { f.DetailFund.CNPJ = strings.TrimSpace(f.DetailFund.CNPJ) f.DetailFund.Acronym = strings.TrimSpace(f.DetailFund.Acronym) tradingCodes := strings.Split( strings.TrimSpace(f.DetailFund.TradingCode), " ") f.DetailFund.TradingCode = tradingCodes[0] } ================================================ FILE: parsers/financial.go ================================================ // financial.go // Parses data from csv files containing financial statements package parsers import ( "bufio" "database/sql" "fmt" "os" "strings" "time" "github.com/pkg/errors" "golang.org/x/text/encoding/charmap" "golang.org/x/text/transform" ) var ( // ErrAccumITR error for accumulatd quarterly results ErrAccumITR = fmt.Errorf("accumulated quarterly results") ) // // ImportCsv start the data import process, including the database creation // if necessary // func ImportCsv(db *sql.DB, dataType string, file string) (err error) { // Create status table if err = createTable(db, "STATUS"); err != nil { return err } // Create companies table if err = createTable(db, "COMPANIES"); err != nil { return err } // Check table version, wipe it if version differs from current version, and // (re)create the table for _, t := range []string{dataType, "MD5"} { if v, table := dbVersion(db, t); v != currentDbVersion { if v > 0 { fmt.Printf("[i] Apagando tabela %s versão %d (versão atual: %d)\n", table, v, currentDbVersion) } if err := wipeDB(db, t); err != nil { return err } } if err := createTable(db, t); err != nil { return err } } isNew, err := isNewFile(db, file) if !isNew && err == nil { // if error, process file fmt.Printf("[ ] %s já processado anteriormente\n", dataType) return } var count int if dataType == "FRE" { count, err = populateFRE(db, file) } else { count, err = populateTable(db, dataType, file) } if err == nil { fmt.Printf("\r[√] %-7s %7d linhas processadas", dataType+":", count) storeFile(db, file) } else { fmt.Print("\r[x") } fmt.Println() return err } // // populateTable loop thru file and insert its lines into DB // and returns the number os lines inserted. // func populateTable(db *sql.DB, dataType, file string) (int, error) { progress := []string{"/", "-", "\\", "|", "-", "\\"} p := 0 table, err := whatTable(dataType) if err != nil { return 0, err } companies, _ := loadCompanies(db) fh, err := os.Open(file) if err != nil { return 0, errors.Wrapf(err, "erro ao abrir arquivo %s", file) } defer fh.Close() dec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder()) // BEGIN TRANSACTION tx, err := db.Begin() if err != nil { return 0, errors.Wrap(err, "Failed to begin transaction") } // Data used inside loop header := make(map[string]int) // stores the header item position (e.g., DT_FIM_EXERC:9) scanner := bufio.NewScanner(dec) count := 0 insert := "" var stmt *sql.Stmt // Loop thru file, line by line fmt.Print("[ ] Processando arquivo ", dataType) for scanner.Scan() { line := scanner.Text() if len(line) == 0 { continue } fields := strings.Split(line, ";") if len(header) == 0 { // HEADER // Get header positioning for i, h := range fields { header[h] = i } // Prepare insert statement insert = fmt.Sprintf(`INSERT OR IGNORE INTO %s ( ID, ID_CIA, CODE, YEAR, DATA_TYPE, VERSAO, MOEDA, ESCALA_MOEDA, DT_FIM_EXERC, CD_CONTA, DS_CONTA, VL_CONTA ) VALUES ( ?, ?, ?, ?, "%s", ?, ?, ?, ?, ?, ?, ? );`, table, dataType) stmt, err = tx.Prepare(insert) if err != nil { err = errors.Wrapf(err, "erro ao preparar insert (verificar cabeçalho do arquivo %s)", file) return count, err } defer stmt.Close() } else { // VALUES if len(fields) <= 12 { continue } // Only use penultimate for 2010 file, that's the last year published, // to get data from 2009 if fields[header["ORDEM_EXERC"]] == "PENÚLTIMO" { dt := fields[header["DT_FIM_EXERC"]] if len(dt) < 4 || dt[:4] != "2009" { continue } } // UPDATE COMPANIES n1, ok1 := header["CNPJ_CIA"] n2, ok2 := header["DENOM_CIA"] if ok1 && ok2 && n1 >= 0 && n1 < len(fields) && n2 >= 0 && n2 < len(fields) { updateCompanies(companies, fields[header["CNPJ_CIA"]], fields[header["DENOM_CIA"]]) } // INSERT f, err := prepareFields(dataType, header, fields, companies) if err == ErrAccumITR { continue // ignore accumulated ITR data } if err != nil { return count, errors.Wrap(err, "falha ao preparar registro") } _, err = stmt.Exec(f...) if err != nil { return count, errors.Wrap(err, "falha ao inserir registro") } } // fmt.Println("-------------------------------") if count++; count%1000 == 0 { fmt.Printf("\r[%s", progress[p%6]) p++ } } fmt.Print("\r[*") // END TRANSACTION err = tx.Commit() if err != nil { return count, errors.Wrap(err, "Failed to commit transaction") } if err := scanner.Err(); err != nil { return count, errors.Wrapf(err, "erro ao ler arquivo %s", file) } err = saveCompanies(db, companies) return count, err } // Cache (optimization) var unixTime = make(map[string]int64) // // prepareFields prepares all fields (columns) to be inserted on the DB. // // Returns: // ID, ID_CIA, CODE, YEAR, // VERSAO, // MOEDA, ESCALA_MOEDA, // DT_FIM_EXERC, // CD_CONTA, DS_CONTA, VL_CONTA // // Tip: to convert Unix timestamp to date on sqlite: strftime('%Y-%m-%d', DT_REFER, 'unixepoch') // func prepareFields(dataType string, header map[string]int, fields []string, companies map[string]company) ([]interface{}, error) { // AUX FUNCTIONS val := func(key string) string { v, ok := header[key] if !ok { return "" } return fields[v] } // Convert date string (YYYY-MM-DD) into Unix timestamp tim := func(key string) int64 { v, ok := header[key] if !ok { return 0 } f := fields[v] if ut, ok := unixTime[f]; ok { return ut } t, err := time.Parse("2006-01-02", f) if err != nil { return 0 } unixTime[f] = t.Unix() return unixTime[f] } // REFERENCE DATE v, ok := header["DT_FIM_EXERC"] if !ok { return nil, fmt.Errorf("DT_FIM_EXERC não encontrado") } if len(fields[v]) < 4 || tim("DT_FIM_EXERC") == 0 { return nil, fmt.Errorf("DT_FIM_EXERC incorreto: %v", fields[v]) } // Check if quarterly data contains data from 90 days, except for "BPA_ITR" and "BPP_ITR" if dataType != "BPA_ITR" && dataType != "BPP_ITR" && strings.HasSuffix(dataType, "_ITR") { t1 := tim("DT_INI_EXERC") t2 := tim("DT_FIM_EXERC") days := (t2 - t1) / 60 / 60 / 24 if days < 80 || days > 100 { return nil, ErrAccumITR } } year := fields[v][:4] // CNPJ_CIA and DENOM_CIA are replaced by company id cnpj := val("CNPJ_CIA") c, ok := companies[cnpj] if !ok { return nil, fmt.Errorf("CNPJ %s não encontrado", cnpj) } companyID := c.id // Unique value to be used as PRIMARY KEY hash := Hash(cnpj + val("GRUPO_DFP") + val("DT_FIM_EXERC") + val("VERSAO") + val("CD_CONTA") + val("VL_CONTA")) // Output -- need to follow INSERT sequence f := make([]interface{}, 11) f[0] = hash // ID f[1] = companyID // ID_CIA f[2] = acctCode(fields[header["CD_CONTA"]], fields[header["DS_CONTA"]]) // CODE f[3] = year // YEAR f[4] = val("VERSAO") f[5] = val("MOEDA") f[6] = val("ESCALA_MOEDA") f[7] = tim("DT_FIM_EXERC") f[8] = val("CD_CONTA") f[9] = val("DS_CONTA") f[10] = val("VL_CONTA") return f, nil } ================================================ FILE: parsers/financial_test.go ================================================ package parsers import ( "database/sql" "os" "testing" _ "github.com/mattn/go-sqlite3" ) func tempFilename(t *testing.T) string { f, err := os.CreateTemp("", "rapina-test-") if err != nil { t.Fatal(err) } f.Close() return f.Name() } func samples(filename string) error { bpa := []byte(` CNPJ_CIA;DT_REFER;VERSAO;DENOM_CIA;CD_CVM;GRUPO_DFP;MOEDA;ESCALA_MOEDA;ORDEM_EXERC;DT_FIM_EXERC;CD_CONTA;DS_CONTA;VL_CONTA 00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1;Ativo Total;1162167882.00 00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.01;Caixa e Equivalentes de Caixa;68841638.00 00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02;Aplica��es Financeiras;110019404.00 00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02.01;Aplica��es Financeiras Avaliadas a Valor Justo;109376121.00 00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02.01.01;T�tulos para Negocia��o;18991047.00 00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02.01.02;T�tulos Dispon�veis para Venda;90385074.00 00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02.02;Aplica��es Financeiras Avaliadas ao Custo Amortizado;643283.00 00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.02.02.01;T�tulos Mantidos at� o Vencimento;643283.00 00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.03;Empr�stimos e Receb�veis;755821983.00 00.000.000/0001-91;2013-12-31;4;BANCO DO BRASIL S.A.;1023;DF Consolidado - Balan�o Patrimonial Ativo;REAL;MILHAR;�LTIMO;2013-12-31;1.04;Tributos Diferidos;21954460.00 `) err := os.WriteFile(filename, bpa, 0600) return err } func TestImportCsv(t *testing.T) { var db *sql.DB var err error fileBPA := tempFilename(t) defer os.Remove(fileBPA) fileDB := tempFilename(t) defer os.Remove(fileDB) if db, err = sql.Open("sqlite3", fileDB); err != nil { t.Errorf("Fail to open db: %v", err) } defer db.Close() if err = samples(fileBPA); err != nil { t.Errorf("Fail to create samples: %v", err) } if err = ImportCsv(db, "BPA", fileBPA); err != nil { t.Errorf("Fail to parse: %v", err) } for _, tp := range []string{"BPA", "MD5"} { if v, table := dbVersion(db, tp); v != currentDbVersion { t.Errorf("Expecting table %s on version %d, received %d", table, currentDbVersion, v) } } isNew, err := isNewFile(db, fileBPA) if isNew && err == nil { t.Errorf("Expecting processed file, got new file") } } func TestGetHash(t *testing.T) { table := []struct { s string h uint32 }{ {"test1", 2569220284}, {"random data", 1626193638}, {"excel", 1973829744}, {"One More...12345!", 2258028052}, } for _, x := range table { h := Hash(x.s) if h != x.h { t.Errorf("Hash was incorrect, got: %d, want: %d.", h, x.h) } } } func TestRemoveDiacritics(t *testing.T) { list := []struct { str string exp string }{ {"ITAÚ", "ITAU"}, {"SÃO", "SAO"}, {"São Paulo", "Sao Paulo"}, {"ÁÉÍÓÚáéíóúÀàÃÕãõÇç", "AEIOUaeiouAaAOaoCc"}, } for _, l := range list { if RemoveDiacritics(l.str) != l.exp { t.Errorf("Expecting %s, received %s", l.exp, RemoveDiacritics(l.str)) } } } func Test_prepareFields(t *testing.T) { companies := make(map[string]company) companies["54321"] = company{1, "A"} type args struct { dataType string header map[string]int fields []string companies map[string]company } tests := []struct { name string args args wantErr bool }{ { "dt_refer not found", args{ "BPA", map[string]int{"a": 0, "b": 1}, []string{"a", "b"}, companies, }, true, }, { "should work", args{ "BPA", map[string]int{"x": 0, "y": 1, "DT_FIM_EXERC": 2, "CNPJ_CIA": 3}, []string{"X", "Y", "2020-02-25", "54321"}, companies, }, false, }, { "cnpj not found", args{ "BPA", map[string]int{"x": 0, "y": 2, "DT_FIM_EXERC": 1}, []string{"X", "2020-02-25", "Y"}, companies, }, true, }, { "DT_FIM_EXERC not found", args{ "BPA", map[string]int{"x": 0, "y": 2, "DT_FIM_EXERC": 1}, []string{"X", "202", "Y"}, companies, }, true, }, { "itr should work", args{ "BPA_ITR", map[string]int{"x": 0, "y": 1, "DT_INI_EXERC": 2, "DT_FIM_EXERC": 3, "CNPJ_CIA": 4}, []string{"X", "Y", "2020-01-01", "2020-06-30", "54321"}, companies, }, false, }, { "itr should fail", args{ "DRE_ITR", map[string]int{"x": 0, "y": 1, "DT_INI_EXERC": 2, "DT_FIM_EXERC": 3, "CNPJ_CIA": 4}, []string{"X", "Y", "2020-01-01", "2020-06-30", "54321"}, companies, }, true, }, { "itr should pass", args{ "DRE_ITR", map[string]int{"x": 0, "y": 1, "DT_INI_EXERC": 2, "DT_FIM_EXERC": 3, "CNPJ_CIA": 4}, []string{"X", "Y", "2020-01-01", "2020-03-30", "54321"}, companies, }, false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { _, err := prepareFields(tt.args.dataType, tt.args.header, tt.args.fields, tt.args.companies) if (err != nil) != tt.wantErr { t.Errorf("prepareFields() error = %v, wantErr %v", err, tt.wantErr) return } }) } } func BenchmarkPrepareFields(b *testing.B) { companies := make(map[string]company) companies["54321"] = company{1, "A"} h := map[string]int{"x": 0, "y": 1, "DT_FIM_EXERC": 2, "CNPJ_CIA": 3} f := []string{"X", "Y", "2020-02-25", "54321"} // run the prepareFields function b.N times for n := 0; n < b.N; n++ { _, err := prepareFields("BPA", h, f, companies) if err != nil { b.Errorf("error: %v", err) return } } } ================================================ FILE: parsers/fre.go ================================================ package parsers import ( "bufio" "database/sql" "fmt" "os" "strconv" "strings" "github.com/pkg/errors" "golang.org/x/text/encoding/charmap" "golang.org/x/text/transform" ) var ( // ErrCNPJNotFound error ErrCNPJNotFound = fmt.Errorf("CNPJ not found") ) func populateFRE(db *sql.DB, file string) (int, error) { progress := []string{"/", "-", "\\", "|", "-", "\\"} p := 0 var err error table, err := whatTable("FRE") if err != nil { return 0, err } companies, _ := loadCompanies(db) fh, err := os.Open(file) if err != nil { return 0, errors.Wrapf(err, "erro ao abrir arquivo %s", file) } defer fh.Close() dec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder()) // BEGIN TRANSACTION tx, err := db.Begin() if err != nil { return 0, errors.Wrap(err, "Failed to begin transaction") } // Data used inside loop sep := func(r rune) bool { return r == ';' } header := make(map[string]int) // stores the header item position (e.g., DT_FIM_EXERC:9) scanner := bufio.NewScanner(dec) count := 0 insert := "" var stmt *sql.Stmt // Loop thru file, line by line fmt.Print("[ ] Processando arquivo FRE") for scanner.Scan() { line := scanner.Text() if len(line) == 0 { continue } fields := strings.FieldsFunc(line, sep) if len(header) == 0 { // HEADER // Get header positioning for i, h := range fields { header[h] = i } // Prepare insert statement insert = fmt.Sprintf(`INSERT OR IGNORE INTO %s ( ID, ID_CIA, YEAR, Versao, Quantidade_Total_Acoes_Circulacao, Percentual_Total_Acoes_Circulacao ) VALUES ( ?, ?, ?, ?, ?, ? );`, table) stmt, err = tx.Prepare(insert) if err != nil { err = errors.Wrapf(err, "erro ao preparar insert (verificar cabeçalho do arquivo %s)", file) return count, err } defer stmt.Close() } else { // VALUES if len(fields) <= 12 { continue } // INSERT f, err := prepareFREFields(header, fields, companies) if err == ErrCNPJNotFound { continue } if err != nil { fmt.Println(line) fmt.Printf("\r[x] Falha ao preparar registro: %v\n", err) fmt.Print("[ ] Processando arquivo FRE") continue } _, err = stmt.Exec(f...) if err != nil { return count, errors.Wrap(err, "falha ao inserir registro") } } if count++; count%60 == 0 { fmt.Printf("\r[%s", progress[p%6]) p++ } } fmt.Print("\r[*") // END TRANSACTION err = tx.Commit() if err != nil { return count, errors.Wrap(err, "Failed to commit transaction") } if err := scanner.Err(); err != nil { return count, errors.Wrapf(err, "erro ao ler arquivo %s", file) } return count, nil } func prepareFREFields(header map[string]int, fields []string, companies map[string]company) ([]interface{}, error) { if len(fields) < len(header)-1 { return nil, fmt.Errorf("len(fields)=%d != len(header)=%d", len(fields), len(header)) } // val checks and gets the value from a map val := func(key string) string { v, ok := header[key] if !ok { return "" } return fields[v] } // YEAR v, ok := header["Data_Referencia"] if !ok { return nil, fmt.Errorf("Data_Referencia não encontrado") } if len(fields[v]) != 10 { return nil, fmt.Errorf("DT_FIM_EXERC incorreto: %v", fields[v]) } year := fields[v][:4] // CNPJ_Companhia is replaced by company id cnpj := val("CNPJ_Companhia") c, ok := companies[cnpj] if !ok { return nil, ErrCNPJNotFound } companyID := c.id // Free float ff := val("Percentual_Total_Acoes_Circulacao") var freeFloat float32 if ff != "" { if f, err := strconv.ParseFloat(ff, 32); err == nil { freeFloat = float32(f / 100) } } // Total shares considering the free float shares := val("Quantidade_Total_Acoes_Circulacao") var totalShares float32 if shares != "" { if f, err := strconv.ParseFloat(shares, 32); err == nil { if freeFloat > 0 { totalShares = float32(f) / freeFloat } } } // Unique value to be used as PRIMARY KEY hash := Hash(cnpj + val("Data_Referencia") + val("Versao") + val("ID_Documento") + val("Quantidade_Total_Acoes_Circulacao")) // Output -- need to match INSERT sequence var f []interface{} f = append(f, hash) // ID f = append(f, companyID) // ID_CIA f = append(f, year) // YEAR f = append(f, val("Versao")) f = append(f, totalShares) f = append(f, freeFloat) return f, nil } ================================================ FILE: parsers/fuzzy.go ================================================ package parsers import ( "strings" "github.com/lithammer/fuzzysearch/fuzzy" ) // // FuzzyMatch measures the Levenshtein distance between // the source and the list, returning true if the distance // is less or equal the 'distance'. // Diacritics are removed from 'src' and 'list'. // func FuzzyMatch(src string, list []string, distance int) bool { return FuzzyFind(src, list, distance) != "" } // // FuzzyFind returns the most approximate string inside 'list' that // matches the 'src' string within a maximum 'distance'. // func FuzzyFind(source string, targets []string, maxDistance int) (found string) { for _, target := range targets { src := fix(source) trg := fix(target) if strings.HasPrefix(src, trg) || strings.HasPrefix(trg, src) { return target } distance := fuzzy.LevenshteinDistance(src, trg) if distance <= maxDistance { maxDistance = distance found = target } } if found == "" { for _, target := range targets { src := strings.Split(fix(source), " ") trg := strings.Split(fix(target), " ") if len(src) > 2 && len(trg) > 2 { if src[0] == trg[0] && src[1] == trg[1] { return target } } } } return } func fix(txt string) string { txt = strings.ToUpper(txt) txt = strings.Replace(txt, "BCO ", "BANCO ", 1) return RemoveDiacritics(txt) } ================================================ FILE: parsers/fuzzy_test.go ================================================ package parsers import "testing" func TestFuzzyFind(t *testing.T) { list := []struct { src string trg []string maxDist int expected string }{ {"ABCD", []string{"ABC", "ACD"}, 2, "ABC"}, {"ABCD", []string{"XYZ", "ACD"}, 1, "ACD"}, {"ABCDÉ", []string{"XYZ", "ACD", "ABCDE"}, 0, "ABCDE"}, {"ABCDÉ FGH", []string{"XYZ", "ACD", "FGH"}, 6, "FGH"}, {"BCO ABC", []string{"XYZ", "BANCO ABC", "FGH"}, 0, "BANCO ABC"}, } for _, l := range list { r := FuzzyFind(l.src, l.trg, l.maxDist) if r != l.expected { t.Errorf("Expected: %s, got: %s", l.expected, r) } } } ================================================ FILE: parsers/md5.go ================================================ package parsers import ( "crypto/md5" "database/sql" "fmt" "io" "os" ) // // isNewFile checks the database to see if this file has been // processed already // func isNewFile(db *sql.DB, filename string) (isNew bool, err error) { isNew = true md5, err := md5FromFile(filename) if err != nil { return } sqlStmt := `SELECT md5 FROM md5 WHERE md5 = ?` err = db.QueryRow(sqlStmt, md5).Scan(&md5) if err != nil { return } isNew = false return } // // storeFile into md5 table (only successfully processed files) // func storeFile(db *sql.DB, filename string) (md5 string) { md5, err := md5FromFile(filename) if err != nil { return "" } insert := fmt.Sprintf(`INSERT OR IGNORE INTO md5 (md5) VALUES ("%s")`, md5) _, err = db.Exec(insert) if err != nil { return "" } return md5 } // // md5FromFile // func md5FromFile(filename string) (string, error) { f, err := os.Open(filename) if err != nil { return "", err } defer f.Close() h := md5.New() if _, err = io.Copy(h, f); err != nil { return "", err } return fmt.Sprintf("%x", h.Sum(nil)), nil } ================================================ FILE: parsers/md5_test.go ================================================ package parsers import ( "database/sql" "os" "testing" _ "github.com/mattn/go-sqlite3" "github.com/pkg/errors" ) func TestIsNewFile(t *testing.T) { if testing.Short() { t.Skip("skipping testing in short mode") // used in CI } db, err := openDatabase() if err != nil { t.Errorf("cannot open db: %v", err) return } if err := createTable(db, "MD5"); err != nil { t.Errorf("could not create table: %v", err) } file := "../cli/.data/bpa_cia_aberta_con_2017.csv" isNew, err := isNewFile(db, file) expected := false if _, err := os.Stat(file); !os.IsNotExist(err) { expected = true } if isNew == expected { t.Errorf("isNewFile returned %v. If 'rapina get' has run before it should've returned false.\nError: [%v]", expected, err) } } func openDatabase() (db *sql.DB, err error) { db, err = sql.Open("sqlite3", "../bin/.data/rapina.db") if err != nil { return db, errors.Wrap(err, "database open failed") } return } ================================================ FILE: parsers/meta/meta_bpa_cia_aberta.txt ================================================ ----------------------- Campo: CNPJ_CIA ----------------------- Descrição: CNPJ da companhia Domínio: Alfanumérico Tipo dados: varchar Precisão: 20 Scale: 0 ----------------------- Campo: DT_REFER ----------------------- Descrição: Data de referência do documento Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: VERSAO ----------------------- Descrição: Versão do documento Domínio: Numérico Tipo dados: smallint Precisão: 5 Scale: 0 ----------------------- Campo: DENOM_CIA ----------------------- Descrição: Nome empresarial da companhia Domínio: Alfanumérico Tipo dados: varchar Precisão: 100 Scale: 0 ----------------------- Campo: CD_CVM ----------------------- Descrição: Código CVM Domínio: Numérico Tipo dados: numeric Precisão: 7 Scale: 0 ----------------------- Campo: GRUPO_DFP ----------------------- Descrição: Nome e nível de agregação da demonstração Domínio: Alfanumérico Tipo dados: varchar Precisão: 206 Scale: 0 ----------------------- Campo: MOEDA ----------------------- Descrição: Moeda Domínio: Alfanumérico Tipo dados: varchar Precisão: 4 Scale: 0 ----------------------- Campo: ESCALA_MOEDA ----------------------- Descrição: Escala monetária Domínio: Alfanumérico Tipo dados: varchar Precisão: 7 Scale: 0 ----------------------- Campo: ORDEM_EXERC ----------------------- Descrição: Ordem do exercício social Domínio: Alfanumérico Tipo dados: varchar Precisão: 9 Scale: 0 ----------------------- Campo: DT_FIM_EXERC ----------------------- Descrição: Data fim do exercício social Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: CD_CONTA ----------------------- Descrição: Código da conta Domínio: Numérico Tipo dados: varchar Precisão: 18 Scale: 0 ----------------------- Campo: DS_CONTA ----------------------- Descrição: Descrição da conta Domínio: Alfanumérico Tipo dados: varchar Precisão: 100 Scale: 0 ----------------------- Campo: VL_CONTA ----------------------- Descrição: Valor da conta Domínio: Numérico Tipo dados: numeric Precisão: 29 Scale: 2 ================================================ FILE: parsers/meta/meta_bpp_cia_aberta.txt ================================================ ----------------------- Campo: CNPJ_CIA ----------------------- Descrição: CNPJ da companhia Domínio: Alfanumérico Tipo dados: varchar Precisão: 20 Scale: 0 ----------------------- Campo: DT_REFER ----------------------- Descrição: Data de referência do documento Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: VERSAO ----------------------- Descrição: Versão do documento Domínio: Numérico Tipo dados: smallint Precisão: 5 Scale: 0 ----------------------- Campo: DENOM_CIA ----------------------- Descrição: Nome empresarial da companhia Domínio: Alfanumérico Tipo dados: varchar Precisão: 100 Scale: 0 ----------------------- Campo: CD_CVM ----------------------- Descrição: Código CVM Domínio: Numérico Tipo dados: numeric Precisão: 7 Scale: 0 ----------------------- Campo: GRUPO_DFP ----------------------- Descrição: Nome e nível de agregação da demonstração Domínio: Alfanumérico Tipo dados: varchar Precisão: 206 Scale: 0 ----------------------- Campo: MOEDA ----------------------- Descrição: Moeda Domínio: Alfanumérico Tipo dados: varchar Precisão: 4 Scale: 0 ----------------------- Campo: ESCALA_MOEDA ----------------------- Descrição: Escala monetária Domínio: Alfanumérico Tipo dados: varchar Precisão: 7 Scale: 0 ----------------------- Campo: ORDEM_EXERC ----------------------- Descrição: Ordem do exercício social Domínio: Alfanumérico Tipo dados: varchar Precisão: 9 Scale: 0 ----------------------- Campo: DT_FIM_EXERC ----------------------- Descrição: Data fim do exercício social Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: CD_CONTA ----------------------- Descrição: Código da conta Domínio: Numérico Tipo dados: varchar Precisão: 18 Scale: 0 ----------------------- Campo: DS_CONTA ----------------------- Descrição: Descrição da conta Domínio: Alfanumérico Tipo dados: varchar Precisão: 100 Scale: 0 ----------------------- Campo: VL_CONTA ----------------------- Descrição: Valor da conta Domínio: Numérico Tipo dados: numeric Precisão: 29 Scale: 2 ================================================ FILE: parsers/meta/meta_dfc_md_cia_aberta.txt ================================================ ----------------------- Campo: CNPJ_CIA ----------------------- Descrição: CNPJ da companhia Domínio: Alfanumérico Tipo dados: varchar Precisão: 20 Scale: 0 ----------------------- Campo: DT_REFER ----------------------- Descrição: Data de referência do documento Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: VERSAO ----------------------- Descrição: Versão do documento Domínio: Numérico Tipo dados: smallint Precisão: 5 Scale: 0 ----------------------- Campo: DENOM_CIA ----------------------- Descrição: Nome empresarial da companhia Domínio: Alfanumérico Tipo dados: varchar Precisão: 100 Scale: 0 ----------------------- Campo: CD_CVM ----------------------- Descrição: Código CVM Domínio: Numérico Tipo dados: numeric Precisão: 7 Scale: 0 ----------------------- Campo: GRUPO_DFP ----------------------- Descrição: Nome e nível de agregação da demonstração Domínio: Alfanumérico Tipo dados: varchar Precisão: 206 Scale: 0 ----------------------- Campo: MOEDA ----------------------- Descrição: Moeda Domínio: Alfanumérico Tipo dados: varchar Precisão: 4 Scale: 0 ----------------------- Campo: ESCALA_MOEDA ----------------------- Descrição: Escala monetária Domínio: Alfanumérico Tipo dados: varchar Precisão: 7 Scale: 0 ----------------------- Campo: ORDEM_EXERC ----------------------- Descrição: Ordem do exercício social Domínio: Alfanumérico Tipo dados: varchar Precisão: 9 Scale: 0 ----------------------- Campo: DT_INI_EXERC ----------------------- Descrição: Data início do exercício social Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: DT_FIM_EXERC ----------------------- Descrição: Data fim do exercício social Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: CD_CONTA ----------------------- Descrição: Código da conta Domínio: Numérico Tipo dados: varchar Precisão: 18 Scale: 0 ----------------------- Campo: DS_CONTA ----------------------- Descrição: Descrição da conta Domínio: Alfanumérico Tipo dados: varchar Precisão: 100 Scale: 0 ----------------------- Campo: VL_CONTA ----------------------- Descrição: Valor da conta Domínio: Numérico Tipo dados: numeric Precisão: 29 Scale: 2 ================================================ FILE: parsers/meta/meta_dfc_mi_cia_aberta.txt ================================================ ----------------------- Campo: CNPJ_CIA ----------------------- Descrição: CNPJ da companhia Domínio: Alfanumérico Tipo dados: varchar Precisão: 20 Scale: 0 ----------------------- Campo: DT_REFER ----------------------- Descrição: Data de referência do documento Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: VERSAO ----------------------- Descrição: Versão do documento Domínio: Numérico Tipo dados: smallint Precisão: 5 Scale: 0 ----------------------- Campo: DENOM_CIA ----------------------- Descrição: Nome empresarial da companhia Domínio: Alfanumérico Tipo dados: varchar Precisão: 100 Scale: 0 ----------------------- Campo: CD_CVM ----------------------- Descrição: Código CVM Domínio: Numérico Tipo dados: numeric Precisão: 7 Scale: 0 ----------------------- Campo: GRUPO_DFP ----------------------- Descrição: Nome e nível de agregação da demonstração Domínio: Alfanumérico Tipo dados: varchar Precisão: 206 Scale: 0 ----------------------- Campo: MOEDA ----------------------- Descrição: Moeda Domínio: Alfanumérico Tipo dados: varchar Precisão: 4 Scale: 0 ----------------------- Campo: ESCALA_MOEDA ----------------------- Descrição: Escala monetária Domínio: Alfanumérico Tipo dados: varchar Precisão: 7 Scale: 0 ----------------------- Campo: ORDEM_EXERC ----------------------- Descrição: Ordem do exercício social Domínio: Alfanumérico Tipo dados: varchar Precisão: 9 Scale: 0 ----------------------- Campo: DT_INI_EXERC ----------------------- Descrição: Data início do exercício social Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: DT_FIM_EXERC ----------------------- Descrição: Data fim do exercício social Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: CD_CONTA ----------------------- Descrição: Código da conta Domínio: Numérico Tipo dados: varchar Precisão: 18 Scale: 0 ----------------------- Campo: DS_CONTA ----------------------- Descrição: Descrição da conta Domínio: Alfanumérico Tipo dados: varchar Precisão: 100 Scale: 0 ----------------------- Campo: VL_CONTA ----------------------- Descrição: Valor da conta Domínio: Numérico Tipo dados: numeric Precisão: 29 Scale: 2 ================================================ FILE: parsers/meta/meta_dre_cia_aberta.txt ================================================ ----------------------- Campo: CNPJ_CIA ----------------------- Descrição: CNPJ da companhia Domínio: Alfanumérico Tipo dados: varchar Precisão: 20 Scale: 0 ----------------------- Campo: DT_REFER ----------------------- Descrição: Data de referência do documento Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: VERSAO ----------------------- Descrição: Versão do documento Domínio: Numérico Tipo dados: smallint Precisão: 5 Scale: 0 ----------------------- Campo: DENOM_CIA ----------------------- Descrição: Nome empresarial da companhia Domínio: Alfanumérico Tipo dados: varchar Precisão: 100 Scale: 0 ----------------------- Campo: CD_CVM ----------------------- Descrição: Código CVM Domínio: Numérico Tipo dados: numeric Precisão: 7 Scale: 0 ----------------------- Campo: GRUPO_DFP ----------------------- Descrição: Nome e nível de agregação da demonstração Domínio: Alfanumérico Tipo dados: varchar Precisão: 206 Scale: 0 ----------------------- Campo: ESCALA_DRE ----------------------- Descrição: Escala monetária Domínio: Alfanumérico Tipo dados: varchar Precisão: 7 Scale: 0 ----------------------- Campo: ORDEM_EXERC ----------------------- Descrição: Ordem do exercício social Domínio: Alfanumérico Tipo dados: varchar Precisão: 9 Scale: 0 ----------------------- Campo: DT_INI_EXERC ----------------------- Descrição: Data início do exercício social Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: DT_FIM_EXERC ----------------------- Descrição: Data fim do exercício social Domínio: AAAA-MM-DD Tipo dados: date Precisão: 10 Scale: 0 ----------------------- Campo: CD_CONTA ----------------------- Descrição: Código da conta Domínio: Numérico Tipo dados: varchar Precisão: 18 Scale: 0 ----------------------- Campo: DS_CONTA ----------------------- Descrição: Descrição da conta Domínio: Alfanumérico Tipo dados: varchar Precisão: 100 Scale: 0 ----------------------- Campo: VL_CONTA ----------------------- Descrição: Valor da conta Domínio: Numérico Tipo dados: numeric Precisão: 29 Scale: 2 ================================================ FILE: parsers/sectors.go ================================================ package parsers import ( "bufio" "fmt" "os" "regexp" "strings" "github.com/PuerkitoBio/goquery" "github.com/dude333/rapina" "github.com/gocolly/colly/v2" "github.com/pkg/errors" yaml "gopkg.in/yaml.v2" ) // SectorsToYaml grab data from B3 website and prints out to a yaml file // with all companies grouped by sector, subsector, segment func SectorsToYaml(yamlFile string) (err error) { progress := []string{"/", "-", "\\", "|", "-", "\\"} var p int32 if !overwritePrompt(yamlFile) { return rapina.ErrFileNotUpdated } f, err := os.Create(yamlFile) if err != nil { return errors.Wrapf(err, "falha ao criar arquivo %s", yamlFile) } defer f.Close() w := bufio.NewWriter(f) c := colly.NewCollector( // Restrict crawling to specific domains // colly.AllowedDomains("bvmf.bmfbovespa.com.br"), colly.AllowURLRevisit(), colly.Async(false), colly.CacheDir(".data/cache"), ) c.OnHTML("tr", func(e *colly.HTMLElement) { var sector string var subsectors []string c := 0 e.ForEach("td", func(_ int, elem *colly.HTMLElement) { elem.DOM.Each(func(_ int, s *goquery.Selection) { h, _ := s.Html() if c == 0 { sector = h fmt.Fprintln(w, " - Setor:", sector) fmt.Fprintln(w, " Subsetores:") } else if c == 1 { subsectors = strings.Split(h, "
") last := subsectors[0] for i := range subsectors { if subsectors[i] == "" { subsectors[i] = last } last = subsectors[i] } } c++ }) lastSub := "" elem.ForEach("a[href]", func(i int, elem *colly.HTMLElement) { if strings.Contains(elem.Attr("href"), "BuscaEmpresaListada.aspx") { // fmt.Printf("\n=> %s > %s > %s:\n", sector, subsectors[i], elem.Text) //, elem.Attr("href")) if subsectors[i] != lastSub { fmt.Fprintln(w, " - Subsetor:", subsectors[i]) fmt.Fprintln(w, " Segmentos:") } lastSub = subsectors[i] fmt.Fprintln(w, " - Segmento:", removeYamlInvalidChar(elem.Text)) fmt.Fprintln(w, " Empresas:") _ = companies(w, "http://bvmf.bmfbovespa.com.br/cias-listadas/empresas-listadas/"+elem.Attr("href")) } fmt.Printf("\r[%s]", progress[p%6]) p++ }) }) }) fmt.Print("[ ] Lendo informações do site da B3") fmt.Fprintln(w, "Setores:") err = c.Visit("http://bvmf.bmfbovespa.com.br/cias-listadas/empresas-listadas/BuscaEmpresaListada.aspx?opcao=1&indiceAba=1&Idioma=pt-br") fmt.Println() w.Flush() return } // companies lists all companies in the same sector/subsector/segment func companies(w *bufio.Writer, url string) error { c := colly.NewCollector( // Restrict crawling to specific domains // colly.AllowedDomains("bvmf.bmfbovespa.com.br"), colly.AllowURLRevisit(), colly.Async(false), colly.CacheDir(".data/cache"), ) // Find and visit all links c.OnHTML("tr", func(e *colly.HTMLElement) { // if e.Attr("class") != "GridRow_SiteBmfBovespa GridBovespaItemStyle" { // return // } e.ForEachWithBreak("a", func(_ int, elem *colly.HTMLElement) bool { if strings.Contains(elem.Attr("href"), "ResumoEmpresaPrincipal.aspx") { fmt.Fprintln(w, " -", removeYamlInvalidChar(elem.Text)) } return false // get only the 1st elem }) }) return c.Visit(url) } // overwritePrompt prompts to overwrite file if it exists func overwritePrompt(filename string) bool { if _, err := os.Stat(filename); err == nil { // check if file exists fmt.Printf("\n[?] Deseja sobrescrever o arquivo \"%s\"? (s/N) ", filename) reader := bufio.NewReader(os.Stdin) prompt, _ := reader.ReadString('\n') if !strings.EqualFold(prompt, "s\n") && !strings.EqualFold(prompt, "sim\n") && !strings.EqualFold(prompt, "s\r\n") && !strings.EqualFold(prompt, "sim\r\n") { return false } } return true } // S contains the sectors type S struct { Sectors []Sector `yaml:"Setores"` } // Sector is divided into subsectors type Sector struct { Name string `yaml:"Setor"` Subsectors []Subsector `yaml:"Subsetores"` } // Subsector is divided into segments type Subsector struct { Name string `yaml:"Subsetor"` Segments []Segment `yaml:"Segmentos"` } // Segment contains companies from the same sector/subsector/segment type Segment struct { Name string `yaml:"Segmento"` Companies []string `yaml:"Empresas"` } // FromSector returns all companies from the same sector as the 'company' func FromSector(company, yamlFile string) (companies []string, sectorName string, err error) { y, err := os.ReadFile(yamlFile) if err != nil { err = errors.Wrapf(err, "ReadFile: %v", err) return } s := S{} if err := yaml.Unmarshal(y, &s); err != nil { return nil, "", err } for _, sector := range s.Sectors { for _, subsector := range sector.Subsectors { for _, segment := range subsector.Segments { if FuzzyMatch(company, segment.Companies, 2) { companies = segment.Companies sectorName = strings.Join([]string{sector.Name, subsector.Name, segment.Name}, " > ") return } } } } return } // removeYamlInvalidChar removes yaml invalid characters func removeYamlInvalidChar(text string) string { yaml_invalid_chars := regexp.MustCompile(`[^/\s.A-zÀ-ú0-9&():-]`) return yaml_invalid_chars.ReplaceAllString(text, "") } ================================================ FILE: parsers/sectors_test.go ================================================ package parsers import ( "os" "testing" ) func TestFromSector(t *testing.T) { tempDir, _ := os.MkdirTemp("", "rapina-test") filename := tempDir + "/test_sectors.yml" createYaml(filename) s, _, _ := FromSector("GRENDENE S.A.", filename) expected := [...]string{"ALPARGATAS S.A.", "CAMBUCI S.A.", "GRENDENE S.A.", "VULCABRAS/AZALEIA S.A."} if len(s) != 4 { t.Errorf("\n- Expected: %v\n- Got: %v", expected, s) } var arr [4]string copy(arr[:], s) if arr != expected { t.Errorf("\n- Expected: %v\n- Got: %v", expected, s) } os.Remove(filename) } func createYaml(filename string) { yaml := []byte( `Setores: - Setor: Bens Industriais Subsetores: - Subsetor: Comércio Segmentos: - Segmento: Material de Transporte Empresas: - MINASMAQUINAS S.A. - WLM PART. E COMÉRCIO DE MÁQUINAS E VEÍCULOS S.A. - Setor: Consumo Cíclico Subsetores: - Subsetor: Tecidos. Vestuário e Calçados Segmentos: - Segmento: Acessórios Empresas: - MUNDIAL S.A. - PRODUTOS DE CONSUMO - TECHNOS S.A. - Segmento: Calçados Empresas: - ALPARGATAS S.A. - CAMBUCI S.A. - GRENDENE S.A. - VULCABRAS/AZALEIA S.A.`) _ = os.WriteFile(filename, yaml, 0644) } ================================================ FILE: parsers/stock.go ================================================ package parsers /* TODO: https://query1.finance.yahoo.com/v7/finance/download/RBVA11.SA?period1=1588395063&period2=1619931063&interval=1d&events=history&includeAdjustedClose=true https://query1.finance.yahoo.com/v7/finance/download/BBPO11.SA?period1=1619654400&period2=1619740800&interval=1d&events=history&includeAdjustedClose=true */ import ( "bufio" "database/sql" "fmt" "io" "os" "strconv" "strings" "sync" "github.com/dude333/rapina" "github.com/dude333/rapina/progress" "github.com/pkg/errors" "golang.org/x/text/encoding/charmap" "golang.org/x/text/transform" ) type stockQuote struct { Stock string Date string Open float64 High float64 Low float64 Close float64 Volume float64 } type stockCode struct { TckrSymb string // Code SgmtNm string // value: CASH SctyCtgyNm string // values: SHARES, UNIT, FUNDS CrpnNm string // Company name SpcfctnCd string // values: ON, ON NM, PN N2, etc. CorpGovnLvlNm string // values: NOVO MERCADO, NIVEL 2, etc. } type StockParser struct { db *sql.DB log rapina.Logger } // // NewStock creates the required tables, if necessary, and returns a StockParser instance. // func NewStock(db *sql.DB, log rapina.Logger) (*StockParser, error) { for _, t := range []string{"status", "stock_quotes", "stock_codes"} { if err := createTable(db, t); err != nil { return nil, err } } s := &StockParser{db: db, log: log} return s, nil } // // Quote returns the quote from DB. // func (s *StockParser) Quote(code, date string) (float64, error) { query := `SELECT close FROM stock_quotes WHERE stock=$1 AND date=$2;` var close float64 err := s.db.QueryRow(query, code, date).Scan(&close) if err == sql.ErrNoRows { return 0, errors.New("não encontrado no bd") } if err != nil { return 0, errors.Wrapf(err, "lendo cotação de %s do bd", code) } return close, nil } // // Quote returns the company ON stock code, where stockType is: // ON, PN, UNT, CI [CI = FII] // func (s *StockParser) Code(companyName, stockType string) (string, error) { query := `SELECT trading_code FROM stock_codes WHERE company_name LIKE ? AND SpcfctnCd LIKE ?;` st := strings.ToUpper(stockType + "%") var code string err := s.db.QueryRow(query, "%"+companyName+"%", st).Scan(&code) if err == sql.ErrNoRows { return "", errors.New("não encontrado no bd") } if err != nil { return "", errors.Wrapf(err, "lendo código de %s do bd", companyName) } return code, nil } func (s *StockParser) SaveB3Quotes(filename string) error { isNew, err := isNewFile(s.db, filename) if !isNew && err == nil { // if error, process file progress.Warning("%s já processado anteriormente", filename) return errors.New("este arquivo de cotações já foi importado anteriormente") } if err := s.populateStockQuotes(filename); err != nil { return err } storeFile(s.db, filename) return nil } func (s *StockParser) populateStockQuotes(filename string) error { fh, err := os.Open(filename) if err != nil { return errors.Wrapf(err, "abrindo arquivo %s", filename) } defer fh.Close() // BEGIN TRANSACTION tx, err := s.db.Begin() if err != nil { return errors.Wrap(err, "Failed to begin transaction") } dec := transform.NewReader(fh, charmap.ISO8859_1.NewDecoder()) scanner := bufio.NewScanner(dec) for scanner.Scan() { line := scanner.Text() if len(line) == 0 { continue } q, err := parseB3Quote(line) if err != nil { continue // ignore line } fmt.Printf("%+v\n", q) } // END TRANSACTION if err := tx.Commit(); err != nil { return errors.Wrap(err, "Failed to commit transaction") } if err := scanner.Err(); err != nil { return errors.Wrapf(err, "lendo arquivo %s", filename) } return nil } // // Save parses the 'stream', get the 'code' stock quotes and // store it on 'db'. Returns the number of registers saved. // func (s *StockParser) Save(stream io.Reader, code string) (int, error) { if s.db == nil { return 0, errors.New("bd inválido") } if stream == nil { return 0, errors.New("sem dados") } scanner := bufio.NewScanner(stream) // Read 1st line scanner.Scan() prov := provider(scanner.Text()) var r rec if err := r.open(s.db, prov); err != nil { return 0, err } defer r.close() // Read stream, line by line var count int for scanner.Scan() { line := scanner.Text() var q *stockQuote var c *stockCode var err error switch prov { case b3Quotes: q, err = parseB3Quote(line) case yahoo: q, err = parseYahoo(line, code) case alphaVantage: q, err = parseAlphaVantage(line, code) case b3Codes: c, err = parseB3Code(line) } if err != nil { continue // ignore lines with error } if q != nil { err = r.storeQuote(q) if err == nil { count++ } } if c != nil { err = r.storeCode(c) if err == nil { count++ } } } if err := scanner.Err(); err != nil { return count, err } return count, nil } // open prepares the insert statement. func (s *rec) open(db *sql.DB, provider int) error { var err error insert := `INSERT OR IGNORE INTO stock_quotes (stock, date, open, high, low, close, volume) VALUES (?,?,?,?,?,?,?);` if provider == b3Codes { insert = `INSERT OR IGNORE INTO stock_codes (trading_code, company_name, SpcfctnCd, CorpGovnLvlNm) VALUES (?,?,?,?);` } s.stmt, err = db.Prepare(insert) if err != nil || s.stmt == nil { return errors.Wrap(err, "insert on db") } return nil } // storeQuote stores the data using the insert statement. func (s *rec) storeQuote(q *stockQuote) error { if s.stmt == nil { return errors.New("sql statement not initalized") } s.mu.Lock() res, err := s.stmt.Exec( q.Stock, q.Date, q.Open, q.High, q.Low, q.Close, q.Volume, ) s.mu.Unlock() if err != nil { return errors.Wrap(err, "salvando cotação") } n, err := res.RowsAffected() if n == 0 || err != nil { return errors.New("registro não salvo (duplicado)") } return nil } // storeQuote stores the data using the insert statement. func (s *rec) storeCode(c *stockCode) error { if s.stmt == nil { return errors.New("sql statement not initalized") } s.mu.Lock() res, err := s.stmt.Exec( c.TckrSymb, // trading_code c.CrpnNm, // company_name c.SpcfctnCd, c.CorpGovnLvlNm, ) s.mu.Unlock() if err != nil { return errors.Wrap(err, "salvando códigos") } n, err := res.RowsAffected() if n == 0 || err != nil { return errors.New("registro não salvo (duplicado)") } return nil } // close closes the insert statement. func (s *rec) close() error { var err error if s.stmt != nil { err = s.stmt.Close() } return err } // API providers. const ( none int = iota alphaVantage yahoo b3Quotes b3Codes ) // provider returns stream type based on header func provider(header string) int { if header == "timestamp,open,high,low,close,volume" { return alphaVantage } if header == "Date,Open,High,Low,Close,Adj Close,Volume" { return yahoo } if strings.HasPrefix(header, "00COTAHIST.") { return b3Quotes } if strings.HasPrefix(header, "RptDt;TckrSymb;Asst;AsstDesc;SgmtNm;MktNm;SctyCtgyNm;XprtnDt;") { return b3Codes } return none } // parseAlphaVantage parses lines downloaded from Alpha Vantage API server // and returns *stockQuote for 'code'. func parseAlphaVantage(line, code string) (*stockQuote, error) { fields := strings.Split(line, ",") if len(fields) != 6 { return nil, errors.New("linha inválida") // ignore lines with error } // Columns: timestamp,open,high,low,close,volume var err error var floats [5]float64 for i := 1; i <= 5; i++ { floats[i-1], err = strconv.ParseFloat(fields[i], 64) if err != nil { return nil, errors.Wrap(err, "campo inválido") } } return &stockQuote{ Stock: code, Date: fields[0], Open: floats[0], High: floats[1], Low: floats[2], Close: floats[3], Volume: floats[4], }, nil } // parseYahoo parses lines downloaded from Yahoo Finance API server // and returns *stockQuote for 'code'. func parseYahoo(line, code string) (*stockQuote, error) { fields := strings.Split(line, ",") if len(fields) != 7 { return nil, errors.New("linha inválida") // ignore lines with error } // Columns: Date,Open,High,Low,Close,Adj Close,Volume var err error var floats [6]float64 for i := 1; i <= 6; i++ { floats[i-1], err = strconv.ParseFloat(fields[i], 64) if err != nil { return nil, errors.Wrap(err, "campo inválido") } } return &stockQuote{ Stock: code, Date: fields[0], Open: floats[0], High: floats[1], Low: floats[2], Close: floats[3], Volume: floats[5], }, nil } // parseB3Quote parses the line based on this layout: // http://www.b3.com.br/data/files/33/67/B9/50/D84057102C784E47AC094EA8/SeriesHistoricas_Layout.pdf // // CAMPO/CONTEÚDO TIPO E TAMANHO POS. INIC. POS. FINAL // TIPREG “01” N(02) 01 02 // DATA “AAAAMMDD” N(08) 03 10 // CODBDI X(02) 11 12 // CODNEG X(12) 13 24 // TPMERC N(03) 25 27 // PREABE (11)V99 57 69 // PREMAX (11)V99 70 82 // PREMIN (11)V99 83 95 // PREULT (11)V99 109 121 // QUATOT N18 153 170 // VOLTOT (16)V99 171 188 // // CODBDI: // 02 LOTE PADRÃO // 12 FUNDO IMOBILIÁRIO // // TPMERC: // 010 VISTA // 020 FRACIONÁRIO func parseB3Quote(line string) (*stockQuote, error) { if len(line) != 245 { return nil, errors.New("linha deve conter 245 bytes") } recType := line[0:2] if recType != "01" { return nil, fmt.Errorf("registro %s ignorado", recType) } codBDI := line[10:12] if codBDI != "02" && codBDI != "12" && codBDI != "13" && codBDI != "14" { return nil, fmt.Errorf("BDI %s ignorado", codBDI) } tpMerc := line[24:27] if tpMerc != "010" && tpMerc != "020" { return nil, fmt.Errorf("tipo de mercado %s ignorado", tpMerc) } date := line[2:6] + "-" + line[6:8] + "-" + line[8:10] code := strings.TrimSpace(line[12:24]) numRanges := [5]struct { i, f int }{ {56, 69}, // PREABE = open {69, 82}, // PREMAX = high {82, 95}, // PREMIN = low {108, 121}, // PREULT = close {170, 188}, // VOLTOT = volume } var vals [5]int for i, r := range numRanges { num, err := strconv.Atoi(line[r.i:r.f]) if err != nil { return nil, err } vals[i] = num } return &stockQuote{ Stock: code, Date: date, Open: float64(vals[0]) / 100, High: float64(vals[1]) / 100, Low: float64(vals[2]) / 100, Close: float64(vals[3]) / 100, Volume: float64(vals[4]) / 100, }, nil } type rec struct { stmt *sql.Stmt mu sync.Mutex // ensures atomic writes to db } // parseB3Code parses lines downloaded from B3 server // and returns *stockCode. // func parseB3Code(line string) (*stockCode, error) { fields := strings.Split(line, ";") // Columns: // RptDt;TckrSymb(2);Asst;AsstDesc;SgmtNm(5);MktNm;SctyCtgyNm(7);XprtnDt;XprtnCd; // TradgStartDt;TradgEndDt;BaseCd;ConvsCritNm;MtrtyDtTrgtPt;ReqrdConvsInd; // ISIN;CFICd;DlvryNtceStartDt;DlvryNtceEndDt;OptnTp;CtrctMltplr;AsstQtnQty; // AllcnRndLot;TradgCcy;DlvryTpNm;WdrwlDays;WrkgDays;ClnrDays;RlvrBasePricNm; // OpngFutrPosDay;SdTpCd1;UndrlygTckrSymb1;SdTpCd2;UndrlygTckrSymb2; // PureGoldWght;ExrcPric;OptnStyle;ValTpNm;PrmUpfrntInd;OpngPosLmtDt; // DstrbtnId;PricFctr;DaysToSttlm;SrsTpNm;PrtcnFlg;AutomtcExrcInd;SpcfctnCd(47); // CrpnNm(48);CorpActnStartDt;CtdyTrtmntTpNm;MktCptlstn;CorpGovnLvlNm(52) if len(fields) != 52 { return nil, fmt.Errorf("linha inválida %d", len(fields)) // ignore lines with error } s := stockCode{ TckrSymb: fields[1], SgmtNm: fields[4], SctyCtgyNm: fields[6], CrpnNm: fields[47], SpcfctnCd: fields[46], CorpGovnLvlNm: fields[51], } if s.SgmtNm != "CASH" || (s.SctyCtgyNm != "SHARES" && s.SctyCtgyNm != "FUNDS" && s.SctyCtgyNm != "UNIT") { return nil, errors.New("linha ignorada") } return &s, nil } ================================================ FILE: parsers/stock_test.go ================================================ package parsers import ( "reflect" "strings" "testing" ) func Test_parseB3(t *testing.T) { const file = `012021010412NSLU11 010FII LOURDES CI ER R$ 000000002840000000000284000000000027700000000002809000000000281900000000028029000000002819000168000000000000001381000000000038793560000000000000009999123100000010000000000000BRNSLUCTF008272 012021010412NVHO11 010FII NOVOHORICI ER R$ 000000000154000000000015900000000001535000000000153700000000015400000000001536000000000154000092000000000000006200000000000009533490000000000000009999123100000010000000000000BRNVHOCTF003186 012021010412ONEF11 010FII THE ONE CI R$ 000000001478800000000148000000000014717000000001478900000000147360000000014735000000001478700035000000000000002546000000000037652878000000000000009999123100000010000000000000BRONEFCTF003200` want := []stockQuote{ {Stock: "NSLU11", Date: "2021-01-04", Open: 284, High: 284, Low: 277, Close: 281.9, Volume: 387935.6}, {Stock: "NVHO11", Date: "2021-01-04", Open: 15.4, High: 15.9, Low: 15.35, Close: 15.4, Volume: 95334.9}, {Stock: "ONEF11", Date: "2021-01-04", Open: 147.88, High: 148, Low: 147.17, Close: 147.36, Volume: 376528.78}, } for i, line := range strings.Split(file, "\n") { got, err := parseB3Quote(line) if err != nil { t.Errorf("parseB3() error = %v", err) return } if err == nil && !reflect.DeepEqual(got, &want[i]) { t.Errorf("parseB3() got %+v, want %+v", got, &want[i]) } } } func Test_parseB3Code(t *testing.T) { type args struct { line string } tests := []struct { name string args args want *stockCode wantErr bool }{ { name: "funds", args: args{ `2021-05-13;ALMI11;ALMI;ALMI;CASH;EQUITY-CASH;FUNDS;;;2018-09-24;9999-12-31;;;;;BRALMICTF003;CICIRU;;;;;;1;BRL;;;;;;;;;;;;;;;;;250;1;2;;;;CI;FDO INV IMOB - FII TORRE ALMIRANTE;9999-12-31;FUNGIBLE;111177;`, }, want: &stockCode{TckrSymb: "ALMI11", SgmtNm: "CASH", SctyCtgyNm: "FUNDS", CrpnNm: "FDO INV IMOB - FII TORRE ALMIRANTE", SpcfctnCd: "CI", CorpGovnLvlNm: ""}, wantErr: false, }, { name: "unit", args: args{ `2021-05-13;ALUP11;ALUP;ALUP;CASH;EQUITY-CASH;UNIT;;;2021-04-28;9999-12-31;;;;;BRALUPCDAM15;EMXXXR;;;;;;1;BRL;;;;;;;;;;;;;;;;;112;1;2;;;;UNT N2;ALUPAR INVESTIMENTO S/A;9999-12-31;FUNGIBLE;136606616;NIVEL 2`, }, want: &stockCode{TckrSymb: "ALUP11", SgmtNm: "CASH", SctyCtgyNm: "UNIT", CrpnNm: "ALUPAR INVESTIMENTO S/A", SpcfctnCd: "UNT N2", CorpGovnLvlNm: "NIVEL 2"}, wantErr: false, }, { name: "shares", args: args{ `2021-05-13;ALPA3;ALPA;ALPA;CASH;EQUITY-CASH;SHARES;;;2020-02-17;9999-12-31;;;;;BRALPAACNOR0;ESVUFR;;;;;;1;BRL;;;;;;;;;;;;;;;;;229;1;2;;;;ON N1;ALPARGATAS S.A.;9999-12-31;FUNGIBLE;302010689;NIVEL 1`, }, want: &stockCode{TckrSymb: "ALPA3", SgmtNm: "CASH", SctyCtgyNm: "SHARES", CrpnNm: "ALPARGATAS S.A.", SpcfctnCd: "ON N1", CorpGovnLvlNm: "NIVEL 1"}, wantErr: false, }, { name: "should fail on odd lot", args: args{ `2021-05-13;ANIM3F;ANIM;ANIM;ODD LOT;EQUITY-CASH;SHARES;;;2021-02-19;9999-12-31;;;;;BRANIMACNOR6;ESVUFR;;;;;;1;BRL;;;;;;;;;;;;;;;;;107;1;2;;;;ON NM;ANIMA HOLDING S.A.;9999-12-31;FUNGIBLE;403868805;NOVO MERCADO`, }, want: &stockCode{}, wantErr: true, }, { name: "should fail on bdr", args: args{ `2021-05-13;AMZO34;AMZO;AMZO;CASH;EQUITY-CASH;BDR;;;2020-11-09;9999-12-31;;;;;BRAMZOBDR002;EDSXPR;;;;;;1;BRL;;;;;;;;;;;;;;;;;102;1;2;;;;DRN;AMAZON.COM, INC;9999-12-31;FUNGIBLE;79059664651;`, }, want: &stockCode{}, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := parseB3Code(tt.args.line) if (err != nil) != tt.wantErr { t.Errorf("parseB3Code() error = %v, wantErr %v", err, tt.wantErr) return } if !tt.wantErr && !reflect.DeepEqual(got, tt.want) { t.Errorf("parseB3Code() = %#v, want %v", got, tt.want) } }) } } ================================================ FILE: parsers/tables.go ================================================ package parsers import ( "database/sql" "fmt" "strings" "github.com/pkg/errors" ) const currentDbVersion = 210514 const currentFIIDbVersion = 210426 const currentStockCodesVersion = 210518 const currentStockQuotesVersion = 210305 var createTableMap = map[string]string{ "dfp": `CREATE TABLE IF NOT EXISTS dfp ( "ID" PRIMARY KEY, "ID_CIA" integer, "CODE" integer, "YEAR" string, "DATA_TYPE" string, "VERSAO" integer, "MOEDA" varchar(4), "ESCALA_MOEDA" varchar(7), "DT_FIM_EXERC" integer, "CD_CONTA" varchar(18), "DS_CONTA" varchar(100), "VL_CONTA" real );`, "itr": `CREATE TABLE IF NOT EXISTS itr ( "ID" PRIMARY KEY, "ID_CIA" integer, "CODE" integer, "YEAR" string, "DATA_TYPE" string, "VERSAO" integer, "MOEDA" varchar(4), "ESCALA_MOEDA" varchar(7), "DT_FIM_EXERC" integer, "CD_CONTA" varchar(18), "DS_CONTA" varchar(100), "VL_CONTA" real );`, "fre": `CREATE TABLE IF NOT EXISTS fre ( "ID" PRIMARY KEY, "ID_CIA" integer, "YEAR" string, "Versao" integer, "Quantidade_Total_Acoes_Circulacao" integer, "Percentual_Total_Acoes_Circulacao" real );`, "codes": `CREATE TABLE IF NOT EXISTS codes ( "CODE" INTEGER NOT NULL PRIMARY KEY, "NAME" varchar(100) );`, "companies": `CREATE TABLE IF NOT EXISTS companies ( "ID" INTEGER NOT NULL PRIMARY KEY, "CNPJ" varchar(20), "NAME" varchar(100) );`, "stock_codes": `CREATE TABLE IF NOT EXISTS stock_codes ( "trading_code" VARCHAR NOT NULL PRIMARY KEY, "company_name" VARCHAR, "SpcfctnCd" VARCHAR, "CorpGovnLvlNm" VARCHAR );`, "md5": `CREATE TABLE IF NOT EXISTS md5 ( md5 NOT NULL PRIMARY KEY );`, "fii_details": `CREATE TABLE IF NOT EXISTS fii_details ( cnpj TEXT NOT NULL PRIMARY KEY, acronym varchar(4), trading_code varchar(6), json varchar );`, "stock_quotes": `CREATE TABLE IF NOT EXISTS stock_quotes ( stock varchar(12) NOT NULL, date varchar(10) NOT NULL, open real, high real, low real, close real, volume real );`, "fii_dividends": `CREATE TABLE IF NOT EXISTS fii_dividends ( trading_code varchar(12) NOT NULL, base_date varchar(10) NOT NULL, payment_date varchar(10), value real );`, "status": `CREATE TABLE IF NOT EXISTS status ( table_name TEXT NOT NULL PRIMARY KEY, version integer );`, } func allTables() []string { keys := make([]string, len(createTableMap)) i := 0 for k := range createTableMap { keys[i] = k i++ } return keys } // // whatTable for the data type // func whatTable(dataType string) (table string, err error) { switch dataType { case "dfp", "BPA", "BPP", "DRE", "DFC_MD", "DFC_MI", "DVA": table = "dfp" case "itr", "BPA_ITR", "BPP_ITR", "DRE_ITR", "DFC_MD_ITR", "DFC_MI_ITR", "DVA_ITR": table = "itr" case "fre", "FRE": table = "fre" case "codes", "CODES": table = "codes" case "md5", "MD5": table = "md5" case "status", "STATUS": table = "status" case "companies", "COMPANIES": table = "companies" case "fii_details": table = dataType case "fii_dividends": table = dataType case "stock_codes": table = dataType case "stock_quotes": table = dataType default: return "", errors.Wrapf(err, "tipo de informação inexistente: %s", dataType) } return } // // createTable creates the table if not created yet // func createTable(db *sql.DB, dataType string) (err error) { table, err := whatTable(dataType) if err != nil { return err } _, err = db.Exec(createTableMap[table]) if err != nil { return errors.Wrapf(err, "erro ao criar tabela '%s'", table) } err = createIndexes(db, table) if err != nil { return errors.Wrap(err, "erro ao criar índice para table "+table) } if strings.ToUpper(dataType) == "STATUS" { return nil } version := currentDbVersion switch dataType { case "fii_details": version = currentFIIDbVersion case "fii_dividends": version = currentFIIDbVersion case "stock_codes": version = currentStockCodesVersion case "stock_quotes": version = currentStockQuotesVersion } _, err = db.Exec(fmt.Sprintf(`INSERT OR REPLACE INTO status (table_name, version) VALUES ("%s",%d)`, table, version)) if err != nil { return errors.Wrap(err, "erro ao atualizar tabela "+table) } return nil } func createAllTables(db *sql.DB) (err error) { if err := createTable(db, "status"); err != nil { return err } for _, t := range allTables() { if t == "status" { continue } if err := createTable(db, t); err != nil { return err } } return nil } // // dbVersion returns the version stored in DB // func dbVersion(db *sql.DB, dataType string) (v int, table string) { table, err := whatTable(dataType) if err != nil { return } sqlStmt := `SELECT version FROM status WHERE table_name = ?` err = db.QueryRow(sqlStmt, table).Scan(&v) if err != nil { return } return } // // wipeDB drops the table! Use with care // func wipeDB(db *sql.DB, dataType string) (err error) { table, err := whatTable(dataType) if err != nil { return } _, err = db.Exec("DROP TABLE IF EXISTS " + table) if err != nil { return errors.Wrap(err, "erro ao apagar tabela") } return } // // createIndexes create indexes based on table name // func createIndexes(db *sql.DB, table string) error { indexes := []string{} switch table { case "dfp": indexes = []string{ "CREATE INDEX IF NOT EXISTS dfp_metrics ON dfp (CODE, ID_CIA, YEAR, VL_CONTA);", "CREATE INDEX IF NOT EXISTS dfp_year_ver ON dfp (ID_CIA, YEAR, VERSAO);", } case "itr": indexes = []string{ "CREATE INDEX IF NOT EXISTS itr_metrics ON itr (CODE, ID_CIA, YEAR, VL_CONTA);", "CREATE INDEX IF NOT EXISTS itr_quarter_ver ON itr (ID_CIA, DT_FIM_EXERC, VERSAO);", } case "stock_quotes": indexes = []string{ "CREATE UNIQUE INDEX IF NOT EXISTS stock_quotes_stockdate ON stock_quotes (stock, date);", } case "fii_dividends": indexes = []string{ "CREATE UNIQUE INDEX IF NOT EXISTS fii_dividends_pk ON fii_dividends (trading_code, base_date);", } } for _, idx := range indexes { _, err := db.Exec(idx) if err != nil { return errors.Wrap(err, "erro ao criar índice") } } return nil } // // hasTable checks if the table exists // func hasTable(db *sql.DB, tableName string) bool { sqlStmt := `SELECT name FROM sqlite_master WHERE type='table' AND name=?;` var n string err := db.QueryRow(sqlStmt, tableName).Scan(&n) if err != nil { return false } return n == tableName } ================================================ FILE: parsers/transform.go ================================================ package parsers import ( "hash/fnv" "unicode" "golang.org/x/text/runes" "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" ) // fnvHash is a global var set to speed up Hash var fnvHash = fnv.New32a() // // Hash returns the FNV-1 non-cryptographic hash // func Hash(s string) uint32 { fnvHash.Write([]byte(s)) defer fnvHash.Reset() return fnvHash.Sum32() } // // RemoveDiacritics transforms, for example, "žůžo" into "zuzo" // func RemoveDiacritics(original string) (result string) { t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC) result, _, _ = transform.String(t, original) return } ================================================ FILE: progress/cmd/main.go ================================================ package main import ( "errors" "time" "github.com/dude333/rapina/progress" ) func main() { progress.Cursor(false) defer progress.Cursor(true) progress.Status("a status msg") progress.Running("start process") progress.Error(errors.New("some error")) time.Sleep(time.Second) progress.RunOK() progress.Running("start another process") time.Sleep(time.Second) progress.Status("middle") time.Sleep(time.Second) progress.RunFail() f1() progress.Running("start spinner") for i := 0; i < 100; i++ { time.Sleep(10 * time.Millisecond) progress.Spinner() if i == 20 { progress.Status("spinner interrupt") } } progress.RunOK() progress.Status("end.") } func f1() { progress.Running("Running *f1*") time.Sleep(time.Second) progress.Warning("f1 warning") time.Sleep(time.Second) progress.RunOK() } ================================================ FILE: progress/progress.go ================================================ // progress prints the program progress on screen. It's similar to a logger, but with // better formatting. package progress import ( "bytes" "fmt" "io" "os" ) // event stores logs IDs and messages. type event struct { id int format string } // Log messages as new Events var ( // evStatus = event{1, "[>] %v"} evError = event{1, "[✗] %v"} evRunning = event{2, "[ ] %v"} evRunOk = event{3, "\r[✓]"} evRunFail = event{4, "\r[✗]"} ) const spinners = `/-\|` const ( colorReset = "\033[0m" colorRed = "\033[31m" // colorGreen = "\033[32m" colorYellow = "\033[33m" colorBlue = "\033[34m" // colorPurple = "\033[35m" colorCyan = "\033[36m" // colorWhite = "\033[37m" ) type Progress struct { out io.Writer // destination for output, usually os.Stderr running []byte seq int // sequence of spinners debug bool } var p *Progress func init() { p = &Progress{out: os.Stderr, debug: false} } func SetDebug(on bool) { p.debug = on } func Cursor(show bool) { if p.out != os.Stdout && p.out != os.Stderr { return } if show { output([]byte("\033[?25h")) // Show cursor } else { output([]byte("\033[?25l")) // Hide cursor } } func Status(format string, a ...interface{}) { if len(p.running) > 0 { clearLine() output([]byte(colorCyan)) } outputln("[>] " + fmt.Sprintf(format, a...)) if len(p.running) > 0 { output([]byte(colorReset)) output(p.running) } } func Error(err error) { if len(p.running) > 0 { clearLine() } output([]byte(colorRed)) outputln(fmt.Sprintf(evError.format, err)) output([]byte(colorReset)) if len(p.running) > 0 { output(p.running) } } func ErrorMsg(format string, a ...interface{}) { if len(p.running) > 0 { clearLine() } output([]byte(colorRed)) outputln("[✗] " + fmt.Sprintf(format, a...)) output([]byte(colorReset)) if len(p.running) > 0 { output(p.running) } } func Warning(format string, a ...interface{}) { if len(p.running) > 0 { clearLine() } output([]byte(colorYellow)) outputln("[!] " + fmt.Sprintf(format, a...)) output([]byte(colorReset)) if len(p.running) > 0 { output(p.running) } } func Debug(format string, a ...interface{}) { if !p.debug { return } if len(p.running) > 0 { clearLine() } output([]byte(colorBlue)) outputln("--- " + fmt.Sprintf(format, a...)) output([]byte(colorReset)) if len(p.running) > 0 { output(p.running) } } func Running(msg string) { p.running = []byte(fmt.Sprintf(evRunning.format, msg)) output(p.running) } func Spinner() { output([]byte{'\r', '[', spinners[p.seq], ']'}) p.seq = (p.seq + 1) % len(spinners) } func RunOK() { outputln(evRunOk.format) p.running = p.running[:0] } func RunFail() { output([]byte(colorRed)) if len(p.running) > 0 { clearLine() output(p.running) } outputln(evRunFail.format) p.running = p.running[:0] output([]byte(colorReset)) } func Download(a string) { output([]byte("[ ] " + a)) } /* ------ static --------- func Status(format string, a ...interface{}) { _progress.Status(format, a...) } func Warning(format string, a ...interface{}) { _progress.Warning(format, a...) } func Error(err error) { _progress.Error(err) } func ErrorMsg(format string, a ...interface{}) { _progress.ErrorMsg(format, a...) } func Download(a string) { _progress.Download(a) } /* ------- output ------- */ func clearLine() { if len(p.running) == 0 { return } buf := bytes.Repeat([]byte(" "), len(p.running)+2) buf[0] = byte('\r') buf[len(buf)-1] = byte('\r') _, _ = p.out.Write(buf) } func output(buf []byte) { _, _ = p.out.Write(buf) } func outputln(s string) { if p.out == nil { return } var buf []byte buf = append(buf, s...) if len(s) == 0 || s[len(s)-1] != '\n' { buf = append(buf, '\n') } output(buf) } ================================================ FILE: reports/db.go ================================================ package reports import ( "database/sql" "fmt" "strconv" "strings" "github.com/dude333/rapina" "github.com/dude333/rapina/parsers" "github.com/pkg/errors" ) type accItems struct { code uint32 cdConta string dsConta string } type AccountValue struct { accItem accItems value float32 year int } // // accountsItems returns all accounts codes and descriptions, e.g.: // [1 Ativo Total, 1.01 Ativo Circulante, ...] // func (r Report) accountsItems(cid int) (items []accItems, err error) { selectItems := fmt.Sprintf(` SELECT DISTINCT CODE, CD_CONTA, DS_CONTA FROM dfp a WHERE ID_CIA = "%d" AND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR) ORDER BY CD_CONTA, DS_CONTA ;`, cid) rows, err := r.db.Query(selectItems) if err != nil { panic(err) } defer rows.Close() var item accItems for rows.Next() { err = rows.Scan(&item.code, &item.cdConta, &item.dsConta) if err != nil { return } items = append(items, item) } return } // // accountsValues stores the values for each account into a map using a hash // of the account code and description as its key // func (r Report) accountsValues(year int) (map[uint32]float32, error) { values := make(map[uint32]float32) lastYear, isITR, err := r.lastYear(r.cid) if err != nil { return nil, err } if year == lastYear && isITR { err = r.ttm(r.cid, values) } else { err = r.dfp(r.cid, year, values) } if err != nil { return values, err } // Stop if year has empty values if sum(values) == 0 { return values, nil } // Financial scale table := "dfp" if isITR { table = "itr" } values[parsers.Escala] = r.scale(r.cid, year, table) // Shares and free float _ = r.shares(r.cid, year, values) var v float32 // Inventory average v, err = r.value(r.cid, year-1, parsers.Estoque) if err == nil { values[parsers.EstoqueMedio] = avg(values[parsers.Estoque], v) } // Equity average v, err = r.value(r.cid, year-1, parsers.Equity) if err == nil { values[parsers.EquityAvg] = avg(values[parsers.Equity], v) } // Stock code if r.code != "" { date := rapina.LastBusinessDayOfYear(year) q, err := r.fetchStock.Quote(r.code, date) if err == nil { values[parsers.Quote] = float32(q) } } return values, nil } // avg returns the average, ignoring numbers <= 0. func avg(nums ...float32) float32 { var total float32 = 0 var n float32 = 0 for _, num := range nums { if num > 0 { total += num n++ } } if n <= 0 { return 0 } return total / n } // // lastYear considers the current year as the latest year recorded on the DB. // Returns this lastest year, if it's to use the ITR table (instead of the DFP), // and the error, if any. // func (r Report) lastYear(cid int) (int, bool, error) { if cid == 0 { return 0, false, fmt.Errorf("customer ID not set") } numErr := 0 selectDfpLastYear := `SELECT MAX(CAST(YEAR AS INTEGER)) YEAR FROM dfp WHERE ID_CIA = ?;` dfp := 0 err := r.db.QueryRow(selectDfpLastYear, cid).Scan(&dfp) if err != nil { numErr++ } selectItrLastYear := `SELECT MAX(CAST(YEAR AS INTEGER)) YEAR FROM itr WHERE ID_CIA = ?;` itr := 0 err = r.db.QueryRow(selectItrLastYear, cid).Scan(&itr) if err != nil { numErr++ } if numErr == 2 { return 0, false, sql.ErrNoRows } if itr > dfp { return itr, true, nil // Use ITR } return dfp, false, nil // Use DFP } // // LastYearRange returns the 1st and last day from last year stored on the DB // for this company id. Return dates in unix epoch format. // func (r Report) LastYearRange(cid int) (int, int, error) { if cid == 0 { return 0, 0, fmt.Errorf("customer ID not set") } s := ` SELECT DISTINCT DT_FIM_EXERC FROM dfp WHERE ID_CIA = ? ORDER BY DT_FIM_EXERC DESC LIMIT 2; ` rows, err := r.db.Query(s, cid) if err != nil { return 0, 0, err } defer rows.Close() var dateRange [2]int // [0] = last day, [1] = first day i := 0 for rows.Next() { _ = rows.Scan(&dateRange[i]) i++ if i >= len(dateRange) { break } } for _, v := range dateRange { if v == 0 { return 0, 0, fmt.Errorf("range not found") } } // return the first and last day of the year return dateRange[1], dateRange[0], nil } func (r Report) dfp(cid, year int, _values map[uint32]float32) error { selectReport := ` SELECT CODE, VL_CONTA FROM dfp a WHERE ID_CIA = $1 AND YEAR = $2 AND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR) ;` rows, err := r.db.Query(selectReport, cid, year) if err != nil { return err } defer rows.Close() for rows.Next() { var code uint32 var vlConta float32 err := rows.Scan(&code, &vlConta) if err == nil { _values[code] = vlConta } } return nil } func (r Report) RawAccounts(cid, year int) ([]AccountValue, error) { selectReport := ` SELECT CD_CONTA, DS_CONTA, CODE, YEAR, VL_CONTA FROM dfp a WHERE ID_CIA = $1 AND YEAR = $2 AND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR) ;` values := make([]AccountValue, 0, 10) rows, err := r.db.Query(selectReport, cid, year) if err != nil { return nil, err } defer rows.Close() for rows.Next() { var codeConta string var descConta string var code uint32 var vlConta float32 var year int err := rows.Scan(&codeConta, &descConta, &code, &year, &vlConta) av := AccountValue{ accItem: accItems{ code: code, cdConta: codeConta, dsConta: descConta, }, year: year, value: vlConta, } if err == nil { values = append(values, av) } } return values, nil } func (r Report) lastDate(cid int) (int, string, error) { rowDfp := r.db.QueryRow("SELECT MAX(DT_FIM_EXERC) FROM dfp WHERE ID_CIA = ? LIMIT 1;", cid) maxDfp := 0 err := rowDfp.Scan(&maxDfp) if err != nil { return 0, "", err } rowItr := r.db.QueryRow("SELECT MAX(DT_FIM_EXERC) FROM itr WHERE ID_CIA = ? LIMIT 1;", cid) maxItr := 0 err = rowItr.Scan(&maxItr) if err != nil { return 0, "", err } if maxDfp >= maxItr { return maxDfp, "dfp", nil } return maxItr, "itr", nil } // // lastBalance returns a hash with the '[code] = value' from the balance sheet // with the newest date available on the dfp or itr tables. // func (r Report) lastBalance(cid int) (map[uint32]float32, error) { d, table, err := r.lastDate(cid) if err != nil { return nil, err } if table != "dfp" && table != "itr" { return nil, fmt.Errorf("table %s is not allowed", table) } selectBalance := fmt.Sprintf(` SELECT date(DT_FIM_EXERC, 'unixepoch') DT, CODE, SUM(VL_CONTA) TOTAL FROM %s t WHERE ID_CIA = $1 AND DT_FIM_EXERC = $2 AND VERSAO = (SELECT MAX(VERSAO) FROM %s WHERE ID_CIA = t.ID_CIA AND DT_FIM_EXERC = t.DT_FIM_EXERC) AND CAST(substr(CD_CONTA, 1, 1) as decimal) <= 2 GROUP BY DT_FIM_EXERC, CODE, CD_CONTA; `, table, table) rows, err := r.db.Query(selectBalance, cid, d) if err != nil { return nil, err } defer rows.Close() balance := make(map[uint32]float32) var maxDt string var code uint32 var vlConta float32 for rows.Next() { err := rows.Scan(&maxDt, &code, &vlConta) if err == nil { balance[code] = vlConta } } return balance, nil } // // ttm (twelve trailling months) retrieves the 4 quarters from // last year, the quarters from the current year and sums up the last 4 // quarters for every account, returning a map with '[account_code] = value'. // func (r Report) ttm(cid int, _values map[uint32]float32) error { lastYear, err := r.lastDFPYear(cid) if err != nil { return err } selectQuarters := `SELECT CODE, sum(VAL) FROM ( -- Last quarter from last year SELECT CODE, sum(VAL) VAL FROM ( SELECT CODE, VL_CONTA VAL -- Year total FROM dfp d WHERE ID_CIA = $1 AND YEAR = $2 AND VERSAO = (SELECT max(VERSAO) FROM dfp WHERE ID_CIA = d.ID_CIA AND YEAR = d.YEAR) AND CAST(substr(CD_CONTA, 1, 1) as decimal) > 2 -- IGNORE BALANCE SHEETS UNION SELECT CODE, -1*VL_CONTA VAL -- Minus 3 semesters FROM itr i WHERE ID_CIA = $1 AND YEAR <= $2 AND CAST(substr(CD_CONTA, 1, 1) as decimal) > 2 -- IGNORE BALANCE SHEETS AND VERSAO = (SELECT MAX(VERSAO) FROM itr WHERE ID_CIA = i.ID_CIA AND CODE = i.CODE AND DT_FIM_EXERC = i.DT_FIM_EXERC) AND ID IN (SELECT ID FROM itr WHERE ID_CIA = i.ID_CIA AND CODE = i.CODE AND DT_FIM_EXERC = i.DT_FIM_EXERC ORDER BY DT_FIM_EXERC desc LIMIT 3) ) GROUP BY CODE UNION -- Last 3 quarters SELECT CODE, sum(VL_CONTA) VAL FROM itr i WHERE ID_CIA = $1 AND CAST(substr(CD_CONTA, 1, 1) as decimal) > 2 -- IGNORE BALANCE SHEETS AND VERSAO = (SELECT MAX(VERSAO) FROM itr WHERE ID_CIA = i.ID_CIA AND CODE = i.CODE AND DT_FIM_EXERC = i.DT_FIM_EXERC) AND ID IN (SELECT ID FROM itr WHERE ID_CIA = i.ID_CIA AND CODE = i.CODE ORDER BY DT_FIM_EXERC desc LIMIT 3) GROUP BY CODE ) GROUP BY CODE ORDER BY CODE;` rows, err := r.db.Query(selectQuarters, cid, lastYear) if err != nil { return err } defer rows.Close() var code uint32 var vlConta float32 for rows.Next() { err := rows.Scan(&code, &vlConta) if err == nil { _values[code] = vlConta } } bal, err := r.lastBalance(cid) if err != nil { return err } for k, v := range bal { _values[k] = v } return nil } func (r Report) lastDFPYear(cid int) (int, error) { if cid == 0 { return 0, fmt.Errorf("customer ID not set") } s := `SELECT MAX(YEAR) FROM dfp WHERE ID_CIA = ?;` row := r.db.QueryRow(s, cid) var lastDate int err := row.Scan(&lastDate) return lastDate, err } // // shares set the 'values' map with the number of shares and the free float of // a given conpany in a given year. // func (r Report) shares(cid int, year int, values map[uint32]float32) error { selectFRE := ` SELECT Quantidade_Total_Acoes_Circulacao, Percentual_Total_Acoes_Circulacao FROM fre f WHERE ID_CIA = $1 AND YEAR = $2 AND Versao = (SELECT MAX(Versao) FROM fre WHERE ID_CIA = f.ID_CIA AND YEAR = f.YEAR); ` row := r.db.QueryRow(selectFRE, cid, year) var shares float32 var freeFloat float32 err := row.Scan(&shares, &freeFloat) if err != nil && err != sql.ErrNoRows { return err } values[parsers.Shares] = shares values[parsers.FreeFloat] = freeFloat return nil } // // sharesAvg set the 'values' map with the average number of shares and // the free float of a given conpany in a given year. // func (r Report) sharesAvg(cids []string, year int, values map[uint32]float32) error { selectFRE := fmt.Sprintf(` SELECT AVG(Quantidade_Total_Acoes_Circulacao), AVG(Percentual_Total_Acoes_Circulacao) FROM fre f WHERE ID_CIA IN (%s) AND YEAR = $1 AND Versao = (SELECT MAX(Versao) FROM fre WHERE ID_CIA = f.ID_CIA AND YEAR = f.YEAR); `, strings.Join(cids, ",")) row := r.db.QueryRow(selectFRE, year) var sharesAvg float32 var freeFloatAvg float32 err := row.Scan(&sharesAvg, &freeFloatAvg) if err != nil && err != sql.ErrNoRows { return err } values[parsers.Shares] = sharesAvg values[parsers.FreeFloat] = freeFloatAvg return nil } // // value returns the account value for company id 'cid', 'year' and code. // func (r Report) value(cid, year int, code uint32) (float32, error) { selectInventory := ` SELECT VL_CONTA FROM dfp a WHERE ID_CIA = $1 AND YEAR = $2 AND CODE = $3 AND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR) ;` val := float32(0) err := r.db.QueryRow(selectInventory, cid, year, code).Scan(&val) if err != nil && err != sql.ErrNoRows { return 0, err } return val, nil } // // accountsAverage stores the average of all companies of the same sector // for each account into a map using a hash of the account code and // description as its key // func (r Report) accountsAverage(company string, year int) (map[uint32]float32, error) { companies, _, err := r.fromSector(company) if len(companies) <= 1 || err != nil { err = errors.Wrap(err, "erro ao ler arquivo de setores "+r.yamlFile) return nil, err } if len(companies) == 0 { err = errors.Errorf("erro ao procurar empresas") return nil, err } cids := make([]string, len(companies)) for i, co := range companies { if id, err := r.getCid(co); err == nil { cids[i] = strconv.Itoa(id) } } selectReport := fmt.Sprintf(` SELECT CODE, AVG(VL_CONTA) FROM dfp a WHERE ID_CIA IN (%s) AND YEAR = "%d" AND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR) GROUP BY CODE; `, strings.Join(cids, ","), year) rows, err := r.db.Query(selectReport) if err != nil { return nil, err } defer rows.Close() values := make(map[uint32]float32) for rows.Next() { var code uint32 var vlConta float32 err := rows.Scan( &code, &vlConta, ) if err == nil { values[code] = vlConta } } var err1, err2 error values[parsers.EstoqueMedio], err1 = r.movingAvg(cids, year, parsers.Estoque) values[parsers.EquityAvg], err2 = r.movingAvg(cids, year, parsers.Equity) if err1 == nil && err2 == nil { _ = r.sharesAvg(cids, year, values) } return values, nil } // movingAvg returns the moving average of account 'code' between year and // last year for all companies listed on 'cids'. func (r Report) movingAvg(cids []string, year int, code uint32) (float32, error) { s := fmt.Sprintf(` SELECT (SELECT AVG(VL_CONTA) FROM dfp d2 WHERE d1.ID_CIA = d2.ID_CIA AND d2.CODE = d1.CODE AND d2.YEAR >= (d1.YEAR - 1) AND d2.YEAR <= d1.YEAR AND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = d2.ID_CIA AND YEAR = d2.YEAR) ) AS MAVG FROM dfp d1 WHERE ID_CIA IN (%s) AND YEAR = $1 AND CODE = $2 AND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = d1.ID_CIA AND YEAR = d1.YEAR) GROUP BY YEAR; `, strings.Join(cids, ",")) mavg := float32(0) err := r.db.QueryRow(s, year, code).Scan(&mavg) if err != nil && err != sql.ErrNoRows { return 0, err } return mavg, nil } func (r Report) fromSector(company string) (companies []string, sectorName string, err error) { // Companies from the same sector secCo, secName, err := parsers.FromSector(company, r.yamlFile) if len(secCo) <= 1 || err != nil { err = errors.Wrap(err, "erro ao ler arquivo dos setores "+r.yamlFile) return } // All companies stored on db list, err := ListCompanies(r.db) if err != nil { err = errors.Wrap(err, "erro ao listar empresas") return } // Translate company names to match the name stored on db for _, s := range secCo { z := parsers.FuzzyFind(s, list, 3) if len(z) > 0 { companies = append(companies, z) } } return removeDuplicates(companies), secName, nil } // CompanyInfo contains the company name and CNPJ type CompanyInfo struct { id int name string } // // companies returns available companies in the DB // func companies(db *sql.DB) ([]CompanyInfo, error) { selectCompanies := ` SELECT ID, NAME FROM companies ORDER BY NAME;` rows, err := db.Query(selectCompanies) if err != nil { err = errors.Wrap(err, "falha ao ler banco de dados") return nil, err } defer rows.Close() var info CompanyInfo var list []CompanyInfo for rows.Next() { err := rows.Scan(&info.id, &info.name) if err == nil { list = append(list, info) } } return list, nil } // TickerInfo contains the ticker name and SpcfctnCd type TickerInfo struct { name string SpcfctnCd string } // // tickers returns available tickers for a company name in the DB // func tickers(db *sql.DB, companyName string) ([]TickerInfo, error) { selectTickers := ` SELECT trading_code, SpcfctnCd FROM stock_codes WHERE company_name LIKE ? ORDER BY trading_code;` rows, err := db.Query(selectTickers, "%"+companyName+"%") if err != nil { err = errors.Wrap(err, "falha ao ler banco de dados") return nil, err } defer rows.Close() var info TickerInfo var list []TickerInfo for rows.Next() { err := rows.Scan(&info.name, &info.SpcfctnCd) if err == nil { list = append(list, info) } } return list, nil } // // setCompany sets the company ID, CNPJ and stock code based on it's name... // // func (r *Report) setCompany(company string) error { // return r.setCompanyAndTicker(company,"ON") // } // // setCompanyAndTicker sets the company ID, CNPJ and stock code based on it's name. // func (r *Report) setCompanyAndTicker(company string, spcfctnCd string) error { if company == "" { return errors.New("company name not set") } if r.fetchStock == nil { return errors.New("fetchStock not set") } // Reset company data r.cid = 0 r.cnpj = "" r.code = "" query := `SELECT DISTINCT ID, NAME, CNPJ FROM companies WHERE NAME LIKE ?` var cid int var name, cnpj string err := r.db.QueryRow(query, "%"+company+"%").Scan(&cid, &name, &cnpj) if err != nil { return err } r.cid = cid r.company = name // reset company name to match the name stored on db r.cnpj = cnpj // Stock code r.code, err = r.fetchStock.Code(r.company, spcfctnCd) if err != nil { fmt.Printf("\n[x] Erro obtendo código negociação: %v\n", err) } return nil } func (r *Report) getCid(companyName string) (int, error) { selectID := `SELECT DISTINCT ID FROM companies WHERE NAME LIKE ?` var cid int err := r.db.QueryRow(selectID, "%"+companyName+"%").Scan(&cid) return cid, err } // // scale returns the financial scale used on the values (unit or thousands). // func (r Report) scale(cid, year int, table string) float32 { s := fmt.Sprintf( `SELECT ESCALA_MOEDA FROM %s WHERE ID_CIA = $1 AND YEAR = $2 limit 1;`, table, ) var scale string err := r.db.QueryRow(s, cid, year).Scan(&scale) if err != nil { return 1000 } switch scale { case "UNIDADE": return 1 case "MIL": return 1000 case "MILHAO": return 1000000 } return 1000 } // // timeRange returns the begin=min(year) and end=max(year) // func timeRange(db *sql.DB) (int, int, error) { selectYears := ` SELECT MIN(CAST(YEAR AS INTEGER)), MAX(CAST(YEAR AS INTEGER)) FROM dfp;` begin := 0 end := 0 err := db.QueryRow(selectYears).Scan(&begin, &end) if err != nil { return 0, 0, err } selectItrYears := ` SELECT MAX(CAST(YEAR AS INTEGER)) FROM itr;` end2 := 0 err = db.QueryRow(selectItrYears).Scan(&end2) if err == nil && end2 > end { end = end2 } // Check year if begin < 1900 || begin > 2100 || end < 1900 || end > 2100 { err = errors.Wrap(err, "ano inválido") return 0, 0, err } if begin > end { aux := end end = begin begin = aux } return begin, end, nil } func removeDuplicates(elements []string) []string { // change string to int here if required // Use map to record duplicates as we find them. encountered := map[string]bool{} // change string to int here if required result := []string{} // change string to int here if required for v := range elements { if encountered[elements[v]] { // Do not add duplicate. } else { // Record this element as an encountered element. encountered[elements[v]] = true // Append to result slice. result = append(result, elements[v]) } } // Return the new slice. return result } type profit struct { year int profit float32 } func companyProfits(db *sql.DB, companyID int) ([]profit, error) { selectProfits := fmt.Sprintf(` SELECT YEAR, VL_CONTA FROM dfp a WHERE ID_CIA = "%d" AND CODE = "%d" AND VERSAO = (SELECT MAX(VERSAO) FROM dfp WHERE ID_CIA = a.ID_CIA AND YEAR = a.YEAR) ORDER BY YEAR;`, companyID, parsers.LucLiq) rows, err := db.Query(selectProfits) if err != nil { err = errors.Wrap(err, "falha ao ler banco de dados") return nil, err } defer rows.Close() var profits []profit for rows.Next() { var year int var val float32 err := rows.Scan(&year, &val) if err == nil { profits = append(profits, profit{year, val}) } } return profits, nil } ================================================ FILE: reports/db_test.go ================================================ package reports import "testing" func Test_avg(t *testing.T) { type args struct { nums []float32 } tests := []struct { name string args args want float32 }{ {"average 2", args{[]float32{1, 2, 3}}, 2}, {"average 10", args{[]float32{10, 2, 18}}, 10}, {"average 12.705", args{[]float32{6, 20.4, 18.1, 6.32}}, 12.705}, {"average 5.5", args{[]float32{5.5, 0}}, 5.5}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := avg(tt.args.nums...); got != tt.want { t.Errorf("avg() = %v, want %v", got, tt.want) } }) } } ================================================ FILE: reports/excel.go ================================================ package reports import ( "encoding/json" "strconv" "github.com/360EntSecGroup-Skylar/excelize" "github.com/pkg/errors" ) // Excel instance reachable data type Excel struct { xlsx *excelize.File } // // newExcel creates a new Excel instance // func newExcel() (e *Excel) { e = &Excel{} e.xlsx = excelize.NewFile() return } // // saveAndCloseExcel saves to filename (need to set the directory as well) // func (e *Excel) saveAndCloseExcel(filename string) (err error) { // newFilename = time.Now().Format("02Jan06_150405.000") + ".xlsx" // DDMMMYY e.xlsx.DeleteSheet("Sheet1") e.xlsx.SetActiveSheet(1) err = e.xlsx.SaveAs(filename) if err != nil { return errors.Wrapf(err, "erro ao salvar planilha") } return } // Sheet struct type Sheet struct { xlsx *excelize.File name string } func (e *Excel) newSheet(name string) (s *Sheet, err error) { s = &Sheet{} s.name = name s.xlsx = e.xlsx // Create a new sheet. // Avoid duplicated sheet if index := e.xlsx.GetSheetIndex(name); index > 0 { return nil, errors.Wrapf(err, "erro ao criar planilha %s", name) } e.xlsx.NewSheet(name) return } func (s Sheet) printCell(row, col int, value interface{}, styleID int) { // Print value cell := axis(col, row) s.xlsx.SetCellValue(s.name, cell, value) // Format cell s.xlsx.SetCellStyle(s.name, cell, cell, styleID) } // // printTitle prints the cols titles in Excel // func (s *Sheet) printTitle(cell string, title string) (err error) { // Print header s.xlsx.SetSheetRow(s.name, cell, &[]string{title}) // Set styles style, err := s.xlsx.NewStyle(`{"number_format": 0,"font":{"bold":true},"alignment":{"horizontal":"center"},"border":[{"type":"bottom","color":"333333","style":3}]}`) if err == nil { s.xlsx.SetCellStyle(s.name, cell, cell, style) } return } // // print cols in Excel // func (s *Sheet) print(startingCel string, slice *[]string, format int, bold bool) error { var err error var style int // Set styles json, err := jsonStyle(10, format, bold) if err != nil { return err } style, err = s.xlsx.NewStyle(string(json)) if style > 0 && err == nil { col, row := cell2axis(startingCel) col += len(*slice) s.xlsx.SetCellStyle(s.name, startingCel, axis(col, row), style) } // Print row s.xlsx.SetSheetRow(s.name, startingCel, slice) return nil } // // printValues prints cols in Excel // Values >0 and <= 100 will be printed as % // func (s *Sheet) printValue(cell string, value float32, format int, bold bool) (err error) { s.xlsx.SetSheetRow(s.name, cell, &[]float32{value}) // Set styles json, err := jsonStyle(10, format, bold) if err == nil { style, err := s.xlsx.NewStyle(string(json)) if err == nil { s.xlsx.SetCellStyle(s.name, cell, cell, style) } } return nil } // // printFormula // func (s *Sheet) printFormula(cell string, formula string, format int, bold bool) (err error) { s.xlsx.SetCellFormula(s.name, cell, formula) // Set styles json, err := jsonStyle(9, format, bold) if err == nil { style, err := s.xlsx.NewStyle(string(json)) if err == nil { s.xlsx.SetCellStyle(s.name, cell, cell, style) } } return } // // jsonStyle // func jsonStyle(size, format int, bold bool) ([]byte, error) { m := map[string]interface{}{ "font": map[string]interface{}{"size": size, "bold": bold}, } switch format { case PERCENT: m["custom_number_format"] = "0%;-0%;- " case INDEX: m["custom_number_format"] = "0.00;-0.00;-" case NUMBER: m["custom_number_format"] = "_-* #,##0,_-;_-* (#,##0,);_-* \"-\"_-;_-@_-" case RIGHT: m["alignment"] = map[string]interface{}{"horizontal": "right"} } j, err := json.Marshal(m) return j, err } // // mergeCell // func (s *Sheet) mergeCell(a, b string) { s.xlsx.MergeCell(s.name, a, b) } // // autoWidth adjust the cols width // func (s *Sheet) autoWidth() { const cols string = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" setColWidth := s.xlsx.SetColWidth setColWidth(s.name, "A", "A", 16) setColWidth(s.name, "B", "B", 48) // Get the space that separates the account numbers from the // vertical analysis numbers var spaced int for col := 2; col < len(cols); col++ { if len(s.xlsx.GetCellValue(s.name, axis(col, 1))) == 0 { spaced = col break } } // 012345678901234567890 // AB2DE5GHI => 5-2+5 = 8 // AB2DEF6HIJK => 6-2+6 = 10 // AB2DEFG7IJKLM => 7-2+7 = 12 setColWidth(s.name, "C", string(cols[spaced-1]), 9.5) // Account values setColWidth(s.name, string(cols[spaced]), "AC", 4.64) // Vertical Analysis values } func (s *Sheet) setColWidth(col int, width float64) { c := excelize.ToAlphaString(col) s.xlsx.SetColWidth(s.name, c, c, width) } // // axis transforms (2, 3) into "B3" // func axis(col, row int) string { return excelize.ToAlphaString(col) + strconv.Itoa(row) } // // cell2axis only works from A1 to Z999 // func cell2axis(cell string) (col, row int) { col = int(cell[0] - 'A') row, _ = strconv.Atoi(cell[1:]) return } // // colLetter transforms '2' into 'B' // func colLetter(col int) string { return excelize.ToAlphaString(col) } ================================================ FILE: reports/format.go ================================================ package reports import ( "encoding/json" "github.com/360EntSecGroup-Skylar/excelize" "github.com/dude333/rapina/parsers" ) // Global var to keep track of added styles var stylesMap = make(map[uint32]int) // Used by style const ( DEFAULT = iota + 1 // Number format GENERAL NUMBER INDEX PERCENT EMPTY // Text position LEFT RIGHT CENTER ) // formatFont directly maps the styles settings of the fonts. type formatFont struct { Bold bool `json:"bold"` Italic bool `json:"italic"` Underline string `json:"underline"` Family string `json:"family"` Size int `json:"size"` Color string `json:"color"` } type formatAlignment struct { Horizontal string `json:"horizontal"` Indent int `json:"indent"` JustifyLastLine bool `json:"justify_last_line"` ReadingOrder uint64 `json:"reading_order"` RelativeIndent int `json:"relative_indent"` ShrinkToFit bool `json:"shrink_to_fit"` TextRotation int `json:"text_rotation"` Vertical string `json:"vertical"` WrapText bool `json:"wrap_text"` } type formatBorder struct { Type string `json:"type"` Color string `json:"color"` Style int `json:"style"` } type formatFill struct { Type string `json:"type"` Pattern int `json:"pattern"` Color []string `json:"color"` Shading int `json:"shading"` } // formatStyle directly maps the styles settings of the cells. type formatStyle struct { Border []formatBorder `json:"border"` Fill formatFill `json:"fill"` Font *formatFont `json:"font"` Alignment *formatAlignment `json:"alignment"` Protection *struct { Hidden bool `json:"hidden"` Locked bool `json:"locked"` } `json:"protection"` NumFmt int `json:"number_format"` DecimalPlaces int `json:"decimal_places"` CustomNumFmt *string `json:"custom_number_format"` Lang string `json:"lang"` NegRed bool `json:"negred"` } // // newFormat provides a struct to create style for cells // func newFormat(format int, position int, bold bool) (f *formatStyle) { f = &formatStyle{} custom := "" switch format { case PERCENT: custom = "0%;-0%;- " case INDEX: custom = "0.00;-0.00;-" case NUMBER: custom = "_-* #,##0,_-;_-* (#,##0,);_-* \"-\"_-;_-@_-" } if custom != "" { f.CustomNumFmt = &custom } switch position { case RIGHT: f.Alignment = &formatAlignment{Horizontal: "right"} case CENTER: f.Alignment = &formatAlignment{Horizontal: "center"} } if bold { f.Font = &formatFont{Bold: true} } return } func (f *formatStyle) size(s int) { f.Font = &formatFont{Size: s} } func (f formatStyle) newStyle(e *excelize.File) (style int) { j, err := json.Marshal(f) if err == nil { s := string(j) k := parsers.Hash(s) // Check if style already exists id, ok := stylesMap[k] if ok { // fmt.Printf("[i] Reusing style %d [%d]\n", id, k) return id } // Create new style style, err = e.NewStyle(s) stylesMap[k] = style if err != nil { return 0 } } // fmt.Printf("[i] New style %d\n", style) return } ================================================ FILE: reports/format_test.go ================================================ package reports import ( "testing" "github.com/360EntSecGroup-Skylar/excelize" ) func TestFormat(t *testing.T) { var f [2]*formatStyle var style [2]int xlsx := excelize.NewFile() f[0] = newFormat(NUMBER, LEFT, false) f[1] = newFormat(INDEX, RIGHT, false) f[0].NumFmt = 10 f[1].Lang = "en-US" for i := range f { style[i] = f[i].newStyle(xlsx) if style[i] == 0 { t.Error("Expecting style > 0, received 0") } } var f2 [2]*formatStyle var style2 [2]int f2[0] = newFormat(NUMBER, LEFT, false) f2[1] = newFormat(INDEX, RIGHT, false) f2[0].NumFmt = 10 f2[1].Lang = "en-US" for i := range f { style2[i] = f[i].newStyle(xlsx) if style2[i] != style[i] { t.Errorf("Expecting style == %d, received %d", style[i], style2[i]) } } } ================================================ FILE: reports/list.go ================================================ package reports import ( "database/sql" "fmt" "math" "strings" "github.com/dude333/rapina/parsers" "github.com/pkg/errors" "golang.org/x/text/collate" "golang.org/x/text/language" "golang.org/x/text/message" ) // // ListCompanies shows all available companies // func ListCompanies(db *sql.DB) (names []string, err error) { info, err := companies(db) if err != nil { fmt.Println("[x] Falha:", err) return } if len(info) == 0 { err = fmt.Errorf("lista vazia") return } // Extract companies names names = make([]string, len(info)) for i, co := range info { names[i] = co.name } // Sort accents correctly cl := collate.New(language.BrazilianPortuguese, collate.Loose) cl.SortStrings(names) return } // // ListTickers shows all available tickers for a companie name // func ListTickers(db *sql.DB, companyName string) (names []string, err error) { info, err := tickers(db, companyName) if err != nil { fmt.Println("[x] Falha:", err) return } if len(info) == 0 { err = fmt.Errorf("lista vazia") return } // Extract companies names names = make([]string, len(info)) for i, co := range info { names[i] = co.name } // Sort accents correctly cl := collate.New(language.BrazilianPortuguese, collate.Loose) cl.SortStrings(names) return } // // ListTickers returns SpcfctnCd of a ticker // func GetSpcfctnCd(db *sql.DB, companyName string, ticker string) string { info, err := tickers(db, companyName) if err != nil { fmt.Println("[x] Falha:", err) return "" } if len(info) == 0 { fmt.Println("[x] Lista vazia") return "" } // Extract companies names var spcfctnCd string = "" for _, co := range info { if co.name == ticker { spcfctnCd = co.SpcfctnCd } } return spcfctnCd } // // ListSector shows all companies from the same sector as 'company' // func ListSector(db *sql.DB, company, yamlFile string) (err error) { // Companies from the same sector secCo, secName, err := parsers.FromSector(company, yamlFile) if len(secCo) <= 1 || err != nil { err = errors.Wrap(err, "erro ao ler arquivo dos setores "+yamlFile) return } // All companies stored on db list, err := ListCompanies(db) if err != nil { err = errors.Wrap(err, "erro ao listar empresas") return } // Translate company names to match the name stored on db fmt.Printf("%-40s %s\n", "ARQUIVO YAML", "BANCO DE DADOS") fmt.Printf("%-40s %s\n", strings.Repeat("-", 40), strings.Repeat("-", 40)) for _, s := range secCo { z := parsers.FuzzyFind(s, list, 3) if len(z) > 0 { fmt.Printf("%-40s %s\n", s, z) } else { fmt.Printf("%-40s %s\n", s, "Nao encontrado") } } fmt.Printf("\nSETOR: %s\n", secName) return } // // ListCompaniesProfits lists companies by net profit: more sustainable growth // listed first // func ListCompaniesProfits(db *sql.DB, rate float32) error { info, err := companies(db) if err != nil { return fmt.Errorf("falha ao obter a lista de empresas (%v)", err) } yi, yf, err := timeRange(db) if err != nil { return fmt.Errorf("falha ao obter a faixa de datas (%v)", err) } // Header var sep string fmt.Printf("%20s ", " ") // Space to match company name for y := yi; y <= yf; y++ { fmt.Printf("%10d ", y) sep += fmt.Sprintf("%s ", strings.Repeat("-", 10)) } fmt.Printf("%10s\n", "CAGR") sep += fmt.Sprintf("%s ", strings.Repeat("-", 10)) fmt.Printf("%20s %s\n", " ", sep) // Profits pt := message.NewPrinter(language.Portuguese) for _, co := range info { profits, err := companyProfits(db, co.id) if err != nil { return fmt.Errorf("falha ao obter lucros de %s (%v)", co.name, err) } // FILTERS ---------------------------- // At least 4 years if len(profits) < 4 { continue } // Ignore if there is no recent data if profits[len(profits)-1].year < yf-1 { continue } pi := profits[0].profit pf := profits[len(profits)-1].profit if pf < pi { continue } // Ignore if next profix < 'rate' * current profit profitable := true for i := 1; i < len(profits); i++ { if profits[i].profit < 0 || profits[i].profit < (1+rate)*profits[i-1].profit { profitable = false break } } if !profitable { continue } // COMPANY NAME ----------------------- fmt.Printf("%-20.20s ", co.name) // PROFIT VALUES ---------------------- i := 0 for y := yi; y <= yf; y++ { if i < len(profits) && profits[i].year == y { pt.Printf("%10.0f ", profits[i].profit) i++ } else { fmt.Printf("%10s ", " ") } } // CAGR ------------------------------- if pi != 0 && pf != 0 && pf*pi >= 0 { cagr := math.Pow(float64(pf/pi), 1/float64(yf-yi-1)) - 1 pt.Printf("%10.1f%%", cagr*100) } fmt.Println() } // next c fmt.Printf("\nEmpresas com lucros crescentes e variação mínima de %0.0f%% de um ano para o outro.\n\n", rate*100) return nil } ================================================ FILE: reports/logger.go ================================================ package reports import ( "fmt" "io" "os" ) type Logger struct { out io.Writer // destination for output buf []byte // for accumulating text to write } // New creates a new Logger func NewLogger(out io.Writer) *Logger { return &Logger{out: out} } func (l *Logger) SetOut(out io.Writer) { l.out = out } // Run prints a message before running a process. func (l *Logger) Run(format string, v ...interface{}) { s := fmt.Sprintf(format, v...) if len(s) > 0 && s[len(s)-1] == '\n' { s = s[:len(s)-1] } l.output("[ ] " + s) } // Ok prints a checkmark after a successful Run() func (l *Logger) Ok() { l.outputln("\r[✓]") } // Nok prints a x mark after a unsuccessful Run() func (l *Logger) Nok() { l.outputln("\r[✗]") } // Printf prints the plain text. func (l *Logger) Printf(format string, v ...interface{}) { l.output(fmt.Sprintf(format, v...)) } // Trace for very low level logs. func (l *Logger) Trace(format string, v ...interface{}) { l.outputln("[TRACE] " + fmt.Sprintf(format, v...)) } // Debug for debugging information. func (l *Logger) Debug(format string, v ...interface{}) { l.outputln("[DEBUG] " + fmt.Sprintf(format, v...)) } // Info for something noteworthy. func (l *Logger) Info(format string, v ...interface{}) { l.outputln("[INFO] " + fmt.Sprintf(format, v...)) } // Warn for a warning message. func (l *Logger) Warn(format string, v ...interface{}) { l.outputln("[WARN] " + fmt.Sprintf(format, v...)) } // Error message. Always print to Stderr. func (l *Logger) Error(format string, v ...interface{}) { hold := l.out l.out = os.Stderr l.outputln("[ERRO] " + fmt.Sprintf(format, v...)) l.out = hold } func (l *Logger) output(s string) { if l.out == nil { return } l.buf = l.buf[:0] l.buf = append(l.buf, s...) _, _ = l.out.Write(l.buf) } func (l *Logger) outputln(s string) { if l.out == nil { return } l.buf = l.buf[:0] l.buf = append(l.buf, s...) if len(s) == 0 || s[len(s)-1] != '\n' { l.buf = append(l.buf, '\n') } _, _ = l.out.Write(l.buf) } ================================================ FILE: reports/logger_test.go ================================================ package reports import ( "bytes" "testing" "github.com/stretchr/testify/assert" ) func TestLogger(t *testing.T) { var buf bytes.Buffer log := NewLogger(&buf) tests := []struct { name string fn func(format string, v ...interface{}) msg string want string }{ { name: "printf", fn: log.Printf, msg: "this is a normal message", want: "this is a normal message", }, { name: "trace", fn: log.Trace, msg: "this is a trace log\n", want: "[TRACE] this is a trace log\n", }, { name: "debug", fn: log.Debug, msg: "this is a debug log", want: "[DEBUG] this is a debug log\n", }, { name: "info", fn: log.Info, msg: "you have been informed", want: "[INFO] you have been informed\n", }, { name: "warn", fn: log.Warn, msg: "this is a warning", want: "[WARN] this is a warning\n", }, // { // name: "error", // fn: log.Error, // msg: "this is an error message", // want: "[ERRO] this is an error message\n", // }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.fn(tt.msg) assert.Equal(t, tt.want, buf.String()) buf.Reset() }) } } func TestLogger_Run(t *testing.T) { var buf bytes.Buffer log := NewLogger(&buf) log.Run("starting process %d\n", 10) log.Ok() assert.Equal(t, "[ ] starting process 10\r[✓]\n", buf.String()) buf.Reset() log.Run("starting process %d", 15) log.Nok() assert.Equal(t, "[ ] starting process 15\r[✗]\n", buf.String()) } ================================================ FILE: reports/reports.go ================================================ package reports import ( "database/sql" "fmt" "os" "os/signal" "path" "sort" "strconv" "strings" "github.com/360EntSecGroup-Skylar/excelize" "github.com/dude333/rapina/fetch" p "github.com/dude333/rapina/parsers" "github.com/pkg/errors" "golang.org/x/text/language" "golang.org/x/text/message" ) const sectorAverage = "MÉDIA DO SETOR" const ( grpAccts int = iota + 100 grpShares grpExtra grpFleuriet ) // metric parameters type metric struct { descr string val float32 format int // mapped by constants NUMBER, INDEX, PERCENT group int // mapped by constants grpX } // Report parameters used in most functions type Report struct { // average metric values/year. Index 0: year, index 1: metric average [][]float32 // groups that will be printed on the output xlsx // - ExtraRatios: enables some extra financial ratios on report // - ShowShares: shows the number of shares and free float on report // - Sector: creates a sheet with the sector report groups map[int]bool // if true will print the reports for comanpanies in the same sector printSector bool // get the stock quotes fetchStock *fetch.Stock /* Current company */ cid int // Company ID cnpj string // Company CNPJ code string // Company stock code /* Parameters from caller */ db *sql.DB // Sqlite3 handler company string // company name to be processed spcfctnCd string // spcfctnCd used to select the correct ticker format string // report format filename string // path and filename of the output xlsx yamlFile string // file with the companies' sectors } func New(parms map[string]interface{}) (*Report, error) { var r Report if v, ok := parms["db"]; ok { r.db = v.(*sql.DB) } if v, ok := parms["company"]; ok { r.company = v.(string) } if v, ok := parms["SpcfctnCd"]; ok { r.spcfctnCd = v.(string) } if v, ok := parms["format"]; ok { r.format = v.(string) } if v, ok := parms["filename"]; ok { r.filename = v.(string) } if v, ok := parms["yamlFile"]; ok { r.yamlFile = v.(string) } if v, ok := parms["reports"]; ok { p := v.(map[string]bool) r.groups = make(map[int]bool, 4) r.groups[grpAccts] = true r.groups[grpShares] = p["ShowShares"] r.groups[grpExtra] = p["ExtraRatios"] r.groups[grpFleuriet] = p["Fleuriet"] r.printSector = true if v, ok := p["PrintSector"]; ok { r.printSector = v } } dataDir := path.Join(".", "data") if v, ok := parms["dataDir"]; ok { dataDir = v.(string) } apiKey := "" if v, ok := parms["apiKey"]; ok { apiKey = v.(string) } var err error log := NewLogger(os.Stderr) r.fetchStock, err = fetch.NewStock(r.db, log, apiKey, dataDir) return &r, err } // // ReportToXlsx reports company financial data from DB to Excel. // func ReportToXlsx(parms map[string]interface{}) error { // Initialize report object r, err := New(parms) if err != nil { return err } err = r.setCompanyAndTicker(r.company,r.spcfctnCd) if err != nil { return fmt.Errorf("empresa '%s' não encontrada no banco de dados", r.company) } e := newExcel() sheet, _ := e.newSheet(r.company) // Company name sheet.mergeCell("A1", "B1") sheet.print("A1", &[]string{r.company}, LEFT, true) // ACCOUNT NUMBERING AND DESCRIPTION (COLS A AND B) ===============\/ accounts, _ := r.accountsItems(r.cid) baseItems, lastStatementsRow, lastMetricsRow := r.printCodesAndDescriptions(sheet, accounts, 'A', 2) // VALUES (COLS C, D, E...) / PER YEAR ===========================\/ begin, end, err := timeRange(r.db) if err != nil { return err } var values map[uint32]float32 // LOOP THROUGH YEARS =============================================\/ for y := begin; y <= end; y++ { // Title row := 2 col := colLetter(2 + y - begin) // start on col 'C' cell := col + "1" title := "[" + strconv.Itoa(y) + "]" lastYear, isTTM, err := r.lastYear(r.cid) if lastYear == y && isTTM && err == nil { title = "[TTM/" + strconv.Itoa(y) + "]" } // ACCOUNT VALUES (COLS C, D, E...) / YEAR ====================\/ values, err = r.accountsValues(y) if err != nil { fmt.Println("[x]", err) continue } // Skip last year if empty if y == end && sum(values) == 0 { end-- break } _ = sheet.printTitle(cell, title) // Print year as title on row 1 for _, acct := range accounts { cell := col + strconv.Itoa(row) _ = sheet.printValue(cell, values[acct.code], NUMBER, baseItems[row]) row++ } // FINANCIAL METRICS (COLS C, D, E...) / YEAR =================\/ row++ cell = col + strconv.Itoa(row) _ = sheet.printTitle(cell, title) // Print year as title row++ // Print report in the sequence defined on metricsList() for _, metric := range metricsList(values) { if !r.groups[metric.group] { continue } if metric.format != EMPTY { cell := col + strconv.Itoa(row) _ = sheet.printValue(cell, metric.val, metric.format, false) } row++ } } // next year // // VERTICAL ANALYSIS // // CODES | DESCRIPTION | Y1 | Y2 | Yn | sp | v1 | v2 | v3 // wide := (end - begin) year := begin top := 2 bottom := top for col := 2; col <= 2+wide; col++ { vCol := col + wide + 2 // Column where the vertical analysis will be printed _ = sheet.printTitle(axis(vCol, 1), "'"+strconv.Itoa(year)) // Print year year++ var ref string for row := top; row <= lastStatementsRow; row++ { idx := row - top if idx < 0 || idx >= len(accounts) { break } if len(accounts[idx].cdConta) == 0 { break } n, _ := strconv.Atoi(accounts[idx].cdConta[:1]) if n > 3 { break } switch accounts[idx].cdConta { case "1", "2", "3.01": ref = axis(col, row) } val := axis(col, row) formula := fmt.Sprintf(`=IfError(%s/%s, "-")`, val, ref) _ = sheet.printFormula(axis(vCol, row), formula, PERCENT, baseItems[row]) bottom = row } } // Print VERTICAL ANALYSIS title sheet.mergeCell(axis(1+wide+2, top), axis(1+wide+2, bottom)) format := newFormat(DEFAULT, RIGHT, true) format.Alignment.Vertical = "top" format.Alignment.TextRotation = 90 rotatedTextStyle := format.newStyle(sheet.xlsx) sheet.printCell(top, 1+wide+2, "ANÁLISE VERTICAL", rotatedTextStyle) // // HORIZONTAL ANALYSIS // // sp | DESCRIPTION | Y1 | Y2 | Yn | sp | h1 | h2 | hn // wide = (end - begin) year = begin top = lastStatementsRow + 2 bottom = lastMetricsRow for col := 0; col <= wide-1; col++ { year++ vCol := (2 + wide + 2) + col // Column where the horizontal analysis will be printed _ = sheet.printTitle(axis(vCol, top), "'"+strconv.Itoa(year)) // Print year for row := top + 1; row <= bottom; row++ { vt0 := axis(col+2, row) vtn := axis(col+3, row) formula := fmt.Sprintf(`=IF(OR(%s="", %s=""), "", IF(MIN(%s, %s)<=0, IF((%s - %s)>0, " ⇧", " ⇩"), (%s/%s)-1))`, vtn, vt0, vtn, vt0, vtn, vt0, vtn, vt0) _ = sheet.printFormula(axis(vCol, row), formula, PERCENT, false) } } // Print HORIZONTAL ANALYSIS title sheet.mergeCell(axis(2+wide+1, top+1), axis(2+wide+1, bottom)) sheet.printCell(top+1, 1+wide+2, "ANÁLISE HORIZONTAL", rotatedTextStyle) // CAGR (compound annual growth rate) // CAGR (t0, tn) = (V(tn)/V(t0))^(1/(tn-t0-1))-1 vCol := (2 + wide + 2) + wide + 1 _ = sheet.printTitle(axis(vCol, top), "CAGR") for row := top + 1; row <= bottom; row++ { vt0 := axis(2, row) vtn := axis(2+wide, row) formula := fmt.Sprintf(`=IF(OR(%s="", %s="", %s=0, (%s*%s)<0), "", (%s/%s)^(1/%d)-1)`, vtn, vt0, vt0, vt0, vtn, vtn, vt0, wide) _ = sheet.printFormula(axis(vCol, row), formula, PERCENT, false) } // ADJUST COLUMNS WIDTH sheet.autoWidth() // SECTOR REPORT if r.printSector { sheet2, err := e.newSheet("SETOR") if err == nil { _ = sheet2.xlsx.SetSheetViewOptions(sheet2.name, 0, excelize.ShowGridLines(false), excelize.ZoomScale(80), ) _ = r.sectorReport(sheet2, r.company) } } err = e.saveAndCloseExcel(r.filename) if err == nil { fmt.Printf("[√] Dados salvos em %s\n", r.filename) } return err } //ReportToStdout reports company financial data from DB to Stdout. func ReportToStdout(parms map[string]interface{}) error { r, err := New(parms) if err != nil { return err } err = r.setCompanyAndTicker(r.company,r.spcfctnCd) if err != nil { return fmt.Errorf("empresa '%s' não encontrada no banco de dados", r.company) } begin, end, err := timeRange(r.db) if err != nil { return err } acc := []AccountValue{} for y := begin; y <= end; y++ { d, err := r.RawAccounts(r.cid, y) if err != nil { return err } acc = append(acc, d...) } accBuf, err := buildStdAccountReport(acc) if err != nil { return err } fmt.Print(accBuf) return err } func buildStdAccountReport(data []AccountValue) (*strings.Builder, error) { buf := &strings.Builder{} p := message.NewPrinter(language.BrazilianPortuguese) sort.Slice(data, func(i, j int) bool { return data[i].year < data[j].year }) for _, acc := range data { fmt.Fprintf(buf, "%d;", acc.year) if _, err := p.Fprintf(buf, "%s;%s;", acc.accItem.cdConta, acc.accItem.dsConta); err != nil { return nil, err } fmt.Fprintf(buf, "%d", int(acc.value)) buf.WriteByte('\n') } return buf, nil } // // sectorReport gets all the companies related to the 'company' and reports // their financial summary // func (r Report) sectorReport(sheet *Sheet, company string) (err error) { var interrupt bool // Handle Ctrl+C c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt) go func() { <-c fmt.Println("\n[ ] Processamento interrompido") interrupt = true }() // Companies from the same sector companies, secName, err := r.fromSector(company) if len(companies) <= 1 || err != nil { err = errors.Wrap(err, "erro ao ler arquivo de setores "+r.yamlFile) return } companies = append([]string{sectorAverage}, companies...) fmt.Println("[i] Criando relatório setorial (Ctrl+C para interromper)") var top, row, col int = 2, 0, 0 var count int for _, co := range companies { row = top col++ fmt.Printf("[ ] - %s", co) avg := false if co == sectorAverage { avg = true co = company } empty, err := r.companySummary(sheet, &row, &col, co, secName, count%3 == 0, avg) ok := "√" if err != nil || empty { ok = "x" col-- } else { count++ if count%3 == 0 { top = row + 2 col = 0 } } if interrupt { return nil } fmt.Printf("\r[%s\n", ok) } sheet.setColWidth(0, 2) return } // // companySummary reports all companies from the same segment into the // 'Setor' sheet. // func (r *Report) companySummary(sheet *Sheet, row, col *int, _company, sectorName string, printDescr, sectorAvg bool) (empty bool, err error) { // if !sectorAvg && !r.isCompany(company) { // return true, nil // } err = r.setCompanyAndTicker(r.company,r.spcfctnCd) if err != nil { err = errors.Errorf("empresa '%s' não encontrada no banco de dados", _company) return } begin, end, err := timeRange(r.db) if err != nil { return } // Formats used in this report sTitle := newFormat(DEFAULT, RIGHT, true).newStyle(sheet.xlsx) fCompanyName := newFormat(DEFAULT, CENTER, true) fCompanyName.size(16) sCompanyName := fCompanyName.newStyle(sheet.xlsx) fSectorName := newFormat(DEFAULT, LEFT, false) fSectorName.size(14) sSectorName := fSectorName.newStyle(sheet.xlsx) // fDescr := newFormat(DEFAULT, RIGHT, false) fDescr.Border = []formatBorder{{Type: "left", Color: "333333", Style: 1}} sDescr := fDescr.newStyle(sheet.xlsx) fDescr.Border = []formatBorder{ {Type: "top", Color: "333333", Style: 1}, {Type: "left", Color: "333333", Style: 1}, } sDescrTop := fDescr.newStyle(sheet.xlsx) fDescr.Border = []formatBorder{ {Type: "top", Color: "333333", Style: 1}, } sDescrBottom := fDescr.newStyle(sheet.xlsx) // Company name if printDescr { *col++ } sheet.mergeCell(axis(*col, *row), axis(*col+end-begin+1, *row)) if sectorAvg { sheet.printCell(*row-1, *col-1, sectorName, sSectorName) sheet.printCell(*row, *col, sectorAverage, sCompanyName) } else { sheet.printCell(*row, *col, _company, sCompanyName) } if printDescr { *col-- } *row++ // Save starting row rw := *row // Set width for the description col if printDescr { sheet.setColWidth(*col, 18) *col++ } // Print values ONE YEAR PER COLUMN for y := begin; y <= end; y++ { var values map[uint32]float32 var err error if sectorAvg { values, err = r.accountsAverage(_company, y) r.average = append(r.average, []float32{}) } else { values, err = r.accountsValues(y) } if err != nil { fmt.Printf(" -- %v", err) return false, err } // Skip last year if empty if y == end && sum(values) == 0 { end-- break } *row = rw // Print year sheet.printCell(*row, *col, "["+strconv.Itoa(y)+"]", sTitle) *row++ // Print financial metrics i := 0 for _, metric := range metricsList(values) { if !r.groups[metric.group] { continue } if sectorAvg { r.average[y-begin] = append(r.average[y-begin], metric.val) } // Description if printDescr { stl := sDescr if i == 0 { stl = sDescrTop } sheet.printCell(*row, *col-1, metric.descr, stl) } // Values if metric.format != EMPTY { fVal := newFormat(metric.format, DEFAULT, false) fVal.Border = []formatBorder{ {Type: "top", Color: "cccccc", Style: 1}, {Type: "right", Color: "cccccc", Style: 1}, {Type: "bottom", Color: "cccccc", Style: 1}, {Type: "left", Color: "cccccc", Style: 1}, } // Color the cell background according to its value compared with the average if len(r.average) > 0 && len(r.average[y-begin]) > 0 && len(r.average[y-begin]) >= i { f := formatFill{Type: "pattern", Pattern: 1} if metric.val > r.average[y-begin][i] { f.Color = []string{"c6efce"} // green fVal.Fill = f } else if metric.val < r.average[y-begin][i] { f.Color = []string{"ffc7ce"} // red fVal.Fill = f } } stl := fVal.newStyle(sheet.xlsx) sheet.printCell(*row, *col, metric.val, stl) } *row++ i++ } if printDescr { sheet.printCell(*row, *col-1, "", sDescrBottom) } printDescr = false *col++ } // next year bottom := *row // CAGR (compound annual growth rate) // CAGR (t0, tn) = (V(tn)/V(t0))^(1/(tn-t0-1))-1 wide := end - begin _ = sheet.printTitle(axis(*col, rw), "CAGR") for r := rw + 1; r <= bottom; r++ { vt0 := axis(*col-wide-1, r) vtn := axis(*col-1, r) formula := fmt.Sprintf(`=IF(OR(%s="", %s="", %s=0, (%s*%s)<0), "", (%s/%s)^(1/%d)-1)`, vtn, vt0, vt0, vt0, vtn, vtn, vt0, wide) _ = sheet.printFormula(axis(*col, r), formula, PERCENT, false) } *col++ return } func (r *Report) Summary(company string) (map[string]string, error) { m := make(map[string]string) err := r.setCompanyAndTicker(r.company,r.spcfctnCd) if err != nil { return m, err } values, err := r.accountsValues(2020) if err != nil { return m, err } for _, metric := range metricsList(values) { if !r.groups[metric.group] { continue } } return m, nil } // // metricsList returns the sequence to be printed after the financial statements // func metricsList(v map[uint32]float32) (metrics []metric) { dividaBruta := v[p.DividaCirc] + v[p.DividaNCirc] caixa := v[p.Caixa] + v[p.AplicFinanceiras] dividaLiquida := dividaBruta - caixa EBITDA := v[p.EBIT] - v[p.Deprec] proventos := v[p.Dividendos] + v[p.JurosCapProp] var roe float32 if v[p.LucLiq] > 0 && v[p.EquityAvg] > 0 { roe = zeroIfNeg(safeDiv(v[p.LucLiq], v[p.EquityAvg])) } var cg float32 = v[p.AtivoCirc] - v[p.PassivoCirc] var st float32 = v[p.Caixa] + v[p.AplicFinanceiras] - (v[p.DividaCirc] + v[p.DividendosJCP] + v[p.DividendosMin]) var ncg float32 = cg - st var lpa float32 = safeDiv(v[p.LucLiq]*v[p.Escala], v[p.Shares]) return []metric{ {"Patrimônio Líquido", v[p.Equity], NUMBER, grpAccts}, {"", 0, EMPTY, grpAccts}, {"Receita Líquida", v[p.Vendas], NUMBER, grpAccts}, {"EBITDA", EBITDA, NUMBER, grpAccts}, {"EBIT", v[p.EBIT], NUMBER, grpAccts}, {"Resultado Financeiro", v[p.ResulFinanc], NUMBER, grpAccts}, {"Operações Descontinuadas", v[p.ResulOpDescont], NUMBER, grpAccts}, {"Lucro Líquido", v[p.LucLiq], NUMBER, grpAccts}, {"", 0, EMPTY, grpAccts}, {"LPA", lpa, INDEX, grpAccts}, {"VPA", safeDiv(v[p.Equity]*v[p.Escala], v[p.Shares]), INDEX, grpAccts}, {"P/L", safeDiv(v[p.Quote], lpa), INDEX, grpAccts}, {"Cotação", v[p.Quote], INDEX, grpAccts}, {"", 0, EMPTY, grpAccts}, {"Marg. EBITDA", zeroIfNeg(safeDiv(EBITDA, v[p.Vendas])), PERCENT, grpAccts}, {"Marg. EBIT", zeroIfNeg(safeDiv(v[p.EBIT], v[p.Vendas])), PERCENT, grpAccts}, {"Marg. Líq.", zeroIfNeg(safeDiv(v[p.LucLiq], v[p.Vendas])), PERCENT, grpAccts}, {"ROE", roe, PERCENT, grpAccts}, {"", 0, EMPTY, grpAccts}, {"Caixa", caixa, NUMBER, grpAccts}, {"Dívida Bruta", dividaBruta, NUMBER, grpAccts}, {"Dívida Líq.", dividaLiquida, NUMBER, grpAccts}, {"Dív. Bru./PL", zeroIfNeg(safeDiv(dividaBruta, v[p.Equity])), PERCENT, grpAccts}, {"Dív.Líq./EBITDA", zeroIfNeg(safeDiv(dividaLiquida, EBITDA)), INDEX, grpAccts}, {"", 0, EMPTY, grpAccts}, {"FCO", v[p.FCO], NUMBER, grpAccts}, {"FCI", v[p.FCI], NUMBER, grpAccts}, {"FCF", v[p.FCF], NUMBER, grpAccts}, {"FCT", v[p.FCO] + v[p.FCI] + v[p.FCF], NUMBER, grpAccts}, {"FCL (FCO+FCI)", v[p.FCO] + v[p.FCI], NUMBER, grpAccts}, {"", 0, EMPTY, grpAccts}, {"Proventos", proventos, NUMBER, grpAccts}, {"Payout", zeroIfNeg(safeDiv(proventos, v[p.LucLiq])), PERCENT, grpAccts}, {"", 0, EMPTY, grpAccts}, {"Total de Ações", v[p.Shares], GENERAL, grpShares}, {"Free Float", v[p.FreeFloat], PERCENT, grpShares}, {"", 0, EMPTY, grpShares}, {"Liquidez Corrente (Ativo Circ./Passivo Circ.)", safeDiv(v[p.AtivoCirc], v[p.PassivoCirc]), INDEX, grpExtra}, {"Liquidez Seco [(Ativo Circ.-Estoque)/Passivo Circ.]", safeDiv(v[p.AtivoCirc]-v[p.Estoque], v[p.PassivoCirc]), INDEX, grpExtra}, {"Giro dos Ativos (Vendas/Ativo)", safeDiv(v[p.Vendas], v[p.AtivoTotal]), INDEX, grpExtra}, {"", 0, EMPTY, grpExtra}, {"Giro de Estoque (dias)", safeDiv(v[p.EstoqueMedio], -v[p.CustoVendas]/360), INDEX, grpExtra}, {"Prazo Médio de Recebimento (dias)", safeDiv(v[p.ContasARecebCirc]+v[p.ContasARecebNCirc], v[p.Vendas]/360), INDEX, grpExtra}, {"", 0, EMPTY, grpExtra}, {"Poder de Ganho Básico (EBITDA/Ativo)", safeDiv(EBITDA, v[p.AtivoTotal]), PERCENT, grpExtra}, {"ROA", safeDiv(v[p.LucLiq], v[p.AtivoTotal]), PERCENT, grpExtra}, {"ROE", roe, PERCENT, grpExtra}, {"", 0, EMPTY, grpExtra}, {"Capital de Giro (CG)", cg, NUMBER, grpFleuriet}, {"Saldo de Tesouraria (ST)", st, NUMBER, grpFleuriet}, {"Necessidade de Capital de Giro (NCG=CG-ST)", ncg, NUMBER, grpFleuriet}, } } func zeroIfNeg(n float32) float32 { if n < 0 { return 0 } return n } func safeDiv(n, d float32) float32 { if d == 0 { return 0 } return n / d } // // ident returns the number of spaces according to the code level, e.g.: // "1.1 ABC" => " " (2 spaces) // "1.1.1 ABC" => " " (4 spaces) // For items equal or above 3, only returns spaces after 2nd level: // "3.01 ABC" => "" // "3.01.01 ABC" => " " // func ident(str string) (spaces string, baseItem bool) { num := strings.SplitN(str, ".", 2)[0] c := strings.Count(str, ".") if num != "1" && num != "2" && c > 0 { c-- } if c > 0 { spaces = strings.Repeat(" ", c) } if num == "1" || num == "2" { baseItem = c <= 1 } else { baseItem = c == 0 } return } // printCodesAndDescription prints 'accounts' codes and descriptions on // columns 'col' and 'col+1' (A <= col <= Z), starting on row 2. // Adjust space related to the group, e.g.: // 3.02 ABC <= print in bold if base item and stores the row position in baseItems[] // 3.02.01 ABC // // Returns: // - []bool indicates if a row is a base item, // - the row of the last statement, // - the row of the last metric item. func (r Report) printCodesAndDescriptions(sheet *Sheet, accounts []accItems, col rune, row int) ([]bool, int, int) { baseItems := make([]bool, len(accounts)+row) for _, it := range accounts { var sp string sp, baseItems[row] = ident(it.cdConta) cell := string(col) + strconv.Itoa(row) sheet.print(cell, &[]string{sp + it.cdConta, sp + it.dsConta}, LEFT, baseItems[row]) row++ } lastStatementsRow := row - 1 row += 2 col++ // Metrics descriptions for _, metric := range metricsList(nil) { if !r.groups[metric.group] { continue } if metric.descr != "" { cell := string(col) + strconv.Itoa(row) sheet.print(cell, &[]string{metric.descr}, RIGHT, false) } row++ } lastMetricsRow := row - 1 return baseItems, lastStatementsRow, lastMetricsRow } // sum returns a float32 with the sum of all values from a map func sum(values map[uint32]float32) float32 { var sum float32 for _, v := range values { sum += v } return sum } ================================================ FILE: reports/reports_fii.go ================================================ package reports import ( "database/sql" "fmt" "math" "strings" "sync" "github.com/dude333/rapina" "github.com/dude333/rapina/fetch" "github.com/dude333/rapina/progress" "golang.org/x/text/language" "golang.org/x/text/message" ) var line = strings.Repeat("-", 67) // Type of report output const ( Rtable = iota + 1 Rcsv Rcsvrend ) // FIITerminal implements reports related to FII funds on the terminal. type FIITerminal struct { fetchFII *fetch.FII fetchStock *fetch.Stock reportFormat int } type FIITerminalOptions struct { APIKey, DataDir string } // NewFIITerminal creates a new instace of a FIITerminal func NewFIITerminal(db *sql.DB, opts FIITerminalOptions) (*FIITerminal, error) { var log rapina.Logger fetchStock, err := fetch.NewStock(db, log, opts.APIKey, opts.DataDir) if err != nil { return nil, err } fetchFII, err := fetch.NewFII(db, log) if err != nil { return nil, err } return &FIITerminal{ fetchFII: fetchFII, fetchStock: fetchStock, reportFormat: Rtable, }, nil } // SetParms set the terminal reports parameters. func (t *FIITerminal) SetParms(parms map[string]string) { if _, ok := parms["verbose"]; ok { progress.SetDebug(true) } if r, ok := parms["format"]; ok { switch r { case "table", "tabela", "tab": t.reportFormat = Rtable case "csv": t.reportFormat = Rcsv case "csvrend": t.reportFormat = Rcsvrend } } } // Dividends prints the dividends report on terminal. func (t FIITerminal) Dividends(codes []string, n int) error { // Header if t.reportFormat == Rcsv { fmt.Println("Código,Data Com,Rendimento,Cotação,Yeld,Yeld a.a.") } if t.reportFormat == Rcsvrend { fmt.Print(`Código/Data-Com`) for _, date := range revMonthsFromToday(n) { fmt.Printf(",%s", date) } fmt.Println() } // Remove codes c := make([]string, 0, len(codes)) for _, code := range codes { if len(code) == 6 { c = append(c, code) } else { progress.ErrorMsg("Código inválido: %s. Padrão esperado: ABCD11.", code) } } codes = c dividends := sync.Map{} var wg sync.WaitGroup for i, code := range codes { wg.Add(1) i := i go func(code string, n int) { defer wg.Done() div, err := t.fetchFII.Dividends(code, n) if err != nil { progress.ErrorMsg("%s: %v", code, err) return } progress.Debug("[go routine %d] dividends (%d): %v", i, len(*div), div) dividends.Store(code, div) }(code, n) } wg.Wait() for _, code := range codes { div, ok := dividends.Load(code) if !ok { continue } dividendsForCode := div.(*[]rapina.Dividend) var buf *strings.Builder var err error switch t.reportFormat { case Rcsv: buf, err = t.csvDividends(code, dividendsForCode) case Rcsvrend: buf, err = t.csvDividendsOnly(code, n, dividendsForCode) default: buf, err = t.printDividends(code, dividendsForCode) } if err != nil { progress.Error(err) } else { fmt.Print(buf) } } // Footer // if t.reportFormat == Rtable { // fmt.Println(line) // } return nil } func (t FIITerminal) printDividends(code string, dividends *[]rapina.Dividend) (*strings.Builder, error) { buf := &strings.Builder{} p := message.NewPrinter(language.BrazilianPortuguese) p.Fprintln(buf, line) p.Fprintln(buf, code) p.Fprintln(buf, line) p.Fprintln(buf, " DATA COM RENDIMENTO COTAÇÃO YELD YELD a.a.") p.Fprintln(buf, " ---------- ---------- ---------- ------ ---------") for _, d := range *dividends { p.Fprintf(buf, " %s R$%8.2f ", d.Date, d.Val) q, err := t.fetchStock.Quote(code, d.Date) if err != nil { progress.ErrorMsg("Cotação de %s (%s): %v", code, d.Date, err) } if q > 0 && err == nil { i := d.Val / q p.Fprintf(buf, "R$%8.2f %8.2f%% %8.2f%%", q, 100*i, 100*(math.Pow(1+i, 12)-1)) } buf.WriteByte('\n') } buf.WriteByte('\n') return buf, nil } func (t FIITerminal) csvDividends(code string, dividends *[]rapina.Dividend) (*strings.Builder, error) { buf := &strings.Builder{} p := message.NewPrinter(language.BrazilianPortuguese) for _, d := range *dividends { p.Fprintf(buf, `%s,%s,"%f",`, code, d.Date, d.Val) q, err := t.fetchStock.Quote(code, d.Date) if err != nil { progress.ErrorMsg("Cotação de %s (%s): %v", code, d.Date, err) } if q > 0 && err == nil { i := d.Val / q p.Fprintf(buf, `"%f","%f%%","%f%%"`, q, 100*i, 100*(math.Pow(1+i, 12)-1)) } else { buf.WriteString(`"","",""`) } buf.WriteByte('\n') } return buf, nil } func (t FIITerminal) csvDividendsOnly(code string, n int, dividends *[]rapina.Dividend) (*strings.Builder, error) { buf := &strings.Builder{} p := message.NewPrinter(language.BrazilianPortuguese) buf.WriteString(code) for _, month := range revMonthsFromToday(n) { found := false for _, div := range *dividends { if div.Date[0:len("YYYY-MM")] == month { p.Fprintf(buf, `,"%f"`, div.Val) found = true break } } if !found { buf.WriteString(`,""`) } } buf.WriteByte('\n') return buf, nil } func revMonthsFromToday(n int) []string { rev := make([]string, 0, n) dates := rapina.MonthsFromToday(n) for i := len(dates) - 1; i >= 0; i-- { rev = append(rev, dates[i][0:len("YYYY-MM")]) } return rev } /* ------- MONTHLY REPORTS -------- */ func (t FIITerminal) Monthly(codes []string, n int) error { for _, c := range codes { ii, err := t.fetchFII.MonthlyReportIDs(c, n) progress.Status("indexes: %v (err: %v)", ii, err) } return nil } ================================================ FILE: reports/reports_html.go ================================================ package reports ================================================ FILE: reports/reports_test.go ================================================ package reports import ( "strings" "testing" p "github.com/dude333/rapina/parsers" ) // AssertEqual checks if values are equal func AssertEqual(t *testing.T, msg string, a interface{}, b interface{}) { if a == b { return } // debug.PrintStack() t.Errorf("%s was incorrect, received %v, expected %v.", msg, a, b) } func TestIdent(t *testing.T) { table := []struct { in string expected string isBaseItem bool }{ {"1", "", true}, {"1.1", " ", true}, {"1.1.2", " ", false}, {"2", "", true}, {"2.3.4.5", " ", false}, {"2.10", " ", true}, {"3", "", true}, {"3.1", "", true}, {"3.1.2", " ", false}, } for _, x := range table { spaces, baseItem := ident(x.in) if spaces != x.expected { t.Errorf("ident was incorrect for %s, got: '%s', want: '%s'.", x.in, spaces, x.expected) } if baseItem != x.isBaseItem { t.Errorf("ident was incorrect for %s, got: %v, want: %v.", x.in, baseItem, x.isBaseItem) } } } func TestZeroIfNeg(t *testing.T) { for x := float32(10); x >= -10; x -= 0.1 { y := zeroIfNeg(x) if (x >= 0 && x != y) || (x < 0 && y != 0) { t.Errorf("zeroIfNeg was incorrect, got: %f, want: %f", y, x) } } } func TestSafeDiv(t *testing.T) { for x := float32(10); x >= -10; x -= 0.1 { y := safeDiv(2, x) if (x == 0 && y != 0) || (y != 2/x) { t.Errorf("safeDiv was incorrect, got: %f, want: %f", y, x) } } } func TestMetricsList(t *testing.T) { v := make(map[uint32]float32) for x := uint32(p.Caixa); x <= uint32(p.Dividendos); x++ { v[x] = float32(x) * 123456 } l := metricsList(v) seq := []float32{ v[p.Equity], 0, v[p.Vendas], v[p.EBIT] - v[p.Deprec], // EBITDA v[p.EBIT], v[p.ResulFinanc], v[p.ResulOpDescont], v[p.LucLiq], 0, } for i, val := range seq { AssertEqual(t, "metricsList ["+l[i].descr+"]", l[i].val, val) } } func TestStdBuildReport(t *testing.T) { data := []AccountValue{ { accItem: accItems{ code: 123, cdConta: "cdConta Second", dsConta: "desc Second Conta", }, value: 456, year: 2012, }, { accItem: accItems{ code: 987, cdConta: "cdConta First", dsConta: "desc First Conta", }, value: 654, year: 2011, }, } builder, err := buildStdAccountReport(data) if err != nil { t.Error(err) } lines := strings.Split(strings.TrimSuffix(builder.String(), "\n"), "\n") strings.EqualFold("2011;cdConta First;desc First Conta;654", lines[0]) strings.EqualFold("2012;cdConta Second;desc Second Conta;456", lines[1]) } ================================================ FILE: server/fs_dev.go ================================================ // +build dev package server import ( "io/fs" "log" "os" ) var _fs = os.DirFS(".") var _contentFS fs.FS func init() { var err error _contentFS, err = fs.Sub(_fs, "templates") if err != nil { log.Fatal(err) } } ================================================ FILE: server/fs_prod.go ================================================ // +build !dev package server import ( "embed" "io/fs" "log" ) //go:embed templates var _fs embed.FS var _contentFS fs.FS func init() { var err error _contentFS, err = fs.Sub(_fs, "templates") if err != nil { log.Fatal(err) } } ================================================ FILE: server/payload.go ================================================ package server import ( "math" "net/url" "strings" "github.com/dude333/rapina/progress" ) // fiiDividendsPayload returns the data to be used in the FII template. func fiiDividendsPayload(srv *Server, fiiCodes []string, months int) interface{} { var payload struct { Codes string Months int Data interface{} } payload.Codes = strings.Join(fiiCodes, " ") payload.Months = months payload.Data = fiiDividends(srv, fiiCodes, months) return &payload } func fiiDividends(srv *Server, codes []string, n int) interface{} { type value struct { Date string Dividend float64 Quote float64 Yeld float64 YeldYear float64 } type data struct { Code string Name string Website string Values []value } var dataset []data // Fill 'data' for every stock code for _, code := range codes { code = strings.ToUpper(code) values := make([]value, 0, n) // Dividends from last "n" months div, err := srv.fetchFII.Dividends(code, n) if err != nil { progress.ErrorMsg("%s: %v", code, err) continue } // Stock quotes from the days when the dividends were received for _, d := range *div { q, err := srv.fetchStock.Quote(code, d.Date) if err != nil { progress.ErrorMsg("Cotação de %s (%s): %v", code, d.Date, err) continue } v := value{ Date: d.Date, Dividend: d.Val, Quote: q, } if q > 0 { i := d.Val / q v.Yeld = 100 * i v.YeldYear = 100 * (math.Pow(1+i, 12) - 1) } values = append(values, v) } // FII details, if found details, err := srv.fetchFII.Details(code) var name, a string if err == nil { name = details.DetailFund.CompanyName u, err := url.Parse(details.DetailFund.WebSite) if err == nil && u.Scheme == "" { u.Scheme = "https" a = u.String() } } d := data{ Code: code, Name: name, Website: a, Values: values, } dataset = append(dataset, d) } // next code return &dataset } // func financialsPayload(srv *Server, stockCode string) interface{} { // var payload struct { // Equity float32 // } // return &payload // } ================================================ FILE: server/server.go ================================================ package server import ( "database/sql" "errors" "html/template" "log" "net/http" "os" "path/filepath" "strconv" "strings" "github.com/dude333/rapina/fetch" "github.com/dude333/rapina/progress" "github.com/dude333/rapina/reports" "golang.org/x/text/language" "golang.org/x/text/message" ) type Server struct { db *sql.DB fetchFII *fetch.FII fetchStock *fetch.Stock report *reports.Report dataDir string apiKey string verbose bool } type ServerOption func(*Server) func WithDB(db *sql.DB) ServerOption { return func(s *Server) { s.db = db } } func WithAPIKey(apiKey string) ServerOption { return func(s *Server) { s.apiKey = apiKey } } func WithDataDir(dataDir string) ServerOption { return func(s *Server) { s.dataDir = dataDir } } func Verbose(on bool) ServerOption { return func(s *Server) { s.verbose = on } } func initServer(opts ...ServerOption) (*Server, error) { var srv Server for _, opt := range opts { opt(&srv) } if srv.db == nil { return nil, errors.New("BD inválido") } progress.SetDebug(srv.verbose) srv.db.SetMaxOpenConns(1) log := reports.NewLogger(os.Stderr) fetchStock, err := fetch.NewStock(srv.db, log, srv.apiKey, srv.dataDir) if err != nil { return nil, err } fetchFII, err := fetch.NewFII(srv.db, log) if err != nil { return nil, err } report, err := reports.New(map[string]interface{}{"db": srv.db}) if err != nil { return nil, err } srv.fetchFII = fetchFII srv.fetchStock = fetchStock srv.report = report return &srv, nil } // HTML is a very basic html server to handle the reports. func HTML(opts ...ServerOption) { srv, err := initServer(opts...) if err != nil { log.Fatal(err) } http.HandleFunc("/", renderTemplate(srv)) log.Println("Listening on :3000...") err = http.ListenAndServe(":3000", nil) if err != nil { log.Fatal(err) } } // renderTemplate renders the file related to the URL path inside the layout // templates. Template files are locates in _contentFS. func renderTemplate(srv *Server) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { fp := filepath.Clean(r.URL.Path) if strings.HasPrefix(fp, `/`) || strings.HasPrefix(fp, `\`) { fp = fp[1:] // remove starting "/" (or "\" on Windows) } if fp == "" { fp = "index.html" } log.Println("rendering", fp) // TODO: load all templates outside this funcion tmpl, err := template.New("").Funcs(template.FuncMap{ "ptFmtFloat": ptFmtFloat, }).ParseFS(_contentFS, "layout.html", fp) if err != nil { log.Println(err) return } // Set the payload according to the URL path var payload interface{} if strings.Contains(fp, "fii.html") && r.Method == http.MethodPost { codes := parseCodes(r.FormValue("codes")) months := parseNumeric(r.FormValue("months"), 1) payload = fiiDividendsPayload(srv, codes, months) } err = tmpl.ExecuteTemplate(w, "layout", payload) if err != nil { log.Println(err) } } } func ptFmtFloat(f float64) string { p := message.NewPrinter(language.BrazilianPortuguese) return p.Sprintf("%.2f", f) } func parseCodes(text string) []string { var codes []string for _, field := range strings.FieldsFunc(text, split) { field = strings.TrimSpace(field) if len(field) == len("ABCD11") { codes = append(codes, field) } } return codes } func split(r rune) bool { return r == ' ' || r == ',' || r == ';' || r == '\n' } // parseNumeric converts "numeric" to integer, or returns "alt" in case of error. func parseNumeric(numeric string, alt int) int { n, err := strconv.Atoi(numeric) if err != nil { n = alt } return n } ================================================ FILE: server/templates/fii.html ================================================ {{define "body"}}

Rendimentos dos FII

{{if not .Data}} {{else}} {{range .Data}}

⯆ {{.Code}}

{{range .Values}} {{end}}
{{.Name}} Yeld Yeld
Data Com Rendimento Cotação a.m. a.a. a.m. a.a.
{{.Date}} R$ {{ptFmtFloat .Dividend}} R$ {{ptFmtFloat .Quote}} {{ptFmtFloat .Yeld}}% {{ptFmtFloat .YeldYear}}%
{{end}} {{end}} {{end}} ================================================ FILE: server/templates/financials.html ================================================ {{define "body"}}

Finanças

{{end}} ================================================ FILE: server/templates/index.html ================================================ {{define "body"}}

Relatórios

Rendimentos dos FII
Finanças {{end}} ================================================ FILE: server/templates/layout.html ================================================ {{define "layout"}} Rapina
Rapina
FII:Rendimentos Ações:Finanças
{{template "body" .}}
{{end}} ================================================ FILE: stock.go ================================================ package rapina import "io" // StockStorage is the interface that contains the methods needed to parse, save and // retrieve stock data to/from a storage. type StockStorage interface { Quote(code, date string) (float64, error) Code(companyName, stockType string) (string, error) Save(stream io.Reader, code string) (int, error) }