Repository: facebook/proxygen
Branch: main
Commit: 2d72b64e5ce4
Files: 1140
Total size: 7.5 MB
Directory structure:
gitextract_yvh0wu4e/
├── .github/
│ ├── dependabot.yml
│ └── workflows/
│ ├── TagIt.yml
│ ├── close_stale.yml
│ ├── getdeps_linux.yml
│ ├── getdeps_mac.yml
│ └── publish_mvfst_interop.yml
├── .gitignore
├── CMakeLists.txt
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── Doxyfile
├── LICENSE
├── README.md
├── build/
│ ├── README.md
│ ├── deps/
│ │ └── github_hashes/
│ │ ├── facebook/
│ │ │ ├── folly-rev.txt
│ │ │ ├── mvfst-rev.txt
│ │ │ └── wangle-rev.txt
│ │ └── facebookincubator/
│ │ └── fizz-rev.txt
│ └── fbcode_builder/
│ ├── .gitignore
│ ├── CMake/
│ │ ├── FBBuildOptions.cmake
│ │ ├── FBCMakeParseArgs.cmake
│ │ ├── FBCompilerSettings.cmake
│ │ ├── FBCompilerSettingsMSVC.cmake
│ │ ├── FBCompilerSettingsUnix.cmake
│ │ ├── FBPythonBinary.cmake
│ │ ├── FBPythonTestAddTests.cmake
│ │ ├── FBThriftCppLibrary.cmake
│ │ ├── FBThriftLibrary.cmake
│ │ ├── FBThriftPyLibrary.cmake
│ │ ├── FindCares.cmake
│ │ ├── FindDoubleConversion.cmake
│ │ ├── FindGMock.cmake
│ │ ├── FindGflags.cmake
│ │ ├── FindGlog.cmake
│ │ ├── FindLMDB.cmake
│ │ ├── FindLibEvent.cmake
│ │ ├── FindLibUnwind.cmake
│ │ ├── FindLibiberty.cmake
│ │ ├── FindPCRE.cmake
│ │ ├── FindPCRE2.cmake
│ │ ├── FindRe2.cmake
│ │ ├── FindSodium.cmake
│ │ ├── FindXxhash.cmake
│ │ ├── FindZstd.cmake
│ │ ├── Findibverbs.cmake
│ │ ├── RustStaticLibrary.cmake
│ │ ├── fb_py_test_main.py
│ │ ├── fb_py_win_main.c
│ │ └── make_fbpy_archive.py
│ ├── LICENSE
│ ├── README.md
│ ├── getdeps/
│ │ ├── __init__.py
│ │ ├── builder.py
│ │ ├── buildopts.py
│ │ ├── cache.py
│ │ ├── cargo.py
│ │ ├── copytree.py
│ │ ├── dyndeps.py
│ │ ├── envfuncs.py
│ │ ├── errors.py
│ │ ├── expr.py
│ │ ├── fetcher.py
│ │ ├── include_rewriter.py
│ │ ├── load.py
│ │ ├── manifest.py
│ │ ├── platform.py
│ │ ├── py_wheel_builder.py
│ │ ├── runcmd.py
│ │ ├── subcmd.py
│ │ └── test/
│ │ ├── expr_test.py
│ │ ├── fixtures/
│ │ │ └── duplicate/
│ │ │ ├── foo
│ │ │ └── subdir/
│ │ │ └── foo
│ │ ├── manifest_test.py
│ │ ├── platform_test.py
│ │ ├── retry_test.py
│ │ ├── scratch_test.py
│ │ └── strip_marker_test.py
│ ├── getdeps.py
│ ├── manifests/
│ │ ├── CLI11
│ │ ├── autoconf
│ │ ├── automake
│ │ ├── benchmark
│ │ ├── blake3
│ │ ├── boost
│ │ ├── boost-python
│ │ ├── bz2
│ │ ├── c-ares
│ │ ├── cabal
│ │ ├── cachelib
│ │ ├── cinderx-3_14
│ │ ├── cinderx-main
│ │ ├── clang
│ │ ├── clang19
│ │ ├── cmake
│ │ ├── cpptoml
│ │ ├── double-conversion
│ │ ├── double-conversion-python
│ │ ├── eden
│ │ ├── edencommon
│ │ ├── exprtk
│ │ ├── fast_float
│ │ ├── fatal
│ │ ├── fb303
│ │ ├── fboss
│ │ ├── fbthrift
│ │ ├── fbthrift-python
│ │ ├── fizz
│ │ ├── fizz-python
│ │ ├── fmt
│ │ ├── fmt-python
│ │ ├── folly
│ │ ├── folly-python
│ │ ├── gcc12
│ │ ├── gcc14
│ │ ├── gflags
│ │ ├── ghc
│ │ ├── git-lfs
│ │ ├── glean
│ │ ├── glog
│ │ ├── googletest
│ │ ├── gperf
│ │ ├── hexdump
│ │ ├── hsthrift
│ │ ├── iproute2
│ │ ├── jom
│ │ ├── jq
│ │ ├── katran
│ │ ├── libaio
│ │ ├── libaio-python
│ │ ├── libbpf
│ │ ├── libcurl
│ │ ├── libdwarf
│ │ ├── libdwarf-python
│ │ ├── libelf
│ │ ├── libevent
│ │ ├── libevent-python
│ │ ├── libffi
│ │ ├── libgit2
│ │ ├── libgpiod
│ │ ├── libiberty
│ │ ├── libiberty-python
│ │ ├── libibverbs
│ │ ├── libmnl
│ │ ├── libnl
│ │ ├── liboqs
│ │ ├── libsai
│ │ ├── libsodium
│ │ ├── libtool
│ │ ├── libunwind
│ │ ├── libusb
│ │ ├── libyaml
│ │ ├── llvm
│ │ ├── lmdb
│ │ ├── lz4
│ │ ├── lz4-python
│ │ ├── magic_enum
│ │ ├── mcrouter
│ │ ├── mononoke
│ │ ├── mononoke_integration
│ │ ├── moxygen
│ │ ├── mvfst
│ │ ├── mvfst-python
│ │ ├── ncurses
│ │ ├── nghttp2
│ │ ├── ninja
│ │ ├── nlohmann-json
│ │ ├── nmap
│ │ ├── numa
│ │ ├── openr
│ │ ├── openssl
│ │ ├── osxfuse
│ │ ├── patchelf
│ │ ├── pcre2
│ │ ├── perl
│ │ ├── pexpect
│ │ ├── proxygen
│ │ ├── python
│ │ ├── python-3_14
│ │ ├── python-click
│ │ ├── python-filelock
│ │ ├── python-main
│ │ ├── python-psutil
│ │ ├── python-ptyprocess
│ │ ├── python-pyyaml
│ │ ├── python-setuptools
│ │ ├── python-setuptools-69
│ │ ├── python-six
│ │ ├── python-toml
│ │ ├── ragel
│ │ ├── range-v3
│ │ ├── rdma-core
│ │ ├── re2
│ │ ├── rebalancer
│ │ ├── ripgrep
│ │ ├── rocksdb
│ │ ├── rust-shed
│ │ ├── sapling
│ │ ├── snappy
│ │ ├── sparsemap
│ │ ├── sqlite3
│ │ ├── systemd
│ │ ├── tabulate
│ │ ├── tree
│ │ ├── wangle
│ │ ├── wangle-python
│ │ ├── watchman
│ │ ├── xxhash
│ │ ├── xz
│ │ ├── yaml-cpp
│ │ ├── zlib
│ │ ├── zlib-python
│ │ ├── zstd
│ │ └── zstd-python
│ └── patches/
│ ├── boost_1_83_0.patch
│ ├── iproute2_oss.patch
│ ├── libiberty_install_pic_lib.patch
│ └── zlib_dont_build_more_than_needed.patch
├── cmake/
│ ├── FindGMock.cmake
│ ├── FindZstd.cmake
│ ├── ProxygenCompatAliases.cmake
│ ├── ProxygenFunctions.cmake
│ ├── ProxygenTest.cmake
│ ├── TARGETS.txt
│ └── proxygen-config.cmake.in
├── cmake_uninstall.cmake.in
├── getdeps.sh
└── proxygen/
├── .clang-format
├── .clang-tidy
├── CMakeLists.txt
├── VERSION
├── build.sh
├── external/
│ ├── CMakeLists.txt
│ └── http_parser/
│ ├── CONTRIBUTIONS
│ ├── LICENSE-MIT
│ ├── README.md
│ ├── http_parser.h
│ ├── http_parser_cpp.cpp
│ └── test.c
├── fuzzers/
│ └── CMakeLists.txt
├── httpclient/
│ ├── CMakeLists.txt
│ └── samples/
│ ├── CMakeLists.txt
│ ├── H3Datagram/
│ │ ├── CMakeLists.txt
│ │ └── H3DatagramClient.cpp
│ ├── curl/
│ │ ├── CMakeLists.txt
│ │ ├── CurlClient.cpp
│ │ ├── CurlClient.h
│ │ └── CurlClientMain.cpp
│ ├── httperf2/
│ │ ├── CMakeLists.txt
│ │ ├── Client.cpp
│ │ ├── Client.h
│ │ ├── HTTPerf2.cpp
│ │ ├── HTTPerf2.h
│ │ ├── HTTPerfStats.h
│ │ └── Main.cpp
│ └── websocket/
│ ├── CMakeLists.txt
│ ├── WebSocketClient.cpp
│ ├── WebSocketClient.h
│ └── main.cpp
├── httpserver/
│ ├── CMakeLists.txt
│ ├── Filters.h
│ ├── HTTPServer.cpp
│ ├── HTTPServer.h
│ ├── HTTPServerAcceptor.cpp
│ ├── HTTPServerAcceptor.h
│ ├── HTTPServerOptions.h
│ ├── HTTPTransactionHandlerAdaptor.h
│ ├── Mocks.h
│ ├── PushHandler.h
│ ├── RequestHandler.h
│ ├── RequestHandlerAdaptor.cpp
│ ├── RequestHandlerAdaptor.h
│ ├── RequestHandlerFactory.h
│ ├── ResponseBuilder.h
│ ├── ResponseHandler.h
│ ├── ScopedHTTPServer.h
│ ├── SignalHandler.cpp
│ ├── SignalHandler.h
│ ├── filters/
│ │ ├── CompressionFilter.h
│ │ ├── DecompressionFilter.cpp
│ │ ├── DecompressionFilter.h
│ │ ├── DirectResponseHandler.h
│ │ ├── RejectConnectFilter.h
│ │ └── tests/
│ │ ├── CMakeLists.txt
│ │ ├── CompressionFilterTest.cpp
│ │ └── DecompressionFilterTest.cpp
│ ├── samples/
│ │ ├── echo/
│ │ │ ├── EchoHandler.cpp
│ │ │ ├── EchoHandler.h
│ │ │ ├── EchoServer.cpp
│ │ │ ├── EchoStats.h
│ │ │ └── test/
│ │ │ ├── CMakeLists.txt
│ │ │ └── EchoHandlerTest.cpp
│ │ ├── hq/
│ │ │ ├── CMakeLists.txt
│ │ │ ├── ConnIdLogger.h
│ │ │ ├── FizzContext.cpp
│ │ │ ├── FizzContext.h
│ │ │ ├── H1QDownstreamSession.h
│ │ │ ├── H1QUpstreamSession.h
│ │ │ ├── H2Server.cpp
│ │ │ ├── H2Server.h
│ │ │ ├── HQClient.cpp
│ │ │ ├── HQClient.h
│ │ │ ├── HQCommandLine.cpp
│ │ │ ├── HQCommandLine.h
│ │ │ ├── HQLoggerHelper.cpp
│ │ │ ├── HQLoggerHelper.h
│ │ │ ├── HQParams.cpp
│ │ │ ├── HQParams.h
│ │ │ ├── HQServer.cpp
│ │ │ ├── HQServer.h
│ │ │ ├── HQServerModule.cpp
│ │ │ ├── HQServerModule.h
│ │ │ ├── InsecureVerifierDangerousDoNotUseInProduction.h
│ │ │ ├── SampleHandlers.cpp
│ │ │ ├── SampleHandlers.h
│ │ │ ├── devious/
│ │ │ │ ├── CMakeLists.txt
│ │ │ │ ├── DeviousBaton.cpp
│ │ │ │ ├── DeviousBaton.h
│ │ │ │ └── test/
│ │ │ │ ├── CMakeLists.txt
│ │ │ │ └── DeviousBatonTests.cpp
│ │ │ ├── main.cpp
│ │ │ ├── pusheen.txt
│ │ │ └── quic-interop/
│ │ │ ├── Dockerfile
│ │ │ ├── README.md
│ │ │ └── run_endpoint.sh
│ │ ├── masque/
│ │ │ └── MasqueClient.cpp
│ │ ├── proxy/
│ │ │ ├── ProxyHandler.cpp
│ │ │ ├── ProxyHandler.h
│ │ │ ├── ProxyServer.cpp
│ │ │ ├── ProxyStats.h
│ │ │ └── SessionWrapper.h
│ │ ├── push/
│ │ │ ├── PushRequestHandler.cpp
│ │ │ ├── PushRequestHandler.h
│ │ │ ├── PushServer.cpp
│ │ │ ├── PushStats.h
│ │ │ └── pusheen.txt
│ │ ├── static/
│ │ │ ├── StaticHandler.cpp
│ │ │ ├── StaticHandler.h
│ │ │ └── StaticServer.cpp
│ │ └── websocket/
│ │ ├── WebSocketHandler.cpp
│ │ ├── WebSocketHandler.h
│ │ └── main.cpp
│ └── tests/
│ ├── CMakeLists.txt
│ ├── HTTPServerTest.cpp
│ ├── RequestHandlerAdaptorTest.cpp
│ └── certs/
│ ├── ca_cert.pem
│ ├── ca_key.pem
│ ├── client_ca_cert.pem
│ ├── client_ca_key.pem
│ ├── client_cert.pem
│ ├── client_key.pem
│ ├── generate_certs.sh
│ ├── test_cert1.pem
│ ├── test_cert2.pem
│ ├── test_key1.pem
│ └── test_key2.pem
├── install.sh
└── lib/
├── CMakeLists.txt
├── dns/
│ ├── AsyncDNSStatsCollector.cpp
│ ├── AsyncDNSStatsCollector.h
│ ├── CAresResolver.cpp
│ ├── CAresResolver.h
│ ├── CMakeLists.txt
│ ├── CachingDNSResolver.cpp
│ ├── CachingDNSResolver.h
│ ├── DNSModule.cpp
│ ├── DNSModule.h
│ ├── DNSResolver.cpp
│ ├── DNSResolver.h
│ ├── FutureDNSResolver.cpp
│ ├── FutureDNSResolver.h
│ ├── NaiveResolutionCallback.cpp
│ ├── NaiveResolutionCallback.h
│ ├── Rfc6724.cpp
│ ├── Rfc6724.h
│ ├── SyncDNSResolver.cpp
│ ├── SyncDNSResolver.h
│ └── test/
│ ├── CAresResolverTest.cpp
│ ├── CachingDNSResolverTest.cpp
│ ├── DNSResolverTest.cpp
│ ├── Dummies.h
│ ├── FutureDNSResolverTest.cpp
│ ├── MockDNSModule.cpp
│ ├── MockDNSModule.h
│ ├── MockDNSResolver.h
│ ├── Mocks.h
│ ├── Rfc6724Test.cpp
│ ├── SyncDNSResolverTest.cpp
│ └── SyncDNSResolverTest.h
├── healthcheck/
│ ├── CMakeLists.txt
│ ├── PoolHealthChecker.h
│ ├── ServerHealthCheckerCallback.cpp
│ └── ServerHealthCheckerCallback.h
├── http/
│ ├── CMakeLists.txt
│ ├── HQConnector.cpp
│ ├── HQConnector.h
│ ├── HTTP3ErrorCode.cpp
│ ├── HTTP3ErrorCode.h
│ ├── HTTPCommonHeaders.txt
│ ├── HTTPConnector.cpp
│ ├── HTTPConnector.h
│ ├── HTTPConnectorWithFizz.cpp
│ ├── HTTPConnectorWithFizz.h
│ ├── HTTPConstants.cpp
│ ├── HTTPConstants.h
│ ├── HTTPException.cpp
│ ├── HTTPException.h
│ ├── HTTPHeaderSize.h
│ ├── HTTPHeaders.cpp
│ ├── HTTPHeaders.h
│ ├── HTTPMessage.cpp
│ ├── HTTPMessage.h
│ ├── HTTPMessageFilters.cpp
│ ├── HTTPMessageFilters.h
│ ├── HTTPMethod.cpp
│ ├── HTTPMethod.h
│ ├── HTTPPriorityFunctions.cpp
│ ├── HTTPPriorityFunctions.h
│ ├── HeaderConstants.cpp
│ ├── HeaderConstants.h
│ ├── ProxyStatus.cpp
│ ├── ProxyStatus.h
│ ├── ProxygenErrorEnum.cpp
│ ├── ProxygenErrorEnum.h
│ ├── RFC2616.cpp
│ ├── RFC2616.h
│ ├── StatusTypeEnum.cpp
│ ├── StatusTypeEnum.h
│ ├── SynchronizedLruQuicPskCache.cpp
│ ├── SynchronizedLruQuicPskCache.h
│ ├── Types.h
│ ├── Window.cpp
│ ├── Window.h
│ ├── codec/
│ │ ├── CMakeLists.txt
│ │ ├── CapsuleCodec.cpp
│ │ ├── CapsuleCodec.h
│ │ ├── CodecProtocol.cpp
│ │ ├── CodecProtocol.h
│ │ ├── CodecUtil.cpp
│ │ ├── CodecUtil.h
│ │ ├── ControlMessageRateLimitFilter.h
│ │ ├── DebugFilter.h
│ │ ├── DefaultHTTPCodecFactory.cpp
│ │ ├── DefaultHTTPCodecFactory.h
│ │ ├── DirectErrorsRateLimitFilter.h
│ │ ├── ErrorCode.cpp
│ │ ├── ErrorCode.h
│ │ ├── FlowControlFilter.cpp
│ │ ├── FlowControlFilter.h
│ │ ├── HQControlCodec.cpp
│ │ ├── HQControlCodec.h
│ │ ├── HQFramedCodec.cpp
│ │ ├── HQFramedCodec.h
│ │ ├── HQFramer.cpp
│ │ ├── HQFramer.h
│ │ ├── HQMultiCodec.h
│ │ ├── HQStreamCodec.cpp
│ │ ├── HQStreamCodec.h
│ │ ├── HQUnidirectionalCodec.cpp
│ │ ├── HQUnidirectionalCodec.h
│ │ ├── HQUtils.cpp
│ │ ├── HQUtils.h
│ │ ├── HTTP1xCodec.cpp
│ │ ├── HTTP1xCodec.h
│ │ ├── HTTP2Codec.cpp
│ │ ├── HTTP2Codec.h
│ │ ├── HTTP2Constants.cpp
│ │ ├── HTTP2Constants.h
│ │ ├── HTTP2Framer.cpp
│ │ ├── HTTP2Framer.h
│ │ ├── HTTPBinaryCodec.cpp
│ │ ├── HTTPBinaryCodec.h
│ │ ├── HTTPChecks.cpp
│ │ ├── HTTPChecks.h
│ │ ├── HTTPCodec.h
│ │ ├── HTTPCodecFactory.cpp
│ │ ├── HTTPCodecFactory.h
│ │ ├── HTTPCodecFilter.cpp
│ │ ├── HTTPCodecFilter.h
│ │ ├── HTTPCodecPrinter.cpp
│ │ ├── HTTPCodecPrinter.h
│ │ ├── HTTPParallelCodec.cpp
│ │ ├── HTTPParallelCodec.h
│ │ ├── HTTPRequestVerifier.h
│ │ ├── HTTPSettings.cpp
│ │ ├── HTTPSettings.h
│ │ ├── HeaderDecodeInfo.cpp
│ │ ├── HeaderDecodeInfo.h
│ │ ├── HeadersRateLimitFilter.h
│ │ ├── QPACKDecoderCodec.h
│ │ ├── QPACKEncoderCodec.h
│ │ ├── RateLimitFilter.cpp
│ │ ├── RateLimitFilter.h
│ │ ├── ResetsRateLimitFilter.h
│ │ ├── SettingsId.h
│ │ ├── TransportDirection.cpp
│ │ ├── TransportDirection.h
│ │ ├── compress/
│ │ │ ├── CMakeLists.txt
│ │ │ ├── CompressionInfo.h
│ │ │ ├── HPACKCodec.cpp
│ │ │ ├── HPACKCodec.h
│ │ │ ├── HPACKConstants.h
│ │ │ ├── HPACKContext.cpp
│ │ │ ├── HPACKContext.h
│ │ │ ├── HPACKDecodeBuffer.cpp
│ │ │ ├── HPACKDecodeBuffer.h
│ │ │ ├── HPACKDecoder.cpp
│ │ │ ├── HPACKDecoder.h
│ │ │ ├── HPACKDecoderBase.cpp
│ │ │ ├── HPACKDecoderBase.h
│ │ │ ├── HPACKEncodeBuffer.cpp
│ │ │ ├── HPACKEncodeBuffer.h
│ │ │ ├── HPACKEncoder.cpp
│ │ │ ├── HPACKEncoder.h
│ │ │ ├── HPACKEncoderBase.cpp
│ │ │ ├── HPACKEncoderBase.h
│ │ │ ├── HPACKHeader.cpp
│ │ │ ├── HPACKHeader.h
│ │ │ ├── HPACKHeaderName.h
│ │ │ ├── HPACKStreamingCallback.h
│ │ │ ├── Header.h
│ │ │ ├── HeaderCodec.h
│ │ │ ├── HeaderIndexingStrategy.cpp
│ │ │ ├── HeaderIndexingStrategy.h
│ │ │ ├── HeaderPiece.h
│ │ │ ├── HeaderTable.cpp
│ │ │ ├── HeaderTable.h
│ │ │ ├── Huffman.cpp
│ │ │ ├── Huffman.h
│ │ │ ├── Logging.cpp
│ │ │ ├── Logging.h
│ │ │ ├── NoPathIndexingStrategy.cpp
│ │ │ ├── NoPathIndexingStrategy.h
│ │ │ ├── QPACKCodec.cpp
│ │ │ ├── QPACKCodec.h
│ │ │ ├── QPACKContext.cpp
│ │ │ ├── QPACKContext.h
│ │ │ ├── QPACKDecoder.cpp
│ │ │ ├── QPACKDecoder.h
│ │ │ ├── QPACKEncoder.cpp
│ │ │ ├── QPACKEncoder.h
│ │ │ ├── QPACKHeaderTable.cpp
│ │ │ ├── QPACKHeaderTable.h
│ │ │ ├── QPACKStaticHeaderTable.cpp
│ │ │ ├── QPACKStaticHeaderTable.h
│ │ │ ├── StaticHeaderTable.cpp
│ │ │ ├── StaticHeaderTable.h
│ │ │ ├── experimental/
│ │ │ │ ├── interop/
│ │ │ │ │ └── QPACKInterop.cpp
│ │ │ │ └── simulator/
│ │ │ │ ├── CMakeLists.txt
│ │ │ │ ├── CompressionScheme.h
│ │ │ │ ├── CompressionSimulator.cpp
│ │ │ │ ├── CompressionSimulator.h
│ │ │ │ ├── CompressionTypes.h
│ │ │ │ ├── CompressionUtils.cpp
│ │ │ │ ├── CompressionUtils.h
│ │ │ │ ├── HPACKQueue.h
│ │ │ │ ├── HPACKQueueTests.cpp
│ │ │ │ ├── HPACKScheme.h
│ │ │ │ ├── Main.cpp
│ │ │ │ ├── QMINScheme.h
│ │ │ │ ├── QPACKScheme.h
│ │ │ │ └── SimStreamingCallback.h
│ │ │ └── test/
│ │ │ ├── CMakeLists.txt
│ │ │ ├── EncoderBenchmark.cpp
│ │ │ ├── HPACKBenchmark.cpp
│ │ │ ├── HPACKBufferTests.cpp
│ │ │ ├── HPACKCodecTests.cpp
│ │ │ ├── HPACKContextTests.cpp
│ │ │ ├── HPACKHeaderTests.cpp
│ │ │ ├── HTTPArchive.cpp
│ │ │ ├── HTTPArchive.h
│ │ │ ├── HeaderPieceTests.cpp
│ │ │ ├── HeaderTableTests.cpp
│ │ │ ├── HuffmanTests.cpp
│ │ │ ├── LoggingTests.cpp
│ │ │ ├── QPACKCodecTests.cpp
│ │ │ ├── QPACKContextTests.cpp
│ │ │ ├── QPACKHeaderTableTests.cpp
│ │ │ ├── RFCExamplesTests.cpp
│ │ │ ├── TestStreamingCallback.h
│ │ │ ├── TestUtil.cpp
│ │ │ └── TestUtil.h
│ │ ├── test/
│ │ │ ├── CMakeLists.txt
│ │ │ ├── CapsuleCodecTest.cpp
│ │ │ ├── CodecUtilTests.cpp
│ │ │ ├── CrossCodecTest.cpp
│ │ │ ├── DefaultHTTPCodecFactoryTest.cpp
│ │ │ ├── FilterTests.cpp
│ │ │ ├── HQCodecTest.cpp
│ │ │ ├── HQFramerTest.cpp
│ │ │ ├── HQFramerTest.h
│ │ │ ├── HQMultiCodecTest.cpp
│ │ │ ├── HTTP1xCodecTest.cpp
│ │ │ ├── HTTP2CodecTest.cpp
│ │ │ ├── HTTP2FramerTest.cpp
│ │ │ ├── HTTP2FramerTest.h
│ │ │ ├── HTTPBinaryCodecTest.cpp
│ │ │ ├── HTTPParallelCodecTest.h
│ │ │ ├── MockHTTPCodec.h
│ │ │ ├── TestUtils.cpp
│ │ │ └── TestUtils.h
│ │ └── webtransport/
│ │ ├── CMakeLists.txt
│ │ ├── WebTransportCapsuleCodec.cpp
│ │ ├── WebTransportCapsuleCodec.h
│ │ ├── WebTransportFramer.cpp
│ │ ├── WebTransportFramer.h
│ │ └── test/
│ │ ├── WebTransportCapsuleCodecTest.cpp
│ │ └── WebTransportFramerTest.cpp
│ ├── connpool/
│ │ ├── CMakeLists.txt
│ │ ├── Endpoint.h
│ │ ├── ServerIdleSessionController.cpp
│ │ ├── ServerIdleSessionController.h
│ │ ├── SessionHolder.cpp
│ │ ├── SessionHolder.h
│ │ ├── SessionPool.cpp
│ │ ├── SessionPool.h
│ │ ├── ThreadIdleSessionController.cpp
│ │ ├── ThreadIdleSessionController.h
│ │ └── test/
│ │ ├── CMakeLists.txt
│ │ ├── SessionPoolTest.cpp
│ │ └── SessionPoolTestFixture.h
│ ├── coro/
│ │ ├── CMakeLists.txt
│ │ ├── HTTPBodyEventQueue.cpp
│ │ ├── HTTPBodyEventQueue.h
│ │ ├── HTTPByteEventHelpers.cpp
│ │ ├── HTTPByteEventHelpers.h
│ │ ├── HTTPByteEvents.h
│ │ ├── HTTPCoroSession.cpp
│ │ ├── HTTPCoroSession.h
│ │ ├── HTTPError.cpp
│ │ ├── HTTPError.h
│ │ ├── HTTPEvents.cpp
│ │ ├── HTTPEvents.h
│ │ ├── HTTPFilterFactoryHandler.cpp
│ │ ├── HTTPFilterFactoryHandler.h
│ │ ├── HTTPFixedSource.h
│ │ ├── HTTPHandlerChain.cpp
│ │ ├── HTTPHandlerChain.h
│ │ ├── HTTPHybridSource.h
│ │ ├── HTTPSource.h
│ │ ├── HTTPSourceFilter.cpp
│ │ ├── HTTPSourceFilter.h
│ │ ├── HTTPSourceFilterChain.h
│ │ ├── HTTPSourceHolder.h
│ │ ├── HTTPSourceReader.cpp
│ │ ├── HTTPSourceReader.h
│ │ ├── HTTPStreamSource.cpp
│ │ ├── HTTPStreamSource.h
│ │ ├── HTTPStreamSourceHolder.h
│ │ ├── HTTPStreamSourceSink.cpp
│ │ ├── HTTPStreamSourceSink.h
│ │ ├── HTTPStreamSourceSinkFactory.h
│ │ ├── HTTPTransactionAdaptorSource.cpp
│ │ ├── HTTPTransactionAdaptorSource.h
│ │ ├── benchmark/
│ │ │ └── HTTPCoroBenchmark.cpp
│ │ ├── client/
│ │ │ ├── CMakeLists.txt
│ │ │ ├── CertReloadSessionPool.cpp
│ │ │ ├── CertReloadSessionPool.h
│ │ │ ├── CoroDNSResolver.cpp
│ │ │ ├── CoroDNSResolver.h
│ │ │ ├── HTTPClient.cpp
│ │ │ ├── HTTPClient.h
│ │ │ ├── HTTPClientConnectionCache.cpp
│ │ │ ├── HTTPClientConnectionCache.h
│ │ │ ├── HTTPCoroConnector.cpp
│ │ │ ├── HTTPCoroConnector.h
│ │ │ ├── HTTPCoroSessionPool.cpp
│ │ │ ├── HTTPCoroSessionPool.h
│ │ │ ├── HTTPSessionFactory.h
│ │ │ ├── main.cpp
│ │ │ ├── samples/
│ │ │ │ └── cocurl/
│ │ │ │ ├── CMakeLists.txt
│ │ │ │ └── CoCurl.cpp
│ │ │ └── test/
│ │ │ ├── CoroDNSResolverTests.cpp
│ │ │ ├── HTTPClientTests.cpp
│ │ │ ├── HTTPClientTestsCommon.cpp
│ │ │ ├── HTTPClientTestsCommon.h
│ │ │ ├── HTTPConnectIntegrationTest.cpp
│ │ │ ├── MockHTTPClient.h
│ │ │ └── certs/
│ │ │ ├── test_cert1.pem
│ │ │ └── test_key1.pem
│ │ ├── filters/
│ │ │ ├── CMakeLists.txt
│ │ │ ├── CompressionFilter.cpp
│ │ │ ├── CompressionFilter.h
│ │ │ ├── CompressionFilterFactory.h
│ │ │ ├── DecompressionFilter.cpp
│ │ │ ├── DecompressionFilter.h
│ │ │ ├── DecompressionFilterFactory.cpp
│ │ │ ├── DecompressionFilterFactory.h
│ │ │ ├── FilterFactory.h
│ │ │ ├── HTTPRedirectHandler.cpp
│ │ │ ├── HTTPRedirectHandler.h
│ │ │ ├── Logger.cpp
│ │ │ ├── Logger.h
│ │ │ ├── MutateFilter.cpp
│ │ │ ├── MutateFilter.h
│ │ │ ├── RateLimitFilter.cpp
│ │ │ ├── RateLimitFilter.h
│ │ │ ├── RequestContextFilterFactory.cpp
│ │ │ ├── RequestContextFilterFactory.h
│ │ │ ├── ServerFilterFactory.h
│ │ │ ├── StatsFilterUtil.cpp
│ │ │ ├── StatsFilterUtil.h
│ │ │ ├── Status1xxFilter.cpp
│ │ │ ├── Status1xxFilter.h
│ │ │ ├── TransformFilter.cpp
│ │ │ ├── TransformFilter.h
│ │ │ ├── VisitorFilter.cpp
│ │ │ ├── VisitorFilter.h
│ │ │ └── test/
│ │ │ ├── CompressionFilterTests.cpp
│ │ │ ├── DecompressionFilterTest.cpp
│ │ │ ├── FakeServerStats.cpp
│ │ │ ├── FakeServerStats.h
│ │ │ ├── HTTPRedirectHandlerTest.cpp
│ │ │ ├── LoggerTest.cpp
│ │ │ ├── MutateFilterTest.cpp
│ │ │ ├── RateLimitFilterTest.cpp
│ │ │ ├── RequestContextFilterFactoryTest.cpp
│ │ │ ├── Status1xxFilterTest.cpp
│ │ │ ├── TransformFilterTest.cpp
│ │ │ └── VisitorFilterTest.cpp
│ │ ├── server/
│ │ │ ├── CMakeLists.txt
│ │ │ ├── HTTPCoroAcceptor.cpp
│ │ │ ├── HTTPCoroAcceptor.h
│ │ │ ├── HTTPServer.cpp
│ │ │ ├── HTTPServer.h
│ │ │ ├── ScopedHTTPServer.h
│ │ │ ├── handlers/
│ │ │ │ ├── CMakeLists.txt
│ │ │ │ ├── ExpectContinueWrapperHandler.cpp
│ │ │ │ ├── ExpectContinueWrapperHandler.h
│ │ │ │ └── test/
│ │ │ │ └── ExpectContinueWrapperHandlerTest.cpp
│ │ │ ├── samples/
│ │ │ │ ├── echo/
│ │ │ │ │ └── EchoServer.cpp
│ │ │ │ ├── fwdproxy/
│ │ │ │ │ ├── ConnectSource.h
│ │ │ │ │ └── FwdProxyServer.cpp
│ │ │ │ └── proxy/
│ │ │ │ └── ProxyServer.cpp
│ │ │ └── test/
│ │ │ ├── HTTPServerTest.cpp
│ │ │ └── certs/
│ │ │ ├── test_cert1.pem
│ │ │ ├── test_cert2.pem
│ │ │ ├── test_key1.pem
│ │ │ └── test_key2.pem
│ │ ├── test/
│ │ │ ├── HTTPCoroSessionTests.cpp
│ │ │ ├── HTTPCoroSessionTests.h
│ │ │ ├── HTTPDownstreamCoroSessionTests.cpp
│ │ │ ├── HTTPFilterFactoryHandlerTests.cpp
│ │ │ ├── HTTPHandlerChainTests.cpp
│ │ │ ├── HTTPSourceTests.cpp
│ │ │ ├── HTTPStreamSourceSinkTests.cpp
│ │ │ ├── HTTPTestSources.h
│ │ │ ├── HTTPTransactionAdaptorSourceTest.cpp
│ │ │ ├── HTTPUpstreamCoroSessionTests.cpp
│ │ │ ├── HttpWtUpstreamTests.cpp
│ │ │ ├── Mocks.h
│ │ │ ├── TestUtils.h
│ │ │ └── WindowContainerTests.cpp
│ │ ├── transport/
│ │ │ ├── CMakeLists.txt
│ │ │ ├── CoroSSLTransport.cpp
│ │ │ ├── CoroSSLTransport.h
│ │ │ ├── HTTPConnectAsyncTransport.cpp
│ │ │ ├── HTTPConnectAsyncTransport.h
│ │ │ ├── HTTPConnectStream.cpp
│ │ │ ├── HTTPConnectStream.h
│ │ │ ├── HTTPConnectTransport.cpp
│ │ │ ├── HTTPConnectTransport.h
│ │ │ └── test/
│ │ │ ├── Client.cpp
│ │ │ ├── CoroSSLTransportTest.cpp
│ │ │ ├── HTTPConnectTransportTest.cpp
│ │ │ ├── HTTPConnectTransportTest.h
│ │ │ ├── TestCoroTransport.cpp
│ │ │ └── TestCoroTransport.h
│ │ └── util/
│ │ ├── AwaitableKeepAlive.h
│ │ ├── CMakeLists.txt
│ │ ├── CancellableBaton.cpp
│ │ ├── CancellableBaton.h
│ │ ├── CoroWtSession.cpp
│ │ ├── CoroWtSession.h
│ │ ├── DetachableExecutor.cpp
│ │ ├── DetachableExecutor.h
│ │ ├── ExecutorSourceFilter.cpp
│ │ ├── ExecutorSourceFilter.h
│ │ ├── Refcount.h
│ │ ├── TimedBaton.h
│ │ ├── Transport.cpp
│ │ ├── Transport.h
│ │ ├── WindowContainer.h
│ │ └── test/
│ │ ├── AwaitableKeepAliveTest.cpp
│ │ ├── CancellableBatonTest.cpp
│ │ ├── DetachableExecutorTest.cpp
│ │ ├── TestHelpers.h
│ │ └── TimedBatonTest.cpp
│ ├── experimental/
│ │ ├── CMakeLists.txt
│ │ ├── RFC1867.cpp
│ │ ├── RFC1867.h
│ │ └── test/
│ │ └── RFC1867Test.cpp
│ ├── gen_HTTPCommonHeaders.sh
│ ├── observer/
│ │ ├── CMakeLists.txt
│ │ ├── HTTPSessionObserverContainer.h
│ │ ├── HTTPSessionObserverInterface.cpp
│ │ ├── HTTPSessionObserverInterface.h
│ │ ├── HTTPTransactionObserverContainer.h
│ │ ├── HTTPTransactionObserverInterface.cpp
│ │ └── HTTPTransactionObserverInterface.h
│ ├── session/
│ │ ├── AckLatencyEvent.h
│ │ ├── ByteEventTracker.cpp
│ │ ├── ByteEventTracker.h
│ │ ├── ByteEvents.cpp
│ │ ├── ByteEvents.h
│ │ ├── CMakeLists.txt
│ │ ├── CodecErrorResponseHandler.cpp
│ │ ├── CodecErrorResponseHandler.h
│ │ ├── HQByteEventTracker.cpp
│ │ ├── HQByteEventTracker.h
│ │ ├── HQDownstreamSession.cpp
│ │ ├── HQDownstreamSession.h
│ │ ├── HQSession.cpp
│ │ ├── HQSession.h
│ │ ├── HQStreamBase.cpp
│ │ ├── HQStreamBase.h
│ │ ├── HQStreamDispatcher.cpp
│ │ ├── HQStreamDispatcher.h
│ │ ├── HQUpstreamSession.cpp
│ │ ├── HQUpstreamSession.h
│ │ ├── HTTP2PriorityQueue.cpp
│ │ ├── HTTP2PriorityQueue.h
│ │ ├── HTTPDefaultSessionCodecFactory.cpp
│ │ ├── HTTPDefaultSessionCodecFactory.h
│ │ ├── HTTPDirectResponseHandler.cpp
│ │ ├── HTTPDirectResponseHandler.h
│ │ ├── HTTPDownstreamSession.cpp
│ │ ├── HTTPDownstreamSession.h
│ │ ├── HTTPErrorPage.cpp
│ │ ├── HTTPErrorPage.h
│ │ ├── HTTPEvent.cpp
│ │ ├── HTTPEvent.h
│ │ ├── HTTPSession.cpp
│ │ ├── HTTPSession.h
│ │ ├── HTTPSessionAcceptor.cpp
│ │ ├── HTTPSessionAcceptor.h
│ │ ├── HTTPSessionActivityTracker.cpp
│ │ ├── HTTPSessionActivityTracker.h
│ │ ├── HTTPSessionBase.cpp
│ │ ├── HTTPSessionBase.h
│ │ ├── HTTPSessionController.h
│ │ ├── HTTPSessionStats.h
│ │ ├── HTTPTransaction.cpp
│ │ ├── HTTPTransaction.h
│ │ ├── HTTPTransactionEgressSM.cpp
│ │ ├── HTTPTransactionEgressSM.h
│ │ ├── HTTPTransactionIngressSM.cpp
│ │ ├── HTTPTransactionIngressSM.h
│ │ ├── HTTPUpstreamSession.cpp
│ │ ├── HTTPUpstreamSession.h
│ │ ├── QuicProtocolInfo.h
│ │ ├── SecondaryAuthManager.cpp
│ │ ├── SecondaryAuthManager.h
│ │ ├── SecondaryAuthManagerBase.h
│ │ ├── ServerPushLifecycle.h
│ │ ├── SimpleController.cpp
│ │ ├── SimpleController.h
│ │ ├── TTLBAStats.h
│ │ ├── TransactionByteEvents.h
│ │ ├── WebTransportFilter.h
│ │ └── test/
│ │ ├── ByteEventTrackerMocks.h
│ │ ├── ByteEventTrackerTest.cpp
│ │ ├── CMakeLists.txt
│ │ ├── DownstreamTransactionTest.cpp
│ │ ├── HQByteEventTrackerTest.cpp
│ │ ├── HQDownstreamSessionTest.cpp
│ │ ├── HQDownstreamSessionTest.h
│ │ ├── HQSessionMocks.h
│ │ ├── HQSessionMocksTest.cpp
│ │ ├── HQSessionTestCommon.cpp
│ │ ├── HQSessionTestCommon.h
│ │ ├── HQStreamBaseTest.cpp
│ │ ├── HQStreamDispatcherTest.cpp
│ │ ├── HQUpstreamSessionTest.cpp
│ │ ├── HQUpstreamSessionTest.h
│ │ ├── HTTP2PriorityQueueBench.cpp
│ │ ├── HTTP2PriorityQueueTest.cpp
│ │ ├── HTTPDefaultSessionCodecFactoryTest.cpp
│ │ ├── HTTPDownstreamSessionTest.cpp
│ │ ├── HTTPSessionAcceptorTest.cpp
│ │ ├── HTTPSessionActivityTrackerTest.cpp
│ │ ├── HTTPSessionBenchmark.cpp
│ │ ├── HTTPSessionMocks.h
│ │ ├── HTTPSessionTest.h
│ │ ├── HTTPTransactionMocks.h
│ │ ├── HTTPTransactionSMTest.cpp
│ │ ├── HTTPTransactionTest.cpp
│ │ ├── HTTPTransactionWebTransportTest.cpp
│ │ ├── HTTPUpstreamSessionTest.cpp
│ │ ├── MockByteEventTracker.h
│ │ ├── MockCodecDownstreamTest.cpp
│ │ ├── MockHTTPSessionStats.h
│ │ ├── MockHTTPTransactionObserver.h
│ │ ├── MockQuicSocketDriver.h
│ │ ├── MockSecondaryAuthManager.h
│ │ ├── MockSessionObserver.h
│ │ ├── SecondaryAuthManagerTest.cpp
│ │ ├── TestUtils.cpp
│ │ ├── TestUtils.h
│ │ ├── WebTransportFilterTest.cpp
│ │ ├── test_cert1.key
│ │ └── test_cert1.pem
│ ├── sink/
│ │ ├── CMakeLists.txt
│ │ ├── FlowControlInfo.h
│ │ ├── HTTPSink.h
│ │ ├── HTTPTransactionSink.cpp
│ │ ├── HTTPTransactionSink.h
│ │ ├── HTTPTunnelSink.cpp
│ │ ├── HTTPTunnelSink.h
│ │ └── test/
│ │ └── HTTPConnectSinkTest.cpp
│ ├── stats/
│ │ ├── ConnectionStats.cpp
│ │ ├── ConnectionStats.h
│ │ ├── HTTPCodecStats.cpp
│ │ ├── HTTPCodecStats.h
│ │ ├── HTTPCodecStatsFilter.cpp
│ │ ├── HTTPCodecStatsFilter.h
│ │ ├── HeaderCodecStats.cpp
│ │ ├── HeaderCodecStats.h
│ │ ├── HttpServerStats.h
│ │ ├── ResponseCodeStatsMinute.cpp
│ │ ├── ResponseCodeStatsMinute.h
│ │ ├── TLResponseCodeStats.cpp
│ │ ├── TLResponseCodeStats.h
│ │ ├── ThreadLocalHTTPSessionStats.cpp
│ │ └── ThreadLocalHTTPSessionStats.h
│ ├── structuredheaders/
│ │ ├── CMakeLists.txt
│ │ ├── StructuredHeadersBuffer.cpp
│ │ ├── StructuredHeadersBuffer.h
│ │ ├── StructuredHeadersConstants.cpp
│ │ ├── StructuredHeadersConstants.h
│ │ ├── StructuredHeadersDecoder.cpp
│ │ ├── StructuredHeadersDecoder.h
│ │ ├── StructuredHeadersEncoder.cpp
│ │ ├── StructuredHeadersEncoder.h
│ │ ├── StructuredHeadersUtilities.cpp
│ │ ├── StructuredHeadersUtilities.h
│ │ └── test/
│ │ ├── CMakeLists.txt
│ │ ├── StructuredHeadersBufferTest.cpp
│ │ ├── StructuredHeadersDecoderTest.cpp
│ │ ├── StructuredHeadersEncoderTest.cpp
│ │ ├── StructuredHeadersStandardTest.cpp
│ │ └── StructuredHeadersUtilitiesTest.cpp
│ ├── test/
│ │ ├── CMakeLists.txt
│ │ ├── HTTPCommonHeadersTests.cpp
│ │ ├── HTTPConnectorWithFizzTest.cpp
│ │ ├── HTTPHeadersBenchmark.cpp
│ │ ├── HTTPMessageFilterTest.cpp
│ │ ├── HTTPMessageTest.cpp
│ │ ├── HTTPPriorityFunctionsTest.cpp
│ │ ├── MockHTTPHeaders.h
│ │ ├── MockHTTPMessageFilter.h
│ │ ├── ProxyStatusTest.cpp
│ │ ├── RFC2616Test.cpp
│ │ └── WindowTest.cpp
│ └── webtransport/
│ ├── CMakeLists.txt
│ ├── FlowController.h
│ ├── HTTPWebTransport.cpp
│ ├── HTTPWebTransport.h
│ ├── QuicWebTransport.cpp
│ ├── QuicWebTransport.h
│ ├── QuicWtSession.cpp
│ ├── QuicWtSession.h
│ ├── StreamPriorityQueue.h
│ ├── WebTransport.cpp
│ ├── WebTransport.h
│ ├── WebTransportImpl.cpp
│ ├── WebTransportImpl.h
│ ├── WtEgressContainer.cpp
│ ├── WtEgressContainer.h
│ ├── WtStreamManager.cpp
│ ├── WtStreamManager.h
│ ├── WtUtils.cpp
│ ├── WtUtils.h
│ └── test/
│ ├── CMakeLists.txt
│ ├── FakeSharedWebTransport.h
│ ├── FlowControllerTest.cpp
│ ├── Mocks.h
│ ├── QuicWebTransportTest.cpp
│ ├── QuicWtSessionTest.cpp
│ ├── WebTransportAPITest.cpp
│ └── WtStreamManagerTest.cpp
├── pools/
│ └── generators/
│ ├── CMakeLists.txt
│ ├── FileServerListGenerator.cpp
│ ├── FileServerListGenerator.h
│ ├── MemberGroupConfig.h
│ ├── ServerConfig.h
│ ├── ServerListGenerator.cpp
│ ├── ServerListGenerator.h
│ └── coro/
│ ├── CMakeLists.txt
│ ├── CoroServerListGenerator.h
│ ├── FileCoroServerListGenerator.cpp
│ ├── FileCoroServerListGenerator.h
│ ├── JsonFileCoroServerListGenerator.cpp
│ ├── JsonFileCoroServerListGenerator.h
│ ├── PlainTextFileCoroServerListGenerator.cpp
│ └── PlainTextFileCoroServerListGenerator.h
├── sampling/
│ ├── CMakeLists.txt
│ ├── MultiSampled.h
│ ├── Sampled.h
│ ├── Sampling.cpp
│ ├── Sampling.h
│ ├── SamplingFunctions.h
│ └── test/
│ ├── CMakeLists.txt
│ └── SamplingTest.cpp
├── services/
│ ├── AcceptorConfiguration.h
│ ├── CMakeLists.txt
│ ├── HTTPAcceptor.h
│ ├── RequestWorkerThread.cpp
│ ├── RequestWorkerThread.h
│ ├── RequestWorkerThreadNoExecutor.cpp
│ ├── RequestWorkerThreadNoExecutor.h
│ ├── Service.cpp
│ ├── Service.h
│ ├── ServiceConfiguration.h
│ ├── ServiceWorker.h
│ ├── WorkerThread.cpp
│ ├── WorkerThread.h
│ └── test/
│ ├── AcceptorTest.cpp
│ ├── CMakeLists.txt
│ └── RequestWorkerThreadTest.cpp
├── ssl/
│ ├── CMakeLists.txt
│ ├── ProxygenSSL.h
│ ├── ThreadLocalSSLStats.cpp
│ └── ThreadLocalSSLStats.h
├── stats/
│ ├── BaseStats.cpp
│ ├── BaseStats.h
│ ├── PeriodicStats.h
│ ├── PeriodicStatsDataBase.h
│ ├── ResourceData.h
│ ├── ResourceStats.cpp
│ ├── ResourceStats.h
│ ├── gen_StatsWrapper.sh
│ └── test/
│ ├── BaseStatsTest.cpp
│ ├── CMakeLists.txt
│ ├── MockARLResourceStats.h
│ ├── MockResources.h
│ ├── PeriodicStatsTest.cpp
│ ├── PeriodicStatsTestHelper.h
│ ├── ResourceDataTest.cpp
│ └── ResourceStatsTest.cpp
├── test/
│ ├── CMakeLists.txt
│ ├── TestAsyncTransport.cpp
│ ├── TestAsyncTransport.h
│ └── TestMain.cpp
├── transport/
│ ├── AsyncUDPSocketFactory.cpp
│ ├── AsyncUDPSocketFactory.h
│ ├── CMakeLists.txt
│ ├── ConnectUDPUtils.cpp
│ ├── ConnectUDPUtils.h
│ ├── H3DatagramAsyncSocket.cpp
│ ├── H3DatagramAsyncSocket.h
│ ├── PersistentFizzPskCache.cpp
│ ├── PersistentFizzPskCache.h
│ ├── PersistentQuicPskCache.cpp
│ ├── PersistentQuicPskCache.h
│ ├── PersistentQuicTokenCache.cpp
│ ├── PersistentQuicTokenCache.h
│ └── test/
│ ├── AsyncUDPSocketFactoryTest.cpp
│ ├── CMakeLists.txt
│ ├── ConnectUDPUtilsTest.cpp
│ ├── H3DatagramAsyncSocketTest.cpp
│ ├── H3DatagramAsyncSocketTest.h
│ ├── MockAsyncTransportCertificate.h
│ ├── PersistentFizzPskCacheTest.cpp
│ └── PersistentQuicPskCacheTest.cpp
└── utils/
├── AcceptorAddress.h
├── AsyncTimeoutSet.cpp
├── AsyncTimeoutSet.h
├── CMakeLists.txt
├── CobHelper.h
├── CompressionFilterUtils.h
├── ConditionalGate.h
├── ConsistentHash.h
├── CryptUtil.cpp
├── CryptUtil.h
├── Exception.cpp
├── Exception.h
├── Export.h
├── FilterChain.h
├── HTTPTime.cpp
├── HTTPTime.h
├── Logging.cpp
├── Logging.h
├── NullTraceEventObserver.h
├── ParseURL.cpp
├── ParseURL.h
├── PerfectIndexMap.h
├── RendezvousHash.cpp
├── RendezvousHash.h
├── SafePathUtils.cpp
├── SafePathUtils.h
├── StateMachine.h
├── StreamCompressor.h
├── StreamDecompressor.h
├── TestUtils.h
├── Time.h
├── TraceEvent.cpp
├── TraceEvent.h
├── TraceEventContext.cpp
├── TraceEventContext.h
├── TraceEventObserver.h
├── URL.h
├── UnionBasedStatic.h
├── UtilInl.h
├── WeakRefCountedPtr.h
├── WheelTimerInstance.cpp
├── WheelTimerInstance.h
├── ZlibStreamCompressor.cpp
├── ZlibStreamCompressor.h
├── ZlibStreamDecompressor.cpp
├── ZlibStreamDecompressor.h
├── ZstdStreamCompressor.cpp
├── ZstdStreamCompressor.h
├── ZstdStreamDecompressor.cpp
├── ZstdStreamDecompressor.h
├── gen_perfect_hash_table.sh
├── gen_trace_event_constants.py
├── perfect_hash_table_template.cpp.gperf
├── perfect_hash_table_template.h
├── samples/
│ ├── TraceEventType.txt
│ └── TraceFieldType.txt
└── test/
├── AsyncTimeoutSetTest.cpp
├── CMakeLists.txt
├── CompressionFilterUtilsTest.cpp
├── ConditionalGateTest.cpp
├── CryptUtilTest.cpp
├── GenericFilterTest.cpp
├── HTTPTimeTest.cpp
├── LoggingTests.cpp
├── MockTime.h
├── ParseURLTest.cpp
├── PerfectIndexMapBenchmark.cpp
├── PerfectIndexMapTest.cpp
├── RendezvousHashTest.cpp
├── TimeTest.cpp
├── TraceEventTest.cpp
├── URLTest.cpp
├── UtilTest.cpp
├── WeakRefCountedPtrTest.cpp
├── ZlibTests.cpp
└── ZstdTests.cpp
================================================
FILE CONTENTS
================================================
================================================
FILE: .github/dependabot.yml
================================================
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
================================================
FILE: .github/workflows/TagIt.yml
================================================
on:
push:
tags:
# Only match TagIt tags, which always start with this prefix
- 'v20*'
name: TagIt
permissions:
contents: read
jobs:
build:
permissions:
contents: write # for actions/create-release to create a release
name: Release
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Archive project
id: archive_project
run: |
FILE_NAME=${GITHUB_REPOSITORY#*/}-${GITHUB_REF##*/}
git archive ${{ github.ref }} -o ${FILE_NAME}.zip
git archive ${{ github.ref }} -o ${FILE_NAME}.tar.gz
echo "file_name=${FILE_NAME}" >> $GITHUB_OUTPUT
- name: Compute digests
id: compute_digests
run: |
echo "tgz_256=$(openssl dgst -sha256 ${{ steps.archive_project.outputs.file_name }}.tar.gz)" >> $GITHUB_OUTPUT
echo "tgz_512=$(openssl dgst -sha512 ${{ steps.archive_project.outputs.file_name }}.tar.gz)" >> $GITHUB_OUTPUT
echo "zip_256=$(openssl dgst -sha256 ${{ steps.archive_project.outputs.file_name }}.zip)" >> $GITHUB_OUTPUT
echo "zip_512=$(openssl dgst -sha512 ${{ steps.archive_project.outputs.file_name }}.zip)" >> $GITHUB_OUTPUT
- name: Create Release
id: create_release
uses: actions/create-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref }}
release_name: ${{ github.ref }}
body: |
Automated release from TagIt
File Hashes
${{ steps.compute_digests.outputs.zip_256 }}
${{ steps.compute_digests.outputs.zip_512 }}
${{ steps.compute_digests.outputs.tgz_256 }}
${{ steps.compute_digests.outputs.tgz_512 }}
draft: false
prerelease: false
- name: Upload zip
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./${{ steps.archive_project.outputs.file_name }}.zip
asset_name: ${{ steps.archive_project.outputs.file_name }}.zip
asset_content_type: application/zip
- name: Upload tar.gz
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: ./${{ steps.archive_project.outputs.file_name }}.tar.gz
asset_name: ${{ steps.archive_project.outputs.file_name }}.tar.gz
asset_content_type: application/gzip
================================================
FILE: .github/workflows/close_stale.yml
================================================
name: Close inactive issues
on:
schedule:
- cron: "30 1 * * *"
jobs:
close-issues:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- uses: actions/stale@v9
with:
days-before-issue-stale: -1
days-before-issue-close: 14
stale-issue-label: "need-input"
stale-issue-message: "This issue is stale because it has been open too long with no activity."
close-issue-message: "This issue was closed because it has been inactive for two weeks since being marked as need-input."
days-before-pr-stale: -1
days-before-pr-close: -1
repo-token: ${{ secrets.GITHUB_TOKEN }}
================================================
FILE: .github/workflows/getdeps_linux.yml
================================================
# This file was @generated by getdeps.py
name: linux
on:
push:
branches:
- main
pull_request:
branches:
- main
permissions:
contents: read # to fetch code (actions/checkout)
jobs:
build:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v6
- name: Update system package info
run: sudo --preserve-env=http_proxy apt-get update
- name: Install system deps
run: sudo --preserve-env=http_proxy python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive proxygen && sudo --preserve-env=http_proxy python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive patchelf
- id: paths
name: Query paths
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages query-paths --recursive --src-dir=. proxygen >> "$GITHUB_OUTPUT"
- name: Fetch ninja
if: ${{ steps.paths.outputs.ninja_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja
- name: Fetch cmake
if: ${{ steps.paths.outputs.cmake_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake
- name: Fetch c-ares
if: ${{ steps.paths.outputs.c-ares_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests c-ares
- name: Fetch zlib
if: ${{ steps.paths.outputs.zlib_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zlib
- name: Fetch zstd
if: ${{ steps.paths.outputs.zstd_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd
- name: Fetch boost
if: ${{ steps.paths.outputs.boost_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost
- name: Fetch double-conversion
if: ${{ steps.paths.outputs.double-conversion_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion
- name: Fetch fast_float
if: ${{ steps.paths.outputs.fast_float_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fast_float
- name: Fetch fmt
if: ${{ steps.paths.outputs.fmt_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt
- name: Fetch gflags
if: ${{ steps.paths.outputs.gflags_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags
- name: Fetch glog
if: ${{ steps.paths.outputs.glog_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog
- name: Fetch googletest
if: ${{ steps.paths.outputs.googletest_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest
- name: Fetch libaio
if: ${{ steps.paths.outputs.libaio_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libaio
- name: Fetch libdwarf
if: ${{ steps.paths.outputs.libdwarf_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libdwarf
- name: Fetch libevent
if: ${{ steps.paths.outputs.libevent_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent
- name: Fetch lz4
if: ${{ steps.paths.outputs.lz4_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests lz4
- name: Fetch snappy
if: ${{ steps.paths.outputs.snappy_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy
- name: Fetch openssl
if: ${{ steps.paths.outputs.openssl_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl
- name: Fetch liboqs
if: ${{ steps.paths.outputs.liboqs_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests liboqs
- name: Fetch autoconf
if: ${{ steps.paths.outputs.autoconf_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf
- name: Fetch automake
if: ${{ steps.paths.outputs.automake_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests automake
- name: Fetch libtool
if: ${{ steps.paths.outputs.libtool_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libtool
- name: Fetch gperf
if: ${{ steps.paths.outputs.gperf_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gperf
- name: Fetch libiberty
if: ${{ steps.paths.outputs.libiberty_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libiberty
- name: Fetch libsodium
if: ${{ steps.paths.outputs.libsodium_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libsodium
- name: Fetch libunwind
if: ${{ steps.paths.outputs.libunwind_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libunwind
- name: Fetch xz
if: ${{ steps.paths.outputs.xz_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xz
- name: Fetch folly
if: ${{ steps.paths.outputs.folly_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly
- name: Fetch fizz
if: ${{ steps.paths.outputs.fizz_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz
- name: Fetch mvfst
if: ${{ steps.paths.outputs.mvfst_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests mvfst
- name: Fetch wangle
if: ${{ steps.paths.outputs.wangle_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests wangle
- name: Restore ninja from cache
id: restore_ninja
if: ${{ steps.paths.outputs.ninja_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.ninja_INSTALL }}
key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install
- name: Build ninja
if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja
- name: Save ninja to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.ninja_INSTALL }}
key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install
- name: Restore cmake from cache
id: restore_cmake
if: ${{ steps.paths.outputs.cmake_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.cmake_INSTALL }}
key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install
- name: Build cmake
if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake
- name: Save cmake to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.cmake_INSTALL }}
key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install
- name: Restore c-ares from cache
id: restore_c-ares
if: ${{ steps.paths.outputs.c-ares_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.c-ares_INSTALL }}
key: ${{ steps.paths.outputs.c-ares_CACHE_KEY }}-install
- name: Build c-ares
if: ${{ steps.paths.outputs.c-ares_SOURCE && ! steps.restore_c-ares.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests c-ares
- name: Save c-ares to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.c-ares_SOURCE && ! steps.restore_c-ares.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.c-ares_INSTALL }}
key: ${{ steps.paths.outputs.c-ares_CACHE_KEY }}-install
- name: Restore zlib from cache
id: restore_zlib
if: ${{ steps.paths.outputs.zlib_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.zlib_INSTALL }}
key: ${{ steps.paths.outputs.zlib_CACHE_KEY }}-install
- name: Build zlib
if: ${{ steps.paths.outputs.zlib_SOURCE && ! steps.restore_zlib.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zlib
- name: Save zlib to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.zlib_SOURCE && ! steps.restore_zlib.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.zlib_INSTALL }}
key: ${{ steps.paths.outputs.zlib_CACHE_KEY }}-install
- name: Restore zstd from cache
id: restore_zstd
if: ${{ steps.paths.outputs.zstd_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.zstd_INSTALL }}
key: ${{ steps.paths.outputs.zstd_CACHE_KEY }}-install
- name: Build zstd
if: ${{ steps.paths.outputs.zstd_SOURCE && ! steps.restore_zstd.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd
- name: Save zstd to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.zstd_SOURCE && ! steps.restore_zstd.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.zstd_INSTALL }}
key: ${{ steps.paths.outputs.zstd_CACHE_KEY }}-install
- name: Restore boost from cache
id: restore_boost
if: ${{ steps.paths.outputs.boost_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.boost_INSTALL }}
key: ${{ steps.paths.outputs.boost_CACHE_KEY }}-install
- name: Build boost
if: ${{ steps.paths.outputs.boost_SOURCE && ! steps.restore_boost.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost
- name: Save boost to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.boost_SOURCE && ! steps.restore_boost.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.boost_INSTALL }}
key: ${{ steps.paths.outputs.boost_CACHE_KEY }}-install
- name: Restore double-conversion from cache
id: restore_double-conversion
if: ${{ steps.paths.outputs.double-conversion_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.double-conversion_INSTALL }}
key: ${{ steps.paths.outputs.double-conversion_CACHE_KEY }}-install
- name: Build double-conversion
if: ${{ steps.paths.outputs.double-conversion_SOURCE && ! steps.restore_double-conversion.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion
- name: Save double-conversion to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.double-conversion_SOURCE && ! steps.restore_double-conversion.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.double-conversion_INSTALL }}
key: ${{ steps.paths.outputs.double-conversion_CACHE_KEY }}-install
- name: Restore fast_float from cache
id: restore_fast_float
if: ${{ steps.paths.outputs.fast_float_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.fast_float_INSTALL }}
key: ${{ steps.paths.outputs.fast_float_CACHE_KEY }}-install
- name: Build fast_float
if: ${{ steps.paths.outputs.fast_float_SOURCE && ! steps.restore_fast_float.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fast_float
- name: Save fast_float to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.fast_float_SOURCE && ! steps.restore_fast_float.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.fast_float_INSTALL }}
key: ${{ steps.paths.outputs.fast_float_CACHE_KEY }}-install
- name: Restore fmt from cache
id: restore_fmt
if: ${{ steps.paths.outputs.fmt_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.fmt_INSTALL }}
key: ${{ steps.paths.outputs.fmt_CACHE_KEY }}-install
- name: Build fmt
if: ${{ steps.paths.outputs.fmt_SOURCE && ! steps.restore_fmt.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt
- name: Save fmt to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.fmt_SOURCE && ! steps.restore_fmt.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.fmt_INSTALL }}
key: ${{ steps.paths.outputs.fmt_CACHE_KEY }}-install
- name: Restore gflags from cache
id: restore_gflags
if: ${{ steps.paths.outputs.gflags_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.gflags_INSTALL }}
key: ${{ steps.paths.outputs.gflags_CACHE_KEY }}-install
- name: Build gflags
if: ${{ steps.paths.outputs.gflags_SOURCE && ! steps.restore_gflags.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags
- name: Save gflags to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.gflags_SOURCE && ! steps.restore_gflags.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.gflags_INSTALL }}
key: ${{ steps.paths.outputs.gflags_CACHE_KEY }}-install
- name: Restore glog from cache
id: restore_glog
if: ${{ steps.paths.outputs.glog_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.glog_INSTALL }}
key: ${{ steps.paths.outputs.glog_CACHE_KEY }}-install
- name: Build glog
if: ${{ steps.paths.outputs.glog_SOURCE && ! steps.restore_glog.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog
- name: Save glog to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.glog_SOURCE && ! steps.restore_glog.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.glog_INSTALL }}
key: ${{ steps.paths.outputs.glog_CACHE_KEY }}-install
- name: Restore googletest from cache
id: restore_googletest
if: ${{ steps.paths.outputs.googletest_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.googletest_INSTALL }}
key: ${{ steps.paths.outputs.googletest_CACHE_KEY }}-install
- name: Build googletest
if: ${{ steps.paths.outputs.googletest_SOURCE && ! steps.restore_googletest.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest
- name: Save googletest to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.googletest_SOURCE && ! steps.restore_googletest.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.googletest_INSTALL }}
key: ${{ steps.paths.outputs.googletest_CACHE_KEY }}-install
- name: Restore libaio from cache
id: restore_libaio
if: ${{ steps.paths.outputs.libaio_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.libaio_INSTALL }}
key: ${{ steps.paths.outputs.libaio_CACHE_KEY }}-install
- name: Build libaio
if: ${{ steps.paths.outputs.libaio_SOURCE && ! steps.restore_libaio.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libaio
- name: Save libaio to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.libaio_SOURCE && ! steps.restore_libaio.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.libaio_INSTALL }}
key: ${{ steps.paths.outputs.libaio_CACHE_KEY }}-install
- name: Restore libdwarf from cache
id: restore_libdwarf
if: ${{ steps.paths.outputs.libdwarf_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.libdwarf_INSTALL }}
key: ${{ steps.paths.outputs.libdwarf_CACHE_KEY }}-install
- name: Build libdwarf
if: ${{ steps.paths.outputs.libdwarf_SOURCE && ! steps.restore_libdwarf.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libdwarf
- name: Save libdwarf to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.libdwarf_SOURCE && ! steps.restore_libdwarf.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.libdwarf_INSTALL }}
key: ${{ steps.paths.outputs.libdwarf_CACHE_KEY }}-install
- name: Restore libevent from cache
id: restore_libevent
if: ${{ steps.paths.outputs.libevent_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.libevent_INSTALL }}
key: ${{ steps.paths.outputs.libevent_CACHE_KEY }}-install
- name: Build libevent
if: ${{ steps.paths.outputs.libevent_SOURCE && ! steps.restore_libevent.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent
- name: Save libevent to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.libevent_SOURCE && ! steps.restore_libevent.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.libevent_INSTALL }}
key: ${{ steps.paths.outputs.libevent_CACHE_KEY }}-install
- name: Restore lz4 from cache
id: restore_lz4
if: ${{ steps.paths.outputs.lz4_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.lz4_INSTALL }}
key: ${{ steps.paths.outputs.lz4_CACHE_KEY }}-install
- name: Build lz4
if: ${{ steps.paths.outputs.lz4_SOURCE && ! steps.restore_lz4.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests lz4
- name: Save lz4 to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.lz4_SOURCE && ! steps.restore_lz4.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.lz4_INSTALL }}
key: ${{ steps.paths.outputs.lz4_CACHE_KEY }}-install
- name: Restore snappy from cache
id: restore_snappy
if: ${{ steps.paths.outputs.snappy_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.snappy_INSTALL }}
key: ${{ steps.paths.outputs.snappy_CACHE_KEY }}-install
- name: Build snappy
if: ${{ steps.paths.outputs.snappy_SOURCE && ! steps.restore_snappy.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy
- name: Save snappy to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.snappy_SOURCE && ! steps.restore_snappy.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.snappy_INSTALL }}
key: ${{ steps.paths.outputs.snappy_CACHE_KEY }}-install
- name: Restore openssl from cache
id: restore_openssl
if: ${{ steps.paths.outputs.openssl_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.openssl_INSTALL }}
key: ${{ steps.paths.outputs.openssl_CACHE_KEY }}-install
- name: Build openssl
if: ${{ steps.paths.outputs.openssl_SOURCE && ! steps.restore_openssl.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests openssl
- name: Save openssl to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.openssl_SOURCE && ! steps.restore_openssl.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.openssl_INSTALL }}
key: ${{ steps.paths.outputs.openssl_CACHE_KEY }}-install
- name: Restore liboqs from cache
id: restore_liboqs
if: ${{ steps.paths.outputs.liboqs_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.liboqs_INSTALL }}
key: ${{ steps.paths.outputs.liboqs_CACHE_KEY }}-install
- name: Build liboqs
if: ${{ steps.paths.outputs.liboqs_SOURCE && ! steps.restore_liboqs.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests liboqs
- name: Save liboqs to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.liboqs_SOURCE && ! steps.restore_liboqs.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.liboqs_INSTALL }}
key: ${{ steps.paths.outputs.liboqs_CACHE_KEY }}-install
- name: Restore autoconf from cache
id: restore_autoconf
if: ${{ steps.paths.outputs.autoconf_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.autoconf_INSTALL }}
key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install
- name: Build autoconf
if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf
- name: Save autoconf to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.autoconf_INSTALL }}
key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install
- name: Restore automake from cache
id: restore_automake
if: ${{ steps.paths.outputs.automake_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.automake_INSTALL }}
key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install
- name: Build automake
if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests automake
- name: Save automake to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.automake_INSTALL }}
key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install
- name: Restore libtool from cache
id: restore_libtool
if: ${{ steps.paths.outputs.libtool_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.libtool_INSTALL }}
key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install
- name: Build libtool
if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libtool
- name: Save libtool to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.libtool_INSTALL }}
key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install
- name: Restore gperf from cache
id: restore_gperf
if: ${{ steps.paths.outputs.gperf_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.gperf_INSTALL }}
key: ${{ steps.paths.outputs.gperf_CACHE_KEY }}-install
- name: Build gperf
if: ${{ steps.paths.outputs.gperf_SOURCE && ! steps.restore_gperf.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gperf
- name: Save gperf to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.gperf_SOURCE && ! steps.restore_gperf.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.gperf_INSTALL }}
key: ${{ steps.paths.outputs.gperf_CACHE_KEY }}-install
- name: Restore libiberty from cache
id: restore_libiberty
if: ${{ steps.paths.outputs.libiberty_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.libiberty_INSTALL }}
key: ${{ steps.paths.outputs.libiberty_CACHE_KEY }}-install
- name: Build libiberty
if: ${{ steps.paths.outputs.libiberty_SOURCE && ! steps.restore_libiberty.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libiberty
- name: Save libiberty to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.libiberty_SOURCE && ! steps.restore_libiberty.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.libiberty_INSTALL }}
key: ${{ steps.paths.outputs.libiberty_CACHE_KEY }}-install
- name: Restore libsodium from cache
id: restore_libsodium
if: ${{ steps.paths.outputs.libsodium_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.libsodium_INSTALL }}
key: ${{ steps.paths.outputs.libsodium_CACHE_KEY }}-install
- name: Build libsodium
if: ${{ steps.paths.outputs.libsodium_SOURCE && ! steps.restore_libsodium.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libsodium
- name: Save libsodium to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.libsodium_SOURCE && ! steps.restore_libsodium.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.libsodium_INSTALL }}
key: ${{ steps.paths.outputs.libsodium_CACHE_KEY }}-install
- name: Restore libunwind from cache
id: restore_libunwind
if: ${{ steps.paths.outputs.libunwind_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.libunwind_INSTALL }}
key: ${{ steps.paths.outputs.libunwind_CACHE_KEY }}-install
- name: Build libunwind
if: ${{ steps.paths.outputs.libunwind_SOURCE && ! steps.restore_libunwind.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libunwind
- name: Save libunwind to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.libunwind_SOURCE && ! steps.restore_libunwind.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.libunwind_INSTALL }}
key: ${{ steps.paths.outputs.libunwind_CACHE_KEY }}-install
- name: Restore xz from cache
id: restore_xz
if: ${{ steps.paths.outputs.xz_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.xz_INSTALL }}
key: ${{ steps.paths.outputs.xz_CACHE_KEY }}-install
- name: Build xz
if: ${{ steps.paths.outputs.xz_SOURCE && ! steps.restore_xz.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests xz
- name: Save xz to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.xz_SOURCE && ! steps.restore_xz.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.xz_INSTALL }}
key: ${{ steps.paths.outputs.xz_CACHE_KEY }}-install
- name: Restore folly from cache
id: restore_folly
if: ${{ steps.paths.outputs.folly_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.folly_INSTALL }}
key: ${{ steps.paths.outputs.folly_CACHE_KEY }}-install
- name: Build folly
if: ${{ steps.paths.outputs.folly_SOURCE && ! steps.restore_folly.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests folly
- name: Save folly to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.folly_SOURCE && ! steps.restore_folly.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.folly_INSTALL }}
key: ${{ steps.paths.outputs.folly_CACHE_KEY }}-install
- name: Restore fizz from cache
id: restore_fizz
if: ${{ steps.paths.outputs.fizz_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.fizz_INSTALL }}
key: ${{ steps.paths.outputs.fizz_CACHE_KEY }}-install
- name: Build fizz
if: ${{ steps.paths.outputs.fizz_SOURCE && ! steps.restore_fizz.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fizz
- name: Save fizz to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.fizz_SOURCE && ! steps.restore_fizz.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.fizz_INSTALL }}
key: ${{ steps.paths.outputs.fizz_CACHE_KEY }}-install
- name: Restore mvfst from cache
id: restore_mvfst
if: ${{ steps.paths.outputs.mvfst_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.mvfst_INSTALL }}
key: ${{ steps.paths.outputs.mvfst_CACHE_KEY }}-install
- name: Build mvfst
if: ${{ steps.paths.outputs.mvfst_SOURCE && ! steps.restore_mvfst.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests mvfst
- name: Save mvfst to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.mvfst_SOURCE && ! steps.restore_mvfst.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.mvfst_INSTALL }}
key: ${{ steps.paths.outputs.mvfst_CACHE_KEY }}-install
- name: Restore wangle from cache
id: restore_wangle
if: ${{ steps.paths.outputs.wangle_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.wangle_INSTALL }}
key: ${{ steps.paths.outputs.wangle_CACHE_KEY }}-install
- name: Build wangle
if: ${{ steps.paths.outputs.wangle_SOURCE && ! steps.restore_wangle.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests wangle
- name: Save wangle to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.wangle_SOURCE && ! steps.restore_wangle.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.wangle_INSTALL }}
key: ${{ steps.paths.outputs.wangle_CACHE_KEY }}-install
- name: Build proxygen
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local
- name: Copy artifacts
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --strip --src-dir=. proxygen _artifacts/linux --project-install-prefix proxygen:/usr/local --final-install-prefix /usr/local
- uses: actions/upload-artifact@v6
with:
name: proxygen
path: _artifacts
- name: Test proxygen
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local
================================================
FILE: .github/workflows/getdeps_mac.yml
================================================
# This file was @generated by getdeps.py
name: mac
on:
push:
branches:
- main
pull_request:
branches:
- main
permissions:
contents: read # to fetch code (actions/checkout)
jobs:
build:
runs-on: macOS-latest
steps:
- uses: actions/checkout@v6
- name: Install system deps
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps --recursive proxygen
- id: paths
name: Query paths
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages query-paths --recursive --src-dir=. proxygen >> "$GITHUB_OUTPUT"
- name: Fetch ninja
if: ${{ steps.paths.outputs.ninja_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests ninja
- name: Fetch cmake
if: ${{ steps.paths.outputs.cmake_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests cmake
- name: Fetch c-ares
if: ${{ steps.paths.outputs.c-ares_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests c-ares
- name: Fetch zlib
if: ${{ steps.paths.outputs.zlib_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zlib
- name: Fetch zstd
if: ${{ steps.paths.outputs.zstd_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests zstd
- name: Fetch boost
if: ${{ steps.paths.outputs.boost_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests boost
- name: Fetch double-conversion
if: ${{ steps.paths.outputs.double-conversion_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests double-conversion
- name: Fetch fast_float
if: ${{ steps.paths.outputs.fast_float_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fast_float
- name: Fetch fmt
if: ${{ steps.paths.outputs.fmt_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fmt
- name: Fetch gflags
if: ${{ steps.paths.outputs.gflags_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gflags
- name: Fetch glog
if: ${{ steps.paths.outputs.glog_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests glog
- name: Fetch googletest
if: ${{ steps.paths.outputs.googletest_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests googletest
- name: Fetch libdwarf
if: ${{ steps.paths.outputs.libdwarf_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libdwarf
- name: Fetch lz4
if: ${{ steps.paths.outputs.lz4_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests lz4
- name: Fetch openssl
if: ${{ steps.paths.outputs.openssl_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests openssl
- name: Fetch snappy
if: ${{ steps.paths.outputs.snappy_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests snappy
- name: Fetch libevent
if: ${{ steps.paths.outputs.libevent_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libevent
- name: Fetch liboqs
if: ${{ steps.paths.outputs.liboqs_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests liboqs
- name: Fetch autoconf
if: ${{ steps.paths.outputs.autoconf_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests autoconf
- name: Fetch automake
if: ${{ steps.paths.outputs.automake_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests automake
- name: Fetch libtool
if: ${{ steps.paths.outputs.libtool_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libtool
- name: Fetch gperf
if: ${{ steps.paths.outputs.gperf_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests gperf
- name: Fetch libsodium
if: ${{ steps.paths.outputs.libsodium_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests libsodium
- name: Fetch xz
if: ${{ steps.paths.outputs.xz_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests xz
- name: Fetch folly
if: ${{ steps.paths.outputs.folly_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests folly
- name: Fetch fizz
if: ${{ steps.paths.outputs.fizz_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests fizz
- name: Fetch mvfst
if: ${{ steps.paths.outputs.mvfst_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests mvfst
- name: Fetch wangle
if: ${{ steps.paths.outputs.wangle_SOURCE }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fetch --no-tests wangle
- name: Restore ninja from cache
id: restore_ninja
if: ${{ steps.paths.outputs.ninja_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.ninja_INSTALL }}
key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install
- name: Build ninja
if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests ninja
- name: Save ninja to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.ninja_SOURCE && ! steps.restore_ninja.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.ninja_INSTALL }}
key: ${{ steps.paths.outputs.ninja_CACHE_KEY }}-install
- name: Restore cmake from cache
id: restore_cmake
if: ${{ steps.paths.outputs.cmake_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.cmake_INSTALL }}
key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install
- name: Build cmake
if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests cmake
- name: Save cmake to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.cmake_SOURCE && ! steps.restore_cmake.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.cmake_INSTALL }}
key: ${{ steps.paths.outputs.cmake_CACHE_KEY }}-install
- name: Restore c-ares from cache
id: restore_c-ares
if: ${{ steps.paths.outputs.c-ares_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.c-ares_INSTALL }}
key: ${{ steps.paths.outputs.c-ares_CACHE_KEY }}-install
- name: Build c-ares
if: ${{ steps.paths.outputs.c-ares_SOURCE && ! steps.restore_c-ares.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests c-ares
- name: Save c-ares to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.c-ares_SOURCE && ! steps.restore_c-ares.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.c-ares_INSTALL }}
key: ${{ steps.paths.outputs.c-ares_CACHE_KEY }}-install
- name: Restore zlib from cache
id: restore_zlib
if: ${{ steps.paths.outputs.zlib_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.zlib_INSTALL }}
key: ${{ steps.paths.outputs.zlib_CACHE_KEY }}-install
- name: Build zlib
if: ${{ steps.paths.outputs.zlib_SOURCE && ! steps.restore_zlib.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zlib
- name: Save zlib to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.zlib_SOURCE && ! steps.restore_zlib.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.zlib_INSTALL }}
key: ${{ steps.paths.outputs.zlib_CACHE_KEY }}-install
- name: Restore zstd from cache
id: restore_zstd
if: ${{ steps.paths.outputs.zstd_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.zstd_INSTALL }}
key: ${{ steps.paths.outputs.zstd_CACHE_KEY }}-install
- name: Build zstd
if: ${{ steps.paths.outputs.zstd_SOURCE && ! steps.restore_zstd.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests zstd
- name: Save zstd to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.zstd_SOURCE && ! steps.restore_zstd.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.zstd_INSTALL }}
key: ${{ steps.paths.outputs.zstd_CACHE_KEY }}-install
- name: Restore boost from cache
id: restore_boost
if: ${{ steps.paths.outputs.boost_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.boost_INSTALL }}
key: ${{ steps.paths.outputs.boost_CACHE_KEY }}-install
- name: Build boost
if: ${{ steps.paths.outputs.boost_SOURCE && ! steps.restore_boost.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests boost
- name: Save boost to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.boost_SOURCE && ! steps.restore_boost.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.boost_INSTALL }}
key: ${{ steps.paths.outputs.boost_CACHE_KEY }}-install
- name: Restore double-conversion from cache
id: restore_double-conversion
if: ${{ steps.paths.outputs.double-conversion_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.double-conversion_INSTALL }}
key: ${{ steps.paths.outputs.double-conversion_CACHE_KEY }}-install
- name: Build double-conversion
if: ${{ steps.paths.outputs.double-conversion_SOURCE && ! steps.restore_double-conversion.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests double-conversion
- name: Save double-conversion to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.double-conversion_SOURCE && ! steps.restore_double-conversion.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.double-conversion_INSTALL }}
key: ${{ steps.paths.outputs.double-conversion_CACHE_KEY }}-install
- name: Restore fast_float from cache
id: restore_fast_float
if: ${{ steps.paths.outputs.fast_float_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.fast_float_INSTALL }}
key: ${{ steps.paths.outputs.fast_float_CACHE_KEY }}-install
- name: Build fast_float
if: ${{ steps.paths.outputs.fast_float_SOURCE && ! steps.restore_fast_float.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fast_float
- name: Save fast_float to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.fast_float_SOURCE && ! steps.restore_fast_float.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.fast_float_INSTALL }}
key: ${{ steps.paths.outputs.fast_float_CACHE_KEY }}-install
- name: Restore fmt from cache
id: restore_fmt
if: ${{ steps.paths.outputs.fmt_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.fmt_INSTALL }}
key: ${{ steps.paths.outputs.fmt_CACHE_KEY }}-install
- name: Build fmt
if: ${{ steps.paths.outputs.fmt_SOURCE && ! steps.restore_fmt.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fmt
- name: Save fmt to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.fmt_SOURCE && ! steps.restore_fmt.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.fmt_INSTALL }}
key: ${{ steps.paths.outputs.fmt_CACHE_KEY }}-install
- name: Restore gflags from cache
id: restore_gflags
if: ${{ steps.paths.outputs.gflags_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.gflags_INSTALL }}
key: ${{ steps.paths.outputs.gflags_CACHE_KEY }}-install
- name: Build gflags
if: ${{ steps.paths.outputs.gflags_SOURCE && ! steps.restore_gflags.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gflags
- name: Save gflags to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.gflags_SOURCE && ! steps.restore_gflags.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.gflags_INSTALL }}
key: ${{ steps.paths.outputs.gflags_CACHE_KEY }}-install
- name: Restore glog from cache
id: restore_glog
if: ${{ steps.paths.outputs.glog_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.glog_INSTALL }}
key: ${{ steps.paths.outputs.glog_CACHE_KEY }}-install
- name: Build glog
if: ${{ steps.paths.outputs.glog_SOURCE && ! steps.restore_glog.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests glog
- name: Save glog to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.glog_SOURCE && ! steps.restore_glog.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.glog_INSTALL }}
key: ${{ steps.paths.outputs.glog_CACHE_KEY }}-install
- name: Restore googletest from cache
id: restore_googletest
if: ${{ steps.paths.outputs.googletest_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.googletest_INSTALL }}
key: ${{ steps.paths.outputs.googletest_CACHE_KEY }}-install
- name: Build googletest
if: ${{ steps.paths.outputs.googletest_SOURCE && ! steps.restore_googletest.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests googletest
- name: Save googletest to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.googletest_SOURCE && ! steps.restore_googletest.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.googletest_INSTALL }}
key: ${{ steps.paths.outputs.googletest_CACHE_KEY }}-install
- name: Restore libdwarf from cache
id: restore_libdwarf
if: ${{ steps.paths.outputs.libdwarf_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.libdwarf_INSTALL }}
key: ${{ steps.paths.outputs.libdwarf_CACHE_KEY }}-install
- name: Build libdwarf
if: ${{ steps.paths.outputs.libdwarf_SOURCE && ! steps.restore_libdwarf.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libdwarf
- name: Save libdwarf to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.libdwarf_SOURCE && ! steps.restore_libdwarf.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.libdwarf_INSTALL }}
key: ${{ steps.paths.outputs.libdwarf_CACHE_KEY }}-install
- name: Restore lz4 from cache
id: restore_lz4
if: ${{ steps.paths.outputs.lz4_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.lz4_INSTALL }}
key: ${{ steps.paths.outputs.lz4_CACHE_KEY }}-install
- name: Build lz4
if: ${{ steps.paths.outputs.lz4_SOURCE && ! steps.restore_lz4.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests lz4
- name: Save lz4 to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.lz4_SOURCE && ! steps.restore_lz4.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.lz4_INSTALL }}
key: ${{ steps.paths.outputs.lz4_CACHE_KEY }}-install
- name: Restore openssl from cache
id: restore_openssl
if: ${{ steps.paths.outputs.openssl_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.openssl_INSTALL }}
key: ${{ steps.paths.outputs.openssl_CACHE_KEY }}-install
- name: Build openssl
if: ${{ steps.paths.outputs.openssl_SOURCE && ! steps.restore_openssl.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests openssl
- name: Save openssl to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.openssl_SOURCE && ! steps.restore_openssl.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.openssl_INSTALL }}
key: ${{ steps.paths.outputs.openssl_CACHE_KEY }}-install
- name: Restore snappy from cache
id: restore_snappy
if: ${{ steps.paths.outputs.snappy_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.snappy_INSTALL }}
key: ${{ steps.paths.outputs.snappy_CACHE_KEY }}-install
- name: Build snappy
if: ${{ steps.paths.outputs.snappy_SOURCE && ! steps.restore_snappy.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests snappy
- name: Save snappy to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.snappy_SOURCE && ! steps.restore_snappy.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.snappy_INSTALL }}
key: ${{ steps.paths.outputs.snappy_CACHE_KEY }}-install
- name: Restore libevent from cache
id: restore_libevent
if: ${{ steps.paths.outputs.libevent_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.libevent_INSTALL }}
key: ${{ steps.paths.outputs.libevent_CACHE_KEY }}-install
- name: Build libevent
if: ${{ steps.paths.outputs.libevent_SOURCE && ! steps.restore_libevent.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libevent
- name: Save libevent to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.libevent_SOURCE && ! steps.restore_libevent.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.libevent_INSTALL }}
key: ${{ steps.paths.outputs.libevent_CACHE_KEY }}-install
- name: Restore liboqs from cache
id: restore_liboqs
if: ${{ steps.paths.outputs.liboqs_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.liboqs_INSTALL }}
key: ${{ steps.paths.outputs.liboqs_CACHE_KEY }}-install
- name: Build liboqs
if: ${{ steps.paths.outputs.liboqs_SOURCE && ! steps.restore_liboqs.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests liboqs
- name: Save liboqs to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.liboqs_SOURCE && ! steps.restore_liboqs.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.liboqs_INSTALL }}
key: ${{ steps.paths.outputs.liboqs_CACHE_KEY }}-install
- name: Restore autoconf from cache
id: restore_autoconf
if: ${{ steps.paths.outputs.autoconf_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.autoconf_INSTALL }}
key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install
- name: Build autoconf
if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests autoconf
- name: Save autoconf to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.autoconf_SOURCE && ! steps.restore_autoconf.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.autoconf_INSTALL }}
key: ${{ steps.paths.outputs.autoconf_CACHE_KEY }}-install
- name: Restore automake from cache
id: restore_automake
if: ${{ steps.paths.outputs.automake_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.automake_INSTALL }}
key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install
- name: Build automake
if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests automake
- name: Save automake to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.automake_SOURCE && ! steps.restore_automake.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.automake_INSTALL }}
key: ${{ steps.paths.outputs.automake_CACHE_KEY }}-install
- name: Restore libtool from cache
id: restore_libtool
if: ${{ steps.paths.outputs.libtool_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.libtool_INSTALL }}
key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install
- name: Build libtool
if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libtool
- name: Save libtool to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.libtool_SOURCE && ! steps.restore_libtool.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.libtool_INSTALL }}
key: ${{ steps.paths.outputs.libtool_CACHE_KEY }}-install
- name: Restore gperf from cache
id: restore_gperf
if: ${{ steps.paths.outputs.gperf_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.gperf_INSTALL }}
key: ${{ steps.paths.outputs.gperf_CACHE_KEY }}-install
- name: Build gperf
if: ${{ steps.paths.outputs.gperf_SOURCE && ! steps.restore_gperf.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests gperf
- name: Save gperf to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.gperf_SOURCE && ! steps.restore_gperf.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.gperf_INSTALL }}
key: ${{ steps.paths.outputs.gperf_CACHE_KEY }}-install
- name: Restore libsodium from cache
id: restore_libsodium
if: ${{ steps.paths.outputs.libsodium_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.libsodium_INSTALL }}
key: ${{ steps.paths.outputs.libsodium_CACHE_KEY }}-install
- name: Build libsodium
if: ${{ steps.paths.outputs.libsodium_SOURCE && ! steps.restore_libsodium.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests libsodium
- name: Save libsodium to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.libsodium_SOURCE && ! steps.restore_libsodium.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.libsodium_INSTALL }}
key: ${{ steps.paths.outputs.libsodium_CACHE_KEY }}-install
- name: Restore xz from cache
id: restore_xz
if: ${{ steps.paths.outputs.xz_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.xz_INSTALL }}
key: ${{ steps.paths.outputs.xz_CACHE_KEY }}-install
- name: Build xz
if: ${{ steps.paths.outputs.xz_SOURCE && ! steps.restore_xz.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests xz
- name: Save xz to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.xz_SOURCE && ! steps.restore_xz.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.xz_INSTALL }}
key: ${{ steps.paths.outputs.xz_CACHE_KEY }}-install
- name: Restore folly from cache
id: restore_folly
if: ${{ steps.paths.outputs.folly_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.folly_INSTALL }}
key: ${{ steps.paths.outputs.folly_CACHE_KEY }}-install
- name: Build folly
if: ${{ steps.paths.outputs.folly_SOURCE && ! steps.restore_folly.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests folly
- name: Save folly to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.folly_SOURCE && ! steps.restore_folly.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.folly_INSTALL }}
key: ${{ steps.paths.outputs.folly_CACHE_KEY }}-install
- name: Restore fizz from cache
id: restore_fizz
if: ${{ steps.paths.outputs.fizz_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.fizz_INSTALL }}
key: ${{ steps.paths.outputs.fizz_CACHE_KEY }}-install
- name: Build fizz
if: ${{ steps.paths.outputs.fizz_SOURCE && ! steps.restore_fizz.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests fizz
- name: Save fizz to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.fizz_SOURCE && ! steps.restore_fizz.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.fizz_INSTALL }}
key: ${{ steps.paths.outputs.fizz_CACHE_KEY }}-install
- name: Restore mvfst from cache
id: restore_mvfst
if: ${{ steps.paths.outputs.mvfst_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.mvfst_INSTALL }}
key: ${{ steps.paths.outputs.mvfst_CACHE_KEY }}-install
- name: Build mvfst
if: ${{ steps.paths.outputs.mvfst_SOURCE && ! steps.restore_mvfst.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests mvfst
- name: Save mvfst to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.mvfst_SOURCE && ! steps.restore_mvfst.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.mvfst_INSTALL }}
key: ${{ steps.paths.outputs.mvfst_CACHE_KEY }}-install
- name: Restore wangle from cache
id: restore_wangle
if: ${{ steps.paths.outputs.wangle_SOURCE }}
uses: actions/cache/restore@v4
with:
path: ${{ steps.paths.outputs.wangle_INSTALL }}
key: ${{ steps.paths.outputs.wangle_CACHE_KEY }}-install
- name: Build wangle
if: ${{ steps.paths.outputs.wangle_SOURCE && ! steps.restore_wangle.outputs.cache-hit }}
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --no-tests wangle
- name: Save wangle to cache
uses: actions/cache/save@v4
if: ${{ steps.paths.outputs.wangle_SOURCE && ! steps.restore_wangle.outputs.cache-hit }}
with:
path: ${{ steps.paths.outputs.wangle_INSTALL }}
key: ${{ steps.paths.outputs.wangle_CACHE_KEY }}-install
- name: Build proxygen
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages build --src-dir=. proxygen --project-install-prefix proxygen:/usr/local
- name: Copy artifacts
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages fixup-dyn-deps --src-dir=. proxygen _artifacts/mac --project-install-prefix proxygen:/usr/local --final-install-prefix /usr/local
- uses: actions/upload-artifact@v6
with:
name: proxygen
path: _artifacts
- name: Test proxygen
run: python3 build/fbcode_builder/getdeps.py --allow-system-packages test --src-dir=. proxygen --project-install-prefix proxygen:/usr/local
================================================
FILE: .github/workflows/publish_mvfst_interop.yml
================================================
# Following instruction from https://docs.github.com/en/actions/publishing-packages/publishing-docker-images
name: Publish mvfst interop image
on:
push:
tags:
# Build a new image weekly with each TagIt release
- 'v20*'
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}/mvfst-interop
jobs:
build-and-push-image:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v5
- name: Log in to the Container registry
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Build and push Docker image
uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0
with:
context: .
file: proxygen/httpserver/samples/hq/quic-interop/Dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
================================================
FILE: .gitignore
================================================
*~
\#*\#
# Build artifacts
.libs
*.log
*.trs
*.o
*.lo
*.a
*.la
gen-cpp
gen-cpp2
# Generated files
/proxygen/lib/http/HTTPCommonHeaders.cpp
/proxygen/lib/http/HTTPCommonHeaders.h
/proxygen/lib/utils/TraceEventType.cpp
/proxygen/lib/utils/TraceEventType.h
/proxygen/lib/utils/TraceFieldType.cpp
/proxygen/lib/utils/TraceFieldType.h
/proxygen/lib/stats/StatsWrapper.h
_build/
# common editor artifacts
*~
*.vscode*
# Generated document
html/
latex/
================================================
FILE: CMakeLists.txt
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
cmake_minimum_required(VERSION 3.10)
include(CheckCXXCompilerFlag)
project(
proxygen
)
# Set for FetchContent to skip find_package(proxygen)
set(proxygen_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}"
CACHE INTERNAL "proxygen source directory")
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
set(CMAKE_MODULE_PATH
"${CMAKE_CURRENT_SOURCE_DIR}/cmake"
# for in-fbsource builds
"${CMAKE_CURRENT_SOURCE_DIR}/../opensource/fbcode_builder/CMake"
# For shipit-transformed builds
"${CMAKE_CURRENT_SOURCE_DIR}/build/fbcode_builder/CMake"
${CMAKE_MODULE_PATH})
option(BUILD_SHARED_LIBS
"If enabled, build proxygen as a shared library. \
This is generally discouraged, since proxygen does not commit to having \
a stable ABI."
OFF
)
option(BUILD_SAMPLES
"If enabled, proxygen will build various examples/samples"
ON
)
# Mark BUILD_SHARED_LIBS as an "advanced" option, since enabling it
# is generally discouraged.
mark_as_advanced(BUILD_SHARED_LIBS)
include(FBBuildOptions)
fb_activate_static_library_option()
# PROXYGEN_FBCODE_ROOT is where the top level proxygen/ directory resides, so
# an #include will resolve to
# $PROXYGEN_FBCODE_ROOT/proxygen/path/to/file on disk
set(PROXYGEN_FBCODE_ROOT ${CMAKE_CURRENT_SOURCE_DIR})
# Similarly, PROXYGEN_GENERATED_ROOT is where the top level proxygen/ directory
# resides for generated files, so a #include
# will be at $PROXYGEN_GENERATED_ROOT/proxygen/path/to/generated/file
set(PROXYGEN_GENERATED_ROOT ${CMAKE_CURRENT_BINARY_DIR}/generated)
file(MAKE_DIRECTORY ${PROXYGEN_GENERATED_ROOT})
# Build-time program requirements.
if(WIN32)
find_program(PROXYGEN_PYTHON python)
else()
find_program(PROXYGEN_PYTHON python3)
endif()
if(NOT PROXYGEN_PYTHON)
message(FATAL_ERROR "python is required for the proxygen build")
endif()
find_program(PROXYGEN_GPERF gperf)
if(NOT PROXYGEN_GPERF)
message(FATAL_ERROR "gperf is required for the proxygen build")
endif()
# Dependencies
#
# IMPORTANT: If you change this, make the analogous update in:
# cmake/proxygen-config.cmake.in
find_package(fmt REQUIRED)
# If deps are being built from source (FetchContent), skip find_package
if (NOT DEFINED folly_SOURCE_DIR)
find_package(folly REQUIRED)
endif()
if (NOT DEFINED fizz_SOURCE_DIR)
find_package(fizz REQUIRED)
endif()
if (NOT DEFINED wangle_SOURCE_DIR)
find_package(wangle REQUIRED)
endif()
if (NOT DEFINED mvfst_SOURCE_DIR)
find_package(mvfst REQUIRED)
endif()
find_package(Zstd REQUIRED)
find_package(ZLIB REQUIRED)
find_package(OpenSSL REQUIRED)
find_package(Threads)
find_package(Cares REQUIRED)
find_package(Glog REQUIRED)
# Propagate glog's required compile definition to all proxygen targets.
# glog 0.7+ requires GLOG_USE_GLOG_EXPORT but since glog::glog is linked
# PRIVATE on the monolithic library, OBJECT libraries don't inherit it.
get_target_property(_glog_defs glog::glog INTERFACE_COMPILE_DEFINITIONS)
if(_glog_defs)
add_compile_definitions(${_glog_defs})
endif()
list(APPEND
_PROXYGEN_COMMON_COMPILE_OPTIONS
-Wall
-Wextra
)
CHECK_CXX_COMPILER_FLAG(-Wnoexcept-type COMPILER_HAS_W_NOEXCEPT_TYPE)
if (COMPILER_HAS_W_NOEXCEPT_TYPE)
list(APPEND _PROXYGEN_COMMON_COMPILE_OPTIONS -Wno-noexcept-type)
endif()
CHECK_CXX_COMPILER_FLAG(-Wunused-parameter COMPILER_HAS_W_UNUSED_PARAMETER)
if (COMPILER_HAS_W_UNUSED_PARAMETER)
list(APPEND _PROXYGEN_COMMON_COMPILE_OPTIONS -Wno-unused-parameter)
endif()
CHECK_CXX_COMPILER_FLAG(-Wmissing-field-initializers COMPILER_HAS_W_MISSING_FIELD_INITIALIZERS)
if (COMPILER_HAS_W_MISSING_FIELD_INITIALIZERS)
list(APPEND _PROXYGEN_COMMON_COMPILE_OPTIONS -Wno-missing-field-initializers)
endif()
CHECK_CXX_COMPILER_FLAG(-Wnullability-completeness COMPILER_HAS_W_NULLABILITY_COMPLETENESS)
if (COMPILER_HAS_W_NULLABILITY_COMPLETENESS)
list(APPEND _PROXYGEN_COMMON_COMPILE_OPTIONS -Wno-nullability-completeness)
endif()
CHECK_CXX_COMPILER_FLAG(-Wdeprecated-register COMPILER_HAS_W_DEPRECATED_REGISTER)
if (COMPILER_HAS_W_DEPRECATED_REGISTER)
list(APPEND _PROXYGEN_COMMON_COMPILE_OPTIONS -Wno-deprecated-register)
endif()
CHECK_CXX_COMPILER_FLAG(-Wregister COMPILER_HAS_W_REGISTER)
if (COMPILER_HAS_W_REGISTER)
list(APPEND _PROXYGEN_COMMON_COMPILE_OPTIONS -Wno-register)
endif()
CHECK_CXX_COMPILER_FLAG(-Wunused-value COMPILER_HAS_W_UNUSED_VALUE)
if (COMPILER_HAS_W_UNUSED_VALUE)
list(APPEND _PROXYGEN_COMMON_COMPILE_OPTIONS -Wno-unused-value)
endif()
SET(GFLAG_DEPENDENCIES "")
SET(PROXYGEN_EXTRA_LINK_LIBRARIES "")
SET(PROXYGEN_EXTRA_INCLUDE_DIRECTORIES "")
find_package(gflags CONFIG QUIET)
if (gflags_FOUND)
message("module path: ${CMAKE_MODULE_PATH}")
message(STATUS "Found gflags from package config")
if (TARGET gflags-shared)
list(APPEND GFLAG_DEPENDENCIES gflags-shared)
elseif (TARGET gflags)
list(APPEND GFLAG_DEPENDENCIES gflags)
else()
message(FATAL_ERROR
"Unable to determine the target name for the GFlags package.")
endif()
list(APPEND CMAKE_REQUIRED_LIBRARIES ${GFLAGS_LIBRARIES})
list(APPEND CMAKE_REQUIRED_INCLUDES ${GFLAGS_INCLUDE_DIR})
else()
find_package(Gflags REQUIRED MODULE)
list(APPEND PROXYGEN_EXTRA_LINK_LIBRARIES ${LIBGFLAGS_LIBRARY})
list(APPEND PROXYGEN_EXTRA_INCLUDE_DIRECTORIES ${LIBGFLAGS_INCLUDE_DIR})
list(APPEND CMAKE_REQUIRED_LIBRARIES ${LIBGFLAGS_LIBRARY})
list(APPEND CMAKE_REQUIRED_INCLUDES ${LIBGFLAGS_INCLUDE_DIR})
endif()
include(ProxygenTest)
include(ProxygenFunctions)
# Set PROXYGEN_DIR for proxygen_add_library to compute target names
set(PROXYGEN_DIR ${CMAKE_CURRENT_SOURCE_DIR}/proxygen)
add_subdirectory(proxygen)
# Resolve deferred dependencies for granular libraries
# (monolithic proxygen target is manually maintained in proxygen/lib/CMakeLists.txt)
proxygen_resolve_deferred_dependencies()
if (NOT DEFINED LIB_INSTALL_DIR)
set(LIB_INSTALL_DIR "lib")
endif()
if (NOT DEFINED INCLUDE_INSTALL_DIR)
set(INCLUDE_INSTALL_DIR "include")
endif()
if (NOT DEFINED CMAKE_INSTALL_DIR)
set(CMAKE_INSTALL_DIR "${LIB_INSTALL_DIR}/cmake/proxygen/")
endif()
install(
EXPORT proxygen-exports
FILE proxygen-targets.cmake
NAMESPACE proxygen::
DESTINATION ${CMAKE_INSTALL_DIR}
)
include(CMakePackageConfigHelpers)
configure_package_config_file(
cmake/proxygen-config.cmake.in
${CMAKE_CURRENT_BINARY_DIR}/proxygen-config.cmake
INSTALL_DESTINATION ${CMAKE_INSTALL_DIR}
)
install(
FILES ${CMAKE_CURRENT_BINARY_DIR}/proxygen-config.cmake
DESTINATION ${CMAKE_INSTALL_DIR}
)
# uninstall target
if(NOT TARGET uninstall)
configure_file(
"${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in"
"${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
IMMEDIATE @ONLY)
add_custom_target(uninstall
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake)
endif()
================================================
FILE: CODE_OF_CONDUCT.md
================================================
# Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to make participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, sex characteristics, gender identity and expression,
level of experience, education, socio-economic status, nationality, personal
appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
## Scope
This Code of Conduct applies within all project spaces, and it also applies when
an individual is representing the project or its community in public spaces.
Examples of representing a project or community include using an official
project e-mail address, posting via an official social media account, or acting
as an appointed representative at an online or offline event. Representation of
a project may be further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at . All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
[homepage]: https://www.contributor-covenant.org
For answers to common questions about this code of conduct, see
https://www.contributor-covenant.org/faq
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing to Proxygen
Here's a quick rundown of how to contribute to this project.
## Code of Conduct
The code of conduct is described in [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md)
## Our Development Process
We develop on a private branch internally at Facebook. We regularly update
this github project with the changes from the internal repo. External pull
requests are cherry-picked into our repo and then pushed back out.
## Pull Requests
We actively welcome your pull requests.
1. Fork the repo and create your branch from `main`.
1. If you've added code that should be tested, add tests
1. If you've changed APIs, update the documentation.
1. Ensure the test suite passes.
1. Make sure your code lints.
1. If you haven't already, complete the Contributor License Agreement ("CLA").
## Contributor License Agreement ("CLA")
In order to accept your pull request, we need you to submit a CLA. You
only need
to do this once to work on any of Facebook's open source projects.
Complete your CLA here:
## Issues
We use GitHub issues to track public bugs. Please ensure your description
is clear and has sufficient instructions to be able to reproduce the issue.
Facebook has a [bounty program](https://www.facebook.com/whitehat/) for
the safe disclosure of security bugs. In those cases, please go through
the process outlined on that page and do not file a public issue.
## Coding Style
* 2 spaces for indentation rather than tabs
* 80 character line length
* Use `Type* foo` not `Type *foo`.
* Align parameters passed to functions.
* Prefer `std::make_unique` to `new Foo`. In general, we discourage
use of raw `new` or `delete`.
## License
By contributing to Proxygen, you agree that your contributions will be
licensed under its BSD license.
================================================
FILE: Doxyfile
================================================
# Doxyfile 1.8.5
# This file describes the settings to be used by the documentation system
# doxygen (www.doxygen.org) for a project.
#
# All text after a double hash (##) is considered a comment and is placed in
# front of the TAG it is preceding.
#
# All text after a single hash (#) is considered a comment and will be ignored.
# The format is:
# TAG = value [value, ...]
# For lists, items can also be appended using:
# TAG += value [value, ...]
# Values that contain spaces should be placed between quotes (\" \").
#---------------------------------------------------------------------------
# Project related configuration options
#---------------------------------------------------------------------------
# This tag specifies the encoding used for all characters in the config file
# that follow. The default is UTF-8 which is also the encoding used for all text
# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
# for the list of possible encodings.
# The default value is: UTF-8.
DOXYFILE_ENCODING = UTF-8
# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
# double-quotes, unless you are using Doxywizard) that should identify the
# project for which the documentation is generated. This name is used in the
# title of most generated pages and in a few other places.
# The default value is: My Project.
PROJECT_NAME = proxygen
# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
# could be handy for archiving the generated documentation or if some version
# control system is used.
PROJECT_NUMBER =
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears at the top of each page and should give viewer a
# quick idea about the purpose of the project. Keep the description short.
PROJECT_BRIEF =
# With the PROJECT_LOGO tag one can specify an logo or icon that is included in
# the documentation. The maximum height of the logo should not exceed 55 pixels
# and the maximum width should not exceed 200 pixels. Doxygen will copy the logo
# to the output directory.
PROJECT_LOGO =
# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
# into which the generated documentation will be written. If a relative path is
# entered, it will be relative to the location where doxygen was started. If
# left blank the current directory will be used.
OUTPUT_DIRECTORY =
# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 4096 sub-
# directories (in 2 levels) under the output directory of each output format and
# will distribute the generated files over these directories. Enabling this
# option can be useful when feeding doxygen a huge amount of source files, where
# putting all generated files in the same directory would otherwise causes
# performance problems for the file system.
# The default value is: NO.
#### OPTION GENERATED IN generate_doxygen.sh
CREATE_SUBDIRS = NO
# The OUTPUT_LANGUAGE tag is used to specify the language in which all
# documentation generated by doxygen is written. Doxygen will use this
# information to generate all constant output in the proper language.
# Possible values are: Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-
# Traditional, Croatian, Czech, Danish, Dutch, English, Esperanto, Farsi,
# Finnish, French, German, Greek, Hungarian, Italian, Japanese, Japanese-en,
# Korean, Korean-en, Latvian, Norwegian, Macedonian, Persian, Polish,
# Portuguese, Romanian, Russian, Serbian, Slovak, Slovene, Spanish, Swedish,
# Turkish, Ukrainian and Vietnamese.
# The default value is: English.
OUTPUT_LANGUAGE = English
# If the BRIEF_MEMBER_DESC tag is set to YES doxygen will include brief member
# descriptions after the members that are listed in the file and class
# documentation (similar to Javadoc). Set to NO to disable this.
# The default value is: YES.
BRIEF_MEMBER_DESC = YES
# If the REPEAT_BRIEF tag is set to YES doxygen will prepend the brief
# description of a member or function before the detailed description
#
# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the
# brief descriptions will be completely suppressed.
# The default value is: YES.
REPEAT_BRIEF = YES
# This tag implements a quasi-intelligent brief description abbreviator that is
# used to form the text in various listings. Each string in this list, if found
# as the leading text of the brief description, will be stripped from the text
# and the result, after processing the whole list, is used as the annotated
# text. Otherwise, the brief description is used as-is. If left blank, the
# following values are used ($name is automatically replaced with the name of
# the entity):The $name class, The $name widget, The $name file, is, provides,
# specifies, contains, represents, a, an and the.
ABBREVIATE_BRIEF =
# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then
# doxygen will generate a detailed section even if there is only a brief
# description.
# The default value is: NO.
ALWAYS_DETAILED_SEC = NO
# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all
# inherited members of a class in the documentation of that class as if those
# members were ordinary class members. Constructors, destructors and assignment
# operators of the base classes will not be shown.
# The default value is: NO.
INLINE_INHERITED_MEMB = NO
# If the FULL_PATH_NAMES tag is set to YES doxygen will prepend the full path
# before files name in the file list and in the header files. If set to NO the
# shortest path that makes the file name unique will be used
# The default value is: YES.
FULL_PATH_NAMES = YES
# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path.
# Stripping is only done if one of the specified strings matches the left-hand
# part of the path. The tag can be used to show relative paths in the file list.
# If left blank the directory from which doxygen is run is used as the path to
# strip.
#
# Note that you can specify absolute paths here, but also relative paths, which
# will be relative from the directory where doxygen is started.
# This tag requires that the tag FULL_PATH_NAMES is set to YES.
STRIP_FROM_PATH =
# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the
# path mentioned in the documentation of a class, which tells the reader which
# header file to include in order to use a class. If left blank only the name of
# the header file containing the class definition is used. Otherwise one should
# specify the list of include paths that are normally passed to the compiler
# using the -I flag.
STRIP_FROM_INC_PATH =
# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but
# less readable) file names. This can be useful is your file systems doesn't
# support long names like on DOS, Mac, or CD-ROM.
# The default value is: NO.
SHORT_NAMES = NO
# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the
# first line (until the first dot) of a Javadoc-style comment as the brief
# description. If set to NO, the Javadoc-style will behave just like regular Qt-
# style comments (thus requiring an explicit @brief command for a brief
# description.)
# The default value is: NO.
JAVADOC_AUTOBRIEF = NO
# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
# line (until the first dot) of a Qt-style comment as the brief description. If
# set to NO, the Qt-style will behave just like regular Qt-style comments (thus
# requiring an explicit \brief command for a brief description.)
# The default value is: NO.
QT_AUTOBRIEF = NO
# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a
# multi-line C++ special comment block (i.e. a block of //! or /// comments) as
# a brief description. This used to be the default behavior. The new default is
# to treat a multi-line C++ comment block as a detailed description. Set this
# tag to YES if you prefer the old behavior instead.
#
# Note that setting this tag to YES also means that rational rose comments are
# not recognized any more.
# The default value is: NO.
MULTILINE_CPP_IS_BRIEF = NO
# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
# documentation from any documented member that it re-implements.
# The default value is: YES.
INHERIT_DOCS = YES
# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce a
# new page for each member. If set to NO, the documentation of a member will be
# part of the file/class/namespace that contains it.
# The default value is: NO.
SEPARATE_MEMBER_PAGES = NO
# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen
# uses this value to replace tabs by spaces in code fragments.
# Minimum value: 1, maximum value: 16, default value: 4.
TAB_SIZE = 2
# This tag can be used to specify a number of aliases that act as commands in
# the documentation. An alias has the form:
# name=value
# For example adding
# "sideeffect=@par Side Effects:\n"
# will allow you to put the command \sideeffect (or @sideeffect) in the
# documentation, which will result in a user-defined paragraph with heading
# "Side Effects:". You can put \n's in the value part of an alias to insert
# newlines.
ALIASES =
# This tag can be used to specify a number of word-keyword mappings (TCL only).
# A mapping has the form "name=value". For example adding "class=itcl::class"
# will allow you to use the command class in the itcl::class meaning.
TCL_SUBST =
# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
# only. Doxygen will then generate output that is more tailored for C. For
# instance, some of the names that are used will be different. The list of all
# members will be omitted, etc.
# The default value is: NO.
OPTIMIZE_OUTPUT_FOR_C = NO
# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or
# Python sources only. Doxygen will then generate output that is more tailored
# for that language. For instance, namespaces will be presented as packages,
# qualified scopes will look different, etc.
# The default value is: NO.
OPTIMIZE_OUTPUT_JAVA = NO
# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran
# sources. Doxygen will then generate output that is tailored for Fortran.
# The default value is: NO.
OPTIMIZE_FOR_FORTRAN = NO
# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL
# sources. Doxygen will then generate output that is tailored for VHDL.
# The default value is: NO.
OPTIMIZE_OUTPUT_VHDL = NO
# Doxygen selects the parser to use depending on the extension of the files it
# parses. With this tag you can assign which parser to use for a given
# extension. Doxygen has a built-in mapping, but you can override or extend it
# using this tag. The format is ext=language, where ext is a file extension, and
# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
# C#, C, C++, D, PHP, Objective-C, Python, Fortran, VHDL. For instance to make
# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C
# (default is Fortran), use: inc=Fortran f=C.
#
# Note For files without extension you can use no_extension as a placeholder.
#
# Note that for custom extensions you also need to set FILE_PATTERNS otherwise
# the files are not read by doxygen.
EXTENSION_MAPPING =
# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
# according to the Markdown format, which allows for more readable
# documentation. See http://daringfireball.net/projects/markdown/ for details.
# The output of markdown processing is further processed by doxygen, so you can
# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
# case of backward compatibilities issues.
# The default value is: YES.
MARKDOWN_SUPPORT = YES
# When enabled doxygen tries to link words that correspond to documented
# classes, or namespaces to their corresponding documentation. Such a link can
# be prevented in individual cases by by putting a % sign in front of the word
# or globally by setting AUTOLINK_SUPPORT to NO.
# The default value is: YES.
AUTOLINK_SUPPORT = YES
# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want
# to include (a tag file for) the STL sources as input, then you should set this
# tag to YES in order to let doxygen match functions declarations and
# definitions whose arguments contain STL classes (e.g. func(std::string);
# versus func(std::string) {}). This also make the inheritance and collaboration
# diagrams that involve STL classes more complete and accurate.
# The default value is: NO.
BUILTIN_STL_SUPPORT = YES
# If you use Microsoft's C++/CLI language, you should set this option to YES to
# enable parsing support.
# The default value is: NO.
CPP_CLI_SUPPORT = NO
# Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
# will parse them like normal C++ but will assume all classes use public instead
# of private inheritance when no explicit protection keyword is present.
# The default value is: NO.
SIP_SUPPORT = NO
# For Microsoft's IDL there are propget and propput attributes to indicate
# getter and setter methods for a property. Setting this option to YES will make
# doxygen to replace the get and set methods by a property in the documentation.
# This will only work if the methods are indeed getting or setting a simple
# type. If this is not the case, or you want to show the methods anyway, you
# should set this option to NO.
# The default value is: YES.
IDL_PROPERTY_SUPPORT = YES
# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC
# tag is set to YES, then doxygen will reuse the documentation of the first
# member in the group (if any) for the other members of the group. By default
# all members of a group must be documented explicitly.
# The default value is: NO.
DISTRIBUTE_GROUP_DOC = NO
# Set the SUBGROUPING tag to YES to allow class member groups of the same type
# (for instance a group of public functions) to be put as a subgroup of that
# type (e.g. under the Public Functions section). Set it to NO to prevent
# subgrouping. Alternatively, this can be done per class using the
# \nosubgrouping command.
# The default value is: YES.
SUBGROUPING = YES
# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions
# are shown inside the group in which they are included (e.g. using \ingroup)
# instead of on a separate page (for HTML and Man pages) or section (for LaTeX
# and RTF).
#
# Note that this feature does not work in combination with
# SEPARATE_MEMBER_PAGES.
# The default value is: NO.
INLINE_GROUPED_CLASSES = NO
# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions
# with only public data fields or simple typedef fields will be shown inline in
# the documentation of the scope in which they are defined (i.e. file,
# namespace, or group documentation), provided this scope is documented. If set
# to NO, structs, classes, and unions are shown on a separate page (for HTML and
# Man pages) or section (for LaTeX and RTF).
# The default value is: NO.
INLINE_SIMPLE_STRUCTS = NO
# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or
# enum is documented as struct, union, or enum with the name of the typedef. So
# typedef struct TypeS {} TypeT, will appear in the documentation as a struct
# with name TypeT. When disabled the typedef will appear as a member of a file,
# namespace, or class. And the struct will be named TypeS. This can typically be
# useful for C code in case the coding convention dictates that all compound
# types are typedef'ed and only the typedef is referenced, never the tag name.
# The default value is: NO.
TYPEDEF_HIDES_STRUCT = NO
# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This
# cache is used to resolve symbols given their name and scope. Since this can be
# an expensive process and often the same symbol appears multiple times in the
# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small
# doxygen will become slower. If the cache is too large, memory is wasted. The
# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range
# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536
# symbols. At the end of a run doxygen will report the cache usage and suggest
# the optimal cache size from a speed point of view.
# Minimum value: 0, maximum value: 9, default value: 0.
LOOKUP_CACHE_SIZE = 0
#---------------------------------------------------------------------------
# Build related configuration options
#---------------------------------------------------------------------------
# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in
# documentation are documented, even if no documentation was available. Private
# class members and static file members will be hidden unless the
# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
# Note: This will also disable the warnings about undocumented members that are
# normally produced when WARNINGS is set to YES.
# The default value is: NO.
EXTRACT_ALL = YES
# If the EXTRACT_PRIVATE tag is set to YES all private members of a class will
# be included in the documentation.
# The default value is: NO.
EXTRACT_PRIVATE = YES
# If the EXTRACT_PACKAGE tag is set to YES all members with package or internal
# scope will be included in the documentation.
# The default value is: NO.
EXTRACT_PACKAGE = NO
# If the EXTRACT_STATIC tag is set to YES all static members of a file will be
# included in the documentation.
# The default value is: NO.
EXTRACT_STATIC = YES
# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) defined
# locally in source files will be included in the documentation. If set to NO
# only classes defined in header files are included. Does not have any effect
# for Java sources.
# The default value is: YES.
EXTRACT_LOCAL_CLASSES = YES
# This flag is only useful for Objective-C code. When set to YES local methods,
# which are defined in the implementation section but not in the interface are
# included in the documentation. If set to NO only methods in the interface are
# included.
# The default value is: NO.
EXTRACT_LOCAL_METHODS = NO
# If this flag is set to YES, the members of anonymous namespaces will be
# extracted and appear in the documentation as a namespace called
# 'anonymous_namespace{file}', where file will be replaced with the base name of
# the file that contains the anonymous namespace. By default anonymous namespace
# are hidden.
# The default value is: NO.
EXTRACT_ANON_NSPACES = NO
# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
# undocumented members inside documented classes or files. If set to NO these
# members will be included in the various overviews, but no documentation
# section is generated. This option has no effect if EXTRACT_ALL is enabled.
# The default value is: NO.
HIDE_UNDOC_MEMBERS = NO
# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
# undocumented classes that are normally visible in the class hierarchy. If set
# to NO these classes will be included in the various overviews. This option has
# no effect if EXTRACT_ALL is enabled.
# The default value is: NO.
HIDE_UNDOC_CLASSES = NO
# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
# (class|struct|union) declarations. If set to NO these declarations will be
# included in the documentation.
# The default value is: NO.
HIDE_FRIEND_COMPOUNDS = NO
# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any
# documentation blocks found inside the body of a function. If set to NO these
# blocks will be appended to the function's detailed documentation block.
# The default value is: NO.
HIDE_IN_BODY_DOCS = NO
# The INTERNAL_DOCS tag determines if documentation that is typed after a
# \internal command is included. If the tag is set to NO then the documentation
# will be excluded. Set it to YES to include the internal documentation.
# The default value is: NO.
INTERNAL_DOCS = NO
# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file
# names in lower-case letters. If set to YES upper-case letters are also
# allowed. This is useful if you have classes or files whose names only differ
# in case and if your file system supports case sensitive file names. Windows
# and Mac users are advised to set this option to NO.
# The default value is: system dependent.
CASE_SENSE_NAMES = YES
# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with
# their full class and namespace scopes in the documentation. If set to YES the
# scope will be hidden.
# The default value is: NO.
HIDE_SCOPE_NAMES = NO
# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of
# the files that are included by a file in the documentation of that file.
# The default value is: YES.
SHOW_INCLUDE_FILES = YES
# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include
# files with double quotes in the documentation rather than with sharp brackets.
# The default value is: NO.
FORCE_LOCAL_INCLUDES = NO
# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the
# documentation for inline members.
# The default value is: YES.
INLINE_INFO = YES
# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the
# (detailed) documentation of file and class members alphabetically by member
# name. If set to NO the members will appear in declaration order.
# The default value is: YES.
SORT_MEMBER_DOCS = YES
# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief
# descriptions of file, namespace and class members alphabetically by member
# name. If set to NO the members will appear in declaration order.
# The default value is: NO.
SORT_BRIEF_DOCS = NO
# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the
# (brief and detailed) documentation of class members so that constructors and
# destructors are listed first. If set to NO the constructors will appear in the
# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS.
# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief
# member documentation.
# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting
# detailed member documentation.
# The default value is: NO.
SORT_MEMBERS_CTORS_1ST = NO
# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy
# of group names into alphabetical order. If set to NO the group names will
# appear in their defined order.
# The default value is: NO.
SORT_GROUP_NAMES = NO
# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
# fully-qualified names, including namespaces. If set to NO, the class list will
# be sorted only by class name, not including the namespace part.
# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
# Note: This option applies only to the class list, not to the alphabetical
# list.
# The default value is: NO.
SORT_BY_SCOPE_NAME = NO
# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper
# type resolution of all parameters of a function it will reject a match between
# the prototype and the implementation of a member function even if there is
# only one candidate or it is obvious which candidate to choose by doing a
# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still
# accept a match between prototype and implementation in such cases.
# The default value is: NO.
STRICT_PROTO_MATCHING = NO
# The GENERATE_TODOLIST tag can be used to enable ( YES) or disable ( NO) the
# todo list. This list is created by putting \todo commands in the
# documentation.
# The default value is: YES.
GENERATE_TODOLIST = YES
# The GENERATE_TESTLIST tag can be used to enable ( YES) or disable ( NO) the
# test list. This list is created by putting \test commands in the
# documentation.
# The default value is: YES.
GENERATE_TESTLIST = YES
# The GENERATE_BUGLIST tag can be used to enable ( YES) or disable ( NO) the bug
# list. This list is created by putting \bug commands in the documentation.
# The default value is: YES.
GENERATE_BUGLIST = YES
# The GENERATE_DEPRECATEDLIST tag can be used to enable ( YES) or disable ( NO)
# the deprecated list. This list is created by putting \deprecated commands in
# the documentation.
# The default value is: YES.
GENERATE_DEPRECATEDLIST= YES
# The ENABLED_SECTIONS tag can be used to enable conditional documentation
# sections, marked by \if ... \endif and \cond
# ... \endcond blocks.
ENABLED_SECTIONS =
# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the
# initial value of a variable or macro / define can have for it to appear in the
# documentation. If the initializer consists of more lines than specified here
# it will be hidden. Use a value of 0 to hide initializers completely. The
# appearance of the value of individual variables and macros / defines can be
# controlled using \showinitializer or \hideinitializer command in the
# documentation regardless of this setting.
# Minimum value: 0, maximum value: 10000, default value: 30.
MAX_INITIALIZER_LINES = 30
# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at
# the bottom of the documentation of classes and structs. If set to YES the list
# will mention the files that were used to generate the documentation.
# The default value is: YES.
SHOW_USED_FILES = YES
# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
# will remove the Files entry from the Quick Index and from the Folder Tree View
# (if specified).
# The default value is: YES.
SHOW_FILES = YES
# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
# page. This will remove the Namespaces entry from the Quick Index and from the
# Folder Tree View (if specified).
# The default value is: YES.
SHOW_NAMESPACES = YES
# The FILE_VERSION_FILTER tag can be used to specify a program or script that
# doxygen should invoke to get the current version for each file (typically from
# the version control system). Doxygen will invoke the program by executing (via
# popen()) the command command input-file, where command is the value of the
# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided
# by doxygen. Whatever the program writes to standard output is used as the file
# version. For an example see the documentation.
FILE_VERSION_FILTER =
# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed
# by doxygen. The layout file controls the global structure of the generated
# output files in an output format independent way. To create the layout file
# that represents doxygen's defaults, run doxygen with the -l option. You can
# optionally specify a file name after the option, if omitted DoxygenLayout.xml
# will be used as the name of the layout file.
#
# Note that if you run doxygen from a directory containing a file called
# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE
# tag is left empty.
LAYOUT_FILE =
# The CITE_BIB_FILES tag can be used to specify one or more bib files containing
# the reference definitions. This must be a list of .bib files. The .bib
# extension is automatically appended if omitted. This requires the bibtex tool
# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
# For LaTeX the style of the bibliography can be controlled using
# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
# search path. Do not use file names with spaces, bibtex cannot handle them. See
# also \cite for info how to create references.
CITE_BIB_FILES =
#---------------------------------------------------------------------------
# Configuration options related to warning and progress messages
#---------------------------------------------------------------------------
# The QUIET tag can be used to turn on/off the messages that are generated to
# standard output by doxygen. If QUIET is set to YES this implies that the
# messages are off.
# The default value is: NO.
QUIET = NO
# The WARNINGS tag can be used to turn on/off the warning messages that are
# generated to standard error ( stderr) by doxygen. If WARNINGS is set to YES
# this implies that the warnings are on.
#
# Tip: Turn warnings on while writing the documentation.
# The default value is: YES.
WARNINGS = YES
# If the WARN_IF_UNDOCUMENTED tag is set to YES, then doxygen will generate
# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag
# will automatically be disabled.
# The default value is: YES.
WARN_IF_UNDOCUMENTED = YES
# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for
# potential errors in the documentation, such as not documenting some parameters
# in a documented function, or documenting parameters that don't exist or using
# markup commands wrongly.
# The default value is: YES.
WARN_IF_DOC_ERROR = YES
# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
# are documented, but have no documentation for their parameters or return
# value. If set to NO doxygen will only warn about wrong or incomplete parameter
# documentation, but not about the absence of documentation.
# The default value is: NO.
WARN_NO_PARAMDOC = NO
# The WARN_FORMAT tag determines the format of the warning messages that doxygen
# can produce. The string should contain the $file, $line, and $text tags, which
# will be replaced by the file and line number from which the warning originated
# and the warning text. Optionally the format may contain $version, which will
# be replaced by the version of the file (if it could be obtained via
# FILE_VERSION_FILTER)
# The default value is: $file:$line: $text.
WARN_FORMAT = "$file:$line: $text"
# The WARN_LOGFILE tag can be used to specify a file to which warning and error
# messages should be written. If left blank the output is written to standard
# error (stderr).
WARN_LOGFILE =
#---------------------------------------------------------------------------
# Configuration options related to the input files
#---------------------------------------------------------------------------
# The INPUT tag is used to specify the files and/or directories that contain
# documented source files. You may enter file names like myfile.cpp or
# directories like /usr/src/myproject. Separate the files or directories with
# spaces.
# Note: If this tag is empty the current directory is searched.
INPUT =
# This tag can be used to specify the character encoding of the source files
# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
# documentation (see: http://www.gnu.org/software/libiconv) for the list of
# possible encodings.
# The default value is: UTF-8.
INPUT_ENCODING = UTF-8
# If the value of the INPUT tag contains directories, you can use the
# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
# *.h) to filter out the source-files in the directories. If left blank the
# following patterns are tested:*.c, *.cc, *.cxx, *.cpp, *.c++, *.java, *.ii,
# *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp,
# *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown,
# *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf,
# *.qsf, *.as and *.js.
#### OPTION GENERATED IN generate_doxygen.sh
FILE_PATTERNS =
# The RECURSIVE tag can be used to specify whether or not subdirectories should
# be searched for input files as well.
# The default value is: NO.
RECURSIVE = YES
# The EXCLUDE tag can be used to specify files and/or directories that should be
# excluded from the INPUT source files. This way you can easily exclude a
# subdirectory from a directory tree whose root is specified with the INPUT tag.
#
# Note that relative paths are relative to the directory from which doxygen is
# run.
EXCLUDE =
# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
# directories that are symbolic links (a Unix file system feature) are excluded
# from the input.
# The default value is: NO.
#### OPTION GENERATED IN generate_doxygen.sh
EXCLUDE_SYMLINKS = NO
# If the value of the INPUT tag contains directories, you can use the
# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
# certain files from those directories.
#
# Note that the wildcards are matched against the file with absolute path, so to
# exclude all test directories for example use the pattern */test/*
EXCLUDE_PATTERNS =
# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
# (namespaces, classes, functions, etc.) that should be excluded from the
# output. The symbol name can be a fully qualified name, a word, or if the
# wildcard * is used, a substring. Examples: ANamespace, AClass,
# AClass::ANamespace, ANamespace::*Test
#
# Note that the wildcards are matched against the file with absolute path, so to
# exclude all test directories use the pattern */test/*
EXCLUDE_SYMBOLS =
# The EXAMPLE_PATH tag can be used to specify one or more files or directories
# that contain example code fragments that are included (see the \include
# command).
EXAMPLE_PATH =
# If the value of the EXAMPLE_PATH tag contains directories, you can use the
# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
# *.h) to filter out the source-files in the directories. If left blank all
# files are included.
EXAMPLE_PATTERNS =
# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be
# searched for input files to be used with the \include or \dontinclude commands
# irrespective of the value of the RECURSIVE tag.
# The default value is: NO.
EXAMPLE_RECURSIVE = NO
# The IMAGE_PATH tag can be used to specify one or more files or directories
# that contain images that are to be included in the documentation (see the
# \image command).
IMAGE_PATH =
# The INPUT_FILTER tag can be used to specify a program that doxygen should
# invoke to filter for each input file. Doxygen will invoke the filter program
# by executing (via popen()) the command:
#
#
#
# where is the value of the INPUT_FILTER tag, and is the
# name of an input file. Doxygen will then use the output that the filter
# program writes to standard output. If FILTER_PATTERNS is specified, this tag
# will be ignored.
#
# Note that the filter must not add or remove lines; it is applied before the
# code is scanned, but not when the output code is generated. If lines are added
# or removed, the anchors will not be placed correctly.
INPUT_FILTER =
# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern
# basis. Doxygen will compare the file name with each pattern and apply the
# filter if there is a match. The filters are a list of the form: pattern=filter
# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how
# filters are used. If the FILTER_PATTERNS tag is empty or if none of the
# patterns match the file name, INPUT_FILTER is applied.
FILTER_PATTERNS =
# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using
# INPUT_FILTER ) will also be used to filter the input files that are used for
# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES).
# The default value is: NO.
FILTER_SOURCE_FILES = NO
# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file
# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and
# it is also possible to disable source filtering for a specific pattern using
# *.ext= (so without naming a filter).
# This tag requires that the tag FILTER_SOURCE_FILES is set to YES.
FILTER_SOURCE_PATTERNS =
# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that
# is part of the input, its contents will be placed on the main page
# (index.html). This can be useful if you have a project on for instance GitHub
# and want to reuse the introduction page also for the doxygen output.
USE_MDFILE_AS_MAINPAGE =
#---------------------------------------------------------------------------
# Configuration options related to source browsing
#---------------------------------------------------------------------------
# If the SOURCE_BROWSER tag is set to YES then a list of source files will be
# generated. Documented entities will be cross-referenced with these sources.
#
# Note: To get rid of all source code in the generated output, make sure that
# also VERBATIM_HEADERS is set to NO.
# The default value is: NO.
SOURCE_BROWSER = YES
# Setting the INLINE_SOURCES tag to YES will include the body of functions,
# classes and enums directly into the documentation.
# The default value is: NO.
INLINE_SOURCES = YES
# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any
# special comment blocks from generated source code fragments. Normal C, C++ and
# Fortran comments will always remain visible.
# The default value is: YES.
STRIP_CODE_COMMENTS = YES
# If the REFERENCED_BY_RELATION tag is set to YES then for each documented
# function all documented functions referencing it will be listed.
# The default value is: NO.
REFERENCED_BY_RELATION = YES
# If the REFERENCES_RELATION tag is set to YES then for each documented function
# all documented entities called/used by that function will be listed.
# The default value is: NO.
REFERENCES_RELATION = YES
# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set
# to YES, then the hyperlinks from functions in REFERENCES_RELATION and
# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will
# link to the documentation.
# The default value is: YES.
REFERENCES_LINK_SOURCE = YES
# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the
# source code will show a tooltip with additional information such as prototype,
# brief description and links to the definition and documentation. Since this
# will make the HTML file larger and loading of large files a bit slower, you
# can opt to disable this feature.
# The default value is: YES.
# This tag requires that the tag SOURCE_BROWSER is set to YES.
SOURCE_TOOLTIPS = YES
# If the USE_HTAGS tag is set to YES then the references to source code will
# point to the HTML generated by the htags(1) tool instead of doxygen built-in
# source browser. The htags tool is part of GNU's global source tagging system
# (see http://www.gnu.org/software/global/global.html). You will need version
# 4.8.6 or higher.
#
# To use it do the following:
# - Install the latest version of global
# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
# - Make sure the INPUT points to the root of the source tree
# - Run doxygen as normal
#
# Doxygen will invoke htags (and that will in turn invoke gtags), so these
# tools must be available from the command line (i.e. in the search path).
#
# The result: instead of the source browser generated by doxygen, the links to
# source code will now point to the output of htags.
# The default value is: NO.
# This tag requires that the tag SOURCE_BROWSER is set to YES.
USE_HTAGS = NO
# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a
# verbatim copy of the header file for each class for which an include is
# specified. Set to NO to disable this.
# See also: Section \class.
# The default value is: YES.
VERBATIM_HEADERS = YES
#---------------------------------------------------------------------------
# Configuration options related to the alphabetical class index
#---------------------------------------------------------------------------
# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all
# compounds will be generated. Enable this if the project contains a lot of
# classes, structs, unions or interfaces.
# The default value is: YES.
ALPHABETICAL_INDEX = NO
# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
# which the alphabetical index list will be split.
# Minimum value: 1, maximum value: 20, default value: 5.
# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
COLS_IN_ALPHA_INDEX = 5
# In case all classes in a project start with a common prefix, all classes will
# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag
# can be used to specify a prefix (or a list of prefixes) that should be ignored
# while generating the index headers.
# This tag requires that the tag ALPHABETICAL_INDEX is set to YES.
IGNORE_PREFIX =
#---------------------------------------------------------------------------
# Configuration options related to the HTML output
#---------------------------------------------------------------------------
# If the GENERATE_HTML tag is set to YES doxygen will generate HTML output
# The default value is: YES.
GENERATE_HTML = YES
# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a
# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
# it.
# The default directory is: html.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_OUTPUT = html
# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each
# generated HTML page (for example: .htm, .php, .asp).
# The default value is: .html.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_FILE_EXTENSION = .html
# The HTML_HEADER tag can be used to specify a user-defined HTML header file for
# each generated HTML page. If the tag is left blank doxygen will generate a
# standard header.
#
# To get valid HTML the header file that includes any scripts and style sheets
# that doxygen needs, which is dependent on the configuration options used (e.g.
# the setting GENERATE_TREEVIEW). It is highly recommended to start with a
# default header using
# doxygen -w html new_header.html new_footer.html new_stylesheet.css
# YourConfigFile
# and then modify the file new_header.html. See also section "Doxygen usage"
# for information on how to generate the default header that doxygen normally
# uses.
# Note: The header is subject to change so you typically have to regenerate the
# default header when upgrading to a newer version of doxygen. For a description
# of the possible markers and block names see the documentation.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_HEADER =
# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each
# generated HTML page. If the tag is left blank doxygen will generate a standard
# footer. See HTML_HEADER for more information on how to generate a default
# footer and what special commands can be used inside the footer. See also
# section "Doxygen usage" for information on how to generate the default footer
# that doxygen normally uses.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_FOOTER =
# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style
# sheet that is used by each HTML page. It can be used to fine-tune the look of
# the HTML output. If left blank doxygen will generate a default style sheet.
# See also section "Doxygen usage" for information on how to generate the style
# sheet that doxygen normally uses.
# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as
# it is more robust and this tag (HTML_STYLESHEET) will in the future become
# obsolete.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_STYLESHEET =
# The HTML_EXTRA_STYLESHEET tag can be used to specify an additional user-
# defined cascading style sheet that is included after the standard style sheets
# created by doxygen. Using this option one can overrule certain style aspects.
# This is preferred over using HTML_STYLESHEET since it does not replace the
# standard style sheet and is therefor more robust against future updates.
# Doxygen will copy the style sheet file to the output directory. For an example
# see the documentation.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_EXTRA_STYLESHEET =
# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or
# other source files which should be copied to the HTML output directory. Note
# that these files will be copied to the base HTML output directory. Use the
# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these
# files. In the HTML_STYLESHEET file, use the file name only. Also note that the
# files will be copied as-is; there are no commands or markers available.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_EXTRA_FILES =
# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
# will adjust the colors in the stylesheet and background images according to
# this color. Hue is specified as an angle on a colorwheel, see
# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
# purple, and 360 is red again.
# Minimum value: 0, maximum value: 359, default value: 220.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_COLORSTYLE_HUE = 220
# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors
# in the HTML output. For a value of 0 the output will use grayscales only. A
# value of 255 will produce the most vivid colors.
# Minimum value: 0, maximum value: 255, default value: 100.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_COLORSTYLE_SAT = 100
# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the
# luminance component of the colors in the HTML output. Values below 100
# gradually make the output lighter, whereas values above 100 make the output
# darker. The value divided by 100 is the actual gamma applied, so 80 represents
# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not
# change the gamma.
# Minimum value: 40, maximum value: 240, default value: 80.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_COLORSTYLE_GAMMA = 80
# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML
# page will contain the date and time when the page was generated. Setting this
# to NO can help when comparing the output of multiple runs.
# The default value is: YES.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_TIMESTAMP = NO
# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
# documentation will contain sections that can be hidden and shown after the
# page has loaded.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_DYNAMIC_SECTIONS = NO
# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries
# shown in the various tree structured indices initially; the user can expand
# and collapse entries dynamically later on. Doxygen will expand the tree to
# such a level that at most the specified number of entries are visible (unless
# a fully collapsed tree already exceeds this amount). So setting the number of
# entries 1 will produce a full collapsed tree by default. 0 is a special value
# representing an infinite number of entries and will result in a full expanded
# tree by default.
# Minimum value: 0, maximum value: 9999, default value: 100.
# This tag requires that the tag GENERATE_HTML is set to YES.
HTML_INDEX_NUM_ENTRIES = 100
# If the GENERATE_DOCSET tag is set to YES, additional index files will be
# generated that can be used as input for Apple's Xcode 3 integrated development
# environment (see: http://developer.apple.com/tools/xcode/), introduced with
# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
# Makefile in the HTML output directory. Running make will produce the docset in
# that directory and running make install will install the docset in
# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
# for more information.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTML is set to YES.
GENERATE_DOCSET = NO
# This tag determines the name of the docset feed. A documentation feed provides
# an umbrella under which multiple documentation sets from a single provider
# (such as a company or product suite) can be grouped.
# The default value is: Doxygen generated docs.
# This tag requires that the tag GENERATE_DOCSET is set to YES.
DOCSET_FEEDNAME = "Doxygen generated docs"
# This tag specifies a string that should uniquely identify the documentation
# set bundle. This should be a reverse domain-name style string, e.g.
# com.mycompany.MyDocSet. Doxygen will append .docset to the name.
# The default value is: org.doxygen.Project.
# This tag requires that the tag GENERATE_DOCSET is set to YES.
DOCSET_BUNDLE_ID = org.doxygen.Project
# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify
# the documentation publisher. This should be a reverse domain-name style
# string, e.g. com.mycompany.MyDocSet.documentation.
# The default value is: org.doxygen.Publisher.
# This tag requires that the tag GENERATE_DOCSET is set to YES.
DOCSET_PUBLISHER_ID = org.doxygen.Publisher
# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher.
# The default value is: Publisher.
# This tag requires that the tag GENERATE_DOCSET is set to YES.
DOCSET_PUBLISHER_NAME = Publisher
# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
# additional HTML index files: index.hhp, index.hhc, and index.hhk. The
# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
# Windows.
#
# The HTML Help Workshop contains a compiler that can convert all HTML output
# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML
# files are now used as the Windows 98 help format, and will replace the old
# Windows help format (.hlp) on all Windows platforms in the future. Compressed
# HTML files also contain an index, a table of contents, and you can search for
# words in the documentation. The HTML workshop also contains a viewer for
# compressed HTML files.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTML is set to YES.
GENERATE_HTMLHELP = NO
# The CHM_FILE tag can be used to specify the file name of the resulting .chm
# file. You can add a path in front of the file if the result should not be
# written to the html output directory.
# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
CHM_FILE =
# The HHC_LOCATION tag can be used to specify the location (absolute path
# including file name) of the HTML help compiler ( hhc.exe). If non-empty
# doxygen will try to run the HTML help compiler on the generated index.hhp.
# The file has to be specified with full path.
# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
HHC_LOCATION =
# The GENERATE_CHI flag controls if a separate .chi index file is generated (
# YES) or that it should be included in the master .chm file ( NO).
# The default value is: NO.
# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
GENERATE_CHI = NO
# The CHM_INDEX_ENCODING is used to encode HtmlHelp index ( hhk), content ( hhc)
# and project file content.
# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
CHM_INDEX_ENCODING =
# The BINARY_TOC flag controls whether a binary table of contents is generated (
# YES) or a normal table of contents ( NO) in the .chm file.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
BINARY_TOC = NO
# The TOC_EXPAND flag can be set to YES to add extra items for group members to
# the table of contents of the HTML help documentation and to the tree view.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTMLHELP is set to YES.
TOC_EXPAND = NO
# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and
# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that
# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help
# (.qch) of the generated HTML documentation.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTML is set to YES.
GENERATE_QHP = NO
# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify
# the file name of the resulting .qch file. The path specified is relative to
# the HTML output folder.
# This tag requires that the tag GENERATE_QHP is set to YES.
QCH_FILE =
# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
# Project output. For more information please see Qt Help Project / Namespace
# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
# The default value is: org.doxygen.Project.
# This tag requires that the tag GENERATE_QHP is set to YES.
QHP_NAMESPACE = org.doxygen.Project
# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
# Help Project output. For more information please see Qt Help Project / Virtual
# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
# folders).
# The default value is: doc.
# This tag requires that the tag GENERATE_QHP is set to YES.
QHP_VIRTUAL_FOLDER = doc
# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
# filter to add. For more information please see Qt Help Project / Custom
# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
# filters).
# This tag requires that the tag GENERATE_QHP is set to YES.
QHP_CUST_FILTER_NAME =
# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
# custom filter to add. For more information please see Qt Help Project / Custom
# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
# filters).
# This tag requires that the tag GENERATE_QHP is set to YES.
QHP_CUST_FILTER_ATTRS =
# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
# project's filter section matches. Qt Help Project / Filter Attributes (see:
# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
# This tag requires that the tag GENERATE_QHP is set to YES.
QHP_SECT_FILTER_ATTRS =
# The QHG_LOCATION tag can be used to specify the location of Qt's
# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the
# generated .qhp file.
# This tag requires that the tag GENERATE_QHP is set to YES.
QHG_LOCATION =
# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be
# generated, together with the HTML files, they form an Eclipse help plugin. To
# install this plugin and make it available under the help contents menu in
# Eclipse, the contents of the directory containing the HTML and XML files needs
# to be copied into the plugins directory of eclipse. The name of the directory
# within the plugins directory should be the same as the ECLIPSE_DOC_ID value.
# After copying Eclipse needs to be restarted before the help appears.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTML is set to YES.
GENERATE_ECLIPSEHELP = NO
# A unique identifier for the Eclipse help plugin. When installing the plugin
# the directory name containing the HTML and XML files should also have this
# name. Each documentation set should have its own identifier.
# The default value is: org.doxygen.Project.
# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES.
ECLIPSE_DOC_ID = org.doxygen.Project
# If you want full control over the layout of the generated HTML pages it might
# be necessary to disable the index and replace it with your own. The
# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top
# of each HTML page. A value of NO enables the index and the value YES disables
# it. Since the tabs in the index contain the same information as the navigation
# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTML is set to YES.
DISABLE_INDEX = NO
# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index
# structure should be generated to display hierarchical information. If the tag
# value is set to YES, a side panel will be generated containing a tree-like
# index structure (just like the one that is generated for HTML Help). For this
# to work a browser that supports JavaScript, DHTML, CSS and frames is required
# (i.e. any modern browser). Windows users are probably better off using the
# HTML help feature. Via custom stylesheets (see HTML_EXTRA_STYLESHEET) one can
# further fine-tune the look of the index. As an example, the default style
# sheet generated by doxygen has an example that shows how to put an image at
# the root of the tree instead of the PROJECT_NAME. Since the tree basically has
# the same information as the tab index, you could consider setting
# DISABLE_INDEX to YES when enabling this option.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTML is set to YES.
GENERATE_TREEVIEW = YES
# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that
# doxygen will group on one line in the generated HTML documentation.
#
# Note that a value of 0 will completely suppress the enum values from appearing
# in the overview section.
# Minimum value: 0, maximum value: 20, default value: 4.
# This tag requires that the tag GENERATE_HTML is set to YES.
ENUM_VALUES_PER_LINE = 4
# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used
# to set the initial width (in pixels) of the frame in which the tree is shown.
# Minimum value: 0, maximum value: 1500, default value: 250.
# This tag requires that the tag GENERATE_HTML is set to YES.
TREEVIEW_WIDTH = 250
# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open links to
# external symbols imported via tag files in a separate window.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTML is set to YES.
EXT_LINKS_IN_WINDOW = NO
# Use this tag to change the font size of LaTeX formulas included as images in
# the HTML documentation. When you change the font size after a successful
# doxygen run you need to manually remove any form_*.png images from the HTML
# output directory to force them to be regenerated.
# Minimum value: 8, maximum value: 50, default value: 10.
# This tag requires that the tag GENERATE_HTML is set to YES.
FORMULA_FONTSIZE = 10
# Use the FORMULA_TRANPARENT tag to determine whether or not the images
# generated for formulas are transparent PNGs. Transparent PNGs are not
# supported properly for IE 6.0, but are supported on all modern browsers.
#
# Note that when changing this option you need to delete any form_*.png files in
# the HTML output directory before the changes have effect.
# The default value is: YES.
# This tag requires that the tag GENERATE_HTML is set to YES.
FORMULA_TRANSPARENT = YES
# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
# http://www.mathjax.org) which uses client side Javascript for the rendering
# instead of using prerendered bitmaps. Use this if you do not have LaTeX
# installed or if you want to formulas look prettier in the HTML output. When
# enabled you may also need to install MathJax separately and configure the path
# to it using the MATHJAX_RELPATH option.
# The default value is: NO.
# This tag requires that the tag GENERATE_HTML is set to YES.
USE_MATHJAX = NO
# When MathJax is enabled you can set the default output format to be used for
# the MathJax output. See the MathJax site (see:
# http://docs.mathjax.org/en/latest/output.html) for more details.
# Possible values are: HTML-CSS (which is slower, but has the best
# compatibility), NativeMML (i.e. MathML) and SVG.
# The default value is: HTML-CSS.
# This tag requires that the tag USE_MATHJAX is set to YES.
MATHJAX_FORMAT = HTML-CSS
# When MathJax is enabled you need to specify the location relative to the HTML
# output directory using the MATHJAX_RELPATH option. The destination directory
# should contain the MathJax.js script. For instance, if the mathjax directory
# is located at the same level as the HTML output directory, then
# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
# Content Delivery Network so you can quickly see the result without installing
# MathJax. However, it is strongly recommended to install a local copy of
# MathJax from http://www.mathjax.org before deployment.
# The default value is: http://cdn.mathjax.org/mathjax/latest.
# This tag requires that the tag USE_MATHJAX is set to YES.
MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest
# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
# extension names that should be enabled during MathJax rendering. For example
# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols
# This tag requires that the tag USE_MATHJAX is set to YES.
MATHJAX_EXTENSIONS =
# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces
# of code that will be used on startup of the MathJax code. See the MathJax site
# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an
# example see the documentation.
# This tag requires that the tag USE_MATHJAX is set to YES.
MATHJAX_CODEFILE =
# When the SEARCHENGINE tag is enabled doxygen will generate a search box for
# the HTML output. The underlying search engine uses javascript and DHTML and
# should work on any modern browser. Note that when using HTML help
# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET)
# there is already a search function so this one should typically be disabled.
# For large projects the javascript based search engine can be slow, then
# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to
# search using the keyboard; to jump to the search box use + S
# (what the is depends on the OS and browser, but it is typically
# , /, or both). Inside the search box use the to jump into the search results window, the results can be navigated
# using the . Press to select an item or to cancel
# the search. The filter options can be selected when the cursor is inside the
# search box by pressing +. Also here use the
# to select a filter and or to activate or cancel the filter
# option.
# The default value is: YES.
# This tag requires that the tag GENERATE_HTML is set to YES.
SEARCHENGINE = NO
# When the SERVER_BASED_SEARCH tag is enabled the search engine will be
# implemented using a web server instead of a web client using Javascript. There
# are two flavours of web server based searching depending on the
# EXTERNAL_SEARCH setting. When disabled, doxygen will generate a PHP script for
# searching and an index file used by the script. When EXTERNAL_SEARCH is
# enabled the indexing and searching needs to be provided by external tools. See
# the section "External Indexing and Searching" for details.
# The default value is: NO.
# This tag requires that the tag SEARCHENGINE is set to YES.
SERVER_BASED_SEARCH = NO
# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP
# script for searching. Instead the search results are written to an XML file
# which needs to be processed by an external indexer. Doxygen will invoke an
# external search engine pointed to by the SEARCHENGINE_URL option to obtain the
# search results.
#
# Doxygen ships with an example indexer ( doxyindexer) and search engine
# (doxysearch.cgi) which are based on the open source search engine library
# Xapian (see: http://xapian.org/).
#
# See the section "External Indexing and Searching" for details.
# The default value is: NO.
# This tag requires that the tag SEARCHENGINE is set to YES.
EXTERNAL_SEARCH = NO
# The SEARCHENGINE_URL should point to a search engine hosted by a web server
# which will return the search results when EXTERNAL_SEARCH is enabled.
#
# Doxygen ships with an example indexer ( doxyindexer) and search engine
# (doxysearch.cgi) which are based on the open source search engine library
# Xapian (see: http://xapian.org/). See the section "External Indexing and
# Searching" for details.
# This tag requires that the tag SEARCHENGINE is set to YES.
SEARCHENGINE_URL =
# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the unindexed
# search data is written to a file for indexing by an external tool. With the
# SEARCHDATA_FILE tag the name of this file can be specified.
# The default file is: searchdata.xml.
# This tag requires that the tag SEARCHENGINE is set to YES.
SEARCHDATA_FILE = searchdata.xml
# When SERVER_BASED_SEARCH and EXTERNAL_SEARCH are both enabled the
# EXTERNAL_SEARCH_ID tag can be used as an identifier for the project. This is
# useful in combination with EXTRA_SEARCH_MAPPINGS to search through multiple
# projects and redirect the results back to the right project.
# This tag requires that the tag SEARCHENGINE is set to YES.
EXTERNAL_SEARCH_ID =
# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen
# projects other than the one defined by this configuration file, but that are
# all added to the same external search index. Each project needs to have a
# unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of
# to a relative location where the documentation can be found. The format is:
# EXTRA_SEARCH_MAPPINGS = tagname1=loc1 tagname2=loc2 ...
# This tag requires that the tag SEARCHENGINE is set to YES.
EXTRA_SEARCH_MAPPINGS =
#---------------------------------------------------------------------------
# Configuration options related to the LaTeX output
#---------------------------------------------------------------------------
# If the GENERATE_LATEX tag is set to YES doxygen will generate LaTeX output.
# The default value is: YES.
GENERATE_LATEX = YES
# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
# it.
# The default directory is: latex.
# This tag requires that the tag GENERATE_LATEX is set to YES.
LATEX_OUTPUT = latex
# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
# invoked.
#
# Note that when enabling USE_PDFLATEX this option is only used for generating
# bitmaps for formulas in the HTML output, but not in the Makefile that is
# written to the output directory.
# The default file is: latex.
# This tag requires that the tag GENERATE_LATEX is set to YES.
LATEX_CMD_NAME = latex
# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
# index for LaTeX.
# The default file is: makeindex.
# This tag requires that the tag GENERATE_LATEX is set to YES.
MAKEINDEX_CMD_NAME = makeindex
# If the COMPACT_LATEX tag is set to YES doxygen generates more compact LaTeX
# documents. This may be useful for small projects and may help to save some
# trees in general.
# The default value is: NO.
# This tag requires that the tag GENERATE_LATEX is set to YES.
COMPACT_LATEX = NO
# The PAPER_TYPE tag can be used to set the paper type that is used by the
# printer.
# Possible values are: a4 (210 x 297 mm), letter (8.5 x 11 inches), legal (8.5 x
# 14 inches) and executive (7.25 x 10.5 inches).
# The default value is: a4.
# This tag requires that the tag GENERATE_LATEX is set to YES.
PAPER_TYPE = a4wide
# The EXTRA_PACKAGES tag can be used to specify one or more LaTeX package names
# that should be included in the LaTeX output. To get the times font for
# instance you can specify
# EXTRA_PACKAGES=times
# If left blank no extra packages will be included.
# This tag requires that the tag GENERATE_LATEX is set to YES.
EXTRA_PACKAGES =
# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the
# generated LaTeX document. The header should contain everything until the first
# chapter. If it is left blank doxygen will generate a standard header. See
# section "Doxygen usage" for information on how to let doxygen write the
# default header to a separate file.
#
# Note: Only use a user-defined header if you know what you are doing! The
# following commands have a special meaning inside the header: $title,
# $datetime, $date, $doxygenversion, $projectname, $projectnumber. Doxygen will
# replace them by respectively the title of the page, the current date and time,
# only the current date, the version number of doxygen, the project name (see
# PROJECT_NAME), or the project number (see PROJECT_NUMBER).
# This tag requires that the tag GENERATE_LATEX is set to YES.
LATEX_HEADER =
# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the
# generated LaTeX document. The footer should contain everything after the last
# chapter. If it is left blank doxygen will generate a standard footer.
#
# Note: Only use a user-defined footer if you know what you are doing!
# This tag requires that the tag GENERATE_LATEX is set to YES.
LATEX_FOOTER =
# The LATEX_EXTRA_FILES tag can be used to specify one or more extra images or
# other source files which should be copied to the LATEX_OUTPUT output
# directory. Note that the files will be copied as-is; there are no commands or
# markers available.
# This tag requires that the tag GENERATE_LATEX is set to YES.
LATEX_EXTRA_FILES =
# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated is
# prepared for conversion to PDF (using ps2pdf or pdflatex). The PDF file will
# contain links (just like the HTML output) instead of page references. This
# makes the output suitable for online browsing using a PDF viewer.
# The default value is: YES.
# This tag requires that the tag GENERATE_LATEX is set to YES.
PDF_HYPERLINKS = NO
# If the LATEX_PDFLATEX tag is set to YES, doxygen will use pdflatex to generate
# the PDF file directly from the LaTeX files. Set this option to YES to get a
# higher quality PDF documentation.
# The default value is: YES.
# This tag requires that the tag GENERATE_LATEX is set to YES.
USE_PDFLATEX = NO
# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode
# command to the generated LaTeX files. This will instruct LaTeX to keep running
# if errors occur, instead of asking the user for help. This option is also used
# when generating formulas in HTML.
# The default value is: NO.
# This tag requires that the tag GENERATE_LATEX is set to YES.
LATEX_BATCHMODE = NO
# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the
# index chapters (such as File Index, Compound Index, etc.) in the output.
# The default value is: NO.
# This tag requires that the tag GENERATE_LATEX is set to YES.
LATEX_HIDE_INDICES = NO
# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source
# code with syntax highlighting in the LaTeX output.
#
# Note that which sources are shown also depends on other settings such as
# SOURCE_BROWSER.
# The default value is: NO.
# This tag requires that the tag GENERATE_LATEX is set to YES.
LATEX_SOURCE_CODE = NO
# The LATEX_BIB_STYLE tag can be used to specify the style to use for the
# bibliography, e.g. plainnat, or ieeetr. See
# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
# The default value is: plain.
# This tag requires that the tag GENERATE_LATEX is set to YES.
LATEX_BIB_STYLE = plain
#---------------------------------------------------------------------------
# Configuration options related to the RTF output
#---------------------------------------------------------------------------
# If the GENERATE_RTF tag is set to YES doxygen will generate RTF output. The
# RTF output is optimized for Word 97 and may not look too pretty with other RTF
# readers/editors.
# The default value is: NO.
GENERATE_RTF = NO
# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. If a
# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
# it.
# The default directory is: rtf.
# This tag requires that the tag GENERATE_RTF is set to YES.
RTF_OUTPUT = rtf
# If the COMPACT_RTF tag is set to YES doxygen generates more compact RTF
# documents. This may be useful for small projects and may help to save some
# trees in general.
# The default value is: NO.
# This tag requires that the tag GENERATE_RTF is set to YES.
COMPACT_RTF = NO
# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated will
# contain hyperlink fields. The RTF file will contain links (just like the HTML
# output) instead of page references. This makes the output suitable for online
# browsing using Word or some other Word compatible readers that support those
# fields.
#
# Note: WordPad (write) and others do not support links.
# The default value is: NO.
# This tag requires that the tag GENERATE_RTF is set to YES.
RTF_HYPERLINKS = NO
# Load stylesheet definitions from file. Syntax is similar to doxygen's config
# file, i.e. a series of assignments. You only have to provide replacements,
# missing definitions are set to their default value.
#
# See also section "Doxygen usage" for information on how to generate the
# default style sheet that doxygen normally uses.
# This tag requires that the tag GENERATE_RTF is set to YES.
RTF_STYLESHEET_FILE =
# Set optional variables used in the generation of an RTF document. Syntax is
# similar to doxygen's config file. A template extensions file can be generated
# using doxygen -e rtf extensionFile.
# This tag requires that the tag GENERATE_RTF is set to YES.
RTF_EXTENSIONS_FILE =
#---------------------------------------------------------------------------
# Configuration options related to the man page output
#---------------------------------------------------------------------------
# If the GENERATE_MAN tag is set to YES doxygen will generate man pages for
# classes and files.
# The default value is: NO.
GENERATE_MAN = NO
# The MAN_OUTPUT tag is used to specify where the man pages will be put. If a
# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
# it. A directory man3 will be created inside the directory specified by
# MAN_OUTPUT.
# The default directory is: man.
# This tag requires that the tag GENERATE_MAN is set to YES.
MAN_OUTPUT = man
# The MAN_EXTENSION tag determines the extension that is added to the generated
# man pages. In case the manual section does not start with a number, the number
# 3 is prepended. The dot (.) at the beginning of the MAN_EXTENSION tag is
# optional.
# The default value is: .3.
# This tag requires that the tag GENERATE_MAN is set to YES.
MAN_EXTENSION = .3
# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it
# will generate one additional man file for each entity documented in the real
# man page(s). These additional files only source the real man page, but without
# them the man command would be unable to find the correct page.
# The default value is: NO.
# This tag requires that the tag GENERATE_MAN is set to YES.
MAN_LINKS = NO
#---------------------------------------------------------------------------
# Configuration options related to the XML output
#---------------------------------------------------------------------------
# If the GENERATE_XML tag is set to YES doxygen will generate an XML file that
# captures the structure of the code including all documentation.
# The default value is: NO.
GENERATE_XML = NO
# The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
# it.
# The default directory is: xml.
# This tag requires that the tag GENERATE_XML is set to YES.
XML_OUTPUT = xml
# The XML_SCHEMA tag can be used to specify a XML schema, which can be used by a
# validating XML parser to check the syntax of the XML files.
# This tag requires that the tag GENERATE_XML is set to YES.
XML_SCHEMA =
# The XML_DTD tag can be used to specify a XML DTD, which can be used by a
# validating XML parser to check the syntax of the XML files.
# This tag requires that the tag GENERATE_XML is set to YES.
XML_DTD =
# If the XML_PROGRAMLISTING tag is set to YES doxygen will dump the program
# listings (including syntax highlighting and cross-referencing information) to
# the XML output. Note that enabling this will significantly increase the size
# of the XML output.
# The default value is: YES.
# This tag requires that the tag GENERATE_XML is set to YES.
XML_PROGRAMLISTING = YES
#---------------------------------------------------------------------------
# Configuration options related to the DOCBOOK output
#---------------------------------------------------------------------------
# If the GENERATE_DOCBOOK tag is set to YES doxygen will generate Docbook files
# that can be used to generate PDF.
# The default value is: NO.
GENERATE_DOCBOOK = NO
# The DOCBOOK_OUTPUT tag is used to specify where the Docbook pages will be put.
# If a relative path is entered the value of OUTPUT_DIRECTORY will be put in
# front of it.
# The default directory is: docbook.
# This tag requires that the tag GENERATE_DOCBOOK is set to YES.
DOCBOOK_OUTPUT = docbook
#---------------------------------------------------------------------------
# Configuration options for the AutoGen Definitions output
#---------------------------------------------------------------------------
# If the GENERATE_AUTOGEN_DEF tag is set to YES doxygen will generate an AutoGen
# Definitions (see http://autogen.sf.net) file that captures the structure of
# the code including all documentation. Note that this feature is still
# experimental and incomplete at the moment.
# The default value is: NO.
GENERATE_AUTOGEN_DEF = NO
#---------------------------------------------------------------------------
# Configuration options related to the Perl module output
#---------------------------------------------------------------------------
# If the GENERATE_PERLMOD tag is set to YES doxygen will generate a Perl module
# file that captures the structure of the code including all documentation.
#
# Note that this feature is still experimental and incomplete at the moment.
# The default value is: NO.
GENERATE_PERLMOD = NO
# If the PERLMOD_LATEX tag is set to YES doxygen will generate the necessary
# Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI
# output from the Perl module output.
# The default value is: NO.
# This tag requires that the tag GENERATE_PERLMOD is set to YES.
PERLMOD_LATEX = NO
# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be nicely
# formatted so it can be parsed by a human reader. This is useful if you want to
# understand what is going on. On the other hand, if this tag is set to NO the
# size of the Perl module output will be much smaller and Perl will parse it
# just the same.
# The default value is: YES.
# This tag requires that the tag GENERATE_PERLMOD is set to YES.
PERLMOD_PRETTY = YES
# The names of the make variables in the generated doxyrules.make file are
# prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. This is useful
# so different doxyrules.make files included by the same Makefile don't
# overwrite each other's variables.
# This tag requires that the tag GENERATE_PERLMOD is set to YES.
PERLMOD_MAKEVAR_PREFIX =
#---------------------------------------------------------------------------
# Configuration options related to the preprocessor
#---------------------------------------------------------------------------
# If the ENABLE_PREPROCESSING tag is set to YES doxygen will evaluate all
# C-preprocessor directives found in the sources and include files.
# The default value is: YES.
ENABLE_PREPROCESSING = YES
# If the MACRO_EXPANSION tag is set to YES doxygen will expand all macro names
# in the source code. If set to NO only conditional compilation will be
# performed. Macro expansion can be done in a controlled way by setting
# EXPAND_ONLY_PREDEF to YES.
# The default value is: NO.
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
MACRO_EXPANSION = YES
# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
# the macro expansion is limited to the macros specified with the PREDEFINED and
# EXPAND_AS_DEFINED tags.
# The default value is: NO.
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
EXPAND_ONLY_PREDEF = NO
# If the SEARCH_INCLUDES tag is set to YES the includes files in the
# INCLUDE_PATH will be searched if a #include is found.
# The default value is: YES.
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
SEARCH_INCLUDES = YES
# The INCLUDE_PATH tag can be used to specify one or more directories that
# contain include files that are not input files but should be processed by the
# preprocessor.
# This tag requires that the tag SEARCH_INCLUDES is set to YES.
INCLUDE_PATH =
# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard
# patterns (like *.h and *.hpp) to filter out the header-files in the
# directories. If left blank, the patterns specified with FILE_PATTERNS will be
# used.
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
#### OPTION GENERATED IN generate_doxygen.sh
INCLUDE_FILE_PATTERNS =
# The PREDEFINED tag can be used to specify one or more macro names that are
# defined before the preprocessor is started (similar to the -D option of e.g.
# gcc). The argument of the tag is a list of macros of the form: name or
# name=definition (no spaces). If the definition and the "=" are omitted, "=1"
# is assumed. To prevent a macro definition from being undefined via #undef or
# recursively expanded use the := operator instead of the = operator.
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
PREDEFINED =
# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
# tag can be used to specify a list of macro names that should be expanded. The
# macro definition that is found in the sources will be used. Use the PREDEFINED
# tag if you want to use a different macro definition that overrules the
# definition found in the source code.
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
EXPAND_AS_DEFINED =
# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will
# remove all refrences to function-like macros that are alone on a line, have an
# all uppercase name, and do not end with a semicolon. Such function macros are
# typically used for boiler-plate code, and will confuse the parser if not
# removed.
# The default value is: YES.
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
SKIP_FUNCTION_MACROS = YES
#---------------------------------------------------------------------------
# Configuration options related to external references
#---------------------------------------------------------------------------
# The TAGFILES tag can be used to specify one or more tag files. For each tag
# file the location of the external documentation should be added. The format of
# a tag file without this location is as follows:
# TAGFILES = file1 file2 ...
# Adding location for the tag files is done as follows:
# TAGFILES = file1=loc1 "file2 = loc2" ...
# where loc1 and loc2 can be relative or absolute paths or URLs. See the
# section "Linking to external documentation" for more information about the use
# of tag files.
# Note: Each tag file must have an unique name (where the name does NOT include
# the path). If a tag file is not located in the directory in which doxygen is
# run, you must also specify the path to the tagfile here.
TAGFILES =
# When a file name is specified after GENERATE_TAGFILE, doxygen will create a
# tag file that is based on the input files it reads. See section "Linking to
# external documentation" for more information about the usage of tag files.
GENERATE_TAGFILE =
# If the ALLEXTERNALS tag is set to YES all external class will be listed in the
# class index. If set to NO only the inherited external classes will be listed.
# The default value is: NO.
ALLEXTERNALS = NO
# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed in
# the modules index. If set to NO, only the current project's groups will be
# listed.
# The default value is: YES.
EXTERNAL_GROUPS = YES
# If the EXTERNAL_PAGES tag is set to YES all external pages will be listed in
# the related pages index. If set to NO, only the current project's pages will
# be listed.
# The default value is: YES.
EXTERNAL_PAGES = YES
# The PERL_PATH should be the absolute path and name of the perl script
# interpreter (i.e. the result of 'which perl').
# The default file (with absolute path) is: /usr/bin/perl.
PERL_PATH = /usr/bin/perl
#---------------------------------------------------------------------------
# Configuration options related to the dot tool
#---------------------------------------------------------------------------
# If the CLASS_DIAGRAMS tag is set to YES doxygen will generate a class diagram
# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to
# NO turns the diagrams off. Note that this option also works with HAVE_DOT
# disabled, but it is recommended to install and use dot, since it yields more
# powerful graphs.
# The default value is: YES.
CLASS_DIAGRAMS = YES
# You can define message sequence charts within doxygen comments using the \msc
# command. Doxygen will then run the mscgen tool (see:
# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
# documentation. The MSCGEN_PATH tag allows you to specify the directory where
# the mscgen tool resides. If left empty the tool is assumed to be found in the
# default search path.
MSCGEN_PATH =
# If set to YES, the inheritance and collaboration graphs will hide inheritance
# and usage relations if the target is undocumented or is not a class.
# The default value is: YES.
HIDE_UNDOC_RELATIONS = YES
# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is
# available from the path. This tool is part of Graphviz (see:
# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent
# Bell Labs. The other options in this section have no effect if this option is
# set to NO
# The default value is: NO.
HAVE_DOT = NO
# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed
# to run in parallel. When set to 0 doxygen will base this on the number of
# processors available in the system. You can set it explicitly to a value
# larger than 0 to get control over the balance between CPU load and processing
# speed.
# Minimum value: 0, maximum value: 32, default value: 0.
# This tag requires that the tag HAVE_DOT is set to YES.
DOT_NUM_THREADS = 0
# When you want a differently looking font n the dot files that doxygen
# generates you can specify the font name using DOT_FONTNAME. You need to make
# sure dot is able to find the font, which can be done by putting it in a
# standard location or by setting the DOTFONTPATH environment variable or by
# setting DOT_FONTPATH to the directory containing the font.
# The default value is: Helvetica.
# This tag requires that the tag HAVE_DOT is set to YES.
DOT_FONTNAME = Helvetica
# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of
# dot graphs.
# Minimum value: 4, maximum value: 24, default value: 10.
# This tag requires that the tag HAVE_DOT is set to YES.
DOT_FONTSIZE = 10
# By default doxygen will tell dot to use the default font as specified with
# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set
# the path where dot can find it using this tag.
# This tag requires that the tag HAVE_DOT is set to YES.
DOT_FONTPATH =
# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for
# each documented class showing the direct and indirect inheritance relations.
# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO.
# The default value is: YES.
# This tag requires that the tag HAVE_DOT is set to YES.
CLASS_GRAPH = YES
# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a
# graph for each documented class showing the direct and indirect implementation
# dependencies (inheritance, containment, and class references variables) of the
# class with other documented classes.
# The default value is: YES.
# This tag requires that the tag HAVE_DOT is set to YES.
COLLABORATION_GRAPH = YES
# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
# groups, showing the direct groups dependencies.
# The default value is: YES.
# This tag requires that the tag HAVE_DOT is set to YES.
GROUP_GRAPHS = YES
# If the UML_LOOK tag is set to YES doxygen will generate inheritance and
# collaboration diagrams in a style similar to the OMG's Unified Modeling
# Language.
# The default value is: NO.
# This tag requires that the tag HAVE_DOT is set to YES.
UML_LOOK = NO
# If the UML_LOOK tag is enabled, the fields and methods are shown inside the
# class node. If there are many fields or methods and many nodes the graph may
# become too big to be useful. The UML_LIMIT_NUM_FIELDS threshold limits the
# number of items for each type to make the size more manageable. Set this to 0
# for no limit. Note that the threshold may be exceeded by 50% before the limit
# is enforced. So when you set the threshold to 10, up to 15 fields may appear,
# but if the number exceeds 15, the total amount of fields shown is limited to
# 10.
# Minimum value: 0, maximum value: 100, default value: 10.
# This tag requires that the tag HAVE_DOT is set to YES.
UML_LIMIT_NUM_FIELDS = 10
# If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and
# collaboration graphs will show the relations between templates and their
# instances.
# The default value is: NO.
# This tag requires that the tag HAVE_DOT is set to YES.
TEMPLATE_RELATIONS = NO
# If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to
# YES then doxygen will generate a graph for each documented file showing the
# direct and indirect include dependencies of the file with other documented
# files.
# The default value is: YES.
# This tag requires that the tag HAVE_DOT is set to YES.
INCLUDE_GRAPH = YES
# If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are
# set to YES then doxygen will generate a graph for each documented file showing
# the direct and indirect include dependencies of the file with other documented
# files.
# The default value is: YES.
# This tag requires that the tag HAVE_DOT is set to YES.
INCLUDED_BY_GRAPH = YES
# If the CALL_GRAPH tag is set to YES then doxygen will generate a call
# dependency graph for every global function or class method.
#
# Note that enabling this option will significantly increase the time of a run.
# So in most cases it will be better to enable call graphs for selected
# functions only using the \callgraph command.
# The default value is: NO.
# This tag requires that the tag HAVE_DOT is set to YES.
CALL_GRAPH = YES
# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller
# dependency graph for every global function or class method.
#
# Note that enabling this option will significantly increase the time of a run.
# So in most cases it will be better to enable caller graphs for selected
# functions only using the \callergraph command.
# The default value is: NO.
# This tag requires that the tag HAVE_DOT is set to YES.
CALLER_GRAPH = YES
# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical
# hierarchy of all classes instead of a textual one.
# The default value is: YES.
# This tag requires that the tag HAVE_DOT is set to YES.
GRAPHICAL_HIERARCHY = YES
# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
# dependencies a directory has on other directories in a graphical way. The
# dependency relations are determined by the #include relations between the
# files in the directories.
# The default value is: YES.
# This tag requires that the tag HAVE_DOT is set to YES.
DIRECTORY_GRAPH = YES
# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
# generated by dot.
# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order
# to make the SVG files visible in IE 9+ (other browsers do not have this
# requirement).
# Possible values are: png, jpg, gif and svg.
# The default value is: png.
# This tag requires that the tag HAVE_DOT is set to YES.
DOT_IMAGE_FORMAT = png
# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to
# enable generation of interactive SVG images that allow zooming and panning.
#
# Note that this requires a modern browser other than Internet Explorer. Tested
# and working are Firefox, Chrome, Safari, and Opera.
# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make
# the SVG files visible. Older versions of IE do not have SVG support.
# The default value is: NO.
# This tag requires that the tag HAVE_DOT is set to YES.
INTERACTIVE_SVG = NO
# The DOT_PATH tag can be used to specify the path where the dot tool can be
# found. If left blank, it is assumed the dot tool can be found in the path.
# This tag requires that the tag HAVE_DOT is set to YES.
DOT_PATH =
# The DOTFILE_DIRS tag can be used to specify one or more directories that
# contain dot files that are included in the documentation (see the \dotfile
# command).
# This tag requires that the tag HAVE_DOT is set to YES.
DOTFILE_DIRS =
# The MSCFILE_DIRS tag can be used to specify one or more directories that
# contain msc files that are included in the documentation (see the \mscfile
# command).
MSCFILE_DIRS =
# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes
# that will be shown in the graph. If the number of nodes in a graph becomes
# larger than this value, doxygen will truncate the graph, which is visualized
# by representing a node as a red box. Note that doxygen if the number of direct
# children of the root node in a graph is already larger than
# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that
# the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
# Minimum value: 0, maximum value: 10000, default value: 50.
# This tag requires that the tag HAVE_DOT is set to YES.
DOT_GRAPH_MAX_NODES = 50
# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the graphs
# generated by dot. A depth value of 3 means that only nodes reachable from the
# root by following a path via at most 3 edges will be shown. Nodes that lay
# further from the root node will be omitted. Note that setting this option to 1
# or 2 may greatly reduce the computation time needed for large code bases. Also
# note that the size of a graph can be further restricted by
# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
# Minimum value: 0, maximum value: 1000, default value: 0.
# This tag requires that the tag HAVE_DOT is set to YES.
MAX_DOT_GRAPH_DEPTH = 0
# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent
# background. This is disabled by default, because dot on Windows does not seem
# to support this out of the box.
#
# Warning: Depending on the platform used, enabling this option may lead to
# badly anti-aliased labels on the edges of a graph (i.e. they become hard to
# read).
# The default value is: NO.
# This tag requires that the tag HAVE_DOT is set to YES.
DOT_TRANSPARENT = NO
# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output
# files in one run (i.e. multiple -o and -T options on the command line). This
# makes dot run faster, but since only newer versions of dot (>1.8.10) support
# this, this feature is disabled by default.
# The default value is: NO.
# This tag requires that the tag HAVE_DOT is set to YES.
DOT_MULTI_TARGETS = NO
# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page
# explaining the meaning of the various boxes and arrows in the dot generated
# graphs.
# The default value is: YES.
# This tag requires that the tag HAVE_DOT is set to YES.
GENERATE_LEGEND = YES
# If the DOT_CLEANUP tag is set to YES doxygen will remove the intermediate dot
# files that are used to generate the various graphs.
# The default value is: YES.
# This tag requires that the tag HAVE_DOT is set to YES.
DOT_CLEANUP = YES
================================================
FILE: LICENSE
================================================
BSD License
For Proxygen software
Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name Facebook nor the names of its contributors may be used to
endorse or promote products derived from this software without specific
prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
================================================
FILE: README.md
================================================
## Proxygen: Facebook's C++ HTTP Libraries
[](https://github.com/facebook/proxygen/actions/workflows/getdeps_linux.yml)
[](https://github.com/facebook/proxygen/actions/workflows/getdeps_mac.yml)
This project comprises the core C++ HTTP abstractions used at
Facebook. Internally, it is used as the basis for building many HTTP
servers, proxies, and clients. This release focuses on the common HTTP
abstractions and our simple HTTPServer framework. Future releases will
provide simple client APIs as well. The framework supports HTTP/1.1,
SPDY/3, SPDY/3.1, HTTP/2, and HTTP/3. The goal is to provide a simple,
performant, and modern C++ HTTP library.
We have a Google group for general discussions at https://groups.google.com/d/forum/facebook-proxygen.
The [original blog post](https://engineering.fb.com/production-engineering/introducing-proxygen-facebook-s-c-http-framework/)
also has more background on the project.
## Learn More in This Intro Video
[](https://www.youtube.com/watch?v=OsrBYHIYCYk)
### Installing
Note that currently this project has been tested on Ubuntu 18.04 and Mac OSX
although it likely works on many other platforms.
You will need at least 3 GiB of memory to compile `proxygen` and its
dependencies.
##### Easy Install
Just run `./build.sh` from the `proxygen/` directory to get and build all
the dependencies and `proxygen`. You can run the tests manually with `cd _build/ && make test`.
Then run `./install.sh` to install it. You can remove the temporary build directory (`_build`) and `./build.sh && ./install.sh`
to rebase the dependencies, and then rebuild and reinstall `proxygen`.
##### Package Managers
You can download and install proxygen using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager:
git clone https://github.com/Microsoft/vcpkg.git
cd vcpkg
./bootstrap-vcpkg.sh
./vcpkg integrate install
./vcpkg install proxygen
The proxygen port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository.
##### Other Platforms
If you are running on another platform, you may need to install several
packages first. Proxygen and `folly` are all Autotools based projects.
### Introduction
Directory structure and contents:
| Directory | Purpose |
|----------------------------|-------------------------------------------------------------------------------|
| `proxygen/external/` | Contains non-installed 3rd-party code proxygen depends on. |
| `proxygen/lib/` | Core networking abstractions. |
| `proxygen/lib/http/` | HTTP specific code. (including HTTP/2 and HTTP/3) |
| `proxygen/lib/services/` | Connection management and server code. |
| `proxygen/lib/utils/` | Miscellaneous helper code. |
| `proxygen/httpserver/` | Contains code wrapping `proxygen/lib/` for building simple C++ http servers. We recommend building on top of these APIs. |
### Architecture
The central abstractions to understand in `proxygen/lib` are the session, codec,
transaction, and handler. These are the lowest level abstractions, and we
don't generally recommend building off of these directly.
When bytes are read off the wire, the `HTTPCodec` stored inside
`HTTPSession` parses these into higher-level objects and associates with
it a transaction identifier. The codec then calls into `HTTPSession` which
is responsible for maintaining the mapping between transaction identifier
and `HTTPTransaction` objects. Each HTTP request/response pair has a
separate `HTTPTransaction` object. Finally, `HTTPTransaction` forwards the
call to a handler object which implements `HTTPTransaction:: Handler`. The
handler is responsible for implementing business logic for the request or
response.
The handler then calls back into the transaction to generate egress
(whether the egress is a request or response). The call flows from the
transaction back to the session, which uses the codec to convert the
higher-level semantics of the particular call into the appropriate bytes
to send on the wire.
The same handler and transaction interfaces are used to both create requests
and handle responses. The API is generic enough to allow
both. `HTTPSession` is specialized slightly differently depending on
whether you are using the connection to issue or respond to HTTP
requests.

Moving into higher levels of abstraction, `proxygen/HTTP server` has a
simpler set of APIs and is the recommended way to interface with `proxygen`
when acting as a server if you don't need the full control of the lower
level abstractions.
The basic components here are `HTTPServer`, `RequestHandlerFactory`, and
`RequestHandler`. An `HTTPServer` takes some configuration and is given a
`RequestHandlerFactory`. Once the server is started, the installed
`RequestHandlerFactory` spawns a `RequestHandler` for each HTTP
request. `RequestHandler` is a simple interface users of the library
implement. Subclasses of `RequestHandler` should use the inherited
protected member `ResponseHandler* downstream_` to send the response.
### Using it
Proxygen is a library. After installing it, you can build your C++
server. Try `cd` ing to the directory containing the echo server at
`proxygen/httpserver/samples/echo/`.
After building proxygen you can start the echo server with `_build/proxygen/httpserver/proxygen_echo`
and verify it works using curl in a different terminal:
```shell
$ curl -v http://localhost:11000/
* Trying 127.0.0.1...
* Connected to localhost (127.0.0.1) port 11000 (#0)
> GET / HTTP/1.1
> User-Agent: curl/7.35.0
> Host: localhost:11000
> Accept: */*
>
< HTTP/1.1 200 OK
< Request-Number: 1
< Date: Thu, 30 Oct 2014 17:07:36 GMT
< Connection: keep-alive
< Content-Length: 0
<
* Connection #0 to host localhost left intact
```
You can find other samples:
* a simple server that supports HTTP/2 server push (`_build/proxygen/httpserver/proxygen_push`),
* a simple server for static files (`_build/proxygen/httpserver/proxygen_static`)
* a simple fwdproxy (`_build/proxygen/httpserver/proxygen_proxy`)
* a curl-like client (`_build/proxygen/httpclient/samples/curl/proxygen_curl`)
### QUIC and HTTP/3
Proxygen supports HTTP/3!
It depends on Facebook's [mvfst](https://github.com/facebook/mvfst)
library for the [IETF QUIC](https://github.com/quicwg/base-drafts) transport
implementation.
This comes with a handy command-line utility that can be used as an HTTP/3
server and client.
Sample usage:
```shell
_build/proxygen/httpserver/hq --mode=server
_build/proxygen/httpserver/hq --mode=client --path=/
```
The utility supports the [qlog](https://github.com/quiclog/internet-drafts)
logging format; just start the server with the `--qlogger_path` option and many
knobs to tune both the quic transport and the http layer.
### Documentation
We use Doxygen for Proxygen's internal documentation. You can generate a
copy of these docs by running `doxygen Doxyfile` from the project
root. You'll want to look at `html/namespaceproxygen.html` to start. This
will also generate `folly` documentation.
### License
See [LICENSE](LICENSE).
### Contributing
Contributions to Proxygen are more than welcome. [Read the guidelines in CONTRIBUTING.md](CONTRIBUTING.md).
Make sure you've [signed the CLA](https://code.facebook.com/cla) before sending in a pull request.
### Whitehat
Facebook has a [bounty program](https://www.facebook.com/whitehat/) for
the safe disclosure of security bugs. If you find a vulnerability, please
go through the process outlined on that page and do not file a public issue.
================================================
FILE: build/README.md
================================================
# Building using `fbcode_builder`
Continuous integration builds are powered by `fbcode_builder`, a tiny tool
shared by several Facebook projects. Its files are in `./fbcode_builder`
(on Github) or in `fbcode/opensource/fbcode_builder` (inside Facebook's
repo).
Start with the READMEs in the `fbcode_builder` directory.
`./fbcode_builder_config.py` contains the project-specific configuration.
================================================
FILE: build/deps/github_hashes/facebook/folly-rev.txt
================================================
Subproject commit 4321dae73b18961089fc62532ee6dc011cbf1915
================================================
FILE: build/deps/github_hashes/facebook/mvfst-rev.txt
================================================
Subproject commit b229e95977aa5d4c271da60775f2b4bdbc40262a
================================================
FILE: build/deps/github_hashes/facebook/wangle-rev.txt
================================================
Subproject commit dbe97d189ad17a732477ecf4defdcf8cdac2c013
================================================
FILE: build/deps/github_hashes/facebookincubator/fizz-rev.txt
================================================
Subproject commit 8d74917ae1e80269c6e8e9da44f9713269d9c2ec
================================================
FILE: build/fbcode_builder/.gitignore
================================================
# Facebook-internal CI builds don't have write permission outside of the
# source tree, so we install all projects into this directory.
/facebook_ci
__pycache__/
*.pyc
================================================
FILE: build/fbcode_builder/CMake/FBBuildOptions.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
function (fb_activate_static_library_option)
option(USE_STATIC_DEPS_ON_UNIX
"If enabled, use static dependencies on unix systems. This is generally discouraged."
OFF
)
# Mark USE_STATIC_DEPS_ON_UNIX as an "advanced" option, since enabling it
# is generally discouraged.
mark_as_advanced(USE_STATIC_DEPS_ON_UNIX)
if(UNIX AND USE_STATIC_DEPS_ON_UNIX)
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a" PARENT_SCOPE)
endif()
option(PREFER_STATIC_DEPS_ON_UNIX
"If enabled, use static dependencies on unix systems as possible as we can. This is generally discouraged."
OFF
)
# Mark PREFER_STATIC_DEPS_ON_UNIX as an "advanced" option, since enabling it
# is generally discouraged.
mark_as_advanced(PREFER_STATIC_DEPS_ON_UNIX)
if(UNIX AND PREFER_STATIC_DEPS_ON_UNIX)
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so" PARENT_SCOPE)
endif()
endfunction()
================================================
FILE: build/fbcode_builder/CMake/FBCMakeParseArgs.cmake
================================================
#
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Helper function for parsing arguments to a CMake function.
#
# This function is very similar to CMake's built-in cmake_parse_arguments()
# function, with some improvements:
# - This function correctly handles empty arguments. (cmake_parse_arguments()
# ignores empty arguments.)
# - If a multi-value argument is specified more than once, the subsequent
# arguments are appended to the original list rather than replacing it. e.g.
# if "SOURCES" is a multi-value argument, and the argument list contains
# "SOURCES a b c SOURCES x y z" then the resulting value for SOURCES will be
# "a;b;c;x;y;z" rather than "x;y;z"
# - This function errors out by default on unrecognized arguments. You can
# pass in an extra "ALLOW_UNPARSED_ARGS" argument to make it behave like
# cmake_parse_arguments(), and return the unparsed arguments in a
# _UNPARSED_ARGUMENTS variable instead.
#
# It does look like cmake_parse_arguments() handled empty arguments correctly
# from CMake 3.0 through 3.3, but it seems like this was probably broken when
# it was turned into a built-in function in CMake 3.4. Here is discussion and
# patches that fixed this behavior prior to CMake 3.0:
# https://cmake.org/pipermail/cmake-developers/2013-November/020607.html
#
# The one downside to this function over the built-in cmake_parse_arguments()
# is that I don't think we can achieve the PARSE_ARGV behavior in a non-builtin
# function, so we can't properly handle arguments that contain ";". CMake will
# treat the ";" characters as list element separators, and treat it as multiple
# separate arguments.
#
function(fb_cmake_parse_args PREFIX OPTIONS ONE_VALUE_ARGS MULTI_VALUE_ARGS ARGS)
foreach(option IN LISTS ARGN)
if ("${option}" STREQUAL "ALLOW_UNPARSED_ARGS")
set(ALLOW_UNPARSED_ARGS TRUE)
else()
message(
FATAL_ERROR
"unknown optional argument for fb_cmake_parse_args(): ${option}"
)
endif()
endforeach()
# Define all options as FALSE in the parent scope to start with
foreach(var_name IN LISTS OPTIONS)
set("${PREFIX}_${var_name}" "FALSE" PARENT_SCOPE)
endforeach()
# TODO: We aren't extremely strict about error checking for one-value
# arguments here. e.g., we don't complain if a one-value argument is
# followed by another option/one-value/multi-value name rather than an
# argument. We also don't complain if a one-value argument is the last
# argument and isn't followed by a value.
list(APPEND all_args ${ONE_VALUE_ARGS})
list(APPEND all_args ${MULTI_VALUE_ARGS})
set(current_variable)
set(unparsed_args)
foreach(arg IN LISTS ARGS)
list(FIND OPTIONS "${arg}" opt_index)
if("${opt_index}" EQUAL -1)
list(FIND all_args "${arg}" arg_index)
if("${arg_index}" EQUAL -1)
# This argument does not match an argument name,
# must be an argument value
if("${current_variable}" STREQUAL "")
list(APPEND unparsed_args "${arg}")
else()
# Ugh, CMake lists have a pretty fundamental flaw: they cannot
# distinguish between an empty list and a list with a single empty
# element. We track our own SEEN_VALUES_arg setting to help
# distinguish this and behave properly here.
if ("${SEEN_${current_variable}}" AND "${${current_variable}}" STREQUAL "")
set("${current_variable}" ";${arg}")
else()
list(APPEND "${current_variable}" "${arg}")
endif()
set("SEEN_${current_variable}" TRUE)
endif()
else()
# We found a single- or multi-value argument name
set(current_variable "VALUES_${arg}")
set("SEEN_${arg}" TRUE)
endif()
else()
# We found an option variable
set("${PREFIX}_${arg}" "TRUE" PARENT_SCOPE)
set(current_variable)
endif()
endforeach()
foreach(arg_name IN LISTS ONE_VALUE_ARGS)
if(NOT "${SEEN_${arg_name}}")
unset("${PREFIX}_${arg_name}" PARENT_SCOPE)
elseif(NOT "${SEEN_VALUES_${arg_name}}")
# If the argument was seen but a value wasn't specified, error out.
# We require exactly one value to be specified.
message(
FATAL_ERROR "argument ${arg_name} was specified without a value"
)
else()
list(LENGTH "VALUES_${arg_name}" num_args)
if("${num_args}" EQUAL 0)
# We know an argument was specified and that we called list(APPEND).
# If CMake thinks the list is empty that means there is really a single
# empty element in the list.
set("${PREFIX}_${arg_name}" "" PARENT_SCOPE)
elseif("${num_args}" EQUAL 1)
list(GET "VALUES_${arg_name}" 0 arg_value)
set("${PREFIX}_${arg_name}" "${arg_value}" PARENT_SCOPE)
else()
message(
FATAL_ERROR "too many arguments specified for ${arg_name}: "
"${VALUES_${arg_name}}"
)
endif()
endif()
endforeach()
foreach(arg_name IN LISTS MULTI_VALUE_ARGS)
# If this argument name was never seen, then unset the parent scope
if (NOT "${SEEN_${arg_name}}")
unset("${PREFIX}_${arg_name}" PARENT_SCOPE)
else()
# TODO: Our caller still won't be able to distinguish between an empty
# list and a list with a single empty element. We can tell which is
# which, but CMake lists don't make it easy to show this to our caller.
set("${PREFIX}_${arg_name}" "${VALUES_${arg_name}}" PARENT_SCOPE)
endif()
endforeach()
# By default we fatal out on unparsed arguments, but return them to the
# caller if ALLOW_UNPARSED_ARGS was specified.
if (DEFINED unparsed_args)
if ("${ALLOW_UNPARSED_ARGS}")
set("${PREFIX}_UNPARSED_ARGUMENTS" "${unparsed_args}" PARENT_SCOPE)
else()
message(FATAL_ERROR "unrecognized arguments: ${unparsed_args}")
endif()
endif()
endfunction()
================================================
FILE: build/fbcode_builder/CMake/FBCompilerSettings.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
# This file applies common compiler settings that are shared across
# a number of Facebook opensource projects.
# Please use caution and your best judgement before making changes
# to these shared compiler settings in order to avoid accidentally
# breaking a build in another project!
if (WIN32)
include(FBCompilerSettingsMSVC)
else()
include(FBCompilerSettingsUnix)
endif()
================================================
FILE: build/fbcode_builder/CMake/FBCompilerSettingsMSVC.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
# This file applies common compiler settings that are shared across
# a number of Facebook opensource projects.
# Please use caution and your best judgement before making changes
# to these shared compiler settings in order to avoid accidentally
# breaking a build in another project!
add_compile_options(
/wd4250 # 'class1' : inherits 'class2::member' via dominance
/Zc:preprocessor # Enable conforming preprocessor for __VA_OPT__ support
)
================================================
FILE: build/fbcode_builder/CMake/FBCompilerSettingsUnix.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
# This file applies common compiler settings that are shared across
# a number of Facebook opensource projects.
# Please use caution and your best judgement before making changes
# to these shared compiler settings in order to avoid accidentally
# breaking a build in another project!
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Wextra -Wno-deprecated -Wno-deprecated-declarations")
================================================
FILE: build/fbcode_builder/CMake/FBPythonBinary.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
include(FBCMakeParseArgs)
#
# This file contains helper functions for building self-executing Python
# binaries.
#
# This is somewhat different than typical python installation with
# distutils/pip/virtualenv/etc. We primarily want to build a standalone
# executable, isolated from other Python packages on the system. We don't want
# to install files into the standard library python paths. This is more
# similar to PEX (https://github.com/pantsbuild/pex) and XAR
# (https://github.com/facebookincubator/xar). (In the future it would be nice
# to update this code to also support directly generating XAR files if XAR is
# available.)
#
# We also want to be able to easily define "libraries" of python files that can
# be shared and re-used between these standalone python executables, and can be
# shared across projects in different repositories. This means that we do need
# a way to "install" libraries so that they are visible to CMake builds in
# other repositories, without actually installing them in the standard python
# library paths.
#
# If the caller has not already found Python, do so now.
# If we fail to find python now we won't fail immediately, but
# add_fb_python_executable() or add_fb_python_library() will fatal out if they
# are used.
if(NOT TARGET Python3::Interpreter)
# CMake 3.12+ ships with a FindPython3.cmake module. Try using it first.
# We find with QUIET here, since otherwise this generates some noisy warnings
# on versions of CMake before 3.12
if (WIN32)
# On Windows we need both the Interpreter as well as the Development
# libraries.
find_package(Python3 COMPONENTS Interpreter Development QUIET)
else()
find_package(Python3 COMPONENTS Interpreter QUIET)
endif()
if(Python3_Interpreter_FOUND)
message(STATUS "Found Python 3: ${Python3_EXECUTABLE}")
else()
# Try with the FindPythonInterp.cmake module available in older CMake
# versions. Check to see if the caller has already searched for this
# themselves first.
if(NOT PYTHONINTERP_FOUND)
set(Python_ADDITIONAL_VERSIONS 3 3.6 3.5 3.4 3.3 3.2 3.1)
find_package(PythonInterp)
# TODO: On Windows we require the Python libraries as well.
# We currently do not search for them on this code path.
# For now we require building with CMake 3.12+ on Windows, so that the
# FindPython3 code path above is available.
endif()
if(PYTHONINTERP_FOUND)
if("${PYTHON_VERSION_MAJOR}" GREATER_EQUAL 3)
set(Python3_EXECUTABLE "${PYTHON_EXECUTABLE}")
add_custom_target(Python3::Interpreter)
else()
string(
CONCAT FBPY_FIND_PYTHON_ERR
"found Python ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}, "
"but need Python 3"
)
endif()
endif()
endif()
endif()
# Find our helper program.
# We typically install this in the same directory as this .cmake file.
find_program(
FB_MAKE_PYTHON_ARCHIVE "make_fbpy_archive.py"
PATHS ${CMAKE_MODULE_PATH}
)
set(FB_PY_TEST_MAIN "${CMAKE_CURRENT_LIST_DIR}/fb_py_test_main.py")
set(
FB_PY_TEST_DISCOVER_SCRIPT
"${CMAKE_CURRENT_LIST_DIR}/FBPythonTestAddTests.cmake"
)
set(
FB_PY_WIN_MAIN_C
"${CMAKE_CURRENT_LIST_DIR}/fb_py_win_main.c"
)
# An option to control the default installation location for
# install_fb_python_library(). This is relative to ${CMAKE_INSTALL_PREFIX}
set(
FBPY_LIB_INSTALL_DIR "lib/fb-py-libs" CACHE STRING
"The subdirectory where FB python libraries should be installed"
)
#
# Build a self-executing python binary.
#
# This accepts the same arguments as add_fb_python_library().
#
# In addition, a MAIN_MODULE argument is accepted. This argument specifies
# which module should be started as the __main__ module when the executable is
# run. If left unspecified, a __main__.py script must be present in the
# manifest.
#
function(add_fb_python_executable TARGET)
fb_py_check_available()
# Parse the arguments
set(one_value_args BASE_DIR NAMESPACE MAIN_MODULE TYPE)
set(multi_value_args SOURCES DEPENDS)
fb_cmake_parse_args(
ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
)
fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR)
# Use add_fb_python_library() to perform most of our source handling
add_fb_python_library(
"${TARGET}.main_lib"
BASE_DIR "${ARG_BASE_DIR}"
NAMESPACE "${ARG_NAMESPACE}"
SOURCES ${ARG_SOURCES}
DEPENDS ${ARG_DEPENDS}
)
set(
manifest_files
"$"
)
set(
source_files
"$"
)
# The command to build the executable archive.
#
# If we are using CMake 3.8+ we can use COMMAND_EXPAND_LISTS.
# CMP0067 isn't really the policy we care about, but seems like the best way
# to check if we are running 3.8+.
if (POLICY CMP0067)
set(extra_cmd_params COMMAND_EXPAND_LISTS)
set(make_py_args "${manifest_files}")
else()
set(extra_cmd_params)
set(make_py_args --manifest-separator "::" "$")
endif()
set(output_file "${TARGET}${CMAKE_EXECUTABLE_SUFFIX}")
if(WIN32)
set(zipapp_output "${TARGET}.py_zipapp")
else()
set(zipapp_output "${output_file}")
endif()
set(zipapp_output_file "${zipapp_output}")
set(is_dir_output FALSE)
if(DEFINED ARG_TYPE)
list(APPEND make_py_args "--type" "${ARG_TYPE}")
if ("${ARG_TYPE}" STREQUAL "dir")
set(is_dir_output TRUE)
# CMake doesn't really seem to like having a directory specified as an
# output; specify the __main__.py file as the output instead.
set(zipapp_output_file "${zipapp_output}/__main__.py")
# Update output_file to match zipapp_output_file for dir type
set(output_file "${zipapp_output_file}")
list(APPEND
extra_cmd_params
COMMAND "${CMAKE_COMMAND}" -E remove_directory "${zipapp_output}"
)
endif()
endif()
if(DEFINED ARG_MAIN_MODULE)
list(APPEND make_py_args "--main" "${ARG_MAIN_MODULE}")
endif()
add_custom_command(
OUTPUT "${zipapp_output_file}"
${extra_cmd_params}
COMMAND
"${Python3_EXECUTABLE}" "${FB_MAKE_PYTHON_ARCHIVE}"
-o "${zipapp_output}"
${make_py_args}
DEPENDS
${source_files}
"${TARGET}.main_lib.py_sources_built"
"${FB_MAKE_PYTHON_ARCHIVE}"
)
if(WIN32)
if(is_dir_output)
# TODO: generate a main executable that will invoke Python3
# with the correct main module inside the output directory
else()
add_executable("${TARGET}.winmain" "${FB_PY_WIN_MAIN_C}")
target_link_libraries("${TARGET}.winmain" Python3::Python)
# The Python3::Python target doesn't seem to be set up completely
# correctly on Windows for some reason, and we have to explicitly add
# ${Python3_LIBRARY_DIRS} to the target link directories.
target_link_directories(
"${TARGET}.winmain"
PUBLIC ${Python3_LIBRARY_DIRS}
)
add_custom_command(
OUTPUT "${output_file}"
DEPENDS "${TARGET}.winmain" "${zipapp_output_file}"
COMMAND
"cmd.exe" "/c" "copy" "/b"
"${TARGET}.winmain${CMAKE_EXECUTABLE_SUFFIX}+${zipapp_output}"
"${output_file}"
)
endif()
endif()
# Add an "ALL" target that depends on force ${TARGET},
# so that ${TARGET} will be included in the default list of build targets.
add_custom_target("${TARGET}.GEN_PY_EXE" ALL DEPENDS "${output_file}")
# Allow resolving the executable path for the target that we generate
# via a generator expression like:
# "WATCHMAN_WAIT_PATH=$"
set_property(TARGET "${TARGET}.GEN_PY_EXE"
PROPERTY EXECUTABLE "${CMAKE_CURRENT_BINARY_DIR}/${output_file}")
endfunction()
# Define a python unittest executable.
# The executable is built using add_fb_python_executable and has the
# following differences:
#
# Each of the source files specified in SOURCES will be imported
# and have unittest discovery performed upon them.
# Those sources will be imported in the top level namespace.
#
# The ENV argument allows specifying a list of "KEY=VALUE"
# pairs that will be used by the test runner to set up the environment
# in the child process prior to running the test. This is useful for
# passing additional configuration to the test.
function(add_fb_python_unittest TARGET)
# Parse the arguments
set(multi_value_args SOURCES DEPENDS ENV PROPERTIES)
set(
one_value_args
WORKING_DIRECTORY BASE_DIR NAMESPACE TEST_LIST DISCOVERY_TIMEOUT TYPE
)
fb_cmake_parse_args(
ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
)
fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR)
if(NOT ARG_WORKING_DIRECTORY)
# Default the working directory to the current binary directory.
# This matches the default behavior of add_test() and other standard
# test functions like gtest_discover_tests()
set(ARG_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
endif()
if(NOT ARG_TEST_LIST)
set(ARG_TEST_LIST "${TARGET}_TESTS")
endif()
if(NOT ARG_DISCOVERY_TIMEOUT)
set(ARG_DISCOVERY_TIMEOUT 5)
endif()
# Tell our test program the list of modules to scan for tests.
# We scan all modules directly listed in our SOURCES argument, and skip
# modules that came from dependencies in the DEPENDS list.
#
# This is written into a __test_modules__.py module that the test runner
# will look at.
set(
test_modules_path
"${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_test_modules.py"
)
file(WRITE "${test_modules_path}" "TEST_MODULES = [\n")
string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}")
if (NOT "${namespace_dir}" STREQUAL "")
set(namespace_dir "${namespace_dir}/")
endif()
set(test_modules)
foreach(src_path IN LISTS ARG_SOURCES)
fb_py_compute_dest_path(
abs_source dest_path
"${src_path}" "${namespace_dir}" "${ARG_BASE_DIR}"
)
string(REPLACE "/" "." module_name "${dest_path}")
string(REGEX REPLACE "\\.py$" "" module_name "${module_name}")
list(APPEND test_modules "${module_name}")
file(APPEND "${test_modules_path}" " '${module_name}',\n")
endforeach()
file(APPEND "${test_modules_path}" "]\n")
# The __main__ is provided by our runner wrapper/bootstrap
list(APPEND ARG_SOURCES "${FB_PY_TEST_MAIN}=__main__.py")
list(APPEND ARG_SOURCES "${test_modules_path}=__test_modules__.py")
if(NOT DEFINED ARG_TYPE)
set(ARG_TYPE "zipapp")
endif()
add_fb_python_executable(
"${TARGET}"
TYPE "${ARG_TYPE}"
NAMESPACE "${ARG_NAMESPACE}"
BASE_DIR "${ARG_BASE_DIR}"
SOURCES ${ARG_SOURCES}
DEPENDS ${ARG_DEPENDS}
)
# Run test discovery after the test executable is built.
# This logic is based on the code for gtest_discover_tests()
set(ctest_file_base "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}")
set(ctest_include_file "${ctest_file_base}_include.cmake")
set(ctest_tests_file "${ctest_file_base}_tests.cmake")
add_custom_command(
TARGET "${TARGET}.GEN_PY_EXE" POST_BUILD
BYPRODUCTS "${ctest_tests_file}"
COMMAND
"${CMAKE_COMMAND}"
-D "TEST_TARGET=${TARGET}"
-D "TEST_INTERPRETER=${Python3_EXECUTABLE}"
-D "TEST_ENV=${ARG_ENV}"
-D "TEST_EXECUTABLE=$"
-D "TEST_WORKING_DIR=${ARG_WORKING_DIRECTORY}"
-D "TEST_LIST=${ARG_TEST_LIST}"
-D "TEST_PREFIX=${TARGET}::"
-D "TEST_PROPERTIES=${ARG_PROPERTIES}"
-D "CTEST_FILE=${ctest_tests_file}"
-P "${FB_PY_TEST_DISCOVER_SCRIPT}"
VERBATIM
)
file(
WRITE "${ctest_include_file}"
"if(EXISTS \"${ctest_tests_file}\")\n"
" include(\"${ctest_tests_file}\")\n"
"else()\n"
" add_test(\"${TARGET}_NOT_BUILT\" \"${TARGET}_NOT_BUILT\")\n"
"endif()\n"
)
set_property(
DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES
"${ctest_include_file}"
)
endfunction()
#
# Define a python library.
#
# If you want to install a python library generated from this rule note that
# you need to use install_fb_python_library() rather than CMake's built-in
# install() function. This will make it available for other downstream
# projects to use in their add_fb_python_executable() and
# add_fb_python_library() calls. (You do still need to use `install(EXPORT)`
# later to install the CMake exports.)
#
# Parameters:
# - BASE_DIR :
# The base directory path to strip off from each source path. All source
# files must be inside this directory. If not specified it defaults to
# ${CMAKE_CURRENT_SOURCE_DIR}.
# - NAMESPACE :
# The destination namespace where these files should be installed in python
# binaries. If not specified, this defaults to the current relative path of
# ${CMAKE_CURRENT_SOURCE_DIR} inside ${CMAKE_SOURCE_DIR}. e.g., a python
# library defined in the directory repo_root/foo/bar will use a default
# namespace of "foo.bar"
# - SOURCES <...>:
# The python source files.
# You may optionally specify as source using the form: PATH=ALIAS where
# PATH is a relative path in the source tree and ALIAS is the relative
# path into which PATH should be rewritten. This is useful for mapping
# an executable script to the main module in a python executable.
# e.g.: `python/bin/watchman-wait=__main__.py`
# - DEPENDS <...>:
# Other python libraries that this one depends on.
# - INSTALL_DIR :
# The directory where this library should be installed.
# install_fb_python_library() must still be called later to perform the
# installation. If a relative path is given it will be treated relative to
# ${CMAKE_INSTALL_PREFIX}
#
# CMake is unfortunately pretty crappy at being able to define custom build
# rules & behaviors. It doesn't support transitive property propagation
# between custom targets; only the built-in add_executable() and add_library()
# targets support transitive properties.
#
# We hack around this janky CMake behavior by (ab)using interface libraries to
# propagate some of the data we want between targets, without actually
# generating a C library.
#
# add_fb_python_library(SOMELIB) generates the following things:
# - An INTERFACE library rule named SOMELIB.py_lib which tracks some
# information about transitive dependencies:
# - the transitive set of source files in the INTERFACE_SOURCES property
# - the transitive set of manifest files that this library depends on in
# the INTERFACE_INCLUDE_DIRECTORIES property.
# - A custom command that generates a SOMELIB.manifest file.
# This file contains the mapping of source files to desired destination
# locations in executables that depend on this library. This manifest file
# will then be read at build-time in order to build executables.
#
function(add_fb_python_library LIB_NAME)
fb_py_check_available()
# Parse the arguments
# We use fb_cmake_parse_args() rather than cmake_parse_arguments() since
# cmake_parse_arguments() does not handle empty arguments, and it is common
# for callers to want to specify an empty NAMESPACE parameter.
set(one_value_args BASE_DIR NAMESPACE INSTALL_DIR)
set(multi_value_args SOURCES DEPENDS)
fb_cmake_parse_args(
ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
)
fb_py_process_default_args(ARG_NAMESPACE ARG_BASE_DIR)
string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}")
if (NOT "${namespace_dir}" STREQUAL "")
set(namespace_dir "${namespace_dir}/")
endif()
if(NOT DEFINED ARG_INSTALL_DIR)
set(install_dir "${FBPY_LIB_INSTALL_DIR}/")
elseif("${ARG_INSTALL_DIR}" STREQUAL "")
set(install_dir "")
else()
set(install_dir "${ARG_INSTALL_DIR}/")
endif()
# message(STATUS "fb py library ${LIB_NAME}: "
# "NS=${namespace_dir} BASE=${ARG_BASE_DIR}")
# TODO: In the future it would be nice to support pre-compiling the source
# files. We could emit a rule to compile each source file and emit a
# .pyc/.pyo file here, and then have the manifest reference the pyc/pyo
# files.
# Define a library target to help pass around information about the library,
# and propagate dependency information.
#
# CMake make a lot of assumptions that libraries are C++ libraries. To help
# avoid confusion we name our target "${LIB_NAME}.py_lib" rather than just
# "${LIB_NAME}". This helps avoid confusion if callers try to use
# "${LIB_NAME}" on their own as a target name. (e.g., attempting to install
# it directly with install(TARGETS) won't work. Callers must use
# install_fb_python_library() instead.)
add_library("${LIB_NAME}.py_lib" INTERFACE)
# Emit the manifest file.
#
# We write the manifest file to a temporary path first, then copy it with
# configure_file(COPYONLY). This is necessary to get CMake to understand
# that "${manifest_path}" is generated by the CMake configure phase,
# and allow using it as a dependency for add_custom_command().
# (https://gitlab.kitware.com/cmake/cmake/issues/16367)
set(manifest_path "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}.manifest")
set(tmp_manifest "${manifest_path}.tmp")
file(WRITE "${tmp_manifest}" "FBPY_MANIFEST 1\n")
set(abs_sources)
foreach(src_path IN LISTS ARG_SOURCES)
fb_py_compute_dest_path(
abs_source dest_path
"${src_path}" "${namespace_dir}" "${ARG_BASE_DIR}"
)
list(APPEND abs_sources "${abs_source}")
target_sources(
"${LIB_NAME}.py_lib" INTERFACE
"$"
"$"
)
file(
APPEND "${tmp_manifest}"
"${abs_source} :: ${dest_path}\n"
)
endforeach()
configure_file("${tmp_manifest}" "${manifest_path}" COPYONLY)
target_include_directories(
"${LIB_NAME}.py_lib" INTERFACE
"$"
"$"
)
# Add a target that depends on all of the source files.
# This is needed in case some of the source files are generated. This will
# ensure that these source files are brought up-to-date before we build
# any python binaries that depend on this library.
add_custom_target("${LIB_NAME}.py_sources_built" DEPENDS ${abs_sources})
add_dependencies("${LIB_NAME}.py_lib" "${LIB_NAME}.py_sources_built")
# Hook up library dependencies, and also make the *.py_sources_built target
# depend on the sources for all of our dependencies also being up-to-date.
foreach(dep IN LISTS ARG_DEPENDS)
target_link_libraries("${LIB_NAME}.py_lib" INTERFACE "${dep}.py_lib")
# Mark that our .py_sources_built target depends on each our our dependent
# libraries. This serves two functions:
# - This causes CMake to generate an error message if one of the
# dependencies is never defined. The target_link_libraries() call above
# won't complain if one of the dependencies doesn't exist (since it is
# intended to allow passing in file names for plain library files rather
# than just targets).
# - It ensures that sources for our dependencies are built before any
# executable that depends on us. Note that we depend on "${dep}.py_lib"
# rather than "${dep}.py_sources_built" for this purpose because the
# ".py_sources_built" target won't be available for imported targets.
add_dependencies("${LIB_NAME}.py_sources_built" "${dep}.py_lib")
endforeach()
# Add a custom command to help with library installation, in case
# install_fb_python_library() is called later for this library.
# add_custom_command() only works with file dependencies defined in the same
# CMakeLists.txt file, so we want to make sure this is defined here, rather
# then where install_fb_python_library() is called.
# This command won't be run by default, but will only be run if it is needed
# by a subsequent install_fb_python_library() call.
#
# This command copies the library contents into the build directory.
# It would be nicer if we could skip this intermediate copy, and just run
# make_fbpy_archive.py at install time to copy them directly to the desired
# installation directory. Unfortunately this is difficult to do, and seems
# to interfere with some of the CMake code that wants to generate a manifest
# of installed files.
set(build_install_dir "${CMAKE_CURRENT_BINARY_DIR}/${LIB_NAME}.lib_install")
add_custom_command(
OUTPUT
"${build_install_dir}/${LIB_NAME}.manifest"
COMMAND "${CMAKE_COMMAND}" -E remove_directory "${build_install_dir}"
COMMAND
"${Python3_EXECUTABLE}" "${FB_MAKE_PYTHON_ARCHIVE}" --type lib-install
--install-dir "${LIB_NAME}"
-o "${build_install_dir}/${LIB_NAME}" "${manifest_path}"
DEPENDS
"${abs_sources}"
"${manifest_path}"
"${FB_MAKE_PYTHON_ARCHIVE}"
)
add_custom_target(
"${LIB_NAME}.py_lib_install"
DEPENDS "${build_install_dir}/${LIB_NAME}.manifest"
)
# Set some properties to pass through the install paths to
# install_fb_python_library()
#
# Passing through ${build_install_dir} allows install_fb_python_library()
# to work even if used from a different CMakeLists.txt file than where
# add_fb_python_library() was called (i.e. such that
# ${CMAKE_CURRENT_BINARY_DIR} is different between the two calls).
set(abs_install_dir "${install_dir}")
if(NOT IS_ABSOLUTE "${abs_install_dir}")
set(abs_install_dir "${CMAKE_INSTALL_PREFIX}/${abs_install_dir}")
endif()
string(REGEX REPLACE "/$" "" abs_install_dir "${abs_install_dir}")
set_target_properties(
"${LIB_NAME}.py_lib_install"
PROPERTIES
INSTALL_DIR "${abs_install_dir}"
BUILD_INSTALL_DIR "${build_install_dir}"
)
endfunction()
#
# Install an FB-style packaged python binary.
#
# - DESTINATION :
# Associate the installed target files with the given export-name.
#
function(install_fb_python_executable TARGET)
# Parse the arguments
set(one_value_args DESTINATION)
set(multi_value_args)
fb_cmake_parse_args(
ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
)
if(NOT DEFINED ARG_DESTINATION)
set(ARG_DESTINATION bin)
endif()
install(
PROGRAMS "$"
DESTINATION "${ARG_DESTINATION}"
)
endfunction()
#
# Install a python library.
#
# - EXPORT :
# Associate the installed target files with the given export-name.
#
# Note that unlike the built-in CMake install() function we do not accept a
# DESTINATION parameter. Instead, use the INSTALL_DIR parameter to
# add_fb_python_library() to set the installation location.
#
function(install_fb_python_library LIB_NAME)
set(one_value_args EXPORT)
fb_cmake_parse_args(ARG "" "${one_value_args}" "" "${ARGN}")
# Export our "${LIB_NAME}.py_lib" target so that it will be available to
# downstream projects in our installed CMake config files.
if(DEFINED ARG_EXPORT)
install(TARGETS "${LIB_NAME}.py_lib" EXPORT "${ARG_EXPORT}")
endif()
# add_fb_python_library() emits a .py_lib_install target that will prepare
# the installation directory. However, it isn't part of the "ALL" target and
# therefore isn't built by default.
#
# Make sure the ALL target depends on it now. We have to do this by
# introducing yet another custom target.
# Add it as a dependency to the ALL target now.
add_custom_target("${LIB_NAME}.py_lib_install_all" ALL)
add_dependencies(
"${LIB_NAME}.py_lib_install_all" "${LIB_NAME}.py_lib_install"
)
# Copy the intermediate install directory generated at build time into
# the desired install location.
get_target_property(dest_dir "${LIB_NAME}.py_lib_install" "INSTALL_DIR")
get_target_property(
build_install_dir "${LIB_NAME}.py_lib_install" "BUILD_INSTALL_DIR"
)
install(
DIRECTORY "${build_install_dir}/${LIB_NAME}"
DESTINATION "${dest_dir}"
)
install(
FILES "${build_install_dir}/${LIB_NAME}.manifest"
DESTINATION "${dest_dir}"
)
endfunction()
# Helper macro to process the BASE_DIR and NAMESPACE arguments for
# add_fb_python_executable() and add_fb_python_executable()
macro(fb_py_process_default_args NAMESPACE_VAR BASE_DIR_VAR)
# If the namespace was not specified, default to the relative path to the
# current directory (starting from the repository root).
if(NOT DEFINED "${NAMESPACE_VAR}")
file(
RELATIVE_PATH "${NAMESPACE_VAR}"
"${CMAKE_SOURCE_DIR}"
"${CMAKE_CURRENT_SOURCE_DIR}"
)
endif()
if(NOT DEFINED "${BASE_DIR_VAR}")
# If the base directory was not specified, default to the current directory
set("${BASE_DIR_VAR}" "${CMAKE_CURRENT_SOURCE_DIR}")
else()
# If the base directory was specified, always convert it to an
# absolute path.
get_filename_component("${BASE_DIR_VAR}" "${${BASE_DIR_VAR}}" ABSOLUTE)
endif()
endmacro()
function(fb_py_check_available)
# Make sure that Python 3 and our make_fbpy_archive.py helper script are
# available.
if(NOT Python3_EXECUTABLE)
if(FBPY_FIND_PYTHON_ERR)
message(FATAL_ERROR "Unable to find Python 3: ${FBPY_FIND_PYTHON_ERR}")
else()
message(FATAL_ERROR "Unable to find Python 3")
endif()
endif()
if (NOT FB_MAKE_PYTHON_ARCHIVE)
message(
FATAL_ERROR "unable to find make_fbpy_archive.py helper program (it "
"should be located in the same directory as FBPythonBinary.cmake)"
)
endif()
endfunction()
function(
fb_py_compute_dest_path
src_path_output dest_path_output src_path namespace_dir base_dir
)
if("${src_path}" MATCHES "=")
# We want to split the string on the `=` sign, but cmake doesn't
# provide much in the way of helpers for this, so we rewrite the
# `=` sign to `;` so that we can treat it as a cmake list and
# then index into the components
string(REPLACE "=" ";" src_path_list "${src_path}")
list(GET src_path_list 0 src_path)
# Note that we ignore the `namespace_dir` in the alias case
# in order to allow aliasing a source to the top level `__main__.py`
# filename.
list(GET src_path_list 1 dest_path)
else()
unset(dest_path)
endif()
get_filename_component(abs_source "${src_path}" ABSOLUTE)
if(NOT DEFINED dest_path)
file(RELATIVE_PATH rel_src "${ARG_BASE_DIR}" "${abs_source}")
if("${rel_src}" MATCHES "^../")
message(
FATAL_ERROR "${LIB_NAME}: source file \"${abs_source}\" is not inside "
"the base directory ${ARG_BASE_DIR}"
)
endif()
set(dest_path "${namespace_dir}${rel_src}")
endif()
set("${src_path_output}" "${abs_source}" PARENT_SCOPE)
set("${dest_path_output}" "${dest_path}" PARENT_SCOPE)
endfunction()
================================================
FILE: build/fbcode_builder/CMake/FBPythonTestAddTests.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
# Add a command to be emitted to the CTest file
set(ctest_script)
function(add_command CMD)
set(escaped_args "")
foreach(arg ${ARGN})
# Escape all arguments using "Bracket Argument" syntax
# We could skip this for argument that don't contain any special
# characters if we wanted to make the output slightly more human-friendly.
set(escaped_args "${escaped_args} [==[${arg}]==]")
endforeach()
set(ctest_script "${ctest_script}${CMD}(${escaped_args})\n" PARENT_SCOPE)
endfunction()
if(NOT EXISTS "${TEST_EXECUTABLE}")
message(FATAL_ERROR "Test executable does not exist: ${TEST_EXECUTABLE}")
endif()
execute_process(
COMMAND ${CMAKE_COMMAND} -E env ${TEST_ENV} "${TEST_INTERPRETER}" "${TEST_EXECUTABLE}" --list-tests
WORKING_DIRECTORY "${TEST_WORKING_DIR}"
OUTPUT_VARIABLE output
RESULT_VARIABLE result
)
if(NOT "${result}" EQUAL 0)
string(REPLACE "\n" "\n " output "${output}")
message(
FATAL_ERROR
"Error running test executable: ${TEST_EXECUTABLE}\n"
"Output:\n"
" ${output}\n"
)
endif()
# Parse output
string(REPLACE "\n" ";" tests_list "${output}")
foreach(test_name ${tests_list})
add_command(
add_test
"${TEST_PREFIX}${test_name}"
${CMAKE_COMMAND} -E env ${TEST_ENV}
"${TEST_INTERPRETER}" "${TEST_EXECUTABLE}" "${test_name}"
)
add_command(
set_tests_properties
"${TEST_PREFIX}${test_name}"
PROPERTIES
WORKING_DIRECTORY "${TEST_WORKING_DIR}"
${TEST_PROPERTIES}
)
endforeach()
# Set a list of discovered tests in the parent scope, in case users
# want access to this list as a CMake variable
if(TEST_LIST)
add_command(set ${TEST_LIST} ${tests_list})
endif()
file(WRITE "${CTEST_FILE}" "${ctest_script}")
================================================
FILE: build/fbcode_builder/CMake/FBThriftCppLibrary.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
include(FBCMakeParseArgs)
# Generate a C++ library from a thrift file
#
# Parameters:
# - SERVICES [ ...]
# The names of the services defined in the thrift file.
# - DEPENDS [ ...]
# A list of other thrift C++ libraries that this library depends on.
# - OPTIONS [ ...]
# A list of options to pass to the thrift compiler.
# - INCLUDE_DIR
# The sub-directory where generated headers will be installed.
# Defaults to "include" if not specified. The caller must still call
# install() to install the thrift library if desired.
# - THRIFT_INCLUDE_DIR
# The sub-directory where generated headers will be installed.
# Defaults to "${INCLUDE_DIR}/thrift-files" if not specified.
# The caller must still call install() to install the thrift library if
# desired.
function(add_fbthrift_cpp_library LIB_NAME THRIFT_FILE)
# Parse the arguments
set(one_value_args INCLUDE_DIR THRIFT_INCLUDE_DIR)
set(multi_value_args SERVICES DEPENDS OPTIONS)
fb_cmake_parse_args(
ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
)
if(NOT DEFINED ARG_INCLUDE_DIR)
set(ARG_INCLUDE_DIR "include")
endif()
if(NOT DEFINED ARG_THRIFT_INCLUDE_DIR)
set(ARG_THRIFT_INCLUDE_DIR "${ARG_INCLUDE_DIR}/thrift-files")
endif()
get_filename_component(base ${THRIFT_FILE} NAME_WE)
get_filename_component(
output_dir
${CMAKE_CURRENT_BINARY_DIR}/${THRIFT_FILE}
DIRECTORY
)
# Generate relative paths in #includes
file(
RELATIVE_PATH include_prefix
"${CMAKE_SOURCE_DIR}"
"${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}"
)
get_filename_component(include_prefix ${include_prefix} DIRECTORY)
if (NOT "${include_prefix}" STREQUAL "")
list(APPEND ARG_OPTIONS "include_prefix=${include_prefix}")
endif()
# CMake 3.12 is finally getting a list(JOIN) function, but until then
# treating the list as a string and replacing the semicolons is good enough.
string(REPLACE ";" "," GEN_ARG_STR "${ARG_OPTIONS}")
# Compute the list of generated files
list(APPEND generated_headers
"${output_dir}/gen-cpp2/${base}_constants.h"
"${output_dir}/gen-cpp2/${base}_types.h"
"${output_dir}/gen-cpp2/${base}_types.tcc"
"${output_dir}/gen-cpp2/${base}_types_custom_protocol.h"
"${output_dir}/gen-cpp2/${base}_metadata.h"
)
list(APPEND generated_sources
"${output_dir}/gen-cpp2/${base}_constants.cpp"
"${output_dir}/gen-cpp2/${base}_data.h"
"${output_dir}/gen-cpp2/${base}_data.cpp"
"${output_dir}/gen-cpp2/${base}_types.cpp"
"${output_dir}/gen-cpp2/${base}_types_binary.cpp"
"${output_dir}/gen-cpp2/${base}_types_compact.cpp"
"${output_dir}/gen-cpp2/${base}_types_serialization.cpp"
"${output_dir}/gen-cpp2/${base}_metadata.cpp"
)
foreach(service IN LISTS ARG_SERVICES)
list(APPEND generated_headers
"${output_dir}/gen-cpp2/${service}.h"
"${output_dir}/gen-cpp2/${service}.tcc"
"${output_dir}/gen-cpp2/${service}AsyncClient.h"
"${output_dir}/gen-cpp2/${service}_custom_protocol.h"
)
list(APPEND generated_sources
"${output_dir}/gen-cpp2/${service}.cpp"
"${output_dir}/gen-cpp2/${service}AsyncClient.cpp"
"${output_dir}/gen-cpp2/${service}_processmap_binary.cpp"
"${output_dir}/gen-cpp2/${service}_processmap_compact.cpp"
)
endforeach()
# This generator expression gets the list of include directories required
# for all of our dependencies.
# It requires using COMMAND_EXPAND_LISTS in the add_custom_command() call
# below. COMMAND_EXPAND_LISTS is only available in CMake 3.8+
# If we really had to support older versions of CMake we would probably need
# to use a wrapper script around the thrift compiler that could take the
# include list as a single argument and split it up before invoking the
# thrift compiler.
if (NOT POLICY CMP0067)
message(FATAL_ERROR "add_fbthrift_cpp_library() requires CMake 3.8+")
endif()
set(
thrift_include_options
"-I;$,;-I;>"
)
# Emit the rule to run the thrift compiler
add_custom_command(
OUTPUT
${generated_headers}
${generated_sources}
COMMAND_EXPAND_LISTS
COMMAND
"${CMAKE_COMMAND}" -E make_directory "${output_dir}"
COMMAND
"${FBTHRIFT_COMPILER}"
--legacy-strict
--gen "mstch_cpp2:${GEN_ARG_STR}"
"${thrift_include_options}"
-I "${FBTHRIFT_INCLUDE_DIR}"
-o "${output_dir}"
"${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}"
WORKING_DIRECTORY
"${CMAKE_BINARY_DIR}"
MAIN_DEPENDENCY
"${THRIFT_FILE}"
DEPENDS
${ARG_DEPENDS}
"${FBTHRIFT_COMPILER}"
)
# Now emit the library rule to compile the sources
if (BUILD_SHARED_LIBS)
set(LIB_TYPE SHARED)
else ()
set(LIB_TYPE STATIC)
endif ()
add_library(
"${LIB_NAME}" ${LIB_TYPE}
${generated_sources}
)
target_include_directories(
"${LIB_NAME}"
PUBLIC
"$"
"$"
${Xxhash_INCLUDE_DIR}
)
target_link_libraries(
"${LIB_NAME}"
PUBLIC
${ARG_DEPENDS}
FBThrift::thriftcpp2
Folly::folly
mvfst::mvfst_server_async_tran
mvfst::mvfst_server
${Xxhash_LIBRARY}
)
# Add ${generated_headers} to the PUBLIC_HEADER property for ${LIB_NAME}
#
# This allows callers to install it using
# "install(TARGETS ${LIB_NAME} PUBLIC_HEADER)"
# However, note that CMake's PUBLIC_HEADER behavior is rather inflexible,
# and does have any way to preserve header directory structure. Callers
# must be careful to use the correct PUBLIC_HEADER DESTINATION parameter
# when doing this, to put the files the correct directory themselves.
# We define a HEADER_INSTALL_DIR property with the include directory prefix,
# so typically callers should specify the PUBLIC_HEADER DESTINATION as
# "$"
set_property(
TARGET "${LIB_NAME}"
PROPERTY PUBLIC_HEADER ${generated_headers}
)
# Define a dummy interface library to help propagate the thrift include
# directories between dependencies.
add_library("${LIB_NAME}.thrift_includes" INTERFACE)
target_include_directories(
"${LIB_NAME}.thrift_includes"
INTERFACE
"$"
"$"
)
foreach(dep IN LISTS ARG_DEPENDS)
target_link_libraries(
"${LIB_NAME}.thrift_includes"
INTERFACE "${dep}.thrift_includes"
)
endforeach()
set_target_properties(
"${LIB_NAME}"
PROPERTIES
EXPORT_PROPERTIES "THRIFT_INSTALL_DIR"
THRIFT_INSTALL_DIR "${ARG_THRIFT_INCLUDE_DIR}/${include_prefix}"
HEADER_INSTALL_DIR "${ARG_INCLUDE_DIR}/${include_prefix}/gen-cpp2"
)
endfunction()
================================================
FILE: build/fbcode_builder/CMake/FBThriftLibrary.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
include(FBCMakeParseArgs)
include(FBThriftPyLibrary)
include(FBThriftCppLibrary)
#
# add_fbthrift_library()
#
# This is a convenience function that generates thrift libraries for multiple
# languages.
#
# For example:
# add_fbthrift_library(
# foo foo.thrift
# LANGUAGES cpp py
# SERVICES Foo
# DEPENDS bar)
#
# will be expanded into two separate calls:
#
# add_fbthrift_cpp_library(foo_cpp foo.thrift SERVICES Foo DEPENDS bar_cpp)
# add_fbthrift_py_library(foo_py foo.thrift SERVICES Foo DEPENDS bar_py)
#
function(add_fbthrift_library LIB_NAME THRIFT_FILE)
# Parse the arguments
set(one_value_args PY_NAMESPACE INCLUDE_DIR THRIFT_INCLUDE_DIR)
set(multi_value_args SERVICES DEPENDS LANGUAGES CPP_OPTIONS PY_OPTIONS)
fb_cmake_parse_args(
ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
)
if(NOT DEFINED ARG_INCLUDE_DIR)
set(ARG_INCLUDE_DIR "include")
endif()
if(NOT DEFINED ARG_THRIFT_INCLUDE_DIR)
set(ARG_THRIFT_INCLUDE_DIR "${ARG_INCLUDE_DIR}/thrift-files")
endif()
# CMake 3.12+ adds list(TRANSFORM) which would be nice to use here, but for
# now we still want to support older versions of CMake.
set(CPP_DEPENDS)
set(PY_DEPENDS)
foreach(dep IN LISTS ARG_DEPENDS)
list(APPEND CPP_DEPENDS "${dep}_cpp")
list(APPEND PY_DEPENDS "${dep}_py")
endforeach()
foreach(lang IN LISTS ARG_LANGUAGES)
if ("${lang}" STREQUAL "cpp")
add_fbthrift_cpp_library(
"${LIB_NAME}_cpp" "${THRIFT_FILE}"
SERVICES ${ARG_SERVICES}
DEPENDS ${CPP_DEPENDS}
OPTIONS ${ARG_CPP_OPTIONS}
INCLUDE_DIR "${ARG_INCLUDE_DIR}"
THRIFT_INCLUDE_DIR "${ARG_THRIFT_INCLUDE_DIR}"
)
elseif ("${lang}" STREQUAL "py" OR "${lang}" STREQUAL "python")
if (DEFINED ARG_PY_NAMESPACE)
set(namespace_args NAMESPACE "${ARG_PY_NAMESPACE}")
endif()
add_fbthrift_py_library(
"${LIB_NAME}_py" "${THRIFT_FILE}"
SERVICES ${ARG_SERVICES}
${namespace_args}
DEPENDS ${PY_DEPENDS}
OPTIONS ${ARG_PY_OPTIONS}
THRIFT_INCLUDE_DIR "${ARG_THRIFT_INCLUDE_DIR}"
)
else()
message(
FATAL_ERROR "unknown language for thrift library ${LIB_NAME}: ${lang}"
)
endif()
endforeach()
endfunction()
================================================
FILE: build/fbcode_builder/CMake/FBThriftPyLibrary.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
include(FBCMakeParseArgs)
include(FBPythonBinary)
# Generate a Python library from a thrift file
function(add_fbthrift_py_library LIB_NAME THRIFT_FILE)
# Parse the arguments
set(one_value_args NAMESPACE THRIFT_INCLUDE_DIR)
set(multi_value_args SERVICES DEPENDS OPTIONS)
fb_cmake_parse_args(
ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
)
if(NOT DEFINED ARG_THRIFT_INCLUDE_DIR)
set(ARG_THRIFT_INCLUDE_DIR "include/thrift-files")
endif()
get_filename_component(base ${THRIFT_FILE} NAME_WE)
set(output_dir "${CMAKE_CURRENT_BINARY_DIR}/${THRIFT_FILE}-py")
# Parse the namespace value
if (NOT DEFINED ARG_NAMESPACE)
set(ARG_NAMESPACE "${base}")
endif()
string(REPLACE "." "/" namespace_dir "${ARG_NAMESPACE}")
set(py_output_dir "${output_dir}/gen-py/${namespace_dir}")
list(APPEND generated_sources
"${py_output_dir}/__init__.py"
"${py_output_dir}/ttypes.py"
"${py_output_dir}/constants.py"
)
foreach(service IN LISTS ARG_SERVICES)
list(APPEND generated_sources
${py_output_dir}/${service}.py
)
endforeach()
# Define a dummy interface library to help propagate the thrift include
# directories between dependencies.
add_library("${LIB_NAME}.thrift_includes" INTERFACE)
target_include_directories(
"${LIB_NAME}.thrift_includes"
INTERFACE
"$"
"$"
)
foreach(dep IN LISTS ARG_DEPENDS)
target_link_libraries(
"${LIB_NAME}.thrift_includes"
INTERFACE "${dep}.thrift_includes"
)
endforeach()
# This generator expression gets the list of include directories required
# for all of our dependencies.
# It requires using COMMAND_EXPAND_LISTS in the add_custom_command() call
# below. COMMAND_EXPAND_LISTS is only available in CMake 3.8+
# If we really had to support older versions of CMake we would probably need
# to use a wrapper script around the thrift compiler that could take the
# include list as a single argument and split it up before invoking the
# thrift compiler.
if (NOT POLICY CMP0067)
message(FATAL_ERROR "add_fbthrift_py_library() requires CMake 3.8+")
endif()
set(
thrift_include_options
"-I;$,;-I;>"
)
# Always force generation of "new-style" python classes for Python 2
list(APPEND ARG_OPTIONS "new_style")
# CMake 3.12 is finally getting a list(JOIN) function, but until then
# treating the list as a string and replacing the semicolons is good enough.
string(REPLACE ";" "," GEN_ARG_STR "${ARG_OPTIONS}")
# Emit the rule to run the thrift compiler
add_custom_command(
OUTPUT
${generated_sources}
COMMAND_EXPAND_LISTS
COMMAND
"${CMAKE_COMMAND}" -E make_directory "${output_dir}"
COMMAND
"${FBTHRIFT_COMPILER}"
--legacy-strict
--gen "py:${GEN_ARG_STR}"
"${thrift_include_options}"
-o "${output_dir}"
"${CMAKE_CURRENT_SOURCE_DIR}/${THRIFT_FILE}"
WORKING_DIRECTORY
"${CMAKE_BINARY_DIR}"
MAIN_DEPENDENCY
"${THRIFT_FILE}"
DEPENDS
"${FBTHRIFT_COMPILER}"
)
# We always want to pass the namespace as "" to this call:
# thrift will already emit the files with the desired namespace prefix under
# gen-py. We don't want add_fb_python_library() to prepend the namespace a
# second time.
add_fb_python_library(
"${LIB_NAME}"
BASE_DIR "${output_dir}/gen-py"
NAMESPACE ""
SOURCES ${generated_sources}
DEPENDS ${ARG_DEPENDS} FBThrift::thrift_py
)
endfunction()
================================================
FILE: build/fbcode_builder/CMake/FindCares.cmake
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
find_path(CARES_INCLUDE_DIR NAMES ares.h)
find_library(CARES_LIBRARIES NAMES cares)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Cares DEFAULT_MSG CARES_LIBRARIES CARES_INCLUDE_DIR)
mark_as_advanced(
CARES_LIBRARIES
CARES_INCLUDE_DIR
)
if(NOT TARGET cares)
if("${CARES_LIBRARIES}" MATCHES ".*.a$")
add_library(cares STATIC IMPORTED)
else()
add_library(cares SHARED IMPORTED)
endif()
set_target_properties(
cares
PROPERTIES
IMPORTED_LOCATION ${CARES_LIBRARIES}
INTERFACE_INCLUDE_DIRECTORIES ${CARES_INCLUDE_DIR}
)
endif()
================================================
FILE: build/fbcode_builder/CMake/FindDoubleConversion.cmake
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# Finds libdouble-conversion.
#
# This module defines:
# DOUBLE_CONVERSION_INCLUDE_DIR
# DOUBLE_CONVERSION_LIBRARY
#
find_path(DOUBLE_CONVERSION_INCLUDE_DIR double-conversion/double-conversion.h)
find_library(DOUBLE_CONVERSION_LIBRARY NAMES double-conversion)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(
DoubleConversion
DEFAULT_MSG
DOUBLE_CONVERSION_LIBRARY DOUBLE_CONVERSION_INCLUDE_DIR)
mark_as_advanced(DOUBLE_CONVERSION_INCLUDE_DIR DOUBLE_CONVERSION_LIBRARY)
================================================
FILE: build/fbcode_builder/CMake/FindGMock.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
# Find libgmock
#
# LIBGMOCK_DEFINES - List of defines when using libgmock.
# LIBGMOCK_INCLUDE_DIR - where to find gmock/gmock.h, etc.
# LIBGMOCK_LIBRARIES - List of libraries when using libgmock.
# LIBGMOCK_FOUND - True if libgmock found.
IF (LIBGMOCK_INCLUDE_DIR)
# Already in cache, be silent
SET(LIBGMOCK_FIND_QUIETLY TRUE)
ENDIF ()
find_package(GTest CONFIG QUIET)
if (TARGET GTest::gmock)
get_target_property(LIBGMOCK_DEFINES GTest::gtest INTERFACE_COMPILE_DEFINITIONS)
if (NOT ${LIBGMOCK_DEFINES})
# Explicitly set to empty string if not found to avoid it being
# set to NOTFOUND and breaking compilation
set(LIBGMOCK_DEFINES "")
endif()
get_target_property(LIBGMOCK_INCLUDE_DIR GTest::gtest INTERFACE_INCLUDE_DIRECTORIES)
set(LIBGMOCK_LIBRARIES GTest::gmock_main GTest::gmock GTest::gtest)
set(LIBGMOCK_FOUND ON)
message(STATUS "Found gmock via config, defines=${LIBGMOCK_DEFINES}, include=${LIBGMOCK_INCLUDE_DIR}, libs=${LIBGMOCK_LIBRARIES}")
else()
FIND_PATH(LIBGMOCK_INCLUDE_DIR gmock/gmock.h)
FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_DEBUG NAMES gmock_maind)
FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY_RELEASE NAMES gmock_main)
FIND_LIBRARY(LIBGMOCK_LIBRARY_DEBUG NAMES gmockd)
FIND_LIBRARY(LIBGMOCK_LIBRARY_RELEASE NAMES gmock)
FIND_LIBRARY(LIBGTEST_LIBRARY_DEBUG NAMES gtestd)
FIND_LIBRARY(LIBGTEST_LIBRARY_RELEASE NAMES gtest)
find_package(Threads REQUIRED)
INCLUDE(SelectLibraryConfigurations)
SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK_MAIN)
SELECT_LIBRARY_CONFIGURATIONS(LIBGMOCK)
SELECT_LIBRARY_CONFIGURATIONS(LIBGTEST)
set(LIBGMOCK_LIBRARIES
${LIBGMOCK_MAIN_LIBRARY}
${LIBGMOCK_LIBRARY}
${LIBGTEST_LIBRARY}
Threads::Threads
)
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
# The GTEST_LINKED_AS_SHARED_LIBRARY macro must be set properly on Windows.
#
# There isn't currently an easy way to determine if a library was compiled as
# a shared library on Windows, so just assume we've been built against a
# shared build of gmock for now.
SET(LIBGMOCK_DEFINES "GTEST_LINKED_AS_SHARED_LIBRARY=1" CACHE STRING "")
endif()
# handle the QUIETLY and REQUIRED arguments and set LIBGMOCK_FOUND to TRUE if
# all listed variables are TRUE
INCLUDE(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(
GMock
DEFAULT_MSG
LIBGMOCK_MAIN_LIBRARY
LIBGMOCK_LIBRARY
LIBGTEST_LIBRARY
LIBGMOCK_LIBRARIES
LIBGMOCK_INCLUDE_DIR
)
MARK_AS_ADVANCED(
LIBGMOCK_DEFINES
LIBGMOCK_MAIN_LIBRARY
LIBGMOCK_LIBRARY
LIBGTEST_LIBRARY
LIBGMOCK_LIBRARIES
LIBGMOCK_INCLUDE_DIR
)
endif()
================================================
FILE: build/fbcode_builder/CMake/FindGflags.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
# Find libgflags.
# There's a lot of compatibility cruft going on in here, both
# to deal with changes across the FB consumers of this and also
# to deal with variances in behavior of cmake itself.
#
# Since this file is named FindGflags.cmake the cmake convention
# is for the module to export both GFLAGS_FOUND and Gflags_FOUND.
# The convention expected by consumers is that we export the
# following variables, even though these do not match the cmake
# conventions:
#
# LIBGFLAGS_INCLUDE_DIR - where to find gflags/gflags.h, etc.
# LIBGFLAGS_LIBRARY - List of libraries when using libgflags.
# LIBGFLAGS_FOUND - True if libgflags found.
#
# We need to be able to locate gflags both from an installed
# cmake config file and just from the raw headers and libs, so
# test for the former and then the latter, and then stick
# the results together and export them into the variables
# listed above.
#
# For forwards compatibility, we export the following variables:
#
# gflags_INCLUDE_DIR - where to find gflags/gflags.h, etc.
# gflags_TARGET / GFLAGS_TARGET / gflags_LIBRARIES
# - List of libraries when using libgflags.
# gflags_FOUND - True if libgflags found.
#
IF (LIBGFLAGS_INCLUDE_DIR)
# Already in cache, be silent
SET(Gflags_FIND_QUIETLY TRUE)
ENDIF ()
find_package(gflags CONFIG QUIET)
if (gflags_FOUND)
if (NOT Gflags_FIND_QUIETLY)
message(STATUS "Found gflags from package config ${gflags_CONFIG}")
endif()
# Re-export the config-specified libs with our local names
set(LIBGFLAGS_LIBRARY ${gflags_LIBRARIES})
set(LIBGFLAGS_INCLUDE_DIR ${gflags_INCLUDE_DIR})
if(NOT EXISTS "${gflags_INCLUDE_DIR}")
# The gflags-devel RPM on recent RedHat-based systems is somewhat broken.
# RedHat symlinks /lib64 to /usr/lib64, and this breaks some of the
# relative path computation performed in gflags-config.cmake. The package
# config file ends up being found via /lib64, but the relative path
# computation it does only works if it was found in /usr/lib64.
# If gflags_INCLUDE_DIR does not actually exist, simply default it to
# /usr/include on these systems.
set(LIBGFLAGS_INCLUDE_DIR "/usr/include")
set(GFLAGS_INCLUDE_DIR "/usr/include")
endif()
set(LIBGFLAGS_FOUND ${gflags_FOUND})
# cmake module compat
set(GFLAGS_FOUND ${gflags_FOUND})
set(Gflags_FOUND ${gflags_FOUND})
else()
FIND_PATH(LIBGFLAGS_INCLUDE_DIR gflags/gflags.h)
FIND_LIBRARY(LIBGFLAGS_LIBRARY_DEBUG NAMES gflagsd gflags_staticd)
FIND_LIBRARY(LIBGFLAGS_LIBRARY_RELEASE NAMES gflags gflags_static)
INCLUDE(SelectLibraryConfigurations)
SELECT_LIBRARY_CONFIGURATIONS(LIBGFLAGS)
# handle the QUIETLY and REQUIRED arguments and set LIBGFLAGS_FOUND to TRUE if
# all listed variables are TRUE
INCLUDE(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(gflags DEFAULT_MSG LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR)
# cmake module compat
set(Gflags_FOUND ${GFLAGS_FOUND})
# compat with some existing FindGflags consumers
set(LIBGFLAGS_FOUND ${GFLAGS_FOUND})
# Compat with the gflags CONFIG based detection
set(gflags_FOUND ${GFLAGS_FOUND})
set(gflags_INCLUDE_DIR ${LIBGFLAGS_INCLUDE_DIR})
set(gflags_LIBRARIES ${LIBGFLAGS_LIBRARY})
set(GFLAGS_TARGET ${LIBGFLAGS_LIBRARY})
set(gflags_TARGET ${LIBGFLAGS_LIBRARY})
MARK_AS_ADVANCED(LIBGFLAGS_LIBRARY LIBGFLAGS_INCLUDE_DIR)
endif()
# Compat with the gflags CONFIG based detection
if (LIBGFLAGS_FOUND AND NOT TARGET gflags)
add_library(gflags UNKNOWN IMPORTED)
if(TARGET gflags-shared)
# If the installed gflags CMake package config defines a gflags-shared
# target but not gflags, just make the gflags target that we define
# depend on the gflags-shared target.
target_link_libraries(gflags INTERFACE gflags-shared)
# Export LIBGFLAGS_LIBRARY as the gflags-shared target in this case.
set(LIBGFLAGS_LIBRARY gflags-shared)
else()
set_target_properties(
gflags
PROPERTIES
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
IMPORTED_LOCATION "${LIBGFLAGS_LIBRARY}"
INTERFACE_INCLUDE_DIRECTORIES "${LIBGFLAGS_INCLUDE_DIR}"
)
endif()
endif()
================================================
FILE: build/fbcode_builder/CMake/FindGlog.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
# - Try to find Glog
# Once done, this will define
#
# GLOG_FOUND - system has Glog
# GLOG_INCLUDE_DIRS - the Glog include directories
# GLOG_LIBRARIES - link these to use Glog
include(FindPackageHandleStandardArgs)
include(SelectLibraryConfigurations)
find_library(GLOG_LIBRARY_RELEASE glog
PATHS ${GLOG_LIBRARYDIR})
find_library(GLOG_LIBRARY_DEBUG glogd
PATHS ${GLOG_LIBRARYDIR})
find_path(GLOG_INCLUDE_DIR glog/logging.h
PATHS ${GLOG_INCLUDEDIR})
select_library_configurations(GLOG)
find_package_handle_standard_args(Glog DEFAULT_MSG
GLOG_LIBRARY
GLOG_INCLUDE_DIR)
mark_as_advanced(
GLOG_LIBRARY
GLOG_INCLUDE_DIR)
set(GLOG_LIBRARIES ${GLOG_LIBRARY})
set(GLOG_INCLUDE_DIRS ${GLOG_INCLUDE_DIR})
if (NOT TARGET glog::glog)
add_library(glog::glog UNKNOWN IMPORTED)
set_target_properties(glog::glog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${GLOG_INCLUDE_DIRS}")
set_target_properties(glog::glog PROPERTIES IMPORTED_LINK_INTERFACE_LANGUAGES "C" IMPORTED_LOCATION "${GLOG_LIBRARIES}")
set_target_properties(glog::glog PROPERTIES
INTERFACE_COMPILE_DEFINITIONS "GLOG_USE_GLOG_EXPORT")
find_package(Gflags)
if(GFLAGS_FOUND)
message(STATUS "Found gflags as a dependency of glog::glog, include=${LIBGFLAGS_INCLUDE_DIR}, libs=${LIBGFLAGS_LIBRARY}")
set_property(TARGET glog::glog APPEND PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES ${LIBGFLAGS_LIBRARY})
endif()
find_package(LibUnwind)
if(LIBUNWIND_FOUND)
message(STATUS "Found LibUnwind as a dependency of glog::glog, include=${LIBUNWIND_INCLUDE_DIR}, libs=${LIBUNWIND_LIBRARY}")
set_property(TARGET glog::glog APPEND PROPERTY IMPORTED_LINK_INTERFACE_LIBRARIES ${LIBUNWIND_LIBRARY})
endif()
endif()
================================================
FILE: build/fbcode_builder/CMake/FindLMDB.cmake
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2.
find_library(LMDB_LIBRARIES NAMES lmdb liblmdb)
mark_as_advanced(LMDB_LIBRARIES)
find_path(LMDB_INCLUDE_DIR NAMES lmdb.h)
mark_as_advanced(LMDB_INCLUDE_DIR)
find_package_handle_standard_args(
LMDB
REQUIRED_VARS LMDB_LIBRARIES LMDB_INCLUDE_DIR)
if(LMDB_FOUND)
set(LMDB_LIBRARIES ${LMDB_LIBRARIES})
set(LMDB_INCLUDE_DIR, ${LMDB_INCLUDE_DIR})
endif()
================================================
FILE: build/fbcode_builder/CMake/FindLibEvent.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
# - Find LibEvent (a cross event library)
# This module defines
# LIBEVENT_INCLUDE_DIR, where to find LibEvent headers
# LIBEVENT_LIB, LibEvent libraries
# LibEvent_FOUND, If false, do not try to use libevent
set(LibEvent_EXTRA_PREFIXES /usr/local /opt/local "$ENV{HOME}")
foreach(prefix ${LibEvent_EXTRA_PREFIXES})
list(APPEND LibEvent_INCLUDE_PATHS "${prefix}/include")
list(APPEND LibEvent_LIB_PATHS "${prefix}/lib")
endforeach()
find_package(Libevent CONFIG QUIET)
if (TARGET event)
# Re-export the config under our own names
# Somewhat gross, but some vcpkg installed libevents have a relative
# `include` path exported into LIBEVENT_INCLUDE_DIRS, which triggers
# a cmake error because it resolves to the `include` dir within the
# folly repo, which is not something cmake allows to be in the
# INTERFACE_INCLUDE_DIRECTORIES. Thankfully on such a system the
# actual include directory is already part of the global include
# directories, so we can just skip it.
if (NOT "${LIBEVENT_INCLUDE_DIRS}" STREQUAL "include")
set(LIBEVENT_INCLUDE_DIR ${LIBEVENT_INCLUDE_DIRS})
else()
set(LIBEVENT_INCLUDE_DIR)
endif()
# Unfortunately, with a bare target name `event`, downstream consumers
# of the package that depends on `Libevent` located via CONFIG end
# up exporting just a bare `event` in their libraries. This is problematic
# because this in interpreted as just `-levent` with no library path.
# When libevent is not installed in the default installation prefix
# this results in linker errors.
# To resolve this, we ask cmake to lookup the full path to the library
# and use that instead.
cmake_policy(PUSH)
if(POLICY CMP0026)
# Allow reading the LOCATION property
cmake_policy(SET CMP0026 OLD)
endif()
get_target_property(LIBEVENT_LIB event LOCATION)
cmake_policy(POP)
set(LibEvent_FOUND ${Libevent_FOUND})
if (NOT LibEvent_FIND_QUIETLY)
message(STATUS "Found libevent from package config include=${LIBEVENT_INCLUDE_DIRS} lib=${LIBEVENT_LIB}")
endif()
else()
find_path(LIBEVENT_INCLUDE_DIR event.h PATHS ${LibEvent_INCLUDE_PATHS})
find_library(LIBEVENT_LIB NAMES event PATHS ${LibEvent_LIB_PATHS})
if (LIBEVENT_LIB AND LIBEVENT_INCLUDE_DIR)
set(LibEvent_FOUND TRUE)
set(LIBEVENT_LIB ${LIBEVENT_LIB})
else ()
set(LibEvent_FOUND FALSE)
endif ()
if (LibEvent_FOUND)
if (NOT LibEvent_FIND_QUIETLY)
message(STATUS "Found libevent: ${LIBEVENT_LIB}")
endif ()
else ()
if (LibEvent_FIND_REQUIRED)
message(FATAL_ERROR "Could NOT find libevent.")
endif ()
message(STATUS "libevent NOT found.")
endif ()
mark_as_advanced(
LIBEVENT_LIB
LIBEVENT_INCLUDE_DIR
)
endif()
================================================
FILE: build/fbcode_builder/CMake/FindLibUnwind.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
include(FindPackageHandleStandardArgs)
# Prefer pkg-config: picks up transitive deps (e.g. lzma, zlib) that
# static libunwind needs but a bare find_library would miss.
find_package(PkgConfig QUIET)
if(PKG_CONFIG_FOUND)
pkg_check_modules(PC_LIBUNWIND QUIET libunwind)
endif()
if(PC_LIBUNWIND_FOUND)
find_path(LIBUNWIND_INCLUDE_DIR NAMES libunwind.h
HINTS ${PC_LIBUNWIND_INCLUDE_DIRS}
PATH_SUFFIXES libunwind)
mark_as_advanced(LIBUNWIND_INCLUDE_DIR)
# Resolve each library from the static set (Libs + Libs.private) to a
# full path. This gives the linker everything it needs for a fully-static
# link without leaking imported targets through cmake exports.
set(LIBUNWIND_LIBRARIES "")
foreach(_lib IN LISTS PC_LIBUNWIND_STATIC_LIBRARIES)
find_library(_libunwind_dep_${_lib} NAMES ${_lib}
HINTS ${PC_LIBUNWIND_STATIC_LIBRARY_DIRS})
if(_libunwind_dep_${_lib})
list(APPEND LIBUNWIND_LIBRARIES ${_libunwind_dep_${_lib}})
else()
# Fall back to bare name; the linker will resolve -l.
list(APPEND LIBUNWIND_LIBRARIES ${_lib})
endif()
unset(_libunwind_dep_${_lib} CACHE)
endforeach()
set(LIBUNWIND_LIBRARY "${LIBUNWIND_LIBRARIES}")
FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibUnwind
REQUIRED_VARS LIBUNWIND_LIBRARIES LIBUNWIND_INCLUDE_DIR)
else()
# Fallback for systems without pkg-config.
# When using prepackaged LLVM libunwind on Ubuntu, its includes are
# installed in a subdirectory.
find_path(LIBUNWIND_INCLUDE_DIR NAMES libunwind.h PATH_SUFFIXES libunwind)
mark_as_advanced(LIBUNWIND_INCLUDE_DIR)
find_library(LIBUNWIND_LIBRARY NAMES unwind)
mark_as_advanced(LIBUNWIND_LIBRARY)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibUnwind
REQUIRED_VARS LIBUNWIND_LIBRARY LIBUNWIND_INCLUDE_DIR)
endif()
if(LibUnwind_FOUND)
if(NOT LIBUNWIND_LIBRARIES)
set(LIBUNWIND_LIBRARIES ${LIBUNWIND_LIBRARY})
endif()
set(LIBUNWIND_INCLUDE_DIRS ${LIBUNWIND_INCLUDE_DIR})
endif()
================================================
FILE: build/fbcode_builder/CMake/FindLibiberty.cmake
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
find_path(LIBIBERTY_INCLUDE_DIR NAMES libiberty.h PATH_SUFFIXES libiberty)
mark_as_advanced(LIBIBERTY_INCLUDE_DIR)
find_library(LIBIBERTY_LIBRARY NAMES iberty)
mark_as_advanced(LIBIBERTY_LIBRARY)
include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(
LIBIBERTY
REQUIRED_VARS LIBIBERTY_LIBRARY LIBIBERTY_INCLUDE_DIR)
if(LIBIBERTY_FOUND)
set(LIBIBERTY_LIBRARIES ${LIBIBERTY_LIBRARY})
set(LIBIBERTY_INCLUDE_DIRS ${LIBIBERTY_INCLUDE_DIR})
endif()
================================================
FILE: build/fbcode_builder/CMake/FindPCRE.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
include(FindPackageHandleStandardArgs)
find_path(PCRE_INCLUDE_DIR NAMES pcre.h)
find_library(PCRE_LIBRARY NAMES pcre)
find_package_handle_standard_args(
PCRE
DEFAULT_MSG
PCRE_LIBRARY
PCRE_INCLUDE_DIR
)
mark_as_advanced(PCRE_INCLUDE_DIR PCRE_LIBRARY)
================================================
FILE: build/fbcode_builder/CMake/FindPCRE2.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
include(FindPackageHandleStandardArgs)
find_path(PCRE2_INCLUDE_DIR NAMES pcre2.h)
find_library(PCRE2_LIBRARY NAMES pcre2-8)
find_package_handle_standard_args(
PCRE2
DEFAULT_MSG
PCRE2_LIBRARY
PCRE2_INCLUDE_DIR
)
set(PCRE2_DEFINES "PCRE2_CODE_UNIT_WIDTH=8")
mark_as_advanced(PCRE2_INCLUDE_DIR PCRE2_LIBRARY PCRE2_DEFINES)
================================================
FILE: build/fbcode_builder/CMake/FindRe2.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2.
find_library(RE2_LIBRARY re2)
mark_as_advanced(RE2_LIBRARY)
find_path(RE2_INCLUDE_DIR NAMES re2/re2.h)
mark_as_advanced(RE2_INCLUDE_DIR)
include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(
RE2
REQUIRED_VARS RE2_LIBRARY RE2_INCLUDE_DIR)
if(RE2_FOUND)
set(RE2_LIBRARY ${RE2_LIBRARY})
set(RE2_INCLUDE_DIR, ${RE2_INCLUDE_DIR})
endif()
================================================
FILE: build/fbcode_builder/CMake/FindSodium.cmake
================================================
# Written in 2016 by Henrik Steffen Gaßmann
#
# To the extent possible under law, the author(s) have dedicated all
# copyright and related and neighboring rights to this software to the
# public domain worldwide. This software is distributed without any warranty.
#
# You should have received a copy of the CC0 Public Domain Dedication
# along with this software. If not, see
#
# http://creativecommons.org/publicdomain/zero/1.0/
#
########################################################################
# Tries to find the local libsodium installation.
#
# On Windows the sodium_DIR environment variable is used as a default
# hint which can be overridden by setting the corresponding cmake variable.
#
# Once done the following variables will be defined:
#
# sodium_FOUND
# sodium_INCLUDE_DIR
# sodium_LIBRARY_DEBUG
# sodium_LIBRARY_RELEASE
#
#
# Furthermore an imported "sodium" target is created.
#
if (CMAKE_C_COMPILER_ID STREQUAL "GNU"
OR CMAKE_C_COMPILER_ID STREQUAL "Clang")
set(_GCC_COMPATIBLE 1)
endif()
# static library option
if (NOT DEFINED sodium_USE_STATIC_LIBS)
option(sodium_USE_STATIC_LIBS "enable to statically link against sodium" OFF)
endif()
if(NOT (sodium_USE_STATIC_LIBS EQUAL sodium_USE_STATIC_LIBS_LAST))
unset(sodium_LIBRARY CACHE)
unset(sodium_LIBRARY_DEBUG CACHE)
unset(sodium_LIBRARY_RELEASE CACHE)
unset(sodium_DLL_DEBUG CACHE)
unset(sodium_DLL_RELEASE CACHE)
set(sodium_USE_STATIC_LIBS_LAST ${sodium_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable")
endif()
########################################################################
# UNIX
if (UNIX)
# import pkg-config
find_package(PkgConfig QUIET)
if (PKG_CONFIG_FOUND)
pkg_check_modules(sodium_PKG QUIET libsodium)
endif()
if(sodium_USE_STATIC_LIBS)
foreach(_libname ${sodium_PKG_STATIC_LIBRARIES})
if (NOT _libname MATCHES "^lib.*\\.a$") # ignore strings already ending with .a
list(INSERT sodium_PKG_STATIC_LIBRARIES 0 "lib${_libname}.a")
endif()
endforeach()
list(REMOVE_DUPLICATES sodium_PKG_STATIC_LIBRARIES)
# if pkgconfig for libsodium doesn't provide
# static lib info, then override PKG_STATIC here..
if (NOT sodium_PKG_STATIC_FOUND)
set(sodium_PKG_STATIC_LIBRARIES libsodium.a)
endif()
set(XPREFIX sodium_PKG_STATIC)
else()
if (NOT sodium_PKG_FOUND)
set(sodium_PKG_LIBRARIES sodium)
endif()
set(XPREFIX sodium_PKG)
endif()
find_path(sodium_INCLUDE_DIR sodium.h
HINTS ${${XPREFIX}_INCLUDE_DIRS}
)
find_library(sodium_LIBRARY_DEBUG NAMES ${${XPREFIX}_LIBRARIES}
HINTS ${${XPREFIX}_LIBRARY_DIRS}
)
find_library(sodium_LIBRARY_RELEASE NAMES ${${XPREFIX}_LIBRARIES}
HINTS ${${XPREFIX}_LIBRARY_DIRS}
)
########################################################################
# Windows
elseif (WIN32)
set(sodium_DIR "$ENV{sodium_DIR}" CACHE FILEPATH "sodium install directory")
mark_as_advanced(sodium_DIR)
find_path(sodium_INCLUDE_DIR sodium.h
HINTS ${sodium_DIR}
PATH_SUFFIXES include
)
if (MSVC)
# detect target architecture
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" [=[
#if defined _M_IX86
#error ARCH_VALUE x86_32
#elif defined _M_X64
#error ARCH_VALUE x86_64
#endif
#error ARCH_VALUE unknown
]=])
try_compile(_UNUSED_VAR "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp"
OUTPUT_VARIABLE _COMPILATION_LOG
)
string(REGEX REPLACE ".*ARCH_VALUE ([a-zA-Z0-9_]+).*" "\\1" _TARGET_ARCH "${_COMPILATION_LOG}")
# construct library path
if (_TARGET_ARCH STREQUAL "x86_32")
string(APPEND _PLATFORM_PATH "Win32")
elseif(_TARGET_ARCH STREQUAL "x86_64")
string(APPEND _PLATFORM_PATH "x64")
else()
message(FATAL_ERROR "the ${_TARGET_ARCH} architecture is not supported by Findsodium.cmake.")
endif()
string(APPEND _PLATFORM_PATH "/$$CONFIG$$")
if (MSVC_VERSION LESS 1900)
math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 60")
else()
math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50")
endif()
string(APPEND _PLATFORM_PATH "/v${_VS_VERSION}")
if (sodium_USE_STATIC_LIBS)
string(APPEND _PLATFORM_PATH "/static")
else()
string(APPEND _PLATFORM_PATH "/dynamic")
endif()
string(REPLACE "$$CONFIG$$" "Debug" _DEBUG_PATH_SUFFIX "${_PLATFORM_PATH}")
string(REPLACE "$$CONFIG$$" "Release" _RELEASE_PATH_SUFFIX "${_PLATFORM_PATH}")
find_library(sodium_LIBRARY_DEBUG libsodium.lib
HINTS ${sodium_DIR}
PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX}
)
find_library(sodium_LIBRARY_RELEASE libsodium.lib
HINTS ${sodium_DIR}
PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX}
)
if (NOT sodium_USE_STATIC_LIBS)
set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES})
set(CMAKE_FIND_LIBRARY_SUFFIXES ".dll")
find_library(sodium_DLL_DEBUG libsodium
HINTS ${sodium_DIR}
PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX}
)
find_library(sodium_DLL_RELEASE libsodium
HINTS ${sodium_DIR}
PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX}
)
set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK})
endif()
elseif(_GCC_COMPATIBLE)
if (sodium_USE_STATIC_LIBS)
find_library(sodium_LIBRARY_DEBUG libsodium.a
HINTS ${sodium_DIR}
PATH_SUFFIXES lib
)
find_library(sodium_LIBRARY_RELEASE libsodium.a
HINTS ${sodium_DIR}
PATH_SUFFIXES lib
)
else()
find_library(sodium_LIBRARY_DEBUG libsodium.dll.a
HINTS ${sodium_DIR}
PATH_SUFFIXES lib
)
find_library(sodium_LIBRARY_RELEASE libsodium.dll.a
HINTS ${sodium_DIR}
PATH_SUFFIXES lib
)
file(GLOB _DLL
LIST_DIRECTORIES false
RELATIVE "${sodium_DIR}/bin"
"${sodium_DIR}/bin/libsodium*.dll"
)
find_library(sodium_DLL_DEBUG ${_DLL} libsodium
HINTS ${sodium_DIR}
PATH_SUFFIXES bin
)
find_library(sodium_DLL_RELEASE ${_DLL} libsodium
HINTS ${sodium_DIR}
PATH_SUFFIXES bin
)
endif()
else()
message(FATAL_ERROR "this platform is not supported by FindSodium.cmake")
endif()
########################################################################
# unsupported
else()
message(FATAL_ERROR "this platform is not supported by FindSodium.cmake")
endif()
########################################################################
# common stuff
# extract sodium version
if (sodium_INCLUDE_DIR)
set(_VERSION_HEADER "${_INCLUDE_DIR}/sodium/version.h")
if (EXISTS _VERSION_HEADER)
file(READ "${_VERSION_HEADER}" _VERSION_HEADER_CONTENT)
string(REGEX REPLACE ".*#[ \t]*define[ \t]*SODIUM_VERSION_STRING[ \t]*\"([^\n]*)\".*" "\\1"
sodium_VERSION "${_VERSION_HEADER_CONTENT}")
set(sodium_VERSION "${sodium_VERSION}" PARENT_SCOPE)
endif()
endif()
# communicate results
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(
Sodium # The name must be either uppercase or match the filename case.
REQUIRED_VARS
sodium_LIBRARY_RELEASE
sodium_LIBRARY_DEBUG
sodium_INCLUDE_DIR
VERSION_VAR
sodium_VERSION
)
if(Sodium_FOUND)
set(sodium_LIBRARIES
optimized ${sodium_LIBRARY_RELEASE} debug ${sodium_LIBRARY_DEBUG})
endif()
# mark file paths as advanced
mark_as_advanced(sodium_INCLUDE_DIR)
mark_as_advanced(sodium_LIBRARY_DEBUG)
mark_as_advanced(sodium_LIBRARY_RELEASE)
if (WIN32)
mark_as_advanced(sodium_DLL_DEBUG)
mark_as_advanced(sodium_DLL_RELEASE)
endif()
# create imported target
if(sodium_USE_STATIC_LIBS)
set(_LIB_TYPE STATIC)
else()
set(_LIB_TYPE SHARED)
endif()
if(NOT TARGET sodium)
add_library(sodium ${_LIB_TYPE} IMPORTED)
endif()
set_target_properties(sodium PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${sodium_INCLUDE_DIR}"
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
)
if (sodium_USE_STATIC_LIBS)
set_target_properties(sodium PROPERTIES
INTERFACE_COMPILE_DEFINITIONS "SODIUM_STATIC"
IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}"
IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}"
)
else()
if (UNIX)
set_target_properties(sodium PROPERTIES
IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}"
IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}"
)
elseif (WIN32)
set_target_properties(sodium PROPERTIES
IMPORTED_IMPLIB "${sodium_LIBRARY_RELEASE}"
IMPORTED_IMPLIB_DEBUG "${sodium_LIBRARY_DEBUG}"
)
if (NOT (sodium_DLL_DEBUG MATCHES ".*-NOTFOUND"))
set_target_properties(sodium PROPERTIES
IMPORTED_LOCATION_DEBUG "${sodium_DLL_DEBUG}"
)
endif()
if (NOT (sodium_DLL_RELEASE MATCHES ".*-NOTFOUND"))
set_target_properties(sodium PROPERTIES
IMPORTED_LOCATION_RELWITHDEBINFO "${sodium_DLL_RELEASE}"
IMPORTED_LOCATION_MINSIZEREL "${sodium_DLL_RELEASE}"
IMPORTED_LOCATION_RELEASE "${sodium_DLL_RELEASE}"
)
endif()
endif()
endif()
================================================
FILE: build/fbcode_builder/CMake/FindXxhash.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# - Try to find Facebook xxhash library
# This will define
# Xxhash_FOUND
# Xxhash_INCLUDE_DIR
# Xxhash_LIBRARY
#
find_path(Xxhash_INCLUDE_DIR NAMES xxhash.h)
find_library(Xxhash_LIBRARY_RELEASE NAMES xxhash)
include(SelectLibraryConfigurations)
SELECT_LIBRARY_CONFIGURATIONS(Xxhash)
include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(
Xxhash DEFAULT_MSG
Xxhash_LIBRARY Xxhash_INCLUDE_DIR
)
if (Xxhash_FOUND)
message(STATUS "Found xxhash: ${Xxhash_LIBRARY}")
endif()
mark_as_advanced(Xxhash_INCLUDE_DIR Xxhash_LIBRARY)
================================================
FILE: build/fbcode_builder/CMake/FindZstd.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# - Try to find Facebook zstd library
# This will define
# ZSTD_FOUND
# ZSTD_INCLUDE_DIR
# ZSTD_LIBRARY
#
find_path(ZSTD_INCLUDE_DIR NAMES zstd.h)
find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd)
find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static)
include(SelectLibraryConfigurations)
SELECT_LIBRARY_CONFIGURATIONS(ZSTD)
include(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(
Zstd DEFAULT_MSG
ZSTD_LIBRARY ZSTD_INCLUDE_DIR
)
if (ZSTD_FOUND)
message(STATUS "Found Zstd: ${ZSTD_LIBRARY}")
endif()
mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY)
================================================
FILE: build/fbcode_builder/CMake/Findibverbs.cmake
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Find the ibverbs libraries
#
# The following variables are optionally searched for defaults
# IBVERBS_ROOT_DIR: Base directory where all ibverbs components are found
# IBVERBS_INCLUDE_DIR: Directory where ibverbs headers are found
# IBVERBS_LIB_DIR: Directory where ibverbs libraries are found
# The following are set after configuration is done:
# IBVERBS_FOUND
# IBVERBS_INCLUDE_DIRS
# IBVERBS_LIBRARIES
# IBVERBS_VERSION
find_path(IBVERBS_INCLUDE_DIRS
NAMES infiniband/verbs.h
HINTS
${IBVERBS_INCLUDE_DIR}
${IBVERBS_ROOT_DIR}
${IBVERBS_ROOT_DIR}/include)
find_library(IBVERBS_LIBRARIES
NAMES ibverbs
HINTS
${IBVERBS_LIB_DIR}
${IBVERBS_ROOT_DIR}
${IBVERBS_ROOT_DIR}/lib)
# Try to determine the rdma-core version
if(IBVERBS_INCLUDE_DIRS AND IBVERBS_LIBRARIES)
# First try using pkg-config if available
find_package(PkgConfig QUIET)
if(PKG_CONFIG_FOUND)
pkg_check_modules(PC_RDMA_CORE QUIET rdma-core)
if(PC_RDMA_CORE_VERSION)
set(IBVERBS_VERSION ${PC_RDMA_CORE_VERSION})
endif()
endif()
# If pkg-config didn't work, try to extract version from library filename
# According to rdma-core Documentation/versioning.md:
# Library filename format:
# libibverbs.so.SONAME.ABI.PACKAGE_VERSION_MAIN[.PACKAGE_VERSION_BRANCH]
# Where:
# - SONAME: Major version (1st field)
# - ABI: ABI version number (2nd field)
# - PACKAGE_VERSION_MAIN: Main package version (3rd field)
# - PACKAGE_VERSION_BRANCH: Optional counter for branched stable
# releases (4th field, part of PACKAGE_VERSION)
# Example: libibverbs.so.1.14.57.0 → SONAME=1, ABI=14,
# PACKAGE_VERSION=57.0
if(NOT IBVERBS_VERSION)
# Get the real path of the library (follows symlinks)
get_filename_component(IBVERBS_REAL_PATH "${IBVERBS_LIBRARIES}" REALPATH)
get_filename_component(IBVERBS_LIB_NAME "${IBVERBS_REAL_PATH}" NAME)
# Extract version from filename
if(IBVERBS_LIB_NAME MATCHES
"libibverbs\\.so\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)")
# Four-component version: PACKAGE_VERSION_MAIN.PACKAGE_VERSION_BRANCH
set(IBVERBS_VERSION_MAJOR ${CMAKE_MATCH_3})
set(IBVERBS_VERSION_MINOR ${CMAKE_MATCH_4})
set(IBVERBS_VERSION "${IBVERBS_VERSION_MAJOR}.${IBVERBS_VERSION_MINOR}")
elseif(IBVERBS_LIB_NAME MATCHES
"libibverbs\\.so\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)")
# Three-component version: PACKAGE_VERSION_MAIN only
set(IBVERBS_VERSION_MAJOR ${CMAKE_MATCH_3})
set(IBVERBS_VERSION "${IBVERBS_VERSION_MAJOR}.0")
else()
# If we can't parse the filename, set to empty string
# Feature detection will be done in CMakeLists.txt
set(IBVERBS_VERSION "")
endif()
endif()
endif()
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(ibverbs
REQUIRED_VARS IBVERBS_INCLUDE_DIRS IBVERBS_LIBRARIES
VERSION_VAR IBVERBS_VERSION)
mark_as_advanced(IBVERBS_INCLUDE_DIRS IBVERBS_LIBRARIES IBVERBS_VERSION)
================================================
FILE: build/fbcode_builder/CMake/RustStaticLibrary.cmake
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
include(FBCMakeParseArgs)
set(
USE_CARGO_VENDOR AUTO CACHE STRING
"Download Rust Crates from an internally vendored location"
)
set_property(CACHE USE_CARGO_VENDOR PROPERTY STRINGS AUTO ON OFF)
set(
GENERATE_CARGO_VENDOR_CONFIG AUTO CACHE STRING
"Whether to generate Rust cargo vendor config or use existing"
)
set_property(CACHE GENERATE_CARGO_VENDOR_CONFIG PROPERTY STRINGS AUTO ON OFF)
set(RUST_VENDORED_CRATES_DIR "$ENV{RUST_VENDORED_CRATES_DIR}")
if("${USE_CARGO_VENDOR}" STREQUAL "AUTO")
if(EXISTS "${RUST_VENDORED_CRATES_DIR}")
set(USE_CARGO_VENDOR ON)
else()
set(USE_CARGO_VENDOR OFF)
endif()
endif()
if("${GENERATE_CARGO_VENDOR_CONFIG}" STREQUAL "AUTO")
set(GENERATE_CARGO_VENDOR_CONFIG "${USE_CARGO_VENDOR}")
endif()
if(GENERATE_CARGO_VENDOR_CONFIG)
if(NOT EXISTS "${RUST_VENDORED_CRATES_DIR}")
message(
FATAL "vendored rust crates not present: "
"${RUST_VENDORED_CRATES_DIR}"
)
endif()
set(RUST_CARGO_HOME "${CMAKE_BINARY_DIR}/_cargo_home")
file(MAKE_DIRECTORY "${RUST_CARGO_HOME}")
file(
TO_NATIVE_PATH "${RUST_VENDORED_CRATES_DIR}"
ESCAPED_RUST_VENDORED_CRATES_DIR
)
string(
REPLACE "\\" "\\\\"
ESCAPED_RUST_VENDORED_CRATES_DIR
"${ESCAPED_RUST_VENDORED_CRATES_DIR}"
)
file(
WRITE "${RUST_CARGO_HOME}/config"
"[source.crates-io]\n"
"replace-with = \"vendored-sources\"\n"
"\n"
"[source.vendored-sources]\n"
"directory = \"${ESCAPED_RUST_VENDORED_CRATES_DIR}\"\n"
)
endif()
find_program(CARGO_COMMAND cargo REQUIRED)
# Cargo is a build system in itself, and thus will try to take advantage of all
# the cores on the system. Unfortunately, this conflicts with Ninja, since it
# also tries to utilize all the cores. This can lead to a system that is
# completely overloaded with compile jobs to the point where nothing else can
# be achieved on the system.
#
# Let's inform Ninja of this fact so it won't try to spawn other jobs while
# Rust being compiled.
set_property(GLOBAL APPEND PROPERTY JOB_POOLS rust_job_pool=1)
# This function creates an interface library target based on the static library
# built by Cargo. It will call Cargo to build a staticlib and generate a CMake
# interface library with it.
#
# This function requires `find_package(Python COMPONENTS Interpreter)`.
#
# You need to set `lib:crate-type = ["staticlib"]` in your Cargo.toml to make
# Cargo build static library.
#
# ```cmake
# rust_static_library( [CRATE ] [FEATURES ] [USE_CXX_INCLUDE])
# ```
#
# Parameters:
# - TARGET:
# Name of the target name. This function will create an interface library
# target with this name.
# - CRATE_NAME:
# Name of the crate. This parameter is optional. If unspecified, it will
# fallback to `${TARGET}`.
# - FEATURE_NAME:
# Name of the Rust feature to enable.
# - USE_CXX_INCLUDE:
# Include cxx.rs include path in `${TARGET}` INTERFACE.
#
# This function creates two targets:
# - "${TARGET}": an interface library target contains the static library built
# from Cargo.
# - "${TARGET}.cargo": an internal custom target that invokes Cargo.
#
# If you are going to use this static library from C/C++, you will need to
# write header files for the library (or generate with cbindgen) and bind these
# headers with the interface library.
#
function(rust_static_library TARGET)
fb_cmake_parse_args(ARG "USE_CXX_INCLUDE" "CRATE;FEATURES" "" "${ARGN}")
if(DEFINED ARG_CRATE)
set(crate_name "${ARG_CRATE}")
else()
set(crate_name "${TARGET}")
endif()
if(DEFINED ARG_FEATURES)
set(features --features ${ARG_FEATURES})
else()
set(features )
endif()
set(cargo_target "${TARGET}.cargo")
set(target_dir $,debug,release>)
set(staticlib_name "${CMAKE_STATIC_LIBRARY_PREFIX}${crate_name}${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(rust_staticlib "${CMAKE_CURRENT_BINARY_DIR}/${target_dir}/${staticlib_name}")
if(DEFINED ARG_FEATURES)
set(cargo_flags build $,,--release> -p ${crate_name} --features ${ARG_FEATURES} --config fbcode_build=false)
else()
set(cargo_flags build $,,--release> -p ${crate_name} --config fbcode_build=false)
endif()
if(USE_CARGO_VENDOR)
set(extra_cargo_env "CARGO_HOME=${RUST_CARGO_HOME}")
set(cargo_flags ${cargo_flags})
endif()
add_custom_target(
${cargo_target}
COMMAND
"${CMAKE_COMMAND}" -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock"
COMMAND
"${CMAKE_COMMAND}" -E env
"CARGO_TARGET_DIR=${CMAKE_CURRENT_BINARY_DIR}"
${extra_cargo_env}
${CARGO_COMMAND}
${cargo_flags}
COMMENT "Building Rust crate '${crate_name}'..."
JOB_POOL rust_job_pool
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
BYPRODUCTS
"${CMAKE_CURRENT_BINARY_DIR}/debug/${staticlib_name}"
"${CMAKE_CURRENT_BINARY_DIR}/release/${staticlib_name}"
)
add_library(${TARGET} INTERFACE)
add_dependencies(${TARGET} ${cargo_target})
set_target_properties(
${TARGET}
PROPERTIES
INTERFACE_STATICLIB_OUTPUT_PATH "${rust_staticlib}"
INTERFACE_INSTALL_LIBNAME
"${CMAKE_STATIC_LIBRARY_PREFIX}${crate_name}_rs${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
if(DEFINED ARG_USE_CXX_INCLUDE)
target_include_directories(
${TARGET}
INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/cxxbridge/
)
endif()
target_link_libraries(
${TARGET}
INTERFACE "$"
)
endfunction()
# This function instructs CMake to define a target that will use `cargo build`
# to build a bin crate referenced by the Cargo.toml file in the current source
# directory.
# It accepts a single `TARGET` parameter which will be passed as the package
# name to `cargo build -p TARGET`. If binary has different name as package,
# use optional flag BINARY_NAME to override it.
# It also accepts a `FEATURES` parameter if you want to enable certain features
# in your Rust binary.
# The CMake target will be registered to build by default as part of the
# ALL target.
function(rust_executable TARGET)
fb_cmake_parse_args(ARG "" "BINARY_NAME;FEATURES" "" "${ARGN}")
set(crate_name "${TARGET}")
set(cargo_target "${TARGET}.cargo")
set(target_dir $,debug,release>)
if(DEFINED ARG_BINARY_NAME)
set(executable_name "${ARG_BINARY_NAME}${CMAKE_EXECUTABLE_SUFFIX}")
else()
set(executable_name "${crate_name}${CMAKE_EXECUTABLE_SUFFIX}")
endif()
if(DEFINED ARG_FEATURES)
set(features --features ${ARG_FEATURES})
else()
set(features )
endif()
if(DEFINED ARG_FEATURES)
set(cargo_flags build $,,--release> -p ${crate_name} --features ${ARG_FEATURES})
else()
set(cargo_flags build $,,--release> -p ${crate_name})
endif()
if(USE_CARGO_VENDOR)
set(extra_cargo_env "CARGO_HOME=${RUST_CARGO_HOME}")
set(cargo_flags ${cargo_flags})
endif()
add_custom_target(
${cargo_target}
ALL
COMMAND
"${CMAKE_COMMAND}" -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock"
COMMAND
"${CMAKE_COMMAND}" -E env
"CARGO_TARGET_DIR=${CMAKE_CURRENT_BINARY_DIR}"
${extra_cargo_env}
${CARGO_COMMAND}
${cargo_flags}
COMMENT "Building Rust executable '${crate_name}'..."
JOB_POOL rust_job_pool
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
BYPRODUCTS
"${CMAKE_CURRENT_BINARY_DIR}/debug/${executable_name}"
"${CMAKE_CURRENT_BINARY_DIR}/release/${executable_name}"
)
set_property(TARGET "${cargo_target}"
PROPERTY EXECUTABLE "${CMAKE_CURRENT_BINARY_DIR}/${target_dir}/${executable_name}")
endfunction()
# This function can be used to install the executable generated by a prior
# call to the `rust_executable` function.
# It requires a `TARGET` parameter to identify the target to be installed,
# and an optional `DESTINATION` parameter to specify the installation
# directory. If DESTINATION is not specified then the `bin` directory
# will be assumed.
function(install_rust_executable TARGET)
# Parse the arguments
set(one_value_args DESTINATION)
set(multi_value_args)
fb_cmake_parse_args(
ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}"
)
if(NOT DEFINED ARG_DESTINATION)
set(ARG_DESTINATION bin)
endif()
get_target_property(foo "${TARGET}.cargo" EXECUTABLE)
install(
PROGRAMS "${foo}"
DESTINATION "${ARG_DESTINATION}"
)
endfunction()
# This function installs the interface target generated from the function
# `rust_static_library`. Use this function if you want to export your Rust
# target to external CMake targets.
#
# ```cmake
# install_rust_static_library(
#
# INSTALL_DIR
# [EXPORT ]
# )
# ```
#
# Parameters:
# - TARGET: Name of the Rust static library target.
# - EXPORT_NAME: Name of the exported target.
# - INSTALL_DIR: Path to the directory where this library will be installed.
#
function(install_rust_static_library TARGET)
fb_cmake_parse_args(ARG "" "EXPORT;INSTALL_DIR" "" "${ARGN}")
get_property(
staticlib_output_path
TARGET "${TARGET}"
PROPERTY INTERFACE_STATICLIB_OUTPUT_PATH
)
get_property(
staticlib_output_name
TARGET "${TARGET}"
PROPERTY INTERFACE_INSTALL_LIBNAME
)
if(NOT DEFINED staticlib_output_path)
message(FATAL_ERROR "Not a rust_static_library target.")
endif()
if(NOT DEFINED ARG_INSTALL_DIR)
message(FATAL_ERROR "Missing required argument.")
endif()
if(DEFINED ARG_EXPORT)
set(install_export_args EXPORT "${ARG_EXPORT}")
endif()
set(install_interface_dir "${ARG_INSTALL_DIR}")
if(NOT IS_ABSOLUTE "${install_interface_dir}")
set(install_interface_dir "\${_IMPORT_PREFIX}/${install_interface_dir}")
endif()
target_link_libraries(
${TARGET} INTERFACE
"$"
)
install(
TARGETS ${TARGET}
${install_export_args}
LIBRARY DESTINATION ${ARG_INSTALL_DIR}
)
install(
FILES ${staticlib_output_path}
RENAME ${staticlib_output_name}
DESTINATION ${ARG_INSTALL_DIR}
)
endfunction()
# This function creates C++ bindings using the [cxx] crate.
#
# Original function found here: https://github.com/corrosion-rs/corrosion/blob/master/cmake/Corrosion.cmake#L1390
# Simplified for use as part of RustStaticLibrary module. License below.
#
# MIT License
#
# Copyright (c) 2018 Andrew Gaspar
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# The rules approximately do the following:
# - Check which version of `cxx` the Rust crate depends on.
# - Check if the exact same version of `cxxbridge-cmd` is installed
# - If not, create a rule to build the exact same version of `cxxbridge-cmd`.
# - Create rules to run `cxxbridge` and generate
# - The `rust/cxx.h` header
# - A header and source file for the specified CXX_BRIDGE_FILE.
# - The generated sources (and header include directories) are added to the
# `${TARGET}` CMake library target.
#
# ```cmake
# rust_cxx_bridge( [CRATE ] [LIBS ])
# ```
#
# Parameters:
# - TARGET:
# Name of the target name. The target that the bridge will be included with.
# - CXX_BRIDGE_FILE:
# Name of the file that include the cxxbridge (e.g., "src/ffi.rs").
# - CRATE_NAME:
# Name of the crate. This parameter is optional. If unspecified, it will
# fallback to `${TARGET}`.
# - LIBS [ ...]:
# A list of libraries that this library depends on.
#
function(rust_cxx_bridge TARGET CXX_BRIDGE_FILE)
fb_cmake_parse_args(ARG "" "CRATE" "LIBS" "${ARGN}")
if(DEFINED ARG_CRATE)
set(crate_name "${ARG_CRATE}")
else()
set(crate_name "${TARGET}")
endif()
if(USE_CARGO_VENDOR)
set(extra_cargo_env "CARGO_HOME=${RUST_CARGO_HOME}")
endif()
execute_process(
COMMAND
"${CMAKE_COMMAND}" -E env
${extra_cargo_env}
"${CARGO_COMMAND}" tree -i cxx --depth=0
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
RESULT_VARIABLE cxx_version_result
OUTPUT_VARIABLE cxx_version_output
)
if(NOT "${cxx_version_result}" EQUAL "0")
message(FATAL_ERROR "Crate ${crate_name} does not depend on cxx.")
endif()
if(cxx_version_output MATCHES "cxx v([0-9]+.[0-9]+.[0-9]+)")
set(cxx_required_version "${CMAKE_MATCH_1}")
else()
message(
FATAL_ERROR
"Failed to parse cxx version from cargo tree output: `cxx_version_output`")
endif()
# First check if a suitable version of cxxbridge is installed
find_program(INSTALLED_CXXBRIDGE cxxbridge PATHS "$ENV{HOME}/.cargo/bin/")
mark_as_advanced(INSTALLED_CXXBRIDGE)
if(INSTALLED_CXXBRIDGE)
execute_process(
COMMAND "${INSTALLED_CXXBRIDGE}" --version
OUTPUT_VARIABLE cxxbridge_version_output
)
if(cxxbridge_version_output MATCHES "cxxbridge ([0-9]+.[0-9]+.[0-9]+)")
set(cxxbridge_version "${CMAKE_MATCH_1}")
else()
set(cxxbridge_version "")
endif()
endif()
set(cxxbridge "")
if(cxxbridge_version)
if(cxxbridge_version VERSION_EQUAL cxx_required_version)
set(cxxbridge "${INSTALLED_CXXBRIDGE}")
if(NOT TARGET "cxxbridge_v${cxx_required_version}")
# Add an empty target.
add_custom_target("cxxbridge_v${cxx_required_version}")
endif()
endif()
endif()
# No suitable version of cxxbridge was installed,
# so use custom target to install correct version.
if(NOT cxxbridge)
if(NOT TARGET "cxxbridge_v${cxx_required_version}")
add_custom_command(
OUTPUT
"${CMAKE_BINARY_DIR}/cxxbridge_v${cxx_required_version}/bin/cxxbridge"
COMMAND
"${CMAKE_COMMAND}" -E make_directory
"${CMAKE_BINARY_DIR}/cxxbridge_v${cxx_required_version}"
COMMAND
"${CMAKE_COMMAND}" -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock"
COMMAND
"${CMAKE_COMMAND}" -E env
${extra_cargo_env}
"${CARGO_COMMAND}" install cxxbridge-cmd
--version "${cxx_required_version}"
--root "${CMAKE_BINARY_DIR}/cxxbridge_v${cxx_required_version}"
--quiet
COMMAND
"${CMAKE_COMMAND}" -E remove -f "${CMAKE_CURRENT_SOURCE_DIR}/Cargo.lock"
COMMENT "Installing cxxbridge (version ${cxx_required_version})"
)
add_custom_target(
"cxxbridge_v${cxx_required_version}"
DEPENDS "${CMAKE_BINARY_DIR}/cxxbridge_v${cxx_required_version}/bin/cxxbridge"
)
endif()
set(
cxxbridge
"${CMAKE_BINARY_DIR}/cxxbridge_v${cxx_required_version}/bin/cxxbridge"
)
endif()
add_library(${crate_name} STATIC)
target_include_directories(
${crate_name}
PUBLIC
$
$
)
target_link_libraries(
${crate_name}
PUBLIC
${ARG_LIBS}
)
file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/rust")
add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/rust/cxx.h"
COMMAND
"${cxxbridge}" --header --output "${CMAKE_CURRENT_BINARY_DIR}/rust/cxx.h"
DEPENDS "cxxbridge_v${cxx_required_version}"
COMMENT "Generating rust/cxx.h header"
)
get_filename_component(filename_component ${CXX_BRIDGE_FILE} NAME)
get_filename_component(directory_component ${CXX_BRIDGE_FILE} DIRECTORY)
set(directory "")
if(directory_component)
set(directory "${directory_component}")
endif()
set(cxx_header ${directory}/${filename_component}.h)
set(cxx_source ${directory}/${filename_component}.cc)
set(rust_source_path "${CMAKE_CURRENT_SOURCE_DIR}/${CXX_BRIDGE_FILE}")
file(
MAKE_DIRECTORY
"${CMAKE_CURRENT_BINARY_DIR}/${directory_component}"
)
add_custom_command(
OUTPUT
"${CMAKE_CURRENT_BINARY_DIR}/${cxx_header}"
"${CMAKE_CURRENT_BINARY_DIR}/${cxx_source}"
COMMAND
${cxxbridge} ${rust_source_path}
--cfg fbcode_build=false
--header
--output "${CMAKE_CURRENT_BINARY_DIR}/${cxx_header}"
COMMAND
${cxxbridge} ${rust_source_path}
--cfg fbcode_build=false
--output "${CMAKE_CURRENT_BINARY_DIR}/${cxx_source}"
--include "${cxx_header}"
DEPENDS "cxxbridge_v${cxx_required_version}" "${rust_source_path}"
COMMENT "Generating cxx bindings for crate ${crate_name}"
)
target_sources(
${crate_name}
PRIVATE
"${CMAKE_CURRENT_BINARY_DIR}/${cxx_header}"
"${CMAKE_CURRENT_BINARY_DIR}/rust/cxx.h"
"${CMAKE_CURRENT_BINARY_DIR}/${cxx_source}"
)
endfunction()
================================================
FILE: build/fbcode_builder/CMake/fb_py_test_main.py
================================================
#!/usr/bin/env python
#
# Copyright (c) Facebook, Inc. and its affiliates.
#
"""
This file contains the main module code for Python test programs.
"""
import contextlib
import ctypes
import fnmatch
import json
import logging
import optparse
import os
import platform
import re
import sys
import tempfile
import time
import traceback
import unittest
import warnings
from importlib.machinery import PathFinder
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
try:
import coverage
except ImportError:
coverage = None # type: ignore
try:
from importlib.machinery import SourceFileLoader
except ImportError:
SourceFileLoader = None # type: ignore
class get_cpu_instr_counter:
def read(self):
# TODO
return 0
EXIT_CODE_SUCCESS = 0
EXIT_CODE_TEST_FAILURE = 70
class TestStatus:
ABORTED = "FAILURE"
PASSED = "SUCCESS"
FAILED = "FAILURE"
EXPECTED_FAILURE = "SUCCESS"
UNEXPECTED_SUCCESS = "FAILURE"
SKIPPED = "ASSUMPTION_VIOLATION"
class PathMatcher:
def __init__(self, include_patterns, omit_patterns):
self.include_patterns = include_patterns
self.omit_patterns = omit_patterns
def omit(self, path):
"""
Omit iff matches any of the omit_patterns or the include patterns are
not empty and none is matched
"""
path = os.path.realpath(path)
return any(fnmatch.fnmatch(path, p) for p in self.omit_patterns) or (
self.include_patterns
and not any(fnmatch.fnmatch(path, p) for p in self.include_patterns)
)
def include(self, path):
return not self.omit(path)
class DebugWipeFinder(PathFinder):
"""
PEP 302 finder that uses a DebugWipeLoader for all files which do not need
coverage
"""
def __init__(self, matcher):
self.matcher = matcher
def find_spec(self, fullname, path=None, target=None):
spec = super().find_spec(fullname, path=path, target=target)
if spec is None or spec.origin is None:
return None
if not spec.origin.endswith(".py"):
return None
if self.matcher.include(spec.origin):
return None
class PyVarObject(ctypes.Structure):
_fields_ = [
("ob_refcnt", ctypes.c_long),
("ob_type", ctypes.c_void_p),
("ob_size", ctypes.c_ulong),
]
class DebugWipeLoader(SourceFileLoader):
"""
PEP302 loader that zeros out debug information before execution
"""
def get_code(self, fullname):
code = super().get_code(fullname)
if code:
# Ideally we'd do
# code.co_lnotab = b''
# But code objects are READONLY. Not to worry though; we'll
# directly modify CPython's object
code_impl = PyVarObject.from_address(id(code.co_lnotab))
code_impl.ob_size = 0
return code
if isinstance(spec.loader, SourceFileLoader):
spec.loader = DebugWipeLoader(fullname, spec.origin)
return spec
def optimize_for_coverage(cov, include_patterns, omit_patterns):
"""
We get better performance if we zero out debug information for files which
we're not interested in. Only available in CPython 3.3+
"""
matcher = PathMatcher(include_patterns, omit_patterns)
if SourceFileLoader and platform.python_implementation() == "CPython":
sys.meta_path.insert(0, DebugWipeFinder(matcher))
class TeeStream:
def __init__(self, *streams):
self._streams = streams
def write(self, data):
for stream in self._streams:
stream.write(data)
def flush(self):
for stream in self._streams:
stream.flush()
def isatty(self):
return False
class CallbackStream:
def __init__(self, callback, bytes_callback=None, orig=None):
self._callback = callback
self._fileno = orig.fileno() if orig else None
# Python 3 APIs:
# - `encoding` is a string holding the encoding name
# - `errors` is a string holding the error-handling mode for encoding
# - `buffer` should look like an io.BufferedIOBase object
self.errors = orig.errors if orig else None
if bytes_callback:
# those members are only on the io.TextIOWrapper
self.encoding = orig.encoding if orig else "UTF-8"
self.buffer = CallbackStream(bytes_callback, orig=orig)
def write(self, data):
self._callback(data)
def flush(self):
pass
def isatty(self):
return False
def fileno(self):
return self._fileno
class BuckTestResult(unittest.TextTestResult):
"""
Our own TestResult class that outputs data in a format that can be easily
parsed by buck's test runner.
"""
_instr_counter = get_cpu_instr_counter()
def __init__(
self, stream, descriptions, verbosity, show_output, main_program, suite
):
super(BuckTestResult, self).__init__(stream, descriptions, verbosity)
self._main_program = main_program
self._suite = suite
self._results = []
self._current_test = None
self._saved_stdout = sys.stdout
self._saved_stderr = sys.stderr
self._show_output = show_output
def getResults(self):
return self._results
def startTest(self, test):
super(BuckTestResult, self).startTest(test)
# Pass in the real stdout and stderr filenos. We can't really do much
# here to intercept callers who directly operate on these fileno
# objects.
sys.stdout = CallbackStream(
self.addStdout, self.addStdoutBytes, orig=sys.stdout
)
sys.stderr = CallbackStream(
self.addStderr, self.addStderrBytes, orig=sys.stderr
)
self._current_test = test
self._test_start_time = time.time()
self._current_status = TestStatus.ABORTED
self._messages = []
self._stacktrace = None
self._stdout = ""
self._stderr = ""
self._start_instr_count = self._instr_counter.read()
def _find_next_test(self, suite):
"""
Find the next test that has not been run.
"""
for test in suite:
# We identify test suites by test that are iterable (as is done in
# the builtin python test harness). If we see one, recurse on it.
if hasattr(test, "__iter__"):
test = self._find_next_test(test)
# The builtin python test harness sets test references to `None`
# after they have run, so we know we've found the next test up
# if it's not `None`.
if test is not None:
return test
def stopTest(self, test):
sys.stdout = self._saved_stdout
sys.stderr = self._saved_stderr
super(BuckTestResult, self).stopTest(test)
# If a failure occurred during module/class setup, then this "test" may
# actually be a `_ErrorHolder`, which doesn't contain explicit info
# about the upcoming test. Since we really only care about the test
# name field (i.e. `_testMethodName`), we use that to detect an actual
# test cases, and fall back to looking the test up from the suite
# otherwise.
if not hasattr(test, "_testMethodName"):
test = self._find_next_test(self._suite)
result = {
"testCaseName": "{0}.{1}".format(
test.__class__.__module__, test.__class__.__name__
),
"testCase": test._testMethodName,
"type": self._current_status,
"time": int((time.time() - self._test_start_time) * 1000),
"message": os.linesep.join(self._messages),
"stacktrace": self._stacktrace,
"stdOut": self._stdout,
"stdErr": self._stderr,
}
# TestPilot supports an instruction count field.
if "TEST_PILOT" in os.environ:
result["instrCount"] = (
int(self._instr_counter.read() - self._start_instr_count),
)
self._results.append(result)
self._current_test = None
def stopTestRun(self):
cov = self._main_program.get_coverage()
if cov is not None:
self._results.append({"coverage": cov})
@contextlib.contextmanager
def _withTest(self, test):
self.startTest(test)
yield
self.stopTest(test)
def _setStatus(self, test, status, message=None, stacktrace=None):
assert test == self._current_test
self._current_status = status
self._stacktrace = stacktrace
if message is not None:
if message.endswith(os.linesep):
message = message[:-1]
self._messages.append(message)
def setStatus(self, test, status, message=None, stacktrace=None):
# addError() may be called outside of a test if one of the shared
# fixtures (setUpClass/tearDownClass/setUpModule/tearDownModule)
# throws an error.
#
# In this case, create a fake test result to record the error.
if self._current_test is None:
with self._withTest(test):
self._setStatus(test, status, message, stacktrace)
else:
self._setStatus(test, status, message, stacktrace)
def setException(self, test, status, excinfo):
exctype, value, tb = excinfo
self.setStatus(
test,
status,
"{0}: {1}".format(exctype.__name__, value),
"".join(traceback.format_tb(tb)),
)
def addSuccess(self, test):
super(BuckTestResult, self).addSuccess(test)
self.setStatus(test, TestStatus.PASSED)
def addError(self, test, err):
super(BuckTestResult, self).addError(test, err)
self.setException(test, TestStatus.ABORTED, err)
def addFailure(self, test, err):
super(BuckTestResult, self).addFailure(test, err)
self.setException(test, TestStatus.FAILED, err)
def addSkip(self, test, reason):
super(BuckTestResult, self).addSkip(test, reason)
self.setStatus(test, TestStatus.SKIPPED, "Skipped: %s" % (reason,))
def addExpectedFailure(self, test, err):
super(BuckTestResult, self).addExpectedFailure(test, err)
self.setException(test, TestStatus.EXPECTED_FAILURE, err)
def addUnexpectedSuccess(self, test):
super(BuckTestResult, self).addUnexpectedSuccess(test)
self.setStatus(test, TestStatus.UNEXPECTED_SUCCESS, "Unexpected success")
def addStdout(self, val):
self._stdout += val
if self._show_output:
self._saved_stdout.write(val)
self._saved_stdout.flush()
def addStdoutBytes(self, val):
string = val.decode("utf-8", errors="backslashreplace")
self.addStdout(string)
def addStderr(self, val):
self._stderr += val
if self._show_output:
self._saved_stderr.write(val)
self._saved_stderr.flush()
def addStderrBytes(self, val):
string = val.decode("utf-8", errors="backslashreplace")
self.addStderr(string)
class BuckTestRunner(unittest.TextTestRunner):
def __init__(self, main_program, suite, show_output=True, **kwargs):
super(BuckTestRunner, self).__init__(**kwargs)
self.show_output = show_output
self._main_program = main_program
self._suite = suite
def _makeResult(self):
return BuckTestResult(
self.stream,
self.descriptions,
self.verbosity,
self.show_output,
self._main_program,
self._suite,
)
def _format_test_name(test_class, attrname):
return "{0}.{1}.{2}".format(test_class.__module__, test_class.__name__, attrname)
class StderrLogHandler(logging.StreamHandler):
"""
This class is very similar to logging.StreamHandler, except that it
always uses the current sys.stderr object.
StreamHandler caches the current sys.stderr object when it is constructed.
This makes it behave poorly in unit tests, which may replace sys.stderr
with a StringIO buffer during tests. The StreamHandler will continue using
the old sys.stderr object instead of the desired StringIO buffer.
"""
def __init__(self):
logging.Handler.__init__(self)
@property
def stream(self):
return sys.stderr
class RegexTestLoader(unittest.TestLoader):
def __init__(self, regex=None):
self.regex = regex
super(RegexTestLoader, self).__init__()
def getTestCaseNames(self, testCaseClass):
"""
Return a sorted sequence of method names found within testCaseClass
"""
testFnNames = super(RegexTestLoader, self).getTestCaseNames(testCaseClass)
if self.regex is None:
return testFnNames
robj = re.compile(self.regex)
matched = []
for attrname in testFnNames:
fullname = _format_test_name(testCaseClass, attrname)
if robj.search(fullname):
matched.append(attrname)
return matched
class Loader:
suiteClass = unittest.TestSuite
def __init__(self, modules, regex=None):
self.modules = modules
self.regex = regex
def load_all(self):
loader = RegexTestLoader(self.regex)
test_suite = self.suiteClass()
for module_name in self.modules:
__import__(module_name, level=0)
module = sys.modules[module_name]
module_suite = loader.loadTestsFromModule(module)
test_suite.addTest(module_suite)
return test_suite
def load_args(self, args):
loader = RegexTestLoader(self.regex)
suites = []
for arg in args:
suite = loader.loadTestsFromName(arg)
# loadTestsFromName() can only process names that refer to
# individual test functions or modules. It can't process package
# names. If there were no module/function matches, check to see if
# this looks like a package name.
if suite.countTestCases() != 0:
suites.append(suite)
continue
# Load all modules whose name is .
prefix = arg + "."
for module in self.modules:
if module.startswith(prefix):
suite = loader.loadTestsFromName(module)
suites.append(suite)
return loader.suiteClass(suites)
_COVERAGE_INI = """\
[report]
exclude_lines =
pragma: no cover
pragma: nocover
pragma:.*no${PLATFORM}
pragma:.*no${PY_IMPL}${PY_MAJOR}${PY_MINOR}
pragma:.*no${PY_IMPL}${PY_MAJOR}
pragma:.*nopy${PY_MAJOR}
pragma:.*nopy${PY_MAJOR}${PY_MINOR}
"""
class MainProgram:
"""
This class implements the main program. It can be subclassed by
users who wish to customize some parts of the main program.
(Adding additional command line options, customizing test loading, etc.)
"""
DEFAULT_VERBOSITY = 2
def __init__(self, argv):
self.init_option_parser()
self.parse_options(argv)
self.setup_logging()
def init_option_parser(self):
usage = "%prog [options] [TEST] ..."
op = optparse.OptionParser(usage=usage, add_help_option=False)
self.option_parser = op
op.add_option(
"--hide-output",
dest="show_output",
action="store_false",
default=True,
help="Suppress data that tests print to stdout/stderr, and only "
"show it if the test fails.",
)
op.add_option(
"-o",
"--output",
help="Write results to a file in a JSON format to be read by Buck",
)
op.add_option(
"-f",
"--failfast",
action="store_true",
default=False,
help="Stop after the first failure",
)
op.add_option(
"-l",
"--list-tests",
action="store_true",
dest="list",
default=False,
help="List tests and exit",
)
op.add_option(
"-r",
"--regex",
default=None,
help="Regex to apply to tests, to only run those tests",
)
op.add_option(
"--collect-coverage",
action="store_true",
default=False,
help="Collect test coverage information",
)
op.add_option(
"--coverage-include",
default="*",
help='File globs to include in converage (split by ",")',
)
op.add_option(
"--coverage-omit",
default="",
help='File globs to omit from converage (split by ",")',
)
op.add_option(
"--logger",
action="append",
metavar="=",
default=[],
help="Configure log levels for specific logger categories",
)
op.add_option(
"-q",
"--quiet",
action="count",
default=0,
help="Decrease the verbosity (may be specified multiple times)",
)
op.add_option(
"-v",
"--verbosity",
action="count",
default=self.DEFAULT_VERBOSITY,
help="Increase the verbosity (may be specified multiple times)",
)
op.add_option(
"-?", "--help", action="help", help="Show this help message and exit"
)
def parse_options(self, argv):
self.options, self.test_args = self.option_parser.parse_args(argv[1:])
self.options.verbosity -= self.options.quiet
if self.options.collect_coverage and coverage is None:
self.option_parser.error("coverage module is not available")
self.options.coverage_include = self.options.coverage_include.split(",")
if self.options.coverage_omit == "":
self.options.coverage_omit = []
else:
self.options.coverage_omit = self.options.coverage_omit.split(",")
def setup_logging(self):
# Configure the root logger to log at INFO level.
# This is similar to logging.basicConfig(), but uses our
# StderrLogHandler instead of a StreamHandler.
fmt = logging.Formatter("%(pathname)s:%(lineno)s: %(message)s")
log_handler = StderrLogHandler()
log_handler.setFormatter(fmt)
root_logger = logging.getLogger()
root_logger.addHandler(log_handler)
root_logger.setLevel(logging.INFO)
level_names = {
"debug": logging.DEBUG,
"info": logging.INFO,
"warn": logging.WARNING,
"warning": logging.WARNING,
"error": logging.ERROR,
"critical": logging.CRITICAL,
"fatal": logging.FATAL,
}
for value in self.options.logger:
parts = value.rsplit("=", 1)
if len(parts) != 2:
self.option_parser.error(
"--logger argument must be of the "
"form =: %s" % value
)
name = parts[0]
level_name = parts[1].lower()
level = level_names.get(level_name)
if level is None:
self.option_parser.error(
"invalid log level %r for log " "category %s" % (parts[1], name)
)
logging.getLogger(name).setLevel(level)
def create_loader(self):
import __test_modules__
return Loader(__test_modules__.TEST_MODULES, self.options.regex)
def load_tests(self):
loader = self.create_loader()
if self.options.collect_coverage:
self.start_coverage()
include = self.options.coverage_include
omit = self.options.coverage_omit
if include and "*" not in include:
optimize_for_coverage(self.cov, include, omit)
if self.test_args:
suite = loader.load_args(self.test_args)
else:
suite = loader.load_all()
if self.options.collect_coverage:
self.cov.start()
return suite
def get_tests(self, test_suite):
tests = []
for test in test_suite:
if isinstance(test, unittest.TestSuite):
tests.extend(self.get_tests(test))
else:
tests.append(test)
return tests
def run(self):
test_suite = self.load_tests()
if self.options.list:
for test in self.get_tests(test_suite):
method_name = getattr(test, "_testMethodName", "")
name = _format_test_name(test.__class__, method_name)
print(name)
return EXIT_CODE_SUCCESS
else:
result = self.run_tests(test_suite)
if self.options.output is not None:
with open(self.options.output, "w") as f:
json.dump(result.getResults(), f, indent=4, sort_keys=True)
if not result.wasSuccessful():
return EXIT_CODE_TEST_FAILURE
return EXIT_CODE_SUCCESS
def run_tests(self, test_suite):
# Install a signal handler to catch Ctrl-C and display the results
# (but only if running >2.6).
if sys.version_info[0] > 2 or sys.version_info[1] > 6:
unittest.installHandler()
# Run the tests
runner = BuckTestRunner(
self,
test_suite,
verbosity=self.options.verbosity,
show_output=self.options.show_output,
)
result = runner.run(test_suite)
if self.options.collect_coverage and self.options.show_output:
self.cov.stop()
try:
self.cov.report(file=sys.stdout)
except coverage.misc.CoverageException:
print("No lines were covered, potentially restricted by file filters")
return result
def get_abbr_impl(self):
"""Return abbreviated implementation name."""
impl = platform.python_implementation()
if impl == "PyPy":
return "pp"
elif impl == "Jython":
return "jy"
elif impl == "IronPython":
return "ip"
elif impl == "CPython":
return "cp"
else:
raise RuntimeError("unknown python runtime")
def start_coverage(self):
if not self.options.collect_coverage:
return
with tempfile.NamedTemporaryFile("w", delete=False) as coverage_ini:
coverage_ini.write(_COVERAGE_INI)
self._coverage_ini_path = coverage_ini.name
# Keep the original working dir in case tests use os.chdir
self._original_working_dir = os.getcwd()
# for coverage config ignores by platform/python version
os.environ["PLATFORM"] = sys.platform
os.environ["PY_IMPL"] = self.get_abbr_impl()
os.environ["PY_MAJOR"] = str(sys.version_info.major)
os.environ["PY_MINOR"] = str(sys.version_info.minor)
self.cov = coverage.Coverage(
include=self.options.coverage_include,
omit=self.options.coverage_omit,
config_file=coverage_ini.name,
)
self.cov.erase()
self.cov.start()
def get_coverage(self):
if not self.options.collect_coverage:
return None
try:
os.remove(self._coverage_ini_path)
except OSError:
pass # Better to litter than to fail the test
# Switch back to the original working directory.
os.chdir(self._original_working_dir)
result = {}
self.cov.stop()
try:
f = StringIO()
self.cov.report(file=f)
lines = f.getvalue().split("\n")
except coverage.misc.CoverageException:
# Nothing was covered. That's fine by us
return result
# N.B.: the format of the coverage library's output differs
# depending on whether one or more files are in the results
for line in lines[2:]:
if line.strip("-") == "":
break
r = line.split()[0]
analysis = self.cov.analysis2(r)
covString = self.convert_to_diff_cov_str(analysis)
if covString:
result[r] = covString
return result
def convert_to_diff_cov_str(self, analysis):
# Info on the format of analysis:
# http://nedbatchelder.com/code/coverage/api.html
if not analysis:
return None
numLines = max(
analysis[1][-1] if len(analysis[1]) else 0,
analysis[2][-1] if len(analysis[2]) else 0,
analysis[3][-1] if len(analysis[3]) else 0,
)
lines = ["N"] * numLines
for l in analysis[1]:
lines[l - 1] = "C"
for l in analysis[2]:
lines[l - 1] = "X"
for l in analysis[3]:
lines[l - 1] = "U"
return "".join(lines)
def main(argv):
return MainProgram(sys.argv).run()
if __name__ == "__main__":
sys.exit(main(sys.argv))
================================================
FILE: build/fbcode_builder/CMake/fb_py_win_main.c
================================================
// Copyright (c) Facebook, Inc. and its affiliates.
#define WIN32_LEAN_AND_MEAN
#include
#include
#include
#define PATH_SIZE 32768
typedef int (*Py_Main)(int, wchar_t**);
int locate_py_main(int argc, wchar_t** argv) {
/*
* We have to dynamically locate Python3.dll because we may be loading a
* Python native module while running. If that module is built with a
* different Python version, we will end up a DLL import error. To resolve
* this, we can either ship an embedded version of Python with us or
* dynamically look up existing Python distribution installed on user's
* machine. This way, we should be able to get a consistent version of
* Python3.dll and .pyd modules.
*/
HINSTANCE python_dll;
Py_Main pymain;
python_dll =
LoadLibraryExW(L"python3.dll", NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
int returncode = 0;
if (python_dll != NULL) {
pymain = (Py_Main)GetProcAddress(python_dll, "Py_Main");
if (pymain != NULL) {
returncode = (pymain)(argc, argv);
} else {
fprintf(stderr, "error: %d unable to load Py_Main\n", GetLastError());
}
FreeLibrary(python_dll);
} else {
fprintf(stderr, "error: %d unable to locate python3.dll\n", GetLastError());
return 1;
}
return returncode;
}
int wmain() {
/*
* This executable will be prepended to the start of a Python ZIP archive.
* Python will be able to directly execute the ZIP archive, so we simply
* need to tell Py_Main() to run our own file. Duplicate the argument list
* and add our file name to the beginning to tell Python what file to invoke.
*/
wchar_t** pyargv = malloc(sizeof(wchar_t*) * (__argc + 1));
if (!pyargv) {
fprintf(stderr, "error: failed to allocate argument vector\n");
return 1;
}
/* Py_Main wants the wide character version of the argv so we pull those
* values from the global __wargv array that has been prepared by MSVCRT.
*
* In order for the zipapp to run we need to insert an extra argument in
* the front of the argument vector that points to ourselves.
*
* An additional complication is that, depending on who prepared the argument
* string used to start our process, the computed __wargv[0] can be a simple
* shell word like `watchman-wait` which is normally resolved together with
* the PATH by the shell.
* That unresolved path isn't sufficient to start the zipapp on windows;
* we need the fully qualified path.
*
* Given:
* __wargv == {"watchman-wait", "-h"}
*
* we want to pass the following to Py_Main:
*
* {
* "z:\build\watchman\python\watchman-wait.exe",
* "z:\build\watchman\python\watchman-wait.exe",
* "-h"
* }
*/
wchar_t full_path_to_argv0[PATH_SIZE];
DWORD len = GetModuleFileNameW(NULL, full_path_to_argv0, PATH_SIZE);
if (len == 0 ||
len == PATH_SIZE && GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
fprintf(
stderr,
"error: %d while retrieving full path to this executable\n",
GetLastError());
return 1;
}
for (int n = 1; n < __argc; ++n) {
pyargv[n + 1] = __wargv[n];
}
pyargv[0] = full_path_to_argv0;
pyargv[1] = full_path_to_argv0;
return locate_py_main(__argc + 1, pyargv);
}
================================================
FILE: build/fbcode_builder/CMake/make_fbpy_archive.py
================================================
#!/usr/bin/env python3
#
# Copyright (c) Facebook, Inc. and its affiliates.
#
import argparse
import collections
import errno
import os
import shutil
import subprocess
import sys
import tempfile
import zipapp
MANIFEST_SEPARATOR = " :: "
MANIFEST_HEADER_V1 = "FBPY_MANIFEST 1\n"
class UsageError(Exception):
def __init__(self, message):
self.message = message
def __str__(self):
return self.message
class BadManifestError(UsageError):
def __init__(self, path, line_num, message):
full_msg = "%s:%s: %s" % (path, line_num, message)
super().__init__(full_msg)
self.path = path
self.line_num = line_num
self.raw_message = message
PathInfo = collections.namedtuple(
"PathInfo", ("src", "dest", "manifest_path", "manifest_line")
)
def parse_manifest(manifest, path_map):
bad_prefix = ".." + os.path.sep
manifest_dir = os.path.dirname(manifest)
with open(manifest, "r") as f:
line_num = 1
line = f.readline()
if line != MANIFEST_HEADER_V1:
raise BadManifestError(
manifest, line_num, "Unexpected manifest file header"
)
for line in f:
line_num += 1
if line.startswith("#"):
continue
line = line.rstrip("\n")
parts = line.split(MANIFEST_SEPARATOR)
if len(parts) != 2:
msg = "line must be of the form SRC %s DEST" % MANIFEST_SEPARATOR
raise BadManifestError(manifest, line_num, msg)
src, dest = parts
dest = os.path.normpath(dest)
if dest.startswith(bad_prefix):
msg = "destination path starts with %s: %s" % (bad_prefix, dest)
raise BadManifestError(manifest, line_num, msg)
if not os.path.isabs(src):
src = os.path.normpath(os.path.join(manifest_dir, src))
if dest in path_map:
prev_info = path_map[dest]
msg = (
"multiple source paths specified for destination "
"path %s. Previous source was %s from %s:%s"
% (
dest,
prev_info.src,
prev_info.manifest_path,
prev_info.manifest_line,
)
)
raise BadManifestError(manifest, line_num, msg)
info = PathInfo(
src=src,
dest=dest,
manifest_path=manifest,
manifest_line=line_num,
)
path_map[dest] = info
def populate_install_tree(inst_dir, path_map):
os.mkdir(inst_dir)
dest_dirs = {"": False}
def make_dest_dir(path):
if path in dest_dirs:
return
parent = os.path.dirname(path)
make_dest_dir(parent)
abs_path = os.path.join(inst_dir, path)
os.mkdir(abs_path)
dest_dirs[path] = False
def install_file(info):
dir_name, base_name = os.path.split(info.dest)
make_dest_dir(dir_name)
if base_name == "__init__.py":
dest_dirs[dir_name] = True
abs_dest = os.path.join(inst_dir, info.dest)
shutil.copy2(info.src, abs_dest)
# Copy all of the destination files
for info in path_map.values():
install_file(info)
# Create __init__ files in any directories that don't have them.
for dir_path, has_init in dest_dirs.items():
if has_init:
continue
init_path = os.path.join(inst_dir, dir_path, "__init__.py")
with open(init_path, "w"):
pass
def build_pex(args, path_map):
"""Create a self executing python binary using the PEX tool
This type of Python binary is more complex as it requires a third-party tool,
but it does support native language extensions (.so/.dll files).
"""
dest_dir = os.path.dirname(args.output)
with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir:
inst_dir = os.path.join(tmpdir, "tree")
populate_install_tree(inst_dir, path_map)
if os.path.exists(os.path.join(inst_dir, "__main__.py")):
os.rename(
os.path.join(inst_dir, "__main__.py"),
os.path.join(inst_dir, "main.py"),
)
args.main = "main"
tmp_output = os.path.abspath(os.path.join(tmpdir, "output.exe"))
subprocess.check_call(
["pex"]
+ ["--output-file", tmp_output]
+ ["--python", args.python]
+ ["--sources-directory", inst_dir]
+ ["-e", args.main]
)
os.replace(tmp_output, args.output)
def build_zipapp(args, path_map):
"""Create a self executing python binary using Python 3's built-in
zipapp module.
This type of Python binary is relatively simple, as zipapp is part of the
standard library, but it does not support native language extensions
(.so/.dll files).
"""
dest_dir = os.path.dirname(args.output)
with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir:
inst_dir = os.path.join(tmpdir, "tree")
populate_install_tree(inst_dir, path_map)
tmp_output = os.path.join(tmpdir, "output.exe")
zipapp.create_archive(
inst_dir, target=tmp_output, interpreter=args.python, main=args.main
)
os.replace(tmp_output, args.output)
def create_main_module(args, inst_dir, path_map):
if not args.main:
assert "__main__.py" in path_map
return
dest_path = os.path.join(inst_dir, "__main__.py")
main_module, main_fn = args.main.split(":")
main_contents = """\
#!{python}
if __name__ == "__main__":
import {main_module}
{main_module}.{main_fn}()
""".format(
python=args.python, main_module=main_module, main_fn=main_fn
)
with open(dest_path, "w") as f:
f.write(main_contents)
os.chmod(dest_path, 0o755)
def build_install_dir(args, path_map):
"""Create a directory that contains all of the sources, with a __main__
module to run the program.
"""
# Populate a temporary directory first, then rename to the destination
# location. This ensures that we don't ever leave a halfway-built
# directory behind at the output path if something goes wrong.
dest_dir = os.path.dirname(args.output)
with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir:
inst_dir = os.path.join(tmpdir, "tree")
populate_install_tree(inst_dir, path_map)
create_main_module(args, inst_dir, path_map)
os.rename(inst_dir, args.output)
def ensure_directory(path):
try:
os.makedirs(path)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
def install_library(args, path_map):
"""Create an installation directory a python library."""
out_dir = args.output
out_manifest = args.output + ".manifest"
install_dir = args.install_dir
if not install_dir:
install_dir = out_dir
os.makedirs(out_dir)
with open(out_manifest, "w") as manifest:
manifest.write(MANIFEST_HEADER_V1)
for info in path_map.values():
abs_dest = os.path.join(out_dir, info.dest)
ensure_directory(os.path.dirname(abs_dest))
print("copy %r --> %r" % (info.src, abs_dest))
shutil.copy2(info.src, abs_dest)
installed_dest = os.path.join(install_dir, info.dest)
manifest.write("%s%s%s\n" % (installed_dest, MANIFEST_SEPARATOR, info.dest))
def parse_manifests(args):
# Process args.manifest_separator to help support older versions of CMake
if args.manifest_separator:
manifests = []
for manifest_arg in args.manifests:
split_arg = manifest_arg.split(args.manifest_separator)
manifests.extend(split_arg)
args.manifests = manifests
path_map = {}
for manifest in args.manifests:
parse_manifest(manifest, path_map)
return path_map
def check_main_module(args, path_map):
# Translate an empty string in the --main argument to None,
# just to allow the CMake logic to be slightly simpler and pass in an
# empty string when it really wants the default __main__.py module to be
# used.
if args.main == "":
args.main = None
if args.type == "lib-install":
if args.main is not None:
raise UsageError("cannot specify a --main argument with --type=lib-install")
return
main_info = path_map.get("__main__.py")
if args.main:
if main_info is not None:
msg = (
"specified an explicit main module with --main, "
"but the file listing already includes __main__.py"
)
raise BadManifestError(
main_info.manifest_path, main_info.manifest_line, msg
)
parts = args.main.split(":")
if len(parts) != 2:
raise UsageError(
"argument to --main must be of the form MODULE:CALLABLE "
"(received %s)" % (args.main,)
)
else:
if main_info is None:
raise UsageError(
"no main module specified with --main, "
"and no __main__.py module present"
)
BUILD_TYPES = {
"pex": build_pex,
"zipapp": build_zipapp,
"dir": build_install_dir,
"lib-install": install_library,
}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("-o", "--output", required=True, help="The output file path")
ap.add_argument(
"--install-dir",
help="When used with --type=lib-install, this parameter specifies the "
"final location where the library where be installed. This can be "
"used to generate the library in one directory first, when you plan "
"to move or copy it to another final location later.",
)
ap.add_argument(
"--manifest-separator",
help="Split manifest arguments around this separator. This is used "
"to support older versions of CMake that cannot supply the manifests "
"as separate arguments.",
)
ap.add_argument(
"--main",
help="The main module to run, specified as :. "
"This must be specified if and only if the archive does not contain "
"a __main__.py file.",
)
ap.add_argument(
"--python",
help="Explicitly specify the python interpreter to use for the " "executable.",
)
ap.add_argument(
"--type", choices=BUILD_TYPES.keys(), help="The type of output to build."
)
ap.add_argument(
"manifests",
nargs="+",
help="The manifest files specifying how to construct the archive",
)
args = ap.parse_args()
if args.python is None:
args.python = sys.executable
if args.type is None:
# In the future we might want different default output types
# for different platforms.
args.type = "zipapp"
build_fn = BUILD_TYPES[args.type]
try:
path_map = parse_manifests(args)
check_main_module(args, path_map)
except UsageError as ex:
print("error: %s" % (ex,), file=sys.stderr)
sys.exit(1)
build_fn(args, path_map)
if __name__ == "__main__":
main()
================================================
FILE: build/fbcode_builder/LICENSE
================================================
MIT License
Copyright (c) Facebook, Inc. and its affiliates.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: build/fbcode_builder/README.md
================================================
# Easy builds for Facebook projects
This directory contains tools designed to simplify continuous-integration
(and other builds) of Facebook open source projects. In particular, this helps
manage builds for cross-project dependencies.
The main entry point is the `getdeps.py` script. This script has several
subcommands, but the most notable is the `build` command. This will download
and build all dependencies for a project, and then build the project itself.
## Deployment
This directory is copied literally into a number of different Facebook open
source repositories. Any change made to code in this directory will be
automatically be replicated by our open source tooling into all GitHub hosted
repositories that use `fbcode_builder`. Typically this directory is copied
into the open source repositories as `build/fbcode_builder/`.
# Project Configuration Files
The `manifests` subdirectory contains configuration files for many different
projects, describing how to build each project. These files also list
dependencies between projects, enabling `getdeps.py` to build all dependencies
for a project before building the project itself.
# Shared CMake utilities
Since this directory is copied into many Facebook open source repositories,
it is also used to help share some CMake utility files across projects. The
`CMake/` subdirectory contains a number of `.cmake` files that are shared by
the CMake-based build systems across several different projects.
# Older Build Scripts
This directory also still contains a handful of older build scripts that
pre-date the current `getdeps.py` build system. Most of the other `.py` files
in this top directory, apart from `getdeps.py` itself, are from this older
build system. This older system is only used by a few remaining projects, and
new projects should generally use the newer `getdeps.py` script, by adding a
new configuration file in the `manifests/` subdirectory.
================================================
FILE: build/fbcode_builder/getdeps/__init__.py
================================================
# pyre-strict
================================================
FILE: build/fbcode_builder/getdeps/builder.py
================================================
#!/usr/bin/env python3
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import glob
import json
import os
import os.path
import pathlib
import re
import shutil
import stat
import subprocess
import sys
import typing
from collections.abc import Callable, Sequence
from shlex import quote as shellquote
from .copytree import rmtree_more, simple_copytree
from .dyndeps import create_dyn_dep_munger
from .envfuncs import add_path_entry, Env, path_search
from .fetcher import copy_if_different, is_public_commit
from .runcmd import make_memory_limit_preexec_fn, run_cmd
if typing.TYPE_CHECKING:
from .buildopts import BuildOptions
from .dyndeps import DepBase
from .load import ManifestLoader
from .manifest import ManifestContext, ManifestParser
class BuilderBase:
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
src_dir: str,
build_dir: str | None,
inst_dir: str,
env: Env | None = None,
final_install_prefix: str | None = None,
) -> None:
self.env: Env = Env()
if env:
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got
# `Env`.
self.env.update(env)
subdir: str | None = manifest.get("build", "subdir", ctx=ctx)
if subdir:
src_dir = os.path.join(src_dir, subdir)
self.patchfile: str | None = manifest.get("build", "patchfile", ctx=ctx)
self.patchfile_opts: str = (
manifest.get("build", "patchfile_opts", ctx=ctx) or ""
)
self.ctx: ManifestContext = ctx
self.src_dir: str = src_dir
self.build_dir: str = build_dir or src_dir
self.inst_dir: str = inst_dir
self.build_opts: BuildOptions = build_opts
self.manifest: ManifestParser = manifest
self.final_install_prefix: str | None = final_install_prefix
self.loader: ManifestLoader = loader
self.dep_manifests: list[ManifestParser] = dep_manifests
self.install_dirs: list[str] = [
loader.get_project_install_dir(m) for m in dep_manifests
]
def _get_cmd_prefix(self) -> list[str]:
if self.build_opts.is_windows():
vcvarsall = self.build_opts.get_vcvars_path()
if vcvarsall is not None:
# Since it sets rather a large number of variables we mildly abuse
# the cmd quoting rules to assemble a command that calls the script
# to prep the environment and then triggers the actual command that
# we wanted to run.
# Due to changes in vscrsall.bat, it now reports an ERRORLEVEL of 1
# even when succeeding. This occurs when an extension is not present.
# To continue, we must ignore the ERRORLEVEL returned. We do this by
# wrapping the call in a batch file that always succeeds.
wrapper = os.path.join(self.build_dir, "succeed.bat")
with open(wrapper, "w") as f:
f.write("@echo off\n")
f.write(f'call "{vcvarsall}" amd64\n')
f.write("set ERRORLEVEL=0\n")
f.write("exit /b 0\n")
return [wrapper, "&&"]
return []
def _check_cmd(self, cmd: list[str], **kwargs: object) -> None:
"""Run the command and abort on failure"""
# pyre-fixme[6]: For 2nd argument expected `Optional[Env]` but got `object`.
# pyre-fixme[6]: For 2nd argument expected `Optional[str]` but got `object`.
# pyre-fixme[6]: For 2nd argument expected `bool` but got `object`.
rc = self._run_cmd(cmd, **kwargs)
if rc != 0:
raise RuntimeError(f"Failure exit code {rc} for command {cmd}")
def _run_cmd(
self,
cmd: list[str],
cwd: str | None = None,
env: Env | None = None,
use_cmd_prefix: bool = True,
allow_fail: bool = False,
preexec_fn: Callable[[], None] | None = None,
) -> int:
if env:
e = self.env.copy()
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got
# `Env`.
e.update(env)
env = e
else:
env = self.env
if use_cmd_prefix:
cmd_prefix = self._get_cmd_prefix()
if cmd_prefix:
cmd = cmd_prefix + cmd
log_file = os.path.join(self.build_dir, "getdeps_build.log")
return run_cmd(
cmd=cmd,
env=env,
cwd=cwd or self.build_dir,
log_file=log_file,
allow_fail=allow_fail,
preexec_fn=preexec_fn,
)
def _reconfigure(self, reconfigure: bool) -> bool:
if self.build_dir is not None:
if not os.path.isdir(self.build_dir):
os.makedirs(self.build_dir)
reconfigure = True
return reconfigure
def _apply_patchfile(self) -> None:
if self.patchfile is None:
return
patched_sentinel_file = pathlib.Path(self.src_dir + "/.getdeps_patched")
if patched_sentinel_file.exists():
return
old_wd = os.getcwd()
os.chdir(self.src_dir)
# Apply patches from the git repo root so paths resolve correctly
# even when src_dir is a subdirectory of the repo.
try:
git_root = subprocess.check_output(
["git", "rev-parse", "--show-toplevel"], text=True
).strip()
os.chdir(git_root)
except subprocess.CalledProcessError:
pass # not a git repo, stay in src_dir
print(f"Patching {self.manifest.name} with {self.patchfile} in {os.getcwd()}")
patchfile = os.path.join(
self.build_opts.fbcode_builder_dir,
"patches",
# pyre-fixme[6]: For 3rd argument expected `Union[PathLike[str], str]`
# but got `Optional[str]`.
self.patchfile,
)
patchcmd = ["git", "apply", "--ignore-space-change"]
if self.patchfile_opts:
patchcmd.append(self.patchfile_opts)
try:
subprocess.check_call(patchcmd + [patchfile])
except subprocess.CalledProcessError:
raise ValueError(f"Failed to apply patch to {self.manifest.name}")
os.chdir(old_wd)
patched_sentinel_file.touch()
def prepare(self, reconfigure: bool) -> None:
print("Preparing %s..." % self.manifest.name)
reconfigure = self._reconfigure(reconfigure)
self._apply_patchfile()
self._prepare(reconfigure=reconfigure)
def debug(self, reconfigure: bool) -> None:
reconfigure = self._reconfigure(reconfigure)
self._apply_patchfile()
self._prepare(reconfigure=reconfigure)
env = self._compute_env()
print("Starting a shell in %s, ^D to exit..." % self.build_dir)
# TODO: print the command to run the build
shell = ["powershell.exe"] if sys.platform == "win32" else ["/bin/sh", "-i"]
self._run_cmd(shell, cwd=self.build_dir, env=env)
def printenv(self, reconfigure: bool) -> None:
"""print the environment in a shell sourcable format"""
reconfigure = self._reconfigure(reconfigure)
self._apply_patchfile()
self._prepare(reconfigure=reconfigure)
env = self._compute_env(env=Env(src={}))
prefix = "export "
sep = ":"
expand = "$"
expandpost = ""
if self.build_opts.is_windows():
prefix = "SET "
sep = ";"
expand = "%"
expandpost = "%"
for k, v in sorted(env.items()):
existing = os.environ.get(k, None)
if k.endswith("PATH") and existing:
v = shellquote(v) + sep + f"{expand}{k}{expandpost}"
else:
v = shellquote(v)
print("%s%s=%s" % (prefix, k, v))
def build(self, reconfigure: bool) -> None:
print("Building %s..." % self.manifest.name)
reconfigure = self._reconfigure(reconfigure)
self._apply_patchfile()
self._prepare(reconfigure=reconfigure)
self._build(reconfigure=reconfigure)
if self.build_opts.free_up_disk:
# don't clean --src-dir=. case as user may want to build again or run tests on the build
if self.src_dir.startswith(self.build_opts.scratch_dir) and os.path.isdir(
self.build_dir
):
if os.path.islink(self.build_dir):
os.remove(self.build_dir)
else:
rmtree_more(self.build_dir)
elif self.build_opts.is_windows():
# On Windows, emit a wrapper script that can be used to run build artifacts
# directly from the build directory, without installing them. On Windows $PATH
# needs to be updated to include all of the directories containing the runtime
# library dependencies in order to run the binaries.
script_path = self.get_dev_run_script_path()
dep_munger = create_dyn_dep_munger(
self.build_opts, self._compute_env(), self.install_dirs
)
dep_dirs = self.get_dev_run_extra_path_dirs(dep_munger)
# pyre-fixme[16]: Optional type has no attribute `emit_dev_run_script`.
dep_munger.emit_dev_run_script(script_path, dep_dirs)
@property
def _job_weight_mib(self) -> int:
# This is a hack, but we don't have a "defaults manifest" that we can
# customize per platform.
# TODO: Introduce some sort of defaults config that can select by
# platform, just like manifest contexts.
if sys.platform.startswith("freebsd"):
# clang on FreeBSD is quite memory-efficient.
default_job_weight = 512
else:
# 1.5 GiB is a lot to assume, but it's typical of Facebook-style C++.
# Some manifests are even heavier and should override.
default_job_weight = 1536
return int(
self.manifest.get(
"build", "job_weight_mib", str(default_job_weight), ctx=self.ctx
)
)
@property
def num_jobs(self) -> int:
return self.build_opts.get_num_jobs(self._job_weight_mib)
@property
def memory_limit_preexec_fn(self) -> Callable[[], None] | None:
"""Return a preexec_fn that caps per-process virtual memory.
Uses the same job_weight_mib that controls parallelism, so the memory
limit is consistent with the parallelism budget.
"""
return make_memory_limit_preexec_fn(self._job_weight_mib)
def run_tests(
self,
schedule_type: str,
owner: str | None,
test_filter: str | None,
test_exclude: str | None,
retry: int,
no_testpilot: bool,
timeout: int | None = None,
) -> None:
"""Execute any tests that we know how to run. If they fail,
raise an exception."""
pass
def _prepare(self, reconfigure: bool) -> None:
"""Prepare the build. Useful when need to generate config,
but builder is not the primary build system.
e.g. cargo when called from cmake"""
pass
def _build(self, reconfigure: bool) -> None:
"""Perform the build.
reconfigure will be set to true if the fetcher determined
that the sources have changed in such a way that the build
system needs to regenerate its rules."""
pass
def _compute_env(self, env: Env | None = None) -> Env:
if env is None:
env = self.env
# CMAKE_PREFIX_PATH is only respected when passed through the
# environment, so we construct an appropriate path to pass down
return self.build_opts.compute_env_for_install_dirs(
self.loader,
self.dep_manifests,
self.ctx,
env=env,
manifest=self.manifest,
)
def get_dev_run_script_path(self) -> str:
assert self.build_opts.is_windows()
return os.path.join(self.build_dir, "run.ps1")
def get_dev_run_extra_path_dirs(
self, dep_munger: DepBase | None = None
) -> list[str]:
assert self.build_opts.is_windows()
if dep_munger is None:
dep_munger = create_dyn_dep_munger(
self.build_opts, self._compute_env(), self.install_dirs
)
# pyre-fixme[16]: Optional type has no attribute `compute_dependency_paths`.
return dep_munger.compute_dependency_paths(self.build_dir)
class MakeBuilder(BuilderBase):
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
src_dir: str,
build_dir: str,
inst_dir: str,
build_args: list[str] | None,
install_args: list[str] | None,
test_args: list[str] | None,
) -> None:
super(MakeBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
self.build_args: list[str] = build_args or []
self.install_args: list[str] = install_args or []
self.test_args: list[str] | None = test_args
@property
def _make_binary(self) -> str | None:
return self.manifest.get("build", "make_binary", "make", ctx=self.ctx)
def _get_prefix(self) -> list[str]:
return ["PREFIX=" + self.inst_dir, "prefix=" + self.inst_dir]
def _build(self, reconfigure: bool) -> None:
env = self._compute_env()
# Need to ensure that PREFIX is set prior to install because
# libbpf uses it when generating its pkg-config file.
# The lowercase prefix is used by some projects.
cmd = (
[self._make_binary, "-j%s" % self.num_jobs]
+ self.build_args
+ self._get_prefix()
)
# pyre-fixme[6]: For 1st argument expected `List[str]` but got
# `List[Optional[str]]`.
self._check_cmd(cmd, env=env)
install_cmd = [self._make_binary] + self.install_args + self._get_prefix()
# pyre-fixme[6]: For 1st argument expected `List[str]` but got
# `List[Optional[str]]`.
self._check_cmd(install_cmd, env=env)
# bz2's Makefile doesn't install its .so properly
if self.manifest and self.manifest.name == "bz2":
libdir = os.path.join(self.inst_dir, "lib")
srcpattern = os.path.join(self.src_dir, "lib*.so.*")
print(f"copying to {libdir} from {srcpattern}")
for file in glob.glob(srcpattern):
shutil.copy(file, libdir)
def run_tests(
self,
schedule_type: str,
owner: str | None,
test_filter: str | None,
test_exclude: str | None,
retry: int,
no_testpilot: bool,
timeout: int | None = None,
) -> None:
if not self.test_args:
return
env = self._compute_env()
if test_filter:
env["GETDEPS_TEST_FILTER"] = test_filter
else:
env["GETDEPS_TEST_FILTER"] = ""
if retry:
# pyre-fixme[6]: Expected `str` but got `int`.
env["GETDEPS_TEST_RETRY"] = retry
else:
# pyre-fixme[6]: Expected `str` but got `int`.
env["GETDEPS_TEST_RETRY"] = 0
if timeout is not None:
env["GETDEPS_TEST_TIMEOUT"] = str(timeout)
cmd = (
[self._make_binary, "-j%s" % self.num_jobs]
# pyre-fixme[58]: `+` is not supported for operand types
# `list[Optional[str]]` and `Optional[list[str]]`.
+ self.test_args
+ self._get_prefix()
)
# pyre-fixme[6]: For 1st argument expected `List[str]` but got
# `List[Optional[str]]`.
self._check_cmd(cmd, allow_fail=False, env=env)
class CMakeBootStrapBuilder(MakeBuilder):
def _build(self, reconfigure: bool) -> None:
self._check_cmd(
[
"./bootstrap",
"--prefix=" + self.inst_dir,
f"--parallel={self.num_jobs}",
]
)
super(CMakeBootStrapBuilder, self)._build(reconfigure)
class AutoconfBuilder(BuilderBase):
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
src_dir: str,
build_dir: str,
inst_dir: str,
args: list[str] | None,
conf_env_args: dict[str, list[str]] | None,
) -> None:
super(AutoconfBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
self.args: list[str] = args or []
if (
not build_opts.shared_libs
and "--disable-shared" not in self.args
and "--enable-shared" not in self.args
):
self.args.append("--disable-shared")
self.conf_env_args: dict[str, list[str]] = conf_env_args or {}
@property
def _make_binary(self) -> str | None:
return self.manifest.get("build", "make_binary", "make", ctx=self.ctx)
def _build(self, reconfigure: bool) -> None:
configure_path = os.path.join(self.src_dir, "configure")
autogen_path = os.path.join(self.src_dir, "autogen.sh")
env = self._compute_env()
# Some configure scripts need additional env values passed derived from cmds
for k, cmd_args in self.conf_env_args.items():
out = (
subprocess.check_output(cmd_args, env=dict(env.items()))
.decode("utf-8")
.strip()
)
if out:
env.set(k, out)
if not os.path.exists(configure_path):
print("%s doesn't exist, so reconfiguring" % configure_path)
# This libtoolize call is a bit gross; the issue is that
# `autoreconf` as invoked by libsodium's `autogen.sh` doesn't
# seem to realize that it should invoke libtoolize and then
# error out when the configure script references a libtool
# related symbol.
self._check_cmd(["libtoolize"], cwd=self.src_dir, env=env)
# We generally prefer to call the `autogen.sh` script provided
# by the project on the basis that it may know more than plain
# autoreconf does.
if os.path.exists(autogen_path):
self._check_cmd(["bash", autogen_path], cwd=self.src_dir, env=env)
else:
self._check_cmd(["autoreconf", "-ivf"], cwd=self.src_dir, env=env)
configure_cmd = [configure_path, "--prefix=" + self.inst_dir] + self.args
self._check_cmd(configure_cmd, env=env)
only_install = self.manifest.get("build", "only_install", ctx=self.ctx)
if not only_install or only_install.lower() == "false":
# pyre-fixme[6]: For 1st argument expected `List[str]` but got
# `List[Optional[str]]`.
self._check_cmd([self._make_binary, "-j%s" % self.num_jobs], env=env)
# pyre-fixme[6]: For 1st argument expected `List[str]` but got
# `List[Union[str, None, str]]`.
self._check_cmd([self._make_binary, "install"], env=env)
class Iproute2Builder(BuilderBase):
# ./configure --prefix does not work for iproute2.
# Thus, explicitly copy sources from src_dir to build_dir, build,
# and then install to inst_dir using DESTDIR
# lastly, also copy include from build_dir to inst_dir
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
src_dir: str,
build_dir: str,
inst_dir: str,
) -> None:
super(Iproute2Builder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
def _build(self, reconfigure: bool) -> None:
configure_path = os.path.join(self.src_dir, "configure")
env = self.env.copy()
self._check_cmd([configure_path], env=env)
shutil.rmtree(self.build_dir)
shutil.copytree(self.src_dir, self.build_dir)
self._check_cmd(["make", "-j%s" % self.num_jobs], env=env)
install_cmd = ["make", "install", "DESTDIR=" + self.inst_dir]
for d in ["include", "lib"]:
if not os.path.isdir(os.path.join(self.inst_dir, d)):
shutil.copytree(
os.path.join(self.build_dir, d), os.path.join(self.inst_dir, d)
)
self._check_cmd(install_cmd, env=env)
class MesonBuilder(BuilderBase):
# MesonBuilder assumes that meson build tool has already been installed on
# the machine.
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
src_dir: str,
build_dir: str,
inst_dir: str,
) -> None:
super(MesonBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
def _build(self, reconfigure: bool) -> None:
env = self._compute_env()
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got `Env`.
meson: str | None = path_search(env, "meson")
if meson is None:
raise Exception("Failed to find Meson")
setup_args = self.manifest.get_section_as_args("meson.setup_args", self.ctx)
# Meson builds typically require setup, compile, and install steps.
# During this setup step we ensure that the static library is built and
# the prefix is empty.
self._check_cmd(
[
meson,
"setup",
]
+ setup_args
+ [
self.build_dir,
self.src_dir,
]
)
# Compile step needs to satisfy the build directory that was previously
# prepared during setup.
self._check_cmd([meson, "compile", "-C", self.build_dir])
# Install step
self._check_cmd(
[meson, "install", "-C", self.build_dir, "--destdir", self.inst_dir]
)
class CMakeBuilder(BuilderBase):
MANUAL_BUILD_SCRIPT = """\
#!{sys.executable}
import argparse
import subprocess
import sys
CMAKE = {cmake!r}
CTEST = {ctest!r}
SRC_DIR = {src_dir!r}
BUILD_DIR = {build_dir!r}
INSTALL_DIR = {install_dir!r}
CMD_PREFIX = {cmd_prefix!r}
CMAKE_ENV = {env_str}
CMAKE_DEFINE_ARGS = {define_args_str}
def get_jobs_argument(num_jobs_arg: int) -> str:
if num_jobs_arg > 0:
return "-j" + str(num_jobs_arg)
import multiprocessing
num_jobs = multiprocessing.cpu_count() // 2
return "-j" + str(num_jobs)
def main():
ap = argparse.ArgumentParser()
ap.add_argument(
"cmake_args",
nargs=argparse.REMAINDER,
help='Any extra arguments after an "--" argument will be passed '
"directly to CMake."
)
ap.add_argument(
"--mode",
choices=["configure", "build", "install", "test"],
default="configure",
help="The mode to run: configure, build, or install. "
"Defaults to configure",
)
ap.add_argument(
"--build",
action="store_const",
const="build",
dest="mode",
help="An alias for --mode=build",
)
ap.add_argument(
"-j",
"--num-jobs",
action="store",
type=int,
default=0,
help="Run the build or tests with the specified number of parallel jobs",
)
ap.add_argument(
"--install",
action="store_const",
const="install",
dest="mode",
help="An alias for --mode=install",
)
ap.add_argument(
"--test",
action="store_const",
const="test",
dest="mode",
help="An alias for --mode=test",
)
args = ap.parse_args()
# Strip off a leading "--" from the additional CMake arguments
if args.cmake_args and args.cmake_args[0] == "--":
args.cmake_args = args.cmake_args[1:]
env = CMAKE_ENV
if args.mode == "configure":
full_cmd = CMD_PREFIX + [CMAKE, SRC_DIR] + CMAKE_DEFINE_ARGS + args.cmake_args
elif args.mode in ("build", "install"):
target = "all" if args.mode == "build" else "install"
full_cmd = CMD_PREFIX + [
CMAKE,
"--build",
BUILD_DIR,
"--target",
target,
"--config",
"{build_type}",
get_jobs_argument(args.num_jobs),
] + args.cmake_args
elif args.mode == "test":
full_cmd = CMD_PREFIX + [
{dev_run_script}CTEST,
"--output-on-failure",
get_jobs_argument(args.num_jobs),
] + args.cmake_args
else:
ap.error("unknown invocation mode: %s" % (args.mode,))
cmd_str = " ".join(full_cmd)
print("Running: %r" % (cmd_str,))
proc = subprocess.run(full_cmd, env=env, cwd=BUILD_DIR)
sys.exit(proc.returncode)
if __name__ == "__main__":
main()
"""
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
src_dir: str,
build_dir: str,
inst_dir: str,
defines: dict[str, str] | None,
final_install_prefix: str | None = None,
extra_cmake_defines: dict[str, str] | None = None,
cmake_targets: list[str] | None = None,
) -> None:
super(CMakeBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
final_install_prefix=final_install_prefix,
)
self.defines: dict[str, str] = defines or {}
if extra_cmake_defines:
self.defines.update(extra_cmake_defines)
self.cmake_targets: list[str] = cmake_targets or ["install"]
if build_opts.is_windows():
try:
from .facebook.vcvarsall import extra_vc_cmake_defines
except ImportError:
pass
else:
self.defines.update(extra_vc_cmake_defines)
self.loader = loader
if build_opts.shared_libs:
self.defines["BUILD_SHARED_LIBS"] = "ON"
self.defines["BOOST_LINK_STATIC"] = "OFF"
def _invalidate_cache(self) -> None:
for name in [
"CMakeCache.txt",
"CMakeFiles/CMakeError.log",
"CMakeFiles/CMakeOutput.log",
]:
name = os.path.join(self.build_dir, name)
if os.path.isdir(name):
shutil.rmtree(name)
elif os.path.exists(name):
os.unlink(name)
def _needs_reconfigure(self) -> bool:
for name in ["CMakeCache.txt", "build.ninja"]:
name = os.path.join(self.build_dir, name)
if not os.path.exists(name):
return True
return False
def _write_build_script(self, **kwargs: object) -> None:
# pyre-fixme[16]: `object` has no attribute `items`.
env_lines = [" {!r}: {!r},".format(k, v) for k, v in kwargs["env"].items()]
kwargs["env_str"] = "\n".join(["{"] + env_lines + ["}"])
if self.build_opts.is_windows():
kwargs["dev_run_script"] = '"powershell.exe", {!r}, '.format(
self.get_dev_run_script_path()
)
else:
kwargs["dev_run_script"] = ""
define_arg_lines = ["["]
# pyre-fixme[16]: `object` has no attribute `__iter__`.
for arg in kwargs["define_args"]:
# Replace the CMAKE_INSTALL_PREFIX argument to use the INSTALL_DIR
# variable that we define in the MANUAL_BUILD_SCRIPT code.
if arg.startswith("-DCMAKE_INSTALL_PREFIX="):
value = " {!r}.format(INSTALL_DIR),".format(
"-DCMAKE_INSTALL_PREFIX={}"
)
else:
value = " {!r},".format(arg)
define_arg_lines.append(value)
define_arg_lines.append("]")
kwargs["define_args_str"] = "\n".join(define_arg_lines)
# In order to make it easier for developers to manually run builds for
# CMake-based projects, write out some build scripts that can be used to invoke
# CMake manually.
build_script_path = os.path.join(self.build_dir, "run_cmake.py")
script_contents = self.MANUAL_BUILD_SCRIPT.format(**kwargs)
with open(build_script_path, "wb") as f:
f.write(script_contents.encode())
os.chmod(build_script_path, 0o755)
def _compute_cmake_define_args(self, env: Env) -> list[str]:
defines = {
"CMAKE_INSTALL_PREFIX": self.final_install_prefix or self.inst_dir,
"BUILD_SHARED_LIBS": "OFF",
# Some of the deps (rsocket) default to UBSAN enabled if left
# unspecified. Some of the deps fail to compile in release mode
# due to warning->error promotion. RelWithDebInfo is the happy
# medium.
"CMAKE_BUILD_TYPE": self.build_opts.build_type,
}
if "SANDCASTLE" not in os.environ:
# We sometimes see intermittent ccache related breakages on some
# of the FB internal CI hosts, so we prefer to disable ccache
# when running in that environment.
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got
# `Env`.
ccache = path_search(env, "ccache")
if ccache:
defines["CMAKE_CXX_COMPILER_LAUNCHER"] = ccache
else:
# rocksdb does its own probing for ccache.
# Ensure that it is disabled on sandcastle
env["CCACHE_DISABLE"] = "1"
# Some sandcastle hosts have broken ccache related dirs, and
# even though we've asked for it to be disabled ccache is
# still invoked by rocksdb's cmake.
# Redirect its config directory to somewhere that is guaranteed
# fresh to us, and that won't have any ccache data inside.
env["CCACHE_DIR"] = f"{self.build_opts.scratch_dir}/ccache"
if "GITHUB_ACTIONS" in os.environ and self.build_opts.is_windows():
# GitHub actions: the host has both gcc and msvc installed, and
# the default behavior of cmake is to prefer gcc.
# Instruct cmake that we want it to use cl.exe; this is important
# because Boost prefers cl.exe and the mismatch results in cmake
# with gcc not being able to find boost built with cl.exe.
defines["CMAKE_C_COMPILER"] = "cl.exe"
defines["CMAKE_CXX_COMPILER"] = "cl.exe"
if self.build_opts.is_darwin():
# Try to persuade cmake to set the rpath to match the lib
# dirs of the dependencies. This isn't automatic, and to
# make things more interesting, cmake uses `;` as the path
# separator, so translate the runtime path to something
# that cmake will parse
defines["CMAKE_INSTALL_RPATH"] = ";".join(
# pyre-fixme[16]: Optional type has no attribute `split`.
env.get("DYLD_LIBRARY_PATH", "").split(":")
)
# Tell cmake that we want to set the rpath in the tree
# at build time. Without this the rpath is only set
# at the moment that the binaries are installed. That
# default is problematic for example when using the
# gtest integration in cmake which runs the built test
# executables during the build to discover the set of
# tests.
defines["CMAKE_BUILD_WITH_INSTALL_RPATH"] = "ON"
defines.update(self.defines)
define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()]
# if self.build_opts.is_windows():
# define_args += ["-G", "Visual Studio 15 2017 Win64"]
define_args += ["-G", "Ninja"]
return define_args
def _run_include_rewriter(self) -> None:
"""Run include path rewriting on source files before building."""
from .include_rewriter import rewrite_includes_from_manifest
print(f"Rewriting include paths for {self.manifest.name}...")
try:
modified_count = rewrite_includes_from_manifest(
self.manifest, self.ctx, self.src_dir, verbose=True
)
if modified_count > 0:
print(f"Successfully modified {modified_count} files")
else:
print("No files needed modification")
except Exception as e:
print(f"Warning: Include path rewriting failed: {e}")
# Don't fail the build for include rewriting issues
def _build(self, reconfigure: bool) -> None:
# Check if include rewriting is enabled
rewrite_includes: str | None = self.manifest.get(
"build", "rewrite_includes", "false", ctx=self.ctx
)
# pyre-fixme[16]: Optional type has no attribute `lower`.
if rewrite_includes.lower() == "true":
self._run_include_rewriter()
reconfigure = reconfigure or self._needs_reconfigure()
env = self._compute_env()
if not self.build_opts.is_windows() and self.final_install_prefix:
env["DESTDIR"] = self.inst_dir
# Resolve the cmake that we installed
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got `Env`.
cmake = path_search(env, "cmake")
if cmake is None:
raise Exception("Failed to find CMake")
if self.build_opts.is_windows():
checkdir = self.src_dir
if os.path.exists(checkdir):
children = os.listdir(checkdir)
print(f"Building from source {checkdir} contents: {children}")
else:
print(f"Source {checkdir} not found")
if reconfigure:
define_args = self._compute_cmake_define_args(env)
self._write_build_script(
cmd_prefix=self._get_cmd_prefix(),
cmake=cmake,
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but
# got `Env`.
ctest=path_search(env, "ctest"),
env=env,
define_args=define_args,
src_dir=self.src_dir,
build_dir=self.build_dir,
install_dir=self.inst_dir,
sys=sys,
build_type=self.build_opts.build_type,
)
self._invalidate_cache()
self._check_cmd([cmake, self.src_dir] + define_args, env=env)
self._check_cmd(
# pyre-fixme[6]: For 1st argument expected `List[str]` but got
# `List[Optional[str]]`.
[cmake, "--build", self.build_dir, "--target"]
+ self.cmake_targets
+ [
"--config",
self.build_opts.build_type,
"-j",
str(self.num_jobs),
],
env=env,
preexec_fn=self.memory_limit_preexec_fn,
)
def _build_targets(self, targets: Sequence[str]) -> None:
"""Build one or more cmake targets in parallel.
Args:
targets: Sequence of target names (strings) to build
"""
if not targets:
return
env = self._compute_env()
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got `Env`.
cmake = path_search(env, "cmake")
if cmake is None:
raise RuntimeError("unable to find cmake")
# Build all targets in a single cmake invocation for better parallelism
cmd = [
cmake,
"--build",
self.build_dir,
]
# Add all targets
for target in targets:
cmd.extend(["--target", target])
cmd.extend(
# pyre-fixme[6]: For 1st argument expected `Iterable[str]` but got
# `Iterable[Union[str, str, None, str]]`.
[
"--config",
self.build_opts.build_type,
"-j",
str(self.num_jobs),
]
)
self._check_cmd(cmd, env=env, preexec_fn=self.memory_limit_preexec_fn)
def _get_missing_test_executables(
self, test_filter: str | None, env: Env, ctest: str | None
) -> set[str]:
"""Discover which test executables are missing for the given filter.
Returns a set of missing executable basenames (without path)."""
if ctest is None:
return set()
# Run ctest -N (show tests without running) with the filter to see which tests match
cmd = [ctest, "-N"]
if test_filter:
cmd += ["-R", test_filter]
try:
output = subprocess.check_output(
cmd,
env=dict(env.items()),
cwd=self.build_dir,
stderr=subprocess.STDOUT,
text=True,
)
except subprocess.CalledProcessError as e:
# If ctest fails, it might be because executables don't exist yet
# Parse the error output to find the missing executables
output = e.output
# Parse output to find missing executable paths
# Look for lines like "Could not find executable /path/to/test_binary"
missing_executables = set()
for line in output.split("\n"):
match = re.search(r"Could not find executable (.+)", line)
if match:
exe_path = match.group(1)
exe_name = os.path.basename(exe_path)
missing_executables.add(exe_name)
return missing_executables
def run_tests(
self,
schedule_type: str,
owner: str | None,
test_filter: str | None,
test_exclude: str | None,
retry: int,
no_testpilot: bool,
timeout: int | None = None,
) -> None:
env = self._compute_env()
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got `Env`.
ctest: str | None = path_search(env, "ctest")
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got `Env`.
cmake = path_search(env, "cmake")
# Build only the missing test executables needed for the given filter.
# This is especially important for LocalDirFetcher projects (like fboss)
# where the build marker gets removed when building specific cmake targets.
missing_test_executables = self._get_missing_test_executables(
test_filter, env, ctest
)
if missing_test_executables:
sorted_executables = sorted(missing_test_executables)
print(f"Building missing test executables: {', '.join(sorted_executables)}")
# Build all missing executables in one cmake invocation for better parallelism
self._build_targets(sorted_executables)
def require_command(path: str | None, name: str) -> str:
if path is None:
raise RuntimeError("unable to find command `{}`".format(name))
return path
# On Windows, we also need to update $PATH to include the directories that
# contain runtime library dependencies. This is not needed on other platforms
# since CMake will emit RPATH properly in the binary so they can find these
# dependencies.
if self.build_opts.is_windows():
path_entries = self.get_dev_run_extra_path_dirs()
path = env.get("PATH")
if path:
path_entries.insert(0, path)
env["PATH"] = ";".join(path_entries)
# Don't use the cmd_prefix when running tests. This is vcvarsall.bat on
# Windows. vcvarsall.bat is only needed for the build, not tests. It
# unfortunately fails if invoked with a long PATH environment variable when
# running the tests.
use_cmd_prefix = False
def get_property(
test: dict[str, object], propname: str, defval: object = None
) -> object:
"""extracts a named property from a cmake test info json blob.
The properties look like:
[{"name": "WORKING_DIRECTORY"},
{"value": "something"}]
We assume that it is invalid for the same named property to be
listed more than once.
"""
props = test.get("properties", [])
# pyre-fixme[16]: `object` has no attribute `__iter__`.
for p in props:
if p.get("name", None) == propname:
return p.get("value", defval)
return defval
# pyre-fixme[53]: Captured variable `cmake` is not annotated.
# pyre-fixme[53]: Captured variable `env` is not annotated.
def list_tests() -> list[dict[str, object]]:
output = subprocess.check_output(
[require_command(ctest, "ctest"), "--show-only=json-v1"],
env=env,
cwd=self.build_dir,
)
try:
data = json.loads(output.decode("utf-8"))
except ValueError as exc:
raise Exception(
"Failed to decode cmake test info using %s: %s. Output was: %r"
% (ctest, str(exc), output)
)
tests = []
machine_suffix = self.build_opts.host_type.as_tuple_string()
for test in data["tests"]:
working_dir = get_property(test, "WORKING_DIRECTORY")
labels = []
machine_suffix = self.build_opts.host_type.as_tuple_string()
labels.append("tpx-fb-test-type=3")
labels.append("tpx_test_config::buildsystem=getdeps")
labels.append("tpx_test_config::platform={}".format(machine_suffix))
if get_property(test, "DISABLED"):
labels.append("disabled")
command = test["command"]
if working_dir:
command = [
require_command(cmake, "cmake"),
"-E",
"chdir",
working_dir,
] + command
tests.append(
{
"type": "custom",
"target": "%s-%s-getdeps-%s"
% (self.manifest.name, test["name"], machine_suffix),
"command": command,
"labels": labels,
"env": {},
"required_paths": [],
"contacts": [],
"cwd": os.getcwd(),
}
)
return tests
discover_like_continuous = False
if schedule_type == "continuous" or (
schedule_type == "base_retry" and is_public_commit(self.build_opts)
):
discover_like_continuous = True
if discover_like_continuous or schedule_type == "testwarden":
# for continuous and testwarden runs, disabling retry can give up
# better signals for flaky tests.
retry = 0
tpx = None
try:
from .facebook.testinfra import start_run
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got
# `Env`.
tpx = path_search(env, "tpx")
except ImportError:
# internal testinfra not available
pass
if tpx and not no_testpilot:
import os
buck_test_info = list_tests()
buck_test_info_name = os.path.join(self.build_dir, ".buck-test-info.json")
with open(buck_test_info_name, "w") as f:
json.dump(buck_test_info, f)
env.set("http_proxy", "")
env.set("https_proxy", "")
runs = []
with start_run(env["FBSOURCE_HASH"]) as run_id:
testpilot_args = [
tpx,
"--force-local-execution",
"--buck-test-info",
buck_test_info_name,
"--retry=%d" % retry,
"-j=%s" % str(self.num_jobs),
"--print-long-results",
]
if owner:
testpilot_args += ["--contacts", owner]
if env:
testpilot_args.append("--env")
testpilot_args.extend(f"{key}={val}" for key, val in env.items())
if run_id is not None:
testpilot_args += ["--run-id", run_id]
if timeout is not None:
testpilot_args += ["--timeout", str(timeout)]
if test_filter:
testpilot_args += ["--", test_filter]
if schedule_type == "diff":
runs.append(["--collection", "oss-diff", "--purpose", "diff"])
elif discover_like_continuous:
runs.append(
[
"--tag-new-tests",
"--collection",
"oss-continuous",
"--purpose",
"continuous",
]
)
elif schedule_type == "testwarden":
# One run to assess new tests
runs.append(
[
"--tag-new-tests",
"--collection",
"oss-new-test-stress",
"--stress-runs",
"10",
"--purpose",
"stress-run-new-test",
]
)
# And another for existing tests
runs.append(
[
"--tag-new-tests",
"--collection",
"oss-existing-test-stress",
"--stress-runs",
"10",
"--purpose",
"stress-run",
]
)
else:
runs.append([])
for run in runs:
# FIXME: What is this trying to accomplish? Should it fail on first or >=1 errors?
self._run_cmd(
testpilot_args + run,
cwd=self.build_opts.fbcode_builder_dir,
env=env,
use_cmd_prefix=use_cmd_prefix,
)
else:
args = [
require_command(ctest, "ctest"),
"--output-on-failure",
"-j",
str(self.num_jobs),
]
if test_filter:
args += ["-R", test_filter]
if test_exclude:
args += ["--exclude-regex", test_exclude]
if timeout is not None:
args += ["--timeout", str(timeout)]
count: int = 0
retcode: int | None = -1
while count <= retry:
# FIXME: What is this trying to accomplish? Should it fail on first or >=1 errors?
retcode = self._check_cmd(
args, env=env, use_cmd_prefix=use_cmd_prefix, allow_fail=True
)
if retcode == 0:
break
if count == 0:
# Only add this option in the second run.
args += ["--rerun-failed"]
count += 1
if retcode is not None and retcode != 0:
# Allow except clause in getdeps.main to catch and exit gracefully
# This allows non-testpilot runs to fail through the same logic as failed testpilot runs, which may become handy in case if post test processing is needed in the future
raise subprocess.CalledProcessError(retcode, args)
class NinjaBootstrap(BuilderBase):
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
build_dir: str,
src_dir: str,
inst_dir: str,
) -> None:
super(NinjaBootstrap, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
def _build(self, reconfigure: bool) -> None:
self._check_cmd(
[sys.executable, "configure.py", "--bootstrap"], cwd=self.src_dir
)
src_ninja = os.path.join(self.src_dir, "ninja")
dest_ninja = os.path.join(self.inst_dir, "bin/ninja")
bin_dir = os.path.dirname(dest_ninja)
if not os.path.exists(bin_dir):
os.makedirs(bin_dir)
shutil.copyfile(src_ninja, dest_ninja)
shutil.copymode(src_ninja, dest_ninja)
class OpenSSLBuilder(BuilderBase):
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
build_dir: str,
src_dir: str,
inst_dir: str,
) -> None:
super(OpenSSLBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
def _build(self, reconfigure: bool) -> None:
configure = os.path.join(self.src_dir, "Configure")
# prefer to resolve the perl that we installed from
# our manifest on windows, but fall back to the system
# path on eg: darwin
env = self.env.copy()
for m in self.dep_manifests:
bindir = os.path.join(self.loader.get_project_install_dir(m), "bin")
add_path_entry(env, "PATH", bindir, append=False)
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got `Env`.
perl = typing.cast(str, path_search(env, "perl", "perl"))
make_j_args = []
extra_args = []
if self.build_opts.is_windows():
# jom is compatible with nmake, adds the /j argument for parallel build
make = "jom.exe"
make_j_args = ["/j%s" % self.num_jobs]
args = ["VC-WIN64A-masm", "-utf-8"]
# fixes "if multiple CL.EXE write to the same .PDB file, please use /FS"
extra_args = ["/FS"]
elif self.build_opts.is_darwin():
make = "make"
make_j_args = ["-j%s" % self.num_jobs]
args = (
["darwin64-x86_64-cc"]
if not self.build_opts.is_arm()
else ["darwin64-arm64-cc"]
)
elif self.build_opts.is_linux():
make = "make"
make_j_args = ["-j%s" % self.num_jobs]
args = (
["linux-x86_64"] if not self.build_opts.is_arm() else ["linux-aarch64"]
)
else:
raise Exception("don't know how to build openssl for %r" % self.ctx)
self._check_cmd(
[
perl,
configure,
"--prefix=%s" % self.inst_dir,
"--openssldir=%s" % self.inst_dir,
]
+ args
+ [
"enable-static-engine",
"enable-capieng",
"no-makedepend",
"no-unit-test",
"no-tests",
]
+ extra_args
)
# show the config produced
self._check_cmd([perl, "configdata.pm", "--dump"], env=env)
make_build = [make] + make_j_args
self._check_cmd(make_build, env=env)
make_install = [make, "install_sw", "install_ssldirs"]
self._check_cmd(make_install, env=env)
class Boost(BuilderBase):
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
src_dir: str,
build_dir: str,
inst_dir: str,
b2_args: list[str],
) -> None:
children = os.listdir(src_dir)
assert len(children) == 1, "expected a single directory entry: %r" % (children,)
boost_src = children[0]
assert boost_src.startswith("boost")
src_dir = os.path.join(src_dir, children[0])
super(Boost, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
self.b2_args: list[str] = b2_args
def _build(self, reconfigure: bool) -> None:
env = self._compute_env()
linkage: list[str] = ["static"]
if self.build_opts.is_windows() or self.build_opts.shared_libs:
linkage.append("shared")
args = []
if self.build_opts.is_darwin():
clang = subprocess.check_output(["xcrun", "--find", "clang"])
user_config = os.path.join(self.build_dir, "project-config.jam")
with open(user_config, "w") as jamfile:
jamfile.write("using clang : : %s ;\n" % clang.decode().strip())
args.append("--user-config=%s" % user_config)
for link in linkage:
bootstrap_args = self.manifest.get_section_as_args(
"bootstrap.args", self.ctx
)
if self.build_opts.is_windows():
bootstrap = os.path.join(self.src_dir, "bootstrap.bat")
self._check_cmd([bootstrap] + bootstrap_args, cwd=self.src_dir, env=env)
args += ["address-model=64"]
else:
bootstrap = os.path.join(self.src_dir, "bootstrap.sh")
self._check_cmd(
[bootstrap, "--prefix=%s" % self.inst_dir] + bootstrap_args,
cwd=self.src_dir,
env=env,
)
b2 = os.path.join(self.src_dir, "b2")
self._check_cmd(
[
b2,
"-j%s" % self.num_jobs,
"--prefix=%s" % self.inst_dir,
"--builddir=%s" % self.build_dir,
]
+ args
+ self.b2_args
+ [
"link=%s" % link,
"runtime-link=shared",
"variant=release",
"threading=multi",
"debug-symbols=on",
"visibility=global",
"-d2",
"install",
],
cwd=self.src_dir,
env=env,
)
class NopBuilder(BuilderBase):
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
src_dir: str,
inst_dir: str,
) -> None:
super(NopBuilder, self).__init__(
loader, dep_manifests, build_opts, ctx, manifest, src_dir, None, inst_dir
)
def build(self, reconfigure: bool) -> None:
print("Installing %s -> %s" % (self.src_dir, self.inst_dir))
parent = os.path.dirname(self.inst_dir)
if not os.path.exists(parent):
os.makedirs(parent)
install_files = self.manifest.get_section_as_ordered_pairs(
"install.files", self.ctx
)
if install_files:
for src_name, dest_name in self.manifest.get_section_as_ordered_pairs(
"install.files", self.ctx
):
# pyre-fixme[6]: For 2nd argument expected `Union[PathLike[str],
# str]` but got `Optional[str]`.
full_dest = os.path.join(self.inst_dir, dest_name)
full_src = os.path.join(self.src_dir, src_name)
dest_parent = os.path.dirname(full_dest)
if not os.path.exists(dest_parent):
os.makedirs(dest_parent)
if os.path.isdir(full_src):
if not os.path.exists(full_dest):
simple_copytree(full_src, full_dest)
else:
shutil.copyfile(full_src, full_dest)
shutil.copymode(full_src, full_dest)
# This is a bit gross, but the mac ninja.zip doesn't
# give ninja execute permissions, so force them on
# for things that look like they live in a bin dir
# pyre-fixme[6]: For 1st argument expected `PathLike[AnyStr]`
# but got `Optional[str]`.
if os.path.dirname(dest_name) == "bin":
st = os.lstat(full_dest)
os.chmod(full_dest, st.st_mode | stat.S_IXUSR)
else:
if not os.path.exists(self.inst_dir):
simple_copytree(self.src_dir, self.inst_dir)
class SetupPyBuilder(BuilderBase):
def _build(self, reconfigure: bool) -> None:
env = self._compute_env()
setup_env = self.manifest.get_section_as_dict("setup-py.env", self.ctx)
for key, value in setup_env.items():
# pyre-fixme[6]: For 2nd argument expected `str` but got `Optional[str]`.
env[key] = value
setup_py_path = os.path.join(self.src_dir, "setup.py")
if not os.path.exists(setup_py_path):
raise RuntimeError(f"setup.py script not found at {setup_py_path}")
self._check_cmd(
# pyre-fixme[6]: For 1st argument expected `List[str]` but got
# `List[Union[str, None, str]]`.
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got
# `Env`.
[path_search(env, "python3"), setup_py_path, "install"],
cwd=self.src_dir,
env=env,
)
# Create the installation directory if it doesn't exist
os.makedirs(self.inst_dir, exist_ok=True)
# Mark the project as built
with open(os.path.join(self.inst_dir, ".built-by-getdeps"), "w") as f:
f.write("built")
def run_tests(
self,
schedule_type: str,
owner: str | None,
test_filter: str | None,
test_exclude: str | None,
retry: int,
no_testpilot: bool,
timeout: int | None = None,
) -> None:
# setup.py actually no longer has a standard command for running tests.
# Instead we let manifest files specify an arbitrary Python file to run
# as a test.
# Get the test command from the manifest
python_script = self.manifest.get(
"setup-py.test", "python_script", ctx=self.ctx
)
if not python_script:
print(f"No test script specified for {self.manifest.name}")
return
# Run the command
env = self._compute_env()
self._check_cmd(["python3", python_script], cwd=self.src_dir, env=env)
class SqliteBuilder(BuilderBase):
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
src_dir: str,
build_dir: str,
inst_dir: str,
) -> None:
super(SqliteBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
def _build(self, reconfigure: bool) -> None:
for f in ["sqlite3.c", "sqlite3.h", "sqlite3ext.h"]:
src = os.path.join(self.src_dir, f)
dest = os.path.join(self.build_dir, f)
copy_if_different(src, dest)
cmake_lists = """
cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
project(sqlite3 C)
add_library(sqlite3 STATIC sqlite3.c)
# These options are taken from the defaults in Makefile.msc in
# the sqlite distribution
target_compile_definitions(sqlite3 PRIVATE
-DSQLITE_ENABLE_COLUMN_METADATA=1
-DSQLITE_ENABLE_FTS3=1
-DSQLITE_ENABLE_RTREE=1
-DSQLITE_ENABLE_GEOPOLY=1
-DSQLITE_ENABLE_JSON1=1
-DSQLITE_ENABLE_STMTVTAB=1
-DSQLITE_ENABLE_DBPAGE_VTAB=1
-DSQLITE_ENABLE_DBSTAT_VTAB=1
-DSQLITE_INTROSPECTION_PRAGMAS=1
-DSQLITE_ENABLE_DESERIALIZE=1
)
install(TARGETS sqlite3)
install(FILES sqlite3.h sqlite3ext.h DESTINATION include)
"""
with open(os.path.join(self.build_dir, "CMakeLists.txt"), "w") as f:
f.write(cmake_lists)
defines = {
"CMAKE_INSTALL_PREFIX": self.inst_dir,
"BUILD_SHARED_LIBS": "ON" if self.build_opts.shared_libs else "OFF",
"CMAKE_BUILD_TYPE": "RelWithDebInfo",
}
define_args = ["-D%s=%s" % (k, v) for (k, v) in defines.items()]
define_args += ["-G", "Ninja"]
env = self._compute_env()
# Resolve the cmake that we installed
# pyre-fixme[6]: For 1st argument expected `Mapping[str, str]` but got `Env`.
cmake = path_search(env, "cmake")
# pyre-fixme[6]: For 1st argument expected `List[str]` but got
# `List[Optional[str]]`.
self._check_cmd([cmake, self.build_dir] + define_args, env=env)
self._check_cmd(
# pyre-fixme[6]: For 1st argument expected `List[str]` but got
# `List[Union[str, str, str, str, str, None, str]]`.
[
cmake,
"--build",
self.build_dir,
"--target",
"install",
"--config",
self.build_opts.build_type,
"-j",
str(self.num_jobs),
],
env=env,
)
================================================
FILE: build/fbcode_builder/getdeps/buildopts.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import argparse
import errno
import glob
import ntpath
import os
import subprocess
import sys
import tempfile
import typing
from collections.abc import Mapping
from .copytree import containing_repo_type
from .envfuncs import add_flag, add_path_entry, Env
from .fetcher import get_fbsource_repo_data, homebrew_package_prefix
from .manifest import ContextGenerator
from .platform import get_available_ram, HostType, is_windows
if typing.TYPE_CHECKING:
from .load import ManifestLoader
from .manifest import ManifestContext, ManifestParser
GITBASH_TMP: str = "c:\\tools\\fb.gitbash\\tmp"
def detect_project(path: str) -> tuple[str | None, str | None]:
repo_type, repo_root = containing_repo_type(path)
if repo_type is None:
return None, None
# Look for a .projectid file. If it exists, read the project name from it.
# pyre-fixme[6]: For 1st argument expected `LiteralString` but got `Optional[str]`.
project_id_path = os.path.join(repo_root, ".projectid")
try:
with open(project_id_path, "r") as f:
project_name = f.read().strip()
return repo_root, project_name
except EnvironmentError as ex:
if ex.errno != errno.ENOENT:
raise
return repo_root, None
class BuildOptions:
def __init__(
self,
fbcode_builder_dir: str,
scratch_dir: str,
host_type: HostType,
install_dir: str | None = None,
num_jobs: int = 0,
use_shipit: bool = False,
vcvars_path: str | None = None,
allow_system_packages: bool = False,
lfs_path: str | None = None,
shared_libs: bool = False,
facebook_internal: bool | None = None,
free_up_disk: bool = False,
build_type: str | None = None,
) -> None:
"""fbcode_builder_dir - the path to either the in-fbsource fbcode_builder dir,
or for shipit-transformed repos, the build dir that
has been mapped into that dir.
scratch_dir - a place where we can store repos and build bits.
This path should be stable across runs and ideally
should not be in the repo of the project being built,
but that is ultimately where we generally fall back
for builds outside of FB
install_dir - where the project will ultimately be installed
num_jobs - the level of concurrency to use while building
use_shipit - use real shipit instead of the simple shipit transformer
vcvars_path - Path to external VS toolchain's vsvarsall.bat
shared_libs - whether to build shared libraries
free_up_disk - take extra actions to save runner disk space
build_type - CMAKE_BUILD_TYPE, used by cmake and cargo builders
"""
if not install_dir:
install_dir = os.path.join(scratch_dir, "installed")
self.project_hashes: str | None = None
for p in ["../deps/github_hashes", "../project_hashes"]:
hashes = os.path.join(fbcode_builder_dir, p)
if os.path.exists(hashes):
self.project_hashes = hashes
break
# Detect what repository and project we are being run from.
# pyre-fixme[4]: Attribute must be annotated.
self.repo_root, self.repo_project = detect_project(os.getcwd())
# If we are running from an fbsource repository, set self.fbsource_dir
# to allow the ShipIt-based fetchers to use it.
if self.repo_project == "fbsource":
self.fbsource_dir: str | None = self.repo_root
else:
self.fbsource_dir = None
if facebook_internal is None:
if self.fbsource_dir:
facebook_internal = True
else:
facebook_internal = False
self.facebook_internal: bool = facebook_internal
self.specified_num_jobs: int = num_jobs
self.scratch_dir: str = scratch_dir
self.install_dir: str = install_dir
self.fbcode_builder_dir: str = fbcode_builder_dir
self.host_type: HostType = host_type
self.use_shipit: bool = use_shipit
self.allow_system_packages: bool = allow_system_packages
self.lfs_path: str | None = lfs_path
self.shared_libs: bool = shared_libs
self.free_up_disk: bool = free_up_disk
self.build_type: str | None = build_type
lib_path: str | None = None
if self.is_darwin():
lib_path = "DYLD_LIBRARY_PATH"
elif self.is_linux():
lib_path = "LD_LIBRARY_PATH"
elif self.is_windows():
lib_path = "PATH"
else:
lib_path = None
self.lib_path: str | None = lib_path
if vcvars_path is None and is_windows():
try:
# Allow a site-specific vcvarsall path.
from .facebook.vcvarsall import build_default_vcvarsall
except ImportError:
vcvarsall: list[str] = []
else:
vcvarsall = (
build_default_vcvarsall(self.fbsource_dir)
if self.fbsource_dir is not None
else []
)
# On Windows, the compiler is not available in the PATH by
# default so we need to run the vcvarsall script to populate the
# environment. We use a glob to find some version of this script
# as deployed with Visual Studio.
if len(vcvarsall) == 0:
# check the 64 bit installs
for year in ["2022"]:
vcvarsall += glob.glob(
os.path.join(
os.environ.get("ProgramFiles", "C:\\Program Files"),
"Microsoft Visual Studio",
year,
"*",
"VC",
"Auxiliary",
"Build",
"vcvarsall.bat",
)
)
# then the 32 bit ones
for year in ["2022", "2019", "2017"]:
vcvarsall += glob.glob(
os.path.join(
os.environ["ProgramFiles(x86)"],
"Microsoft Visual Studio",
year,
"*",
"VC",
"Auxiliary",
"Build",
"vcvarsall.bat",
)
)
if len(vcvarsall) == 0:
raise Exception(
"Could not find vcvarsall.bat. Please install Visual Studio."
)
vcvars_path = vcvarsall[0]
print(f"Using vcvarsall.bat from {vcvars_path}", file=sys.stderr)
self.vcvars_path: str | None = vcvars_path
@property
def manifests_dir(self) -> str:
return os.path.join(self.fbcode_builder_dir, "manifests")
def is_darwin(self) -> bool:
return self.host_type.is_darwin()
def is_windows(self) -> bool:
return self.host_type.is_windows()
def is_arm(self) -> bool:
return self.host_type.is_arm()
def get_vcvars_path(self) -> str | None:
return self.vcvars_path
def is_linux(self) -> bool:
return self.host_type.is_linux()
def is_freebsd(self) -> bool:
return self.host_type.is_freebsd()
def get_num_jobs(self, job_weight: int) -> int:
"""Given an estimated job_weight in MiB, compute a reasonable concurrency limit."""
if self.specified_num_jobs:
return self.specified_num_jobs
available_ram = get_available_ram()
import multiprocessing
return max(1, min(multiprocessing.cpu_count(), available_ram // job_weight))
def get_context_generator(
self, host_tuple: str | HostType | None = None
) -> ContextGenerator:
"""Create a manifest ContextGenerator for the specified target platform."""
if host_tuple is None:
host_type = self.host_type
elif isinstance(host_tuple, HostType):
host_type = host_tuple
else:
host_type = HostType.from_tuple_string(host_tuple)
return ContextGenerator(
{
"os": host_type.ostype,
"distro": host_type.distro,
"distro_vers": host_type.distrovers,
"fb": "on" if self.facebook_internal else "off",
"fbsource": "on" if self.fbsource_dir else "off",
"test": "off",
"shared_libs": "on" if self.shared_libs else "off",
}
)
def compute_env_for_install_dirs(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
ctx: ManifestContext,
env: Env | None = None,
manifest: ManifestParser | None = None,
) -> Env: # noqa: C901
if env is not None:
env = env.copy()
else:
env = Env()
env["GETDEPS_BUILD_DIR"] = os.path.join(self.scratch_dir, "build")
env["GETDEPS_INSTALL_DIR"] = self.install_dir
# Python setuptools attempts to discover a local MSVC for
# building Python extensions. On Windows, getdeps already
# supports invoking a vcvarsall prior to compilation.
#
# Tell setuptools to bypass its own search. This fixes a bug
# where setuptools would fail when run from CMake on GitHub
# Actions with the inscrutable message 'error: Microsoft
# Visual C++ 14.0 is required. Get it with "Build Tools for
# Visual Studio"'. I suspect the actual error is that the
# environment or PATH is overflowing.
#
# For extra credit, someone could patch setuptools to
# propagate the actual error message from vcvarsall, because
# often it does not mean Visual C++ is not available.
#
# Related discussions:
# - https://github.com/pypa/setuptools/issues/2028
# - https://github.com/pypa/setuptools/issues/2307
# - https://developercommunity.visualstudio.com/t/error-microsoft-visual-c-140-is-required/409173
# - https://github.com/OpenMS/OpenMS/pull/4779
# - https://github.com/actions/virtual-environments/issues/1484
if self.is_windows() and self.get_vcvars_path():
env["DISTUTILS_USE_SDK"] = "1"
# On macOS we need to set `SDKROOT` when we use clang for system
# header files.
if self.is_darwin() and "SDKROOT" not in env:
sdkroot = subprocess.check_output(["xcrun", "--show-sdk-path"])
env["SDKROOT"] = sdkroot.decode().strip()
if (
self.is_darwin()
and self.allow_system_packages
and self.host_type.get_package_manager() == "homebrew"
and manifest
and manifest.resolved_system_packages
):
# Homebrew packages may not be on the default PATHs
brew_packages = manifest.resolved_system_packages.get("homebrew", [])
for p in brew_packages:
found = self.add_homebrew_package_to_env(p, env)
# Try extra hard to find openssl, needed with homebrew on macOS
if found and p.startswith("openssl"):
candidate = homebrew_package_prefix("openssl@1.1")
# pyre-fixme[6]: For 1st argument expected
# `Union[PathLike[bytes], PathLike[str], bytes, int, str]` but got
# `Optional[str]`.
if os.path.exists(candidate):
# pyre-fixme[6]: For 2nd argument expected `str` but got
# `Optional[str]`.
os.environ["OPENSSL_ROOT_DIR"] = candidate
env["OPENSSL_ROOT_DIR"] = os.environ["OPENSSL_ROOT_DIR"]
if self.fbsource_dir:
env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join(
self.fbsource_dir, "xplat/third-party/yarn/offline-mirror"
)
yarn_exe = "yarn.bat" if self.is_windows() else "yarn"
env["YARN_PATH"] = os.path.join(
# pyre-fixme[6]: For 1st argument expected `LiteralString` but got
# `Optional[str]`.
self.fbsource_dir,
"xplat/third-party/yarn/",
yarn_exe,
)
node_exe = "node-win-x64.exe" if self.is_windows() else "node"
env["NODE_BIN"] = os.path.join(
# pyre-fixme[6]: For 1st argument expected `LiteralString` but got
# `Optional[str]`.
self.fbsource_dir,
"xplat/third-party/node/bin/",
node_exe,
)
env["RUST_VENDORED_CRATES_DIR"] = os.path.join(
# pyre-fixme[6]: For 1st argument expected `LiteralString` but got
# `Optional[str]`.
self.fbsource_dir,
"third-party/rust/vendor",
)
hash_data = get_fbsource_repo_data(self)
env["FBSOURCE_HASH"] = hash_data.hash
env["FBSOURCE_DATE"] = hash_data.date
# reverse as we are prepending to the PATHs
for m in reversed(dep_manifests):
is_direct_dep = (
manifest is not None and m.name in manifest.get_dependencies(ctx)
)
d = loader.get_project_install_dir(m)
if os.path.exists(d):
self.add_prefix_to_env(
d,
env,
append=False,
is_direct_dep=is_direct_dep,
)
# Linux is always system openssl
system_openssl: bool = self.is_linux()
# For other systems lets see if package is requested
if not system_openssl and manifest and manifest.resolved_system_packages:
for _pkg_type, pkgs in manifest.resolved_system_packages.items():
for p in pkgs:
if p.startswith("openssl") or p.startswith("libssl"):
system_openssl = True
break
# Let openssl know to pick up the system certs if present
if system_openssl or "OPENSSL_DIR" in env:
for system_ssl_cfg in ["/etc/pki/tls", "/etc/ssl"]:
if os.path.isdir(system_ssl_cfg):
cert_dir = system_ssl_cfg + "/certs"
if os.path.isdir(cert_dir):
env["SSL_CERT_DIR"] = cert_dir
cert_file = system_ssl_cfg + "/cert.pem"
if os.path.isfile(cert_file):
env["SSL_CERT_FILE"] = cert_file
return env
def add_homebrew_package_to_env(self, package: str, env: Env) -> bool:
prefix = homebrew_package_prefix(package)
if prefix and os.path.exists(prefix):
return self.add_prefix_to_env(
prefix, env, append=False, add_library_path=True
)
return False
def add_prefix_to_env(
self,
d: str,
env: Env,
append: bool = True,
add_library_path: bool = False,
is_direct_dep: bool = False,
) -> bool: # noqa: C901
bindir: str = os.path.join(d, "bin")
found: bool = False
has_pkgconfig: bool = False
pkgconfig: str = os.path.join(d, "lib", "pkgconfig")
if os.path.exists(pkgconfig):
found = True
has_pkgconfig = True
add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append)
pkgconfig = os.path.join(d, "lib64", "pkgconfig")
if os.path.exists(pkgconfig):
found = True
has_pkgconfig = True
add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig, append=append)
add_path_entry(env, "CMAKE_PREFIX_PATH", d, append=append)
# Tell the thrift compiler about includes it needs to consider
thriftdir: str = os.path.join(d, "include", "thrift-files")
if os.path.exists(thriftdir):
found = True
add_path_entry(env, "THRIFT_INCLUDE_PATH", thriftdir, append=append)
# module detection for python is old fashioned and needs flags
includedir: str = os.path.join(d, "include")
if os.path.exists(includedir):
found = True
ncursesincludedir: str = os.path.join(d, "include", "ncurses")
if os.path.exists(ncursesincludedir):
add_path_entry(env, "C_INCLUDE_PATH", ncursesincludedir, append=append)
add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append)
add_flag(env, "CPPFLAGS", f"-I{ncursesincludedir}", append=append)
elif "/bz2-" in d:
add_flag(env, "CPPFLAGS", f"-I{includedir}", append=append)
# For non-pkgconfig projects Cabal has no way to find the includes or
# libraries, so we provide a set of extra Cabal flags in the env
if not has_pkgconfig and is_direct_dep:
add_flag(
env,
"GETDEPS_CABAL_FLAGS",
f"--extra-include-dirs={includedir}",
append=append,
)
# The thrift compiler's built-in includes are installed directly to the include dir
includethriftdir: str = os.path.join(d, "include", "thrift")
if os.path.exists(includethriftdir):
add_path_entry(env, "THRIFT_INCLUDE_PATH", includedir, append=append)
# Map from FB python manifests to PYTHONPATH
pydir: str = os.path.join(d, "lib", "fb-py-libs")
if os.path.exists(pydir):
found = True
manifest_ext: str = ".manifest"
pymanifestfiles: list[str] = [
f
for f in os.listdir(pydir)
if f.endswith(manifest_ext) and os.path.isfile(os.path.join(pydir, f))
]
for f in pymanifestfiles:
subdir = f[: -len(manifest_ext)]
add_path_entry(
env, "PYTHONPATH", os.path.join(pydir, subdir), append=append
)
# Allow resolving shared objects built earlier (eg: zstd
# doesn't include the full path to the dylib in its linkage
# so we need to give it an assist)
if self.lib_path:
for lib in ["lib", "lib64"]:
libdir: str = os.path.join(d, lib)
if os.path.exists(libdir):
found = True
# pyre-fixme[6]: For 2nd argument expected `str` but got
# `Optional[str]`.
add_path_entry(env, self.lib_path, libdir, append=append)
# module detection for python is old fashioned and needs flags
if "/ncurses-" in d:
add_flag(env, "LDFLAGS", f"-L{libdir}", append=append)
elif "/bz2-" in d:
add_flag(env, "LDFLAGS", f"-L{libdir}", append=append)
if add_library_path:
add_path_entry(env, "LIBRARY_PATH", libdir, append=append)
if not has_pkgconfig and is_direct_dep:
add_flag(
env,
"GETDEPS_CABAL_FLAGS",
f"--extra-lib-dirs={libdir}",
append=append,
)
# Allow resolving binaries (eg: cmake, ninja) and dlls
# built by earlier steps
if os.path.exists(bindir):
found = True
add_path_entry(env, "PATH", bindir, append=append)
# If rustc is present in the `bin` directory, set RUSTC to prevent
# cargo uses the rustc installed in the system.
if self.is_windows():
cargo_path: str = os.path.join(bindir, "cargo.exe")
rustc_path: str = os.path.join(bindir, "rustc.exe")
rustdoc_path: str = os.path.join(bindir, "rustdoc.exe")
else:
cargo_path = os.path.join(bindir, "cargo")
rustc_path = os.path.join(bindir, "rustc")
rustdoc_path = os.path.join(bindir, "rustdoc")
if os.path.isfile(rustc_path):
env["CARGO_BIN"] = cargo_path
env["RUSTC"] = rustc_path
env["RUSTDOC"] = rustdoc_path
openssl_include: str = os.path.join(d, "include", "openssl")
if os.path.isdir(openssl_include) and any(
os.path.isfile(os.path.join(d, "lib", libcrypto))
for libcrypto in ("libcrypto.lib", "libcrypto.so", "libcrypto.a")
):
# This must be the openssl library, let Rust know about it
env["OPENSSL_DIR"] = d
return found
def list_win32_subst_letters() -> dict[str, str]:
output = subprocess.check_output(["subst"]).decode("utf-8")
# The output is a set of lines like: `F:\: => C:\open\some\where`
lines = output.strip().split("\r\n")
mapping: dict[str, str] = {}
for line in lines:
fields = line.split(": => ")
if len(fields) != 2:
continue
letter = fields[0]
path = fields[1]
mapping[letter] = path
return mapping
def find_existing_win32_subst_for_path(
path: str,
subst_mapping: Mapping[str, str],
) -> str | None:
path = ntpath.normcase(ntpath.normpath(path))
for letter, target in subst_mapping.items():
if ntpath.normcase(target) == path:
return letter
return None
def find_unused_drive_letter() -> str | None:
import ctypes
buffer_len = 256
blen = ctypes.c_uint(buffer_len)
rv = ctypes.c_uint()
bufs = ctypes.create_string_buffer(buffer_len)
# pyre-fixme[16]: Module `ctypes` has no attribute `windll`.
rv = ctypes.windll.kernel32.GetLogicalDriveStringsA(blen, bufs)
if rv > buffer_len:
raise Exception("GetLogicalDriveStringsA result too large for buffer")
nul = "\x00".encode("ascii")
used: list[str] = [
drive.decode("ascii")[0] for drive in bufs.raw.strip(nul).split(nul)
]
possible: list[str] = [c for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"]
available: list[str] = sorted(list(set(possible) - set(used)))
if len(available) == 0:
return None
# Prefer to assign later letters rather than earlier letters
return available[-1]
def map_subst_path(path: str) -> str:
"""find a short drive letter mapping for a path"""
for _attempt in range(0, 24):
drive = find_existing_win32_subst_for_path(
path, subst_mapping=list_win32_subst_letters()
)
if drive:
return drive
available = find_unused_drive_letter()
if available is None:
raise Exception(
(
"unable to make shorter subst mapping for %s; "
"no available drive letters"
)
% path
)
# Try to set up a subst mapping; note that we may be racing with
# other processes on the same host, so this may not succeed.
try:
subprocess.check_call(["subst", "%s:" % available, path])
subst = "%s:\\" % available
print("Mapped scratch dir %s -> %s" % (path, subst), file=sys.stderr)
return subst
except Exception:
print("Failed to map %s -> %s" % (available, path), file=sys.stderr)
raise Exception("failed to set up a subst path for %s" % path)
def _check_host_type(args: argparse.Namespace, host_type: HostType | None) -> HostType:
if host_type is None:
host_tuple_string: str | None = getattr(args, "host_type", None)
if host_tuple_string:
host_type = HostType.from_tuple_string(host_tuple_string)
else:
host_type = HostType()
assert isinstance(host_type, HostType)
return host_type
def setup_build_options(
args: argparse.Namespace, host_type: HostType | None = None
) -> BuildOptions:
"""Create a BuildOptions object based on the arguments"""
fbcode_builder_dir: str = os.path.dirname(
os.path.dirname(os.path.abspath(__file__))
)
scratch_dir: str | None = args.scratch_path
if not scratch_dir:
# TODO: `mkscratch` doesn't currently know how best to place things on
# sandcastle, so whip up something reasonable-ish
if "SANDCASTLE" in os.environ:
if "DISK_TEMP" not in os.environ:
raise Exception(
(
"I need DISK_TEMP to be set in the sandcastle environment "
"so that I can store build products somewhere sane"
)
)
disk_temp: str = os.environ["DISK_TEMP"]
if is_windows():
# force use gitbash tmp dir for windows, as its less likely to have a tmp cleaner
# that removes extracted prior dated source files
os.makedirs(GITBASH_TMP, exist_ok=True)
print(
f"Using {GITBASH_TMP} instead of DISK_TEMP {disk_temp} for scratch dir",
file=sys.stderr,
)
disk_temp = GITBASH_TMP
scratch_dir = os.path.join(disk_temp, "fbcode_builder_getdeps")
if not scratch_dir:
try:
scratch_dir = (
subprocess.check_output(
["mkscratch", "path", "--subdir", "fbcode_builder_getdeps"]
)
.strip()
.decode("utf-8")
)
except OSError as exc:
if exc.errno != errno.ENOENT:
# A legit failure; don't fall back, surface the error
raise
# This system doesn't have mkscratch so we fall back to
# something local.
munged: str = fbcode_builder_dir.replace("Z", "zZ")
for s in ["/", "\\", ":"]:
munged = munged.replace(s, "Z")
if is_windows() and os.path.isdir("c:/open"):
temp: str = "c:/open/scratch"
else:
temp = tempfile.gettempdir()
scratch_dir = os.path.join(temp, "fbcode_builder_getdeps-%s" % munged)
if not is_windows() and os.geteuid() == 0:
# Running as root; in the case where someone runs
# sudo getdeps.py install-system-deps
# and then runs as build without privs, we want to avoid creating
# a scratch dir that the second stage cannot write to.
# So we generate a different path if we are root.
scratch_dir += "-root"
if not os.path.exists(scratch_dir):
os.makedirs(scratch_dir)
if is_windows():
subst = map_subst_path(scratch_dir)
scratch_dir = subst
else:
if not os.path.exists(scratch_dir):
os.makedirs(scratch_dir)
# Make sure we normalize the scratch path. This path is used as part of the hash
# computation for detecting if projects have been updated, so we need to always
# use the exact same string to refer to a given directory.
# But! realpath in some combinations of Windows/Python3 versions can expand the
# drive substitutions on Windows, so avoid that!
if not is_windows():
scratch_dir = os.path.realpath(scratch_dir)
# Save these args passed by the user in an env variable, so it
# can be used while hashing this build.
os.environ["GETDEPS_CMAKE_DEFINES"] = getattr(args, "extra_cmake_defines", "") or ""
host_type = _check_host_type(args, host_type)
build_args: dict[str, object] = {
k: v
for (k, v) in vars(args).items()
if k
in {
"num_jobs",
"use_shipit",
"vcvars_path",
"allow_system_packages",
"lfs_path",
"shared_libs",
"free_up_disk",
"build_type",
}
}
return BuildOptions(
fbcode_builder_dir,
scratch_dir,
host_type,
install_dir=args.install_prefix,
facebook_internal=args.facebook_internal,
# pyre-fixme[6]: For 6th argument expected `Optional[str]` but got `object`.
# pyre-fixme[6]: For 6th argument expected `bool` but got `object`.
# pyre-fixme[6]: For 6th argument expected `int` but got `object`.
**build_args,
)
================================================
FILE: build/fbcode_builder/getdeps/cache.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
class ArtifactCache:
"""The ArtifactCache is a small abstraction that allows caching
named things in some external storage mechanism.
The primary use case is for storing the build products on CI
systems to accelerate the build"""
def download_to_file(self, name: str, dest_file_name: str) -> bool:
"""If `name` exists in the cache, download it and place it
in the specified `dest_file_name` location on the filesystem.
If a transient issue was encountered a TransientFailure shall
be raised.
If `name` doesn't exist in the cache `False` shall be returned.
If `dest_file_name` was successfully updated `True` shall be
returned.
All other conditions shall raise an appropriate exception."""
return False
def upload_from_file(self, name: str, source_file_name: str) -> None:
"""Causes `name` to be populated in the cache by uploading
the contents of `source_file_name` to the storage system.
If a transient issue was encountered a TransientFailure shall
be raised.
If the upload failed for some other reason, an appropriate
exception shall be raised."""
pass
def create_cache() -> ArtifactCache | None:
"""This function is monkey patchable to provide an actual
implementation"""
return None
================================================
FILE: build/fbcode_builder/getdeps/cargo.py
================================================
#!/usr/bin/env python3
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import os
import re
import shutil
import sys
import typing
from .builder import BuilderBase
from .copytree import rmtree_more, simple_copytree
if typing.TYPE_CHECKING:
from .buildopts import BuildOptions
from .load import ManifestLoader
from .manifest import ManifestContext, ManifestParser
class CargoBuilder(BuilderBase):
def __init__(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser], # manifests of dependencies
build_opts: BuildOptions,
ctx: ManifestContext,
manifest: ManifestParser,
src_dir: str,
build_dir: str,
inst_dir: str,
build_doc: bool,
workspace_dir: str | None,
manifests_to_build: str | None,
cargo_config_file: str | None,
) -> None:
super(CargoBuilder, self).__init__(
loader,
dep_manifests,
build_opts,
ctx,
manifest,
src_dir,
build_dir,
inst_dir,
)
self.build_doc = build_doc
self.ws_dir: str | None = workspace_dir
# pyre-fixme[8]: Attribute has type `Optional[List[str]]`; used as
# `Union[None, List[str], str]`.
self.manifests_to_build: list[str] | None = (
manifests_to_build and manifests_to_build.split(",")
)
self.loader: ManifestLoader = loader
self.cargo_config_file_subdir: str | None = cargo_config_file
def run_cargo(
self,
install_dirs: list[str],
operation: str,
args: list[str] | None = None,
) -> None:
args = args or []
env = self._compute_env()
# Enable using nightly features with stable compiler
env["RUSTC_BOOTSTRAP"] = "1"
env["LIBZ_SYS_STATIC"] = "1"
cmd = [
"cargo",
operation,
"--workspace",
"-j%s" % self.num_jobs,
] + args
self._check_cmd(cmd, cwd=self.workspace_dir(), env=env)
def build_source_dir(self) -> str:
return os.path.join(self.build_dir, "source")
def workspace_dir(self) -> str:
return os.path.join(self.build_source_dir(), self.ws_dir or "")
def manifest_dir(self, manifest: str) -> str:
return os.path.join(self.build_source_dir(), manifest)
def recreate_dir(self, src: str, dst: str) -> None:
if os.path.isdir(dst):
if os.path.islink(dst):
os.remove(dst)
else:
rmtree_more(dst)
simple_copytree(src, dst)
def recreate_linked_dir(self, src: str, dst: str) -> None:
if os.path.isdir(dst):
if os.path.islink(dst):
os.remove(dst)
elif os.path.isdir(dst):
shutil.rmtree(dst)
os.symlink(src, dst)
def cargo_config_file(self) -> str:
build_source_dir = self.build_dir
if self.cargo_config_file_subdir:
return os.path.join(build_source_dir, self.cargo_config_file_subdir)
else:
return os.path.join(build_source_dir, ".cargo", "config.toml")
def _create_cargo_config(self) -> dict[str, dict[str, str]]:
cargo_config_file = self.cargo_config_file()
cargo_config_dir = os.path.dirname(cargo_config_file)
if not os.path.isdir(cargo_config_dir):
os.mkdir(cargo_config_dir)
dep_to_git = self._resolve_dep_to_git()
if os.path.isfile(cargo_config_file):
with open(cargo_config_file, "r") as f:
print(f"Reading {cargo_config_file}", file=sys.stderr)
cargo_content = f.read()
else:
cargo_content = ""
new_content = cargo_content
if "# Generated by getdeps.py" not in cargo_content:
new_content += """\
# Generated by getdeps.py
[build]
target-dir = '''{}'''
[profile.dev]
debug = false
incremental = false
[profile.release]
opt-level = "{}"
""".format(
self.build_dir.replace("\\", "\\\\"),
"z" if self.build_opts.build_type == "MinSizeRel" else "s",
)
# Point to vendored sources from getdeps manifests
for _dep, git_conf in dep_to_git.items():
if "cargo_vendored_sources" in git_conf:
vendored_dir = git_conf["cargo_vendored_sources"].replace("\\", "\\\\")
override = (
f'[source."{git_conf["repo_url"]}"]\ndirectory = "{vendored_dir}"\n'
)
if override not in cargo_content:
new_content += override
if self.build_opts.fbsource_dir:
# Point to vendored crates.io if possible
try:
from .facebook.rust import vendored_crates
new_content = vendored_crates(
self.build_opts.fbsource_dir, new_content
)
except ImportError:
# This FB internal module isn't shippped to github,
# so just rely on cargo downloading crates on it's own
pass
if new_content != cargo_content:
with open(cargo_config_file, "w") as f:
print(
f"Writing cargo config for {self.manifest.name} to {cargo_config_file}",
file=sys.stderr,
)
f.write(new_content)
return dep_to_git
def _prepare(self, reconfigure: bool) -> None:
build_source_dir = self.build_source_dir()
self.recreate_dir(self.src_dir, build_source_dir)
dep_to_git = self._create_cargo_config()
if self.ws_dir is not None:
self._patchup_workspace(dep_to_git)
def _build(self, reconfigure: bool) -> None:
# _prepare has been run already. Actually do the build
build_source_dir = self.build_source_dir()
build_args = [
"--artifact-dir",
os.path.join(self.inst_dir, "bin"),
"-Zunstable-options",
]
if self.build_opts.build_type != "Debug":
build_args.append("--release")
if self.manifests_to_build is None:
self.run_cargo(
self.install_dirs,
"build",
build_args,
)
else:
# pyre-fixme[16]: Optional type has no attribute `__iter__`.
for manifest in self.manifests_to_build:
self.run_cargo(
self.install_dirs,
"build",
build_args
+ [
"--manifest-path",
self.manifest_dir(manifest),
],
)
self.recreate_linked_dir(
build_source_dir, os.path.join(self.inst_dir, "source")
)
def run_tests(
self,
schedule_type: str,
owner: str | None,
test_filter: str | None,
test_exclude: str | None,
retry: int,
no_testpilot: bool,
timeout: int | None = None,
) -> None:
build_args: list[str] = []
if self.build_opts.build_type != "Debug":
build_args.append("--release")
if test_filter:
filter_args = ["--", test_filter]
else:
filter_args = []
if self.manifests_to_build is None:
self.run_cargo(self.install_dirs, "test", build_args + filter_args)
if self.build_doc and not filter_args:
self.run_cargo(self.install_dirs, "doc", ["--no-deps"])
else:
# pyre-fixme[16]: Optional type has no attribute `__iter__`.
for manifest in self.manifests_to_build:
margs = ["--manifest-path", self.manifest_dir(manifest)]
self.run_cargo(
self.install_dirs, "test", build_args + filter_args + margs
)
if self.build_doc and not filter_args:
self.run_cargo(self.install_dirs, "doc", ["--no-deps"] + margs)
def _patchup_workspace(self, dep_to_git: dict[str, dict[str, str]]) -> None:
"""
This method makes some assumptions about the state of the project and
its cargo dependendies:
1. Crates from cargo dependencies can be extracted from Cargo.toml files
using _extract_crates function. It is using a heuristic so check its
code to understand how it is done.
2. The extracted cargo dependencies crates can be found in the
dependency's install dir using _resolve_crate_to_path function
which again is using a heuristic.
Notice that many things might go wrong here. E.g. if someone depends
on another getdeps crate by writing in their Cargo.toml file:
my-rename-of-crate = { package = "crate", git = "..." }
they can count themselves lucky because the code will raise an
Exception. There might be more cases where the code will silently pass
producing bad results.
"""
workspace_dir = self.workspace_dir()
git_url_to_crates_and_paths = self._resolve_config(dep_to_git)
if git_url_to_crates_and_paths:
patch_cargo = os.path.join(workspace_dir, "Cargo.toml")
if os.path.isfile(patch_cargo):
with open(patch_cargo, "r") as f:
manifest_content = f.read()
else:
manifest_content = ""
new_content = manifest_content
if "[package]" not in manifest_content:
# A fake manifest has to be crated to change the virtual
# manifest into a non-virtual. The virtual manifests are limited
# in many ways and the inability to define patches on them is
# one. Check https://github.com/rust-lang/cargo/issues/4934 to
# see if it is resolved.
null_file = "/dev/null"
if self.build_opts.is_windows():
null_file = "nul"
new_content += f"""
[package]
name = "fake_manifest_of_{self.manifest.name}"
version = "0.0.0"
[lib]
path = "{null_file}"
"""
config: list[str] = []
for git_url, crates_to_patch_path in git_url_to_crates_and_paths.items():
crates_patches = [
'{} = {{ path = "{}" }}'.format(
crate,
crates_to_patch_path[crate].replace("\\", "\\\\"),
)
for crate in sorted(crates_to_patch_path.keys())
]
patch_key = f'[patch."{git_url}"]'
if patch_key not in manifest_content:
config.append(f"\n{patch_key}\n" + "\n".join(crates_patches))
new_content += "\n".join(config)
if new_content != manifest_content:
with open(patch_cargo, "w") as f:
print(
f"writing patch to {patch_cargo}",
file=sys.stderr,
)
f.write(new_content)
def _resolve_config(
self, dep_to_git: dict[str, dict[str, str]]
) -> dict[str, dict[str, str]]:
"""
Returns a configuration to be put inside root Cargo.toml file which
patches the dependencies git code with local getdeps versions.
See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section
"""
dep_to_crates = self._resolve_dep_to_crates(self.build_source_dir(), dep_to_git)
git_url_to_crates_and_paths: dict[str, dict[str, str]] = {}
for dep_name in sorted(dep_to_git.keys()):
git_conf = dep_to_git[dep_name]
req_crates = sorted(dep_to_crates.get(dep_name, []))
if not req_crates:
continue # nothing to patch, move along
git_url = git_conf.get("repo_url", None)
crate_source_map = git_conf["crate_source_map"]
if git_url and crate_source_map:
crates_to_patch_path = git_url_to_crates_and_paths.get(git_url, {})
for c in req_crates:
if c in crate_source_map and c not in crates_to_patch_path:
# pyre-fixme[6]: For 1st argument expected `Union[slice[Any,
# Any, Any], SupportsIndex]` but got `str`.
crates_to_patch_path[c] = crate_source_map[c]
print(
f"{self.manifest.name}: Patching crate {c} via virtual manifest in {self.workspace_dir()}",
file=sys.stderr,
)
if crates_to_patch_path:
git_url_to_crates_and_paths[git_url] = crates_to_patch_path
return git_url_to_crates_and_paths
def _resolve_dep_to_git(self) -> dict[str, dict[str, str]]:
"""
For each direct dependency of the currently build manifest check if it
is also cargo-builded and if yes then extract it's git configs and
install dir
"""
dependencies = self.manifest.get_dependencies(self.ctx)
if not dependencies:
return {}
dep_to_git: dict[str, dict[str, str]] = {}
for dep in dependencies:
dep_manifest = self.loader.load_manifest(dep)
dep_builder = dep_manifest.get("build", "builder", ctx=self.ctx)
dep_cargo_conf = dep_manifest.get_section_as_dict("cargo", self.ctx)
dep_crate_map = dep_manifest.get_section_as_dict("crate.pathmap", self.ctx)
if (
not (dep_crate_map or dep_cargo_conf)
and dep_builder not in ["cargo"]
or dep == "rust"
):
# This dependency has no cargo rust content so ignore it.
# The "rust" dependency is an exception since it contains the
# toolchain.
continue
git_conf = dep_manifest.get_section_as_dict("git", self.ctx)
if dep != "rust" and "repo_url" not in git_conf:
raise Exception(
f"{dep}: A cargo dependency requires git.repo_url to be defined."
)
if dep_builder == "cargo":
dep_source_dir = self.loader.get_project_install_dir(dep_manifest)
dep_source_dir = os.path.join(dep_source_dir, "source")
else:
fetcher = self.loader.create_fetcher(dep_manifest)
dep_source_dir = fetcher.get_src_dir()
crate_source_map: dict[str, str] = {}
if dep_crate_map:
for crate, subpath in dep_crate_map.items():
if crate not in crate_source_map:
if self.build_opts.is_windows():
# pyre-fixme[16]: Optional type has no attribute `replace`.
subpath = subpath.replace("/", "\\")
crate_path = os.path.join(dep_source_dir, subpath)
print(
f"{self.manifest.name}: Mapped crate {crate} to dep {dep} dir {crate_path}",
file=sys.stderr,
)
crate_source_map[crate] = crate_path
elif dep_cargo_conf:
# We don't know what crates are defined buy the dep, look for them
search_pattern = re.compile('\\[package\\]\nname = "(.*)"')
for crate_root, _, files in os.walk(dep_source_dir):
if "Cargo.toml" in files:
with open(os.path.join(crate_root, "Cargo.toml"), "r") as f:
content = f.read()
match = search_pattern.search(content)
if match:
crate = match.group(1)
if crate:
print(
f"{self.manifest.name}: Discovered crate {crate} in dep {dep} dir {crate_root}",
file=sys.stderr,
)
crate_source_map[crate] = crate_root
# pyre-fixme[6]: For 2nd argument expected `Optional[str]` but got
# `Dict[str, str]`.
git_conf["crate_source_map"] = crate_source_map
if not dep_crate_map and dep_cargo_conf:
dep_cargo_dir = self.loader.get_project_build_dir(dep_manifest)
dep_cargo_dir = os.path.join(dep_cargo_dir, "source")
dep_ws_dir = dep_cargo_conf.get("workspace_dir", None)
if dep_ws_dir:
dep_cargo_dir = os.path.join(dep_cargo_dir, dep_ws_dir)
git_conf["cargo_vendored_sources"] = dep_cargo_dir
# pyre-fixme[6]: For 2nd argument expected `Dict[str, str]` but got
# `Dict[str, Optional[str]]`.
dep_to_git[dep] = git_conf
return dep_to_git
def _resolve_dep_to_crates(
self,
build_source_dir: str,
dep_to_git: dict[str, dict[str, str]],
) -> dict[str, set[str]]:
"""
This function traverse the build_source_dir in search of Cargo.toml
files, extracts the crate names from them using _extract_crates
function and returns a merged result containing crate names per
dependency name from all Cargo.toml files in the project.
"""
if not dep_to_git:
return {} # no deps, so don't waste time traversing files
dep_to_crates: dict[str, set[str]] = {}
# First populate explicit crate paths from dependencies
for name, git_conf in dep_to_git.items():
# pyre-fixme[16]: `str` has no attribute `keys`.
crates = git_conf["crate_source_map"].keys()
if crates:
dep_to_crates.setdefault(name, set()).update(crates)
# Now find from Cargo.tomls
for root, _, files in os.walk(build_source_dir):
for f in files:
if f == "Cargo.toml":
more_dep_to_crates = CargoBuilder._extract_crates_used(
os.path.join(root, f), dep_to_git
)
for dep_name, crates in more_dep_to_crates.items():
existing_crates = dep_to_crates.get(dep_name, set())
for c in crates:
if c not in existing_crates:
print(
f"Patch {self.manifest.name} uses {dep_name} crate {crates}",
file=sys.stderr,
)
existing_crates.add(c)
# pyre-fixme[61]: `name` is undefined, or not always defined.
dep_to_crates.setdefault(name, set()).update(existing_crates)
return dep_to_crates
@staticmethod
def _extract_crates_used(
cargo_toml_file: str,
dep_to_git: dict[str, dict[str, str]],
) -> dict[str, set[str]]:
"""
This functions reads content of provided cargo toml file and extracts
crate names per each dependency. The extraction is done by a heuristic
so it might be incorrect.
"""
deps_to_crates: dict[str, set[str]] = {}
with open(cargo_toml_file, "r") as f:
for line in f.readlines():
if line.startswith("#") or "git = " not in line:
continue # filter out commented lines and ones without git deps
for dep_name, conf in dep_to_git.items():
# Only redirect deps that point to git URLS
if 'git = "{}"'.format(conf["repo_url"]) in line:
pkg_template = ' package = "'
if pkg_template in line:
crate_name, _, _ = line.partition(pkg_template)[
2
].partition('"')
else:
crate_name, _, _ = line.partition("=")
deps_to_crates.setdefault(dep_name, set()).add(
crate_name.strip()
)
return deps_to_crates
def _resolve_crate_to_path(
self,
crate: str,
crate_source_map: dict[str, str],
) -> str:
"""
Tries to find in source_dir by searching a [package]
keyword followed by name = "".
"""
search_pattern = '[package]\nname = "{}"'.format(crate)
for _crate, crate_source_dir in crate_source_map.items():
for crate_root, _, files in os.walk(crate_source_dir):
if "Cargo.toml" in files:
with open(os.path.join(crate_root, "Cargo.toml"), "r") as f:
content = f.read()
if search_pattern in content:
return crate_root
raise Exception(
f"{self.manifest.name}: Failed to find dep crate {crate} in paths {crate_source_map}"
)
================================================
FILE: build/fbcode_builder/getdeps/copytree.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import os
import shutil
import stat
import subprocess
from collections.abc import Callable
from .platform import is_windows
from .runcmd import run_cmd
PREFETCHED_DIRS: set[str] = set()
def containing_repo_type(path: str) -> tuple[str | None, str | None]:
while True:
if os.path.exists(os.path.join(path, ".git")):
return ("git", path)
if os.path.exists(os.path.join(path, ".hg")):
return ("hg", path)
parent = os.path.dirname(path)
if parent == path:
return None, None
path = parent
def find_eden_root(dirpath: str) -> str | None:
"""If the specified directory is inside an EdenFS checkout, returns
the canonical absolute path to the root of that checkout.
Returns None if the specified directory is not in an EdenFS checkout.
"""
if is_windows():
repo_type, repo_root = containing_repo_type(dirpath)
if repo_root is not None:
if os.path.exists(os.path.join(repo_root, ".eden", "config")):
return repo_root
return None
try:
return os.readlink(os.path.join(dirpath, ".eden", "root"))
except OSError:
return None
def prefetch_dir_if_eden(dirpath: str) -> None:
"""After an amend/rebase, Eden may need to fetch a large number
of trees from the servers. The simplistic single threaded walk
performed by copytree makes this more expensive than is desirable
so we help accelerate things by performing a prefetch on the
source directory"""
global PREFETCHED_DIRS
if dirpath in PREFETCHED_DIRS:
return
root = find_eden_root(dirpath)
if root is None:
return
glob = f"{os.path.relpath(dirpath, root).replace(os.sep, '/')}/**"
print(f"Prefetching {glob}")
subprocess.call(["edenfsctl", "prefetch", "--repo", root, glob, "--background"])
PREFETCHED_DIRS.add(dirpath)
def simple_copytree(src_dir: str, dest_dir: str, symlinks: bool = False) -> str:
"""A simple version of shutil.copytree() that can delegate to native tools if faster"""
if is_windows():
os.makedirs(dest_dir, exist_ok=True)
cmd = [
"robocopy.exe",
src_dir,
dest_dir,
# copy directories, including empty ones
"/E",
# Ignore Extra files in destination
"/XX",
# enable parallel copy
"/MT",
# be quiet
"/NFL",
"/NDL",
"/NJH",
"/NJS",
"/NP",
]
if symlinks:
cmd.append("/SL")
# robocopy exits with code 1 if it copied ok, hence allow_fail
# https://learn.microsoft.com/en-us/troubleshoot/windows-server/backup-and-storage/return-codes-used-robocopy-utility
exit_code = run_cmd(cmd, allow_fail=True)
if exit_code > 1:
raise subprocess.CalledProcessError(exit_code, cmd)
return dest_dir
else:
return shutil.copytree(src_dir, dest_dir, symlinks=symlinks)
def _remove_readonly_and_try_again(
func: Callable[..., object],
path: str,
# pyre-fixme[24]: Generic type `type` expects 1 type parameter, use
# `typing.Type[ ]` to avoid runtime subscripting errors.
exc_info: tuple[type, BaseException, object],
) -> None:
"""
Error handler for shutil.rmtree.
If the error is due to an access error (read only file)
it attempts to add write permission and then retries the operation.
Any other failure propagates.
"""
# exc_info is a tuple (exc_type, exc_value, traceback)
exc_type = exc_info[0]
if exc_type is PermissionError:
os.chmod(path, stat.S_IWRITE)
# Retry the original function (os.remove or os.rmdir)
try:
func(path)
except Exception:
# If it still fails, the original exception from func() will propagate
raise
else:
# If the error is not a PermissionError, re-raise the original exception
raise exc_info[1]
def rmtree_more(path: str) -> None:
"""Wrapper around shutil.rmtree() that makes it remove readonly files as well.
Useful when git on windows decides to make some files readonly on checkout"""
shutil.rmtree(path, onerror=_remove_readonly_and_try_again)
================================================
FILE: build/fbcode_builder/getdeps/dyndeps.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import errno
import glob
import os
import re
import shlex
import shutil
import stat
import subprocess
import sys
import typing
from collections.abc import Generator
from struct import unpack
if typing.TYPE_CHECKING:
from .buildopts import BuildOptions
from .envfuncs import Env
OBJECT_SUBDIRS: tuple[str, ...] = ("bin", "lib", "lib64")
def copyfile(src: str, dest: str) -> None:
shutil.copyfile(src, dest)
shutil.copymode(src, dest)
class DepBase:
def __init__(
self,
buildopts: BuildOptions,
env: Env,
install_dirs: list[str],
strip: bool,
) -> None:
self.buildopts: BuildOptions = buildopts
self.env: Env = env
self.install_dirs: list[str] = install_dirs
self.strip: bool = strip
# Deduplicates dependency processing. Keyed on the library
# destination path.
self.processed_deps: set[str] = set()
self.munged_lib_dir: str = ""
def list_dynamic_deps(self, objfile: str) -> list[str]:
raise RuntimeError("list_dynamic_deps not implemented")
def interesting_dep(self, d: str) -> bool:
return True
# final_install_prefix must be the equivalent path to `destdir` on the
# installed system. For example, if destdir is `/tmp/RANDOM/usr/local' which
# is intended to map to `/usr/local` in the install image, then
# final_install_prefix='/usr/local'.
# If left unspecified, destdir will be used.
def process_deps(
self, destdir: str, final_install_prefix: str | None = None
) -> None:
if self.buildopts.is_windows():
lib_dir = "bin"
else:
lib_dir = "lib"
self.munged_lib_dir = os.path.join(destdir, lib_dir)
final_lib_dir: str = os.path.join(final_install_prefix or destdir, lib_dir)
if not os.path.isdir(self.munged_lib_dir):
os.makedirs(self.munged_lib_dir)
# Look only at the things that got installed in the leaf package,
# which will be the last entry in the install dirs list
inst_dir: str = self.install_dirs[-1]
print("Process deps under %s" % inst_dir, file=sys.stderr)
for dir in OBJECT_SUBDIRS:
src_dir: str = os.path.join(inst_dir, dir)
if not os.path.isdir(src_dir):
continue
dest_dir: str = os.path.join(destdir, dir)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
for objfile in self.list_objs_in_dir(src_dir):
print("Consider %s/%s" % (dir, objfile))
dest_obj: str = os.path.join(dest_dir, objfile)
copyfile(os.path.join(src_dir, objfile), dest_obj)
self.munge_in_place(dest_obj, final_lib_dir)
def find_all_dependencies(self, build_dir: str) -> list[str]:
all_deps: set[str] = set()
for objfile in self.list_objs_in_dir(
build_dir, recurse=True, output_prefix=build_dir
):
for d in self.list_dynamic_deps(objfile):
all_deps.add(d)
interesting_deps: set[str] = {d for d in all_deps if self.interesting_dep(d)}
dep_paths: list[str] = []
for dep in interesting_deps:
dep_path: str | None = self.resolve_loader_path(dep)
if dep_path:
dep_paths.append(dep_path)
return dep_paths
def munge_in_place(self, objfile: str, final_lib_dir: str) -> None:
print("Munging %s" % objfile)
for d in self.list_dynamic_deps(objfile):
if not self.interesting_dep(d):
continue
# Resolve this dep: does it exist in any of our installation
# directories? If so, then it is a candidate for processing
dep: str | None = self.resolve_loader_path(d)
if dep:
dest_dep: str = os.path.join(self.munged_lib_dir, os.path.basename(dep))
print("dep: %s -> %s" % (d, dest_dep))
if dest_dep in self.processed_deps:
# A previous dependency with the same name has already
# been installed at dest_dep, so there is no need to copy
# or munge the dependency again.
# TODO: audit that both source paths have the same inode number
pass
else:
self.processed_deps.add(dest_dep)
copyfile(dep, dest_dep)
self.munge_in_place(dest_dep, final_lib_dir)
self.rewrite_dep(objfile, d, dep, dest_dep, final_lib_dir)
if self.strip:
self.strip_debug_info(objfile)
def rewrite_dep(
self,
objfile: str,
depname: str,
old_dep: str,
new_dep: str,
final_lib_dir: str,
) -> None:
raise RuntimeError("rewrite_dep not implemented")
def resolve_loader_path(self, dep: str) -> str | None:
if os.path.isabs(dep):
return dep
d: str = os.path.basename(dep)
for inst_dir in self.install_dirs:
for libdir in OBJECT_SUBDIRS:
candidate: str = os.path.join(inst_dir, libdir, d)
if os.path.exists(candidate):
return candidate
return None
def list_objs_in_dir(
self, dir: str, recurse: bool = False, output_prefix: str = ""
) -> Generator[str, None, None]:
for entry in os.listdir(dir):
entry_path: str = os.path.join(dir, entry)
st: os.stat_result = os.lstat(entry_path)
if stat.S_ISREG(st.st_mode):
if self.is_objfile(entry_path):
relative_result: str = os.path.join(output_prefix, entry)
yield os.path.normcase(relative_result)
elif recurse and stat.S_ISDIR(st.st_mode):
child_prefix: str = os.path.join(output_prefix, entry)
for result in self.list_objs_in_dir(
entry_path, recurse=recurse, output_prefix=child_prefix
):
yield result
def is_objfile(self, objfile: str) -> bool:
return True
def strip_debug_info(self, objfile: str) -> None:
"""override this to define how to remove debug information
from an object file"""
pass
def check_call_verbose(self, args: list[str]) -> None:
print(" ".join(map(shlex.quote, args)))
subprocess.check_call(args)
class WinDeps(DepBase):
def __init__(
self,
buildopts: BuildOptions,
env: Env,
install_dirs: list[str],
strip: bool,
) -> None:
super(WinDeps, self).__init__(buildopts, env, install_dirs, strip)
self.dumpbin: str = self.find_dumpbin()
def find_dumpbin(self) -> str:
# Looking for dumpbin in the following hardcoded paths.
# The registry option to find the install dir doesn't work anymore.
globs: list[str] = [
(
"C:/Program Files/"
"Microsoft Visual Studio/"
"*/*/VC/Tools/"
"MSVC/*/bin/Hostx64/x64/dumpbin.exe"
),
(
"C:/Program Files (x86)/"
"Microsoft Visual Studio/"
"*/*/VC/Tools/"
"MSVC/*/bin/Hostx64/x64/dumpbin.exe"
),
(
"C:/Program Files (x86)/"
"Common Files/"
"Microsoft/Visual C++ for Python/*/"
"VC/bin/dumpbin.exe"
),
("c:/Program Files (x86)/Microsoft Visual Studio */VC/bin/dumpbin.exe"),
(
"C:/Program Files/Microsoft Visual Studio/*/Professional/VC/Tools/MSVC/*/bin/HostX64/x64/dumpbin.exe"
),
]
for pattern in globs:
for exe in glob.glob(pattern):
return exe
raise RuntimeError("could not find dumpbin.exe")
# pyre-fixme[14]: `list_dynamic_deps` overrides method defined in `DepBase`
# inconsistently.
def list_dynamic_deps(self, exe: str) -> list[str]:
deps: list[str] = []
print("Resolve deps for %s" % exe)
output: str = subprocess.check_output(
[self.dumpbin, "/nologo", "/dependents", exe]
).decode("utf-8")
lines: list[str] = output.split("\n")
for line in lines:
m: re.Match[str] | None = re.match("\\s+(\\S+.dll)", line, re.IGNORECASE)
if m:
deps.append(m.group(1).lower())
return deps
def rewrite_dep(
self,
objfile: str,
depname: str,
old_dep: str,
new_dep: str,
final_lib_dir: str,
) -> None:
# We can't rewrite on windows, but we will
# place the deps alongside the exe so that
# they end up in the search path
pass
# These are the Windows system dll, which we don't want to copy while
# packaging.
SYSTEM_DLLS: set[str] = set( # noqa: C405
[
"advapi32.dll",
"dbghelp.dll",
"kernel32.dll",
"msvcp140.dll",
"vcruntime140.dll",
"ws2_32.dll",
"ntdll.dll",
"shlwapi.dll",
]
)
def interesting_dep(self, d: str) -> bool:
if "api-ms-win-crt" in d:
return False
if d in self.SYSTEM_DLLS:
return False
return True
def is_objfile(self, objfile: str) -> bool:
if not os.path.isfile(objfile):
return False
if objfile.lower().endswith(".exe"):
return True
return False
def emit_dev_run_script(self, script_path: str, dep_dirs: list[str]) -> None:
"""Emit a script that can be used to run build artifacts directly from the
build directory, without installing them.
The dep_dirs parameter should be a list of paths that need to be added to $PATH.
This can be computed by calling compute_dependency_paths() or
compute_dependency_paths_fast().
This is only necessary on Windows, which does not have RPATH, and instead
requires the $PATH environment variable be updated in order to find the proper
library dependencies.
"""
contents: str = self._get_dev_run_script_contents(dep_dirs)
with open(script_path, "w") as f:
f.write(contents)
def compute_dependency_paths(self, build_dir: str) -> list[str]:
"""Return a list of all directories that need to be added to $PATH to ensure
that library dependencies can be found correctly. This is computed by scanning
binaries to determine exactly the right list of dependencies.
The compute_dependency_paths_fast() is a alternative function that runs faster
but may return additional extraneous paths.
"""
dep_dirs: set[str] = set()
# Find paths by scanning the binaries.
for dep in self.find_all_dependencies(build_dir):
dep_dirs.add(os.path.dirname(dep))
dep_dirs.update(self.read_custom_dep_dirs(build_dir))
return sorted(dep_dirs)
def compute_dependency_paths_fast(self, build_dir: str) -> list[str]:
"""Similar to compute_dependency_paths(), but rather than actually scanning
binaries, just add all library paths from the specified installation
directories. This is much faster than scanning the binaries, but may result in
more paths being returned than actually necessary.
"""
dep_dirs: set[str] = set()
for inst_dir in self.install_dirs:
for subdir in OBJECT_SUBDIRS:
path: str = os.path.join(inst_dir, subdir)
if os.path.exists(path):
dep_dirs.add(path)
dep_dirs.update(self.read_custom_dep_dirs(build_dir))
return sorted(dep_dirs)
def read_custom_dep_dirs(self, build_dir: str) -> set[str]:
# The build system may also have included libraries from other locations that
# we might not be able to find normally in find_all_dependencies().
# To handle this situation we support reading additional library paths
# from a LIBRARY_DEP_DIRS.txt file that may have been generated in the build
# output directory.
dep_dirs: set[str] = set()
try:
explicit_dep_dirs_path: str = os.path.join(
build_dir, "LIBRARY_DEP_DIRS.txt"
)
with open(explicit_dep_dirs_path, "r") as f:
for line in f.read().splitlines():
dep_dirs.add(line)
except OSError as ex:
if ex.errno != errno.ENOENT:
raise
return dep_dirs
def _get_dev_run_script_contents(self, path_dirs: list[str]) -> str:
path_entries: list[str] = ["$env:PATH"] + path_dirs
path_str: str = ";".join(path_entries)
return """\
$orig_env = $env:PATH
$env:PATH = "{path_str}"
try {{
$cmd_args = $args[1..$args.length]
& $args[0] @cmd_args
}} finally {{
$env:PATH = $orig_env
}}
""".format(
path_str=path_str
)
class ElfDeps(DepBase):
def __init__(
self,
buildopts: BuildOptions,
env: Env,
install_dirs: list[str],
strip: bool,
) -> None:
super(ElfDeps, self).__init__(buildopts, env, install_dirs, strip)
# We need patchelf to rewrite deps, so ensure that it is built...
args: list[str] = [sys.executable, sys.argv[0]]
if buildopts.allow_system_packages:
args.append("--allow-system-packages")
subprocess.check_call(args + ["build", "patchelf"])
# ... and that we know where it lives
patchelf_install: str = os.fsdecode(
subprocess.check_output(args + ["show-inst-dir", "patchelf"]).strip()
)
if not patchelf_install:
# its a system package, so we assume it is in the path
patchelf_install = "patchelf"
else:
patchelf_install = os.path.join(patchelf_install, "bin", "patchelf")
self.patchelf: str = patchelf_install
def list_dynamic_deps(self, objfile: str) -> list[str]:
out: str = (
subprocess.check_output(
[self.patchelf, "--print-needed", objfile], env=dict(self.env.items())
)
.decode("utf-8")
.strip()
)
lines: list[str] = out.split("\n")
return lines
def rewrite_dep(
self,
objfile: str,
depname: str,
old_dep: str,
new_dep: str,
final_lib_dir: str,
) -> None:
final_dep: str = os.path.join(
final_lib_dir,
os.path.relpath(new_dep, self.munged_lib_dir),
)
self.check_call_verbose(
[self.patchelf, "--replace-needed", depname, final_dep, objfile]
)
def is_objfile(self, objfile: str) -> bool:
if not os.path.isfile(objfile):
return False
with open(objfile, "rb") as f:
# https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header
magic: bytes = f.read(4)
return magic == b"\x7fELF"
def strip_debug_info(self, objfile: str) -> None:
self.check_call_verbose(["strip", objfile])
# MACH-O magic number
MACH_MAGIC: int = 0xFEEDFACF
class MachDeps(DepBase):
def interesting_dep(self, d: str) -> bool:
if d.startswith("/usr/lib/") or d.startswith("/System/"):
return False
return True
def is_objfile(self, objfile: str) -> bool:
if not os.path.isfile(objfile):
return False
with open(objfile, "rb") as f:
# mach stores the magic number in native endianness,
# so unpack as native here and compare
header: bytes = f.read(4)
if len(header) != 4:
return False
magic: int = unpack("I", header)[0]
return magic == MACH_MAGIC
def list_dynamic_deps(self, objfile: str) -> list[str]:
if not self.interesting_dep(objfile):
return []
out: str = (
subprocess.check_output(
["otool", "-L", objfile], env=dict(self.env.items())
)
.decode("utf-8")
.strip()
)
lines: list[str] = out.split("\n")
deps: list[str] = []
for line in lines:
m: re.Match[str] | None = re.match("\t(\\S+)\\s", line)
if m:
if os.path.basename(m.group(1)) != os.path.basename(objfile):
deps.append(os.path.normcase(m.group(1)))
return deps
def rewrite_dep(
self,
objfile: str,
depname: str,
old_dep: str,
new_dep: str,
final_lib_dir: str,
) -> None:
if objfile.endswith(".dylib"):
# Erase the original location from the id of the shared
# object. It doesn't appear to hurt to retain it, but
# it does look weird, so let's rewrite it to be sure.
self.check_call_verbose(
["install_name_tool", "-id", os.path.basename(objfile), objfile]
)
final_dep: str = os.path.join(
final_lib_dir,
os.path.relpath(new_dep, self.munged_lib_dir),
)
self.check_call_verbose(
["install_name_tool", "-change", depname, final_dep, objfile]
)
def create_dyn_dep_munger(
buildopts: BuildOptions,
env: Env,
install_dirs: list[str],
strip: bool = False,
) -> DepBase | None:
if buildopts.is_linux():
return ElfDeps(buildopts, env, install_dirs, strip)
if buildopts.is_darwin():
return MachDeps(buildopts, env, install_dirs, strip)
if buildopts.is_windows():
return WinDeps(buildopts, env, install_dirs, strip)
if buildopts.is_freebsd():
return ElfDeps(buildopts, env, install_dirs, strip)
return None
================================================
FILE: build/fbcode_builder/getdeps/envfuncs.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import os
import shlex
import sys
from collections.abc import ItemsView, Iterator, KeysView, Mapping, ValuesView
class Env:
def __init__(self, src: Mapping[str, str] | None = None) -> None:
self._dict: dict[str, str] = {}
if src is None:
self.update(os.environ)
else:
self.update(src)
def update(self, src: Mapping[str, str]) -> None:
for k, v in src.items():
self.set(k, v)
def copy(self) -> Env:
return Env(self._dict)
def _key(self, key: str) -> str | None:
# The `str` cast may not appear to be needed, but without it we run
# into issues when passing the environment to subprocess. The main
# issue is that in python2 `os.environ` (which is the initial source
# of data for the environment) uses byte based strings, but this
# project uses `unicode_literals`. `subprocess` will raise an error
# if the environment that it is passed has a mixture of byte and
# unicode strings.
# It is simplest to force everything to be `str` for the sake of
# consistency.
key = str(key)
if sys.platform.startswith("win"):
# Windows env var names are case insensitive but case preserving.
# An implementation of PAR files on windows gets confused if
# the env block contains keys with conflicting case, so make a
# pass over the contents to remove any.
# While this O(n) scan is technically expensive and gross, it
# is practically not a problem because the volume of calls is
# relatively low and the cost of manipulating the env is dwarfed
# by the cost of spawning a process on windows. In addition,
# since the processes that we run are expensive anyway, this
# overhead is not the worst thing to worry about.
for k in list(self._dict.keys()):
if str(k).lower() == key.lower():
return k
elif key in self._dict:
return key
return None
def get(self, key: str, defval: str | None = None) -> str | None:
resolved_key = self._key(key)
if resolved_key is None:
return defval
return self._dict[resolved_key]
def __getitem__(self, key: str) -> str:
val = self.get(key)
if val is None:
raise KeyError(key)
return val
def unset(self, key: str) -> None:
if key is None:
raise KeyError("attempting to unset env[None]")
resolved_key = self._key(key)
if resolved_key:
del self._dict[resolved_key]
def __delitem__(self, key: str) -> None:
self.unset(key)
def __repr__(self) -> str:
return repr(self._dict)
def set(self, key: str, value: str) -> None:
if key is None:
raise KeyError("attempting to assign env[None] = %r" % value)
if value is None:
raise ValueError("attempting to assign env[%s] = None" % key)
# The `str` conversion is important to avoid triggering errors
# with subprocess if we pass in a unicode value; see commentary
# in the `_key` method.
key = str(key)
value = str(value)
# The `unset` call is necessary on windows where the keys are
# case insensitive. Since this dict is case sensitive, simply
# assigning the value to the new key is not sufficient to remove
# the old value. The `unset` call knows how to match keys and
# remove any potential duplicates.
self.unset(key)
self._dict[key] = value
def __setitem__(self, key: str, value: str) -> None:
self.set(key, value)
def __iter__(self) -> Iterator[str]:
return self._dict.__iter__()
def __len__(self) -> int:
return len(self._dict)
def keys(self) -> KeysView[str]:
return self._dict.keys()
def values(self) -> ValuesView[str]:
return self._dict.values()
def items(self) -> ItemsView[str, str]:
return self._dict.items()
def add_path_entry(
env: Env, name: str, item: str, append: bool = True, separator: str = os.pathsep
) -> None:
"""Cause `item` to be added to the path style env var named
`name` held in the `env` dict. `append` specifies whether
the item is added to the end (the default) or should be
prepended if `name` already exists."""
val = env.get(name, "")
if val is not None and len(val) > 0:
val_list = val.split(separator)
else:
val_list = []
if append:
val_list.append(item)
else:
val_list.insert(0, item)
env.set(name, separator.join(val_list))
def add_flag(env: Env, name: str, flag: str, append: bool = True) -> None:
"""Cause `flag` to be added to the CXXFLAGS-style env var named
`name` held in the `env` dict. `append` specifies whether the
flag is added to the end (the default) or should be prepended if
`name` already exists."""
val = shlex.split(env.get(name, "") or "")
if append:
val.append(flag)
else:
val.insert(0, flag)
env.set(name, " ".join(val))
_path_search_cache: dict[object, str | None] = {}
_not_found: object = object()
def tpx_path() -> str:
return "xplat/testinfra/tpx/ctp.tpx"
def path_search(
env: Mapping[str, str], exename: str, defval: str | None = None
) -> str | None:
"""Search for exename in the PATH specified in env.
exename is eg: `ninja` and this function knows to append a .exe
to the end on windows.
Returns the path to the exe if found, or None if either no
PATH is set in env or no executable is found."""
path = env.get("PATH", None)
if path is None:
return defval
# The project hash computation code searches for C++ compilers (g++, clang, etc)
# repeatedly. Cache the result so we don't end up searching for these over and over
# again.
cache_key = (path, exename)
result = _path_search_cache.get(cache_key, _not_found)
if result is _not_found:
result = _perform_path_search(path, exename)
_path_search_cache[cache_key] = result
# pyre-fixme[7]: Expected `Optional[str]` but got `Optional[object]`.
return result
def _perform_path_search(path: str, exename: str) -> str | None:
is_win = sys.platform.startswith("win")
if is_win:
exename = "%s.exe" % exename
for bindir in path.split(os.pathsep):
full_name = os.path.join(bindir, exename)
if os.path.exists(full_name) and os.path.isfile(full_name):
if not is_win and not os.access(full_name, os.X_OK):
continue
return full_name
return None
================================================
FILE: build/fbcode_builder/getdeps/errors.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
class TransientFailure(Exception):
"""Raising this error causes getdeps to return with an error code
that Sandcastle will consider to be a retryable transient
infrastructure error"""
pass
class ManifestNotFound(Exception):
def __init__(self, manifest_name: str) -> None:
super(Exception, self).__init__("Unable to find manifest '%s'" % manifest_name)
================================================
FILE: build/fbcode_builder/getdeps/expr.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import re
import shlex
from collections.abc import Callable
def parse_expr(expr_text: str, valid_variables: set[str]) -> ExprNode:
"""parses the simple criteria expression syntax used in
dependency specifications.
Returns an ExprNode instance that can be evaluated like this:
```
expr = parse_expr("os=windows")
ok = expr.eval({
"os": "windows"
})
```
Whitespace is allowed between tokens. The following terms
are recognized:
KEY = VALUE # Evaluates to True if ctx[KEY] == VALUE
not(EXPR) # Evaluates to True if EXPR evaluates to False
# and vice versa
all(EXPR1, EXPR2, ...) # Evaluates True if all of the supplied
# EXPR's also evaluate True
any(EXPR1, EXPR2, ...) # Evaluates True if any of the supplied
# EXPR's also evaluate True, False if
# none of them evaluated true.
"""
p = Parser(expr_text, valid_variables)
return p.parse()
class ExprNode:
def eval(self, ctx: dict[str, str | None]) -> bool:
return False
class TrueExpr(ExprNode):
def eval(self, ctx: dict[str, str | None]) -> bool:
return True
def __str__(self) -> str:
return "true"
class NotExpr(ExprNode):
def __init__(self, node: ExprNode) -> None:
self._node: ExprNode = node
def eval(self, ctx: dict[str, str | None]) -> bool:
return not self._node.eval(ctx)
def __str__(self) -> str:
return "not(%s)" % self._node
class AllExpr(ExprNode):
def __init__(self, nodes: list[ExprNode]) -> None:
self._nodes: list[ExprNode] = nodes
def eval(self, ctx: dict[str, str | None]) -> bool:
for node in self._nodes:
if not node.eval(ctx):
return False
return True
def __str__(self) -> str:
items: list[str] = []
for node in self._nodes:
items.append(str(node))
return "all(%s)" % ",".join(items)
class AnyExpr(ExprNode):
def __init__(self, nodes: list[ExprNode]) -> None:
self._nodes: list[ExprNode] = nodes
def eval(self, ctx: dict[str, str | None]) -> bool:
for node in self._nodes:
if node.eval(ctx):
return True
return False
def __str__(self) -> str:
items: list[str] = []
for node in self._nodes:
items.append(str(node))
return "any(%s)" % ",".join(items)
class EqualExpr(ExprNode):
def __init__(self, key: str, value: str) -> None:
self._key: str = key
self._value: str = value
def eval(self, ctx: dict[str, str | None]) -> bool:
return ctx.get(self._key) == self._value
def __str__(self) -> str:
return "%s=%s" % (self._key, self._value)
class Parser:
def __init__(self, text: str, valid_variables: set[str]) -> None:
self.text: str = text
self.lex: shlex.shlex = shlex.shlex(text)
self.valid_variables: set[str] = valid_variables
def parse(self) -> ExprNode:
expr = self.top()
garbage = self.lex.get_token()
if garbage != "":
raise Exception(
"Unexpected token %s after EqualExpr in %s" % (garbage, self.text)
)
return expr
def top(self) -> ExprNode:
name = self.ident()
op = self.lex.get_token()
if op == "(":
parsers: dict[str, Callable[[], ExprNode]] = {
"not": self.parse_not,
"any": self.parse_any,
"all": self.parse_all,
}
func = parsers.get(name)
if not func:
raise Exception("invalid term %s in %s" % (name, self.text))
return func()
if op == "=":
if name not in self.valid_variables:
raise Exception("unknown variable %r in expression" % (name,))
# remove shell quote from value so can test things with period in them, e.g "18.04"
token = self.lex.get_token()
if token is None:
raise Exception("unexpected end of expression in %s" % self.text)
unquoted = " ".join(shlex.split(token))
return EqualExpr(name, unquoted)
raise Exception(
"Unexpected token sequence '%s %s' in %s" % (name, op, self.text)
)
def ident(self) -> str:
ident = self.lex.get_token()
if ident is None or not re.match("[a-zA-Z]+", ident):
raise Exception("expected identifier found %s" % ident)
return ident
def parse_not(self) -> NotExpr:
node = self.top()
expr = NotExpr(node)
tok = self.lex.get_token()
if tok != ")":
raise Exception("expected ')' found %s" % tok)
return expr
def parse_any(self) -> AnyExpr:
nodes: list[ExprNode] = []
while True:
nodes.append(self.top())
tok = self.lex.get_token()
if tok == ")":
break
if tok != ",":
raise Exception("expected ',' or ')' but found %s" % tok)
return AnyExpr(nodes)
def parse_all(self) -> AllExpr:
nodes: list[ExprNode] = []
while True:
nodes.append(self.top())
tok = self.lex.get_token()
if tok == ")":
break
if tok != ",":
raise Exception("expected ',' or ')' but found %s" % tok)
return AllExpr(nodes)
================================================
FILE: build/fbcode_builder/getdeps/fetcher.py
================================================
#!/usr/bin/env python3
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
from __future__ import annotations
# pyre-strict
import errno
import hashlib
import os
import random
import re
import shlex
import shutil
import stat
import subprocess
import sys
import tarfile
import time
import zipfile
from abc import ABC, abstractmethod
from collections.abc import Iterator
from datetime import datetime
from typing import NamedTuple, TYPE_CHECKING
from urllib.parse import urlparse
from urllib.request import Request, urlopen
from .copytree import prefetch_dir_if_eden
from .envfuncs import Env
from .errors import TransientFailure
from .platform import HostType, is_windows
from .runcmd import run_cmd
if TYPE_CHECKING:
from .buildopts import BuildOptions
from .manifest import ManifestContext, ManifestParser
def file_name_is_cmake_file(file_name: str) -> bool:
file_name = file_name.lower()
base = os.path.basename(file_name)
return (
base.endswith(".cmake")
or base.endswith(".cmake.in")
or base == "cmakelists.txt"
)
class ChangeStatus:
"""Indicates the nature of changes that happened while updating
the source directory. There are two broad uses:
* When extracting archives for third party software we want to
know that we did something (eg: we either extracted code or
we didn't do anything)
* For 1st party code where we use shipit to transform the code,
we want to know if we changed anything so that we can perform
a build, but we generally want to be a little more nuanced
and be able to distinguish between just changing a source file
and whether we might need to reconfigure the build system.
"""
def __init__(self, all_changed: bool = False) -> None:
"""Construct a ChangeStatus object. The default is to create
a status that indicates no changes, but passing all_changed=True
will create one that indicates that everything changed"""
if all_changed:
self.source_files: int = 1
self.make_files: int = 1
else:
self.source_files: int = 0
self.make_files: int = 0
def record_change(self, file_name: str) -> None:
"""Used by the shipit fetcher to record changes as it updates
files in the destination. If the file name might be one used
in the cmake build system that we use for 1st party code, then
record that as a "make file" change. We could broaden this
to match any file used by various build systems, but it is
only really useful for our internal cmake stuff at this time.
If the file isn't a build file and is under the `fbcode_builder`
dir then we don't class that as an interesting change that we
might need to rebuild, so we ignore it.
Otherwise we record the file as a source file change."""
file_name = file_name.lower()
if file_name_is_cmake_file(file_name):
self.make_files += 1
elif "/fbcode_builder/cmake" in file_name:
self.source_files += 1
elif "/fbcode_builder/" not in file_name:
self.source_files += 1
def sources_changed(self) -> bool:
"""Returns true if any source files were changed during
an update operation. This will typically be used to decide
that the build system to be run on the source dir in an
incremental mode"""
return self.source_files > 0
def build_changed(self) -> bool:
"""Returns true if any build files were changed during
an update operation. This will typically be used to decidfe
that the build system should be reconfigured and re-run
as a full build"""
return self.make_files > 0
class Fetcher(ABC):
"""The Fetcher is responsible for fetching and extracting the
sources for project. The Fetcher instance defines where the
extracted data resides and reports this to the consumer via
its `get_src_dir` method."""
def update(self) -> ChangeStatus:
"""Brings the src dir up to date, ideally minimizing
changes so that a subsequent build doesn't over-build.
Returns a ChangeStatus object that helps the caller to
understand the nature of the changes required during
the update."""
return ChangeStatus()
@abstractmethod
def clean(self) -> None:
"""Reverts any changes that might have been made to
the src dir"""
pass
@abstractmethod
def hash(self) -> str:
"""Returns a hash that identifies the version of the code in the
working copy. For a git repo this is commit hash for the working
copy. For other Fetchers this should relate to the version of
the code in the src dir. The intent is that if a manifest
changes the version/rev of a project that the hash be different.
Importantly, this should be computable without actually fetching
the code, as we want this to factor into a hash used to download
a pre-built version of the code, without having to first download
and extract its sources (eg: boost on windows is pretty painful).
"""
pass
@abstractmethod
def get_src_dir(self) -> str:
"""Returns the source directory that the project was
extracted into"""
pass
class LocalDirFetcher:
"""This class exists to override the normal fetching behavior, and
use an explicit user-specified directory for the project sources.
This fetcher cannot update or track changes. It always reports that the
project has changed, forcing it to always be built."""
def __init__(self, path: str) -> None:
self.path: str = os.path.realpath(path)
def update(self) -> ChangeStatus:
return ChangeStatus(all_changed=True)
def hash(self) -> str:
return "0" * 40
def get_src_dir(self) -> str:
return self.path
def clean(self) -> None:
pass
class SystemPackageFetcher:
def __init__(
self, build_options: BuildOptions, packages: dict[str, list[str]]
) -> None:
self.manager: str | None = build_options.host_type.get_package_manager()
# pyre-fixme[6]: For 1st argument expected `str` but got `Optional[str]`.
self.packages: list[str] | None = packages.get(self.manager)
self.host_type: HostType = build_options.host_type
if self.packages:
self.installed: bool | None = None
else:
self.installed = False
def packages_are_installed(self) -> bool:
if self.installed is not None:
return self.installed
cmd = None
if self.manager == "rpm":
# pyre-fixme[6]: For 1st argument expected
# `pyre_extensions.PyreReadOnly[Iterable[SupportsRichComparisonT]]` but
# got `Optional[List[str]]`.
cmd = ["rpm", "-q"] + sorted(self.packages)
elif self.manager == "deb":
# pyre-fixme[6]: For 1st argument expected
# `pyre_extensions.PyreReadOnly[Iterable[SupportsRichComparisonT]]` but
# got `Optional[List[str]]`.
cmd = ["dpkg", "-s"] + sorted(self.packages)
elif self.manager == "homebrew":
# pyre-fixme[6]: For 1st argument expected
# `pyre_extensions.PyreReadOnly[Iterable[SupportsRichComparisonT]]` but
# got `Optional[List[str]]`.
cmd = ["brew", "ls", "--versions"] + sorted(self.packages)
if cmd:
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if proc.returncode == 0:
# captured as binary as we will hash this later
# pyre-fixme[8]: Attribute has type `Optional[bool]`; used as `bytes`.
self.installed = proc.stdout
else:
# Need all packages to be present to consider us installed
self.installed = False
else:
self.installed = False
return bool(self.installed)
def update(self) -> ChangeStatus:
assert self.installed
return ChangeStatus(all_changed=False)
def hash(self) -> str:
if self.packages_are_installed():
return hashlib.sha256(self.installed).hexdigest()
else:
return "0" * 40
def get_src_dir(self) -> None:
return None
class PreinstalledNopFetcher(SystemPackageFetcher):
def __init__(self) -> None:
self.installed = True
class GitFetcher(Fetcher):
DEFAULT_DEPTH = 1
def __init__(
self,
build_options: BuildOptions,
manifest: ManifestParser,
repo_url: str,
rev: str,
depth: int,
branch: str,
) -> None:
# Extract the host/path portions of the URL and generate a flattened
# directory name. eg:
# github.com/facebook/folly.git -> github.com-facebook-folly.git
url = urlparse(repo_url)
directory = "%s%s%s" % (url.netloc, url.path, branch if branch else "")
for s in ["/", "\\", ":"]:
directory = directory.replace(s, "-")
# Place it in a repos dir in the scratch space
repos_dir = os.path.join(build_options.scratch_dir, "repos")
if not os.path.exists(repos_dir):
os.makedirs(repos_dir)
self.repo_dir: str = os.path.join(repos_dir, directory)
if not rev and build_options.project_hashes:
hash_file = os.path.join(
build_options.project_hashes,
re.sub("\\.git$", "-rev.txt", url.path[1:]),
)
if os.path.exists(hash_file):
with open(hash_file, "r") as f:
data = f.read()
m = re.match("Subproject commit ([a-fA-F0-9]{40})", data)
if not m:
raise Exception("Failed to parse rev from %s" % hash_file)
rev = m.group(1)
print(
"Using pinned rev %s for %s" % (rev, repo_url), file=sys.stderr
)
self.rev: str = rev or branch or "main"
self.origin_repo: str = repo_url
self.manifest: ManifestParser = manifest
self.depth: int = depth if depth else GitFetcher.DEFAULT_DEPTH
self.branch: str = branch
def _update(self) -> ChangeStatus:
current_hash = (
subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.repo_dir)
.strip()
.decode("utf-8")
)
target_hash = (
subprocess.check_output(["git", "rev-parse", self.rev], cwd=self.repo_dir)
.strip()
.decode("utf-8")
)
if target_hash == current_hash:
# It's up to date, so there are no changes. This doesn't detect eg:
# if origin/main moved and rev='main', but that's ok for our purposes;
# we should be using explicit hashes or eg: a stable branch for the cases
# that we care about, and it isn't unreasonable to require that the user
# explicitly perform a clean build if those have moved. For the most
# part we prefer that folks build using a release tarball from github
# rather than use the git protocol, as it is generally a bit quicker
# to fetch and easier to hash and verify tarball downloads.
return ChangeStatus()
print("Updating %s -> %s" % (self.repo_dir, self.rev))
run_cmd(["git", "fetch", "origin", self.rev], cwd=self.repo_dir)
run_cmd(["git", "checkout", self.rev], cwd=self.repo_dir)
run_cmd(["git", "submodule", "update", "--init"], cwd=self.repo_dir)
return ChangeStatus(True)
def update(self) -> ChangeStatus:
if os.path.exists(self.repo_dir):
return self._update()
self._clone()
return ChangeStatus(True)
def _clone(self) -> None:
print("Cloning %s..." % self.origin_repo)
# The basename/dirname stuff allows us to dance around issues where
# eg: this python process is native win32, but the git.exe is cygwin
# or msys and doesn't like the absolute windows path that we'd otherwise
# pass to it. Careful use of cwd helps avoid headaches with cygpath.
cmd = [
"git",
"clone",
"--depth=" + str(self.depth),
]
if self.branch:
cmd.append("--branch=" + self.branch)
cmd += [
"--",
self.origin_repo,
os.path.basename(self.repo_dir),
]
run_cmd(cmd, cwd=os.path.dirname(self.repo_dir))
self._update()
def clean(self) -> None:
if os.path.exists(self.repo_dir):
run_cmd(["git", "clean", "-fxd"], cwd=self.repo_dir)
def hash(self) -> str:
return self.rev
def get_src_dir(self) -> str:
return self.repo_dir
def does_file_need_update(
src_name: str, src_st: os.stat_result, dest_name: str
) -> bool:
try:
target_st = os.lstat(dest_name)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
return True
if src_st.st_size != target_st.st_size:
return True
if stat.S_IFMT(src_st.st_mode) != stat.S_IFMT(target_st.st_mode):
return True
if stat.S_ISLNK(src_st.st_mode):
return os.readlink(src_name) != os.readlink(dest_name)
if not stat.S_ISREG(src_st.st_mode):
return True
# They might have the same content; compare.
with open(src_name, "rb") as sf, open(dest_name, "rb") as df:
chunk_size = 8192
while True:
src_data = sf.read(chunk_size)
dest_data = df.read(chunk_size)
if src_data != dest_data:
return True
if len(src_data) < chunk_size:
# EOF
break
return False
def copy_if_different(src_name: str, dest_name: str) -> bool:
"""Copy src_name -> dest_name, but only touch dest_name
if src_name is different from dest_name, making this a
more build system friendly way to copy."""
src_st = os.lstat(src_name)
if not does_file_need_update(src_name, src_st, dest_name):
return False
dest_parent = os.path.dirname(dest_name)
if not os.path.exists(dest_parent):
os.makedirs(dest_parent)
if stat.S_ISLNK(src_st.st_mode):
try:
os.unlink(dest_name)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
target = os.readlink(src_name)
os.symlink(target, dest_name)
else:
shutil.copy2(src_name, dest_name)
return True
def filter_strip_marker(dest_name: str, marker: str) -> None:
"""Strip lines/blocks tagged with the given marker from a file."""
try:
with open(dest_name, "r") as f:
content = f.read()
except (UnicodeDecodeError, PermissionError):
return
if marker not in content:
return
escaped = re.escape(marker)
block_re = re.compile(
r"[^\n]*" + escaped + r"-start[^\n]*\n.*?[^\n]*" + escaped + r"-end[^\n]*\n?",
re.DOTALL,
)
line_re = re.compile(r".*" + escaped + r".*\n?")
filtered = block_re.sub("", content)
filtered = line_re.sub("", filtered)
if filtered != content:
with open(dest_name, "w") as f:
f.write(filtered)
def list_files_under_dir_newer_than_timestamp(
dir_to_scan: str, ts: int
) -> Iterator[str]:
for root, _dirs, files in os.walk(dir_to_scan):
for src_file in files:
full_name = os.path.join(root, src_file)
st = os.lstat(full_name)
if st.st_mtime > ts:
yield full_name
class ShipitPathMap:
def __init__(self) -> None:
self.roots: list[str] = []
self.mapping: list[str] = []
self.exclusion: list[str] = []
self.strip_marker: str = "@fb-only"
def add_mapping(self, fbsource_dir: str, target_dir: str) -> None:
"""Add a posix path or pattern. We cannot normpath the input
here because that would change the paths from posix to windows
form and break the logic throughout this class."""
self.roots.append(fbsource_dir)
# pyre-fixme[6]: For 1st argument expected `str` but got `Tuple[str, str]`.
self.mapping.append((fbsource_dir, target_dir))
def add_exclusion(self, pattern: str) -> None:
# pyre-fixme[6]: For 1st argument expected `str` but got `Pattern[str]`.
self.exclusion.append(re.compile(pattern))
def _minimize_roots(self) -> None:
"""compute the de-duplicated set of roots within fbsource.
We take the shortest common directory prefix to make this
determination"""
self.roots.sort(key=len)
minimized = []
for r in self.roots:
add_this_entry = True
for existing in minimized:
if r.startswith(existing + "/"):
add_this_entry = False
break
if add_this_entry:
minimized.append(r)
self.roots = minimized
def _sort_mapping(self) -> None:
self.mapping.sort(reverse=True, key=lambda x: len(x[0]))
def _map_name(self, norm_name: str, dest_root: str) -> str | None:
if norm_name.endswith(".pyc") or norm_name.endswith(".swp"):
# Ignore some incidental garbage while iterating
return None
for excl in self.exclusion:
# pyre-fixme[16]: `str` has no attribute `match`.
if excl.match(norm_name):
return None
for src_name, dest_name in self.mapping:
if norm_name == src_name or norm_name.startswith(src_name + "/"):
rel_name = os.path.relpath(norm_name, src_name)
# We can have "." as a component of some paths, depending
# on the contents of the shipit transformation section.
# normpath doesn't always remove `.` as the final component
# of the path, which be problematic when we later mkdir
# the dirname of the path that we return. Take care to avoid
# returning a path with a `.` in it.
rel_name = os.path.normpath(rel_name)
if dest_name == ".":
return os.path.normpath(os.path.join(dest_root, rel_name))
dest_name = os.path.normpath(dest_name)
return os.path.normpath(os.path.join(dest_root, dest_name, rel_name))
raise Exception("%s did not match any rules" % norm_name)
def mirror(self, fbsource_root: str, dest_root: str) -> ChangeStatus:
self._minimize_roots()
self._sort_mapping()
change_status = ChangeStatus()
# Record the full set of files that should be in the tree
full_file_list = set()
if sys.platform == "win32":
# Let's not assume st_dev has a consistent value on Windows.
def st_dev(path: str) -> int:
return 1
else:
def st_dev(path: str) -> int:
return os.lstat(path).st_dev
for fbsource_subdir in self.roots:
dir_to_mirror = os.path.join(fbsource_root, fbsource_subdir)
root_dev = st_dev(dir_to_mirror)
prefetch_dir_if_eden(dir_to_mirror)
if not os.path.exists(dir_to_mirror):
raise Exception(
"%s doesn't exist; check your sparse profile!" % dir_to_mirror
)
update_count = 0
for root, dirs, files in os.walk(dir_to_mirror):
dirs[:] = [d for d in dirs if root_dev == st_dev(os.path.join(root, d))]
for src_file in files:
full_name = os.path.join(root, src_file)
rel_name = os.path.relpath(full_name, fbsource_root)
norm_name = rel_name.replace("\\", "/")
target_name = self._map_name(norm_name, dest_root)
if target_name:
full_file_list.add(target_name)
if copy_if_different(full_name, target_name):
filter_strip_marker(target_name, self.strip_marker)
change_status.record_change(target_name)
if update_count < 10:
print("Updated %s -> %s" % (full_name, target_name))
elif update_count == 10:
print("...")
update_count += 1
if update_count:
print("Updated %s for %s" % (update_count, fbsource_subdir))
# Compare the list of previously shipped files; if a file is
# in the old list but not the new list then it has been
# removed from the source and should be removed from the
# destination.
# Why don't we simply create this list by walking dest_root?
# Some builds currently have to be in-source builds and
# may legitimately need to keep some state in the source tree :-/
installed_name = os.path.join(dest_root, ".shipit_shipped")
if os.path.exists(installed_name):
with open(installed_name, "rb") as f:
for name in f.read().decode("utf-8").splitlines():
name = name.strip()
if name not in full_file_list:
print("Remove %s" % name)
os.unlink(name)
change_status.record_change(name)
with open(installed_name, "wb") as f:
for name in sorted(full_file_list):
f.write(("%s\n" % name).encode("utf-8"))
return change_status
class FbsourceRepoData(NamedTuple):
hash: str
date: str
FBSOURCE_REPO_DATA: dict[str, FbsourceRepoData] = {}
def get_fbsource_repo_data(build_options: BuildOptions) -> FbsourceRepoData:
"""Returns the commit metadata for the fbsource repo.
Since we may have multiple first party projects to
hash, and because we don't mutate the repo, we cache
this hash in a global."""
# pyre-fixme[6]: For 1st argument expected `str` but got `Optional[str]`.
cached_data = FBSOURCE_REPO_DATA.get(build_options.fbsource_dir)
if cached_data:
return cached_data
if "GETDEPS_HG_REPO_DATA" in os.environ:
log_data = os.environ["GETDEPS_HG_REPO_DATA"]
else:
cmd = ["hg", "log", "-r.", "-T{node}\n{date|hgdate}"]
env = Env()
env.set("HGPLAIN", "1")
log_data = subprocess.check_output(
cmd, cwd=build_options.fbsource_dir, env=dict(env.items())
).decode("ascii")
(hash, datestr) = log_data.split("\n")
# datestr is like "seconds fractionalseconds"
# We want "20200324.113140"
(unixtime, _fractional) = datestr.split(" ")
date = datetime.fromtimestamp(int(unixtime)).strftime("%Y%m%d.%H%M%S")
cached_data = FbsourceRepoData(hash=hash, date=date)
# pyre-fixme[6]: For 1st argument expected `str` but got `Optional[str]`.
FBSOURCE_REPO_DATA[build_options.fbsource_dir] = cached_data
return cached_data
def is_public_commit(build_options: BuildOptions) -> bool: # noqa: C901
"""Check if the current commit is public (shipped/will be shipped to remote).
Works across git, sapling (sl), and hg repositories:
- For hg/sapling: Uses 'phase' command to check if commit is public
- For git: Checks if commit exists in remote branches
Returns True if public, False if draft/local-only or on error (conservative).
"""
# Use fbsource_dir if available (Meta internal), otherwise fall back to repo_root
repo_dir = build_options.fbsource_dir or build_options.repo_root
if not repo_dir:
# No repository detected, be conservative
return False
env = Env()
env.set("HGPLAIN", "1")
env_dict = dict(env.items())
try:
# Try hg/sapling phase command first (works for both hg and sl)
# Try 'sl' first as it's the preferred tool at Meta
for cmd in [["sl", "phase", "-r", "."], ["hg", "phase", "-r", "."]]:
try:
output = (
subprocess.check_output(
cmd, cwd=repo_dir, env=env_dict, stderr=subprocess.DEVNULL
)
.decode("ascii")
.strip()
)
# Output format: "hash: public" or "hash: draft"
return "public" in output
except (subprocess.CalledProcessError, FileNotFoundError):
continue
# Try git if hg/sl didn't work
try:
# Detect the default branch for origin remote
default_branch = None
try:
# Get the symbolic ref for origin/HEAD to find default branch
output = (
subprocess.check_output(
["git", "symbolic-ref", "refs/remotes/origin/HEAD"],
cwd=repo_dir,
stderr=subprocess.DEVNULL,
)
.decode("ascii")
.strip()
)
# Output format: "refs/remotes/origin/main"
if output.startswith("refs/remotes/"):
default_branch = output
except subprocess.CalledProcessError:
# If symbolic-ref fails, fall back to common names
pass
# Build list of branches to check
branches_to_check = []
if default_branch:
branches_to_check.append(default_branch)
# Also try common defaults as fallback
branches_to_check.extend(["origin/main", "origin/master"])
# Check if HEAD is an ancestor of any of these branches
for branch in branches_to_check:
try:
subprocess.check_output(
["git", "merge-base", "--is-ancestor", "HEAD", branch],
cwd=repo_dir,
stderr=subprocess.DEVNULL,
)
# If command succeeds (exit 0), HEAD is an ancestor of the branch
return True
except subprocess.CalledProcessError:
# Not an ancestor of this branch, try next
continue
# HEAD is not in any default branch
return False
except FileNotFoundError:
pass
# If all VCS commands failed, be conservative and don't upload
return False
except Exception:
# On any unexpected error, be conservative and don't upload
return False
class SimpleShipitTransformerFetcher(Fetcher):
def __init__(
self,
build_options: BuildOptions,
manifest: ManifestParser,
ctx: ManifestContext,
) -> None:
self.build_options: BuildOptions = build_options
self.manifest: ManifestParser = manifest
self.repo_dir: str = os.path.join(
build_options.scratch_dir, "shipit", manifest.name
)
self.ctx: ManifestContext = ctx
def clean(self) -> None:
if os.path.exists(self.repo_dir):
shutil.rmtree(self.repo_dir)
def update(self) -> ChangeStatus:
mapping = ShipitPathMap()
for src, dest in self.manifest.get_section_as_ordered_pairs(
"shipit.pathmap", self.ctx
):
# pyre-fixme[6]: For 2nd argument expected `str` but got `Optional[str]`.
mapping.add_mapping(src, dest)
if self.manifest.shipit_fbcode_builder:
mapping.add_mapping(
"fbcode/opensource/fbcode_builder", "build/fbcode_builder"
)
for pattern in self.manifest.get_section_as_args("shipit.strip", self.ctx):
mapping.add_exclusion(pattern)
# pyre-fixme[8]: Attribute has type `str`; used as `Optional[str]`.
mapping.strip_marker = self.manifest.shipit_strip_marker
# pyre-fixme[6]: In call `ShipitPathMap.mirror`, for 1st positional argument, expected `str` but got `Optional[str]`
return mapping.mirror(self.build_options.fbsource_dir, self.repo_dir)
def hash(self) -> str:
# We return a fixed non-hash string for in-fbsource builds.
# We're relying on the `update` logic to correctly invalidate
# the build in the case that files have changed.
return "fbsource"
def get_src_dir(self) -> str:
return self.repo_dir
class SubFetcher(Fetcher):
"""Fetcher for a project with subprojects"""
def __init__(self, base: Fetcher, subs: list[tuple[Fetcher, str]]) -> None:
self.base: Fetcher = base
self.subs: list[tuple[Fetcher, str]] = subs
def update(self) -> ChangeStatus:
base = self.base.update()
changed = base.build_changed() or base.sources_changed()
for fetcher, dir in self.subs:
stat = fetcher.update()
if stat.build_changed() or stat.sources_changed():
changed = True
link = self.base.get_src_dir() + "/" + dir
if not os.path.exists(link):
os.symlink(fetcher.get_src_dir(), link)
return ChangeStatus(changed)
def clean(self) -> None:
self.base.clean()
for fetcher, _ in self.subs:
fetcher.clean()
def hash(self) -> str:
my_hash = self.base.hash()
for fetcher, _ in self.subs:
my_hash += fetcher.hash()
return my_hash
def get_src_dir(self) -> str:
return self.base.get_src_dir()
class ShipitTransformerFetcher(Fetcher):
@classmethod
def _shipit_paths(cls, build_options: BuildOptions) -> list[str]:
www_path = ["/var/www/scripts/opensource/codesync"]
if build_options.fbsource_dir:
fbcode_path = [
os.path.join(
build_options.fbsource_dir,
"fbcode/opensource/codesync/codesync-cli/codesync",
)
]
else:
fbcode_path = []
return www_path + fbcode_path
def __init__(
self, build_options: BuildOptions, project_name: str, external_branch: str
) -> None:
self.build_options: BuildOptions = build_options
self.project_name: str = project_name
self.external_branch: str = external_branch
self.repo_dir: str = os.path.join(
build_options.scratch_dir, "shipit", project_name
)
self.shipit: str | None = None
for path in ShipitTransformerFetcher._shipit_paths(build_options):
if os.path.exists(path):
self.shipit = path
break
def update(self) -> ChangeStatus:
if os.path.exists(self.repo_dir):
return ChangeStatus()
self.run_shipit()
return ChangeStatus(True)
def clean(self) -> None:
if os.path.exists(self.repo_dir):
shutil.rmtree(self.repo_dir)
@classmethod
def available(cls, build_options: BuildOptions) -> bool:
return any(
os.path.exists(path)
for path in ShipitTransformerFetcher._shipit_paths(build_options)
)
def run_shipit(self) -> None:
tmp_path = self.repo_dir + ".new"
try:
if os.path.exists(tmp_path):
shutil.rmtree(tmp_path)
os.makedirs(os.path.dirname(tmp_path), exist_ok=True)
cmd = [
self.shipit,
"shipit",
"--project=" + self.project_name,
"--create-new-repo",
# pyre-fixme[58]: `+` is not supported for operand types `str` and
# `Optional[str]`.
"--source-repo-dir=" + self.build_options.fbsource_dir,
"--source-branch=.",
"--skip-source-init",
"--skip-source-pull",
"--skip-source-clean",
"--skip-push",
"--destination-use-anonymous-https",
"--create-new-repo-output-path=" + tmp_path,
]
if self.external_branch:
cmd += [
f"--external-branch={self.external_branch}",
]
# Run shipit
# pyre-fixme[6]: For 1st argument expected `List[str]` but got
# `List[Optional[str]]`.
run_cmd(cmd)
# Remove the .git directory from the repository it generated.
# There is no need to commit this.
repo_git_dir = os.path.join(tmp_path, ".git")
shutil.rmtree(repo_git_dir)
os.rename(tmp_path, self.repo_dir)
except Exception:
# Clean up after a failed extraction
if os.path.exists(tmp_path):
shutil.rmtree(tmp_path)
self.clean()
raise
def hash(self) -> str:
# We return a fixed non-hash string for in-fbsource builds.
return "fbsource"
def get_src_dir(self) -> str:
return self.repo_dir
def download_url_to_file_with_progress(url: str, file_name: str) -> None:
print("Download with %s -> %s ..." % (url, file_name))
class Progress:
last_report: float = 0
def write_update(self, total: int, amount: int) -> None:
if total == -1:
total = "(Unknown)"
if sys.stdout.isatty():
sys.stdout.write("\r downloading %s of %s " % (amount, total))
else:
# When logging to CI logs, avoid spamming the logs and print
# status every few seconds
now = time.time()
if now - self.last_report > 5:
sys.stdout.write(".. %s of %s " % (amount, total))
self.last_report = now
sys.stdout.flush()
def progress_pycurl(
self, total: float, amount: float, _uploadtotal: float, _uploadamount: float
) -> None:
self.write_update(total, amount)
progress = Progress()
start = time.time()
try:
if os.environ.get("GETDEPS_USE_WGET") is not None:
procargs = (
[
"wget",
]
+ os.environ.get("GETDEPS_WGET_ARGS", "").split()
+ [
"-O",
file_name,
url,
]
)
subprocess.run(procargs, capture_output=True)
headers = None
elif os.environ.get("GETDEPS_USE_LIBCURL") is not None:
import pycurl
with open(file_name, "wb") as f:
c = pycurl.Curl()
c.setopt(pycurl.URL, url)
c.setopt(pycurl.WRITEDATA, f)
# display progress
c.setopt(pycurl.NOPROGRESS, False)
c.setopt(pycurl.XFERINFOFUNCTION, progress.progress_pycurl)
c.perform()
c.close()
headers = None
else:
try:
req_header = {"Accept": "application/*"}
res = urlopen(Request(url, None, req_header))
chunk_size = 8192 # urlretrieve uses this value
headers = res.headers
content_length = res.headers.get("Content-Length")
total = int(content_length.strip()) if content_length else -1
amount = 0
with open(file_name, "wb") as f:
chunk = res.read(chunk_size)
while chunk:
f.write(chunk)
amount += len(chunk)
progress.write_update(total, amount)
chunk = res.read(chunk_size)
except (OSError, IOError) as exc: # noqa: B014
# Downloading from within Meta's network needs to use a proxy.
if shutil.which("fwdproxy-config") is None:
print(
"Note: Could not find Meta-specific fallback 'fwdproxy-config'. "
"If you are working externally, you can ignore this message."
)
raise
print("Default download failed, retrying with curl and fwdproxy...")
cmd = f"curl -L $(fwdproxy-config curl) -o {shlex.quote(file_name)} {shlex.quote(url)}"
print(f"Running command: {cmd}")
result = subprocess.run(cmd, shell=True, capture_output=True)
if result.returncode != 0:
raise TransientFailure(
f"Failed to download {url} to {file_name}: {exc} (fwdproxy fallback failed: {result.stderr.decode()})"
)
headers = None
except (OSError, IOError) as exc: # noqa: B014
raise TransientFailure(
"Failed to download %s to %s: %s" % (url, file_name, str(exc))
)
end = time.time()
sys.stdout.write(" [Complete in %f seconds]\n" % (end - start))
sys.stdout.flush()
if headers is not None:
print(f"{headers}")
class ArchiveFetcher(Fetcher):
def __init__(
self,
build_options: BuildOptions,
manifest: ManifestParser,
url: str,
sha256: str,
) -> None:
self.manifest: ManifestParser = manifest
self.url: str = url
self.sha256: str = sha256
self.build_options: BuildOptions = build_options
parsed_url = urlparse(self.url)
basename = "%s-%s" % (manifest.name, os.path.basename(parsed_url.path))
self.file_name: str = os.path.join(
build_options.scratch_dir, "downloads", basename
)
self.src_dir: str = os.path.join(
build_options.scratch_dir, "extracted", basename
)
self.hash_file: str = self.src_dir + ".hash"
def _verify_hash(self) -> None:
h = hashlib.sha256()
with open(self.file_name, "rb") as f:
while True:
block = f.read(8192)
if not block:
break
h.update(block)
digest = h.hexdigest()
if digest != self.sha256:
os.unlink(self.file_name)
raise Exception(
"%s: expected sha256 %s but got %s" % (self.url, self.sha256, digest)
)
def _download_dir(self) -> str:
"""returns the download dir, creating it if it doesn't already exist"""
download_dir = os.path.dirname(self.file_name)
if not os.path.exists(download_dir):
os.makedirs(download_dir)
return download_dir
def _download(self) -> None:
self._download_dir()
max_attempts = 5
delay = 1
for attempt in range(max_attempts):
try:
download_url_to_file_with_progress(self.url, self.file_name)
break
except TransientFailure as tf:
if attempt < max_attempts - 1:
delay *= 2
delay_with_jitter = delay * (1 + random.random() * 0.1)
time.sleep(min(delay_with_jitter, 10))
else:
print(f"Failed after retries: {tf}")
raise
self._verify_hash()
def clean(self) -> None:
if os.path.exists(self.src_dir):
shutil.rmtree(self.src_dir)
def update(self) -> ChangeStatus:
try:
with open(self.hash_file, "r") as f:
saved_hash = f.read().strip()
if saved_hash == self.sha256 and os.path.exists(self.src_dir):
# Everything is up to date
return ChangeStatus()
print(
"saved hash %s doesn't match expected hash %s, re-validating"
% (saved_hash, self.sha256)
)
os.unlink(self.hash_file)
except EnvironmentError:
pass
# If we got here we know the contents of src_dir are either missing
# or wrong, so blow away whatever happened to be there first.
if os.path.exists(self.src_dir):
shutil.rmtree(self.src_dir)
# If we already have a file here, make sure it looks legit before
# proceeding: any errors and we just remove it and re-download
if os.path.exists(self.file_name):
try:
self._verify_hash()
except Exception:
if os.path.exists(self.file_name):
os.unlink(self.file_name)
if not os.path.exists(self.file_name):
self._download()
self._verify_hash()
if tarfile.is_tarfile(self.file_name):
opener = tarfile.open
elif zipfile.is_zipfile(self.file_name):
opener = zipfile.ZipFile
else:
raise Exception("don't know how to extract %s" % self.file_name)
os.makedirs(self.src_dir)
print("Extract %s -> %s" % (self.file_name, self.src_dir))
if is_windows():
# Ensure that we don't fall over when dealing with long paths
# on windows
src = r"\\?\%s" % os.path.normpath(self.src_dir)
else:
src = self.src_dir
with opener(self.file_name) as t:
# The `str` here is necessary to ensure that we don't pass a unicode
# object down to tarfile.extractall on python2. When extracting
# the boost tarball it makes some assumptions and tries to convert
# a non-ascii path to ascii and throws.
src = str(src)
t.extractall(src)
if is_windows():
subdir = self.manifest.get("build", "subdir")
checkdir = src
if subdir:
checkdir = src + "\\" + subdir
if os.path.exists(checkdir):
children = os.listdir(checkdir)
print(f"Extracted to {checkdir} contents: {children}")
with open(self.hash_file, "w") as f:
f.write(self.sha256)
return ChangeStatus(True)
def hash(self) -> str:
return self.sha256
def get_src_dir(self) -> str:
return self.src_dir
def homebrew_package_prefix(package: str) -> str | None:
cmd = ["brew", "--prefix", package]
try:
proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except FileNotFoundError:
return None
if proc.returncode == 0:
return proc.stdout.decode("utf-8").rstrip()
================================================
FILE: build/fbcode_builder/getdeps/include_rewriter.py
================================================
#!/usr/bin/env python3
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
"""
Include Path Rewriter for getdeps
This module provides functionality to rewrite #include statements in C++ files
to handle differences between fbcode and open source project structures.
"""
import os
import re
import typing
from pathlib import Path
from typing import Any
if typing.TYPE_CHECKING:
from .manifest import ManifestParser
class IncludePathRewriter:
"""Rewrites #include paths in C++ source files based on path mappings."""
# C++ file extensions to process
CPP_EXTENSIONS: set[str] = {
".cpp",
".cc",
".cxx",
".c",
".h",
".hpp",
".hxx",
".tcc",
".inc",
}
def __init__(self, mappings: list[tuple[str, str]], verbose: bool = False) -> None:
"""
Initialize the rewriter with path mappings.
Args:
mappings: List of (old_path_prefix, new_path_prefix) tuples
verbose: Enable verbose output
"""
self.mappings: list[tuple[str, str]] = mappings
self.verbose: bool = verbose
# Compile regex patterns for efficiency
self.patterns: list[tuple[re.Pattern[str], str, str]] = []
for old_prefix, new_prefix in mappings:
# Match both quoted and angle bracket includes
# Pattern matches: #include "old_prefix/rest" or #include
pattern = re.compile(
r'(#\s*include\s*[<"])(' + re.escape(old_prefix) + r'/[^">]+)([">])',
re.MULTILINE,
)
self.patterns.append((pattern, old_prefix, new_prefix))
def rewrite_file(self, file_path: Path, dry_run: bool = False) -> bool:
"""
Rewrite includes in a single file.
Args:
file_path: Path to the file to process
dry_run: If True, don't actually modify files
Returns:
True if file was modified, False otherwise
"""
try:
with open(file_path, "r", encoding="utf-8") as f:
original_content: str = f.read()
except (IOError, UnicodeDecodeError) as e:
if self.verbose:
print(f"Warning: Could not read {file_path}: {e}")
return False
modified_content: str = original_content
changes_made: bool = False
for pattern, old_prefix, new_prefix in self.patterns:
def make_replace_func(
old_prefix: str, new_prefix: str
) -> typing.Callable[[re.Match[str]], str]:
def replace_func(match: re.Match[str]) -> str:
nonlocal changes_made
prefix: str = match.group(1) # #include [<"]
full_path: str = match.group(2) # full path
suffix: str = match.group(3) # [">]
# Replace the old prefix with new prefix
new_path: str = full_path.replace(old_prefix, new_prefix, 1)
if self.verbose and not changes_made:
print(f" {full_path} -> {new_path}")
changes_made = True
return f"{prefix}{new_path}{suffix}"
return replace_func
modified_content = pattern.sub(
make_replace_func(old_prefix, new_prefix), modified_content
)
if changes_made and not dry_run:
try:
with open(file_path, "w", encoding="utf-8") as f:
f.write(modified_content)
if self.verbose:
print(f"Modified: {file_path}")
except IOError as e:
print(f"Error: Could not write {file_path}: {e}")
return False
elif changes_made and dry_run:
if self.verbose:
print(f"Would modify: {file_path}")
return changes_made
def process_directory(self, source_dir: Path, dry_run: bool = False) -> int:
"""
Process all C++ files in a directory recursively.
Args:
source_dir: Root directory to process
dry_run: If True, don't actually modify files
Returns:
Number of files modified
"""
if not source_dir.exists():
if self.verbose:
print(f"Warning: Directory {source_dir} does not exist")
return 0
modified_count: int = 0
processed_count: int = 0
for root, dirs, files in os.walk(source_dir):
# Skip hidden directories and common build directories
dirs[:] = [
d
for d in dirs
if not d.startswith(".")
and d not in {"build", "_build", "__pycache__", "CMakeFiles"}
]
for file in files:
file_path: Path = Path(root) / file
# Only process C++ files
if file_path.suffix.lower() not in self.CPP_EXTENSIONS:
continue
processed_count += 1
if self.verbose:
print(f"Processing: {file_path}")
if self.rewrite_file(file_path, dry_run):
modified_count += 1
if self.verbose or modified_count > 0:
print(f"Processed {processed_count} files, modified {modified_count} files")
return modified_count
def rewrite_includes_from_manifest(
manifest: ManifestParser, ctx: Any, source_dir: str, verbose: bool = False
) -> int:
"""
Rewrite includes using mappings from a manifest file.
Args:
manifest: The manifest object containing shipit.pathmap section
ctx: The manifest context
source_dir: Directory containing source files to process
verbose: Enable verbose output
Returns:
Number of files modified
"""
mappings: list[tuple[str, str]] = []
# Get mappings from the manifest's shipit.pathmap section
for src, dest in manifest.get_section_as_ordered_pairs("shipit.pathmap", ctx):
# Remove fbcode/ or xplat/ prefixes from src since they won't appear in #include statements
if src.startswith("fbcode/"):
src = src[len("fbcode/") :]
elif src.startswith("xplat/"):
src = src[len("xplat/") :]
# pyre-fixme[6]: For 1st argument expected `Tuple[str, str]` but got
# `Tuple[str, Optional[str]]`.
mappings.append((src, dest))
if not mappings:
if verbose:
print("No include path mappings found in manifest")
return 0
if verbose:
print("Include path mappings:")
for old_path, new_path in mappings:
print(f" {old_path} -> {new_path}")
rewriter: IncludePathRewriter = IncludePathRewriter(mappings, verbose)
return rewriter.process_directory(Path(source_dir), dry_run=False)
================================================
FILE: build/fbcode_builder/getdeps/load.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import base64
import copy
import hashlib
import os
import typing
from collections.abc import Iterator
from . import fetcher
from .envfuncs import path_search
from .errors import ManifestNotFound
from .manifest import ManifestParser
if typing.TYPE_CHECKING:
from .buildopts import BuildOptions
from .manifest import ContextGenerator, ManifestContext
class Loader:
"""The loader allows our tests to patch the load operation"""
def _list_manifests(self, build_opts: BuildOptions) -> Iterator[str]:
"""Returns a generator that iterates all the available manifests"""
for path, _, files in os.walk(build_opts.manifests_dir):
for name in files:
# skip hidden files
if name.startswith("."):
continue
yield os.path.join(path, name)
def _load_manifest(self, path: str) -> ManifestParser:
return ManifestParser(path)
def load_project(
self, build_opts: BuildOptions, project_name: str
) -> ManifestParser:
if "/" in project_name or "\\" in project_name:
# Assume this is a path already
return ManifestParser(project_name)
for manifest in self._list_manifests(build_opts):
if os.path.basename(manifest) == project_name:
return ManifestParser(manifest)
raise ManifestNotFound(project_name)
def load_all(self, build_opts: BuildOptions) -> dict[str, ManifestParser]:
manifests_by_name: dict[str, ManifestParser] = {}
for manifest in self._list_manifests(build_opts):
m = self._load_manifest(manifest)
if m.name in manifests_by_name:
raise Exception("found duplicate manifest '%s'" % m.name)
manifests_by_name[m.name] = m
return manifests_by_name
class ResourceLoader(Loader):
def __init__(self, namespace: str, manifests_dir: str) -> None:
self.namespace: str = namespace
self.manifests_dir: str = manifests_dir
def _list_manifests(self, build_opts: BuildOptions) -> Iterator[str]:
import pkg_resources
dirs: list[str] = [self.manifests_dir]
while dirs:
current = dirs.pop(0)
for name in pkg_resources.resource_listdir(self.namespace, current):
path = "%s/%s" % (current, name)
if pkg_resources.resource_isdir(self.namespace, path):
dirs.append(path)
else:
yield "%s/%s" % (current, name)
def _find_manifest(self, project_name: str) -> str:
# pyre-fixme[20]: Call `ResourceLoader._list_manifests` expects argument `build_opts`.
for name in self._list_manifests():
if name.endswith("/%s" % project_name):
return name
raise ManifestNotFound(project_name)
def _load_manifest(self, path: str) -> ManifestParser:
import pkg_resources
contents = pkg_resources.resource_string(self.namespace, path).decode("utf8")
return ManifestParser(file_name=path, fp=contents)
def load_project(
self, build_opts: BuildOptions, project_name: str
) -> ManifestParser:
project_name = self._find_manifest(project_name)
# pyre-fixme[16]: `ResourceLoader` has no attribute `_load_resource_manifest`.
return self._load_resource_manifest(project_name)
LOADER: Loader = Loader()
def patch_loader(namespace: str, manifests_dir: str = "manifests") -> None:
global LOADER
LOADER = ResourceLoader(namespace, manifests_dir)
def load_project(build_opts: BuildOptions, project_name: str) -> ManifestParser:
"""given the name of a project or a path to a manifest file,
load up the ManifestParser instance for it and return it"""
return LOADER.load_project(build_opts, project_name)
def load_all_manifests(build_opts: BuildOptions) -> dict[str, ManifestParser]:
return LOADER.load_all(build_opts)
class ManifestLoader:
"""ManifestLoader stores information about project manifest relationships for a
given set of (build options + platform) configuration.
The ManifestLoader class primarily serves as a location to cache project dependency
relationships and project hash values for this build configuration.
"""
def __init__(
self, build_opts: BuildOptions, ctx_gen: ContextGenerator | None = None
) -> None:
self._loader: Loader = LOADER
self.build_opts: BuildOptions = build_opts
if ctx_gen is None:
self.ctx_gen: ContextGenerator = self.build_opts.get_context_generator()
else:
self.ctx_gen = ctx_gen
self.manifests_by_name: dict[str, ManifestParser] = {}
self._loaded_all: bool = False
self._project_hashes: dict[str, str] = {}
self._fetcher_overrides: dict[str, fetcher.LocalDirFetcher] = {}
self._build_dir_overrides: dict[str, str] = {}
self._install_dir_overrides: dict[str, str] = {}
self._install_prefix_overrides: dict[str, str] = {}
def load_manifest(self, name: str) -> ManifestParser:
manifest = self.manifests_by_name.get(name)
if manifest is None:
manifest = self._loader.load_project(self.build_opts, name)
self.manifests_by_name[name] = manifest
return manifest
def load_all_manifests(self) -> dict[str, ManifestParser]:
if not self._loaded_all:
all_manifests_by_name = self._loader.load_all(self.build_opts)
if self.manifests_by_name:
# To help ensure that we only ever have a single manifest object for a
# given project, and that it can't change once we have loaded it,
# only update our mapping for projects that weren't already loaded.
for name, manifest in all_manifests_by_name.items():
self.manifests_by_name.setdefault(name, manifest)
else:
self.manifests_by_name = all_manifests_by_name
self._loaded_all = True
return self.manifests_by_name
def dependencies_of(self, manifest: ManifestParser) -> list[ManifestParser]:
"""Returns the dependencies of the given project, not including the project itself, in topological order."""
return [
dep
for dep in self.manifests_in_dependency_order(manifest)
if dep != manifest
]
def manifests_in_dependency_order(
self, manifest: ManifestParser | None = None
) -> list[ManifestParser]:
"""Compute all dependencies of the specified project. Returns a list of the
dependencies plus the project itself, in topologically sorted order.
Each entry in the returned list only depends on projects that appear before it
in the list.
If the input manifest is None, the dependencies for all currently loaded
projects will be computed. i.e., if you call load_all_manifests() followed by
manifests_in_dependency_order() this will return a global dependency ordering of
all projects."""
# The list of deps that have been fully processed
seen: set[str] = set()
# The list of deps which have yet to be evaluated. This
# can potentially contain duplicates.
if manifest is None:
deps: list[ManifestParser] = list(self.manifests_by_name.values())
else:
assert manifest.name in self.manifests_by_name
deps = [manifest]
# The list of manifests in dependency order
dep_order: list[ManifestParser] = []
system_packages: dict[str, list[str]] = {}
while len(deps) > 0:
m = deps.pop(0)
if m.name in seen:
continue
# Consider its deps, if any.
# We sort them for increased determinism; we'll produce
# a correct order even if they aren't sorted, but we prefer
# to produce the same order regardless of how they are listed
# in the project manifest files.
ctx: ManifestContext = self.ctx_gen.get_context(m.name)
dep_list: list[str] = m.get_dependencies(ctx)
dep_count: int = 0
for dep_name in dep_list:
# If we're not sure whether it is done, queue it up
if dep_name not in seen:
dep = self.manifests_by_name.get(dep_name)
if dep is None:
dep = self._loader.load_project(self.build_opts, dep_name)
self.manifests_by_name[dep.name] = dep
deps.append(dep)
dep_count += 1
if dep_count > 0:
# If we queued anything, re-queue this item, as it depends
# those new item(s) and their transitive deps.
deps.append(m)
continue
# Its deps are done, so we can emit it
seen.add(m.name)
# Capture system packages as we may need to set PATHs to then later
if (
self.build_opts.allow_system_packages
and self.build_opts.host_type.get_package_manager()
):
packages: dict[str, list[str]] = m.get_required_system_packages(ctx)
for pkg_type, v in packages.items():
merged: list[str] = system_packages.get(pkg_type, [])
if v not in merged:
merged += v
system_packages[pkg_type] = merged
# A manifest depends on all system packages in it dependencies as well
# pyre-fixme[8]: Attribute has type `Dict[str, str]`; used as
# `Dict[str, List[str]]`.
m.resolved_system_packages = copy.copy(system_packages)
dep_order.append(m)
return dep_order
def set_project_src_dir(self, project_name: str, path: str) -> None:
self._fetcher_overrides[project_name] = fetcher.LocalDirFetcher(path)
def set_project_build_dir(self, project_name: str, path: str) -> None:
self._build_dir_overrides[project_name] = path
def set_project_install_dir(self, project_name: str, path: str) -> None:
self._install_dir_overrides[project_name] = path
def set_project_install_prefix(self, project_name: str, path: str) -> None:
self._install_prefix_overrides[project_name] = path
def create_fetcher(
self, manifest: ManifestParser
) -> fetcher.Fetcher | fetcher.LocalDirFetcher:
override = self._fetcher_overrides.get(manifest.name)
if override is not None:
return override
ctx: ManifestContext = self.ctx_gen.get_context(manifest.name)
return manifest.create_fetcher(self.build_opts, self, ctx)
def get_project_hash(self, manifest: ManifestParser) -> str:
h = self._project_hashes.get(manifest.name)
if h is None:
h = self._compute_project_hash(manifest)
self._project_hashes[manifest.name] = h
return h
def _compute_project_hash(self, manifest: ManifestParser) -> str:
"""This recursive function computes a hash for a given manifest.
The hash takes into account some environmental factors on the
host machine and includes the hashes of its dependencies.
No caching of the computation is performed, which is theoretically
wasteful but the computation is fast enough that it is not required
to cache across multiple invocations."""
ctx: ManifestContext = self.ctx_gen.get_context(manifest.name)
hasher = hashlib.sha256()
# Some environmental and configuration things matter
env: dict[str, str | None] = {}
env["install_dir"] = self.build_opts.install_dir
env["scratch_dir"] = self.build_opts.scratch_dir
env["vcvars_path"] = self.build_opts.vcvars_path
env["os"] = self.build_opts.host_type.ostype
env["distro"] = self.build_opts.host_type.distro
env["distro_vers"] = self.build_opts.host_type.distrovers
env["shared_libs"] = str(self.build_opts.shared_libs)
for name in [
"CXXFLAGS",
"CPPFLAGS",
"LDFLAGS",
"CXX",
"CC",
"GETDEPS_CMAKE_DEFINES",
]:
env[name] = os.environ.get(name)
for tool in ["cc", "c++", "gcc", "g++", "clang", "clang++"]:
env["tool-%s" % tool] = path_search(os.environ, tool)
for name in manifest.get_section_as_args("depends.environment", ctx):
env[name] = os.environ.get(name)
fetcher_inst: fetcher.Fetcher | fetcher.LocalDirFetcher = self.create_fetcher(
manifest
)
env["fetcher.hash"] = fetcher_inst.hash()
for name in sorted(env.keys()):
hasher.update(name.encode("utf-8"))
value = env.get(name)
if value is not None:
try:
hasher.update(value.encode("utf-8"))
except AttributeError as exc:
raise AttributeError("name=%r, value=%r: %s" % (name, value, exc))
manifest.update_hash(hasher, ctx)
# If a patchfile is specified, include its contents in the hash
patchfile: str | None = manifest.get("build", "patchfile", ctx=ctx)
if patchfile:
patchfile_path: str = os.path.join(
self.build_opts.fbcode_builder_dir, "patches", patchfile
)
if os.path.exists(patchfile_path):
with open(patchfile_path, "rb") as f:
hasher.update(f.read())
dep_list: list[str] = manifest.get_dependencies(ctx)
for dep in dep_list:
dep_manifest: ManifestParser = self.load_manifest(dep)
dep_hash: str = self.get_project_hash(dep_manifest)
hasher.update(dep_hash.encode("utf-8"))
# Use base64 to represent the hash, rather than the simple hex digest,
# so that the string is shorter. Use the URL-safe encoding so that
# the hash can also be safely used as a filename component.
h: str = base64.urlsafe_b64encode(hasher.digest()).decode("ascii")
# ... and because cmd.exe is troublesome with `=` signs, nerf those.
# They tend to be padding characters at the end anyway, so we can
# safely discard them.
h = h.replace("=", "")
return h
def _get_project_dir_name(self, manifest: ManifestParser) -> str:
if manifest.is_first_party_project():
return manifest.name
else:
project_hash: str = self.get_project_hash(manifest)
return "%s-%s" % (manifest.name, project_hash)
def get_project_install_dir(self, manifest: ManifestParser) -> str:
override = self._install_dir_overrides.get(manifest.name)
if override:
return override
project_dir_name: str = self._get_project_dir_name(manifest)
return os.path.join(self.build_opts.install_dir, project_dir_name)
def get_project_build_dir(self, manifest: ManifestParser) -> str:
override = self._build_dir_overrides.get(manifest.name)
if override:
return override
project_dir_name: str = self._get_project_dir_name(manifest)
return os.path.join(self.build_opts.scratch_dir, "build", project_dir_name)
def get_project_install_prefix(self, manifest: ManifestParser) -> str | None:
return self._install_prefix_overrides.get(manifest.name)
def get_project_install_dir_respecting_install_prefix(
self, manifest: ManifestParser
) -> str:
inst_dir: str = self.get_project_install_dir(manifest)
prefix: str | None = self.get_project_install_prefix(manifest)
if prefix:
return inst_dir + prefix
return inst_dir
================================================
FILE: build/fbcode_builder/getdeps/manifest.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import configparser
import hashlib
import io
import os
import sys
import typing
from .builder import (
AutoconfBuilder,
Boost,
CMakeBootStrapBuilder,
CMakeBuilder,
Iproute2Builder,
MakeBuilder,
MesonBuilder,
NinjaBootstrap,
NopBuilder,
OpenSSLBuilder,
SetupPyBuilder,
SqliteBuilder,
)
from .cargo import CargoBuilder
from .expr import ExprNode, parse_expr
from .fetcher import (
ArchiveFetcher,
GitFetcher,
PreinstalledNopFetcher,
ShipitTransformerFetcher,
SimpleShipitTransformerFetcher,
SubFetcher,
SystemPackageFetcher,
)
from .py_wheel_builder import PythonWheelBuilder
if typing.TYPE_CHECKING:
from .builder import BuilderBase
from .buildopts import BuildOptions
from .fetcher import Fetcher
from .load import ManifestLoader
REQUIRED: str = "REQUIRED"
OPTIONAL: str = "OPTIONAL"
SCHEMA: dict[str, dict[str, object]] = {
"manifest": {
"optional_section": False,
"fields": {
"name": REQUIRED,
"fbsource_path": OPTIONAL,
"shipit_project": OPTIONAL,
"shipit_fbcode_builder": OPTIONAL,
"use_shipit": OPTIONAL,
"shipit_external_branch": OPTIONAL,
"shipit_strip_marker": OPTIONAL,
},
},
"dependencies": {"optional_section": True, "allow_values": False},
"depends.environment": {"optional_section": True},
"git": {
"optional_section": True,
"fields": {
"repo_url": REQUIRED,
"rev": OPTIONAL,
"depth": OPTIONAL,
"branch": OPTIONAL,
},
},
"download": {
"optional_section": True,
"fields": {"url": REQUIRED, "sha256": REQUIRED},
},
"build": {
"optional_section": True,
"fields": {
"builder": REQUIRED,
"subdir": OPTIONAL,
"make_binary": OPTIONAL,
"build_in_src_dir": OPTIONAL,
"only_install": OPTIONAL,
"job_weight_mib": OPTIONAL,
"patchfile": OPTIONAL,
"patchfile_opts": OPTIONAL,
"rewrite_includes": OPTIONAL,
},
},
"msbuild": {"optional_section": True, "fields": {"project": REQUIRED}},
"cargo": {
"optional_section": True,
"fields": {
"build_doc": OPTIONAL,
"workspace_dir": OPTIONAL,
"manifests_to_build": OPTIONAL,
# Where to write cargo config (defaults to build_dir/.cargo/config.toml)
"cargo_config_file": OPTIONAL,
},
},
"github.actions": {
"optional_section": True,
"fields": {
"run_tests": OPTIONAL,
"required_locales": OPTIONAL,
"rust_version": OPTIONAL,
"build_type": OPTIONAL,
},
},
"crate.pathmap": {"optional_section": True},
"cmake.defines": {"optional_section": True},
"autoconf.args": {"optional_section": True},
"autoconf.envcmd.LDFLAGS": {"optional_section": True},
"rpms": {"optional_section": True},
"debs": {"optional_section": True},
"homebrew": {"optional_section": True},
"pps": {"optional_section": True},
"preinstalled.env": {"optional_section": True},
"bootstrap.args": {"optional_section": True},
"b2.args": {"optional_section": True},
"make.build_args": {"optional_section": True},
"make.install_args": {"optional_section": True},
"make.test_args": {"optional_section": True},
"meson.setup_args": {"optional_section": True},
"header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}},
"shipit.pathmap": {"optional_section": True},
"shipit.strip": {"optional_section": True},
"install.files": {"optional_section": True},
"subprojects": {"optional_section": True},
# fb-only
"sandcastle": {"optional_section": True, "fields": {"run_tests": OPTIONAL}},
"setup-py.test": {"optional_section": True, "fields": {"python_script": REQUIRED}},
"setup-py.env": {"optional_section": True},
}
# These sections are allowed to vary for different platforms
# using the expression syntax to enable/disable sections
ALLOWED_EXPR_SECTIONS: list[str] = [
"autoconf.args",
"autoconf.envcmd.LDFLAGS",
"build",
"cmake.defines",
"dependencies",
"make.build_args",
"make.install_args",
"bootstrap.args",
"b2.args",
"download",
"git",
"install.files",
"rpms",
"debs",
"shipit.pathmap",
"shipit.strip",
"homebrew",
"github.actions",
"pps",
]
def parse_conditional_section_name(name: str, section_def: str) -> ExprNode:
expr = name[len(section_def) + 1 :]
return parse_expr(expr, ManifestContext.ALLOWED_VARIABLES)
def validate_allowed_fields(
file_name: str,
section: str,
config: configparser.RawConfigParser,
allowed_fields: dict[str, str],
) -> None:
for field in config.options(section):
if not allowed_fields.get(field):
raise Exception(
("manifest file %s section '%s' contains " "unknown field '%s'")
% (file_name, section, field)
)
for field in allowed_fields:
if allowed_fields[field] == REQUIRED and not config.has_option(section, field):
raise Exception(
("manifest file %s section '%s' is missing " "required field '%s'")
% (file_name, section, field)
)
def validate_allow_values(
file_name: str, section: str, config: configparser.RawConfigParser
) -> None:
for field in config.options(section):
value = config.get(section, field)
if value is not None:
raise Exception(
(
"manifest file %s section '%s' has '%s = %s' but "
"this section doesn't allow specifying values "
"for its entries"
)
% (file_name, section, field, value)
)
def validate_section(
file_name: str, section: str, config: configparser.RawConfigParser
) -> str:
section_def = SCHEMA.get(section)
if not section_def:
for name in ALLOWED_EXPR_SECTIONS:
if section.startswith(name + "."):
# Verify that the conditional parses, but discard it
try:
parse_conditional_section_name(section, name)
except Exception as exc:
raise Exception(
("manifest file %s section '%s' has invalid " "conditional: %s")
% (file_name, section, str(exc))
)
section_def = SCHEMA.get(name)
canonical_section_name = name
break
if not section_def:
raise Exception(
"manifest file %s contains unknown section '%s'" % (file_name, section)
)
else:
canonical_section_name = section
allowed_fields = section_def.get("fields")
if allowed_fields:
# pyre-ignore[6]: Expected `dict[str, str]` but got `object`.
validate_allowed_fields(file_name, section, config, allowed_fields)
elif not section_def.get("allow_values", True):
validate_allow_values(file_name, section, config)
# pyre-fixme[61]: `canonical_section_name` is undefined, or not always defined.
return canonical_section_name
class ManifestParser:
def __init__(self, file_name: str, fp: str | typing.IO[str] | None = None) -> None:
# allow_no_value enables listing parameters in the
# autoconf.args section one per line
config = configparser.RawConfigParser(allow_no_value=True)
config.optionxform = str # type: ignore[assignment] # make it case sensitive
if fp is None:
with open(file_name, "r") as fp:
config.read_file(fp)
elif isinstance(fp, type("")):
# For testing purposes, parse from a string (str
# or unicode)
config.read_file(io.StringIO(fp))
else:
config.read_file(fp)
# validate against the schema
seen_sections: set[str] = set()
for section in config.sections():
seen_sections.add(validate_section(file_name, section, config))
for section in SCHEMA.keys():
section_def = SCHEMA[section]
if (
not section_def.get("optional_section", False)
and section not in seen_sections
):
raise Exception(
"manifest file %s is missing required section %s"
% (file_name, section)
)
self._config: configparser.RawConfigParser = config
self.name: str = config.get("manifest", "name")
self.fbsource_path: str | None = self.get("manifest", "fbsource_path")
self.shipit_project: str | None = self.get("manifest", "shipit_project")
self.shipit_fbcode_builder: str | None = self.get(
"manifest", "shipit_fbcode_builder"
)
self.resolved_system_packages: dict[str, str] = {}
self.shipit_strip_marker: str | None = self.get(
"manifest", "shipit_strip_marker", defval="@fb-only"
)
if self.name != os.path.basename(file_name):
raise Exception(
"filename of the manifest '%s' does not match the manifest name '%s'"
% (file_name, self.name)
)
if "." in self.name:
raise Exception(
f"manifest name ({self.name}) must not contain the '.' character (it is incompatible with github actions)"
)
def get(
self,
section: str,
key: str,
defval: str | None = None,
ctx: ManifestContext | dict[str, str | None] | None = None,
) -> str | None:
ctx = ctx or {}
for s in self._config.sections():
if s == section:
if self._config.has_option(s, key):
return self._config.get(s, key)
return defval
if s.startswith(section + "."):
expr = parse_conditional_section_name(s, section)
# pyre-fixme[6]: For 1st argument expected `Dict[str,
# Optional[str]]` but got `Union[Dict[str, Optional[str]],
# ManifestContext]`.
if not expr.eval(ctx):
continue
if self._config.has_option(s, key):
return self._config.get(s, key)
return defval
def get_dependencies(self, ctx: ManifestContext) -> list[str]:
dep_list = list(self.get_section_as_dict("dependencies", ctx).keys())
dep_list.sort()
builder = self.get("build", "builder", ctx=ctx)
if builder in ("cmake", "python-wheel"):
dep_list.insert(0, "cmake")
elif builder == "autoconf" and self.name not in (
"autoconf",
"libtool",
"automake",
):
# they need libtool and its deps (automake, autoconf) so add
# those as deps (but obviously not if we're building those
# projects themselves)
dep_list.insert(0, "libtool")
return dep_list
def get_section_as_args(
self,
section: str,
ctx: ManifestContext | dict[str, str | None] | None = None,
) -> list[str]:
"""Intended for use with the make.[build_args/install_args] and
autoconf.args sections, this method collects the entries and returns an
array of strings.
If the manifest contains conditional sections, ctx is used to
evaluate the condition and merge in the values.
"""
args: list[str] = []
ctx = ctx or {}
for s in self._config.sections():
if s != section:
if not s.startswith(section + "."):
continue
expr = parse_conditional_section_name(s, section)
# pyre-fixme[6]: For 1st argument expected `Dict[str,
# Optional[str]]` but got `Union[Dict[str, Optional[str]],
# ManifestContext]`.
if not expr.eval(ctx):
continue
for field in self._config.options(s):
value = self._config.get(s, field)
if value is None:
args.append(field)
else:
args.append("%s=%s" % (field, value))
return args
def get_section_as_ordered_pairs(
self,
section: str,
ctx: ManifestContext | dict[str, str | None] | None = None,
) -> list[tuple[str, str | None]]:
"""Used for eg: shipit.pathmap which has strong
ordering requirements"""
res: list[tuple[str, str | None]] = []
ctx = ctx or {}
for s in self._config.sections():
if s != section:
if not s.startswith(section + "."):
continue
expr = parse_conditional_section_name(s, section)
# pyre-fixme[6]: For 1st argument expected `Dict[str,
# Optional[str]]` but got `Union[Dict[str, Optional[str]],
# ManifestContext]`.
if not expr.eval(ctx):
continue
for key in self._config.options(s):
value = self._config.get(s, key)
res.append((key, value))
return res
def get_section_as_dict(
self,
section: str,
ctx: ManifestContext | dict[str, str | None] | None,
) -> dict[str, str | None]:
d: dict[str, str | None] = {}
for s in self._config.sections():
if s != section:
if not s.startswith(section + "."):
continue
expr = parse_conditional_section_name(s, section)
# pyre-fixme[6]: For 1st argument expected `Dict[str,
# Optional[str]]` but got `Union[None, Dict[str, Optional[str]],
# ManifestContext]`.
if not expr.eval(ctx):
continue
for field in self._config.options(s):
value = self._config.get(s, field)
d[field] = value
return d
def update_hash(self, hasher: hashlib._Hash, ctx: ManifestContext) -> None:
"""Compute a hash over the configuration for the given
context. The goal is for the hash to change if the config
for that context changes, but not if a change is made to
the config only for a different platform than that expressed
by ctx. The hash is intended to be used to help invalidate
a future cache for the third party build products.
The hasher argument is a hash object returned from hashlib."""
for section in sorted(SCHEMA.keys()):
hasher.update(section.encode("utf-8"))
# Note: at the time of writing, nothing in the implementation
# relies on keys in any config section being ordered.
# In theory we could have conflicting flags in different
# config sections and later flags override earlier flags.
# For the purposes of computing a hash we're not super
# concerned about this: manifest changes should be rare
# enough and we'd rather that this trigger an invalidation
# than strive for a cache hit at this time.
pairs = self.get_section_as_ordered_pairs(section, ctx)
pairs.sort(key=lambda pair: pair[0])
for key, value in pairs:
hasher.update(key.encode("utf-8"))
if value is not None:
hasher.update(value.encode("utf-8"))
def is_first_party_project(self) -> bool:
"""returns true if this is an FB first-party project"""
return self.shipit_project is not None
def get_required_system_packages(
self, ctx: ManifestContext
) -> dict[str, list[str]]:
"""Returns dictionary of packager system -> list of packages"""
return {
"rpm": self.get_section_as_args("rpms", ctx),
"deb": self.get_section_as_args("debs", ctx),
"homebrew": self.get_section_as_args("homebrew", ctx),
"pacman-package": self.get_section_as_args("pps", ctx),
}
def _is_satisfied_by_preinstalled_environment(self, ctx: ManifestContext) -> bool:
envs = self.get_section_as_args("preinstalled.env", ctx)
if not envs:
return False
for key in envs:
val = os.environ.get(key, None)
print(
f"Testing ENV[{key}]: {repr(val)}",
file=sys.stderr,
)
if val is None:
return False
if len(val) == 0:
return False
return True
def get_repo_url(self, ctx: ManifestContext) -> str | None:
return self.get("git", "repo_url", ctx=ctx)
def _create_fetcher(
self, build_options: BuildOptions, ctx: ManifestContext
) -> Fetcher:
real_shipit_available = ShipitTransformerFetcher.available(build_options)
use_real_shipit = real_shipit_available and (
build_options.use_shipit
or self.get("manifest", "use_shipit", defval="false", ctx=ctx) == "true"
)
if (
not use_real_shipit
and self.fbsource_path
and build_options.fbsource_dir
and self.shipit_project
):
return SimpleShipitTransformerFetcher(build_options, self, ctx)
if (
self.fbsource_path
and build_options.fbsource_dir
and self.shipit_project
and real_shipit_available
):
# We can use the code from fbsource
return ShipitTransformerFetcher(
build_options,
self.shipit_project,
# pyre-fixme[6]: For 3rd argument expected `str` but got
# `Optional[str]`.
self.get("manifest", "shipit_external_branch"),
)
# If both of these are None, the package can only be coming from
# preinstalled toolchain or system packages
repo_url = self.get_repo_url(ctx)
url = self.get("download", "url", ctx=ctx)
# Can we satisfy this dep with system packages?
if (repo_url is None and url is None) or build_options.allow_system_packages:
if self._is_satisfied_by_preinstalled_environment(ctx):
# pyre-fixme[7]: Expected `Fetcher` but got `PreinstalledNopFetcher`.
return PreinstalledNopFetcher()
if build_options.host_type.get_package_manager():
packages = self.get_required_system_packages(ctx)
package_fetcher = SystemPackageFetcher(build_options, packages)
if package_fetcher.packages_are_installed():
# pyre-fixme[7]: Expected `Fetcher` but got `SystemPackageFetcher`.
return package_fetcher
if repo_url:
rev = self.get("git", "rev")
depth = self.get("git", "depth")
branch = self.get("git", "branch")
# pyre-fixme[6]: For 4th argument expected `str` but got `Optional[str]`.
# pyre-fixme[6]: For 5th argument expected `int` but got `Optional[str]`.
# pyre-fixme[6]: For 6th argument expected `str` but got `Optional[str]`.
return GitFetcher(build_options, self, repo_url, rev, depth, branch)
if url:
# We need to defer this import until now to avoid triggering
# a cycle when the facebook/__init__.py is loaded.
try:
from .facebook.lfs import LFSCachingArchiveFetcher
return LFSCachingArchiveFetcher(
build_options,
self,
url,
# pyre-fixme[6]: For 4th argument expected `str` but got
# `Optional[str]`.
self.get("download", "sha256", ctx=ctx),
)
except ImportError:
# This FB internal module isn't shippped to github,
# so just use its base class
return ArchiveFetcher(
build_options,
self,
url,
# pyre-fixme[6]: For 4th argument expected `str` but got
# `Optional[str]`.
self.get("download", "sha256", ctx=ctx),
)
raise KeyError(
f"project {self.name} has no fetcher configuration or system packages matching {ctx} - have you run `getdeps.py install-system-deps --recursive`?"
)
def create_fetcher(
self,
build_options: BuildOptions,
loader: ManifestLoader,
ctx: ManifestContext,
) -> Fetcher:
fetcher = self._create_fetcher(build_options, ctx)
subprojects = self.get_section_as_ordered_pairs("subprojects", ctx)
if subprojects:
subs: list[tuple[Fetcher, str | None]] = []
for project, subdir in subprojects:
submanifest = loader.load_manifest(project)
subfetcher = submanifest.create_fetcher(build_options, loader, ctx)
subs.append((subfetcher, subdir))
# pyre-fixme[6]: For 2nd argument expected `List[Tuple[Fetcher, str]]`
# but got `List[Tuple[Fetcher, Optional[str]]]`.
return SubFetcher(fetcher, subs)
else:
return fetcher
def get_builder_name(self, ctx: ManifestContext) -> str:
builder = self.get("build", "builder", ctx=ctx)
if not builder:
raise Exception("project %s has no builder for %r" % (self.name, ctx))
return builder
def create_builder( # noqa:C901
self,
build_options: BuildOptions,
src_dir: str,
build_dir: str,
inst_dir: str,
ctx: ManifestContext,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
final_install_prefix: str | None = None,
extra_cmake_defines: dict[str, str] | None = None,
cmake_targets: list[str] | None = None,
extra_b2_args: list[str] | None = None,
) -> BuilderBase:
builder = self.get_builder_name(ctx)
build_in_src_dir = self.get("build", "build_in_src_dir", "false", ctx=ctx)
if build_in_src_dir == "true":
# Some scripts don't work when they are configured and build in
# a different directory than source (or when the build directory
# is not a subdir of source).
build_dir = src_dir
subdir = self.get("build", "subdir", None, ctx=ctx)
if subdir is not None:
build_dir = os.path.join(build_dir, subdir)
print("build_dir is %s" % build_dir) # just to quiet lint
if builder == "make" or builder == "cmakebootstrap":
build_args = self.get_section_as_args("make.build_args", ctx)
install_args = self.get_section_as_args("make.install_args", ctx)
test_args = self.get_section_as_args("make.test_args", ctx)
if builder == "cmakebootstrap":
return CMakeBootStrapBuilder(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
# pyre-fixme[6]: For 7th argument expected `str` but got `None`.
None,
inst_dir,
build_args,
install_args,
test_args,
)
else:
return MakeBuilder(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
# pyre-fixme[6]: For 7th argument expected `str` but got `None`.
None,
inst_dir,
build_args,
install_args,
test_args,
)
if builder == "autoconf":
args = self.get_section_as_args("autoconf.args", ctx)
conf_env_args: dict[str, list[str]] = {}
ldflags_cmd = self.get_section_as_args("autoconf.envcmd.LDFLAGS", ctx)
if ldflags_cmd:
conf_env_args["LDFLAGS"] = ldflags_cmd
return AutoconfBuilder(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
args,
conf_env_args,
)
if builder == "boost":
args = self.get_section_as_args("b2.args", ctx)
if extra_b2_args is not None:
args += extra_b2_args
return Boost(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
args,
)
if builder == "cmake":
defines = self.get_section_as_dict("cmake.defines", ctx)
return CMakeBuilder(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
# pyre-fixme[6]: For 9th argument expected `Optional[Dict[str,
# str]]` but got `Dict[str, Optional[str]]`.
defines,
final_install_prefix,
extra_cmake_defines,
cmake_targets,
)
if builder == "python-wheel":
return PythonWheelBuilder(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
)
if builder == "sqlite":
return SqliteBuilder(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
)
if builder == "ninja_bootstrap":
return NinjaBootstrap(
loader,
dep_manifests,
build_options,
ctx,
self,
build_dir,
src_dir,
inst_dir,
)
if builder == "nop":
return NopBuilder(
loader, dep_manifests, build_options, ctx, self, src_dir, inst_dir
)
if builder == "openssl":
return OpenSSLBuilder(
loader,
dep_manifests,
build_options,
ctx,
self,
build_dir,
src_dir,
inst_dir,
)
if builder == "iproute2":
return Iproute2Builder(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
)
if builder == "meson":
return MesonBuilder(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
)
if builder == "setup-py":
return SetupPyBuilder(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
)
if builder == "cargo":
return self.create_cargo_builder(
loader,
dep_manifests,
build_options,
ctx,
src_dir,
build_dir,
inst_dir,
)
raise KeyError("project %s has no known builder" % (self.name))
def create_prepare_builders(
self,
build_options: BuildOptions,
ctx: ManifestContext,
src_dir: str,
build_dir: str,
inst_dir: str,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
) -> list[BuilderBase]:
"""Create builders that have a prepare step run, e.g. to write config files"""
prepare_builders: list[BuilderBase] = []
builder = self.get_builder_name(ctx)
cargo = self.get_section_as_dict("cargo", ctx)
if not builder == "cargo" and cargo:
cargo_builder = self.create_cargo_builder(
loader,
dep_manifests,
build_options,
ctx,
src_dir,
build_dir,
inst_dir,
)
prepare_builders.append(cargo_builder)
return prepare_builders
def create_cargo_builder(
self,
loader: ManifestLoader,
dep_manifests: list[ManifestParser],
build_options: BuildOptions,
ctx: ManifestContext,
src_dir: str,
build_dir: str,
inst_dir: str,
) -> CargoBuilder:
# pyre-fixme[6]: For 3rd argument expected `Optional[str]` but got `bool`.
build_doc = self.get("cargo", "build_doc", False, ctx)
workspace_dir = self.get("cargo", "workspace_dir", None, ctx)
manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx)
cargo_config_file = self.get("cargo", "cargo_config_file", None, ctx)
return CargoBuilder(
loader,
dep_manifests,
build_options,
ctx,
self,
src_dir,
build_dir,
inst_dir,
# pyre-fixme[6]: For 9th argument expected `bool` but got `Optional[str]`.
build_doc,
workspace_dir,
manifests_to_build,
cargo_config_file,
)
class ManifestContext:
"""ProjectContext contains a dictionary of values to use when evaluating boolean
expressions in a project manifest.
This object should be passed as the `ctx` parameter in ManifestParser.get() calls.
"""
ALLOWED_VARIABLES: set[str] = {
"os",
"distro",
"distro_vers",
"fb",
"fbsource",
"test",
"shared_libs",
}
def __init__(self, ctx_dict: dict[str, str | None]) -> None:
assert set(ctx_dict.keys()) == self.ALLOWED_VARIABLES
self.ctx_dict: dict[str, str | None] = ctx_dict
def get(self, key: str) -> str | None:
return self.ctx_dict[key]
def set(self, key: str, value: str | None) -> None:
assert key in self.ALLOWED_VARIABLES
self.ctx_dict[key] = value
def copy(self) -> ManifestContext:
return ManifestContext(dict(self.ctx_dict))
def __str__(self) -> str:
s = ", ".join(
"%s=%s" % (key, value) for key, value in sorted(self.ctx_dict.items())
)
return "{" + s + "}"
class ContextGenerator:
"""ContextGenerator allows creating ManifestContext objects on a per-project basis.
This allows us to evaluate different projects with slightly different contexts.
For instance, this can be used to only enable tests for some projects."""
def __init__(self, default_ctx: dict[str, str | None]) -> None:
self.default_ctx: ManifestContext = ManifestContext(default_ctx)
self.ctx_by_project: dict[str, ManifestContext] = {}
def set_value_for_project(
self, project_name: str, key: str, value: str | None
) -> None:
project_ctx = self.ctx_by_project.get(project_name)
if project_ctx is None:
project_ctx = self.default_ctx.copy()
self.ctx_by_project[project_name] = project_ctx
project_ctx.set(key, value)
def set_value_for_all_projects(self, key: str, value: str | None) -> None:
self.default_ctx.set(key, value)
for ctx in self.ctx_by_project.values():
ctx.set(key, value)
def get_context(self, project_name: str) -> ManifestContext:
return self.ctx_by_project.get(project_name, self.default_ctx)
================================================
FILE: build/fbcode_builder/getdeps/platform.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import os
import platform
import re
import shlex
import sys
def is_windows() -> bool:
"""Returns true if the system we are currently running on
is a Windows system"""
return sys.platform.startswith("win")
def get_linux_type() -> tuple[str | None, str | None, str | None]:
try:
with open("/etc/os-release") as f:
data = f.read()
except EnvironmentError:
return (None, None, None)
os_vars: dict[str, str] = {}
for line in data.splitlines():
parts = line.split("=", 1)
if len(parts) != 2:
continue
key = parts[0].strip()
value_parts = shlex.split(parts[1].strip())
if not value_parts:
value = ""
else:
value = value_parts[0]
os_vars[key] = value
name = os_vars.get("NAME")
if name:
name = name.lower()
name = re.sub("linux", "", name)
name = name.strip().replace(" ", "_")
version_id = os_vars.get("VERSION_ID")
if version_id:
version_id = version_id.lower()
return "linux", name, version_id
# Ideally we'd use a common library like `psutil` to read system information,
# but getdeps can't take third-party dependencies.
def _get_available_ram_linux() -> int:
# TODO: Ideally, this function would inspect the current cgroup for any
# limits, rather than solely relying on system RAM.
meminfo_path = "/proc/meminfo"
try:
with open(meminfo_path) as f:
for line in f:
try:
key, value = line.split(":", 1)
except ValueError:
continue
suffix = " kB\n"
if key == "MemAvailable" and value.endswith(suffix):
value = value[: -len(suffix)]
try:
return int(value) // 1024
except ValueError:
continue
except OSError:
print("error opening {}".format(meminfo_path), end="", file=sys.stderr)
else:
print(
"{} had no valid MemAvailable".format(meminfo_path), end="", file=sys.stderr
)
guess = 8
print(", guessing {} GiB".format(guess), file=sys.stderr)
return guess * 1024
def _get_available_ram_macos() -> int:
import ctypes.util
libc = ctypes.CDLL(ctypes.util.find_library("libc"), use_errno=True)
sysctlbyname = libc.sysctlbyname
sysctlbyname.restype = ctypes.c_int
sysctlbyname.argtypes = [
ctypes.c_char_p,
ctypes.c_void_p,
ctypes.POINTER(ctypes.c_size_t),
ctypes.c_void_p,
ctypes.c_size_t,
]
# TODO: There may be some way to approximate an availability
# metric, but just use total RAM for now.
memsize = ctypes.c_int64()
memsizesize = ctypes.c_size_t(8)
res = sysctlbyname(
b"hw.memsize", ctypes.byref(memsize), ctypes.byref(memsizesize), None, 0
)
if res != 0:
raise NotImplementedError(
f"failed to retrieve hw.memsize sysctl: {ctypes.get_errno()}"
)
return memsize.value // (1024 * 1024)
def _get_available_ram_windows() -> int:
import ctypes
DWORD = ctypes.c_uint32
QWORD = ctypes.c_uint64
class MEMORYSTATUSEX(ctypes.Structure):
_fields_ = [
("dwLength", DWORD),
("dwMemoryLoad", DWORD),
("ullTotalPhys", QWORD),
("ullAvailPhys", QWORD),
("ullTotalPageFile", QWORD),
("ullAvailPageFile", QWORD),
("ullTotalVirtual", QWORD),
("ullAvailVirtual", QWORD),
("ullExtendedVirtual", QWORD),
]
ms = MEMORYSTATUSEX()
ms.dwLength = ctypes.sizeof(ms)
# pyre-ignore[16]
res = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(ms))
if res == 0:
raise NotImplementedError("error calling GlobalMemoryStatusEx")
# This is fuzzy, but AvailPhys is too conservative, and AvailTotal is too
# aggressive, so average the two. It's okay for builds to use some swap.
return (ms.ullAvailPhys + ms.ullTotalPhys) // (2 * 1024 * 1024)
def _get_available_ram_freebsd() -> int:
import ctypes.util
libc = ctypes.CDLL(ctypes.util.find_library("libc"), use_errno=True)
sysctlbyname = libc.sysctlbyname
sysctlbyname.restype = ctypes.c_int
sysctlbyname.argtypes = [
ctypes.c_char_p,
ctypes.c_void_p,
ctypes.POINTER(ctypes.c_size_t),
ctypes.c_void_p,
ctypes.c_size_t,
]
# hw.usermem is pretty close to what we want.
memsize = ctypes.c_int64()
memsizesize = ctypes.c_size_t(8)
res = sysctlbyname(
b"hw.usermem", ctypes.byref(memsize), ctypes.byref(memsizesize), None, 0
)
if res != 0:
raise NotImplementedError(
f"failed to retrieve hw.memsize sysctl: {ctypes.get_errno()}"
)
return memsize.value // (1024 * 1024)
def get_available_ram() -> int:
"""
Returns a platform-appropriate available RAM metric in MiB.
"""
if sys.platform == "linux":
return _get_available_ram_linux()
elif sys.platform == "darwin":
return _get_available_ram_macos()
elif sys.platform == "win32":
return _get_available_ram_windows()
elif sys.platform.startswith("freebsd"):
return _get_available_ram_freebsd()
else:
raise NotImplementedError(
f"platform {sys.platform} does not have an implementation of get_available_ram"
)
def is_current_host_arm() -> bool:
if sys.platform.startswith("darwin"):
# platform.machine() can be fooled by rosetta for python < 3.9.2
return "ARM64" in os.uname().version
else:
machine = platform.machine().lower()
return "arm" in machine or "aarch" in machine
class HostType:
def __init__(
self,
ostype: str | None = None,
distro: str | None = None,
distrovers: str | None = None,
) -> None:
# Maybe we should allow callers to indicate whether this machine uses
# an ARM architecture, but we need to change HostType serialization
# and deserialization in that case and hunt down anywhere that is
# persisting that serialized data.
isarm = False
if ostype is None:
distro = None
distrovers = None
if sys.platform.startswith("linux"):
ostype, distro, distrovers = get_linux_type()
elif sys.platform.startswith("darwin"):
ostype = "darwin"
elif is_windows():
ostype = "windows"
distrovers = str(sys.getwindowsversion().major)
elif sys.platform.startswith("freebsd"):
ostype = "freebsd"
else:
ostype = sys.platform
isarm = is_current_host_arm()
# The operating system type
self.ostype: str | None = ostype
# The distribution, if applicable
self.distro: str | None = distro
# The OS/distro version if known
self.distrovers: str | None = distrovers
# Does the CPU use an ARM architecture? ARM includes Apple Silicon
# Macs as well as other ARM systems that might be running Linux or
# something.
self.isarm: bool = isarm
def is_windows(self) -> bool:
return self.ostype == "windows"
# is_arm is kinda half implemented at the moment. This method is only
# intended to be used when HostType represents information about the
# current machine we are running on.
# When HostType is being used to enumerate platform types (represent
# information about machine types that we may or may not be running on)
# the result could be nonsense (under the current implementation its always
# false.)
def is_arm(self) -> bool:
return self.isarm
def is_darwin(self) -> bool:
return self.ostype == "darwin"
def is_linux(self) -> bool:
return self.ostype == "linux"
def is_freebsd(self) -> bool:
return self.ostype == "freebsd"
def as_tuple_string(self) -> str:
return "%s-%s-%s" % (
self.ostype,
self.distro or "none",
self.distrovers or "none",
)
def get_package_manager(self) -> str | None:
if not self.is_linux() and not self.is_darwin():
return None
if self.is_darwin():
return "homebrew"
if self.distro in ("fedora", "centos", "centos_stream", "rocky"):
return "rpm"
if self.distro is not None and self.distro.startswith(
("debian", "ubuntu", "pop!_os", "mint")
):
return "deb"
if self.distro == "arch":
return "pacman-package"
return None
@staticmethod
def from_tuple_string(s: str) -> HostType:
ostype, distro, distrovers = s.split("-")
return HostType(ostype=ostype, distro=distro, distrovers=distrovers)
def __eq__(self, b: object) -> bool:
if not isinstance(b, HostType):
return False
return (
self.ostype == b.ostype
and self.distro == b.distro
and self.distrovers == b.distrovers
)
================================================
FILE: build/fbcode_builder/getdeps/py_wheel_builder.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import codecs
import collections
import email
import email.message
import os
import re
import stat
from .builder import BuilderBase, CMakeBuilder
WheelNameInfo = collections.namedtuple(
"WheelNameInfo", ("distribution", "version", "build", "python", "abi", "platform")
)
CMAKE_HEADER = """
cmake_minimum_required(VERSION 3.8)
project("{manifest_name}" LANGUAGES C)
set(CMAKE_MODULE_PATH
"{cmake_dir}"
${{CMAKE_MODULE_PATH}}
)
include(FBPythonBinary)
set(CMAKE_INSTALL_DIR lib/cmake/{manifest_name} CACHE STRING
"The subdirectory where CMake package config files should be installed")
"""
CMAKE_FOOTER = """
install_fb_python_library({lib_name} EXPORT all)
install(
EXPORT all
FILE {manifest_name}-targets.cmake
NAMESPACE {namespace}::
DESTINATION ${{CMAKE_INSTALL_DIR}}
)
include(CMakePackageConfigHelpers)
configure_package_config_file(
${{CMAKE_BINARY_DIR}}/{manifest_name}-config.cmake.in
{manifest_name}-config.cmake
INSTALL_DESTINATION ${{CMAKE_INSTALL_DIR}}
PATH_VARS
CMAKE_INSTALL_DIR
)
install(
FILES ${{CMAKE_CURRENT_BINARY_DIR}}/{manifest_name}-config.cmake
DESTINATION ${{CMAKE_INSTALL_DIR}}
)
"""
CMAKE_CONFIG_FILE = """
@PACKAGE_INIT@
include(CMakeFindDependencyMacro)
set_and_check({upper_name}_CMAKE_DIR "@PACKAGE_CMAKE_INSTALL_DIR@")
if (NOT TARGET {namespace}::{lib_name})
include("${{{upper_name}_CMAKE_DIR}}/{manifest_name}-targets.cmake")
endif()
set({upper_name}_LIBRARIES {namespace}::{lib_name})
{find_dependency_lines}
if (NOT {manifest_name}_FIND_QUIETLY)
message(STATUS "Found {manifest_name}: ${{PACKAGE_PREFIX_DIR}}")
endif()
"""
# Note: for now we are manually manipulating the wheel packet contents.
# The wheel format is documented here:
# https://www.python.org/dev/peps/pep-0491/#file-format
#
# We currently aren't particularly smart about correctly handling the full wheel
# functionality, but this is good enough to handle simple pure-python wheels,
# which is the main thing we care about right now.
#
# We could potentially use pip to install the wheel to a temporary location and
# then copy its "installed" files, but this has its own set of complications.
# This would require pip to already be installed and available, and we would
# need to correctly find the right version of pip or pip3 to use.
# If we did ever want to go down that path, we would probably want to use
# something like the following pip3 command:
# pip3 --isolated install --no-cache-dir --no-index --system \
# --target
class PythonWheelBuilder(BuilderBase):
"""This Builder can take Python wheel archives and install them as python libraries
that can be used by add_fb_python_library()/add_fb_python_executable() CMake rules.
"""
# pyre-fixme[13]: Attribute `dist_info_dir` is never initialized.
dist_info_dir: str
# pyre-fixme[13]: Attribute `template_format_dict` is never initialized.
template_format_dict: dict[str, str]
def _build(self, reconfigure: bool) -> None:
# When we are invoked, self.src_dir contains the unpacked wheel contents.
#
# Since a wheel file is just a zip file, the Fetcher code recognizes it as such
# and goes ahead and unpacks it. (We could disable that Fetcher behavior in the
# future if we ever wanted to, say if we wanted to call pip here.)
wheel_name = self._parse_wheel_name()
name_version_prefix = "-".join((wheel_name.distribution, wheel_name.version))
dist_info_name = name_version_prefix + ".dist-info"
data_dir_name = name_version_prefix + ".data"
self.dist_info_dir = os.path.join(self.src_dir, dist_info_name)
wheel_metadata = self._read_wheel_metadata(wheel_name)
# Check that we can understand the wheel version.
# We don't really care about wheel_metadata["Root-Is-Purelib"] since
# we are generating our own standalone python archives rather than installing
# into site-packages.
version = wheel_metadata["Wheel-Version"]
if not version.startswith("1."):
raise Exception("unsupported wheel version %s" % (version,))
# Add a find_dependency() call for each of our dependencies.
# The dependencies are also listed in the wheel METADATA file, but it is simpler
# to pull this directly from the getdeps manifest.
dep_list = sorted(
self.manifest.get_section_as_dict("dependencies", self.ctx).keys()
)
find_dependency_lines = ["find_dependency({})".format(dep) for dep in dep_list]
getdeps_cmake_dir = os.path.join(
os.path.dirname(os.path.dirname(__file__)), "CMake"
)
self.template_format_dict = {
# Note that CMake files always uses forward slash separators in path names,
# even on Windows. Therefore replace path separators here.
"cmake_dir": _to_cmake_path(getdeps_cmake_dir),
"lib_name": self.manifest.name,
"manifest_name": self.manifest.name,
"namespace": self.manifest.name,
"upper_name": self.manifest.name.upper().replace("-", "_"),
"find_dependency_lines": "\n".join(find_dependency_lines),
}
# Find sources from the root directory
path_mapping: dict[str, str] = {}
for entry in os.listdir(self.src_dir):
if entry == data_dir_name:
continue
self._add_sources(path_mapping, os.path.join(self.src_dir, entry), entry)
# Files under the .data directory also need to be installed in the correct
# locations
if os.path.exists(data_dir_name):
# TODO: process the subdirectories of data_dir_name
# This isn't implemented yet since for now we have only needed dependencies
# on some simple pure Python wheels, so I haven't tested against wheels with
# additional files in the .data directory.
raise Exception(
"handling of the subdirectories inside %s is not implemented yet"
% data_dir_name
)
# Emit CMake files
self._write_cmakelists(path_mapping, dep_list)
self._write_cmake_config_template()
# Run the build
self._run_cmake_build(reconfigure)
def _run_cmake_build(self, reconfigure: bool) -> None:
cmake_builder = CMakeBuilder(
loader=self.loader,
dep_manifests=self.dep_manifests,
build_opts=self.build_opts,
ctx=self.ctx,
manifest=self.manifest,
# Note that we intentionally supply src_dir=build_dir,
# since we wrote out our generated CMakeLists.txt in the build directory
src_dir=self.build_dir,
build_dir=self.build_dir,
inst_dir=self.inst_dir,
defines={},
final_install_prefix=None,
)
cmake_builder.build(reconfigure=reconfigure)
def _write_cmakelists(
self, path_mapping: dict[str, str], dependencies: list[str]
) -> None:
cmake_path = os.path.join(self.build_dir, "CMakeLists.txt")
with open(cmake_path, "w") as f:
f.write(CMAKE_HEADER.format(**self.template_format_dict))
for dep in dependencies:
f.write("find_package({0} REQUIRED)\n".format(dep))
f.write(
"add_fb_python_library({lib_name}\n".format(**self.template_format_dict)
)
f.write(' BASE_DIR "%s"\n' % _to_cmake_path(self.src_dir))
f.write(" SOURCES\n")
for src_path, install_path in path_mapping.items():
f.write(
' "%s=%s"\n'
% (_to_cmake_path(src_path), _to_cmake_path(install_path))
)
if dependencies:
f.write(" DEPENDS\n")
for dep in dependencies:
f.write(' "{0}::{0}"\n'.format(dep))
f.write(")\n")
f.write(CMAKE_FOOTER.format(**self.template_format_dict))
def _write_cmake_config_template(self) -> None:
config_path_name = self.manifest.name + "-config.cmake.in"
output_path = os.path.join(self.build_dir, config_path_name)
with open(output_path, "w") as f:
f.write(CMAKE_CONFIG_FILE.format(**self.template_format_dict))
def _add_sources(
self, path_mapping: dict[str, str], src_path: str, install_path: str
) -> None:
s = os.lstat(src_path)
if not stat.S_ISDIR(s.st_mode):
path_mapping[src_path] = install_path
return
for entry in os.listdir(src_path):
self._add_sources(
path_mapping,
os.path.join(src_path, entry),
os.path.join(install_path, entry),
)
def _parse_wheel_name(self) -> WheelNameInfo:
# The ArchiveFetcher prepends "manifest_name-", so strip that off first.
wheel_name = os.path.basename(self.src_dir)
prefix = self.manifest.name + "-"
if not wheel_name.startswith(prefix):
raise Exception(
"expected wheel source directory to be of the form %s-NAME.whl"
% (prefix,)
)
wheel_name = wheel_name[len(prefix) :]
wheel_name_re = re.compile(
r"(?P[^-]+)"
r"-(?P\d+[^-]*)"
r"(-(?P\d+[^-]*))?"
r"-(?P\w+\d+(\.\w+\d+)*)"
r"-(?P\w+)"
r"-(?P\w+(\.\w+)*)"
r"\.whl"
)
match = wheel_name_re.match(wheel_name)
if not match:
raise Exception(
"bad python wheel name %s: expected to have the form "
"DISTRIBUTION-VERSION-[-BUILD]-PYTAG-ABI-PLATFORM"
)
return WheelNameInfo(
distribution=match.group("distribution"),
version=match.group("version"),
build=match.group("build"),
python=match.group("python"),
abi=match.group("abi"),
platform=match.group("platform"),
)
# pyre-fixme[24]: Generic type `email.message.Message` expects 2 type parameters.
def _read_wheel_metadata(self, wheel_name: WheelNameInfo) -> email.message.Message:
metadata_path = os.path.join(self.dist_info_dir, "WHEEL")
with codecs.open(metadata_path, "r", encoding="utf-8") as f:
return email.message_from_file(f)
def _to_cmake_path(path: str) -> str:
# CMake always uses forward slashes to separate paths in CMakeLists.txt files,
# even on Windows. It treats backslashes as character escapes, so using
# backslashes in the path will cause problems. Therefore replace all path
# separators with forward slashes to make sure the paths are correct on Windows.
# e.g. "C:\foo\bar.txt" becomes "C:/foo/bar.txt"
return path.replace(os.path.sep, "/")
================================================
FILE: build/fbcode_builder/getdeps/runcmd.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import os
import select
import subprocess
import sys
from collections.abc import Callable
from shlex import quote as shellquote
from .envfuncs import Env
from .platform import is_windows
class RunCommandError(Exception):
pass
def make_memory_limit_preexec_fn(
job_weight_mib: int,
) -> Callable[[], None] | None:
"""Create a preexec_fn that sets a per-process virtual memory limit.
When getdeps spawns build commands (cmake -> ninja -> N compiler processes),
the parallelism is computed from available RAM divided by job_weight_mib.
However, there is no enforcement of that budget: if a compiler or linker
process exceeds its expected memory usage, the system can run out of RAM
and the Linux OOM killer may terminate arbitrary processes — including the
user's shell or terminal.
This function returns a callable suitable for subprocess.Popen's preexec_fn
parameter. It runs in each child process after fork() but before exec(),
setting RLIMIT_AS (virtual address space limit) so that a runaway process
gets a failed allocation (std::bad_alloc / ENOMEM) instead of triggering
the OOM killer. The limit is inherited by all descendant processes (ninja,
compiler invocations, etc.).
The per-process limit is set to job_weight_mib * 10. The 10x multiplier
accounts for the fact that RLIMIT_AS caps virtual address space, which is
typically 2-4x larger than resident (physical) memory for C++ compilers
due to memory-mapped files, shared libraries, and address space reservations
that don't consume physical RAM. The multiplier is intentionally generous:
the goal is a safety net that catches genuine runaways before the OOM killer
fires, not a tight per-job budget.
Only applies on Linux, where the OOM killer is the problem. Returns None
on other platforms.
"""
if sys.platform != "linux":
return None
# Each job is budgeted job_weight_mib of physical RAM. Virtual address
# space is typically 2-4x RSS. Use 10x as a generous safety net: tight
# enough to stop a runaway process before the OOM killer fires, but loose
# enough to avoid false positives from normal virtual memory overhead.
limit_bytes: int = job_weight_mib * 10 * 1024 * 1024
def _set_memory_limit() -> None:
import resource
resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes))
return _set_memory_limit
def _print_env_diff(env: Env, log_fn: Callable[[str], None]) -> None:
current_keys = set(os.environ.keys())
wanted_env = set(env.keys())
unset_keys = current_keys.difference(wanted_env)
for k in sorted(unset_keys):
log_fn("+ unset %s\n" % k)
added_keys = wanted_env.difference(current_keys)
for k in wanted_env.intersection(current_keys):
if os.environ[k] != env[k]:
added_keys.add(k)
for k in sorted(added_keys):
if ("PATH" in k) and (os.pathsep in env[k]):
log_fn("+ %s=\\\n" % k)
for elem in env[k].split(os.pathsep):
log_fn("+ %s%s\\\n" % (shellquote(elem), os.pathsep))
else:
log_fn("+ %s=%s \\\n" % (k, shellquote(env[k])))
def check_cmd(
cmd: list[str],
env: Env | None = None,
cwd: str | None = None,
allow_fail: bool = False,
log_file: str | None = None,
) -> None:
"""Run the command and abort on failure"""
rc = run_cmd(cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_file=log_file)
if rc != 0:
raise RuntimeError(f"Failure exit code {rc} for command {cmd}")
def run_cmd(
cmd: list[str],
env: Env | None = None,
cwd: str | None = None,
allow_fail: bool = False,
log_file: str | None = None,
preexec_fn: Callable[[], None] | None = None,
) -> int:
def log_to_stdout(msg: str) -> None:
sys.stdout.buffer.write(msg.encode(errors="surrogateescape"))
if log_file is not None:
with open(log_file, "a", encoding="utf-8", errors="surrogateescape") as log:
# pyre-fixme[53]: Captured variable `log` is not annotated.
def log_function(msg: str) -> None:
log.write(msg)
log_to_stdout(msg)
return _run_cmd(
cmd,
env=env,
cwd=cwd,
allow_fail=allow_fail,
log_fn=log_function,
preexec_fn=preexec_fn,
)
else:
return _run_cmd(
cmd,
env=env,
cwd=cwd,
allow_fail=allow_fail,
log_fn=log_to_stdout,
preexec_fn=preexec_fn,
)
def _run_cmd(
cmd: list[str],
env: Env | None,
cwd: str | None,
allow_fail: bool,
log_fn: Callable[[str], None],
preexec_fn: Callable[[], None] | None = None,
) -> int:
log_fn("---\n")
try:
cmd_str = " \\\n+ ".join(shellquote(arg) for arg in cmd)
except TypeError:
# eg: one of the elements is None
raise RunCommandError("problem quoting cmd: %r" % cmd)
if env:
assert isinstance(env, Env)
_print_env_diff(env, log_fn)
# Convert from our Env type to a regular dict.
# This is needed because python3 looks up b'PATH' and 'PATH'
# and emits an error if both are present. In our Env type
# we'll return the same value for both requests, but we don't
# have duplicate potentially conflicting values which is the
# spirit of the check.
env_dict: dict[str, str] | None = dict(env.items())
else:
env_dict = None
if cwd:
log_fn("+ cd %s && \\\n" % shellquote(cwd))
# Our long path escape sequence may confuse cmd.exe, so if the cwd
# is short enough, strip that off.
if is_windows() and (len(cwd) < 250) and cwd.startswith("\\\\?\\"):
cwd = cwd[4:]
log_fn("+ %s\n" % cmd_str)
isinteractive = os.isatty(sys.stdout.fileno())
if isinteractive:
stdout = None
sys.stdout.buffer.flush()
else:
stdout = subprocess.PIPE
try:
p = subprocess.Popen(
cmd,
env=env_dict,
cwd=cwd,
stdout=stdout,
stderr=subprocess.STDOUT,
preexec_fn=preexec_fn,
)
except (TypeError, ValueError, OSError) as exc:
log_fn("error running `%s`: %s" % (cmd_str, exc))
raise RunCommandError(
"%s while running `%s` with env=%r\nos.environ=%r"
% (str(exc), cmd_str, env_dict, os.environ)
)
if not isinteractive:
_pipe_output(p, log_fn)
p.wait()
if p.returncode != 0 and not allow_fail:
raise subprocess.CalledProcessError(p.returncode, cmd)
return p.returncode
if hasattr(select, "poll"):
def _pipe_output(p: subprocess.Popen[bytes], log_fn: Callable[[str], None]) -> None:
"""Read output from p.stdout and call log_fn() with each chunk of data as it
becomes available."""
# Perform non-blocking reads
import fcntl
assert p.stdout is not None
fcntl.fcntl(p.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
poll = select.poll()
poll.register(p.stdout.fileno(), select.POLLIN)
buffer_size = 4096
while True:
poll.poll()
data = p.stdout.read(buffer_size)
if not data:
break
# log_fn() accepts arguments as str (binary in Python 2, unicode in
# Python 3). In Python 3 the subprocess output will be plain bytes,
# and need to be decoded.
if not isinstance(data, str):
data = data.decode("utf-8", errors="surrogateescape")
log_fn(data)
else:
def _pipe_output(p: subprocess.Popen[bytes], log_fn: Callable[[str], None]) -> None:
"""Read output from p.stdout and call log_fn() with each chunk of data as it
becomes available."""
# Perform blocking reads. Use a smaller buffer size to avoid blocking
# for very long when data is available.
assert p.stdout is not None
buffer_size = 64
while True:
# pyre-fixme[16]: Optional type has no attribute `read`.
data = p.stdout.read(buffer_size)
if not data:
break
# log_fn() accepts arguments as str (binary in Python 2, unicode in
# Python 3). In Python 3 the subprocess output will be plain bytes,
# and need to be decoded.
if not isinstance(data, str):
data = data.decode("utf-8", errors="surrogateescape")
log_fn(data)
================================================
FILE: build/fbcode_builder/getdeps/subcmd.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
from __future__ import annotations
import argparse
from collections.abc import Callable
class SubCmd:
NAME: str | None = None
HELP: str | None = None
def run(self, args: argparse.Namespace) -> int:
"""perform the command"""
return 0
def setup_parser(self, parser: argparse.ArgumentParser) -> None:
# Subclasses should override setup_parser() if they have any
# command line options or arguments.
pass
CmdTable: list[type[SubCmd]] = []
def add_subcommands(
parser: argparse._SubParsersAction[argparse.ArgumentParser],
common_args: argparse.ArgumentParser,
cmd_table: list[type[SubCmd]] = CmdTable,
) -> None:
"""Register parsers for the defined commands with the provided parser"""
for cls in cmd_table:
command = cls()
command_parser = parser.add_parser(
# pyre-fixme[6]: For 1st argument expected `str` but got `Optional[str]`.
command.NAME,
help=command.HELP,
parents=[common_args],
)
command.setup_parser(command_parser)
command_parser.set_defaults(func=command.run)
def cmd(
name: str,
help: str | None = None,
cmd_table: list[type[SubCmd]] = CmdTable,
) -> Callable[[type[SubCmd]], type[SubCmd]]:
"""
@cmd() is a decorator that can be used to help define Subcmd instances
Example usage:
@subcmd('list', 'Show the result list')
class ListCmd(Subcmd):
def run(self, args):
# Perform the command actions here...
pass
"""
def wrapper(cls: type[SubCmd]) -> type[SubCmd]:
class SubclassedCmd(cls):
NAME = name
HELP = help
# pyre-fixme[6]: For 1st argument expected `Type[SubCmd]` but got
# `Type[SubclassedCmd]`.
# pyre-fixme[16]: Callable `cmd` has no attribute `wrapper`.
cmd_table.append(SubclassedCmd)
# pyre-fixme[7]: Expected `Type[SubCmd]` but got `Type[SubclassedCmd]`.
return SubclassedCmd
return wrapper
================================================
FILE: build/fbcode_builder/getdeps/test/expr_test.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-unsafe
import unittest
from ..expr import parse_expr
class ExprTest(unittest.TestCase):
def test_equal(self) -> None:
valid_variables = {"foo", "some_var", "another_var"}
e = parse_expr("foo=bar", valid_variables)
self.assertTrue(e.eval({"foo": "bar"}))
self.assertFalse(e.eval({"foo": "not-bar"}))
self.assertFalse(e.eval({"not-foo": "bar"}))
def test_not_equal(self) -> None:
valid_variables = {"foo"}
e = parse_expr("not(foo=bar)", valid_variables)
self.assertFalse(e.eval({"foo": "bar"}))
self.assertTrue(e.eval({"foo": "not-bar"}))
def test_bad_not(self) -> None:
valid_variables = {"foo"}
with self.assertRaises(Exception):
parse_expr("foo=not(bar)", valid_variables)
def test_bad_variable(self) -> None:
valid_variables = {"bar"}
with self.assertRaises(Exception):
parse_expr("foo=bar", valid_variables)
def test_all(self) -> None:
valid_variables = {"foo", "baz"}
e = parse_expr("all(foo = bar, baz = qux)", valid_variables)
self.assertTrue(e.eval({"foo": "bar", "baz": "qux"}))
self.assertFalse(e.eval({"foo": "bar", "baz": "nope"}))
self.assertFalse(e.eval({"foo": "nope", "baz": "nope"}))
def test_any(self) -> None:
valid_variables = {"foo", "baz"}
e = parse_expr("any(foo = bar, baz = qux)", valid_variables)
self.assertTrue(e.eval({"foo": "bar", "baz": "qux"}))
self.assertTrue(e.eval({"foo": "bar", "baz": "nope"}))
self.assertFalse(e.eval({"foo": "nope", "baz": "nope"}))
================================================
FILE: build/fbcode_builder/getdeps/test/fixtures/duplicate/foo
================================================
[manifest]
name = foo
================================================
FILE: build/fbcode_builder/getdeps/test/fixtures/duplicate/subdir/foo
================================================
[manifest]
name = foo
================================================
FILE: build/fbcode_builder/getdeps/test/manifest_test.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-unsafe
import sys
import unittest
from ..load import load_all_manifests, patch_loader
from ..manifest import ManifestParser
class ManifestTest(unittest.TestCase):
def test_missing_section(self) -> None:
with self.assertRaisesRegex(
Exception, "manifest file test is missing required section manifest"
):
ManifestParser("test", "")
def test_missing_name(self) -> None:
with self.assertRaisesRegex(
Exception,
"manifest file test section 'manifest' is missing required field 'name'",
):
ManifestParser(
"test",
"""
[manifest]
""",
)
def test_minimal(self) -> None:
p = ManifestParser(
"test",
"""
[manifest]
name = test
""",
)
self.assertEqual(p.name, "test")
self.assertEqual(p.fbsource_path, None)
def test_minimal_with_fbsource_path(self) -> None:
p = ManifestParser(
"test",
"""
[manifest]
name = test
fbsource_path = fbcode/wat
""",
)
self.assertEqual(p.name, "test")
self.assertEqual(p.fbsource_path, "fbcode/wat")
def test_unknown_field(self) -> None:
with self.assertRaisesRegex(
Exception,
(
"manifest file test section 'manifest' contains "
"unknown field 'invalid.field'"
),
):
ManifestParser(
"test",
"""
[manifest]
name = test
invalid.field = woot
""",
)
def test_invalid_section_name(self) -> None:
with self.assertRaisesRegex(
Exception, "manifest file test contains unknown section 'invalid.section'"
):
ManifestParser(
"test",
"""
[manifest]
name = test
[invalid.section]
foo = bar
""",
)
def test_value_in_dependencies_section(self) -> None:
with self.assertRaisesRegex(
Exception,
(
"manifest file test section 'dependencies' has "
"'foo = bar' but this section doesn't allow "
"specifying values for its entries"
),
):
ManifestParser(
"test",
"""
[manifest]
name = test
[dependencies]
foo = bar
""",
)
def test_invalid_conditional_section_name(self) -> None:
with self.assertRaisesRegex(
Exception,
(
"manifest file test section 'dependencies.=' "
"has invalid conditional: expected "
"identifier found ="
),
):
ManifestParser(
"test",
"""
[manifest]
name = test
[dependencies.=]
""",
)
def test_section_as_args(self) -> None:
p = ManifestParser(
"test",
"""
[manifest]
name = test
[dependencies]
a
b
c
[dependencies.test=on]
foo
""",
)
self.assertEqual(p.get_section_as_args("dependencies"), ["a", "b", "c"])
self.assertEqual(
p.get_section_as_args("dependencies", {"test": "off"}), ["a", "b", "c"]
)
self.assertEqual(
p.get_section_as_args("dependencies", {"test": "on"}),
["a", "b", "c", "foo"],
)
p2 = ManifestParser(
"test",
"""
[manifest]
name = test
[autoconf.args]
--prefix=/foo
--with-woot
""",
)
self.assertEqual(
p2.get_section_as_args("autoconf.args"), ["--prefix=/foo", "--with-woot"]
)
def test_section_as_dict(self) -> None:
p = ManifestParser(
"test",
"""
[manifest]
name = test
[cmake.defines]
foo = bar
[cmake.defines.test=on]
foo = baz
""",
)
self.assertEqual(p.get_section_as_dict("cmake.defines", {}), {"foo": "bar"})
self.assertEqual(
p.get_section_as_dict("cmake.defines", {"test": "on"}), {"foo": "baz"}
)
p2 = ManifestParser(
"test",
"""
[manifest]
name = test
[cmake.defines.test=on]
foo = baz
[cmake.defines]
foo = bar
""",
)
self.assertEqual(
p2.get_section_as_dict("cmake.defines", {"test": "on"}),
{"foo": "bar"},
msg="sections cascade in the order they appear in the manifest",
)
def test_parse_common_manifests(self) -> None:
patch_loader(__name__)
# pyre-fixme[6]: For 1st argument expected `BuildOptions` but got `None`.
manifests = load_all_manifests(None)
self.assertNotEqual(0, len(manifests), msg="parsed some number of manifests")
def test_mismatch_name(self) -> None:
with self.assertRaisesRegex(
Exception,
"filename of the manifest 'foo' does not match the manifest name 'bar'",
):
ManifestParser(
"foo",
"""
[manifest]
name = bar
""",
)
def test_duplicate_manifest(self) -> None:
patch_loader(__name__, "fixtures/duplicate")
with self.assertRaisesRegex(Exception, "found duplicate manifest 'foo'"):
# pyre-fixme[6]: For 1st argument expected `BuildOptions` but got `None`.
load_all_manifests(None)
if sys.version_info < (3, 2):
def assertRaisesRegex(self, *args, **kwargs):
return self.assertRaisesRegex(*args, **kwargs)
================================================
FILE: build/fbcode_builder/getdeps/test/platform_test.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-unsafe
import unittest
from ..platform import HostType
class PlatformTest(unittest.TestCase):
def test_create(self) -> None:
p = HostType()
self.assertNotEqual(p.ostype, None, msg="probed and returned something")
tuple_string = p.as_tuple_string()
round_trip = HostType.from_tuple_string(tuple_string)
self.assertEqual(round_trip, p)
def test_rendering_of_none(self) -> None:
p = HostType(ostype="foo")
self.assertEqual(p.as_tuple_string(), "foo-none-none")
def test_is_methods(self) -> None:
p = HostType(ostype="windows")
self.assertTrue(p.is_windows())
self.assertFalse(p.is_darwin())
self.assertFalse(p.is_linux())
p = HostType(ostype="darwin")
self.assertFalse(p.is_windows())
self.assertTrue(p.is_darwin())
self.assertFalse(p.is_linux())
p = HostType(ostype="linux")
self.assertFalse(p.is_windows())
self.assertFalse(p.is_darwin())
self.assertTrue(p.is_linux())
================================================
FILE: build/fbcode_builder/getdeps/test/retry_test.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-unsafe
import unittest
from unittest.mock import call, MagicMock, patch
from ..buildopts import BuildOptions
from ..errors import TransientFailure
from ..fetcher import ArchiveFetcher
from ..manifest import ManifestParser
class RetryTest(unittest.TestCase):
def _get_build_opts(self) -> BuildOptions:
mock_build_opts = MagicMock(spec=BuildOptions)
mock_build_opts.scratch_dir = "/path/to/scratch_dir"
return mock_build_opts
def _get_manifest(self) -> ManifestParser:
mock_manifest_parser = MagicMock(spec=ManifestParser)
mock_manifest_parser.name = "mock_manifest_parser"
return mock_manifest_parser
def _get_archive_fetcher(self) -> ArchiveFetcher:
return ArchiveFetcher(
build_options=self._get_build_opts(),
manifest=self._get_manifest(),
url="https://github.com/systemd/systemd/archive/refs/tags/v256.7.tar.gz",
sha256="896d76ff65c88f5fd9e42f90d152b0579049158a163431dd77cdc57748b1d7b0",
)
@patch("os.makedirs")
@patch("os.environ.get")
@patch("time.sleep")
@patch("subprocess.run")
def test_no_retries(
self, mock_run, mock_sleep, mock_os_environ_get, mock_makedirs
) -> None:
def custom_makedirs(path, exist_ok=False):
return None
def custom_get(key, default=None):
if key == "GETDEPS_USE_WGET":
return "1"
elif key == "GETDEPS_WGET_ARGS":
return ""
else:
return None
mock_makedirs.side_effect = custom_makedirs
mock_os_environ_get.side_effect = custom_get
mock_sleep.side_effect = None
fetcher = self._get_archive_fetcher()
fetcher._verify_hash = MagicMock(return_value=None)
fetcher._download()
mock_sleep.assert_has_calls([], any_order=False)
mock_run.assert_called_once_with(
[
"wget",
"-O",
"/path/to/scratch_dir/downloads/mock_manifest_parser-v256.7.tar.gz",
"https://github.com/systemd/systemd/archive/refs/tags/v256.7.tar.gz",
],
capture_output=True,
)
@patch("random.random")
@patch("os.makedirs")
@patch("os.environ.get")
@patch("time.sleep")
@patch("subprocess.run")
def test_retries(
self, mock_run, mock_sleep, mock_os_environ_get, mock_makedirs, mock_random
) -> None:
def custom_makedirs(path, exist_ok=False):
return None
def custom_get(key, default=None):
if key == "GETDEPS_USE_WGET":
return "1"
elif key == "GETDEPS_WGET_ARGS":
return ""
else:
return None
mock_random.return_value = 0
mock_run.side_effect = [
IOError(""),
IOError(""),
None,
]
mock_makedirs.side_effect = custom_makedirs
mock_os_environ_get.side_effect = custom_get
mock_sleep.side_effect = None
fetcher = self._get_archive_fetcher()
fetcher._verify_hash = MagicMock(return_value=None)
fetcher._download()
mock_sleep.assert_has_calls([call(2), call(4)], any_order=False)
calls = [
call(
[
"wget",
"-O",
"/path/to/scratch_dir/downloads/mock_manifest_parser-v256.7.tar.gz",
"https://github.com/systemd/systemd/archive/refs/tags/v256.7.tar.gz",
],
capture_output=True,
),
] * 3
mock_run.assert_has_calls(calls, any_order=False)
@patch("random.random")
@patch("os.makedirs")
@patch("os.environ.get")
@patch("time.sleep")
@patch("subprocess.run")
def test_all_retries(
self, mock_run, mock_sleep, mock_os_environ_get, mock_makedirs, mock_random
) -> None:
def custom_makedirs(path, exist_ok=False):
return None
def custom_get(key, default=None):
if key == "GETDEPS_USE_WGET":
return "1"
elif key == "GETDEPS_WGET_ARGS":
return ""
else:
return None
mock_random.return_value = 0
mock_run.side_effect = IOError(
""
)
mock_makedirs.side_effect = custom_makedirs
mock_os_environ_get.side_effect = custom_get
mock_sleep.side_effect = None
fetcher = self._get_archive_fetcher()
fetcher._verify_hash = MagicMock(return_value=None)
with self.assertRaises(TransientFailure):
fetcher._download()
mock_sleep.assert_has_calls(
[call(2), call(4), call(8), call(10)], any_order=False
)
calls = [
call(
[
"wget",
"-O",
"/path/to/scratch_dir/downloads/mock_manifest_parser-v256.7.tar.gz",
"https://github.com/systemd/systemd/archive/refs/tags/v256.7.tar.gz",
],
capture_output=True,
),
] * 5
mock_run.assert_has_calls(calls, any_order=False)
================================================
FILE: build/fbcode_builder/getdeps/test/scratch_test.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-unsafe
import unittest
from ..buildopts import find_existing_win32_subst_for_path
class Win32SubstTest(unittest.TestCase):
def test_no_existing_subst(self) -> None:
self.assertIsNone(
find_existing_win32_subst_for_path(
r"C:\users\alice\appdata\local\temp\fbcode_builder_getdeps",
subst_mapping={},
)
)
self.assertIsNone(
find_existing_win32_subst_for_path(
r"C:\users\alice\appdata\local\temp\fbcode_builder_getdeps",
subst_mapping={"X:\\": r"C:\users\alice\appdata\local\temp\other"},
)
)
def test_exact_match_returns_drive_path(self) -> None:
self.assertEqual(
find_existing_win32_subst_for_path(
r"C:\temp\fbcode_builder_getdeps",
subst_mapping={"X:\\": r"C:\temp\fbcode_builder_getdeps"},
),
"X:\\",
)
self.assertEqual(
find_existing_win32_subst_for_path(
r"C:/temp/fbcode_builder_getdeps",
subst_mapping={"X:\\": r"C:/temp/fbcode_builder_getdeps"},
),
"X:\\",
)
def test_multiple_exact_matches_returns_arbitrary_drive_path(self) -> None:
self.assertIn(
find_existing_win32_subst_for_path(
r"C:\temp\fbcode_builder_getdeps",
subst_mapping={
"X:\\": r"C:\temp\fbcode_builder_getdeps",
"Y:\\": r"C:\temp\fbcode_builder_getdeps",
"Z:\\": r"C:\temp\fbcode_builder_getdeps",
},
),
("X:\\", "Y:\\", "Z:\\"),
)
def test_drive_letter_is_case_insensitive(self) -> None:
self.assertEqual(
find_existing_win32_subst_for_path(
r"C:\temp\fbcode_builder_getdeps",
subst_mapping={"X:\\": r"c:\temp\fbcode_builder_getdeps"},
),
"X:\\",
)
def test_path_components_are_case_insensitive(self) -> None:
self.assertEqual(
find_existing_win32_subst_for_path(
r"C:\TEMP\FBCODE_builder_getdeps",
subst_mapping={"X:\\": r"C:\temp\fbcode_builder_getdeps"},
),
"X:\\",
)
self.assertEqual(
find_existing_win32_subst_for_path(
r"C:\temp\fbcode_builder_getdeps",
subst_mapping={"X:\\": r"C:\TEMP\FBCODE_builder_getdeps"},
),
"X:\\",
)
================================================
FILE: build/fbcode_builder/getdeps/test/strip_marker_test.py
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
# pyre-strict
import os
import tempfile
import unittest
from ..fetcher import filter_strip_marker
from ..manifest import ManifestParser
class ManifestStripMarkerTest(unittest.TestCase):
def test_default_strip_marker(self) -> None:
p = ManifestParser(
"test",
"""
[manifest]
name = test
""",
)
self.assertEqual(p.shipit_strip_marker, "@fb-only")
def test_custom_strip_marker(self) -> None:
p = ManifestParser(
"test",
"""
[manifest]
name = test
shipit_strip_marker = @oss-disable
""",
)
self.assertEqual(p.shipit_strip_marker, "@oss-disable")
class FilterStripMarkerTest(unittest.TestCase):
def _write_temp(self, content: str) -> str:
fd, path = tempfile.mkstemp(suffix=".txt")
os.close(fd)
with open(path, "w") as f:
f.write(content)
return path
def _read(self, path: str) -> str:
with open(path, "r") as f:
return f.read()
def test_single_line_removal(self) -> None:
path = self._write_temp("keep this\nremove this @fb-only\nkeep this too\n")
try:
filter_strip_marker(path, "@fb-only")
self.assertEqual(self._read(path), "keep this\nkeep this too\n")
finally:
os.unlink(path)
def test_block_removal(self) -> None:
content = (
"before\n"
"// @fb-only-start\n"
"secret stuff\n"
"more secret\n"
"// @fb-only-end\n"
"after\n"
)
path = self._write_temp(content)
try:
filter_strip_marker(path, "@fb-only")
self.assertEqual(self._read(path), "before\nafter\n")
finally:
os.unlink(path)
def test_no_marker_present_no_change(self) -> None:
original = "nothing special here\njust plain code\n"
path = self._write_temp(original)
try:
filter_strip_marker(path, "@fb-only")
self.assertEqual(self._read(path), original)
finally:
os.unlink(path)
def test_custom_marker_single_line(self) -> None:
content = "keep\nremove @oss-disable\nkeep too\n"
path = self._write_temp(content)
try:
filter_strip_marker(path, "@oss-disable")
self.assertEqual(self._read(path), "keep\nkeep too\n")
finally:
os.unlink(path)
def test_custom_marker_block(self) -> None:
content = (
"before\n"
"# @oss-disable-start\n"
"internal only\n"
"# @oss-disable-end\n"
"after\n"
)
path = self._write_temp(content)
try:
filter_strip_marker(path, "@oss-disable")
self.assertEqual(self._read(path), "before\nafter\n")
finally:
os.unlink(path)
def test_custom_marker_ignores_default(self) -> None:
"""When using a custom marker, @fb-only lines should be kept."""
content = "keep @fb-only\nremove @oss-disable\nplain\n"
path = self._write_temp(content)
try:
filter_strip_marker(path, "@oss-disable")
self.assertEqual(self._read(path), "keep @fb-only\nplain\n")
finally:
os.unlink(path)
def test_mixed_single_and_block(self) -> None:
content = (
"line1\n"
"line2 @fb-only\n"
"line3\n"
"// @fb-only-start\n"
"block content\n"
"// @fb-only-end\n"
"line4\n"
)
path = self._write_temp(content)
try:
filter_strip_marker(path, "@fb-only")
self.assertEqual(self._read(path), "line1\nline3\nline4\n")
finally:
os.unlink(path)
def test_marker_with_regex_metacharacters(self) -> None:
"""Markers containing regex metacharacters should be escaped properly."""
content = "keep\nremove @fb.only\nkeep too\n"
path = self._write_temp(content)
try:
# With proper escaping, the dot is literal, not a wildcard
filter_strip_marker(path, "@fb.only")
self.assertEqual(self._read(path), "keep\nkeep too\n")
finally:
os.unlink(path)
def test_binary_file_skipped(self) -> None:
"""Binary files that can't be decoded as UTF-8 should be skipped."""
fd, path = tempfile.mkstemp(suffix=".bin")
os.close(fd)
binary_content = b"\x80\x81\x82\xff\xfe"
with open(path, "wb") as f:
f.write(binary_content)
try:
filter_strip_marker(path, "@fb-only")
with open(path, "rb") as f:
self.assertEqual(f.read(), binary_content)
finally:
os.unlink(path)
================================================
FILE: build/fbcode_builder/getdeps.py
================================================
#!/usr/bin/env python3
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import argparse
import json
import os
import shutil
import subprocess
import sys
import tarfile
import tempfile
# We don't import cache.create_cache directly as the facebook
# specific import below may monkey patch it, and we want to
# observe the patched version of this function!
import getdeps.cache as cache_module
from getdeps.buildopts import setup_build_options
from getdeps.dyndeps import create_dyn_dep_munger
from getdeps.errors import TransientFailure
from getdeps.fetcher import (
file_name_is_cmake_file,
is_public_commit,
list_files_under_dir_newer_than_timestamp,
SystemPackageFetcher,
)
from getdeps.load import ManifestLoader
from getdeps.manifest import ManifestParser
from getdeps.platform import HostType
from getdeps.runcmd import check_cmd
from getdeps.subcmd import add_subcommands, cmd, SubCmd
try:
import getdeps.facebook # noqa: F401
except ImportError:
# we don't ship the facebook specific subdir,
# so allow that to fail silently
pass
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "getdeps"))
class UsageError(Exception):
pass
# Shared argument definition for --build-type used by multiple commands
BUILD_TYPE_ARG = {
"help": "Set the build type explicitly: Debug (unoptimized, debug symbols), RelWithDebInfo (optimized with debug symbols, default), MinSizeRel (size-optimized, no debug), or Release (optimized, no debug).",
"choices": ["Debug", "Release", "RelWithDebInfo", "MinSizeRel"],
"action": "store",
"default": "RelWithDebInfo",
}
@cmd("validate-manifest", "parse a manifest and validate that it is correct")
class ValidateManifest(SubCmd):
def run(self, args):
try:
ManifestParser(file_name=args.file_name)
print("OK", file=sys.stderr)
return 0
except Exception as exc:
print("ERROR: %s" % str(exc), file=sys.stderr)
return 1
def setup_parser(self, parser):
parser.add_argument("file_name", help="path to the manifest file")
@cmd("show-host-type", "outputs the host type tuple for the host machine")
class ShowHostType(SubCmd):
def run(self, args):
host = HostType()
print("%s" % host.as_tuple_string())
return 0
class ProjectCmdBase(SubCmd):
def run(self, args):
opts = setup_build_options(args)
if args.current_project is not None:
opts.repo_project = args.current_project
if args.project is None:
if opts.repo_project is None:
raise UsageError(
"no project name specified, and no .projectid file found"
)
if opts.repo_project == "fbsource":
# The fbsource repository is a little special. There is no project
# manifest file for it. A specific project must always be explicitly
# specified when building from fbsource.
raise UsageError(
"no project name specified (required when building in fbsource)"
)
args.project = opts.repo_project
ctx_gen = opts.get_context_generator()
if args.test_dependencies:
ctx_gen.set_value_for_all_projects("test", "on")
if args.enable_tests:
ctx_gen.set_value_for_project(args.project, "test", "on")
else:
ctx_gen.set_value_for_project(args.project, "test", "off")
if opts.shared_libs:
ctx_gen.set_value_for_all_projects("shared_libs", "on")
loader = ManifestLoader(opts, ctx_gen)
self.process_project_dir_arguments(args, loader)
manifest = loader.load_manifest(args.project)
return self.run_project_cmd(args, loader, manifest)
def process_project_dir_arguments(self, args, loader):
def parse_project_arg(arg, arg_type):
parts = arg.split(":")
if len(parts) == 2:
project, path = parts
elif len(parts) == 1:
project = args.project
path = parts[0]
# On Windows path contains colon, e.g. C:\open
elif os.name == "nt" and len(parts) == 3:
project = parts[0]
path = parts[1] + ":" + parts[2]
else:
raise UsageError(
"invalid %s argument; too many ':' characters: %s" % (arg_type, arg)
)
return project, os.path.abspath(path)
# If we are currently running from a project repository,
# use the current repository for the project sources.
build_opts = loader.build_opts
if build_opts.repo_project is not None and build_opts.repo_root is not None:
loader.set_project_src_dir(build_opts.repo_project, build_opts.repo_root)
for arg in args.src_dir:
project, path = parse_project_arg(arg, "--src-dir")
loader.set_project_src_dir(project, path)
for arg in args.build_dir:
project, path = parse_project_arg(arg, "--build-dir")
loader.set_project_build_dir(project, path)
for arg in args.install_dir:
project, path = parse_project_arg(arg, "--install-dir")
loader.set_project_install_dir(project, path)
for arg in args.project_install_prefix:
project, path = parse_project_arg(arg, "--install-prefix")
loader.set_project_install_prefix(project, path)
def setup_parser(self, parser):
parser.add_argument(
"project",
nargs="?",
help=(
"name of the project or path to a manifest "
"file describing the project"
),
)
parser.add_argument(
"--no-tests",
action="store_false",
dest="enable_tests",
default=True,
help="Disable building tests for this project.",
)
parser.add_argument(
"--test-dependencies",
action="store_true",
help="Enable building tests for dependencies as well.",
)
parser.add_argument(
"--current-project",
help="Specify the name of the fbcode_builder manifest file for the "
"current repository. If not specified, the code will attempt to find "
"this in a .projectid file in the repository root.",
)
parser.add_argument(
"--src-dir",
default=[],
action="append",
help="Specify a local directory to use for the project source, "
"rather than fetching it.",
)
parser.add_argument(
"--build-dir",
default=[],
action="append",
help="Explicitly specify the build directory to use for the "
"project, instead of the default location in the scratch path. "
"This only affects the project specified, and not its dependencies.",
)
parser.add_argument(
"--install-dir",
default=[],
action="append",
help="Explicitly specify the install directory to use for the "
"project, instead of the default location in the scratch path. "
"This only affects the project specified, and not its dependencies.",
)
parser.add_argument(
"--project-install-prefix",
default=[],
action="append",
help="Specify the final deployment installation path for a project",
)
self.setup_project_cmd_parser(parser)
def setup_project_cmd_parser(self, parser):
pass
def create_builder(self, loader, manifest):
fetcher = loader.create_fetcher(manifest)
src_dir = fetcher.get_src_dir()
ctx = loader.ctx_gen.get_context(manifest.name)
build_dir = loader.get_project_build_dir(manifest)
inst_dir = loader.get_project_install_dir(manifest)
return manifest.create_builder(
loader.build_opts,
src_dir,
build_dir,
inst_dir,
ctx,
loader,
loader.dependencies_of(manifest),
)
def check_built(self, loader, manifest):
built_marker = os.path.join(
loader.get_project_install_dir(manifest), ".built-by-getdeps"
)
return os.path.exists(built_marker)
class CachedProject:
"""A helper that allows calling the cache logic for a project
from both the build and the fetch code"""
def __init__(self, cache, loader, m):
self.m = m
self.inst_dir = loader.get_project_install_dir(m)
self.project_hash = loader.get_project_hash(m)
self.ctx = loader.ctx_gen.get_context(m.name)
self.loader = loader
self.cache = cache
self.cache_key = "-".join(
(
m.name,
self.ctx.get("os"),
self.ctx.get("distro") or "none",
self.ctx.get("distro_vers") or "none",
self.project_hash,
)
)
self.cache_file_name = self.cache_key + "-buildcache.tgz"
def is_cacheable(self):
"""We only cache third party projects"""
return self.cache and self.m.shipit_project is None
def was_cached(self):
cached_marker = os.path.join(self.inst_dir, ".getdeps-cached-build")
return os.path.exists(cached_marker)
def download(self):
if self.is_cacheable() and not os.path.exists(self.inst_dir):
print("check cache for %s" % self.cache_file_name)
dl_dir = os.path.join(self.loader.build_opts.scratch_dir, "downloads")
if not os.path.exists(dl_dir):
os.makedirs(dl_dir)
try:
target_file_name = os.path.join(dl_dir, self.cache_file_name)
if self.cache.download_to_file(self.cache_file_name, target_file_name):
tf = tarfile.open(target_file_name, "r")
print(
"Extracting %s -> %s..." % (self.cache_file_name, self.inst_dir)
)
tf.extractall(self.inst_dir)
cached_marker = os.path.join(self.inst_dir, ".getdeps-cached-build")
with open(cached_marker, "w") as f:
f.write("\n")
return True
except Exception as exc:
print("%s" % str(exc))
return False
def upload(self):
if self.is_cacheable():
# We can prepare an archive and stick it in LFS
tempdir = tempfile.mkdtemp()
tarfilename = os.path.join(tempdir, self.cache_file_name)
print("Archiving for cache: %s..." % tarfilename)
tf = tarfile.open(tarfilename, "w:gz")
tf.add(self.inst_dir, arcname=".")
tf.close()
try:
self.cache.upload_from_file(self.cache_file_name, tarfilename)
except Exception as exc:
print(
"Failed to upload to cache (%s), continue anyway" % str(exc),
file=sys.stderr,
)
shutil.rmtree(tempdir)
@cmd("fetch", "fetch the code for a given project")
class FetchCmd(ProjectCmdBase):
def setup_project_cmd_parser(self, parser):
parser.add_argument(
"--recursive",
help="fetch the transitive deps also",
action="store_true",
default=False,
)
parser.add_argument(
"--host-type",
help=(
"When recursively fetching, fetch deps for "
"this host type rather than the current system"
),
)
def run_project_cmd(self, args, loader, manifest):
if args.recursive:
projects = loader.manifests_in_dependency_order()
else:
projects = [manifest]
cache = cache_module.create_cache()
for m in projects:
fetcher = loader.create_fetcher(m)
if isinstance(fetcher, SystemPackageFetcher):
# We are guaranteed that if the fetcher is set to
# SystemPackageFetcher then this item is completely
# satisfied by the appropriate system packages
continue
cached_project = CachedProject(cache, loader, m)
if cached_project.download():
continue
inst_dir = loader.get_project_install_dir(m)
built_marker = os.path.join(inst_dir, ".built-by-getdeps")
if os.path.exists(built_marker):
with open(built_marker, "r") as f:
built_hash = f.read().strip()
project_hash = loader.get_project_hash(m)
if built_hash == project_hash:
continue
# We need to fetch the sources
fetcher.update()
@cmd("install-system-deps", "Install system packages to satisfy the deps for a project")
class InstallSysDepsCmd(ProjectCmdBase):
def setup_project_cmd_parser(self, parser):
parser.add_argument(
"--recursive",
help="install the transitive deps also",
action="store_true",
default=False,
)
parser.add_argument(
"--dry-run",
action="store_true",
default=False,
help="Don't install, just print the commands specs we would run",
)
parser.add_argument(
"--os-type",
help="Filter to just this OS type to run",
choices=["linux", "darwin", "windows", "pacman-package"],
action="store",
dest="ostype",
default=None,
)
parser.add_argument(
"--distro",
help="Filter to just this distro to run",
choices=["ubuntu", "centos_stream"],
action="store",
dest="distro",
default=None,
)
parser.add_argument(
"--distro-version",
help="Filter to just this distro version",
action="store",
dest="distrovers",
default=None,
)
def run_project_cmd(self, args, loader, manifest):
if args.recursive:
projects = loader.manifests_in_dependency_order()
else:
projects = [manifest]
rebuild_ctx_gen = False
if args.ostype:
loader.build_opts.host_type.ostype = args.ostype
loader.build_opts.host_type.distro = None
loader.build_opts.host_type.distrovers = None
rebuild_ctx_gen = True
if args.distro:
loader.build_opts.host_type.distro = args.distro
loader.build_opts.host_type.distrovers = None
rebuild_ctx_gen = True
if args.distrovers:
loader.build_opts.host_type.distrovers = args.distrovers
rebuild_ctx_gen = True
if rebuild_ctx_gen:
loader.ctx_gen = loader.build_opts.get_context_generator()
manager = loader.build_opts.host_type.get_package_manager()
all_packages = {}
for m in projects:
ctx = loader.ctx_gen.get_context(m.name)
packages = m.get_required_system_packages(ctx)
for k, v in packages.items():
merged = all_packages.get(k, [])
merged += v
all_packages[k] = merged
cmd_argss = []
if manager == "rpm":
packages = sorted(set(all_packages["rpm"]))
if packages:
cmd_argss.append(
["sudo", "dnf", "install", "-y", "--skip-broken"] + packages
)
elif manager == "deb":
packages = sorted(set(all_packages["deb"]))
if packages:
cmd_argss.append(
[
"sudo",
"--preserve-env=http_proxy",
"apt-get",
"install",
"-y",
]
+ packages
)
cmd_argss.append(["pip", "install", "pex"])
elif manager == "homebrew":
packages = sorted(set(all_packages["homebrew"]))
if packages:
cmd_argss.append(["brew", "install"] + packages)
elif manager == "pacman-package":
packages = sorted(list(set(all_packages["pacman-package"])))
if packages:
cmd_argss.append(["pacman", "-S"] + packages)
else:
host_tuple = loader.build_opts.host_type.as_tuple_string()
print(
f"I don't know how to install any packages on this system {host_tuple}"
)
return
for cmd_args in cmd_argss:
if args.dry_run:
print(" ".join(cmd_args))
else:
check_cmd(cmd_args)
else:
print("no packages to install")
@cmd("list-deps", "lists the transitive deps for a given project")
class ListDepsCmd(ProjectCmdBase):
def run_project_cmd(self, args, loader, manifest):
for m in loader.manifests_in_dependency_order():
print(m.name)
return 0
def setup_project_cmd_parser(self, parser):
parser.add_argument(
"--host-type",
help=(
"Produce the list for the specified host type, "
"rather than that of the current system"
),
)
def clean_dirs(opts):
for d in ["build", "installed", "extracted", "shipit"]:
d = os.path.join(opts.scratch_dir, d)
print("Cleaning %s..." % d)
if os.path.exists(d):
shutil.rmtree(d)
@cmd("clean", "clean up the scratch dir")
class CleanCmd(SubCmd):
def run(self, args):
opts = setup_build_options(args)
clean_dirs(opts)
@cmd("show-scratch-dir", "show the scratch dir")
class ShowScratchDirCmd(SubCmd):
def run(self, args):
opts = setup_build_options(args)
print(opts.scratch_dir)
@cmd("show-build-dir", "print the build dir for a given project")
class ShowBuildDirCmd(ProjectCmdBase):
def run_project_cmd(self, args, loader, manifest):
if args.recursive:
manifests = loader.manifests_in_dependency_order()
else:
manifests = [manifest]
for m in manifests:
inst_dir = loader.get_project_build_dir(m)
print(inst_dir)
def setup_project_cmd_parser(self, parser):
parser.add_argument(
"--recursive",
help="print the transitive deps also",
action="store_true",
default=False,
)
@cmd("show-inst-dir", "print the installation dir for a given project")
class ShowInstDirCmd(ProjectCmdBase):
def run_project_cmd(self, args, loader, manifest):
if args.recursive:
manifests = loader.manifests_in_dependency_order()
else:
manifests = [manifest]
for m in manifests:
fetcher = loader.create_fetcher(m)
if isinstance(fetcher, SystemPackageFetcher):
# We are guaranteed that if the fetcher is set to
# SystemPackageFetcher then this item is completely
# satisfied by the appropriate system packages
continue
inst_dir = loader.get_project_install_dir_respecting_install_prefix(m)
print(inst_dir)
def setup_project_cmd_parser(self, parser):
parser.add_argument(
"--recursive",
help="print the transitive deps also",
action="store_true",
default=False,
)
@cmd("query-paths", "print the paths for tooling to use")
class QueryPathsCmd(ProjectCmdBase):
def run_project_cmd(self, args, loader, manifest):
if args.recursive:
manifests = loader.manifests_in_dependency_order()
else:
manifests = [manifest]
cache = cache_module.create_cache()
for m in manifests:
fetcher = loader.create_fetcher(m)
if isinstance(fetcher, SystemPackageFetcher):
# We are guaranteed that if the fetcher is set to
# SystemPackageFetcher then this item is completely
# satisfied by the appropriate system packages
continue
src_dir = fetcher.get_src_dir()
print(f"{m.name}_SOURCE={src_dir}")
inst_dir = loader.get_project_install_dir_respecting_install_prefix(m)
print(f"{m.name}_INSTALL={inst_dir}")
cached_project = CachedProject(cache, loader, m)
print(f"{m.name}_CACHE_KEY={cached_project.cache_key}")
def setup_project_cmd_parser(self, parser):
parser.add_argument(
"--recursive",
help="print the transitive deps also",
action="store_true",
default=False,
)
@cmd("show-source-dir", "print the source dir for a given project")
class ShowSourceDirCmd(ProjectCmdBase):
def run_project_cmd(self, args, loader, manifest):
if args.recursive:
manifests = loader.manifests_in_dependency_order()
else:
manifests = [manifest]
for m in manifests:
fetcher = loader.create_fetcher(m)
print(fetcher.get_src_dir())
def setup_project_cmd_parser(self, parser):
parser.add_argument(
"--recursive",
help="print the transitive deps also",
action="store_true",
default=False,
)
@cmd("build", "build a given project")
class BuildCmd(ProjectCmdBase):
def run_project_cmd(self, args, loader, manifest):
if args.clean:
clean_dirs(loader.build_opts)
print("Building on %s" % loader.ctx_gen.get_context(args.project))
projects = loader.manifests_in_dependency_order()
cache = cache_module.create_cache() if args.use_build_cache else None
dep_manifests = []
for m in projects:
dep_manifests.append(m)
fetcher = loader.create_fetcher(m)
if args.build_skip_lfs_download and hasattr(fetcher, "skip_lfs_download"):
print("skipping lfs download for %s" % m.name)
fetcher.skip_lfs_download()
if isinstance(fetcher, SystemPackageFetcher):
# We are guaranteed that if the fetcher is set to
# SystemPackageFetcher then this item is completely
# satisfied by the appropriate system packages
continue
if args.clean:
fetcher.clean()
build_dir = loader.get_project_build_dir(m)
inst_dir = loader.get_project_install_dir(m)
if (
m == manifest
and not args.only_deps
or m != manifest
and not args.no_deps
):
print("Assessing %s..." % m.name)
project_hash = loader.get_project_hash(m)
ctx = loader.ctx_gen.get_context(m.name)
built_marker = os.path.join(inst_dir, ".built-by-getdeps")
cached_project = CachedProject(cache, loader, m)
reconfigure, sources_changed = self.compute_source_change_status(
cached_project, fetcher, m, built_marker, project_hash
)
if os.path.exists(built_marker) and not cached_project.was_cached():
# We've previously built this. We may need to reconfigure if
# our deps have changed, so let's check them.
dep_reconfigure, dep_build = self.compute_dep_change_status(
m, built_marker, loader
)
if dep_reconfigure:
reconfigure = True
if dep_build:
sources_changed = True
extra_cmake_defines = (
json.loads(args.extra_cmake_defines)
if args.extra_cmake_defines
else {}
)
extra_b2_args = args.extra_b2_args or []
cmake_targets = args.cmake_target or ["install"]
if sources_changed or reconfigure or not os.path.exists(built_marker):
if os.path.exists(built_marker):
os.unlink(built_marker)
src_dir = fetcher.get_src_dir()
# Prepare builders write out config before the main builder runs
prepare_builders = m.create_prepare_builders(
loader.build_opts,
ctx,
src_dir,
build_dir,
inst_dir,
loader,
dep_manifests,
)
for preparer in prepare_builders:
preparer.prepare(reconfigure=reconfigure)
builder = m.create_builder(
loader.build_opts,
src_dir,
build_dir,
inst_dir,
ctx,
loader,
dep_manifests,
final_install_prefix=loader.get_project_install_prefix(m),
extra_cmake_defines=extra_cmake_defines,
cmake_targets=(cmake_targets if m == manifest else ["install"]),
extra_b2_args=extra_b2_args,
)
builder.build(reconfigure=reconfigure)
# If we are building the project (not dependency) and a specific
# cmake_target (not 'install') has been requested, then we don't
# set the built_marker. This allows subsequent runs of getdeps.py
# for the project to run with different cmake_targets to trigger
# cmake
has_built_marker = False
if not (m == manifest and "install" not in cmake_targets):
os.makedirs(os.path.dirname(built_marker), exist_ok=True)
with open(built_marker, "w") as f:
f.write(project_hash)
has_built_marker = True
# Only populate the cache from continuous build runs, and
# only if we have a built_marker.
if not args.skip_upload and has_built_marker:
if args.schedule_type == "continuous":
cached_project.upload()
elif args.schedule_type == "base_retry":
# Check if on public commit before uploading
if is_public_commit(loader.build_opts):
cached_project.upload()
elif args.verbose:
print("found good %s" % built_marker)
def compute_dep_change_status(self, m, built_marker, loader):
reconfigure = False
sources_changed = False
st = os.lstat(built_marker)
ctx = loader.ctx_gen.get_context(m.name)
dep_list = m.get_dependencies(ctx)
for dep in dep_list:
if reconfigure and sources_changed:
break
dep_manifest = loader.load_manifest(dep)
dep_root = loader.get_project_install_dir(dep_manifest)
for dep_file in list_files_under_dir_newer_than_timestamp(
dep_root, st.st_mtime
):
if os.path.basename(dep_file) == ".built-by-getdeps":
continue
if file_name_is_cmake_file(dep_file):
if not reconfigure:
reconfigure = True
print(
f"Will reconfigure cmake because {dep_file} is newer than {built_marker}"
)
else:
if not sources_changed:
sources_changed = True
print(
f"Will run build because {dep_file} is newer than {built_marker}"
)
if reconfigure and sources_changed:
break
return reconfigure, sources_changed
def compute_source_change_status(
self, cached_project, fetcher, m, built_marker, project_hash
):
reconfigure = False
sources_changed = False
if cached_project.download():
if not os.path.exists(built_marker):
fetcher.update()
else:
check_fetcher = True
if os.path.exists(built_marker):
check_fetcher = False
with open(built_marker, "r") as f:
built_hash = f.read().strip()
if built_hash == project_hash:
if cached_project.is_cacheable():
# We can blindly trust the build status
reconfigure = False
sources_changed = False
else:
# Otherwise, we may have changed the source, so let's
# check in with the fetcher layer
check_fetcher = True
else:
# Some kind of inconsistency with a prior build,
# let's run it again to be sure
os.unlink(built_marker)
reconfigure = True
sources_changed = True
# While we don't need to consult the fetcher for the
# status in this case, we may still need to have eg: shipit
# run in order to have a correct source tree.
fetcher.update()
if check_fetcher:
change_status = fetcher.update()
reconfigure = change_status.build_changed()
sources_changed = change_status.sources_changed()
return reconfigure, sources_changed
def setup_project_cmd_parser(self, parser):
parser.add_argument(
"--clean",
action="store_true",
default=False,
help=(
"Clean up the build and installation area prior to building, "
"causing the projects to be built from scratch"
),
)
parser.add_argument(
"--no-deps",
action="store_true",
default=False,
help=(
"Only build the named project, not its deps. "
"This is most useful after you've built all of the deps, "
"and helps to avoid waiting for relatively "
"slow up-to-date-ness checks"
),
)
parser.add_argument(
"--only-deps",
action="store_true",
default=False,
help=(
"Only build the named project's deps. "
"This is most useful when you want to separate out building "
"of all of the deps and your project"
),
)
parser.add_argument(
"--no-build-cache",
action="store_false",
default=True,
dest="use_build_cache",
help="Do not attempt to use the build cache.",
)
parser.add_argument(
"--cmake-target",
help=("Repeatable argument that specifies targets for cmake build."),
default=[],
action="append",
)
parser.add_argument(
"--extra-b2-args",
help=(
"Repeatable argument that contains extra arguments to pass "
"to b2, which compiles boost. "
"e.g.: 'cxxflags=-fPIC' 'cflags=-fPIC'"
),
action="append",
)
parser.add_argument(
"--free-up-disk",
help="Remove unused tools and clean up intermediate files if possible to maximise space for the build",
action="store_true",
default=False,
)
parser.add_argument("--build-type", **BUILD_TYPE_ARG)
@cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes")
class FixupDeps(ProjectCmdBase):
def run_project_cmd(self, args, loader, manifest):
projects = loader.manifests_in_dependency_order()
# Accumulate the install directories so that the build steps
# can find their dep installation
install_dirs = []
dep_manifests = []
for m in projects:
inst_dir = loader.get_project_install_dir_respecting_install_prefix(m)
install_dirs.append(inst_dir)
dep_manifests.append(m)
if m == manifest:
ctx = loader.ctx_gen.get_context(m.name)
env = loader.build_opts.compute_env_for_install_dirs(
loader, dep_manifests, ctx
)
dep_munger = create_dyn_dep_munger(
loader.build_opts, env, install_dirs, args.strip
)
if dep_munger is None:
print(f"dynamic dependency fixups not supported on {sys.platform}")
else:
dep_munger.process_deps(args.destdir, args.final_install_prefix)
def setup_project_cmd_parser(self, parser):
parser.add_argument("destdir", help="Where to copy the fixed up executables")
parser.add_argument(
"--final-install-prefix", help="specify the final installation prefix"
)
parser.add_argument(
"--strip",
action="store_true",
default=False,
help="Strip debug info while processing executables",
)
@cmd("test", "test a given project")
class TestCmd(ProjectCmdBase):
def run_project_cmd(self, args, loader, manifest):
if not self.check_built(loader, manifest):
print("project %s has not been built" % manifest.name)
return 1
return self.create_builder(loader, manifest).run_tests(
schedule_type=args.schedule_type,
owner=args.test_owner,
test_filter=args.filter,
test_exclude=args.exclude,
retry=args.retry,
no_testpilot=args.no_testpilot,
timeout=args.timeout,
)
def setup_project_cmd_parser(self, parser):
parser.add_argument("--test-owner", help="Owner for testpilot")
parser.add_argument("--filter", help="Only run the tests matching the regex")
parser.add_argument("--exclude", help="Exclude tests matching the regex")
parser.add_argument(
"--retry",
type=int,
default=3,
help="Number of immediate retries for failed tests "
"(noop in continuous and testwarden runs)",
)
parser.add_argument(
"--no-testpilot",
help="Do not use Test Pilot even when available",
action="store_true",
)
parser.add_argument(
"--timeout",
type=int,
default=None,
help="Timeout in seconds for each individual test",
)
parser.add_argument("--build-type", **BUILD_TYPE_ARG)
@cmd(
"debug",
"start a shell in the given project's build dir with the correct environment for running the build",
)
class DebugCmd(ProjectCmdBase):
def run_project_cmd(self, args, loader, manifest):
self.create_builder(loader, manifest).debug(reconfigure=False)
@cmd(
"env",
"print the environment in a shell sourceable format",
)
class EnvCmd(ProjectCmdBase):
def setup_project_cmd_parser(self, parser):
parser.add_argument(
"--os-type",
help="Filter to just this OS type to run",
choices=["linux", "darwin", "windows"],
action="store",
dest="ostype",
default=None,
)
def run_project_cmd(self, args, loader, manifest):
if args.ostype:
loader.build_opts.host_type.ostype = args.ostype
self.create_builder(loader, manifest).printenv(reconfigure=False)
@cmd("generate-github-actions", "generate a GitHub actions configuration")
class GenerateGitHubActionsCmd(ProjectCmdBase):
RUN_ON_ALL = """ [push, pull_request]"""
WORKFLOW_DISPATCH_TMATE = """
workflow_dispatch:
inputs:
tmate_enabled:
description: 'Start a tmate SSH session on failure'
required: false
default: false
type: boolean"""
def run_project_cmd(self, args, loader, manifest):
platforms = [
HostType("linux", "ubuntu", "24"),
HostType("darwin", None, None),
HostType("windows", None, None),
]
for p in platforms:
if args.os_types and p.ostype not in args.os_types:
continue
self.write_job_for_platform(p, args)
def get_run_on(self, args):
if args.run_on_all_branches:
return (
"""
push:
pull_request:"""
+ self.WORKFLOW_DISPATCH_TMATE
)
if args.cron:
if args.cron == "never":
return " {}"
elif args.cron == "workflow_dispatch":
return self.WORKFLOW_DISPATCH_TMATE
else:
return (
f"""
schedule:
- cron: '{args.cron}'"""
+ self.WORKFLOW_DISPATCH_TMATE
)
return (
f"""
push:
branches:
- {args.main_branch}
pull_request:
branches:
- {args.main_branch}"""
+ self.WORKFLOW_DISPATCH_TMATE
)
# TODO: Break up complex function
def write_job_for_platform(self, platform, args): # noqa: C901
build_opts = setup_build_options(args, platform)
ctx_gen = build_opts.get_context_generator()
if args.enable_tests:
ctx_gen.set_value_for_project(args.project, "test", "on")
else:
ctx_gen.set_value_for_project(args.project, "test", "off")
loader = ManifestLoader(build_opts, ctx_gen)
self.process_project_dir_arguments(args, loader)
manifest = loader.load_manifest(args.project)
manifest_ctx = loader.ctx_gen.get_context(manifest.name)
run_tests = (
args.enable_tests
and manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off"
)
rust_version = (
manifest.get("github.actions", "rust_version", ctx=manifest_ctx) or "stable"
)
override_build_type = args.build_type or manifest.get(
"github.actions", "build_type", ctx=manifest_ctx
)
if run_tests:
manifest_ctx.set("test", "on")
run_on = self.get_run_on(args)
tests_arg = "--no-tests "
if run_tests:
tests_arg = ""
# Some projects don't do anything "useful" as a leaf project, only
# as a dep for a leaf project. Check for those here; we don't want
# to waste the effort scheduling them on CI.
# We do this by looking at the builder type in the manifest file
# rather than creating a builder and checking its type because we
# don't know enough to create the full builder instance here.
builder_name = manifest.get("build", "builder", ctx=manifest_ctx)
if builder_name == "nop":
return None
# We want to be sure that we're running things with python 3
# but python versioning is honestly a bit of a frustrating mess.
# `python` may be version 2 or version 3 depending on the system.
# python3 may not be a thing at all!
# Assume an optimistic default
py3 = "python3"
if build_opts.is_linux():
artifacts = "linux"
if args.runs_on:
runs_on = args.runs_on
else:
runs_on = f"ubuntu-{args.ubuntu_version}"
if args.cpu_cores:
runs_on = f"{args.cpu_cores}-core-ubuntu-{args.ubuntu_version}"
elif build_opts.is_windows():
artifacts = "windows"
if args.runs_on:
runs_on = args.runs_on
else:
runs_on = "windows-2022"
# The windows runners are python 3 by default; python2.exe
# is available if needed.
py3 = "python"
else:
artifacts = "mac"
if args.runs_on:
runs_on = args.runs_on
else:
runs_on = "macOS-latest"
os.makedirs(args.output_dir, exist_ok=True)
job_file_prefix = "getdeps_"
if args.job_file_prefix:
job_file_prefix = args.job_file_prefix
output_file = os.path.join(args.output_dir, f"{job_file_prefix}{artifacts}.yml")
if args.job_name_prefix:
job_name = args.job_name_prefix + artifacts.capitalize()
else:
job_name = artifacts
with open(output_file, "w") as out:
# Deliberate line break here because the @ and the generated
# symbols are meaningful to our internal tooling when they
# appear in a single token
out.write("# This file was @")
out.write("generated by getdeps.py\n")
out.write(
f"""
name: {job_name}
on:{run_on}
permissions:
contents: read # to fetch code (actions/checkout)
jobs:
"""
)
getdepscmd = f"{py3} build/fbcode_builder/getdeps.py"
out.write(" build:\n")
out.write(" runs-on: %s\n" % runs_on)
out.write(" steps:\n")
if build_opts.is_windows():
# cmake relies on BOOST_ROOT but GH deliberately don't set it in order
# to avoid versioning issues:
# https://github.com/actions/virtual-environments/issues/319
# Instead, set the version we think we need; this is effectively
# coupled with the boost manifest
# This is the unusual syntax for setting an env var for the rest of
# the steps in a workflow:
# https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/
out.write(" - name: Export boost environment\n")
out.write(
' run: "echo BOOST_ROOT=%BOOST_ROOT_1_83_0% >> %GITHUB_ENV%"\n'
)
out.write(" shell: cmd\n")
out.write(" - name: Fix Git config\n")
out.write(" run: >\n")
out.write(" git config --system core.longpaths true &&\n")
out.write(" git config --system core.autocrlf false &&\n")
# cxx crate needs symlinks enabled
out.write(" git config --system core.symlinks true\n")
# && is not supported on default windows powershell, so use cmd
out.write(" shell: cmd\n")
out.write(" - uses: actions/checkout@v6\n")
build_type_arg = ""
if override_build_type:
build_type_arg = f"--build-type {override_build_type} "
if args.shared_libs:
build_type_arg += "--shared-libs "
if build_opts.free_up_disk:
free_up_disk = "--free-up-disk "
if not build_opts.is_windows():
out.write(" - name: Show disk space at start\n")
out.write(" run: df -h\n")
# remove the unused github supplied android dev tools
out.write(" - name: Free up disk space\n")
out.write(" run: sudo rm -rf /usr/local/lib/android\n")
out.write(" - name: Show disk space after freeing up\n")
out.write(" run: df -h\n")
else:
free_up_disk = ""
allow_sys_arg = ""
if (
build_opts.allow_system_packages
and build_opts.host_type.get_package_manager()
):
sudo_arg = "sudo --preserve-env=http_proxy "
allow_sys_arg = " --allow-system-packages"
if build_opts.host_type.get_package_manager() == "deb":
out.write(" - name: Update system package info\n")
out.write(f" run: {sudo_arg}apt-get update\n")
out.write(" - name: Install system deps\n")
if build_opts.is_darwin():
# brew is installed as regular user
sudo_arg = ""
system_deps_cmd = f"{sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive {manifest.name}"
if build_opts.is_linux() or build_opts.is_freebsd():
system_deps_cmd += f" && {sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive patchelf"
out.write(f" run: {system_deps_cmd}\n")
required_locales = manifest.get(
"github.actions", "required_locales", ctx=manifest_ctx
)
if (
build_opts.host_type.get_package_manager() == "deb"
and required_locales
):
# ubuntu doesn't include this by default
out.write(" - name: Install locale-gen\n")
out.write(f" run: {sudo_arg}apt-get install locales\n")
for loc in required_locales.split():
out.write(f" - name: Ensure {loc} locale present\n")
out.write(f" run: {sudo_arg}locale-gen {loc}\n")
out.write(" - id: paths\n")
out.write(" name: Query paths\n")
if build_opts.is_windows():
out.write(
f" run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name} >> $env:GITHUB_OUTPUT\n"
)
out.write(" shell: pwsh\n")
else:
out.write(
f' run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name} >> "$GITHUB_OUTPUT"\n'
)
projects = loader.manifests_in_dependency_order()
main_repo_url = manifest.get_repo_url(manifest_ctx)
has_same_repo_dep = False
# Add the rust dep which doesn't have a manifest
for m in projects:
if m == manifest:
continue
mbuilder_name = m.get("build", "builder", ctx=manifest_ctx)
if (
m.name == "rust"
or builder_name == "cargo"
or mbuilder_name == "cargo"
):
out.write(f" - name: Install Rust {rust_version.capitalize()}\n")
out.write(f" uses: dtolnay/rust-toolchain@{rust_version}\n")
break
# Normal deps that have manifests
for m in projects:
if m == manifest or m.name == "rust":
continue
ctx = loader.ctx_gen.get_context(m.name)
if m.get_repo_url(ctx) != main_repo_url:
out.write(" - name: Fetch %s\n" % m.name)
out.write(
f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n"
)
out.write(
f" run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
)
for m in projects:
if m == manifest or m.name == "rust":
continue
src_dir_arg = ""
ctx = loader.ctx_gen.get_context(m.name)
if main_repo_url and m.get_repo_url(ctx) == main_repo_url:
# Its in the same repo, so src-dir is also .
src_dir_arg = "--src-dir=. "
has_same_repo_dep = True
if args.use_build_cache and not src_dir_arg:
out.write(f" - name: Restore {m.name} from cache\n")
out.write(f" id: restore_{m.name}\n")
# only need to restore if would build it
out.write(
f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n"
)
out.write(" uses: actions/cache/restore@v4\n")
out.write(" with:\n")
out.write(
f" path: ${{{{ steps.paths.outputs.{m.name}_INSTALL }}}}\n"
)
out.write(
f" key: ${{{{ steps.paths.outputs.{m.name}_CACHE_KEY }}}}-install\n"
)
out.write(" - name: Build %s\n" % m.name)
if not src_dir_arg:
if args.use_build_cache:
out.write(
f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE && ! steps.restore_{m.name}.outputs.cache-hit }}}}\n"
)
else:
out.write(
f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n"
)
out.write(
f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{src_dir_arg}{free_up_disk}--no-tests {m.name}\n"
)
if args.use_build_cache and not src_dir_arg:
out.write(f" - name: Save {m.name} to cache\n")
out.write(" uses: actions/cache/save@v4\n")
out.write(
f" if: ${{{{ steps.paths.outputs.{m.name}_SOURCE && ! steps.restore_{m.name}.outputs.cache-hit }}}}\n"
)
out.write(" with:\n")
out.write(
f" path: ${{{{ steps.paths.outputs.{m.name}_INSTALL }}}}\n"
)
out.write(
f" key: ${{{{ steps.paths.outputs.{m.name}_CACHE_KEY }}}}-install\n"
)
out.write(" - name: Build %s\n" % manifest.name)
project_prefix = ""
if not build_opts.is_windows():
prefix = loader.get_project_install_prefix(manifest) or "/usr/local"
project_prefix = " --project-install-prefix %s:%s" % (
manifest.name,
prefix,
)
# If we have dep from same repo, we already built it and don't want to rebuild it again
no_deps_arg = ""
if has_same_repo_dep:
no_deps_arg = "--no-deps "
out.write(
f" run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{tests_arg}{no_deps_arg}--src-dir=. {manifest.name}{project_prefix}\n"
)
out.write(" - name: Copy artifacts\n")
if build_opts.is_linux():
# Strip debug info from the binaries, but only on linux.
# While the `strip` utility is also available on macOS,
# attempting to strip there results in an error.
# The `strip` utility is not available on Windows.
strip = " --strip"
else:
strip = ""
out.write(
f" run: {getdepscmd}{allow_sys_arg} fixup-dyn-deps{strip} "
f"--src-dir=. {manifest.name} _artifacts/{artifacts}{project_prefix} "
f"--final-install-prefix /usr/local\n"
)
out.write(" - uses: actions/upload-artifact@v6\n")
out.write(" with:\n")
out.write(" name: %s\n" % manifest.name)
out.write(" path: _artifacts\n")
if run_tests:
num_jobs_arg = ""
if args.num_jobs:
num_jobs_arg = f"--num-jobs {args.num_jobs} "
out.write(" - name: Test %s\n" % manifest.name)
out.write(
f" run: {getdepscmd}{allow_sys_arg} test {build_type_arg}{num_jobs_arg}--src-dir=. {manifest.name}{project_prefix}\n"
)
if build_opts.free_up_disk and not build_opts.is_windows():
out.write(" - name: Show disk space at end\n")
out.write(" if: always()\n")
out.write(" run: df -h\n")
out.write(" - name: Setup tmate session\n")
out.write(
" if: failure() && github.event_name == 'workflow_dispatch' && inputs.tmate_enabled\n"
)
out.write(" uses: mxschmitt/action-tmate@v3\n")
def setup_project_cmd_parser(self, parser):
parser.add_argument(
"--disallow-system-packages",
help="Disallow satisfying third party deps from installed system packages",
action="store_true",
default=False,
)
parser.add_argument(
"--output-dir", help="The directory that will contain the yml files"
)
parser.add_argument(
"--run-on-all-branches",
action="store_true",
help="Allow CI to fire on all branches - Handy for testing",
)
parser.add_argument(
"--ubuntu-version", default="24.04", help="Version of Ubuntu to use"
)
parser.add_argument(
"--cpu-cores",
help="Number of CPU cores to use (applicable for Linux OS)",
)
parser.add_argument(
"--runs-on",
help="Allow specifying explicit runs-on: for github actions",
)
parser.add_argument(
"--cron",
help="Specify that the job runs on a cron schedule instead of on pushes. Pass never to disable the action.",
)
parser.add_argument(
"--main-branch",
default="main",
help="Main branch to trigger GitHub Action on",
)
parser.add_argument(
"--os-type",
help="Filter to just this OS type to run",
choices=["linux", "darwin", "windows"],
action="append",
dest="os_types",
default=[],
)
parser.add_argument(
"--job-file-prefix",
type=str,
help="add a prefix to all job file names",
default=None,
)
parser.add_argument(
"--job-name-prefix",
type=str,
help="add a prefix to all job names",
default=None,
)
parser.add_argument(
"--free-up-disk",
help="Remove unused tools and clean up intermediate files if possible to maximise space for the build",
action="store_true",
default=False,
)
parser.add_argument("--build-type", **BUILD_TYPE_ARG)
parser.add_argument(
"--no-build-cache",
action="store_false",
default=True,
dest="use_build_cache",
help="Do not attempt to use the build cache.",
)
def get_arg_var_name(args):
for arg in args:
if arg.startswith("--"):
return arg[2:].replace("-", "_")
raise Exception("unable to determine argument variable name from %r" % (args,))
def parse_args():
# We want to allow common arguments to be specified either before or after
# the subcommand name. In order to do this we add them to the main parser
# and to subcommand parsers. In order for this to work, we need to tell
# argparse that the default value is SUPPRESS, so that the default values
# from the subparser arguments won't override values set by the user from
# the main parser. We maintain our own list of desired defaults in the
# common_defaults dictionary, and manually set those if the argument wasn't
# present at all.
common_args = argparse.ArgumentParser(add_help=False)
common_defaults = {}
def add_common_arg(*args, **kwargs):
var_name = get_arg_var_name(args)
default_value = kwargs.pop("default", None)
common_defaults[var_name] = default_value
kwargs["default"] = argparse.SUPPRESS
common_args.add_argument(*args, **kwargs)
add_common_arg("--scratch-path", help="Where to maintain checkouts and build dirs")
add_common_arg(
"--vcvars-path", default=None, help="Path to the vcvarsall.bat on Windows."
)
add_common_arg(
"--install-prefix",
help=(
"Where the final build products will be installed "
"(default is [scratch-path]/installed)"
),
)
add_common_arg(
"--num-jobs",
type=int,
help=(
"Number of concurrent jobs to use while building. "
"(default=number of cpu cores)"
),
)
add_common_arg(
"--use-shipit",
help="use the real ShipIt instead of the simple shipit transformer",
action="store_true",
default=False,
)
add_common_arg(
"--facebook-internal",
help="Setup the build context as an FB internal build",
action="store_true",
default=None,
)
add_common_arg(
"--no-facebook-internal",
help="Perform a non-FB internal build, even when in an fbsource repository",
action="store_false",
dest="facebook_internal",
)
add_common_arg(
"--shared-libs",
help="Build shared libraries if possible",
action="store_true",
default=False,
)
add_common_arg(
"--extra-cmake-defines",
help=(
"Input json map that contains extra cmake defines to be used "
"when compiling the current project and all its deps. "
'e.g: \'{"CMAKE_CXX_FLAGS": "--bla"}\''
),
)
add_common_arg(
"--allow-system-packages",
help="Allow satisfying third party deps from installed system packages",
action="store_true",
default=False,
)
add_common_arg(
"-v",
"--verbose",
help="Print more output",
action="store_true",
default=False,
)
add_common_arg(
"-su",
"--skip-upload",
help="skip upload steps",
action="store_true",
default=False,
)
add_common_arg(
"--lfs-path",
help="Provide a parent directory for lfs when fbsource is unavailable",
default=None,
)
add_common_arg(
"--build-skip-lfs-download",
action="store_true",
default=False,
help=(
"Download from the URL, rather than LFS. This is useful "
"in cases where the upstream project has uploaded a new "
"version of the archive with a different hash"
),
)
add_common_arg(
"--schedule-type",
nargs="?",
help="Indicates how the build was activated",
)
ap = argparse.ArgumentParser(
description="Get and build dependencies and projects", parents=[common_args]
)
sub = ap.add_subparsers(
# metavar suppresses the long and ugly default list of subcommands on a
# single line. We still render the nicer list below where we would
# have shown the nasty one.
metavar="",
title="Available commands",
help="",
)
add_subcommands(sub, common_args)
args = ap.parse_args()
for var_name, default_value in common_defaults.items():
if not hasattr(args, var_name):
setattr(args, var_name, default_value)
return ap, args
def main():
ap, args = parse_args()
if getattr(args, "func", None) is None:
ap.print_help()
return 0
try:
return args.func(args)
except UsageError as exc:
ap.error(str(exc))
return 1
except TransientFailure as exc:
print("TransientFailure: %s" % str(exc))
# This return code is treated as a retryable transient infrastructure
# error by Facebook's internal CI, rather than eg: a build or code
# related error that needs to be fixed before progress can be made.
return 128
except subprocess.CalledProcessError as exc:
print("%s" % str(exc), file=sys.stderr)
print("!! Failed", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())
================================================
FILE: build/fbcode_builder/manifests/CLI11
================================================
[manifest]
name = CLI11
[download]
url = https://github.com/CLIUtils/CLI11/archive/v2.0.0.tar.gz
sha256 = 2c672f17bf56e8e6223a3bfb74055a946fa7b1ff376510371902adb9cb0ab6a3
[build]
builder = cmake
subdir = CLI11-2.0.0
[cmake.defines]
CLI11_BUILD_TESTS = OFF
CLI11_BUILD_EXAMPLES = OFF
================================================
FILE: build/fbcode_builder/manifests/autoconf
================================================
[manifest]
name = autoconf
[debs]
autoconf
[homebrew]
autoconf
[rpms]
autoconf
[pps]
autoconf
[download]
url = https://ftpmirror.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz
sha256 = 954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969
[build]
builder = autoconf
subdir = autoconf-2.69
================================================
FILE: build/fbcode_builder/manifests/automake
================================================
[manifest]
name = automake
[homebrew]
automake
[debs]
automake
[rpms]
automake
[pps]
automake
[download]
url = https://ftpmirror.gnu.org/gnu/automake/automake-1.16.1.tar.gz
sha256 = 608a97523f97db32f1f5d5615c98ca69326ced2054c9f82e65bade7fc4c9dea8
[build]
builder = autoconf
subdir = automake-1.16.1
[dependencies]
autoconf
================================================
FILE: build/fbcode_builder/manifests/benchmark
================================================
[manifest]
name = benchmark
[download]
url = https://github.com/google/benchmark/archive/refs/tags/v1.8.0.tar.gz
sha256 = ea2e94c24ddf6594d15c711c06ccd4486434d9cf3eca954e2af8a20c88f9f172
[build]
builder = cmake
subdir = benchmark-1.8.0/
[cmake.defines]
BENCHMARK_ENABLE_TESTING=OFF
================================================
FILE: build/fbcode_builder/manifests/blake3
================================================
[manifest]
name = blake3
[download]
url = https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/1.5.1.tar.gz
sha256 = 822cd37f70152e5985433d2c50c8f6b2ec83aaf11aa31be9fe71486a91744f37
[build]
builder = cmake
subdir = BLAKE3-1.5.1/c
================================================
FILE: build/fbcode_builder/manifests/boost
================================================
[manifest]
name = boost
[download.not(os=windows)]
url = https://archives.boost.io/release/1.83.0/source/boost_1_83_0.tar.gz
sha256 = c0685b68dd44cc46574cce86c4e17c0f611b15e195be9848dfd0769a0a207628
[download.os=windows]
url = https://archives.boost.io/release/1.83.0/source/boost_1_83_0.zip
sha256 = c86bd9d9eef795b4b0d3802279419fde5221922805b073b9bd822edecb1ca28e
[preinstalled.env]
# Here we list the acceptable versions that cmake needs a hint to find
BOOST_ROOT_1_69_0
BOOST_ROOT_1_83_0
[debs]
libboost-all-dev
[homebrew]
boost
# Boost cmake detection on homebrew adds this as requirement: https://github.com/Homebrew/homebrew-core/issues/67427#issuecomment-754187345
icu4c
[pps]
boost
[rpms.all(distro=centos_stream,distro_vers=8)]
boost169
boost169-math
boost169-test
boost169-fiber
boost169-graph
boost169-log
boost169-openmpi
boost169-timer
boost169-chrono
boost169-locale
boost169-thread
boost169-atomic
boost169-random
boost169-static
boost169-contract
boost169-date-time
boost169-iostreams
boost169-container
boost169-coroutine
boost169-filesystem
boost169-system
boost169-stacktrace
boost169-regex
boost169-devel
boost169-context
boost169-python3-devel
boost169-type_erasure
boost169-wave
boost169-python3
boost169-serialization
boost169-program-options
[rpms.distro=fedora]
boost-devel
boost-static
[build]
builder = boost
job_weight_mib = 512
patchfile = boost_1_83_0.patch
[b2.args]
--with-atomic
--with-chrono
--with-container
--with-context
--with-contract
--with-coroutine
--with-date_time
--with-exception
--with-fiber
--with-filesystem
--with-graph
--with-graph_parallel
--with-iostreams
--with-locale
--with-log
--with-math
--with-mpi
--with-program_options
--with-python
--with-random
--with-regex
--with-serialization
--with-stacktrace
--with-system
--with-test
--with-thread
--with-timer
--with-type_erasure
[bootstrap.args.os=darwin]
# Not really gcc, but CI puts a broken clang in the PATH, and saying gcc
# here selects the correct one from Xcode.
--with-toolset=gcc
[b2.args.os=linux]
# RHEL hardened gcc is not compatible with PCH
# https://bugzilla.redhat.com/show_bug.cgi?id=1806545
pch=off
[b2.args.os=darwin]
toolset=clang
# Since Xcode 15.3 std::piecewise_construct is only visible in C++17 and later modes
cxxflags="-DBOOST_UNORDERED_HAVE_PIECEWISE_CONSTRUCT=0"
[b2.args.all(os=windows,fb=on)]
toolset=msvc-14.3
================================================
FILE: build/fbcode_builder/manifests/boost-python
================================================
[manifest]
name = boost-python
[download.not(os=windows)]
url = https://archives.boost.io/release/1.83.0/source/boost_1_83_0.tar.gz
sha256 = c0685b68dd44cc46574cce86c4e17c0f611b15e195be9848dfd0769a0a207628
[download.os=windows]
url = https://archives.boost.io/release/1.83.0/source/boost_1_83_0.zip
sha256 = c86bd9d9eef795b4b0d3802279419fde5221922805b073b9bd822edecb1ca28e
[preinstalled.env]
# Here we list the acceptable versions that cmake needs a hint to find
BOOST_ROOT_1_69_0
BOOST_ROOT_1_83_0
[homebrew]
boost
# Boost cmake detection on homebrew adds this as requirement: https://github.com/Homebrew/homebrew-core/issues/67427#issuecomment-754187345
icu4c
[pps]
boost
[rpms.all(distro=centos_stream,distro_vers=8)]
boost169
boost169-math
boost169-test
boost169-fiber
boost169-graph
boost169-log
boost169-openmpi
boost169-timer
boost169-chrono
boost169-locale
boost169-thread
boost169-atomic
boost169-random
boost169-static
boost169-contract
boost169-date-time
boost169-iostreams
boost169-container
boost169-coroutine
boost169-filesystem
boost169-system
boost169-stacktrace
boost169-regex
boost169-devel
boost169-context
boost169-python3-devel
boost169-type_erasure
boost169-wave
boost169-python3
boost169-serialization
boost169-program-options
[rpms.distro=fedora]
boost-devel
boost-static
[build]
builder = boost
job_weight_mib = 512
patchfile = boost_1_83_0.patch
[build.not(os=linux)]
builder = nop
[b2.args]
--with-atomic
--with-chrono
--with-container
--with-context
--with-contract
--with-coroutine
--with-date_time
--with-exception
--with-fiber
--with-filesystem
--with-graph
--with-graph_parallel
--with-iostreams
--with-locale
--with-log
--with-math
--with-mpi
--with-program_options
--with-python
--with-random
--with-regex
--with-serialization
--with-stacktrace
--with-system
--with-test
--with-thread
--with-timer
--with-type_erasure
[bootstrap.args.os=darwin]
# Not really gcc, but CI puts a broken clang in the PATH, and saying gcc
# here selects the correct one from Xcode.
--with-toolset=gcc
[b2.args.os=linux]
# RHEL hardened gcc is not compatible with PCH
# https://bugzilla.redhat.com/show_bug.cgi?id=1806545
pch=off
# Python extensions need -fPIC for static library linking into shared objects
cxxflags="-fPIC"
[b2.args.os=darwin]
toolset=clang
# Since Xcode 15.3 std::piecewise_construct is only visible in C++17 and later modes
cxxflags="-DBOOST_UNORDERED_HAVE_PIECEWISE_CONSTRUCT=0"
[b2.args.all(os=windows,fb=on)]
toolset=msvc-14.3
================================================
FILE: build/fbcode_builder/manifests/bz2
================================================
[manifest]
name = bz2
[debs]
libbz2-dev
bzip2
[homebrew]
bzip2
[rpms]
bzip2-devel
bzip2
[download]
url = https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
sha256 = ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269
[build.not(os=windows)]
builder = make
subdir = bzip2-1.0.8
[make.build_args.os=linux]
# python bz2 support on linux needs dynamic library
-f
Makefile-libbz2_so
[make.install_args]
install
[build.os=windows]
builder = nop
================================================
FILE: build/fbcode_builder/manifests/c-ares
================================================
[manifest]
name = c-ares
[download]
url = https://github.com/c-ares/c-ares/releases/download/v1.34.6/c-ares-1.34.6.tar.gz
sha256 = 912dd7cc3b3e8a79c52fd7fb9c0f4ecf0aaa73e45efda880266a2d6e26b84ef5
[build]
builder = cmake
subdir = c-ares-1.34.6
[cmake.defines]
CARES_STATIC = ON
[cmake.defines.shared_libs=off]
CARES_SHARED = OFF
================================================
FILE: build/fbcode_builder/manifests/cabal
================================================
[manifest]
name = cabal
[download.os=linux]
url = https://downloads.haskell.org/~cabal/cabal-install-3.6.2.0/cabal-install-3.6.2.0-x86_64-linux-deb10.tar.xz
sha256 = 4759b56e9257e02f29fa374a6b25d6cb2f9d80c7e3a55d4f678a8e570925641c
[build]
builder = nop
[install.files]
cabal = bin/cabal
================================================
FILE: build/fbcode_builder/manifests/cachelib
================================================
[manifest]
name = cachelib
fbsource_path = fbcode/cachelib
shipit_project = cachelib
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/cachelib.git
[build]
builder = cmake
subdir = cachelib
job_weight_mib = 2048
[dependencies]
zlib
fizz
fmt
folly
fbthrift
googletest
sparsemap
wangle
zstd
mvfst
numa
libaio
magic_enum
# cachelib also depends on openssl but since the latter requires a platform-
# specific configuration we rely on the folly manifest to provide this
# dependency to avoid duplication.
[dependencies.all(distro=centos_stream,distro_vers=9)]
gcc14
[cmake.defines.all(distro=centos_stream,distro_vers=9)]
CMAKE_C_COMPILER=/opt/rh/gcc-toolset-14/root/usr/bin/gcc
CMAKE_CXX_COMPILER=/opt/rh/gcc-toolset-14/root/usr/bin/g++
[shipit.pathmap]
fbcode/cachelib = cachelib
fbcode/cachelib/public_tld = .
[shipit.strip]
^fbcode/cachelib/examples(/|$)
^fbcode/cachelib/facebook(/|$)
^fbcode/cachelib/public_tld/website/docs/facebook(/|$)
^fbcode/cachelib/public_tld/website/node_modules(/|$)
^fbcode/cachelib/public_tld/website/build(/|$)
================================================
FILE: build/fbcode_builder/manifests/cinderx-3_14
================================================
[manifest]
name = cinderx-3_14
fbsource_path = fbcode/cinderx
shipit_project = facebookincubator/cinderx
[git]
repo_url = https://github.com/facebookincubator/cinderx.git
[build.os=linux]
builder = setup-py
[build.not(os=linux)]
builder = nop
[dependencies]
python-setuptools
python-3_14
[shipit.pathmap]
fbcode/cinderx = cinderx
fbcode/cinderx/oss_toplevel = .
[setup-py.test]
python_script = cinderx/PythonLib/test_cinderx/test_oss_quick.py
[setup-py.env]
CINDERX_ENABLE_PGO=1
CINDERX_ENABLE_LTO=1
================================================
FILE: build/fbcode_builder/manifests/cinderx-main
================================================
# For building CinderX against CPython main.
# Note that externally this can be broken because in that environment we will
# be checking out the head of the CPython repo. However CinderX is only built
# and tested against our internal copy of CPython which updates ~daily, and so
# may be behind CPython head.
[manifest]
name = cinderx-main
fbsource_path = fbcode/cinderx
shipit_project = facebookincubator/cinderx
[git]
repo_url = https://github.com/facebookincubator/cinderx.git
[build.os=linux]
builder = setup-py
[build.not(os=linux)]
builder = nop
[dependencies]
python-setuptools
python-main
[shipit.pathmap]
fbcode/cinderx = cinderx
fbcode/cinderx/oss_toplevel = .
[setup-py.test]
python_script = cinderx/PythonLib/test_cinderx/test_oss_quick.py
[setup-py.env]
CINDERX_ENABLE_PGO=1
CINDERX_ENABLE_LTO=1
================================================
FILE: build/fbcode_builder/manifests/clang
================================================
[manifest]
name = clang
[rpms]
clang15-devel
================================================
FILE: build/fbcode_builder/manifests/clang19
================================================
[manifest]
name = clang19
[debs.os=linux]
clang-19
[rpms.os=linux]
clang
================================================
FILE: build/fbcode_builder/manifests/cmake
================================================
[manifest]
name = cmake
[homebrew]
cmake
# 18.04 cmake is too old
[debs.not(all(distro=ubuntu,distro_vers="18.04"))]
cmake
[rpms]
cmake
[pps]
cmake
[dependencies]
ninja
[download.os=windows]
url = https://github.com/Kitware/CMake/releases/download/v3.20.4/cmake-3.20.4-windows-x86_64.zip
sha256 = 965d2f001c3ca807d288f2b6b15c42b25579a0e73ef12c2a72c95f4c69123638
[download.os=darwin]
url = https://github.com/Kitware/CMake/releases/download/v3.20.4/cmake-3.20.4-macos-universal.tar.gz
sha256 = df90016635e3183834143c6d94607f0804fe9762f7cc6032f6a4afd7c19cd43b
[download.any(os=linux,os=freebsd)]
url = https://github.com/Kitware/CMake/releases/download/v3.20.4/cmake-3.20.4.tar.gz
sha256 = 87a4060298f2c6bb09d479de1400bc78195a5b55a65622a7dceeb3d1090a1b16
[build.os=windows]
builder = nop
subdir = cmake-3.20.4-windows-x86_64
[build.os=darwin]
builder = nop
subdir = cmake-3.20.4-macos-universal
[install.files.os=darwin]
CMake.app/Contents/bin = bin
CMake.app/Contents/share = share
[build.any(os=linux,os=freebsd)]
builder = cmakebootstrap
subdir = cmake-3.20.4
[make.install_args.any(os=linux,os=freebsd)]
install
================================================
FILE: build/fbcode_builder/manifests/cpptoml
================================================
[manifest]
name = cpptoml
[homebrew]
cpptoml
[download]
url = https://github.com/chadaustin/cpptoml/archive/refs/tags/v0.1.2.tar.gz
sha256 = beda37e94f9746874436c8090c045fd80ae6f8a51f7c668c932a2b110a4fc277
[build]
builder = cmake
subdir = cpptoml-0.1.2
[cmake.defines.os=freebsd]
ENABLE_LIBCXX=NO
================================================
FILE: build/fbcode_builder/manifests/double-conversion
================================================
[manifest]
name = double-conversion
[download]
url = https://github.com/google/double-conversion/archive/v3.1.4.tar.gz
sha256 = 95004b65e43fefc6100f337a25da27bb99b9ef8d4071a36a33b5e83eb1f82021
[homebrew]
double-conversion
[debs]
libdouble-conversion-dev
[rpms]
double-conversion
double-conversion-devel
[pps]
double-conversion
[build]
builder = cmake
subdir = double-conversion-3.1.4
================================================
FILE: build/fbcode_builder/manifests/double-conversion-python
================================================
[manifest]
name = double-conversion-python
[download]
url = https://github.com/google/double-conversion/archive/v3.1.4.tar.gz
sha256 = 95004b65e43fefc6100f337a25da27bb99b9ef8d4071a36a33b5e83eb1f82021
[homebrew]
double-conversion
[debs]
libdouble-conversion-dev
[rpms]
double-conversion
double-conversion-devel
[pps]
double-conversion
[build]
builder = cmake
subdir = double-conversion-3.1.4
[build.not(os=linux)]
builder = nop
[cmake.defines]
CMAKE_POSITION_INDEPENDENT_CODE=ON
================================================
FILE: build/fbcode_builder/manifests/eden
================================================
[manifest]
name = eden
fbsource_path = fbcode/eden
shipit_project = eden
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/sapling.git
[github.actions]
run_tests = off
[sandcastle]
run_tests = off
[build]
builder = cmake
[dependencies]
blake3
googletest
folly
fbthrift
fb303
cpptoml
rocksdb
re2
libgit2
pexpect
python-psutil
python-toml
python-filelock
edencommon
rust-shed
[dependencies.fbsource=on]
rust
# macOS ships with sqlite3, and some of the core system
# frameworks require that that version be linked rather
# than the one we might build for ourselves here, so we
# skip building it on macos.
[dependencies.not(os=darwin)]
sqlite3
[dependencies.os=darwin]
osxfuse
[dependencies.not(os=windows)]
# TODO: teach getdeps to compile curl on Windows.
# Enabling curl on Windows requires us to find a way to compile libcurl with
# msvc.
libcurl
# Added so that OSS doesn't see system "python" which is python 2 on darwin and some linux
python
# TODO: teach getdeps to compile lmdb on Windows.
lmdb
[dependencies.test=on]
# sapling CLI is needed to run the tests
sapling
[shipit.pathmap.fb=on]
# for internal builds that use getdeps
fbcode/fb303 = fb303
fbcode/common/rust/shed = common/rust/shed
fbcode/thrift/lib/cpp = thrift/lib/cpp
fbcode/thrift/lib/cpp2 = thrift/lib/cpp2
fbcode/thrift/lib/java = thrift/lib/java
fbcode/thrift/lib/py = thrift/lib/py
fbcode/thrift/lib/python = thrift/lib/python
fbcode/thrift/lib/rust = thrift/lib/rust
[shipit.pathmap]
# Map hostcaps for now as eden C++ includes its .h. Rust-shed should install it
fbcode/common/rust/shed/hostcaps = common/rust/shed/hostcaps
fbcode/configerator/structs/scm/hg = configerator/structs/scm/hg
fbcode/eden/oss = .
fbcode/eden = eden
fbcode/tools/lfs = tools/lfs
[shipit.pathmap.fb=off]
fbcode/eden/fs/public_autocargo = eden/fs
fbcode/eden/scm/public_autocargo = eden/scm
fbcode/common/rust/shed/hostcaps/public_cargo = common/rust/shed/hostcaps
fbcode/configerator/structs/scm/hg/public_autocargo = configerator/structs/scm/hg
[shipit.strip]
^fbcode/eden/addons/.*$
^fbcode/eden/fs/eden-config\.h$
^fbcode/eden/fs/py/eden/config\.py$
^fbcode/eden/hg-server/.*$
^fbcode/eden/mononoke/(?!lfs_protocol)
^fbcode/eden/scm/build/.*$
^fbcode/eden/scm/lib/third-party/rust/.*/Cargo.toml$
^fbcode/eden/website/.*$
^fbcode/eden/.*/\.cargo/.*$
/Cargo\.lock$
\.pyc$
[shipit.strip.fb=off]
^fbcode/common/rust/shed(?!/public_autocargo).*/Cargo\.toml$
^fbcode/configerator/structs/scm/hg(?!/public_autocargo).*/Cargo\.toml$
^fbcode/eden/fs(?!/public_autocargo).*/Cargo\.toml$
^fbcode/eden/scm(?!/public_autocargo|/saplingnative).*/Cargo\.toml$
^.*/facebook/.*$
^.*/fb/.*$
[cmake.defines.all(fb=on,os=windows)]
ENABLE_GIT=OFF
INSTALL_PYTHON_LIB=ON
[cmake.defines.all(not(fb=on),os=windows)]
ENABLE_GIT=OFF
[cmake.defines.fbsource=on]
USE_CARGO_VENDOR=ON
[cmake.defines.fb=on]
IS_FB_BUILD=ON
[depends.environment]
EDEN_VERSION_OVERRIDE
================================================
FILE: build/fbcode_builder/manifests/edencommon
================================================
[manifest]
name = edencommon
fbsource_path = fbcode/eden/common
shipit_project = edencommon
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebookexperimental/edencommon.git
[build]
builder = cmake
[dependencies]
fbthrift
fb303
fmt
folly
gflags
glog
[cmake.defines.test=on]
BUILD_TESTS=ON
[cmake.defines.test=off]
BUILD_TESTS=OFF
[shipit.pathmap]
fbcode/eden/common = eden/common
fbcode/eden/common/oss = .
[shipit.strip]
@README.facebook@
================================================
FILE: build/fbcode_builder/manifests/exprtk
================================================
[manifest]
name = exprtk
[download]
url = https://github.com/ArashPartow/exprtk/archive/refs/tags/0.0.1.tar.gz
sha256 = fb72791c88ae3b3426e14fdad630027715682584daf56b973569718c56e33f28
[build.not(os=windows)]
builder = nop
subdir = exprtk-0.0.1
[install.files]
exprtk.hpp = exprtk.hpp
[dependencies]
================================================
FILE: build/fbcode_builder/manifests/fast_float
================================================
[manifest]
name = fast_float
[download]
url = https://github.com/fastfloat/fast_float/archive/refs/tags/v8.0.0.tar.gz
sha256 = f312f2dc34c61e665f4b132c0307d6f70ad9420185fa831911bc24408acf625d
[build]
builder = cmake
subdir = fast_float-8.0.0
[cmake.defines]
FASTFLOAT_TEST = OFF
FASTFLOAT_SANITIZE = OFF
[debs.not(all(distro=ubuntu,any(distro_vers="18.04",distro_vers="20.04",distro_vers="22.04",distro_vers="24.04")))]
libfast-float-dev
[rpms.distro=fedora]
fast_float-devel
================================================
FILE: build/fbcode_builder/manifests/fatal
================================================
[manifest]
name = fatal
fbsource_path = fbcode/fatal
shipit_project = fatal
[git]
repo_url = https://github.com/facebook/fatal.git
[shipit.pathmap]
fbcode/fatal = fatal
fbcode/fatal/public_tld = .
[build]
builder = nop
subdir = .
[install.files]
fatal/portability.h = fatal/portability.h
fatal/preprocessor.h = fatal/preprocessor.h
fatal/container = fatal/container
fatal/functional = fatal/functional
fatal/math = fatal/math
fatal/string = fatal/string
fatal/type = fatal/type
================================================
FILE: build/fbcode_builder/manifests/fb303
================================================
[manifest]
name = fb303
fbsource_path = fbcode/fb303
shipit_project = fb303
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/fb303.git
[cargo]
cargo_config_file = source/fb303/thrift/.cargo/config.toml
[crate.pathmap]
fb303_core = fb303/thrift/rust
[build]
builder = cmake
[dependencies]
folly
gflags
glog
fbthrift
[cmake.defines.test=on]
BUILD_TESTS=ON
[cmake.defines.test=off]
BUILD_TESTS=OFF
[shipit.pathmap]
fbcode/fb303/github = .
fbcode/fb303/public_autocargo = fb303
fbcode/fb303 = fb303
[shipit.strip]
^fbcode/fb303/(?!public_autocargo).+/Cargo\.toml$
================================================
FILE: build/fbcode_builder/manifests/fboss
================================================
[manifest]
name = fboss
fbsource_path = fbcode/fboss
shipit_project = fboss
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/fboss.git
[build.os=linux]
builder = cmake
# fboss files take a lot of RAM to compile.
job_weight_mib = 3072
[build.not(os=linux)]
builder = nop
[dependencies]
folly
fb303
wangle
fizz
mvfst
fmt
libsodium
googletest
zstd
fatal
fbthrift
iproute2
libusb
libcurl
libnl
libsai
re2
python
yaml-cpp
libyaml
CLI11
exprtk
nlohmann-json
libgpiod
systemd
range-v3
tabulate
gcc12
python-pyyaml
# ShipitPathMap always assume to use directory to fetch the source code.
# If you need to sync the files to fboss github repo, please make changes in
# configerator/source/opensource/shipit_config/facebook/fboss.cconf
[shipit.pathmap]
fbcode/fboss/github = .
fbcode/fboss/common = common
fbcode/fboss = fboss
# NOTE: Although this directory has other thrift files might not be ready for
# opensource, this pathmap should only be used internally `getdeps.py fetch`
fbcode/configerator/structs/neteng/fboss/thrift = configerator/structs/neteng/fboss/thrift
[shipit.strip]
^fbcode/fboss/github/docs/.*
^fbcode/fboss/oss/.*
^fbcode/fboss/github/.github/.*
^fbcode/fboss/github/fboss-image/.*
^fbcode/fboss/github/.pre-commit-config.yaml
^fbcode/fboss/github/requirements-dev.txt
^fbcode/fboss/.llms/.*
[sandcastle]
run_tests = off
================================================
FILE: build/fbcode_builder/manifests/fbthrift
================================================
[manifest]
name = fbthrift
fbsource_path = xplat/thrift
shipit_project = fbthrift
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/fbthrift.git
[cargo]
cargo_config_file = source/thrift/lib/rust/.cargo/config.toml
[crate.pathmap]
fbthrift = thrift/lib/rust
[build]
builder = cmake
job_weight_mib = 2048
[cmake.defines.all(not(os=windows),test=on)]
enable_tests=ON
[cmake.defines.any(os=windows,test=off)]
enable_tests=OFF
[dependencies]
fizz
fmt
folly
googletest
libsodium
wangle
zstd
mvfst
xxhash
# Thrift also depends on openssl but since the latter requires a platform-
# specific configuration we rely on the folly manifest to provide this
# dependency to avoid duplication.
[shipit.pathmap]
xplat/thrift/public_tld = .
xplat/thrift = thrift
[shipit.strip]
^xplat/thrift/thrift-config\.h$
^xplat/thrift/perf/canary.py$
^xplat/thrift/perf/loadtest.py$
^xplat/thrift/.castle/.*
================================================
FILE: build/fbcode_builder/manifests/fbthrift-python
================================================
[manifest]
name = fbthrift-python
fbsource_path = xplat/thrift
shipit_project = fbthrift
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/fbthrift.git
[cargo]
cargo_config_file = source/thrift/lib/rust/.cargo/config.toml
[crate.pathmap]
fbthrift = thrift/lib/rust
[build]
builder = cmake
job_weight_mib = 2048
[build.not(os=linux)]
builder = nop
[cmake.defines.all(not(os=windows),test=on)]
enable_tests=ON
[cmake.defines.any(os=windows,test=off)]
enable_tests=OFF
[cmake.defines.os=linux]
thrift_python=ON
enable_tests=ON
[dependencies]
fizz-python
fmt-python
folly-python
googletest
libsodium
wangle-python
zstd-python
mvfst-python
xxhash
# Thrift also depends on openssl but since the latter requires a platform-
# specific configuration we rely on the folly manifest to provide this
# dependency to avoid duplication.
[dependencies.os=linux]
libaio-python
libevent-python
[shipit.pathmap]
xplat/thrift/public_tld = .
xplat/thrift = thrift
[shipit.strip]
^xplat/thrift/thrift-config\.h$
^xplat/thrift/perf/canary.py$
^xplat/thrift/perf/loadtest.py$
^xplat/thrift/.castle/.*
================================================
FILE: build/fbcode_builder/manifests/fizz
================================================
[manifest]
name = fizz
fbsource_path = fbcode/fizz
shipit_project = fizz
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebookincubator/fizz.git
[build]
builder = cmake
subdir = fizz
[cmake.defines]
BUILD_EXAMPLES = OFF
[cmake.defines.test=on]
BUILD_TESTS = ON
[cmake.defines.all(os=windows, test=on)]
BUILD_TESTS = OFF
[cmake.defines.test=off]
BUILD_TESTS = OFF
[dependencies]
folly
liboqs
libsodium
zlib
zstd
[dependencies.all(test=on, not(os=windows))]
googletest
[shipit.pathmap]
fbcode/fizz/public_tld = .
fbcode/fizz = fizz
================================================
FILE: build/fbcode_builder/manifests/fizz-python
================================================
[manifest]
name = fizz-python
fbsource_path = fbcode/fizz
shipit_project = fizz
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebookincubator/fizz.git
[build]
builder = cmake
subdir = fizz
[build.not(os=linux)]
builder = nop
[cmake.defines]
BUILD_EXAMPLES = OFF
[cmake.defines.os=linux]
CMAKE_POSITION_INDEPENDENT_CODE = ON
BUILD_SHARED_LIBS = ON
[cmake.defines.test=on]
BUILD_TESTS = ON
[cmake.defines.all(os=windows, test=on)]
BUILD_TESTS = OFF
[cmake.defines.test=off]
BUILD_TESTS = OFF
[dependencies]
folly-python
liboqs
libsodium
zlib-python
zstd-python
[dependencies.all(test=on, not(os=windows))]
googletest
[shipit.pathmap]
fbcode/fizz/public_tld = .
fbcode/fizz = fizz
================================================
FILE: build/fbcode_builder/manifests/fmt
================================================
[manifest]
name = fmt
[download]
url = https://github.com/fmtlib/fmt/archive/refs/tags/12.1.0.tar.gz
sha256 = ea7de4299689e12b6dddd392f9896f08fb0777ac7168897a244a6d6085043fea
[build]
builder = cmake
subdir = fmt-12.1.0
[cmake.defines]
FMT_TEST = OFF
FMT_DOC = OFF
[homebrew]
fmt
[rpms.distro=fedora]
fmt-devel
================================================
FILE: build/fbcode_builder/manifests/fmt-python
================================================
[manifest]
name = fmt-python
[download]
url = https://github.com/fmtlib/fmt/archive/refs/tags/12.1.0.tar.gz
sha256 = ea7de4299689e12b6dddd392f9896f08fb0777ac7168897a244a6d6085043fea
[build]
builder = cmake
subdir = fmt-12.1.0
[build.not(os=linux)]
builder = nop
[cmake.defines]
FMT_TEST = OFF
FMT_DOC = OFF
# Build as shared library so Python extensions can find fmt symbols at runtime
# (fmt uses -fvisibility=hidden, so static linking leaves symbols unexported)
BUILD_SHARED_LIBS = ON
[homebrew]
fmt
[rpms.distro=fedora]
fmt-devel
================================================
FILE: build/fbcode_builder/manifests/folly
================================================
[manifest]
name = folly
fbsource_path = fbcode/folly
shipit_project = folly
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/folly.git
[build]
builder = cmake
job_weight_mib = 1024
[dependencies]
gflags
glog
googletest
boost
libdwarf
libevent
libsodium
double-conversion
fast_float
fmt
lz4
snappy
zstd
# no openssl or zlib in the linux case, why?
# these are usually installed on the system
# and are the easiest system deps to pull in.
# In the future we want to be able to express
# that a system dep is sufficient in the manifest
# for eg: openssl and zlib, but for now we don't
# have it.
# macOS doesn't expose the openssl api so we need
# to build our own.
[dependencies.os=darwin]
openssl
# Windows has neither openssl nor zlib, so we get
# to provide both
[dependencies.os=windows]
openssl
zlib
[dependencies.os=linux]
libaio
libiberty
libunwind
# xz depends on autoconf which does not build on
# Windows
[dependencies.not(os=windows)]
xz
[shipit.pathmap]
fbcode/folly/public_tld = .
fbcode/folly = folly
[shipit.strip]
^fbcode/folly/folly-config\.h$
^fbcode/folly/public_tld/build/facebook_.*
[cmake.defines]
BUILD_SHARED_LIBS=OFF
[cmake.defines.not(os=windows)]
BOOST_LINK_STATIC=ON
[cmake.defines.os=freebsd]
LIBDWARF_FOUND=NO
[cmake.defines.test=on]
BUILD_TESTS=ON
BUILD_BENCHMARKS=OFF
[cmake.defines.test=off]
BUILD_TESTS=OFF
BUILD_BENCHMARKS=OFF
================================================
FILE: build/fbcode_builder/manifests/folly-python
================================================
[manifest]
name = folly-python
fbsource_path = fbcode/folly
shipit_project = folly
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/folly.git
[build]
builder = cmake
job_weight_mib = 1024
[build.not(os=linux)]
builder = nop
[dependencies]
gflags
glog
googletest
boost-python
libdwarf-python
libevent-python
libsodium
double-conversion-python
fast_float
fmt-python
lz4-python
snappy
zstd-python
# no openssl or zlib in the linux case, why?
# these are usually installed on the system
# and are the easiest system deps to pull in.
# In the future we want to be able to express
# that a system dep is sufficient in the manifest
# for eg: openssl and zlib, but for now we don't
# have it.
# macOS doesn't expose the openssl api so we need
# to build our own.
[dependencies.os=darwin]
openssl
# Windows has neither openssl nor zlib, so we get
# to provide both
[dependencies.os=windows]
openssl
zlib
[dependencies.os=linux]
libaio-python
libiberty-python
libunwind
# xz depends on autoconf which does not build on
# Windows
[dependencies.not(os=windows)]
xz
[shipit.pathmap]
fbcode/folly/public_tld = .
fbcode/folly = folly
[shipit.strip]
^fbcode/folly/folly-config\.h$
^fbcode/folly/public_tld/build/facebook_.*
[cmake.defines.os=linux]
PYTHON_EXTENSIONS=ON
BUILD_SHARED_LIBS=ON
[cmake.defines.not(os=windows)]
BOOST_LINK_STATIC=ON
[cmake.defines.os=freebsd]
LIBDWARF_FOUND=NO
[cmake.defines.test=on]
BUILD_TESTS=ON
BUILD_BENCHMARKS=OFF
[cmake.defines.test=off]
BUILD_TESTS=OFF
BUILD_BENCHMARKS=OFF
================================================
FILE: build/fbcode_builder/manifests/gcc12
================================================
[manifest]
name = gcc12
[rpms.all(distro=centos_stream,distro_vers=9)]
gcc-toolset-12
================================================
FILE: build/fbcode_builder/manifests/gcc14
================================================
[manifest]
name = gcc14
[rpms.all(distro=centos_stream,distro_vers=9)]
gcc-toolset-14
================================================
FILE: build/fbcode_builder/manifests/gflags
================================================
[manifest]
name = gflags
[download]
url = https://github.com/gflags/gflags/archive/v2.2.2.tar.gz
sha256 = 34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf
[build]
builder = cmake
subdir = gflags-2.2.2
[cmake.defines]
BUILD_SHARED_LIBS = ON
BUILD_STATIC_LIBS = ON
#BUILD_gflags_nothreads_LIB = OFF
BUILD_gflags_LIB = ON
[homebrew]
gflags
[debs]
libgflags-dev
[rpms.distro=fedora]
gflags-devel
================================================
FILE: build/fbcode_builder/manifests/ghc
================================================
[manifest]
name = ghc
[download.os=linux]
url = https://downloads.haskell.org/~ghc/9.2.8/ghc-9.2.8-x86_64-fedora27-linux.tar.xz
sha256 = 845f63cd365317bb764d81025554a2527dbe315d6fa268c9859e21b911bf2d3c
[build]
builder = autoconf
subdir = ghc-9.2.8
build_in_src_dir = true
only_install = true
[make.install_args]
install
================================================
FILE: build/fbcode_builder/manifests/git-lfs
================================================
[manifest]
name = git-lfs
[rpms]
git-lfs
[debs]
git-lfs
[homebrew]
git-lfs
# only used from system packages currently
[build]
builder = nop
================================================
FILE: build/fbcode_builder/manifests/glean
================================================
[manifest]
name = glean
fbsource_path = fbcode/glean
shipit_project = facebookincubator/Glean
use_shipit = true
[shipit.pathmap]
# These are only used by target determinator to trigger builds, the
# real path mappings are in the ShipIt config.
fbcode/glean = glean
fbcode/common/hs = hsthrift
[subprojects]
hsthrift = hsthrift
[dependencies]
cabal
ghc
gflags
glog
folly
rocksdb
xxhash
llvm
clang
re2
[build]
builder = make
[make.build_args]
setup-folly
setup-folly-version
cabal-update
all
glean-hie
glass
glean-clang
EXTRA_GHC_OPTS=-j4 +RTS -A32m -n4m -RTS
CABAL_CONFIG_FLAGS=-f-hack-tests -f-typescript-tests -f-python-tests -f-dotnet-tests -f-go-tests -f-rust-tests -f-java-lsif-tests -f-flow-tests -f-bundled-folly
[make.install_args]
install
[make.test_args]
test
EXTRA_GHC_OPTS=-j4 +RTS -A32m -n4m -RTS
CABAL_CONFIG_FLAGS=-f-hack-tests -f-typescript-tests -f-python-tests -f-dotnet-tests -f-go-tests -f-rust-tests -f-java-lsif-tests -f-flow-tests -f-bundled-folly
================================================
FILE: build/fbcode_builder/manifests/glog
================================================
[manifest]
name = glog
[download]
url = https://github.com/google/glog/archive/v0.5.0.tar.gz
sha256 = eede71f28371bf39aa69b45de23b329d37214016e2055269b3b5e7cfd40b59f5
[build]
builder = cmake
subdir = glog-0.5.0
[dependencies]
gflags
[cmake.defines]
BUILD_SHARED_LIBS=ON
BUILD_TESTING=NO
WITH_PKGCONFIG=ON
[cmake.defines.os=freebsd]
HAVE_TR1_UNORDERED_MAP=OFF
HAVE_TR1_UNORDERED_SET=OFF
[homebrew]
glog
# on ubuntu glog brings in liblzma-dev, which in turn breaks watchman tests
[debs.not(distro=ubuntu)]
libgoogle-glog-dev
[rpms.distro=fedora]
glog-devel
================================================
FILE: build/fbcode_builder/manifests/googletest
================================================
[manifest]
name = googletest
[download]
url = https://github.com/google/googletest/archive/refs/tags/v1.17.0.tar.gz
sha256 = 65fab701d9829d38cb77c14acdc431d2108bfdbf8979e40eb8ae567edf10b27c
[build]
builder = cmake
subdir = googletest-1.17.0
[cmake.defines]
# Everything else defaults to the shared runtime, so tell gtest that
# it should not use its choice of the static runtime
gtest_force_shared_crt=ON
[cmake.defines.os=windows]
BUILD_SHARED_LIBS=ON
[homebrew]
googletest
# packaged googletest is too old
[debs.not(all(distro=ubuntu,any(distro_vers="18.04",distro_vers="20.04",distro_vers="22.04")))]
libgtest-dev
libgmock-dev
[rpms.distro=fedora]
gmock-devel
gtest-devel
================================================
FILE: build/fbcode_builder/manifests/gperf
================================================
[manifest]
name = gperf
[download]
url = https://ftpmirror.gnu.org/gnu/gperf/gperf-3.1.tar.gz
sha256 = 588546b945bba4b70b6a3a616e80b4ab466e3f33024a352fc2198112cdbb3ae2
[build.not(os=windows)]
builder = autoconf
subdir = gperf-3.1
[build.os=windows]
builder = nop
================================================
FILE: build/fbcode_builder/manifests/hexdump
================================================
[manifest]
name = hexdump
[rpms]
util-linux
[debs]
bsdmainutils
# only used from system packages currently
[build]
builder = nop
================================================
FILE: build/fbcode_builder/manifests/hsthrift
================================================
[manifest]
name = hsthrift
fbsource_path = fbcode/common/hs
shipit_project = facebookincubator/hsthrift
use_shipit = true
[shipit.pathmap]
# These are only used by target determinator to trigger builds, the
# real path mappings are in the ShipIt config.
fbcode/common/hs = .
[dependencies]
cabal
ghc
gflags
glog
folly
fbthrift
wangle
fizz
boost
[build]
builder = make
[make.build_args]
setup-folly
setup-meta
cabal-update
all
[make.install_args]
install
[make.test_args]
test
================================================
FILE: build/fbcode_builder/manifests/iproute2
================================================
[manifest]
name = iproute2
[download]
url = https://mirrors.edge.kernel.org/pub/linux/utils/net/iproute2/iproute2-4.12.0.tar.gz
sha256 = 46612a1e2d01bb31932557bccdb1b8618cae9a439dfffc08ef35ed8e197f14ce
[build.os=linux]
builder = iproute2
subdir = iproute2-4.12.0
patchfile = iproute2_oss.patch
[build.not(os=linux)]
builder = nop
================================================
FILE: build/fbcode_builder/manifests/jom
================================================
# jom is compatible with MSVC nmake, but adds the /j argment which
# speeds up openssl build a lot
[manifest]
name = jom
# see https://download.qt.io/official_releases/jom/changelog.txt for latest version
[download.os=windows]
url = https://download.qt.io/official_releases/jom/jom_1_1_4.zip
sha256 = d533c1ef49214229681e90196ed2094691e8c4a0a0bef0b2c901debcb562682b
[build.os=windows]
builder = nop
[install.files.os=windows]
. = bin
================================================
FILE: build/fbcode_builder/manifests/jq
================================================
[manifest]
name = jq
[rpms.distro=fedora]
jq
[homebrew]
jq
[download.not(os=windows)]
# we use jq-1.7+ to get fix for number truncation https://github.com/jqlang/jq/pull/1752
url = https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-1.7.1.tar.gz
sha256 = 478c9ca129fd2e3443fe27314b455e211e0d8c60bc8ff7df703873deeee580c2
[build.not(os=windows)]
builder = autoconf
subdir = jq-1.7.1
[build.os=windows]
builder = nop
[autoconf.args]
# This argument turns off some developers tool and it is recommended in jq's
# README
--disable-maintainer-mode
================================================
FILE: build/fbcode_builder/manifests/katran
================================================
[manifest]
name = katran
fbsource_path = fbcode/katran
shipit_project = katran
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebookincubator/katran.git
[build.not(os=linux)]
builder = nop
[build.os=linux]
builder = cmake
subdir = .
[cmake.defines.test=on]
BUILD_TESTS=ON
[cmake.defines.test=off]
BUILD_TESTS=OFF
[dependencies]
folly
fizz
libbpf
libmnl
zlib
googletest
fmt
[debs]
libssl-dev
[shipit.pathmap]
fbcode/katran/public_root = .
fbcode/katran = katran
[shipit.strip]
^fbcode/katran/facebook
^fbcode/katran/OSS_SYNC
================================================
FILE: build/fbcode_builder/manifests/libaio
================================================
[manifest]
name = libaio
[debs]
libaio-dev
[rpms.distro=centos_stream]
libaio-devel
[download]
url = https://pagure.io/libaio/archive/libaio-0.3.113/libaio-libaio-0.3.113.tar.gz
sha256 = 716c7059703247344eb066b54ecbc3ca2134f0103307192e6c2b7dab5f9528ab
[build]
builder = make
subdir = libaio-libaio-0.3.113
[make.build_args.shared_libs=off]
ENABLE_SHARED=0
[make.install_args]
install
[make.install_args.shared_libs=off]
ENABLE_SHARED=0
================================================
FILE: build/fbcode_builder/manifests/libaio-python
================================================
[manifest]
name = libaio-python
[debs]
libaio-dev
[rpms.distro=centos_stream]
libaio-devel
[download]
url = https://pagure.io/libaio/archive/libaio-0.3.113/libaio-libaio-0.3.113.tar.gz
sha256 = 716c7059703247344eb066b54ecbc3ca2134f0103307192e6c2b7dab5f9528ab
[build]
builder = make
subdir = libaio-libaio-0.3.113
[build.not(os=linux)]
builder = nop
[make.build_args]
CFLAGS=-fPIC -O2
[make.install_args]
install
================================================
FILE: build/fbcode_builder/manifests/libbpf
================================================
[manifest]
name = libbpf
[download]
url = https://github.com/libbpf/libbpf/archive/refs/tags/v1.6.2.tar.gz
sha256 = 16f31349c70764cba8e0fad3725cc9f52f6cf952554326aa0229daaa21ef4fbd
# BPF only builds on linux, so make it a NOP on other platforms
[build.not(os=linux)]
builder = nop
[build.os=linux]
builder = make
subdir = libbpf-1.6.2/src
[make.build_args]
BUILD_STATIC_ONLY=y
# libbpf-0.3 requires uapi headers >= 5.8
[make.install_args]
install
install_uapi_headers
BUILD_STATIC_ONLY=y
[dependencies]
libelf
================================================
FILE: build/fbcode_builder/manifests/libcurl
================================================
[manifest]
name = libcurl
[rpms]
libcurl-devel
libcurl-minimal
[debs]
libcurl4-openssl-dev
[pps]
libcurl-gnutls
[download]
url = https://curl.haxx.se/download/curl-7.65.1.tar.gz
sha256 = 821aeb78421375f70e55381c9ad2474bf279fc454b791b7e95fc83562951c690
[dependencies]
nghttp2
# We use system OpenSSL on Linux (see folly's manifest for details)
[dependencies.not(os=linux)]
openssl
[build.not(os=windows)]
builder = autoconf
subdir = curl-7.65.1
[autoconf.args]
# fboss (which added the libcurl dep) doesn't need ldap so it is disabled here.
# if someone in the future wants to add ldap for something else, it won't hurt
# fboss. However, that would require adding an ldap manifest.
#
# For the same reason, we disable libssh2 and libidn2 which aren't really used
# but would require adding manifests if we don't disable them.
--disable-ldap
--without-libssh2
--without-libidn2
[build.os=windows]
builder = cmake
subdir = curl-7.65.1
================================================
FILE: build/fbcode_builder/manifests/libdwarf
================================================
[manifest]
name = libdwarf
[rpms]
libdwarf-devel
libdwarf
[debs]
libdwarf-dev
[homebrew]
dwarfutils
[download]
url = https://www.prevanders.net/libdwarf-0.9.2.tar.xz
sha256 = 22b66d06831a76f6a062126cdcad3fcc58540b89a1acb23c99f8861f50999ec3
[build]
builder = cmake
subdir = libdwarf-0.9.2
================================================
FILE: build/fbcode_builder/manifests/libdwarf-python
================================================
[manifest]
name = libdwarf-python
[rpms]
libdwarf-devel
libdwarf
[debs]
libdwarf-dev
[homebrew]
dwarfutils
[download]
url = https://www.prevanders.net/libdwarf-0.9.2.tar.xz
sha256 = 22b66d06831a76f6a062126cdcad3fcc58540b89a1acb23c99f8861f50999ec3
[build]
builder = cmake
subdir = libdwarf-0.9.2
[build.not(os=linux)]
builder = nop
[cmake.defines]
CMAKE_POSITION_INDEPENDENT_CODE=ON
================================================
FILE: build/fbcode_builder/manifests/libelf
================================================
[manifest]
name = libelf
[rpms]
elfutils-libelf-devel-static
[debs]
libelf-dev
[pps]
libelf
[download]
url = https://sourceware.org/elfutils/ftp/0.193/elfutils-0.193.tar.bz2
sha256 = 7857f44b624f4d8d421df851aaae7b1402cfe6bcdd2d8049f15fc07d3dde7635
# libelf only makes sense on linux, so make it a NOP on other platforms
[build.not(os=linux)]
builder = nop
[build.os=linux]
builder = autoconf
subdir = elfutils-0.193
================================================
FILE: build/fbcode_builder/manifests/libevent
================================================
[manifest]
name = libevent
[debs]
libevent-dev
[homebrew]
libevent
[rpms]
libevent-devel
[pps]
libevent
# Note that the CMakeLists.txt file is present only in
# git repo and not in the release tarball, so take care
# to use the github generated source tarball rather than
# the explicitly uploaded source tarball
[download]
url = https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz
sha256 = 92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb
[build]
builder = cmake
subdir = libevent-2.1.12-stable
[cmake.defines]
EVENT__DISABLE_TESTS = ON
EVENT__DISABLE_BENCHMARK = ON
EVENT__DISABLE_SAMPLES = ON
EVENT__DISABLE_REGRESS = ON
[cmake.defines.shared_libs=on]
EVENT__BUILD_SHARED_LIBRARIES = ON
[cmake.defines.shared_libs=off]
EVENT__LIBRARY_TYPE = STATIC
[cmake.defines.os=windows]
EVENT__LIBRARY_TYPE = STATIC
[dependencies.not(any(os=linux, os=freebsd))]
openssl
================================================
FILE: build/fbcode_builder/manifests/libevent-python
================================================
[manifest]
name = libevent-python
# NOTE: System packages (debs, rpms) removed because they don't include
# LibeventConfig.cmake which is required by find_package(Libevent REQUIRED CONFIG).
# Building from source ensures CMake config files are present.
[homebrew]
libevent
[pps]
libevent
# Note that the CMakeLists.txt file is present only in
# git repo and not in the release tarball, so take care
# to use the github generated source tarball rather than
# the explicitly uploaded source tarball
[download]
url = https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz
sha256 = 92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb
[build]
builder = cmake
subdir = libevent-2.1.12-stable
[build.not(os=linux)]
builder = nop
[cmake.defines]
EVENT__DISABLE_TESTS = ON
EVENT__DISABLE_BENCHMARK = ON
EVENT__DISABLE_SAMPLES = ON
EVENT__DISABLE_REGRESS = ON
CMAKE_POSITION_INDEPENDENT_CODE=ON
[cmake.defines.shared_libs=on]
EVENT__BUILD_SHARED_LIBRARIES = ON
[cmake.defines.os=windows]
EVENT__LIBRARY_TYPE = STATIC
[dependencies.not(any(os=linux, os=freebsd))]
openssl
================================================
FILE: build/fbcode_builder/manifests/libffi
================================================
[manifest]
name = libffi
[debs]
libffi-dev
[homebrew]
libffi
[rpms]
libffi-devel
libffi
[pps]
libffi
[download]
url = https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz
sha256 = 540fb721619a6aba3bdeef7d940d8e9e0e6d2c193595bc243241b77ff9e93620
[build]
builder = autoconf
subdir = libffi-3.4.2
================================================
FILE: build/fbcode_builder/manifests/libgit2
================================================
[manifest]
name = libgit2
[homebrew]
libgit2
[rpms]
libgit2-devel
[pps]
libgit2
# Ubuntu 18.04 libgit2 has clash with libcurl4-openssl-dev as it depends on
# libcurl4-gnutls-dev. Should be ok from 20.04 again
# There is a description at https://github.com/r-hub/sysreqsdb/issues/77
[debs.not(all(distro=ubuntu,distro_vers="18.04"))]
libgit2-dev
[download]
url = https://github.com/libgit2/libgit2/archive/v0.28.1.tar.gz
sha256 = 0ca11048795b0d6338f2e57717370208c2c97ad66c6d5eac0c97a8827d13936b
[build]
builder = cmake
subdir = libgit2-0.28.1
[cmake.defines]
# Could turn this on if we also wanted to add a manifest for libssh2
USE_SSH = OFF
BUILD_CLAR = OFF
# Have to build shared to work around annoying problems with cmake
# mis-parsing the frameworks required to link this on macos :-/
BUILD_SHARED_LIBS = ON
================================================
FILE: build/fbcode_builder/manifests/libgpiod
================================================
[manifest]
name = libgpiod
[download]
url = https://cdn.kernel.org/pub/software/libs/libgpiod/libgpiod-1.6.tar.xz
sha256 = 62908023d59e8cbb9137ddd14deec50ced862d8f9b8749f288d3dbe7967151ef
[build]
builder = autoconf
subdir = libgpiod-1.6
================================================
FILE: build/fbcode_builder/manifests/libiberty
================================================
[manifest]
name = libiberty
[rpms]
binutils-devel
binutils
[debs.not(all(distro=ubuntu,distro_vers="24.04"))]
binutils-dev
[debs.all(distro=ubuntu,distro_vers="24.04")]
binutils-x86-64-linux-gnu
[download]
url = https://ftpmirror.gnu.org/gnu/binutils/binutils-2.43.tar.xz
sha256 = b53606f443ac8f01d1d5fc9c39497f2af322d99e14cea5c0b4b124d630379365
[dependencies]
zlib
[build]
builder = autoconf
subdir = binutils-2.43/libiberty
patchfile = libiberty_install_pic_lib.patch
# only build the parts needed for demangling
# as we still want to use system linker and assembler etc
[autoconf.args]
--enable-install-libiberty
================================================
FILE: build/fbcode_builder/manifests/libiberty-python
================================================
[manifest]
name = libiberty-python
[rpms]
binutils-devel
binutils
[debs.not(all(distro=ubuntu,distro_vers="24.04"))]
binutils-dev
[debs.all(distro=ubuntu,distro_vers="24.04")]
binutils-x86-64-linux-gnu
[download]
url = https://ftpmirror.gnu.org/gnu/binutils/binutils-2.43.tar.xz
sha256 = b53606f443ac8f01d1d5fc9c39497f2af322d99e14cea5c0b4b124d630379365
[dependencies]
zlib-python
[build]
builder = autoconf
subdir = binutils-2.43/libiberty
patchfile = libiberty_install_pic_lib.patch
[build.not(os=linux)]
builder = nop
# only build the parts needed for demangling
# as we still want to use system linker and assembler etc
[autoconf.args]
--enable-install-libiberty
CFLAGS=-fPIC -O2
PICFLAG=-fPIC
================================================
FILE: build/fbcode_builder/manifests/libibverbs
================================================
[manifest]
name = libibverbs
[debs]
libibverbs-dev
rdma-core
[rpms]
libibverbs
rdma-core-devel
[download]
url = https://github.com/linux-rdma/rdma-core/releases/download/v60.0/rdma-core-60.0.tar.gz
sha256 = 9b1b892e4eaaaa5dfbade07a290fbf5079e39117724fa1ef80d0ad78839328de
[build]
builder = cmake
subdir = rdma-core-60.0
[dependencies]
libnl
[cmake.defines]
NO_MAN_PAGES=1
NO_PYVERBS=1
ENABLE_RESOLVE_NEIGH=0
# Use absolute short path for runtime dir to avoid Unix socket path length limit (108 chars)
CMAKE_INSTALL_RUNDIR=/tmp/ibacm
================================================
FILE: build/fbcode_builder/manifests/libmnl
================================================
[manifest]
name = libmnl
[rpms]
libmnl-devel
# all centos 8 distros are missing this,
# but its in fedora so may be back in a later version
[rpms.not(all(any(distro=centos_stream,distro=centos),distro_vers=8))]
libmnl-static
[debs]
libmnl-dev
[pps]
libmnl
[download]
url = https://www.netfilter.org/pub/libmnl/libmnl-1.0.4.tar.bz2
sha256 = 171f89699f286a5854b72b91d06e8f8e3683064c5901fb09d954a9ab6f551f81
[build.os=linux]
builder = autoconf
subdir = libmnl-1.0.4
================================================
FILE: build/fbcode_builder/manifests/libnl
================================================
[manifest]
name = libnl
[rpms]
libnl3-devel
libnl3
[debs]
libnl-3-dev
libnl-route-3-dev
[pps]
libnl
[download]
url = https://github.com/thom311/libnl/releases/download/libnl3_2_25/libnl-3.2.25.tar.gz
sha256 = 8beb7590674957b931de6b7f81c530b85dc7c1ad8fbda015398bc1e8d1ce8ec5
[build.os=linux]
builder = autoconf
subdir = libnl-3.2.25
================================================
FILE: build/fbcode_builder/manifests/liboqs
================================================
[manifest]
name = liboqs
[download]
url = https://github.com/open-quantum-safe/liboqs/archive/refs/tags/0.12.0.tar.gz
sha256 = df999915204eb1eba311d89e83d1edd3a514d5a07374745d6a9e5b2dd0d59c08
[build]
builder = cmake
subdir = liboqs-0.12.0
[cmake.defines]
OQS_MINIMAL_BUILD = KEM_kyber_512;KEM_kyber_768;KEM_kyber_1024;KEM_ml_kem_512;KEM_ml_kem_768;KEM_ml_kem_1024
[dependencies]
openssl
================================================
FILE: build/fbcode_builder/manifests/libsai
================================================
[manifest]
name = libsai
[download]
url = https://github.com/opencomputeproject/SAI/archive/v1.16.3.tar.gz
sha256 = 5c89cdb6b2e4f1b42ced6b78d43d06d22434ddbf423cdc551f7c2001f12e63d9
[build]
builder = nop
subdir = SAI-1.16.3
[install.files]
inc = include
experimental = experimental
================================================
FILE: build/fbcode_builder/manifests/libsodium
================================================
[manifest]
name = libsodium
[debs]
libsodium-dev
[homebrew]
libsodium
[rpms]
libsodium-devel
libsodium-static
[pps]
libsodium
[download.not(os=windows)]
url = https://github.com/jedisct1/libsodium/releases/download/1.0.20-RELEASE/libsodium-1.0.20.tar.gz
sha256 = ebb65ef6ca439333c2bb41a0c1990587288da07f6c7fd07cb3a18cc18d30ce19
[build.not(os=windows)]
builder = autoconf
subdir = libsodium-1.0.20
[download.os=windows]
url = https://github.com/jedisct1/libsodium/releases/download/1.0.20-RELEASE/libsodium-1.0.20-msvc.zip
sha256 = 2ff97f9e3f5b341bdc808e698057bea1ae454f99e29ff6f9b62e14d0eb1b1baa
[build.os=windows]
builder = nop
[install.files.os=windows]
libsodium/x64/Release/v143/dynamic/libsodium.dll = bin/libsodium.dll
libsodium/x64/Release/v143/dynamic/libsodium.lib = lib/libsodium.lib
libsodium/x64/Release/v143/dynamic/libsodium.exp = lib/libsodium.exp
libsodium/x64/Release/v143/dynamic/libsodium.pdb = lib/libsodium.pdb
libsodium/include = include
[autoconf.args]
--with-pic
================================================
FILE: build/fbcode_builder/manifests/libtool
================================================
[manifest]
name = libtool
[homebrew]
libtool
[rpms]
libtool
[debs]
libtool
[pps]
libtool
[download]
url = https://ftpmirror.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz
sha256 = e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3
[build]
builder = autoconf
subdir = libtool-2.4.6
[dependencies]
automake
[autoconf.args]
--enable-ltdl-install
================================================
FILE: build/fbcode_builder/manifests/libunwind
================================================
[manifest]
name = libunwind
[rpms]
libunwind-devel
libunwind
# on ubuntu this brings in liblzma-dev, which in turn breaks watchman tests
[debs.not(distro=ubuntu)]
libunwind-dev
# The current libunwind v1.8.1 release has compiler issues with aarch64 (https://github.com/libunwind/libunwind/issues/702).
# This more recent libunwind version (based on the latest commit, not a release version) got it fixed.
[download]
url = https://github.com/libunwind/libunwind/archive/f081cf42917bdd5c428b77850b473f31f81767cf.tar.gz
sha256 = 4ff5c335c02d225491d6c885db827fb5fa505fee4e68b4d7e866efc0087e7264
[build]
builder = autoconf
subdir = libunwind-f081cf42917bdd5c428b77850b473f31f81767cf
[autoconf.args]
--with-pic
================================================
FILE: build/fbcode_builder/manifests/libusb
================================================
[manifest]
name = libusb
[debs]
libusb-1.0-0-dev
[homebrew]
libusb
[rpms]
libusb-devel
libusb
[pps]
libusb
[download]
url = https://github.com/libusb/libusb/releases/download/v1.0.22/libusb-1.0.22.tar.bz2
sha256 = 75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157
[build.os=linux]
builder = autoconf
subdir = libusb-1.0.22
[autoconf.args]
# fboss (which added the libusb dep) doesn't need udev so it is disabled here.
# if someone in the future wants to add udev for something else, it won't hurt
# fboss.
--disable-udev
================================================
FILE: build/fbcode_builder/manifests/libyaml
================================================
[manifest]
name = libyaml
[download]
url = https://pyyaml.org/download/libyaml/yaml-0.1.7.tar.gz
sha256 = 8088e457264a98ba451a90b8661fcb4f9d6f478f7265d48322a196cec2480729
[build.os=linux]
builder = autoconf
subdir = yaml-0.1.7
[build.not(os=linux)]
builder = nop
================================================
FILE: build/fbcode_builder/manifests/llvm
================================================
[manifest]
name = llvm
[rpms]
llvm15-devel
================================================
FILE: build/fbcode_builder/manifests/lmdb
================================================
[manifest]
name = lmdb
[build]
builder = make
subdir = lmdb-LMDB_0.9.31/libraries/liblmdb
[download]
url = https://github.com/LMDB/lmdb/archive/refs/tags/LMDB_0.9.31.tar.gz
sha256 = dd70a8c67807b3b8532b3e987b0a4e998962ecc28643e1af5ec77696b081c9b0
[make.build_args]
BUILD_STATIC_ONLY=y
[make.install_args]
install
BUILD_STATIC_ONLY=y
================================================
FILE: build/fbcode_builder/manifests/lz4
================================================
[manifest]
name = lz4
[homebrew]
lz4
[rpms]
lz4-devel
# centos 8 and centos_stream 9 are missing this rpm
[rpms.not(any(all(distro=centos,distro_vers=8),all(distro=centos_stream,distro_vers=9)))]
lz4-static
[debs]
liblz4-dev
[pps]
lz4
[download]
url = https://github.com/lz4/lz4/releases/download/v1.10.0/lz4-1.10.0.tar.gz
sha256 = 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b
[build]
builder = cmake
subdir = lz4-1.10.0/build/cmake
================================================
FILE: build/fbcode_builder/manifests/lz4-python
================================================
[manifest]
name = lz4-python
[homebrew]
lz4
[rpms]
lz4-devel
# centos 8 and centos_stream 9 are missing this rpm
[rpms.not(any(all(distro=centos,distro_vers=8),all(distro=centos_stream,distro_vers=9)))]
lz4-static
[debs]
liblz4-dev
[pps]
lz4
[download]
url = https://github.com/lz4/lz4/releases/download/v1.10.0/lz4-1.10.0.tar.gz
sha256 = 537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b
[build]
builder = cmake
subdir = lz4-1.10.0/build/cmake
[build.not(os=linux)]
builder = nop
[cmake.defines]
CMAKE_POSITION_INDEPENDENT_CODE=ON
================================================
FILE: build/fbcode_builder/manifests/magic_enum
================================================
[manifest]
name = magic_enum
[download]
url = https://github.com/Neargye/magic_enum/releases/download/v0.9.7/magic_enum-v0.9.7.tar.gz
sha256 = c047bc7ca0b76752168140e7ae9a4a30d72bf6530c196fdfbf5105a39d40cc46
[build]
builder = cmake
[cmake.defines]
MAGIC_ENUM_OPT_BUILD_EXAMPLES = OFF
MAGIC_ENUM_OPT_BUILD_TESTS = OFF
MAGIC_ENUM_OPT_INSTALL = ON
================================================
FILE: build/fbcode_builder/manifests/mcrouter
================================================
[manifest]
name = mcrouter
fbsource_path = fbcode/mcrouter
shipit_project = mcrouter
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/mcrouter.git
[dependencies]
folly
wangle
fizz
fbthrift
mvfst
ragel
gflags
glog
boost
libevent
openssl
zlib
double-conversion
[shipit.pathmap]
fbcode/mcrouter/public_tld = .
fbcode/mcrouter = mcrouter
[shipit.strip]
^fbcode/mcrouter/(.*/)?facebook/
[build]
builder = cmake
[cmake.defines]
BUILD_SHARED_LIBS=OFF
[cmake.defines.test=on]
BUILD_TESTS=ON
[cmake.defines.test=off]
BUILD_TESTS=OFF
================================================
FILE: build/fbcode_builder/manifests/mononoke
================================================
[manifest]
name = mononoke
fbsource_path = fbcode/eden
shipit_project = eden
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/sapling.git
[build.not(os=windows)]
builder = cargo
[build.os=windows]
# building Mononoke on windows is not supported
builder = nop
[cargo]
build_doc = true
workspace_dir = eden/mononoke
[github.actions]
rust_version = 1.91
build_type = MinSizeRel
[shipit.pathmap]
fbcode/configerator/structs/scm/hg = configerator/structs/scm/hg
fbcode/configerator/structs/scm/hg/public_autocargo = configerator/structs/scm/hg
fbcode/configerator/structs/scm/mononoke/public_autocargo = configerator/structs/scm/mononoke
fbcode/configerator/structs/scm/mononoke = configerator/structs/scm/mononoke
fbcode/eden/oss = .
fbcode/eden = eden
fbcode/eden/fs/public_autocargo = eden/fs
fbcode/eden/mononoke/public_autocargo = eden/mononoke
fbcode/eden/scm/public_autocargo = eden/scm
fbcode/tools/lfs = tools/lfs
tools/rust/ossconfigs = .
[shipit.strip]
^fbcode/configerator/structs/scm/hg(?!/public_autocargo).*/Cargo\.toml$
^fbcode/configerator/structs/scm/mononoke(?!/public_autocargo).*/Cargo\.toml$
^fbcode/eden/fs(?!/public_autocargo).*/Cargo\.toml$
^fbcode/eden/scm/lib/third-party/rust/.*/Cargo\.toml$
^fbcode/eden/mononoke(?!/public_autocargo).*/Cargo\.toml$
# strip other scm code unrelated to mononoke to prevent triggering unnecessary checks
^fbcode/eden(?!/mononoke|/scm/(lib|public_autocargo))/.*$
^.*/facebook/.*$
^.*/fb/.*$
[dependencies]
fb303
fbthrift
rust-shed
[dependencies.fb=on]
rust
================================================
FILE: build/fbcode_builder/manifests/mononoke_integration
================================================
[manifest]
name = mononoke_integration
fbsource_path = fbcode/eden
shipit_project = eden
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/sapling.git
[build.not(os=windows)]
builder = make
subdir = eden/mononoke/tests/integration
[build.os=windows]
# building Mononoke on windows is not supported
builder = nop
[make.build_args]
build-getdeps
[make.install_args]
install-getdeps
[make.test_args]
test-getdeps
[shipit.pathmap]
fbcode/eden/mononoke/tests/integration = eden/mononoke/tests/integration
[shipit.strip]
^.*/facebook/.*$
^.*/fb/.*$
[dependencies]
git-lfs
jq
mononoke
nmap
python
python-click
ripgrep
sapling
tree
zstd
[dependencies.os=linux]
sqlite3
================================================
FILE: build/fbcode_builder/manifests/moxygen
================================================
[manifest]
name = moxygen
fbsource_path = fbcode/moxygen
shipit_project = moxygen
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebookexperimental/moxygen.git
[build.os=windows]
builder = nop
[build]
builder = cmake
subdir = .
job_weight_mib = 3072
rewrite_includes = true
[cmake.defines.test=on]
BUILD_TESTS = ON
[cmake.defines.test=off]
BUILD_TESTS = OFF
[dependencies]
folly
fizz
wangle
mvfst
proxygen
[dependencies.test=on]
googletest
[shipit.pathmap]
fbcode/ti/experimental/moxygen/project_root = .
fbcode/ti/experimental/moxygen = moxygen
================================================
FILE: build/fbcode_builder/manifests/mvfst
================================================
[manifest]
name = mvfst
fbsource_path = fbcode/quic
shipit_project = mvfst
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/mvfst.git
[build]
builder = cmake
subdir = .
[cmake.defines.test=on]
BUILD_TESTS = ON
[cmake.defines.all(os=windows, test=on)]
BUILD_TESTS = OFF
[cmake.defines.test=off]
BUILD_TESTS = OFF
[dependencies]
folly
fizz
[dependencies.all(test=on, not(os=windows))]
googletest
[shipit.pathmap]
fbcode/quic/public_root = .
fbcode/quic = quic
================================================
FILE: build/fbcode_builder/manifests/mvfst-python
================================================
[manifest]
name = mvfst-python
fbsource_path = fbcode/quic
shipit_project = mvfst
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/mvfst.git
[build]
builder = cmake
subdir = .
[build.not(os=linux)]
builder = nop
[cmake.defines.test=on]
BUILD_TESTS = ON
[cmake.defines.all(os=windows, test=on)]
BUILD_TESTS = OFF
[cmake.defines.test=off]
BUILD_TESTS = OFF
[cmake.defines.os=linux]
CMAKE_POSITION_INDEPENDENT_CODE = ON
BUILD_SHARED_LIBS = ON
[dependencies]
folly-python
fizz-python
[dependencies.all(test=on, not(os=windows))]
googletest
[shipit.pathmap]
fbcode/quic/public_root = .
fbcode/quic = quic
================================================
FILE: build/fbcode_builder/manifests/ncurses
================================================
[manifest]
name = ncurses
[debs]
libncurses-dev
[homebrew]
ncurses
[rpms]
ncurses-devel
[download]
url = https://ftpmirror.gnu.org/gnu/ncurses/ncurses-6.3.tar.gz
sha256 = 97fc51ac2b085d4cde31ef4d2c3122c21abc217e9090a43a30fc5ec21684e059
[build.not(os=windows)]
builder = autoconf
subdir = ncurses-6.3
[autoconf.args]
--without-cxx-binding
--without-ada
[autoconf.args.os=linux]
--enable-shared
--with-shared
[build.os=windows]
builder = nop
================================================
FILE: build/fbcode_builder/manifests/nghttp2
================================================
[manifest]
name = nghttp2
[rpms]
libnghttp2-devel
libnghttp2
[debs]
libnghttp2-dev
[pps]
libnghttp2
[download]
url = https://github.com/nghttp2/nghttp2/releases/download/v1.47.0/nghttp2-1.47.0.tar.gz
sha256 = 62f50f0e9fc479e48b34e1526df8dd2e94136de4c426b7680048181606832b7c
[build]
builder = autoconf
subdir = nghttp2-1.47.0
[autoconf.args]
--enable-lib-only
--disable-dependency-tracking
================================================
FILE: build/fbcode_builder/manifests/ninja
================================================
[manifest]
name = ninja
[debs]
ninja-build
[homebrew]
ninja
[rpms]
ninja-build
[pps]
ninja
[download.os=windows]
url = https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-win.zip
sha256 = f550fec705b6d6ff58f2db3c374c2277a37691678d6aba463adcbb129108467a
[build.os=windows]
builder = nop
[install.files.os=windows]
ninja.exe = bin/ninja.exe
[download.not(os=windows)]
url = https://github.com/ninja-build/ninja/archive/v1.12.1.tar.gz
sha256 = 821bdff48a3f683bc4bb3b6f0b5fe7b2d647cf65d52aeb63328c91a6c6df285a
[build.not(os=windows)]
builder = ninja_bootstrap
subdir = ninja-1.12.1
================================================
FILE: build/fbcode_builder/manifests/nlohmann-json
================================================
[manifest]
name = nlohmann-json
[download]
url = https://github.com/nlohmann/json/archive/refs/tags/v3.10.5.tar.gz
sha256 = 5daca6ca216495edf89d167f808d1d03c4a4d929cef7da5e10f135ae1540c7e4
[dependencies]
[build]
builder = cmake
subdir = json-3.10.5
================================================
FILE: build/fbcode_builder/manifests/nmap
================================================
[manifest]
name = nmap
[rpms]
nmap
nmap-ncat
[debs]
nmap
# 18.04 combines ncat into the nmap package, newer need the separate one
[debs.not(all(distro=ubuntu,distro_vers="18.04"))]
ncat
[download.not(os=windows)]
url = https://api.github.com/repos/nmap/nmap/tarball/ef8213a36c2e89233c806753a57b5cd473605408
sha256 = eda39e5a8ef4964fac7db16abf91cc11ff568eac0fa2d680b0bfa33b0ed71f4a
[build.not(os=windows)]
builder = autoconf
subdir = nmap-nmap-ef8213a
build_in_src_dir = true
[build.os=windows]
builder = nop
[autoconf.args]
# Without this option the build was filing to find some third party libraries
# that we don't need
enable_rdma=no
================================================
FILE: build/fbcode_builder/manifests/numa
================================================
[manifest]
name = numa
[download]
url = https://github.com/numactl/numactl/releases/download/v2.0.19/numactl-2.0.19.tar.gz
sha256 = f2672a0381cb59196e9c246bf8bcc43d5568bc457700a697f1a1df762b9af884
[build]
builder = autoconf
subdir = numactl-2.0.19
[rpms.distro=centos_stream]
numactl-devel
================================================
FILE: build/fbcode_builder/manifests/openr
================================================
[manifest]
name = openr
fbsource_path = facebook/openr
shipit_project = openr
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/openr.git
[build.os=linux]
builder = cmake
# openr files take a lot of RAM to compile.
job_weight_mib = 3072
[build.not(os=linux)]
# boost.fiber is required and that is not available on macos.
builder = nop
[dependencies]
boost
fb303
fbthrift
folly
googletest
re2
range-v3
[cmake.defines.test=on]
BUILD_TESTS=ON
ADD_ROOT_TESTS=OFF
[cmake.defines.test=off]
BUILD_TESTS=OFF
[shipit.pathmap]
fbcode/openr = openr
fbcode/openr/public_tld = .
================================================
FILE: build/fbcode_builder/manifests/openssl
================================================
[manifest]
name = openssl
[debs]
libssl-dev
[homebrew]
openssl
# on homebrew need the matching curl and ca-
[rpms]
openssl
openssl-devel
openssl-libs
[pps]
openssl
# no need to download on the systems where we always use the system libs
[download.not(any(os=linux, os=freebsd))]
# match the openssl version packages in ubuntu LTS folly current supports
url = https://www.openssl.org/source/openssl-3.0.15.tar.gz
sha256 = 23c666d0edf20f14249b3d8f0368acaee9ab585b09e1de82107c66e1f3ec9533
# We use the system openssl on these platforms even without --allow-system-packages
[build.any(os=linux, os=freebsd)]
builder = nop
[build.not(any(os=linux, os=freebsd))]
builder = openssl
subdir = openssl-3.0.15
[dependencies.os=windows]
jom
perl
================================================
FILE: build/fbcode_builder/manifests/osxfuse
================================================
[manifest]
name = osxfuse
[download]
url = https://github.com/osxfuse/osxfuse/archive/osxfuse-3.8.3.tar.gz
sha256 = 93bab6731bdfe8dc1ef069483437270ce7fe5a370f933d40d8d0ef09ba846c0c
[build]
builder = nop
[install.files]
osxfuse-osxfuse-3.8.3/common = include
================================================
FILE: build/fbcode_builder/manifests/patchelf
================================================
[manifest]
name = patchelf
[rpms]
patchelf
[debs]
patchelf
[pps]
patchelf
[download]
url = https://github.com/NixOS/patchelf/archive/0.10.tar.gz
sha256 = b3cb6bdedcef5607ce34a350cf0b182eb979f8f7bc31eae55a93a70a3f020d13
[build]
builder = autoconf
subdir = patchelf-0.10
================================================
FILE: build/fbcode_builder/manifests/pcre2
================================================
[manifest]
name = pcre2
[homebrew]
pcre2
[rpms]
pcre2-devel
pcre-static
[debs]
libpcre2-dev
[download]
url = https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.40/pcre2-10.40.tar.bz2
sha256 = 14e4b83c4783933dc17e964318e6324f7cae1bc75d8f3c79bc6969f00c159d68
[build]
builder = cmake
subdir = pcre2-10.40
================================================
FILE: build/fbcode_builder/manifests/perl
================================================
[manifest]
name = perl
[download.os=windows]
url = https://strawberryperl.com/download/5.28.1.1/strawberry-perl-5.28.1.1-64bit-portable.zip
sha256 = 935c95ba096fa11c4e1b5188732e3832d330a2a79e9882ab7ba8460ddbca810d
[build.os=windows]
builder = nop
subdir = perl
================================================
FILE: build/fbcode_builder/manifests/pexpect
================================================
[manifest]
name = pexpect
[download]
url = https://files.pythonhosted.org/packages/0e/3e/377007e3f36ec42f1b84ec322ee12141a9e10d808312e5738f52f80a232c/pexpect-4.7.0-py2.py3-none-any.whl
sha256 = 2094eefdfcf37a1fdbfb9aa090862c1a4878e5c7e0e7e7088bdb511c558e5cd1
[build]
builder = python-wheel
[dependencies]
python-ptyprocess
================================================
FILE: build/fbcode_builder/manifests/proxygen
================================================
[manifest]
name = proxygen
fbsource_path = fbcode/proxygen
shipit_project = proxygen
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/proxygen.git
[build.os=windows]
builder = nop
[build]
builder = cmake
subdir = .
job_weight_mib = 3072
[cmake.defines.test=on]
BUILD_TESTS = ON
[cmake.defines.test=off]
BUILD_TESTS = OFF
[dependencies]
zlib
gperf
folly
fizz
wangle
mvfst
c-ares
[dependencies.test=on]
googletest
[shipit.pathmap]
fbcode/proxygen/public_tld = .
fbcode/proxygen = proxygen
================================================
FILE: build/fbcode_builder/manifests/python
================================================
[manifest]
name = python
[homebrew]
python@3.10
# sapling needs match statements with arrive in python 3.12 in centos 10
[rpms.not(all(distro=centos_stream,distro_vers=9))]
python3
python3-devel
# Centos Stream 9 default python is 3.9, sapling needs 3.10+
[rpms.all(distro=centos_stream,distro_vers=9)]
python3.12
python3.12-devel
# sapling needs match statements with arrive in python 3.10 in ubuntu 22.04
[debs.not(all(distro=ubuntu,any(distro_vers="18.04",distro_vers="20.04")))]
python3-all-dev
[pps]
python3
[download]
url = https://www.python.org/ftp/python/3.10.19/Python-3.10.19.tgz
sha256 = a078fb2d7a216071ebbe2e34b5f5355dd6b6e9b0cd1bacc4a41c63990c5a0eec
[build]
builder = autoconf
subdir = Python-3.10.19
[autoconf.args]
--enable-shared
--with-ensurepip=install
# python's pkg-config libffi detection is broken
# See https://bugs.python.org/issue34823 for clearest description
# and pending PR https://github.com/python/cpython/pull/20451
# The documented workaround requires an environment variable derived from
# pkg-config to be passed into its configure step
[autoconf.envcmd.LDFLAGS]
pkg-config
--libs-only-L
libffi
[dependencies]
libffi
# eden tests expect the python bz2 support
bz2
# eden tests expect the python curses support
ncurses
================================================
FILE: build/fbcode_builder/manifests/python-3_14
================================================
# This is primarily to support CinderX's CI, so it's not heavily configured.
[manifest]
name = python-3_14
[download]
url = https://github.com/python/cpython/archive/refs/tags/v3.14.3.tar.gz
sha256 = f229a232052ae318d2fc8eb0aca4a02d631e7e1a8790ef1f9b65e1632743a469
[build]
builder = autoconf
subdir = cpython-3.14.3
================================================
FILE: build/fbcode_builder/manifests/python-click
================================================
[manifest]
name = python-click
[download]
url = https://files.pythonhosted.org/packages/d2/3d/fa76db83bf75c4f8d338c2fd15c8d33fdd7ad23a9b5e57eb6c5de26b430e/click-7.1.2-py2.py3-none-any.whl
sha256 = dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc
[build]
builder = python-wheel
[rpms]
python3-click
[debs]
python3-click
================================================
FILE: build/fbcode_builder/manifests/python-filelock
================================================
[manifest]
name = python-filelock
[download]
url = https://files.pythonhosted.org/packages/31/24/ee722b92f23b9ebd87783e893a75352c048bbbc1f67dce0d63b58b46cb48/filelock-3.3.2-py3-none-any.whl
sha256 = bb2a1c717df74c48a2d00ed625e5a66f8572a3a30baacb7657add1d7bac4097b
[build]
builder = python-wheel
================================================
FILE: build/fbcode_builder/manifests/python-main
================================================
# This is primarily to support CinderX's CI, so it's not heavily configured.
[manifest]
name = python-main
fbsource_path = third-party/python/main/pristine
# We don't actually have a shipit project for python-main, but we use getdeps
# built-in shipit implementation which just needs a shipit.pathmap.
shipit_project = dummy-name
[git]
repo_url = https://github.com/python/cpython.git
[shipit.pathmap]
third-party/python/main/pristine = .
[build]
builder = autoconf
================================================
FILE: build/fbcode_builder/manifests/python-psutil
================================================
[manifest]
name = python-psutil
[download]
url = https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl
sha256 = 4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34
[build]
builder = python-wheel
================================================
FILE: build/fbcode_builder/manifests/python-ptyprocess
================================================
[manifest]
name = python-ptyprocess
[download]
url = https://files.pythonhosted.org/packages/d1/29/605c2cc68a9992d18dada28206eeada56ea4bd07a239669da41674648b6f/ptyprocess-0.6.0-py2.py3-none-any.whl
sha256 = d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f
[build]
builder = python-wheel
================================================
FILE: build/fbcode_builder/manifests/python-pyyaml
================================================
[manifest]
name = python-pyyaml
[download]
url = https://files.pythonhosted.org/packages/25/a2/b725b61ac76a75583ae7104b3209f75ea44b13cfd026aa535ece22b7f22e/PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl
sha256 = 22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6
[build]
builder = python-wheel
================================================
FILE: build/fbcode_builder/manifests/python-setuptools
================================================
[manifest]
name = python-setuptools
[download]
url = https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl
sha256 = 062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922
[build]
builder = python-wheel
[rpms]
python3-setuptools
# Centos Stream 9 default python is 3.9, sapling needs 3.10+
[rpms.all(distro=centos_stream,distro_vers=9)]
python3.12-setuptools
[homebrew]
python-setuptools
[debs]
python3-setuptools
================================================
FILE: build/fbcode_builder/manifests/python-setuptools-69
================================================
[manifest]
name = python-setuptools-69
[download]
url = https://files.pythonhosted.org/packages/c0/7a/3da654f49c95d0cc6e9549a855b5818e66a917e852ec608e77550c8dc08b/setuptools-69.1.1-py3-none-any.whl
sha256 = 02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56
[build]
builder = python-wheel
[rpms]
python3-setuptools
[homebrew]
python-setuptools
[debs]
python3-setuptools
================================================
FILE: build/fbcode_builder/manifests/python-six
================================================
[manifest]
name = python-six
[download]
url = https://files.pythonhosted.org/packages/73/fb/00a976f728d0d1fecfe898238ce23f502a721c0ac0ecfedb80e0d88c64e9/six-1.12.0-py2.py3-none-any.whl
sha256 = 3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c
[build]
builder = python-wheel
================================================
FILE: build/fbcode_builder/manifests/python-toml
================================================
[manifest]
name = python-toml
[download]
url = https://files.pythonhosted.org/packages/a2/12/ced7105d2de62fa7c8fb5fce92cc4ce66b57c95fb875e9318dba7f8c5db0/toml-0.10.0-py2.py3-none-any.whl
sha256 = 235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e
[build]
builder = python-wheel
================================================
FILE: build/fbcode_builder/manifests/ragel
================================================
[manifest]
name = ragel
[debs]
ragel
[homebrew]
ragel
[rpms]
ragel
[download]
url = https://www.colm.net/files/ragel/ragel-6.10.tar.gz
sha256 = 5f156edb65d20b856d638dd9ee2dfb43285914d9aa2b6ec779dac0270cd56c3f
[build]
builder = autoconf
subdir = ragel-6.10
================================================
FILE: build/fbcode_builder/manifests/range-v3
================================================
[manifest]
name = range-v3
[download]
url = https://github.com/ericniebler/range-v3/archive/refs/tags/0.11.0.tar.gz
sha256 = 376376615dbba43d3bef75aa590931431ecb49eb36d07bb726a19f680c75e20c
[build]
builder = cmake
subdir = range-v3-0.11.0
[cmake.defines]
RANGE_V3_EXAMPLES=OFF
================================================
FILE: build/fbcode_builder/manifests/rdma-core
================================================
[manifest]
name = rdma-core
[debs]
rdma-core
[rpms]
rdma-core-devel
================================================
FILE: build/fbcode_builder/manifests/re2
================================================
[manifest]
name = re2
[homebrew]
re2
[debs]
libre2-dev
[rpms]
re2
re2-devel
[pps]
re2
[download]
url = https://github.com/google/re2/archive/2020-11-01.tar.gz
sha256 = 8903cc66c9d34c72e2bc91722288ebc7e3ec37787ecfef44d204b2d6281954d7
[build]
builder = cmake
subdir = re2-2020-11-01
================================================
FILE: build/fbcode_builder/manifests/rebalancer
================================================
[manifest]
name = rebalancer
fbsource_path = fbcode/algopt/rebalancer/
shipit_project = rebalancer
shipit_fbcode_builder = true
use_shipit = true
[git]
# To git clone on devserver, setup fwdproxy:
# https://www.internalfb.com/wiki/Open_Source/Maintain_a_FB_OSS_Project/Devserver_GitHub_Access/
repo_url = https://github.com/facebookincubator/rebalancer.git
branch = main
[build]
builder = cmake
[dependencies]
boost
fbthrift
fizz
fmt
folly
gflags
glog
googletest
xxhash
[dependencies.os=linux]
clang19
[cmake.defines.os=linux]
CMAKE_C_COMPILER=clang-19
CMAKE_CXX_COMPILER=clang++-19
[cmake.defines.os=darwin]
REBALANCER_USE_SCIP=0
[shipit.strip]
^.*/fb/.*$
[shipit.pathmap]
fbcode/algopt/rebalancer/oss_root = .
fbcode/algopt/rebalancer = algopt/rebalancer
fbcode/algopt/lp = algopt/lp
[dependencies.all(distro=centos_stream,distro_vers=9)]
clang19
[cmake.defines.all(distro=centos_stream,distro_vers=9)]
CMAKE_C_COMPILER=clang-19
CMAKE_CXX_COMPILER=clang++-19
================================================
FILE: build/fbcode_builder/manifests/ripgrep
================================================
[manifest]
name = ripgrep
[rpms]
ripgrep
[debs]
ripgrep
[homebrew]
ripgrep
# only used from system packages currently
[build]
builder = nop
================================================
FILE: build/fbcode_builder/manifests/rocksdb
================================================
[manifest]
name = rocksdb
[download]
url = https://github.com/facebook/rocksdb/archive/refs/tags/v8.7.3.zip
sha256 = 36c06b61dc167f2455990d60dd88d734b73aa8c4dfc095243efd0243834c6cd3
[dependencies]
lz4
snappy
[build]
builder = cmake
subdir = rocksdb-8.7.3
[cmake.defines]
WITH_SNAPPY=ON
WITH_LZ4=ON
WITH_TESTS=OFF
WITH_BENCHMARK_TOOLS=OFF
# We get relocation errors with the static gflags lib,
# and there's no clear way to make it pick the shared gflags
# so just turn it off.
WITH_GFLAGS=OFF
# Disable the use of -Werror
FAIL_ON_WARNINGS = OFF
[cmake.defines.os=windows]
ROCKSDB_INSTALL_ON_WINDOWS=ON
# RocksDB hard codes the paths to the snappy libs to something
# that doesn't exist; ignoring the usual cmake rules. As a result,
# we can't build it with snappy without either patching rocksdb or
# without introducing more complex logic to the build system to
# connect the snappy build outputs to rocksdb's custom logic here.
# Let's just turn it off on windows.
WITH_SNAPPY=OFF
WITH_LZ4=ON
ROCKSDB_SKIP_THIRDPARTY=ON
================================================
FILE: build/fbcode_builder/manifests/rust-shed
================================================
[manifest]
name = rust-shed
fbsource_path = fbcode/common/rust/shed
shipit_project = rust-shed
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebookexperimental/rust-shed.git
[build]
builder = cargo
[cargo]
build_doc = true
workspace_dir =
[shipit.pathmap]
fbcode/common/rust/shed = shed
fbcode/common/rust/shed/public_autocargo = shed
fbcode/common/rust/shed/public_tld = .
tools/rust/ossconfigs = .
[shipit.strip]
^fbcode/common/rust/shed/(?!public_autocargo|public_tld).+/Cargo\.toml$
[dependencies]
fbthrift
fb303
# We use the system openssl on linux
[dependencies.not(os=linux)]
openssl
[dependencies.fbsource=on]
rust
================================================
FILE: build/fbcode_builder/manifests/sapling
================================================
[manifest]
name = sapling
fbsource_path = fbcode/eden
shipit_project = eden
shipit_fbcode_builder = true
[github.actions]
required_locales = en_US.UTF-8
[git]
repo_url = https://github.com/facebook/sapling.git
[build.not(os=windows)]
builder = make
subdir = eden/scm
[build.os=windows]
# For now the biggest blocker is missing "make" on windows, but there are bound
# to be more
builder = nop
[make.build_args]
getdepsbuild
[make.install_args]
install-getdeps
[make.test_args]
test-getdeps
[shipit.pathmap]
fbcode/configerator/structs/scm/hg = configerator/structs/scm/hg
fbcode/configerator/structs/scm/hg/public_autocargo = configerator/structs/scm/hg
fbcode/eden/oss = .
fbcode/eden = eden
fbcode/eden/fs/public_autocargo = eden/fs
fbcode/eden/mononoke/public_autocargo = eden/mononoke
fbcode/eden/scm/public_autocargo = eden/scm
fbcode/tools/lfs = tools/lfs
[shipit.strip]
^fbcode/configerator/structs/scm/hg(?!/public_autocargo).*/Cargo\.toml$
^fbcode/eden/addons/.*$
^fbcode/eden/fs/eden-config\.h$
^fbcode/eden/fs/py/eden/config\.py$
^fbcode/eden/hg-server/.*$
^fbcode/eden/fs(?!/public_autocargo).*/Cargo\.toml$
^fbcode/eden/mononoke(?!/public_autocargo).*/Cargo\.toml$
^fbcode/eden/scm(?!/public_autocargo|/edenscmnative/bindings).*/Cargo\.toml$
^fbcode/eden/scm/build/.*$
^fbcode/eden/website/.*$
^fbcode/eden/.*/\.cargo/.*$
^.*/facebook/.*$
^.*/fb/.*$
/Cargo\.lock$
\.pyc$
[dependencies]
fb303
fbthrift
rust-shed
[dependencies.all(test=on,not(os=darwin))]
hexdump
[dependencies.not(os=windows)]
python
python-setuptools
# We use the system openssl on linux
[dependencies.not(os=linux)]
openssl
[dependencies.fbsource=on]
rust
================================================
FILE: build/fbcode_builder/manifests/snappy
================================================
[manifest]
name = snappy
[homebrew]
snappy
[debs]
libsnappy-dev
[rpms]
snappy-devel
[pps]
snappy
[download]
url = https://github.com/google/snappy/archive/1.1.7.tar.gz
sha256 = 3dfa02e873ff51a11ee02b9ca391807f0c8ea0529a4924afa645fbf97163f9d4
[build]
builder = cmake
subdir = snappy-1.1.7
[cmake.defines]
SNAPPY_BUILD_TESTS = OFF
# Avoid problems like `relocation R_X86_64_PC32 against symbol` on ELF systems
# when linking rocksdb, which builds PIC even when building a static lib
[cmake.defines.os=linux]
BUILD_SHARED_LIBS = ON
================================================
FILE: build/fbcode_builder/manifests/sparsemap
================================================
[manifest]
name = sparsemap
[download]
url = https://github.com/Tessil/sparse-map/archive/refs/tags/v0.6.2.tar.gz
sha256 = 7020c21e8752e59d72e37456cd80000e18671c803890a3e55ae36b295eba99f6
[build]
builder = cmake
subdir = sparse-map-0.6.2/
================================================
FILE: build/fbcode_builder/manifests/sqlite3
================================================
[manifest]
name = sqlite3
[debs]
libsqlite3-dev
sqlite3
[homebrew]
sqlite
[rpms]
sqlite-devel
sqlite-libs
sqlite
[pps]
sqlite3
[download]
url = https://sqlite.org/2019/sqlite-amalgamation-3280000.zip
sha256 = d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327
[dependencies]
cmake
ninja
[build]
builder = sqlite
subdir = sqlite-amalgamation-3280000
================================================
FILE: build/fbcode_builder/manifests/systemd
================================================
[manifest]
name = systemd
[rpms]
systemd
systemd-devel
[download]
url = https://github.com/systemd/systemd/archive/refs/tags/v256.7.tar.gz
sha256 = 896d76ff65c88f5fd9e42f90d152b0579049158a163431dd77cdc57748b1d7b0
[build.os=linux]
builder = meson
subdir = systemd-256.7
[meson.setup_args]
-Dstatic-libsystemd=true
-Dprefix=/
================================================
FILE: build/fbcode_builder/manifests/tabulate
================================================
[manifest]
name = tabulate
[download]
url = https://github.com/p-ranav/tabulate/archive/refs/tags/v1.5.tar.gz
sha256 = 16b289f46306283544bb593f4601e80d6ea51248fde52e910cc569ef08eba3fb
[build]
builder = cmake
subdir = tabulate-1.5
[cmake.defines]
tabulate_BUILD_TESTS = OFF
tabulate_BUILD_SAMPLES = OFF
================================================
FILE: build/fbcode_builder/manifests/tree
================================================
[manifest]
name = tree
[debs]
tree
[homebrew]
tree
[rpms]
tree
[download.os=linux]
url = https://salsa.debian.org/debian/tree-packaging/-/archive/debian/1.8.0-1/tree-packaging-debian-1.8.0-1.tar.gz
sha256 = a841eee1d52bfd64a48f54caab9937b9bd92935055c48885c4ab1ae4dab7fae5
[download.os=darwin]
# The official package of tree source requires users of non-Linux platform to
# comment/uncomment certain lines in the Makefile to build for their platform.
# Besauce getdeps.py doesn't have that functionality we just use this custom
# fork of tree which has proper lines uncommented for a OSX build
url = https://github.com/lukaspiatkowski/tree-command/archive/debian/1.8.0-1-macos.tar.gz
sha256 = 9cbe889553d95cf5a2791dd0743795d46a3c092c5bba691769c0e5c52e11229e
[build.os=linux]
builder = make
subdir = tree-packaging-debian-1.8.0-1
[build.os=darwin]
builder = make
subdir = tree-command-debian-1.8.0-1-macos
[build.os=windows]
builder = nop
[make.install_args]
install
================================================
FILE: build/fbcode_builder/manifests/wangle
================================================
[manifest]
name = wangle
fbsource_path = fbcode/wangle
shipit_project = wangle
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/wangle.git
[build]
builder = cmake
subdir = wangle
[cmake.defines.test=on]
BUILD_TESTS=ON
[cmake.defines.test=off]
BUILD_TESTS=OFF
[dependencies]
folly
googletest
fizz
[shipit.pathmap]
fbcode/wangle/public_tld = .
fbcode/wangle = wangle
================================================
FILE: build/fbcode_builder/manifests/wangle-python
================================================
[manifest]
name = wangle-python
fbsource_path = fbcode/wangle
shipit_project = wangle
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/wangle.git
[build]
builder = cmake
subdir = wangle
[build.not(os=linux)]
builder = nop
[cmake.defines.test=on]
BUILD_TESTS=ON
[cmake.defines.test=off]
BUILD_TESTS=OFF
[cmake.defines.os=linux]
CMAKE_POSITION_INDEPENDENT_CODE = ON
BUILD_SHARED_LIBS = ON
[dependencies]
folly-python
googletest
fizz-python
[shipit.pathmap]
fbcode/wangle/public_tld = .
fbcode/wangle = wangle
================================================
FILE: build/fbcode_builder/manifests/watchman
================================================
[manifest]
name = watchman
fbsource_path = fbcode/watchman
shipit_project = watchman
shipit_fbcode_builder = true
[git]
repo_url = https://github.com/facebook/watchman.git
[build]
builder = cmake
[dependencies]
boost
cpptoml
edencommon
fb303
fbthrift
folly
pcre2
googletest
python-setuptools-69
[dependencies.fbsource=on]
rust
[shipit.pathmap]
fbcode/watchman = watchman
fbcode/watchman/oss = .
fbcode/eden/fs = eden/fs
[shipit.strip]
^fbcode/eden/fs/(?!.*\.thrift|service/shipit_test_file\.txt)
[cmake.defines.fb=on]
ENABLE_EDEN_SUPPORT=ON
IS_FB_BUILD=ON
# FB macos specific settings
[cmake.defines.all(fb=on,os=darwin)]
# this path is coupled with the FB internal watchman-osx.spec
WATCHMAN_STATE_DIR=/opt/facebook/watchman/var/run/watchman
# tell cmake not to try to create /opt/facebook/...
INSTALL_WATCHMAN_STATE_DIR=OFF
USE_SYS_PYTHON=OFF
[depends.environment]
WATCHMAN_VERSION_OVERRIDE
================================================
FILE: build/fbcode_builder/manifests/xxhash
================================================
[manifest]
name = xxhash
[download]
url = https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.2.tar.gz
sha256 = baee0c6afd4f03165de7a4e67988d16f0f2b257b51d0e3cb91909302a26a79c4
[rpms]
xxhash-devel
[debs]
libxxhash-dev
xxhash
[homebrew]
xxhash
[build]
builder = cmake
subdir = xxHash-0.8.2/cmake_unofficial
[cmake.defines]
CMAKE_POSITION_INDEPENDENT_CODE = ON
================================================
FILE: build/fbcode_builder/manifests/xz
================================================
[manifest]
name = xz
# ubuntu's package causes watchman's tests to hang
[debs.not(distro=ubuntu)]
liblzma-dev
[homebrew]
xz
[rpms]
xz-devel
[download]
url = https://tukaani.org/xz/xz-5.2.5.tar.gz
sha256 = f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10
[build]
builder = autoconf
subdir = xz-5.2.5
[autoconf.args]
--with-pic
================================================
FILE: build/fbcode_builder/manifests/yaml-cpp
================================================
[manifest]
name = yaml-cpp
[download]
url = https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-0.6.2.tar.gz
sha256 = e4d8560e163c3d875fd5d9e5542b5fd5bec810febdcba61481fe5fc4e6b1fd05
[build.os=linux]
builder = cmake
subdir = yaml-cpp-yaml-cpp-0.6.2
[build.not(os=linux)]
builder = nop
[dependencies]
boost
googletest
[cmake.defines]
YAML_CPP_BUILD_TESTS=OFF
================================================
FILE: build/fbcode_builder/manifests/zlib
================================================
[manifest]
name = zlib
[debs]
zlib1g-dev
[homebrew]
zlib
[rpms.not(distro=fedora)]
zlib-devel
zlib-static
[rpms.distro=fedora]
zlib-ng-compat-devel
zlib-ng-compat-static
[pps]
zlib
[download]
url = https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz
sha256 = 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23
[build]
builder = cmake
subdir = zlib-1.3.1
patchfile = zlib_dont_build_more_than_needed.patch
================================================
FILE: build/fbcode_builder/manifests/zlib-python
================================================
[manifest]
name = zlib-python
[debs]
zlib1g-dev
[homebrew]
zlib
[rpms.not(distro=fedora)]
zlib-devel
zlib-static
[rpms.distro=fedora]
zlib-ng-compat-devel
zlib-ng-compat-static
[pps]
zlib
[download]
url = https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz
sha256 = 9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23
[build]
builder = cmake
subdir = zlib-1.3.1
patchfile = zlib_dont_build_more_than_needed.patch
[build.not(os=linux)]
builder = nop
[cmake.defines]
CMAKE_POSITION_INDEPENDENT_CODE=ON
================================================
FILE: build/fbcode_builder/manifests/zstd
================================================
[manifest]
name = zstd
[homebrew]
zstd
# 18.04 zstd is too old
[debs.not(all(distro=ubuntu,distro_vers="18.04"))]
libclang-dev
libzstd-dev
zstd
[rpms]
libzstd-devel
libzstd
[pps]
zstd
[download]
url = https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz
sha256 = 9c4396cc829cfae319a6e2615202e82aad41372073482fce286fac78646d3ee4
[build]
builder = cmake
subdir = zstd-1.5.5/build/cmake
# The zstd cmake build explicitly sets the install name
# for the shared library in such a way that cmake discards
# the path to the library from the install_name, rendering
# the library non-resolvable during the build. The short
# term solution for this is just to link static on macos.
#
# And while we're at it, let's just always link statically.
[cmake.defines]
ZSTD_BUILD_SHARED = OFF
================================================
FILE: build/fbcode_builder/manifests/zstd-python
================================================
[manifest]
name = zstd-python
[homebrew]
zstd
# 18.04 zstd is too old
[debs.not(all(distro=ubuntu,distro_vers="18.04"))]
libclang-dev
libzstd-dev
zstd
[rpms]
libzstd-devel
libzstd
[pps]
zstd
[download]
url = https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz
sha256 = 9c4396cc829cfae319a6e2615202e82aad41372073482fce286fac78646d3ee4
[build]
builder = cmake
subdir = zstd-1.5.5/build/cmake
[build.not(os=linux)]
builder = nop
# The zstd cmake build explicitly sets the install name
# for the shared library in such a way that cmake discards
# the path to the library from the install_name, rendering
# the library non-resolvable during the build. The short
# term solution for this is just to link static on macos.
#
# And while we're at it, let's just always link statically.
[cmake.defines]
ZSTD_BUILD_SHARED = OFF
CMAKE_POSITION_INDEPENDENT_CODE=ON
================================================
FILE: build/fbcode_builder/patches/boost_1_83_0.patch
================================================
diff --git a/boost/serialization/strong_typedef.hpp b/boost/serialization/strong_typedef.hpp
--- a/boost/serialization/strong_typedef.hpp
+++ b/boost/serialization/strong_typedef.hpp
@@ -44,6 +44,7 @@
operator const T&() const {return t;} \
operator T&() {return t;} \
bool operator==(const D& rhs) const {return t == rhs.t;} \
+ bool operator==(const T& lhs) const {return t == lhs;} \
bool operator<(const D& rhs) const {return t < rhs.t;} \
};
diff --git a/tools/build/src/tools/msvc.jam b/tools/build/src/tools/msvc.jam
--- a/tools/build/src/tools/msvc.jam
+++ b/tools/build/src/tools/msvc.jam
@@ -1137,6 +1137,14 @@
}
else
{
+ if [ MATCH "(14.4)" : $(version) ]
+ {
+ if $(.debug-configuration)
+ {
+ ECHO "notice: [generate-setup-cmd] $(version) is 14.4x" ;
+ }
+ parent = [ path.native [ path.join $(parent) "..\\..\\..\\..\\..\\Auxiliary\\Build" ] ] ;
+ }
if [ MATCH "(14.3)" : $(version) ]
{
if $(.debug-configuration)
================================================
FILE: build/fbcode_builder/patches/iproute2_oss.patch
================================================
diff --git a/bridge/fdb.c b/bridge/fdb.c
--- a/bridge/fdb.c
+++ b/bridge/fdb.c
@@ -31,7 +31,7 @@
static unsigned int filter_index, filter_vlan, filter_state;
-json_writer_t *jw_global;
+static json_writer_t *jw_global;
static void usage(void)
{
diff --git a/ip/ipmroute.c b/ip/ipmroute.c
--- a/ip/ipmroute.c
+++ b/ip/ipmroute.c
@@ -44,7 +44,7 @@
exit(-1);
}
-struct rtfilter {
+static struct rtfilter {
int tb;
int af;
int iif;
diff --git a/ip/xfrm_monitor.c b/ip/xfrm_monitor.c
--- a/ip/xfrm_monitor.c
+++ b/ip/xfrm_monitor.c
@@ -34,7 +34,7 @@
#include "ip_common.h"
static void usage(void) __attribute__((noreturn));
-int listen_all_nsid;
+static int listen_all_nsid;
static void usage(void)
{
================================================
FILE: build/fbcode_builder/patches/libiberty_install_pic_lib.patch
================================================
diff --git a/Makefile.in b/Makefile.in
index b77a41c..cbe71fe 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -389,7 +389,7 @@ MULTIOSDIR = `$(CC) $(CFLAGS) -print-multi-os-directory`
install_to_libdir: all
if test -n "${target_header_dir}"; then \
${mkinstalldirs} $(DESTDIR)$(libdir)/$(MULTIOSDIR); \
- $(INSTALL_DATA) $(TARGETLIB) $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB)n; \
+ $(INSTALL_DATA) pic/$(TARGETLIB) $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB)n; \
( cd $(DESTDIR)$(libdir)/$(MULTIOSDIR) ; chmod 644 $(TARGETLIB)n ;$(RANLIB) $(TARGETLIB)n ); \
mv -f $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB)n $(DESTDIR)$(libdir)/$(MULTIOSDIR)/$(TARGETLIB); \
case "${target_header_dir}" in \
================================================
FILE: build/fbcode_builder/patches/zlib_dont_build_more_than_needed.patch
================================================
diff -Naur ../zlib-1.3.1/CMakeLists.txt ./CMakeLists.txt
--- ../zlib-1.3.1/CMakeLists.txt 2024-01-22 10:32:37.000000000 -0800
+++ ./CMakeLists.txt 2024-01-23 13:14:09.870289968 -0800
@@ -149,10 +149,8 @@
set(ZLIB_DLL_SRCS ${CMAKE_CURRENT_BINARY_DIR}/zlib1rc.obj)
endif(MINGW)
-add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
+add_library(zlib ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
target_include_directories(zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
-add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS})
-target_include_directories(zlibstatic PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL)
set_target_properties(zlib PROPERTIES SOVERSION 1)
@@ -169,7 +167,7 @@
if(UNIX)
# On unix-like platforms the library is almost always called libz
- set_target_properties(zlib zlibstatic PROPERTIES OUTPUT_NAME z)
+ set_target_properties(zlib PROPERTIES OUTPUT_NAME z)
if(NOT APPLE AND NOT(CMAKE_SYSTEM_NAME STREQUAL AIX))
set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"")
endif()
@@ -179,7 +177,7 @@
endif()
if(NOT SKIP_INSTALL_LIBRARIES AND NOT SKIP_INSTALL_ALL )
- install(TARGETS zlib zlibstatic
+ install(TARGETS zlib
RUNTIME DESTINATION "${INSTALL_BIN_DIR}"
ARCHIVE DESTINATION "${INSTALL_LIB_DIR}"
LIBRARY DESTINATION "${INSTALL_LIB_DIR}" )
================================================
FILE: cmake/FindGMock.cmake
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
#
# Find libgmock
#
# LIBGMOCK_DEFINES - List of defines when using libgmock.
# LIBGMOCK_INCLUDE_DIR - where to find gmock/gmock.h, etc.
# LIBGMOCK_LIBRARIES - List of libraries when using libgmock.
# LIBGMOCK_FOUND - True if libgmock found.
IF (LIBGMOCK_INCLUDE_DIR)
# Already in cache, be silent
SET(LIBGMOCK_FIND_QUIETLY TRUE)
ENDIF ()
FIND_PATH(LIBGTEST_INCLUDE_DIR gtest/gtest.h)
FIND_PATH(LIBGMOCK_INCLUDE_DIR gmock/gmock.h)
FIND_LIBRARY(LIBGMOCK_MAIN_LIBRARY gmock_main)
FIND_LIBRARY(LIBGMOCK_LIBRARY gmock)
FIND_LIBRARY(LIBGTEST_LIBRARY gtest)
set(LIBGMOCK_LIBRARIES
${LIBGMOCK_MAIN_LIBRARY}
${LIBGMOCK_LIBRARY}
${LIBGTEST_LIBRARY}
)
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
# The GTEST_LINKED_AS_SHARED_LIBRARY macro must be set properly on Windows.
#
# There isn't currently an easy way to determine if a library was compiled as
# a shared library on Windows, so just assume we've been built against a
# shared build of gmock for now.
SET(LIBGMOCK_DEFINES "GTEST_LINKED_AS_SHARED_LIBRARY=1" CACHE STRING "")
endif()
# handle the QUIETLY and REQUIRED arguments and set LIBGMOCK_FOUND to TRUE if
# all listed variables are TRUE
INCLUDE(FindPackageHandleStandardArgs)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(
LIBGMOCK
DEFAULT_MSG
LIBGMOCK_MAIN_LIBRARY
LIBGMOCK_LIBRARY
LIBGTEST_LIBRARY
LIBGMOCK_LIBRARIES
LIBGMOCK_INCLUDE_DIR
LIBGTEST_INCLUDE_DIR
)
MARK_AS_ADVANCED(
LIBGMOCK_DEFINES
LIBGMOCK_MAIN_LIBRARY
LIBGMOCK_LIBRARY
LIBGTEST_LIBRARY
LIBGMOCK_LIBRARIES
LIBGMOCK_INCLUDE_DIR
)
================================================
FILE: cmake/FindZstd.cmake
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
# - Find zstd
# Find the zstd compression library and includes
#
# ZSTD_INCLUDE_DIR - where to find zstd.h, etc.
# ZSTD_LIBRARIES - List of libraries when using zstd.
# ZSTD_FOUND - True if zstd found.
find_path(ZSTD_INCLUDE_DIR
NAMES zstd.h
HINTS ${ZSTD_ROOT_DIR}/include)
find_library(ZSTD_LIBRARIES
NAMES zstd
HINTS ${ZSTD_ROOT_DIR}/lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(zstd DEFAULT_MSG ZSTD_LIBRARIES ZSTD_INCLUDE_DIR)
mark_as_advanced(
ZSTD_LIBRARIES
ZSTD_INCLUDE_DIR
)
if(NOT TARGET zstd)
if("${ZSTD_LIBRARIES}" MATCHES ".*.a$")
add_library(zstd STATIC IMPORTED)
else()
add_library(zstd SHARED IMPORTED)
endif()
set_target_properties(
zstd
PROPERTIES
IMPORTED_LOCATION ${ZSTD_LIBRARIES}
INTERFACE_INCLUDE_DIRECTORIES ${ZSTD_INCLUDE_DIR}
)
endif()
================================================
FILE: cmake/ProxygenCompatAliases.cmake
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
# Helper macro to create a compat alias with multiple dependencies
macro(proxygen_compat_alias _name)
add_library(${_name} INTERFACE)
target_link_libraries(${_name} INTERFACE ${ARGN})
install(TARGETS ${_name} EXPORT proxygen-exports)
add_library(proxygen::${_name} ALIAS ${_name})
endmacro()
# =============================================================================
# Backwards compatibility aliases (legacy names used by downstream projects)
# =============================================================================
# quicwebtransport: used by moxygen for QUIC WebTransport support
proxygen_compat_alias(quicwebtransport
proxygen_http_webtransport_quicwebtransport
)
# =============================================================================
# Bundle aliases (group multiple granular targets for convenience)
#
# See TARGETS.txt for the rationale behind these groupings.
# =============================================================================
# Core HTTP types and utilities
proxygen_compat_alias(proxygen_http_core
proxygen_error
proxygen_http_status_type
proxygen_http_http_utils
proxygen_http_types
)
# HTTP/1.1 and HTTP/2 codecs
proxygen_compat_alias(proxygen_codec
proxygen_http_codec
proxygen_http_codec_direction
proxygen_http_codec_error_code
proxygen_http_codec_util
)
# HQ codec, framer, H3 errors
proxygen_compat_alias(proxygen_hq_core
proxygen_http_codec_hq_codec
proxygen_http_h3_errors
)
# HPACK compression
proxygen_compat_alias(proxygen_hpack
proxygen_http_codec_compress_hpack
)
# QPACK compression
proxygen_compat_alias(proxygen_qpack
proxygen_http_codec_compress_qpack
)
# Session base
proxygen_compat_alias(proxygen_http_session
proxygen_http_session_session
proxygen_http_session_http_transaction
)
# HQ Session
proxygen_compat_alias(proxygen_hq_session
proxygen_http_session_hq_session
proxygen_http_session_hq_upstream_session
proxygen_http_session_hq_downstream_session
)
# WebTransport
proxygen_compat_alias(proxygen_webtransport
proxygen_http_webtransport
)
# Combined wt_stream_manager + wt_egress_container
proxygen_compat_alias(proxygen_webtransport_helpers
proxygen_http_webtransport_wt_stream_manager
proxygen_http_webtransport_wt_egress_container
)
# Coro client
proxygen_compat_alias(proxygen_coro_client
proxygen_http_coro_client_http_client_lib
proxygen_http_coro_client_http_client_connection_cache
)
# Coro server
proxygen_compat_alias(proxygen_coro_server
proxygen_http_coro_server_coro_acceptor
proxygen_http_coro_server_coro_httpserver
)
# Coro filters
proxygen_compat_alias(proxygen_coro_filters
proxygen_http_coro_filters_compression_filter
proxygen_http_coro_filters_decompression_filter
)
# Connpool
proxygen_compat_alias(proxygen_connpool
proxygen_http_connpool
proxygen_http_connpool_session_holder
)
# Utils
proxygen_compat_alias(proxygen_utils
proxygen_utils_time_util
proxygen_utils_url
proxygen_utils_util
)
================================================
FILE: cmake/ProxygenFunctions.cmake
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
# Initialize global properties for tracking targets and deferred dependencies
set_property(GLOBAL PROPERTY PROXYGEN_COMPONENT_TARGETS)
set_property(GLOBAL PROPERTY PROXYGEN_DEFERRED_DEPS)
set_property(GLOBAL PROPERTY PROXYGEN_GRANULAR_INTERFACE_TARGETS)
# Define a granular proxygen library that:
# 1. Compiles sources ONCE via OBJECT library
# 2. Creates a STATIC library for individual linking (static builds)
# 3. Creates an INTERFACE library linking to monolithic proxygen (shared builds)
# 4. Defers internal proxygen deps to be resolved later
# 5. Tracks OBJECT target for monolithic aggregation
# 6. Creates proxygen:: namespace alias
#
# Usage:
# proxygen_add_library(proxygen_http_message
# SRCS HTTPMessage.cpp
# DEPS proxygen_http_headers # Private dependencies
# EXPORTED_DEPS Folly::folly_io_iobuf # Public dependencies (propagated)
# EXCLUDE_FROM_MONOLITH # Don't include in monolithic proxygen library
# )
function(proxygen_add_library _target_name)
cmake_parse_arguments(
PROXYGEN_LIB
"EXCLUDE_FROM_MONOLITH" # Options (boolean flags)
"" # Single-value args
"SRCS;DEPS;EXPORTED_DEPS" # Multi-value args
${ARGN}
)
set(_sources ${PROXYGEN_LIB_SRCS})
if(NOT _sources)
# Legacy support: if no SRCS keyword, treat remaining args as sources
set(_sources ${PROXYGEN_LIB_UNPARSED_ARGUMENTS})
endif()
# Object library name - used for monolithic aggregation
set(_obj_target "${_target_name}_obj")
# Skip if no sources (header-only library)
list(LENGTH _sources _src_count)
if(_src_count EQUAL 0)
# Header-only: create INTERFACE library
add_library(${_target_name} INTERFACE)
target_include_directories(${_target_name}
INTERFACE
$
$
$
)
# Link exported deps for INTERFACE libraries
if(PROXYGEN_LIB_EXPORTED_DEPS)
target_link_libraries(${_target_name} INTERFACE ${PROXYGEN_LIB_EXPORTED_DEPS})
endif()
install(TARGETS ${_target_name} EXPORT proxygen-exports)
add_library(proxygen::${_target_name} ALIAS ${_target_name})
return()
endif()
# 1. Create OBJECT library (compiles sources once)
add_library(${_obj_target} OBJECT ${_sources})
# Ensure generated headers are built before any proxygen sources
# This is needed because transitive deps don't carry build-order dependencies
if(TARGET proxygen-generated)
add_dependencies(${_obj_target} proxygen-generated)
endif()
if(DEFINED PACKAGE_VERSION)
set_property(TARGET ${_obj_target} PROPERTY VERSION ${PACKAGE_VERSION})
endif()
if(BUILD_SHARED_LIBS)
set_property(TARGET ${_obj_target} PROPERTY POSITION_INDEPENDENT_CODE ON)
endif()
target_include_directories(${_obj_target}
PUBLIC
$
$
$
)
target_compile_options(${_obj_target}
PRIVATE
${_PROXYGEN_COMMON_COMPILE_OPTIONS}
)
target_compile_features(${_obj_target} PUBLIC cxx_std_20)
# Separate proxygen internal deps (defer) from external deps (link immediately)
# Also separate utility deps (like proxygen-generated) that need add_dependencies
set(_immediate_deps "")
set(_proxygen_deps "")
set(_utility_deps "")
foreach(_dep IN LISTS PROXYGEN_LIB_EXPORTED_DEPS)
if(_dep STREQUAL "proxygen-generated")
# Utility target - use add_dependencies, not target_link_libraries
list(APPEND _utility_deps ${_dep})
elseif(_dep MATCHES "^proxygen_")
list(APPEND _proxygen_deps ${_dep})
else()
# Folly::*, fizz::*, mvfst::*, wangle::*, external libs - link immediately
list(APPEND _immediate_deps ${_dep})
endif()
endforeach()
# Add build-order dependencies on utility targets (like proxygen-generated)
if(_utility_deps)
add_dependencies(${_obj_target} ${_utility_deps})
endif()
# Link non-proxygen deps immediately - they provide include paths needed at compile time
if(_immediate_deps)
target_link_libraries(${_obj_target} PUBLIC ${_immediate_deps})
endif()
# For shared builds: link Folly::folly, fizz::fizz, wangle::wangle to OBJECT libraries
# to get transitive includes
if(BUILD_SHARED_LIBS)
target_link_libraries(${_obj_target} PUBLIC Folly::folly fizz::fizz wangle::wangle)
endif()
# Defer internal proxygen dependencies until all targets are created
# This is needed for both static and shared builds because:
# - Static: avoid circular dependencies during library creation
# - Shared: OBJECT libraries need include paths from deps that may not exist yet
if(_proxygen_deps)
list(JOIN _proxygen_deps "," _deps_str)
set_property(GLOBAL APPEND PROPERTY PROXYGEN_DEFERRED_DEPS
"${_obj_target}|PUBLIC|${_deps_str}"
)
endif()
if(PROXYGEN_LIB_DEPS)
# Separate proxygen internal deps (defer) from external deps (link immediately)
set(_private_immediate_deps "")
set(_private_proxygen_deps "")
foreach(_dep IN LISTS PROXYGEN_LIB_DEPS)
if(_dep MATCHES "^proxygen_")
list(APPEND _private_proxygen_deps ${_dep})
else()
list(APPEND _private_immediate_deps ${_dep})
endif()
endforeach()
if(_private_immediate_deps)
target_link_libraries(${_obj_target} PRIVATE ${_private_immediate_deps})
endif()
if(_private_proxygen_deps)
list(JOIN _private_proxygen_deps "," _deps_str)
set_property(GLOBAL APPEND PROPERTY PROXYGEN_DEFERRED_DEPS
"${_obj_target}|PRIVATE|${_deps_str}"
)
endif()
endif()
# Track OBJECT target for monolithic aggregation (unless excluded)
if(NOT PROXYGEN_LIB_EXCLUDE_FROM_MONOLITH)
set_property(GLOBAL APPEND PROPERTY PROXYGEN_COMPONENT_TARGETS ${_obj_target})
endif()
# 2. Create the granular library target
if(BUILD_SHARED_LIBS AND NOT PROXYGEN_LIB_EXCLUDE_FROM_MONOLITH)
# For shared builds: create INTERFACE library that will link to monolithic proxygen
add_library(${_target_name} INTERFACE)
target_include_directories(${_target_name}
INTERFACE
$
$
$
)
# Track this target to link to proxygen after monolithic library is created
set_property(GLOBAL APPEND PROPERTY PROXYGEN_GRANULAR_INTERFACE_TARGETS ${_target_name})
install(TARGETS ${_target_name} EXPORT proxygen-exports)
elseif(BUILD_SHARED_LIBS AND PROXYGEN_LIB_EXCLUDE_FROM_MONOLITH)
# For excluded targets in shared builds: create SHARED library with actual code
# These are NOT in the monolithic proxygen, so they need their own implementation
add_library(${_target_name} SHARED $)
if(DEFINED PACKAGE_VERSION)
set_property(TARGET ${_target_name} PROPERTY VERSION ${PACKAGE_VERSION})
endif()
target_include_directories(${_target_name}
PUBLIC
$
$
$
)
target_compile_features(${_target_name} PUBLIC cxx_std_20)
# Add build-order dependencies on utility targets
if(_utility_deps)
add_dependencies(${_target_name} ${_utility_deps})
endif()
# Link non-proxygen deps immediately
if(_immediate_deps)
target_link_libraries(${_target_name} PUBLIC ${_immediate_deps})
endif()
# Defer internal proxygen dependencies
if(_proxygen_deps)
list(JOIN _proxygen_deps "," _deps_str)
set_property(GLOBAL APPEND PROPERTY PROXYGEN_DEFERRED_DEPS
"${_target_name}|PUBLIC|${_deps_str}"
)
endif()
if(PROXYGEN_LIB_DEPS)
set(_priv_imm "")
set(_priv_prox "")
foreach(_dep IN LISTS PROXYGEN_LIB_DEPS)
if(_dep MATCHES "^proxygen_")
list(APPEND _priv_prox ${_dep})
else()
list(APPEND _priv_imm ${_dep})
endif()
endforeach()
if(_priv_imm)
target_link_libraries(${_target_name} PRIVATE ${_priv_imm})
endif()
if(_priv_prox)
list(JOIN _priv_prox "," _deps_str)
set_property(GLOBAL APPEND PROPERTY PROXYGEN_DEFERRED_DEPS
"${_target_name}|PRIVATE|${_deps_str}"
)
endif()
endif()
install(
TARGETS ${_target_name}
EXPORT proxygen-exports
LIBRARY DESTINATION ${LIB_INSTALL_DIR}
ARCHIVE DESTINATION ${LIB_INSTALL_DIR}
)
else()
# For static builds: create STATIC library
add_library(${_target_name} STATIC $)
if(DEFINED PACKAGE_VERSION)
set_property(TARGET ${_target_name} PROPERTY VERSION ${PACKAGE_VERSION})
endif()
target_include_directories(${_target_name}
PUBLIC
$
$
$
)
target_compile_features(${_target_name} PUBLIC cxx_std_20)
# Add build-order dependencies on utility targets (reuse _utility_deps from above)
if(_utility_deps)
add_dependencies(${_target_name} ${_utility_deps})
endif()
# Link non-proxygen deps immediately (reuse _immediate_deps computed above)
if(_immediate_deps)
target_link_libraries(${_target_name} PUBLIC ${_immediate_deps})
endif()
# Defer internal proxygen dependencies for STATIC library too (reuse _proxygen_deps)
if(_proxygen_deps)
list(JOIN _proxygen_deps "," _deps_str)
set_property(GLOBAL APPEND PROPERTY PROXYGEN_DEFERRED_DEPS
"${_target_name}|PUBLIC|${_deps_str}"
)
endif()
if(PROXYGEN_LIB_DEPS)
set(_priv_imm "")
set(_priv_prox "")
foreach(_dep IN LISTS PROXYGEN_LIB_DEPS)
if(_dep MATCHES "^proxygen_")
list(APPEND _priv_prox ${_dep})
else()
list(APPEND _priv_imm ${_dep})
endif()
endforeach()
if(_priv_imm)
target_link_libraries(${_target_name} PRIVATE ${_priv_imm})
endif()
if(_priv_prox)
list(JOIN _priv_prox "," _deps_str)
set_property(GLOBAL APPEND PROPERTY PROXYGEN_DEFERRED_DEPS
"${_target_name}|PRIVATE|${_deps_str}"
)
endif()
endif()
install(
TARGETS ${_target_name}
EXPORT proxygen-exports
LIBRARY DESTINATION ${LIB_INSTALL_DIR}
ARCHIVE DESTINATION ${LIB_INSTALL_DIR}
)
endif()
# Create alias for the library
add_library(proxygen::${_target_name} ALIAS ${_target_name})
endfunction()
# Create a backwards-compatible alias target
# This creates an INTERFACE library with the old name that links to the new target
function(proxygen_add_compat_alias _old_name _new_name)
if(NOT TARGET ${_new_name})
message(WARNING "Cannot create compat alias ${_old_name}: target ${_new_name} does not exist")
return()
endif()
add_library(${_old_name} INTERFACE)
target_link_libraries(${_old_name} INTERFACE ${_new_name})
install(TARGETS ${_old_name} EXPORT proxygen-exports)
add_library(proxygen::${_old_name} ALIAS ${_old_name})
endfunction()
# Create the monolithic proxygen library from all component OBJECT libraries
# Call this after all add_subdirectory() calls, before proxygen_resolve_deferred_dependencies()
function(proxygen_create_monolithic_library)
get_property(_component_targets GLOBAL PROPERTY PROXYGEN_COMPONENT_TARGETS)
if(NOT _component_targets)
message(STATUS "No component targets found, skipping monolithic library creation")
return()
endif()
# Collect all object files from component targets
set(_all_objects)
foreach(_target IN LISTS _component_targets)
list(APPEND _all_objects $)
endforeach()
# Create the monolithic library
add_library(proxygen ${_all_objects})
if(BUILD_SHARED_LIBS)
set_property(TARGET proxygen PROPERTY POSITION_INDEPENDENT_CODE ON)
if(DEFINED PACKAGE_VERSION)
set_target_properties(proxygen PROPERTIES VERSION ${PACKAGE_VERSION})
endif()
endif()
target_include_directories(proxygen
PUBLIC
$
$
$
)
target_compile_features(proxygen PUBLIC cxx_std_20)
# Link all dependencies
target_link_libraries(proxygen
PUBLIC
Folly::folly
fizz::fizz
wangle::wangle
${ZSTD_LIBRARIES}
ZLIB::ZLIB
${OPENSSL_LIBRARIES}
Threads::Threads
cares
PRIVATE
glog::glog
${GFLAG_DEPENDENCIES}
${CMAKE_DL_LIBS}
)
# Create alias for consistency
add_library(proxygen::proxygen ALIAS proxygen)
# For shared builds: link all granular INTERFACE targets to the monolithic library
if(BUILD_SHARED_LIBS)
cmake_policy(SET CMP0079 NEW)
get_property(_interface_targets GLOBAL PROPERTY PROXYGEN_GRANULAR_INTERFACE_TARGETS)
foreach(_target IN LISTS _interface_targets)
target_link_libraries(${_target} INTERFACE proxygen)
endforeach()
endif()
endfunction()
# Resolve all deferred dependencies after all targets have been created
# Call this after all add_subdirectory() calls
function(proxygen_resolve_deferred_dependencies)
# Allow linking targets defined in other directories
cmake_policy(SET CMP0079 NEW)
# For shared builds: link all granular INTERFACE targets to the monolithic library
# This is needed because proxygen_add_library creates INTERFACE libraries for shared builds
# that need to link to the monolithic proxygen target
if(BUILD_SHARED_LIBS)
get_property(_interface_targets GLOBAL PROPERTY PROXYGEN_GRANULAR_INTERFACE_TARGETS)
foreach(_target IN LISTS _interface_targets)
if(TARGET ${_target} AND TARGET proxygen)
target_link_libraries(${_target} INTERFACE proxygen)
endif()
endforeach()
endif()
get_property(_deferred_deps GLOBAL PROPERTY PROXYGEN_DEFERRED_DEPS)
foreach(_spec IN LISTS _deferred_deps)
# Parse the spec: "target|visibility|dep1,dep2,..."
string(REPLACE "|" ";" _parts "${_spec}")
list(LENGTH _parts _len)
if(_len LESS 3)
continue()
endif()
list(GET _parts 0 _target)
list(GET _parts 1 _visibility)
list(GET _parts 2 _deps_str)
# Split deps by comma
string(REPLACE "," ";" _deps "${_deps_str}")
# Filter to only existing targets (skip deps that weren't generated)
# For shared builds and OBJECT targets, prefer linking to _obj version for include paths
set(_valid_deps "")
foreach(_dep IN LISTS _deps)
# For shared builds: if target is an OBJECT library (*_obj) and dep has an _obj version,
# link to _obj for include path propagation
if(BUILD_SHARED_LIBS AND _target MATCHES "_obj$" AND TARGET ${_dep}_obj)
list(APPEND _valid_deps ${_dep}_obj)
elseif(BUILD_SHARED_LIBS AND _target MATCHES "_obj$")
# No _obj version exists; skip to avoid cycle through monolithic library
elseif(TARGET ${_dep})
list(APPEND _valid_deps ${_dep})
endif()
endforeach()
if(_valid_deps)
target_link_libraries(${_target} ${_visibility} ${_valid_deps})
endif()
endforeach()
endfunction()
# =============================================================================
# Header installation function
# =============================================================================
# Install headers preserving directory structure relative to rootDir
# Usage: proxygen_install_headers(proxygen ${CMAKE_CURRENT_SOURCE_DIR} ${HEADERS})
function(proxygen_install_headers rootName rootDir)
file(TO_CMAKE_PATH "${rootDir}" rootDir)
string(LENGTH "${rootDir}" rootDirLength)
foreach(fil ${ARGN})
file(TO_CMAKE_PATH "${fil}" filePath)
string(FIND "${filePath}" "/" rIdx REVERSE)
if(rIdx EQUAL -1)
continue()
endif()
string(SUBSTRING "${filePath}" 0 ${rIdx} filePath)
string(LENGTH "${filePath}" filePathLength)
string(FIND "${filePath}" "${rootDir}" rIdx)
if(rIdx EQUAL 0)
math(EXPR filePathLength "${filePathLength} - ${rootDirLength}")
string(SUBSTRING "${filePath}" ${rootDirLength} ${filePathLength} fileGroup)
install(FILES ${fil}
DESTINATION ${INCLUDE_INSTALL_DIR}/${rootName}${fileGroup})
endif()
endforeach()
endfunction()
================================================
FILE: cmake/ProxygenTest.cmake
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
option(BUILD_TESTS "Enable tests" OFF)
include(CTest)
if(BUILD_TESTS)
find_package(GMock 1.10.0 MODULE REQUIRED)
find_package(GTest 1.10.0 MODULE REQUIRED)
endif()
function(proxygen_add_test)
if(NOT BUILD_TESTS)
return()
endif()
set(options)
set(one_value_args TARGET WORKING_DIRECTORY PREFIX)
set(multi_value_args SOURCES DEPENDS INCLUDES EXTRA_ARGS)
cmake_parse_arguments(PARSE_ARGV 0 PROXYGEN_TEST "${options}" "${one_value_args}" "${multi_value_args}")
if(NOT PROXYGEN_TEST_TARGET)
message(FATAL_ERROR "The TARGET parameter is mandatory.")
endif()
if(NOT PROXYGEN_TEST_SOURCES)
set(PROXYGEN_TEST_SOURCES "${PROXYGEN_TEST_TARGET}.cpp")
endif()
add_executable(${PROXYGEN_TEST_TARGET}
"${PROXYGEN_TEST_SOURCES}"
)
# Ensure generated headers are built before test sources compile
if(TARGET proxygen-generated)
add_dependencies(${PROXYGEN_TEST_TARGET} proxygen-generated)
endif()
set_property(TARGET ${PROXYGEN_TEST_TARGET} PROPERTY ENABLE_EXPORTS true)
target_include_directories(${PROXYGEN_TEST_TARGET} PUBLIC
"${PROXYGEN_TEST_INCLUDES}"
${LIBGMOCK_INCLUDE_DIR}
${LIBGTEST_INCLUDE_DIRS}
)
target_compile_definitions(${PROXYGEN_TEST_TARGET} PUBLIC
${LIBGMOCK_DEFINES}
)
target_link_libraries(${PROXYGEN_TEST_TARGET} PUBLIC
"${PROXYGEN_TEST_DEPENDS}"
${LIBGMOCK_LIBRARIES}
${GLOG_LIBRARY}
)
target_compile_options(${PROXYGEN_TEST_TARGET} PRIVATE
"${_PROXYGEN_COMMON_COMPILE_OPTIONS}"
)
gtest_add_tests(TARGET ${PROXYGEN_TEST_TARGET}
EXTRA_ARGS "${PROXYGEN_TEST_EXTRA_ARGS}"
WORKING_DIRECTORY ${PROXYGEN_TEST_WORKING_DIRECTORY}
TEST_PREFIX ${PROXYGEN_TEST_PREFIX}
TEST_LIST PROXYGEN_TEST_CASES)
set_tests_properties(${PROXYGEN_TEST_CASES} PROPERTIES TIMEOUT 120)
endfunction()
================================================
FILE: cmake/TARGETS.txt
================================================
Proxygen CMake Target Compatibility Aliases
============================================
This file explains the rationale behind the compatibility aliases defined in
ProxygenCompatAliases.cmake.
Background
----------
Proxygen's CMake build now uses granular library targets generated from BUCK
files (via generate_cmake.py). Each BUCK target maps to a CMake target with
a name like proxygen__.
However, downstream projects (like moxygen) depend on higher-level groupings
rather than individual fine-grained targets. The compat aliases provide stable,
semantic names that bundle related granular targets together.
Alias Categories
----------------
http_core
Core HTTP types: error codes, status types, utilities, and common types.
Groups: proxygen_error, proxygen_http_status_type, proxygen_http_http_utils,
proxygen_http_types
codec
HTTP/1.1 and HTTP/2 codec infrastructure.
Groups: proxygen_http_codec, proxygen_http_codec_direction,
proxygen_http_codec_error_code, proxygen_http_codec_util
hq_core
HTTP/3 (HQ) codec and error types.
Groups: proxygen_http_codec_hq_codec, proxygen_http_h3_errors
hpack / qpack
Header compression libraries for HTTP/2 and HTTP/3 respectively.
session / hq_session
HTTP session management for HTTP/1-2 and HTTP/3.
webtransport / quicwebtransport
WebTransport protocol support over HTTP/2 and QUIC.
coro / coro_client / coro_server / coro_filters
C++20 coroutine-based HTTP client and server APIs.
connpool
Connection pooling infrastructure.
dns
DNS resolution using c-ares.
services
Acceptor and service configuration.
utils
Common utilities (time, URL parsing, etc.).
httpserver
HTTP server framework including ScopedHTTPServer for testing.
hq_server
HQ (HTTP/3) server implementation.
Adding New Aliases
------------------
When adding new aliases:
1. Choose a semantic name that describes the functionality, not the path
2. Group related granular targets that are typically used together
3. Document the grouping in this file
4. Avoid breaking existing downstream consumers
================================================
FILE: cmake/proxygen-config.cmake.in
================================================
# Copyright (c) 2018, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
# This module sets the following variables:
# proxygen_FOUND
# proxygen_INCLUDE_DIRS
#
# This module exports the following target:
# proxygen::proxygen
#
# which can be used with target_link_libraries() to pull in the proxygen
# library.
@PACKAGE_INIT@
include(CMakeFindDependencyMacro)
find_dependency(Glog)
find_dependency(fmt)
find_dependency(folly)
find_dependency(wangle)
find_dependency(mvfst)
find_dependency(Fizz)
# For now, anything that depends on Proxygen has to copy its FindZstd.cmake
# and issue a `find_package(Zstd)`. Uncommenting this won't work because
# this Zstd module exposes a library called `zstd`. The right fix is
# discussed on D24686032.
#
# find_dependency(Zstd)
find_dependency(ZLIB)
find_dependency(OpenSSL)
find_dependency(Threads)
find_dependency(c-ares REQUIRED)
if(NOT TARGET proxygen::proxygen)
include("${CMAKE_CURRENT_LIST_DIR}/proxygen-targets.cmake")
get_target_property(proxygen_INCLUDE_DIRS proxygen::proxygen INTERFACE_INCLUDE_DIRECTORIES)
endif()
if(NOT proxygen_FIND_QUIETLY)
message(STATUS "Found proxygen: ${PACKAGE_PREFIX_DIR}")
endif()
set(proxygen_LIBRARIES
proxygen::proxygen
proxygen::proxygencurl
proxygen::proxygendeviousbaton
proxygen::proxygenhqloggerhelper
proxygen::proxygenhttpserver
proxygen::proxygenhqserver
)
================================================
FILE: cmake_uninstall.cmake.in
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
if(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
message(FATAL_ERROR "Cannot find install manifest: @CMAKE_BINARY_DIR@/install_manifest.txt")
endif(NOT EXISTS "@CMAKE_BINARY_DIR@/install_manifest.txt")
file(READ "@CMAKE_BINARY_DIR@/install_manifest.txt" files)
string(REGEX REPLACE "\n" ";" files "${files}")
foreach(file ${files})
message(STATUS "Uninstalling $ENV{DESTDIR}${file}")
if(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
exec_program(
"@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\""
OUTPUT_VARIABLE rm_out
RETURN_VALUE rm_retval
)
if(NOT "${rm_retval}" STREQUAL 0)
message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}")
endif(NOT "${rm_retval}" STREQUAL 0)
else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
message(STATUS "File $ENV{DESTDIR}${file} does not exist.")
endif(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}")
endforeach(file)
================================================
FILE: getdeps.sh
================================================
#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
set -xeo pipefail
TOOLCHAIN_DIR=/opt/rh/devtoolset-8/root/usr/bin
if [[ -d "$TOOLCHAIN_DIR" ]]; then
PATH="$TOOLCHAIN_DIR:$PATH"
fi
PROJECT_DIR=$(dirname "$0")
GETDEPS_PATHS=(
"$PROJECT_DIR/build/fbcode_builder/getdeps.py"
"$PROJECT_DIR/../../opensource/fbcode_builder/getdeps.py"
)
ROOT_DIR=$(pwd)
STAGE=${ROOT_DIR}/_build/
mkdir -p "$STAGE"
for getdeps in "${GETDEPS_PATHS[@]}"; do
if [[ -x "$getdeps" ]]; then
"$getdeps" build proxygen --current-project proxygen "$@" --install-prefix=${STAGE}
exit 0
fi
done
echo "Could not find getdeps.py!?" >&2
exit 1
================================================
FILE: proxygen/.clang-format
================================================
---
AccessModifierOffset: -1
AlignAfterOpenBracket: Align
AlignEscapedNewlinesLeft: true
AlignOperands: true
AlignTrailingComments: true
AllowAllParametersOfDeclarationOnNextLine: true
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: false
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakBeforeMultilineStrings: true
AlwaysBreakTemplateDeclarations: true
BinPackArguments: false
BinPackParameters: false
BreakBeforeBinaryOperators: false
BreakBeforeBraces: Attach
BreakBeforeInheritanceComma: true
BreakBeforeTernaryOperators: true
BreakConstructorInitializersBeforeComma: false
ColumnLimit: 80
CompactNamespaces: true
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: true
DerivePointerBinding: true
ExperimentalAutoDetectBinPacking: true
FixNamespaceComments: true
IndentCaseLabels: true
IndentFunctionDeclarationAfterType: false
IndentWidth: 2
Language: Cpp
MaxEmptyLinesToKeep: 1
NamespaceIndentation: None
ObjCSpaceBeforeProtocolList: false
PenaltyBreakBeforeFirstCallParameter: 10
PenaltyBreakComment: 60
PenaltyBreakFirstLessLess: 20
PenaltyBreakString: 1000
PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
PointerBindsToType: true
SortIncludes: true
SortUsingDeclarations: true
SpaceAfterControlStatementKeyword: true
SpaceBeforeAssignmentOperators: true
SpaceBeforeParens: ControlStatements
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 1
SpacesInAngles: false
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
Standard: Cpp11
TabWidth: 8
UseTab: Never
...
================================================
FILE: proxygen/.clang-tidy
================================================
# NOTE there must be no spaces before the '-', so put the comma after.
# When making changes, be sure to verify the output of the following command to ensure
# the desired checks are enabled (run from the directory containing a .clang-tidy file):
# `clang-tidy -list-checks`
# NOTE: Please don't disable inheritance from the parent to make sure that common checks get propagated.
---
InheritParentConfig: true
Checks: '
boost-*,
bugprone-*,
clang-analyzer-*,
modernize-*,
performance-*,
-modernize-use-trailing-return-type,
facebook-hte-BadCall-mlock,
facebook-hte-PortabilityInclude-WinSock2.h,
facebook-hte-PortabilityInclude-Windows.h,
facebook-hte-PortabilityInclude-arpa/inet.h,
facebook-hte-PortabilityInclude-direct.h,
facebook-hte-PortabilityInclude-dirent.h,
facebook-hte-PortabilityInclude-gflags/gflags.h,
facebook-hte-PortabilityInclude-io.h,
facebook-hte-PortabilityInclude-libgen.h,
facebook-hte-PortabilityInclude-netdb.h,
facebook-hte-PortabilityInclude-netinet/in.h,
facebook-hte-PortabilityInclude-netinet/tcp.h,
facebook-hte-PortabilityInclude-openssl/asn1.h,
facebook-hte-PortabilityInclude-openssl/bio.h,
facebook-hte-PortabilityInclude-openssl/crypto.h,
facebook-hte-PortabilityInclude-openssl/dh.h,
facebook-hte-PortabilityInclude-openssl/ec.h,
facebook-hte-PortabilityInclude-openssl/ecdsa.h,
facebook-hte-PortabilityInclude-openssl/err.h,
facebook-hte-PortabilityInclude-openssl/evp.h,
facebook-hte-PortabilityInclude-openssl/hmac.h,
facebook-hte-PortabilityInclude-openssl/opensslv.h,
facebook-hte-PortabilityInclude-openssl/rand.h,
facebook-hte-PortabilityInclude-openssl/rsa.h,
facebook-hte-PortabilityInclude-openssl/sha.h,
facebook-hte-PortabilityInclude-openssl/ssl.h,
facebook-hte-PortabilityInclude-openssl/tls1.h,
facebook-hte-PortabilityInclude-openssl/x509.h,
facebook-hte-PortabilityInclude-openssl/x509v3.h,
facebook-hte-PortabilityInclude-poll.h,
facebook-hte-PortabilityInclude-pthread.h,
facebook-hte-PortabilityInclude-sched.h,
facebook-hte-PortabilityInclude-semaphore.h,
facebook-hte-PortabilityInclude-strings.h,
facebook-hte-PortabilityInclude-sys/file.h,
facebook-hte-PortabilityInclude-sys/mman.h,
facebook-hte-PortabilityInclude-sys/resource.h,
facebook-hte-PortabilityInclude-sys/socket.h,
facebook-hte-PortabilityInclude-sys/syscall.h,
facebook-hte-PortabilityInclude-sys/time.h,
facebook-hte-PortabilityInclude-sys/uio.h,
facebook-hte-PortabilityInclude-sys/un.h,
facebook-hte-PortabilityInclude-syslog.h,
facebook-hte-PortabilityInclude-unistd.h,
facebook-hte-StdToStringUse,
cppcoreguidelines-pro-type-member-init,
cppcoreguidelines-init-variables,
'
...
================================================
FILE: proxygen/CMakeLists.txt
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
if (NOT DEFINED LIB_INSTALL_DIR)
set(LIB_INSTALL_DIR "lib")
endif()
add_subdirectory(external)
add_subdirectory(lib)
add_subdirectory(httpserver)
add_subdirectory(httpclient)
add_subdirectory(fuzzers)
# =============================================================================
# Backwards compatibility aliases for downstream projects
# =============================================================================
include(ProxygenCompatAliases)
# =============================================================================
# Resolve deferred dependencies after ALL subdirectories are processed
# =============================================================================
proxygen_resolve_deferred_dependencies()
================================================
FILE: proxygen/VERSION
================================================
32:0
================================================
FILE: proxygen/build.sh
================================================
#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
## Run this script to build proxygen and run the tests. If you want to
## install proxygen to use in another C++ project on this machine, run
## the sibling file `reinstall.sh`.
# Obtain the base directory this script resides in.
BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
# Useful constants
COLOR_RED="\033[0;31m"
COLOR_GREEN="\033[0;32m"
COLOR_OFF="\033[0m"
function detect_platform() {
unameOut="$(uname -s)"
case "${unameOut}" in
Linux*)
PLATFORM=Linux
if [ -x "$(command -v lsb_release)" ]; then
DISTRO="$(lsb_release -is)"
else
DISTRO="CentOS"
fi
;;
Darwin*) PLATFORM=Mac;;
*) PLATFORM="UNKNOWN:${unameOut}"
esac
echo -e "${COLOR_GREEN}Detected platform: $PLATFORM Distribution $DISTRO ${COLOR_OFF}"
}
function install_dependencies_linux_default() {
sudo apt-get install -yq \
$deps_universal \
libgflags-dev \
libgoogle-glog-dev \
libkrb5-dev \
libsasl2-dev \
libnuma-dev \
pkg-config \
libssl-dev \
libcap-dev \
libevent-dev \
libtool \
libboost-all-dev \
libjemalloc-dev \
libsnappy-dev \
libiberty-dev \
liblz4-dev \
liblzma-dev \
zlib1g-dev \
binutils-dev \
libsodium-dev \
libdouble-conversion-dev
}
function install_dependencies_linux_fedora() {
sudo dnf install -y \
$deps_universal \
m4 \
g++ \
flex \
bison \
gflags-devel \
glog-devel \
krb5-libs \
double-conversion-devel \
libzstd-devel \
libsodium-devel \
binutils-devel \
zlib-devel \
make \
lz4-devel \
wget \
unzip \
snappy-devel \
jemalloc-devel \
boost-devel\
cyrus-sasl-devel \
numactl-libs \
openssl-devel \
libcap-devel \
libevent-devel \
libtool \
gperf
}
function install_dependencies_linux_centos() {
sudo dnf install -y \
$deps_universal \
m4 \
g++ \
flex \
bison \
fast_float-devel \
gflags-devel \
glog-devel \
krb5-libs \
double-conversion-devel \
libzstd-devel \
libsodium-devel \
binutils-devel \
zlib-devel \
make \
lz4-devel \
wget \
unzip \
snappy-devel \
jemalloc-devel \
boost-devel\
cyrus-sasl-devel \
numactl-libs \
openssl-devel \
libcap-devel \
libevent-devel \
libtool \
gperf
}
function install_dependencies_linux {
deps_universal="\
git \
cmake \
m4 \
g++ \
flex \
bison \
gperf \
wget \
unzip \
make"
case "$DISTRO" in
Fedora*) install_dependencies_linux_fedora;;
CentOS*) install_dependencies_linux_centos;;
*) install_dependencies_linux_default;;
esac
}
function install_dependencies_mac() {
# install the default dependencies from homebrew
brew install -f \
cmake \
m4 \
boost \
double-conversion \
gflags \
glog \
gperf \
libevent \
lz4 \
snappy \
xz \
openssl \
libsodium
brew link \
cmake \
boost \
double-conversion \
gflags \
glog \
gperf \
libevent \
lz4 \
snappy \
openssl \
xz \
libsodium
}
function install_dependencies() {
echo -e "${COLOR_GREEN}[ INFO ] install dependencies ${COLOR_OFF}"
if [ "$PLATFORM" = "Linux" ]; then
install_dependencies_linux
elif [ "$PLATFORM" = "Mac" ]; then
install_dependencies_mac
else
echo -e "${COLOR_RED}[ ERROR ] Unknown platform: $PLATFORM ${COLOR_OFF}"
exit 1
fi
}
function synch_dependency_to_commit() {
# Utility function to synch a dependency to a specific commit. Takes two arguments:
# - $1: folder of the dependency's git repository
# - $2: path to the text file containing the desired commit hash
if [ "$FETCH_DEPENDENCIES" = false ] ; then
return
fi
DEP_REV=$(sed 's/Subproject commit //' "$2")
pushd "$1"
git fetch
# Disable git warning about detached head when checking out a specific commit.
git -c advice.detachedHead=false checkout "$DEP_REV"
popd
}
function setup_fmt() {
FMT_DIR=$DEPS_DIR/fmt
FMT_BUILD_DIR=$DEPS_DIR/fmt/build/
FMT_TAG=$(grep "subdir = " ../../build/fbcode_builder/manifests/fmt | cut -d "-" -f 2)
if [ ! -d "$FMT_DIR" ] ; then
echo -e "${COLOR_GREEN}[ INFO ] Cloning fmt repo ${COLOR_OFF}"
git clone https://github.com/fmtlib/fmt.git "$FMT_DIR"
fi
cd "$FMT_DIR"
git fetch --tags
git checkout "${FMT_TAG}"
echo -e "${COLOR_GREEN}Building fmt ${COLOR_OFF}"
mkdir -p "$FMT_BUILD_DIR"
cd "$FMT_BUILD_DIR" || exit
cmake \
-DCMAKE_PREFIX_PATH="$DEPS_DIR" \
-DCMAKE_INSTALL_PREFIX="$DEPS_DIR" \
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
"$MAYBE_OVERRIDE_CXX_FLAGS" \
-DFMT_DOC=OFF \
-DFMT_TEST=OFF \
..
make -j "$JOBS"
make install
echo -e "${COLOR_GREEN}fmt is installed ${COLOR_OFF}"
cd "$BWD" || exit
}
function setup_googletest() {
GTEST_DIR=$DEPS_DIR/googletest
GTEST_BUILD_DIR=$DEPS_DIR/googletest/build/
GTEST_TAG=$(grep "subdir = " ../../build/fbcode_builder/manifests/googletest | cut -d "-" -f 2,3)
if [ ! -d "$GTEST_DIR" ] ; then
echo -e "${COLOR_GREEN}[ INFO ] Cloning googletest repo ${COLOR_OFF}"
git clone https://github.com/google/googletest.git "$GTEST_DIR"
fi
cd "$GTEST_DIR"
git fetch --tags
git checkout "${GTEST_TAG}"
echo -e "${COLOR_GREEN}Building googletest ${COLOR_OFF}"
mkdir -p "$GTEST_BUILD_DIR"
cd "$GTEST_BUILD_DIR" || exit
cmake \
-DCMAKE_PREFIX_PATH="$DEPS_DIR" \
-DCMAKE_INSTALL_PREFIX="$DEPS_DIR" \
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
..
make -j "$JOBS"
make install
echo -e "${COLOR_GREEN}googletest is installed ${COLOR_OFF}"
cd "$BWD" || exit
}
function setup_zstd() {
ZSTD_DIR=$DEPS_DIR/zstd
ZSTD_BUILD_DIR=$DEPS_DIR/zstd/build/cmake/builddir
ZSTD_INSTALL_DIR=$DEPS_DIR
ZSTD_TAG=$(grep "subdir = " ../../build/fbcode_builder/manifests/zstd | cut -d "-" -f 2 | cut -d "/" -f 1)
if [ ! -d "$ZSTD_DIR" ] ; then
echo -e "${COLOR_GREEN}[ INFO ] Cloning zstd repo ${COLOR_OFF}"
git clone https://github.com/facebook/zstd.git "$ZSTD_DIR"
fi
cd "$ZSTD_DIR"
git fetch --tags
git checkout "v${ZSTD_TAG}"
echo -e "${COLOR_GREEN}Building Zstd ${COLOR_OFF}"
mkdir -p "$ZSTD_BUILD_DIR"
cd "$ZSTD_BUILD_DIR" || exit
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DBUILD_TESTS=OFF \
-DCMAKE_PREFIX_PATH="$ZSTD_INSTALL_DIR" \
-DCMAKE_INSTALL_PREFIX="$ZSTD_INSTALL_DIR" \
${CMAKE_EXTRA_ARGS[@]+"${CMAKE_EXTRA_ARGS[@]}"} \
..
make -j "$JOBS"
make install
echo -e "${COLOR_GREEN}Zstd is installed ${COLOR_OFF}"
cd "$BWD" || exit
}
function setup_folly() {
FOLLY_DIR=$DEPS_DIR/folly
FOLLY_BUILD_DIR=$DEPS_DIR/folly/build/
if [ ! -d "$FOLLY_DIR" ] ; then
echo -e "${COLOR_GREEN}[ INFO ] Cloning folly repo ${COLOR_OFF}"
git clone https://github.com/facebook/folly.git "$FOLLY_DIR"
fi
synch_dependency_to_commit "$FOLLY_DIR" "$BASE_DIR"/../build/deps/github_hashes/facebook/folly-rev.txt
if [ "$PLATFORM" = "Mac" ]; then
# Homebrew installs OpenSSL in a non-default location on MacOS >= Mojave
# 10.14 because MacOS has its own SSL implementation. If we find the
# typical Homebrew OpenSSL dir, load OPENSSL_ROOT_DIR so that cmake
# will find the Homebrew version.
dir=/usr/local/opt/openssl
if [ -d $dir ]; then
export OPENSSL_ROOT_DIR=$dir
fi
fi
echo -e "${COLOR_GREEN}Building Folly ${COLOR_OFF}"
mkdir -p "$FOLLY_BUILD_DIR"
cd "$FOLLY_BUILD_DIR" || exit
MAYBE_DISABLE_JEMALLOC=""
if [ "$NO_JEMALLOC" == true ] ; then
MAYBE_DISABLE_JEMALLOC="-DFOLLY_USE_JEMALLOC=0"
fi
MAYBE_USE_STATIC_DEPS=""
MAYBE_USE_STATIC_BOOST=""
MAYBE_BUILD_SHARED_LIBS=""
if [ "$BUILD_FOR_FUZZING" == true ] ; then
MAYBE_USE_STATIC_DEPS="-DUSE_STATIC_DEPS_ON_UNIX=ON"
MAYBE_USE_STATIC_BOOST="-DBOOST_LINK_STATIC=ON"
MAYBE_BUILD_SHARED_LIBS="-DBUILD_SHARED_LIBS=OFF"
fi
cmake \
-DCMAKE_PREFIX_PATH="$DEPS_DIR" \
-DCMAKE_INSTALL_PREFIX="$DEPS_DIR" \
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DBUILD_TESTS=OFF \
"$MAYBE_USE_STATIC_DEPS" \
"$MAYBE_USE_STATIC_BOOST" \
"$MAYBE_BUILD_SHARED_LIBS" \
"$MAYBE_OVERRIDE_CXX_FLAGS" \
$MAYBE_DISABLE_JEMALLOC \
..
make -j "$JOBS"
make install
echo -e "${COLOR_GREEN}Folly is installed ${COLOR_OFF}"
cd "$BWD" || exit
}
function setup_fizz() {
FIZZ_DIR=$DEPS_DIR/fizz
FIZZ_BUILD_DIR=$DEPS_DIR/fizz/build/
if [ ! -d "$FIZZ_DIR" ] ; then
echo -e "${COLOR_GREEN}[ INFO ] Cloning fizz repo ${COLOR_OFF}"
git clone https://github.com/facebookincubator/fizz "$FIZZ_DIR"
fi
synch_dependency_to_commit "$FIZZ_DIR" "$BASE_DIR"/../build/deps/github_hashes/facebookincubator/fizz-rev.txt
echo -e "${COLOR_GREEN}Building Fizz ${COLOR_OFF}"
mkdir -p "$FIZZ_BUILD_DIR"
cd "$FIZZ_BUILD_DIR" || exit
MAYBE_USE_STATIC_DEPS=""
MAYBE_USE_SODIUM_STATIC_LIBS=""
MAYBE_BUILD_SHARED_LIBS=""
if [ "$BUILD_FOR_FUZZING" == true ] ; then
MAYBE_USE_STATIC_DEPS="-DUSE_STATIC_DEPS_ON_UNIX=ON"
MAYBE_USE_SODIUM_STATIC_LIBS="-Dsodium_USE_STATIC_LIBS=ON"
MAYBE_BUILD_SHARED_LIBS="-DBUILD_SHARED_LIBS=OFF"
fi
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DCMAKE_PREFIX_PATH="$DEPS_DIR" \
-DCMAKE_INSTALL_PREFIX="$DEPS_DIR" \
-DBUILD_TESTS=OFF \
"$MAYBE_USE_STATIC_DEPS" \
"$MAYBE_BUILD_SHARED_LIBS" \
"$MAYBE_OVERRIDE_CXX_FLAGS" \
"$MAYBE_USE_SODIUM_STATIC_LIBS" \
"$FIZZ_DIR/fizz"
make -j "$JOBS"
make install
echo -e "${COLOR_GREEN}Fizz is installed ${COLOR_OFF}"
cd "$BWD" || exit
}
function setup_wangle() {
WANGLE_DIR=$DEPS_DIR/wangle
WANGLE_BUILD_DIR=$DEPS_DIR/wangle/build/
if [ ! -d "$WANGLE_DIR" ] ; then
echo -e "${COLOR_GREEN}[ INFO ] Cloning wangle repo ${COLOR_OFF}"
git clone https://github.com/facebook/wangle "$WANGLE_DIR"
fi
synch_dependency_to_commit "$WANGLE_DIR" "$BASE_DIR"/../build/deps/github_hashes/facebook/wangle-rev.txt
echo -e "${COLOR_GREEN}Building Wangle ${COLOR_OFF}"
mkdir -p "$WANGLE_BUILD_DIR"
cd "$WANGLE_BUILD_DIR" || exit
MAYBE_USE_STATIC_DEPS=""
MAYBE_BUILD_SHARED_LIBS=""
if [ "$BUILD_FOR_FUZZING" == true ] ; then
MAYBE_USE_STATIC_DEPS="-DUSE_STATIC_DEPS_ON_UNIX=ON"
MAYBE_BUILD_SHARED_LIBS="-DBUILD_SHARED_LIBS=OFF"
fi
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DCMAKE_PREFIX_PATH="$DEPS_DIR" \
-DCMAKE_INSTALL_PREFIX="$DEPS_DIR" \
-DBUILD_TESTS=OFF \
"$MAYBE_USE_STATIC_DEPS" \
"$MAYBE_BUILD_SHARED_LIBS" \
"$MAYBE_OVERRIDE_CXX_FLAGS" \
"$WANGLE_DIR/wangle"
make -j "$JOBS"
make install
echo -e "${COLOR_GREEN}Wangle is installed ${COLOR_OFF}"
cd "$BWD" || exit
}
function setup_mvfst() {
MVFST_DIR=$DEPS_DIR/mvfst
MVFST_BUILD_DIR=$DEPS_DIR/mvfst/build/
if [ ! -d "$MVFST_DIR" ] ; then
echo -e "${COLOR_GREEN}[ INFO ] Cloning mvfst repo ${COLOR_OFF}"
git clone https://github.com/facebook/mvfst "$MVFST_DIR"
fi
synch_dependency_to_commit "$MVFST_DIR" "$BASE_DIR"/../build/deps/github_hashes/facebook/mvfst-rev.txt
echo -e "${COLOR_GREEN}Building Mvfst ${COLOR_OFF}"
mkdir -p "$MVFST_BUILD_DIR"
cd "$MVFST_BUILD_DIR" || exit
MAYBE_USE_STATIC_DEPS=""
MAYBE_BUILD_SHARED_LIBS=""
if [ "$BUILD_FOR_FUZZING" == true ] ; then
MAYBE_USE_STATIC_DEPS="-DUSE_STATIC_DEPS_ON_UNIX=ON"
MAYBE_BUILD_SHARED_LIBS="-DBUILD_SHARED_LIBS=OFF"
fi
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DCMAKE_PREFIX_PATH="$DEPS_DIR" \
-DCMAKE_INSTALL_PREFIX="$DEPS_DIR" \
-DBUILD_TESTS=OFF \
"$MAYBE_USE_STATIC_DEPS" \
"$MAYBE_BUILD_SHARED_LIBS" \
"$MAYBE_OVERRIDE_CXX_FLAGS" \
"$MVFST_DIR"
make -j "$JOBS"
make install
echo -e "${COLOR_GREEN}Mvfst is installed ${COLOR_OFF}"
cd "$BWD" || exit
}
# Parse args
JOBS=8
INSTALL_DEPENDENCIES=true
FETCH_DEPENDENCIES=true
PREFIX=""
COMPILER_FLAGS=""
PROXY_SERVER_HOST=""
PROXY_SERVER_PORT="8080"
USAGE="./build.sh [-j num_jobs] [-m|--no-jemalloc] [--no-install-dependencies] [-p|--prefix] [-x|--compiler-flags] [--no-fetch-dependencies] [--proxy_server_host] [--proxy_server_port]"
while [ "$1" != "" ]; do
case $1 in
-j | --jobs ) shift
JOBS=$1
;;
-m | --no-jemalloc )
NO_JEMALLOC=true
;;
--no-install-dependencies )
INSTALL_DEPENDENCIES=false
;;
--no-fetch-dependencies )
FETCH_DEPENDENCIES=false
;;
--build-for-fuzzing )
BUILD_FOR_FUZZING=true
;;
-t | --no-tests )
NO_BUILD_TESTS=true
;;
-p | --prefix )
shift
PREFIX=$1
;;
-x | --compiler-flags )
shift
COMPILER_FLAGS=$1
;;
--proxy_server_host )
shift
PROXY_SERVER_HOST=$1
;;
--proxy_server_port )
shift
PROXY_SERVER_PORT=$1
;;
* ) echo $USAGE
exit 1
esac
shift
done
detect_platform
if [ "$INSTALL_DEPENDENCIES" == true ] ; then
install_dependencies
fi
MAYBE_OVERRIDE_CXX_FLAGS=""
if [ -n "$COMPILER_FLAGS" ] ; then
MAYBE_OVERRIDE_CXX_FLAGS="-DCMAKE_CXX_FLAGS=$COMPILER_FLAGS"
fi
BUILD_DIR=_build
mkdir -p $BUILD_DIR
set -e nounset
trap 'cd $BASE_DIR' EXIT
cd $BUILD_DIR || exit
BWD=$(pwd)
DEPS_DIR=$BWD/deps
mkdir -p "$DEPS_DIR"
# Must execute from the directory containing this script
cd "$(dirname "$0")"
if [ -n "$PROXY_SERVER_HOST" ]; then
export https_proxy=http://$PROXY_SERVER_HOST:$PROXY_SERVER_PORT
export http_proxy=http://$PROXY_SERVER_HOST:$PROXY_SERVER_PORT
fi
setup_fmt
setup_googletest
setup_zstd
setup_folly
setup_fizz
setup_wangle
setup_mvfst
MAYBE_BUILD_FUZZERS=""
MAYBE_USE_STATIC_DEPS=""
MAYBE_LIB_FUZZING_ENGINE=""
MAYBE_BUILD_SHARED_LIBS=""
MAYBE_BUILD_TESTS="-DBUILD_TESTS=ON"
if [ "$NO_BUILD_TESTS" == true ] ; then
MAYBE_BUILD_TESTS="-DBUILD_TESTS=OFF"
fi
if [ "$BUILD_FOR_FUZZING" == true ] ; then
MAYBE_BUILD_FUZZERS="-DBUILD_FUZZERS=ON"
MAYBE_USE_STATIC_DEPS="-DUSE_STATIC_DEPS_ON_UNIX=ON"
MAYBE_LIB_FUZZING_ENGINE="-DLIB_FUZZING_ENGINE='$LIB_FUZZING_ENGINE'"
MAYBE_BUILD_SHARED_LIBS="-DBUILD_SHARED_LIBS=OFF"
fi
if [ -z "$PREFIX" ]; then
PREFIX=$BWD
fi
# Build proxygen with cmake
cd "$BWD" || exit
cmake \
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
-DCMAKE_PREFIX_PATH="$DEPS_DIR" \
-DCMAKE_INSTALL_PREFIX="$PREFIX" \
"$MAYBE_BUILD_TESTS" \
"$MAYBE_BUILD_FUZZERS" \
"$MAYBE_BUILD_SHARED_LIBS" \
"$MAYBE_OVERRIDE_CXX_FLAGS" \
"$MAYBE_USE_STATIC_DEPS" \
"$MAYBE_LIB_FUZZING_ENGINE" \
../..
make -j "$JOBS"
echo -e "${COLOR_GREEN}Proxygen build is complete. To run unit test: \
cd _build/ && make test ${COLOR_OFF}"
================================================
FILE: proxygen/external/CMakeLists.txt
================================================
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
add_library(
proxygen_http_parser OBJECT
http_parser/http_parser_cpp.cpp
)
target_compile_options(
proxygen_http_parser PRIVATE
${_PROXYGEN_COMMON_COMPILE_OPTIONS}
"-DHTTP_PARSER_STRICT_URL=1"
"-Wno-implicit-fallthrough"
)
if (BUILD_SHARED_LIBS)
set_property(TARGET proxygen_http_parser PROPERTY POSITION_INDEPENDENT_CODE ON)
endif()
target_include_directories(
proxygen_http_parser PRIVATE
${PROXYGEN_FBCODE_ROOT}
)
install(FILES http_parser/http_parser.h DESTINATION
include/proxygen/external/http_parser)
================================================
FILE: proxygen/external/http_parser/CONTRIBUTIONS
================================================
Contributors must agree to the Contributor License Agreement before patches
can be accepted.
http://spreadsheets2.google.com/viewform?hl=en&formkey=dDJXOGUwbzlYaWM4cHN1MERwQS1CSnc6MQ
================================================
FILE: proxygen/external/http_parser/LICENSE-MIT
================================================
http_parser.c is based on src/http/ngx_http_parse.c from NGINX copyright
Igor Sysoev.
Additional changes are licensed under the same terms as NGINX and
copyright Joyent, Inc. and other Node contributors. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.
================================================
FILE: proxygen/external/http_parser/README.md
================================================
HTTP Parser
===========
This is a parser for HTTP messages written in C. It parses both requests and
responses. The parser is designed to be used in performance HTTP
applications. It does not make any syscalls nor allocations, it does not
buffer data, it can be interrupted at anytime. Depending on your
architecture, it only requires about 40 bytes of data per message
stream (in a web server that is per connection).
Features:
* No dependencies
* Handles persistent streams (keep-alive).
* Decodes chunked encoding.
* Upgrade support
* Defends against buffer overflow attacks.
The parser extracts the following information from HTTP messages:
* Header fields and values
* Content-Length
* Request method
* Response status code
* Transfer-Encoding
* HTTP version
* Request URL
* Message body
Usage
-----
One `http_parser` object is used per TCP connection. Initialize the struct
using `http_parser_init()` and set the callbacks. That might look something
like this for a request parser:
http_parser_settings settings;
settings.on_path = my_path_callback;
settings.on_header_field = my_header_field_callback;
/* ... */
http_parser *parser = malloc(sizeof(http_parser));
http_parser_init(parser, HTTP_REQUEST);
parser->data = my_socket;
When data is received on the socket execute the parser and check for errors.
size_t len = 80*1024, nparsed;
char buf[len];
ssize_t recved;
recved = recv(fd, buf, len, 0);
if (recved < 0) {
/* Handle error. */
}
/* Start up / continue the parser.
* Note we pass recved==0 to signal that EOF has been received.
*/
nparsed = http_parser_execute(parser, &settings, buf, recved);
if (parser->upgrade) {
/* handle new protocol */
} else if (nparsed != recved) {
/* Handle error. Usually just close the connection. */
}
HTTP needs to know where the end of the stream is. For example, sometimes
servers send responses without Content-Length and expect the client to
consume input (for the body) until EOF. To tell http_parser about EOF, give
`0` as the fourth parameter to `http_parser_execute()`. Callbacks and errors
can still be encountered during an EOF, so one must still be prepared
to receive them.
Scalar valued message information such as `status_code`, `method`, and the
HTTP version are stored in the parser structure. This data is only
temporally stored in `http_parser` and gets reset on each new message. If
this information is needed later, copy it out of the structure during the
`headers_complete` callback.
The parser decodes the transfer-encoding for both requests and responses
transparently. That is, a chunked encoding is decoded before being sent to
the on_body callback.
The Special Problem of Upgrade
------------------------------
HTTP supports upgrading the connection to a different protocol. An
increasingly common example of this is the Web Socket protocol which sends
a request like
GET /demo HTTP/1.1
Upgrade: WebSocket
Connection: Upgrade
Host: example.com
Origin: http://example.com
WebSocket-Protocol: sample
followed by non-HTTP data.
(See http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-75 for more
information the Web Socket protocol.)
To support this, the parser will treat this as a normal HTTP message without a
body. Issuing both on_headers_complete and on_message_complete callbacks. However
http_parser_execute() will stop parsing at the end of the headers and return.
The user is expected to check if `parser->upgrade` has been set to 1 after
`http_parser_execute()` returns. Non-HTTP data begins at the buffer supplied
offset by the return value of `http_parser_execute()`.
Callbacks
---------
During the `http_parser_execute()` call, the callbacks set in
`http_parser_settings` will be executed. The parser maintains state and
never looks behind, so buffering the data is not necessary. If you need to
save certain data for later usage, you can do that from the callbacks.
There are two types of callbacks:
* notification `typedef int (*http_cb) (http_parser*);`
Callbacks: on_message_begin, on_headers_complete, on_message_complete.
* data `typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);`
Callbacks: (requests only) on_uri,
(common) on_header_field, on_header_value, on_body;
Callbacks must return 0 on success. Returning a non-zero value indicates
error to the parser, making it exit immediately.
In case you parse HTTP message in chunks (i.e. `read()` request line
from socket, parse, read half headers, parse, etc) your data callbacks
may be called more than once. Http-parser guarantees that data pointer is only
valid for the lifetime of callback. You can also `read()` into a heap allocated
buffer to avoid copying memory around if this fits your application.
Reading headers may be a tricky task if you read/parse headers partially.
Basically, you need to remember whether last header callback was field or value
and apply following logic:
(on_header_field and on_header_value shortened to on_h_*)
------------------------ ------------ --------------------------------------------
| State (prev. callback) | Callback | Description/action |
------------------------ ------------ --------------------------------------------
| nothing (first call) | on_h_field | Allocate new buffer and copy callback data |
| | | into it |
------------------------ ------------ --------------------------------------------
| value | on_h_field | New header started. |
| | | Copy current name,value buffers to headers |
| | | list and allocate new buffer for new name |
------------------------ ------------ --------------------------------------------
| field | on_h_field | Previous name continues. Reallocate name |
| | | buffer and append callback data to it |
------------------------ ------------ --------------------------------------------
| field | on_h_value | Value for current header started. Allocate |
| | | new buffer and copy callback data to it |
------------------------ ------------ --------------------------------------------
| value | on_h_value | Value continues. Reallocate value buffer |
| | | and append callback data to it |
------------------------ ------------ --------------------------------------------
See examples of reading in headers:
* [partial example](http://gist.github.com/155877) in C
* [from http-parser tests](http://github.com/ry/http-parser/blob/37a0ff8928fb0d83cec0d0d8909c5a4abcd221af/test.c#L403) in C
* [from Node library](http://github.com/ry/node/blob/842eaf446d2fdcb33b296c67c911c32a0dabc747/src/http.js#L284) in Javascript
================================================
FILE: proxygen/external/http_parser/http_parser.h
================================================
/* Copyright Joyent, Inc. and other Node contributors. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#ifndef http_parser_h
#define http_parser_h
#define HTTP_PARSER_VERSION_MAJOR 1
#define HTTP_PARSER_VERSION_MINOR 0
#include
#if defined(_WIN32) && !defined(__MINGW32__) && \
(!defined(_MSC_VER) || _MSC_VER<1600) && !defined(__WINE__)
#include // @manual
#include
typedef __int8 int8_t;
typedef unsigned __int8 uint8_t;
typedef __int16 int16_t;
typedef unsigned __int16 uint16_t;
typedef __int32 int32_t;
typedef unsigned __int32 uint32_t;
typedef __int64 int64_t;
typedef unsigned __int64 uint64_t;
#else
#include
#endif
#if __cplusplus
namespace proxygen {
#endif /* __cplusplus */
/* Compile with -DHTTP_PARSER_STRICT_URL=1 to parse URLs
* strictly according to the RFCs
*/
#ifndef HTTP_PARSER_STRICT_URL
# define HTTP_PARSER_STRICT_URL 0
#endif
/* Compile with -DHTTP_PARSER_STRICT_HOSTNAME=1 to parse hostnames
* strictly according to the RFCs
*/
#ifndef HTTP_PARSER_STRICT_HOSTNAME
# define HTTP_PARSER_STRICT_HOSTNAME 0
#endif
/* Compile with -DHTTP_PARSER_DEBUG=1 to add extra debugging information to
* the error reporting facility.
*/
#ifndef HTTP_PARSER_DEBUG
# define HTTP_PARSER_DEBUG 0
#endif
/* Maximium header size allowed */
#define HTTP_MAX_HEADER_SIZE (80*1024)
typedef struct http_parser http_parser;
typedef struct http_parser_settings http_parser_settings;
typedef struct http_parser_result http_parser_result;
/* Callbacks should return non-zero to indicate an error. The parser will
* then halt execution.
*
* The one exception is on_headers_complete. In a HTTP_RESPONSE parser
* returning '1' from on_headers_complete will tell the parser that it
* should not expect a body. This is used when receiving a response to a
* HEAD request which may contain 'Content-Length' or 'Transfer-Encoding:
* chunked' headers that indicate the presence of a body.
*
* http_data_cb does not return data chunks. It will be call arbitrarally
* many times for each string. E.G. you might get 10 callbacks for "on_path"
* each providing just a few characters more data.
*/
typedef int (*http_data_cb) (http_parser*, const char *at, size_t length);
typedef int (*http_cb) (http_parser*);
/* Request Methods */
enum http_method
{ HTTP_DELETE = 0
, HTTP_GET
, HTTP_HEAD
, HTTP_POST
, HTTP_PUT
/* pathological */
, HTTP_CONNECT
, HTTP_OPTIONS
, HTTP_TRACE
/* webdav */
, HTTP_COPY
, HTTP_LOCK
, HTTP_MKCOL
, HTTP_MOVE
, HTTP_PROPFIND
, HTTP_PROPPATCH
, HTTP_UNLOCK
/* subversion */
, HTTP_REPORT
, HTTP_MKACTIVITY
, HTTP_CHECKOUT
, HTTP_MERGE
/* upnp */
, HTTP_MSEARCH
, HTTP_NOTIFY
, HTTP_SUBSCRIBE
, HTTP_UNSUBSCRIBE
/* RFC-5789 */
, HTTP_PATCH
};
enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH };
/* Flag values for http_parser.flags field */
enum flags
{ F_CHUNKED = 1 << 0
, F_TRAILING = 1 << 3
, F_UPGRADE = 1 << 4
, F_SKIPBODY = 1 << 5
};
/* Map for errno-related constants
*
* The provided argument should be a macro that takes 2 arguments.
*/
#define HTTP_ERRNO_MAP(XX) \
/* No error */ \
XX(OK, "success") \
\
/* Callback-related errors */ \
XX(CB_message_begin, "the on_message_begin callback failed") \
XX(CB_path, "the on_path callback failed") \
XX(CB_query_string, "the on_query_string callback failed") \
XX(CB_url, "the on_url callback failed") \
XX(CB_fragment, "the on_fragment callback failed") \
XX(CB_header_field, "the on_header_field callback failed") \
XX(CB_header_value, "the on_header_value callback failed") \
XX(CB_headers_complete, "the on_headers_complete callback failed") \
XX(CB_body, "the on_body callback failed") \
XX(CB_message_complete, "the on_message_complete callback failed") \
XX(CB_reason, "the on_reason callback failed") \
XX(CB_chunk_header, "the on_chunk_header callback failed") \
XX(CB_chunk_complete, "the on_chunk_complete callback failed") \
\
/* Parsing-related errors */ \
XX(INVALID_EOF_STATE, "stream ended at an unexpected time") \
XX(HEADER_OVERFLOW, \
"too many header bytes seen; overflow detected") \
XX(CLOSED_CONNECTION, \
"data received after completed connection: close message") \
XX(INVALID_VERSION, "invalid HTTP version") \
XX(INVALID_STATUS, "invalid HTTP status code") \
XX(INVALID_METHOD, "invalid HTTP method") \
XX(INVALID_URL, "invalid URL") \
XX(INVALID_HOST, "invalid host") \
XX(INVALID_PORT, "invalid port") \
XX(INVALID_PATH, "invalid path") \
XX(INVALID_QUERY_STRING, "invalid query string") \
XX(INVALID_FRAGMENT, "invalid fragment") \
XX(LF_EXPECTED, "LF character expected") \
XX(INVALID_HEADER_TOKEN, "invalid character in header") \
XX(INVALID_CONTENT_LENGTH, \
"invalid character in content-length header") \
XX(HUGE_CONTENT_LENGTH, \
"content-length header too large") \
XX(INVALID_CHUNK_SIZE, \
"invalid character in chunk size header") \
XX(HUGE_CHUNK_SIZE, \
"chunk header size too large") \
XX(INVALID_TRANSFER_ENCODING, \
"invalid character in transfer-encoding header") \
XX(INVALID_UPGRADE, \
"invalid character in upgrade header") \
XX(INVALID_CONSTANT, "invalid constant string") \
XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\
XX(STRICT, "strict mode assertion failed") \
XX(PAUSED, "parser is paused") \
XX(UNKNOWN, "an unknown error occurred")
/* Define HPE_* values for each errno value above */
#define HTTP_ERRNO_GEN(n, s) HPE_##n,
enum http_errno {
HTTP_ERRNO_MAP(HTTP_ERRNO_GEN)
};
#undef HTTP_ERRNO_GEN
/* Get an http_errno value from an http_parser */
#define HTTP_PARSER_ERRNO(p) ((enum http_errno) (p)->http_errno)
/* Get the line number that generated the current error */
#if HTTP_PARSER_DEBUG
#define HTTP_PARSER_ERRNO_LINE(p) ((p)->error_lineno)
#else
#define HTTP_PARSER_ERRNO_LINE(p) 0
#endif
struct http_parser {
/** PRIVATE **/
unsigned char type : 2; /* enum http_parser_type */
unsigned char flags : 6; /* F_* values from 'flags' enum; semi-public */
unsigned char state; /* enum state from http_parser.c */
unsigned char header_state; /* enum header_state from http_parser.c */
unsigned char index; /* index into current matcher */
uint32_t nread; /* # bytes read in various scenarios */
int64_t content_length; /* # bytes in body (0 if no Content-Length header) */
/** READ-ONLY **/
unsigned short http_major;
unsigned short http_minor;
unsigned short status_code; /* responses only */
unsigned char method; /* requests only */
unsigned char http_errno : 7;
/* 1 = Upgrade header was present and the parser has exited because of that.
* 0 = No upgrade header present.
* Should be checked when http_parser_execute() returns in addition to
* error checking.
*/
char upgrade : 1;
#if HTTP_PARSER_DEBUG
uint32_t error_lineno;
#endif
/** PUBLIC **/
void *data; /* A pointer to get hook to the "connection" or "socket" object */
};
struct http_parser_settings {
http_cb on_message_begin;
http_data_cb on_url;
http_data_cb on_header_field;
http_data_cb on_header_value;
http_data_cb on_headers_complete;
http_data_cb on_body;
http_cb on_message_complete;
http_data_cb on_reason;
/* When on_chunk_header is called, the current chunk length is stored
* in parser->content_length.
*/
http_cb on_chunk_header;
http_cb on_chunk_complete;
};
enum http_parser_url_fields
{ UF_SCHEMA = 0
, UF_HOST = 1
, UF_PORT = 2
, UF_PATH = 3
, UF_QUERY = 4
, UF_FRAGMENT = 5
, UF_USERINFO = 6
, UF_MAX = 7
};
/* Result structure for http_parser_parse_url().
*
* Callers should index into field_data[] with UF_* values iff field_set
* has the relevant (1 << UF_*) bit set. As a courtesy to clients (and
* because we probably have padding left over), we convert any port to
* a uint16_t.
*/
struct http_parser_url {
uint16_t field_set; /* Bitmask of (1 << UF_*) values */
uint16_t port; /* Converted UF_PORT string */
struct {
uint16_t off; /* Offset into buffer in which field starts */
uint16_t len; /* Length of run in buffer */
} field_data[UF_MAX];
};
void http_parser_init(http_parser *parser, enum http_parser_type type);
size_t http_parser_execute(http_parser *parser,
const http_parser_settings *settings,
const char *data,
size_t len);
/* Begin Facebook */
enum http_parser_options
{
F_HTTP_PARSER_OPTIONS_URL_STRICT = (1 << 0)
};
size_t http_parser_execute_options(http_parser *parser,
const http_parser_settings *settings,
uint8_t options,
const char *data,
size_t len);
/* End Facebook */
/* Returns a string version of the HTTP method. */
const char *http_method_str(enum http_method m);
/* Return a string name of the given error */
const char *http_errno_name(enum http_errno err);
/* Return a string description of the given error */
const char *http_errno_description(enum http_errno err);
/* Parse a URL; return nonzero on failure */
int http_parser_parse_url(const char *buf, size_t buflen,
int is_connect,
struct http_parser_url *u);
/* Begin Facebook */
enum http_parser_parse_url_options
{
F_PARSE_URL_OPTIONS_URL_STRICT = (1 << 0)
};
int http_parser_parse_url_options(
const char *buf, size_t buflen,
int is_connect,
struct http_parser_url *u,
uint8_t options);
/* End Facebook */
/* Pause or un-pause the parser; a nonzero value pauses */
void http_parser_pause(http_parser *parser, int paused);
#if __cplusplus
}
#endif /* __cplusplus */
#endif
================================================
FILE: proxygen/external/http_parser/http_parser_cpp.cpp
================================================
/* Based on src/http/ngx_http_parse.c from NGINX copyright Igor Sysoev
*
* Additional changes are licensed under the same terms as NGINX and
* copyright Joyent, Inc. and other Node contributors. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "proxygen/external/http_parser/http_parser.h"
#include
#include
#include
#if __cplusplus
#include
namespace proxygen {
#ifndef INT64_MAX
# define INT64_MAX std::numeric_limits::max()
#endif
#else
#include
#define nullptr NULL
#endif /* __cplusplus */
#ifndef MIN
# define MIN(a,b) ((a) < (b) ? (a) : (b))
#endif
#if HTTP_PARSER_DEBUG
#define SET_ERRNO(e) \
do { \
parser->http_errno = (e); \
parser->error_lineno = __LINE__; \
} while (0)
#else
#define SET_ERRNO(e) \
do { \
parser->http_errno = (e); \
} while(0)
#endif
#define RETURN(r) \
do { \
parser->state = state; \
return (r); \
} while(0)
/* Run the notify callback FOR, returning ER if it fails */
#define _CALLBACK_NOTIFY(FOR, ER) \
do { \
parser->state = state; \
assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \
\
if (0 != settings->on_##FOR(parser)) { \
SET_ERRNO(HPE_CB_##FOR); \
} \
\
/* We either errored above or got paused; get out */ \
if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { \
return (ER); \
} \
} while (0)
/* Run the notify callback FOR and consume the current byte */
#define CALLBACK_NOTIFY(FOR) _CALLBACK_NOTIFY(FOR, p - data + 1)
/* Run the notify callback FOR and don't consume the current byte */
#define CALLBACK_NOTIFY_NOADVANCE(FOR) _CALLBACK_NOTIFY(FOR, p - data)
/* Run data callback FOR with LEN bytes, returning ER if it fails */
#define _CALLBACK_DATA(FOR, LEN, ER) \
do { \
parser->state = state; \
assert(HTTP_PARSER_ERRNO(parser) == HPE_OK); \
\
if (FOR##_mark) { \
if (0 != settings->on_##FOR(parser, FOR##_mark, (LEN))) { \
SET_ERRNO(HPE_CB_##FOR); \
} \
\
/* We either errored above or got paused; get out */ \
if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { \
return (ER); \
} \
FOR##_mark = nullptr; \
} \
} while (0)
/* Run the data callback FOR and consume the current byte */
#define CALLBACK_DATA(FOR) \
_CALLBACK_DATA(FOR, p - FOR##_mark, p - data + 1)
/* Run the data callback FOR and don't consume the current byte */
#define CALLBACK_DATA_NOADVANCE(FOR) \
_CALLBACK_DATA(FOR, p - FOR##_mark, p - data)
/* We just saw a synthetic space */
#define CALLBACK_SPACE(FOR) \
do { \
parser->state = state; \
if (0 != settings->on_##FOR(parser, SPACE, 1)) { \
SET_ERRNO(HPE_CB_##FOR); \
return (p - data); \
} \
\
/* We either errored above or got paused; get out */ \
if (HTTP_PARSER_ERRNO(parser) != HPE_OK) { \
return (p - data); \
} \
} while (0)
/* Set the mark FOR; non-destructive if mark is already set */
#define MARK(FOR) \
do { \
if (!FOR##_mark) { \
FOR##_mark = p; \
} \
} while (0)
#define CONTENT_LENGTH "content-length"
#define TRANSFER_ENCODING "transfer-encoding"
#define UPGRADE "upgrade"
#define CHUNKED "chunked"
#define SPACE " "
static const char *method_strings[] =
{ "DELETE"
, "GET"
, "HEAD"
, "POST"
, "PUT"
, "CONNECT"
, "OPTIONS"
, "TRACE"
, "COPY"
, "LOCK"
, "MKCOL"
, "MOVE"
, "PROPFIND"
, "PROPPATCH"
, "UNLOCK"
, "REPORT"
, "MKACTIVITY"
, "CHECKOUT"
, "MERGE"
, "M-SEARCH"
, "NOTIFY"
, "SUBSCRIBE"
, "UNSUBSCRIBE"
, "PATCH"
};
/* Tokens as defined by rfc 2616. Also lowercases them.
* token = 1*
* separators = "(" | ")" | "<" | ">" | "@"
* | "," | ";" | ":" | "\" | <">
* | "/" | "[" | "]" | "?" | "="
* | "{" | "}" | SP | HT
*/
static const char tokens[256] = {
/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */
0, 0, 0, 0, 0, 0, 0, 0,
/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */
0, 0, 0, 0, 0, 0, 0, 0,
/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */
0, 0, 0, 0, 0, 0, 0, 0,
/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */
0, 0, 0, 0, 0, 0, 0, 0,
/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */
0, '!', 0, '#', '$', '%', '&', '\'',
/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */
0, 0, '*', '+', 0, '-', '.', 0,
/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */
'0', '1', '2', '3', '4', '5', '6', '7',
/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */
'8', '9', 0, 0, 0, 0, 0, 0,
/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */
0, 'a', 'b', 'c', 'd', 'e', 'f', 'g',
/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */
'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */
'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */
'x', 'y', 'z', 0, 0, 0, '^', '_',
/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */
'`', 'a', 'b', 'c', 'd', 'e', 'f', 'g',
/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */
'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */
'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */
'x', 'y', 'z', 0, '|', 0, '~', 0 };
static const int8_t unhex[256] =
{-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1
,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
};
#if HTTP_PARSER_STRICT_URL
/* The value here is "0b10000000" so when shifted by
* F_PARSE_URL_OPTIONS_URL_STRICT/F_HTTP_PARSER_OPTIONS_URL_STRICT it remains
* > 0 for non-strict (0) and becomes 0 for strict (1 << 0 = 1)
*/
static_assert(F_PARSE_URL_OPTIONS_URL_STRICT == 1, "must be 1!");
static_assert(F_HTTP_PARSER_OPTIONS_URL_STRICT == 1, "must be 1!");
# define T(v) 0x80
# define IS_PARSER_STRICT(options) \
((options) & F_HTTP_PARSER_OPTIONS_URL_STRICT)
#else
# define T(v) v
# define IS_PARSER_STRICT(options) (0)
#endif
static const uint8_t normal_url_char[256] = {
/* 0 nul 1 soh 2 stx 3 etx 4 eot 5 enq 6 ack 7 bel */
0, 0, 0, 0, 0, 0, 0, 0,
/* 8 bs 9 ht 10 nl 11 vt 12 np 13 cr 14 so 15 si */
0, T(1), 0, 0, T(1), 0, 0, 0,
/* 16 dle 17 dc1 18 dc2 19 dc3 20 dc4 21 nak 22 syn 23 etb */
0, 0, 0, 0, 0, 0, 0, 0,
/* 24 can 25 em 26 sub 27 esc 28 fs 29 gs 30 rs 31 us */
0, 0, 0, 0, 0, 0, 0, 0,
/* 32 sp 33 ! 34 " 35 # 36 $ 37 % 38 & 39 ' */
0, 1, 1, 0, 1, 1, 1, 1,
/* 40 ( 41 ) 42 * 43 + 44 , 45 - 46 . 47 / */
1, 1, 1, 1, 1, 1, 1, 1,
/* 48 0 49 1 50 2 51 3 52 4 53 5 54 6 55 7 */
1, 1, 1, 1, 1, 1, 1, 1,
/* 56 8 57 9 58 : 59 ; 60 < 61 = 62 > 63 ? */
1, 1, 1, 1, 1, 1, 1, 0,
/* 64 @ 65 A 66 B 67 C 68 D 69 E 70 F 71 G */
1, 1, 1, 1, 1, 1, 1, 1,
/* 72 H 73 I 74 J 75 K 76 L 77 M 78 N 79 O */
1, 1, 1, 1, 1, 1, 1, 1,
/* 80 P 81 Q 82 R 83 S 84 T 85 U 86 V 87 W */
1, 1, 1, 1, 1, 1, 1, 1,
/* 88 X 89 Y 90 Z 91 [ 92 \ 93 ] 94 ^ 95 _ */
1, 1, 1, 1, 1, 1, 1, 1,
/* 96 ` 97 a 98 b 99 c 100 d 101 e 102 f 103 g */
1, 1, 1, 1, 1, 1, 1, 1,
/* 104 h 105 i 106 j 107 k 108 l 109 m 110 n 111 o */
1, 1, 1, 1, 1, 1, 1, 1,
/* 112 p 113 q 114 r 115 s 116 t 117 u 118 v 119 w */
1, 1, 1, 1, 1, 1, 1, 1,
/* 120 x 121 y 122 z 123 { 124 | 125 } 126 ~ 127 del */
1, 1, 1, 1, 1, 1, 1, 0, };
#undef T
enum state
{ s_dead = 1 /* important that this is > 0 */
, s_pre_start_req_or_res
, s_start_req_or_res
, s_res_or_resp_H
, s_pre_start_res
, s_start_res
, s_res_H
, s_res_HT
, s_res_HTT
, s_res_HTTP
, s_res_first_http_major
, s_res_http_major
, s_res_first_http_minor
, s_res_http_minor
, s_res_first_status_code
, s_res_status_code
, s_res_status_start
, s_res_status
, s_res_line_almost_done
, s_pre_start_req
, s_start_req
, s_req_method
, s_req_spaces_before_url
, s_req_schema
, s_req_schema_slash
, s_req_schema_slash_slash
, s_req_server_start
, s_req_server
, s_req_server_with_at
, s_req_host_start
, s_req_host
, s_req_host_ipv6
, s_req_host_done
, s_req_port
, s_req_path
, s_req_query_string_start
, s_req_query_string
, s_req_fragment_start
, s_req_fragment
, s_req_http_start
, s_req_http_H
, s_req_http_HT
, s_req_http_HTT
, s_req_http_HTTP
, s_req_first_http_major
, s_req_http_major
, s_req_first_http_minor
, s_req_http_minor
, s_req_line_almost_done
, s_header_field_start
, s_header_field
, s_header_value_start
, s_header_value
, s_header_value_lws
, s_header_almost_done
, s_chunk_size_start
, s_chunk_size
, s_chunk_parameters
, s_chunk_size_almost_done
, s_headers_almost_done
, s_headers_done
/* Important: 's_headers_done' must be the last 'header' state. All
* states beyond this must be 'body' states. It is used for overflow
* checking. See the PARSING_HEADER() macro.
*/
, s_chunk_data
, s_chunk_data_almost_done
, s_chunk_data_done
, s_body_identity
, s_body_identity_eof
, s_message_done
};
#define PARSING_HEADER(state) (state <= s_headers_done)
enum header_states
{ h_general = 0
, h_general_and_quote
, h_general_and_quote_and_escape
, h_matching_content_length
, h_matching_transfer_encoding
, h_matching_upgrade
, h_content_length
, h_transfer_encoding
, h_upgrade
, h_matching_transfer_encoding_chunked
, h_transfer_encoding_chunked
};
enum http_host_state
{
s_http_host_dead = 1
, s_http_userinfo_start
, s_http_userinfo
, s_http_host_start
, s_http_host_v6_start
, s_http_host
, s_http_host_v6
, s_http_host_v6_end
, s_http_host_port_start
, s_http_host_port
};
/* Macros for character classes; depends on strict-mode */
#define CR '\r'
#define LF '\n'
#define QT '"'
#define BS '\\'
#define LOWER(c) (unsigned char)(c | 0x20)
#define TOKEN(c) (tokens[(unsigned char)c])
#define IS_ALPHA(c) (LOWER(c) >= 'a' && LOWER(c) <= 'z')
#define IS_NUM(c) ((c) >= '0' && (c) <= '9')
#define IS_ALPHANUM(c) (IS_ALPHA(c) || IS_NUM(c))
#define IS_HEX(c) (IS_NUM(c) || (LOWER(c) >= 'a' && LOWER(c) <= 'f'))
#define IS_MARK(c) ((c) == '-' || (c) == '_' || (c) == '.' || \
(c) == '!' || (c) == '~' || (c) == '*' || (c) == '\'' || (c) == '(' || \
(c) == ')')
#define IS_USERINFO_CHAR(c) (IS_ALPHANUM(c) || IS_MARK(c) || (c) == '%' || \
(c) == ';' || (c) == ':' || (c) == '&' || (c) == '=' || (c) == '+' || \
(c) == '$' || (c) == ',')
#if HTTP_PARSER_STRICT_URL
#define IS_URL_CHAR(c, strict) \
(((normal_url_char[(unsigned char) (c)] << (strict)) != 0) || \
(((c) & 0x80) && !(strict)))
#else
#define IS_URL_CHAR(c, strict) \
(normal_url_char[(unsigned char) (c)] || ((c) & 0x80))
#endif
#if HTTP_PARSER_STRICT_HOSTNAME
#define IS_HOST_CHAR(c) (IS_ALPHANUM(c) || (c) == '.' || (c) == '-')
#else
#define IS_HOST_CHAR(c) \
(IS_ALPHANUM(c) || (c) == '.' || (c) == '-' || (c) == '_')
#endif
/**
* Verify that a char is a valid visible (printable) US-ASCII
* character or %x80-FF
**/
#define IS_HEADER_CHAR(ch) \
(ch == CR || ch == LF || ch == 9 || ((unsigned char)ch > 31 && ch != 127))
#define start_state (parser->type == HTTP_REQUEST ? s_pre_start_req : s_pre_start_res)
#define NOOP_CHECK(cond)
#define STRICT_CHECK(cond) \
do { \
if (cond) { \
SET_ERRNO(HPE_STRICT); \
goto error; \
} \
} while (0)
#define NEW_MESSAGE() start_state
/* Map errno values to strings for human-readable output */
#define HTTP_STRERROR_GEN(n, s) { "HPE_" #n, s },
static struct {
const char *name;
const char *description;
} http_strerror_tab[] = {
HTTP_ERRNO_MAP(HTTP_STRERROR_GEN)
};
#undef HTTP_STRERROR_GEN
/* Our URL parser.
*
* This is designed to be shared by http_parser_execute() for URL validation,
* hence it has a state transition + byte-for-byte interface. In addition, it
* is meant to be embedded in http_parser_parse_url(), which does the dirty
* work of turning state transitions URL components for its API.
*
* This function should only be invoked with non-space characters. It is
* assumed that the caller cares about (and can detect) the transition between
* URL and non-URL states by looking for these.
*/
static enum state
parse_url_char(enum state s, const char ch, int strict_flag)
{
if (ch == ' ' || ch == '\r' || ch == '\n') {
return s_dead;
}
#if HTTP_PARSER_STRICT_URL
if (ch == '\t' || ch == '\f') {
return s_dead;
}
#endif
switch (s) {
case s_req_spaces_before_url:
/* Proxied requests are followed by scheme of an absolute URI (alpha).
* All methods except CONNECT are followed by '/' or '*'.
*/
if (ch == '/' || ch == '*') {
return s_req_path;
}
if (IS_ALPHA(ch)) {
return s_req_schema;
}
break;
case s_req_schema:
if (IS_ALPHA(ch)) {
return s;
}
if (ch == ':') {
return s_req_schema_slash;
}
break;
case s_req_schema_slash:
if (ch == '/') {
return s_req_schema_slash_slash;
}
break;
case s_req_schema_slash_slash:
if (ch == '/') {
return s_req_server_start;
}
break;
case s_req_server_with_at:
if (ch == '@') {
return s_dead;
}
#if __cplusplus
[[fallthrough]];
#else /* __cplusplus */
__attribute__((fallthrough));
#endif /* __cplusplus */
case s_req_server_start:
case s_req_server:
if (ch == '/') {
return s_req_path;
}
if (ch == '?') {
return s_req_query_string_start;
}
if (ch == '@') {
return s_req_server_with_at;
}
if (IS_USERINFO_CHAR(ch) || ch == '[' || ch == ']') {
return s_req_server;
}
break;
case s_req_path:
if (IS_URL_CHAR(ch, strict_flag)) {
return s;
}
switch (ch) {
case '?':
return s_req_query_string_start;
case '#':
return s_req_fragment_start;
}
break;
case s_req_query_string_start:
case s_req_query_string:
if (IS_URL_CHAR(ch, strict_flag)) {
return s_req_query_string;
}
switch (ch) {
case '?':
/* allow extra '?' in query string */
return s_req_query_string;
case '#':
return s_req_fragment_start;
}
break;
case s_req_fragment_start:
if (IS_URL_CHAR(ch, strict_flag)) {
return s_req_fragment;
}
switch (ch) {
case '?':
return s_req_fragment;
case '#':
return s;
}
break;
case s_req_fragment:
if (IS_URL_CHAR(ch, strict_flag)) {
return s;
}
switch (ch) {
case '?':
case '#':
return s;
}
break;
default:
break;
}
/* We should never fall out of the switch above unless there's an error */
return s_dead;
}
size_t http_parser_execute (http_parser *parser,
const http_parser_settings *settings,
const char *data,
size_t len)
{
return http_parser_execute_options(parser, settings, 0, data, len);
}
size_t http_parser_execute_options (http_parser *parser,
const http_parser_settings *settings,
uint8_t options,
const char *data,
size_t len)
{
char c, ch;
int8_t unhex_val;
const char *p = data;
/* Optimization: within the parsing loop below, we refer to this
* local copy of the state rather than parser->state. The compiler
* can't be sure whether parser->state will change during a callback,
* so it generates a lot of memory loads and stores to keep a register
* copy of the state in sync with the memory copy. We know, however,
* that the callbacks aren't allowed to change the parser state, so
* the parsing loop works with this local variable and only copies
* the value back to parser->loop before returning or invoking a
* callback.
*/
unsigned char state = parser->state;
const unsigned int lenient = 0;
/* We're in an error state. Don't bother doing anything. */
if (HTTP_PARSER_ERRNO(parser) != HPE_OK) {
RETURN(0);
}
if (len == 0) {
switch (state) {
case s_body_identity_eof:
/* Use of CALLBACK_NOTIFY() here would erroneously return 1 byte read if
* we got paused.
*/
CALLBACK_NOTIFY_NOADVANCE(message_complete);
RETURN(0);
case s_pre_start_req_or_res:
case s_pre_start_res:
case s_pre_start_req:
RETURN(0);
default:
SET_ERRNO(HPE_INVALID_EOF_STATE);
RETURN(1);
}
}
/* technically we could combine all of these (except for url_mark) into one
variable, saving stack space, but it seems more clear to have them
separated. */
const char *header_field_mark = nullptr;
const char *header_value_mark = nullptr;
const char *url_mark = nullptr;
const char *reason_mark = nullptr;
const char *body_mark = nullptr;
if (state == s_header_field)
header_field_mark = data;
if (state == s_header_value)
header_value_mark = data;
if (state == s_req_path ||
state == s_req_schema ||
state == s_req_schema_slash ||
state == s_req_schema_slash_slash ||
state == s_req_port ||
state == s_req_query_string_start ||
state == s_req_query_string ||
state == s_req_host_start ||
state == s_req_host ||
state == s_req_host_ipv6 ||
state == s_req_host_done ||
state == s_req_fragment_start ||
state == s_req_fragment)
url_mark = data;
if (state == s_res_status)
reason_mark = data;
/* Used only for overflow checking. If the parser is in a parsing-headers
* state, then its value is equal to max(data, the beginning of the current
* message or chunk). If the parser is in a not-parsing-headers state, then
* its value is irrelevant.
*/
const char* data_or_header_data_start = data;
for (p = data; p != data + len; p++) {
ch = *p;
reexecute_byte:
switch (state) {
case s_pre_start_req_or_res:
if (ch == CR || ch == LF)
break;
state = s_start_req_or_res;
CALLBACK_NOTIFY_NOADVANCE(message_begin);
goto reexecute_byte;
case s_start_req_or_res:
{
parser->flags = 0;
parser->content_length = -1;
if (ch == 'H') {
state = s_res_or_resp_H;
} else {
parser->type = HTTP_REQUEST;
state = s_start_req;
goto reexecute_byte;
}
break;
}
case s_res_or_resp_H:
if (ch == 'T') {
parser->type = HTTP_RESPONSE;
state = s_res_HT;
} else {
if (ch != 'E') {
SET_ERRNO(HPE_INVALID_CONSTANT);
goto error;
}
parser->type = HTTP_REQUEST;
parser->method = HTTP_HEAD;
parser->index = 2;
state = s_req_method;
}
break;
case s_pre_start_res:
if (ch == CR || ch == LF)
break;
state = s_start_res;
CALLBACK_NOTIFY_NOADVANCE(message_begin);
goto reexecute_byte;
case s_start_res:
{
parser->flags = 0;
parser->content_length = -1;
switch (ch) {
case 'H':
state = s_res_H;
break;
default:
SET_ERRNO(HPE_INVALID_CONSTANT);
goto error;
}
break;
}
case s_res_H:
NOOP_CHECK(ch != 'T');
state = s_res_HT;
break;
case s_res_HT:
NOOP_CHECK(ch != 'T');
state = s_res_HTT;
break;
case s_res_HTT:
NOOP_CHECK(ch != 'P');
state = s_res_HTTP;
break;
case s_res_HTTP:
NOOP_CHECK(ch != '/');
state = s_res_first_http_major;
break;
case s_res_first_http_major:
if (ch < '0' || ch > '9') {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
parser->http_major = ch - '0';
state = s_res_http_major;
break;
/* major HTTP version or dot */
case s_res_http_major:
{
if (ch == '.') {
state = s_res_first_http_minor;
break;
}
if (!IS_NUM(ch)) {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
parser->http_major *= 10;
parser->http_major += ch - '0';
if (parser->http_major > 999) {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
break;
}
/* first digit of minor HTTP version */
case s_res_first_http_minor:
if (!IS_NUM(ch)) {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
parser->http_minor = ch - '0';
state = s_res_http_minor;
break;
/* minor HTTP version or end of request line */
case s_res_http_minor:
{
if (ch == ' ') {
state = s_res_first_status_code;
break;
}
if (!IS_NUM(ch)) {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
parser->http_minor *= 10;
parser->http_minor += ch - '0';
if (parser->http_minor > 999) {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
break;
}
case s_res_first_status_code:
{
if (!IS_NUM(ch)) {
if (ch == ' ') {
break;
}
SET_ERRNO(HPE_INVALID_STATUS);
goto error;
}
parser->status_code = ch - '0';
state = s_res_status_code;
break;
}
case s_res_status_code:
{
if (!IS_NUM(ch)) {
switch (ch) {
case ' ':
state = s_res_status;
break;
case CR:
state = s_res_line_almost_done;
break;
case LF:
state = s_header_field_start;
break;
default:
SET_ERRNO(HPE_INVALID_STATUS);
goto error;
}
break;
}
parser->status_code *= 10;
parser->status_code += ch - '0';
if (parser->status_code > 999) {
SET_ERRNO(HPE_INVALID_STATUS);
goto error;
}
break;
}
case s_res_status:
/* the human readable status. e.g. "NOT FOUND" */
MARK(reason);
if (ch == CR) {
state = s_res_line_almost_done;
CALLBACK_DATA(reason);
break;
}
if (ch == LF) {
state = s_header_field_start;
CALLBACK_DATA(reason);
break;
}
break;
case s_res_line_almost_done:
NOOP_CHECK(ch != LF);
state = s_header_field_start;
break;
case s_pre_start_req:
if (ch == CR || ch == LF) {
break;
}
state = s_start_req;
CALLBACK_NOTIFY_NOADVANCE(message_begin);
goto reexecute_byte;
case s_start_req:
{
parser->flags = 0;
parser->content_length = -1;
if (!IS_ALPHA(ch)) {
SET_ERRNO(HPE_INVALID_METHOD);
goto error;
}
parser->method = (enum http_method) 0;
parser->index = 1;
switch (ch) {
case 'C': parser->method = HTTP_CONNECT; /* or COPY, CHECKOUT */ break;
case 'D': parser->method = HTTP_DELETE; break;
case 'G': parser->method = HTTP_GET; break;
case 'H': parser->method = HTTP_HEAD; break;
case 'L': parser->method = HTTP_LOCK; break;
case 'M': parser->method = HTTP_MKCOL; /* or MOVE, MKACTIVITY, MERGE, M-SEARCH */ break;
case 'N': parser->method = HTTP_NOTIFY; break;
case 'O': parser->method = HTTP_OPTIONS; break;
case 'P': parser->method = HTTP_POST;
/* or PROPFIND or PROPPATCH or PUT or PATCH */
break;
case 'R': parser->method = HTTP_REPORT; break;
case 'S': parser->method = HTTP_SUBSCRIBE; break;
case 'T': parser->method = HTTP_TRACE; break;
case 'U': parser->method = HTTP_UNLOCK; /* or UNSUBSCRIBE */ break;
default:
SET_ERRNO(HPE_INVALID_METHOD);
goto error;
}
state = s_req_method;
break;
}
case s_req_method:
{
if (ch == '\0') {
SET_ERRNO(HPE_INVALID_METHOD);
goto error;
}
const char *matcher = method_strings[parser->method];
if (ch == ' ' && matcher[parser->index] == '\0') {
state = s_req_spaces_before_url;
} else if (ch == matcher[parser->index]) {
/* nada */
} else if (parser->method == HTTP_CONNECT) {
if (parser->index == 1 && ch == 'H') {
parser->method = HTTP_CHECKOUT;
} else if (parser->index == 2 && ch == 'P') {
parser->method = HTTP_COPY;
} else {
goto error;
}
} else if (parser->method == HTTP_MKCOL) {
if (parser->index == 1 && ch == 'O') {
parser->method = HTTP_MOVE;
} else if (parser->index == 1 && ch == 'E') {
parser->method = HTTP_MERGE;
} else if (parser->index == 1 && ch == '-') {
parser->method = HTTP_MSEARCH;
} else if (parser->index == 2 && ch == 'A') {
parser->method = HTTP_MKACTIVITY;
} else {
goto error;
}
} else if (parser->index == 1 && parser->method == HTTP_POST) {
if (ch == 'R') {
parser->method = HTTP_PROPFIND; /* or HTTP_PROPPATCH */
} else if (ch == 'U') {
parser->method = HTTP_PUT;
} else if (ch == 'A') {
parser->method = HTTP_PATCH;
} else {
goto error;
}
} else if (parser->index == 2 && parser->method == HTTP_UNLOCK && ch == 'S') {
parser->method = HTTP_UNSUBSCRIBE;
} else if (parser->index == 4 && parser->method == HTTP_PROPFIND && ch == 'P') {
parser->method = HTTP_PROPPATCH;
} else {
SET_ERRNO(HPE_INVALID_METHOD);
goto error;
}
++parser->index;
break;
}
case s_req_spaces_before_url:
{
if (ch == ' ') break;
// CONNECT requests must be followed by a :
if (parser->method == HTTP_CONNECT) {
MARK(url);
state = s_req_host_start;
goto reexecute_byte;
}
if (ch == '/' || ch == '*') {
MARK(url);
state = s_req_path;
break;
}
/* Proxied requests are followed by scheme of an absolute URI (alpha).
* All other methods are followed by '/' or '*' (handled above).
*/
if (IS_ALPHA(ch)) {
MARK(url);
state = s_req_schema;
break;
}
SET_ERRNO(HPE_INVALID_URL);
goto error;
}
case s_req_schema:
{
if (IS_ALPHA(ch)) break;
if (ch == ':') {
state = s_req_schema_slash;
break;
}
SET_ERRNO(HPE_INVALID_URL);
goto error;
}
case s_req_schema_slash:
NOOP_CHECK(ch != '/');
state = s_req_schema_slash_slash;
break;
case s_req_schema_slash_slash:
NOOP_CHECK(ch != '/');
state = s_req_host_start;
break;
case s_req_host_start:
if (ch == '[') {
state = s_req_host_ipv6;
break;
} else if (IS_ALPHANUM(ch)) {
state = s_req_host;
break;
}
SET_ERRNO(HPE_INVALID_HOST);
goto error;
case s_req_host:
if (IS_HOST_CHAR(ch)) break;
state = s_req_host_done;
goto reexecute_byte;
case s_req_host_ipv6:
if (IS_HEX(ch) || ch == ':') break;
if (ch == ']') {
state = s_req_host_done;
break;
}
SET_ERRNO(HPE_INVALID_HOST);
goto error;
case s_req_host_done:
switch (ch) {
case ':':
state = s_req_port;
break;
case '/':
state = s_req_path;
break;
case ' ':
/* The request line looks like:
* "GET http://foo.bar.com HTTP/1.1"
* That is, there is no path.
*/
state = s_req_http_start;
CALLBACK_DATA(url);
break;
case '?':
state = s_req_query_string_start;
break;
default:
SET_ERRNO(HPE_INVALID_HOST);
goto error;
}
break;
case s_req_port:
{
if (IS_NUM(ch)) break;
switch (ch) {
case '/':
state = s_req_path;
break;
case ' ':
/* The request line looks like:
* "GET http://foo.bar.com:1234 HTTP/1.1"
* That is, there is no path.
*/
state = s_req_http_start;
CALLBACK_DATA(url);
break;
case '?':
state = s_req_query_string_start;
break;
default:
SET_ERRNO(HPE_INVALID_PORT);
goto error;
}
break;
}
case s_req_path:
{
if (IS_URL_CHAR(ch, IS_PARSER_STRICT(options))) break;
switch (ch) {
case ' ':
state = s_req_http_start;
CALLBACK_DATA(url);
break;
case CR:
parser->http_major = 0;
parser->http_minor = 9;
state = s_headers_almost_done;
CALLBACK_DATA(url);
break;
case LF:
parser->http_major = 0;
parser->http_minor = 9;
state = s_headers_almost_done;
CALLBACK_DATA(url);
goto reexecute_byte;
case '?':
state = s_req_query_string_start;
break;
case '#':
state = s_req_fragment_start;
break;
default:
SET_ERRNO(HPE_INVALID_PATH);
goto error;
}
break;
}
case s_req_query_string_start:
{
if (IS_URL_CHAR(ch, IS_PARSER_STRICT(options))) {
state = s_req_query_string;
break;
}
switch (ch) {
case '?':
break; /* XXX ignore extra '?' ... is this right? */
case ' ':
state = s_req_http_start;
CALLBACK_DATA(url);
break;
case CR:
parser->http_major = 0;
parser->http_minor = 9;
state = s_headers_almost_done;
CALLBACK_DATA(url);
break;
case LF:
parser->http_major = 0;
parser->http_minor = 9;
state = s_headers_almost_done;
CALLBACK_DATA(url);
goto reexecute_byte;
case '#':
state = s_req_fragment_start;
break;
default:
SET_ERRNO(HPE_INVALID_QUERY_STRING);
goto error;
}
break;
}
case s_req_query_string:
{
if (IS_URL_CHAR(ch, IS_PARSER_STRICT(options))) break;
switch (ch) {
case '?':
/* allow extra '?' in query string */
break;
case ' ':
state = s_req_http_start;
CALLBACK_DATA(url);
break;
case CR:
parser->http_major = 0;
parser->http_minor = 9;
state = s_headers_almost_done;
CALLBACK_DATA(url);
break;
case LF:
parser->http_major = 0;
parser->http_minor = 9;
state = s_headers_almost_done;
CALLBACK_DATA(url);
goto reexecute_byte;
case '#':
state = s_req_fragment_start;
break;
default:
SET_ERRNO(HPE_INVALID_QUERY_STRING);
goto error;
}
break;
}
case s_req_fragment_start:
{
if (IS_URL_CHAR(ch, IS_PARSER_STRICT(options))) {
state = s_req_fragment;
break;
}
switch (ch) {
case ' ':
state = s_req_http_start;
CALLBACK_DATA(url);
break;
case CR:
parser->http_major = 0;
parser->http_minor = 9;
state = s_headers_almost_done;
CALLBACK_DATA(url);
break;
case LF:
parser->http_major = 0;
parser->http_minor = 9;
state = s_headers_almost_done;
CALLBACK_DATA(url);
goto reexecute_byte;
case '?':
state = s_req_fragment;
break;
case '#':
break;
default:
SET_ERRNO(HPE_INVALID_FRAGMENT);
goto error;
}
break;
}
case s_req_fragment:
{
if (IS_URL_CHAR(ch, IS_PARSER_STRICT(options))) break;
switch (ch) {
case ' ':
state = s_req_http_start;
CALLBACK_DATA(url);
break;
case CR:
parser->http_major = 0;
parser->http_minor = 9;
state = s_headers_almost_done;
CALLBACK_DATA(url);
break;
case LF:
parser->http_major = 0;
parser->http_minor = 9;
state = s_headers_almost_done;
CALLBACK_DATA(url);
goto reexecute_byte;
case '?':
case '#':
break;
default:
SET_ERRNO(HPE_INVALID_FRAGMENT);
goto error;
}
break;
}
case s_req_http_start:
switch (ch) {
case 'H':
state = s_req_http_H;
break;
case ' ':
break;
default:
SET_ERRNO(HPE_INVALID_CONSTANT);
goto error;
}
break;
case s_req_http_H:
NOOP_CHECK(ch != 'T');
state = s_req_http_HT;
break;
case s_req_http_HT:
NOOP_CHECK(ch != 'T');
state = s_req_http_HTT;
break;
case s_req_http_HTT:
NOOP_CHECK(ch != 'P');
state = s_req_http_HTTP;
break;
case s_req_http_HTTP:
NOOP_CHECK(ch != '/');
state = s_req_first_http_major;
break;
/* first digit of major HTTP version */
case s_req_first_http_major:
if (ch < '0' || ch > '9') {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
parser->http_major = ch - '0';
state = s_req_http_major;
break;
/* major HTTP version or dot */
case s_req_http_major:
{
if (ch == '.') {
state = s_req_first_http_minor;
break;
}
if (!IS_NUM(ch)) {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
parser->http_major *= 10;
parser->http_major += ch - '0';
if (parser->http_major > 999) {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
break;
}
/* first digit of minor HTTP version */
case s_req_first_http_minor:
if (!IS_NUM(ch)) {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
parser->http_minor = ch - '0';
state = s_req_http_minor;
break;
/* minor HTTP version or end of request line */
case s_req_http_minor:
{
if (ch == CR) {
if (parser->http_major== 0 && parser->http_minor == 9) {
state = s_headers_almost_done;
} else {
state = s_req_line_almost_done;
}
break;
}
if (ch == LF) {
if (parser->http_major == 0 && parser->http_minor == 9) {
state = s_headers_almost_done;
goto reexecute_byte;
} else {
state = s_header_field_start;
}
break;
}
/* XXX allow spaces after digit? */
if (!IS_NUM(ch)) {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
parser->http_minor *= 10;
parser->http_minor += ch - '0';
if (parser->http_minor > 999) {
SET_ERRNO(HPE_INVALID_VERSION);
goto error;
}
break;
}
/* end of request line */
case s_req_line_almost_done:
{
if (ch != LF) {
SET_ERRNO(HPE_LF_EXPECTED);
goto error;
}
state = s_header_field_start;
break;
}
case s_header_field_start:
{
if (ch == CR) {
state = s_headers_almost_done;
break;
}
if (ch == LF) {
/* they might be just sending \n instead of \r\n so this would be
* the second \n to denote the end of headers*/
state = s_headers_almost_done;
goto reexecute_byte;
}
c = TOKEN(ch);
if (!c) {
SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
goto error;
}
MARK(header_field);
parser->index = 0;
state = s_header_field;
switch (c) {
case 'c':
parser->header_state = h_matching_content_length;
break;
case 't':
parser->header_state = h_matching_transfer_encoding;
break;
case 'u':
parser->header_state = h_matching_upgrade;
break;
default:
parser->header_state = h_general;
break;
}
break;
}
case s_header_field:
{
c = TOKEN(ch);
if (c) {
switch (parser->header_state) {
case h_general:
// fast-forwarding, wheeeeeee!
#define MOVE_THE_HEAD do { \
++p; \
if (!TOKEN(*p)) { \
ch = *p; \
goto notatoken; \
} \
} while(0);
if (data + len - p >= 9) {
MOVE_THE_HEAD
MOVE_THE_HEAD
MOVE_THE_HEAD
MOVE_THE_HEAD
MOVE_THE_HEAD
MOVE_THE_HEAD
MOVE_THE_HEAD
MOVE_THE_HEAD
} else if (data + len - p >= 4) {
MOVE_THE_HEAD
MOVE_THE_HEAD
MOVE_THE_HEAD
}
break;
/* content-length */
case h_matching_content_length:
parser->index++;
if (parser->index > sizeof(CONTENT_LENGTH)-1
|| c != CONTENT_LENGTH[parser->index]) {
parser->header_state = h_general;
} else if (parser->index == sizeof(CONTENT_LENGTH)-2) {
parser->header_state = h_content_length;
}
break;
/* transfer-encoding */
case h_matching_transfer_encoding:
parser->index++;
if (parser->index > sizeof(TRANSFER_ENCODING)-1
|| c != TRANSFER_ENCODING[parser->index]) {
parser->header_state = h_general;
} else if (parser->index == sizeof(TRANSFER_ENCODING)-2) {
parser->header_state = h_transfer_encoding;
}
break;
/* upgrade */
case h_matching_upgrade:
parser->index++;
if (parser->index > sizeof(UPGRADE)-1
|| c != UPGRADE[parser->index]) {
parser->header_state = h_general;
} else if (parser->index == sizeof(UPGRADE)-2) {
parser->header_state = h_upgrade;
}
break;
case h_content_length:
case h_transfer_encoding:
case h_upgrade:
if (ch != ' ') parser->header_state = h_general;
break;
default:
assert(0 && "Unknown header_state");
break;
}
break;
}
notatoken:
if (ch == ':') {
state = s_header_value_start;
// do not allow headers with trailing whitespaces
// https://tools.ietf.org/html/rfc7230#section-3.2.4
if (p - header_field_mark > 1 &&
data[p - data - 1] == ' ') {
SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
goto error;
}
CALLBACK_DATA(header_field);
break;
}
SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
goto error;
}
case s_header_value_start:
{
if (ch == ' ' || ch == '\t') break;
MARK(header_value);
state = s_header_value;
parser->index = 0;
// Error out if a content_length, transfer_encoding, or upgrade header
// was present with no actual value. These headers correspond with
// special parser states that without the below accept empty header
// values and so we can reject such requests here in the parser.
// If more headers are added, can consider moving to a hash/map based
// model below.
if (ch == CR || ch == LF) {
if (parser->header_state == h_content_length) {
SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
} else if (parser->header_state == h_transfer_encoding) {
SET_ERRNO(HPE_INVALID_TRANSFER_ENCODING);
} else if (parser->header_state == h_upgrade) {
SET_ERRNO(HPE_INVALID_UPGRADE);
}
if (parser->http_errno != HPE_OK) {
goto error;
}
}
if (ch == CR) {
NOOP_CHECK(parser->quote != 0);
parser->header_state = h_general;
state = s_header_almost_done;
CALLBACK_DATA(header_value);
break;
}
if (ch == LF) {
NOOP_CHECK(parser->quote != 0);
state = s_header_field_start;
CALLBACK_DATA(header_value);
break;
}
c = LOWER(ch);
switch (parser->header_state) {
case h_upgrade:
parser->flags |= F_UPGRADE;
parser->header_state = h_general;
break;
case h_transfer_encoding:
/* looking for 'Transfer-Encoding: chunked' */
if ('c' == c) {
parser->header_state = h_matching_transfer_encoding_chunked;
} else {
parser->header_state = h_general;
}
break;
case h_content_length:
if (!IS_NUM(ch)) {
SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
goto error;
}
parser->content_length = ch - '0';
break;
default:
parser->header_state = ch == QT ? h_general_and_quote : h_general;
break;
}
break;
}
case s_header_value:
{
cr_or_lf_or_qt:
if (ch == CR &&
parser->header_state != h_general_and_quote_and_escape) {
state = s_header_almost_done;
CALLBACK_DATA(header_value);
break;
}
if (ch == LF &&
parser->header_state != h_general_and_quote_and_escape) {
state = s_header_almost_done;
CALLBACK_DATA_NOADVANCE(header_value);
goto reexecute_byte;
}
if (!lenient && !IS_HEADER_CHAR(ch) &&
parser->header_state != h_general_and_quote_and_escape) {
SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
goto error;
}
switch (parser->header_state) {
case h_general:
if (ch == QT) {
parser->header_state = h_general_and_quote;
}
// fast-forwarding, wheee!
#define MOVE_FAST do { \
++p; \
ch = *p; \
if (ch == CR || ch == LF || ch == QT || \
ch == BS || !IS_HEADER_CHAR(ch)) { \
goto cr_or_lf_or_qt; \
} \
} while(0);
if (data + len - p >= 12) {
MOVE_FAST
MOVE_FAST
MOVE_FAST
MOVE_FAST
MOVE_FAST
MOVE_FAST
MOVE_FAST
MOVE_FAST
MOVE_FAST
MOVE_FAST
MOVE_FAST
} else if (data + len - p >= 5) {
MOVE_FAST
MOVE_FAST
MOVE_FAST
MOVE_FAST
}
break;
case h_general_and_quote:
if (ch == QT) {
parser->header_state = h_general;
} else if (ch == BS) {
parser->header_state = h_general_and_quote_and_escape;
}
break;
case h_general_and_quote_and_escape:
parser->header_state = h_general_and_quote;
break;
// Not sure the below is relevant anymore as from
// s_header_value_start it appears as though we can never
// be in the situation below
case h_transfer_encoding:
SET_ERRNO(HPE_INVALID_HEADER_TOKEN);
goto error;
case h_content_length:
if (ch == ' ') break;
if (!IS_NUM(ch)) {
SET_ERRNO(HPE_INVALID_CONTENT_LENGTH);
goto error;
}
if (parser->content_length > ((INT64_MAX - 10) / 10)) {
/* overflow */
SET_ERRNO(HPE_HUGE_CONTENT_LENGTH);
goto error;
}
parser->content_length *= 10;
parser->content_length += ch - '0';
break;
/* Transfer-Encoding: chunked */
case h_matching_transfer_encoding_chunked:
parser->index++;
if (parser->index > sizeof(CHUNKED)-1
|| LOWER(ch) != CHUNKED[parser->index]) {
parser->header_state = h_general;
} else if (parser->index == sizeof(CHUNKED)-2) {
parser->header_state = h_transfer_encoding_chunked;
}
break;
case h_transfer_encoding_chunked:
if (ch != ' ') {
parser->header_state = h_general;
}
break;
default:
state = s_header_value;
parser->header_state = h_general;
break;
}
break;
}
case s_header_almost_done:
{
if (ch == LF) {
state = s_header_value_lws;
} else {
state = s_header_value;
}
switch (parser->header_state) {
case h_transfer_encoding_chunked:
parser->flags |= F_CHUNKED;
break;
default:
break;
}
if (ch != LF) {
CALLBACK_SPACE(header_value);
}
break;
}
case s_header_value_lws:
{
if (ch == ' ' || ch == '\t')
{
state = s_header_value_start;
CALLBACK_SPACE(header_value);
}
else
{
state = s_header_field_start;
goto reexecute_byte;
}
break;
}
case s_headers_almost_done:
{
NOOP_CHECK(ch != LF);
if (ch != LF) {
SET_ERRNO(HPE_STRICT);
goto error;
}
if (parser->flags & F_TRAILING) {
/* End of a chunked request */
state = s_message_done;
CALLBACK_NOTIFY_NOADVANCE(chunk_complete);
goto reexecute_byte;
}
state = s_headers_done;
/* Set this here so that on_headers_complete() callbacks can see it */
parser->upgrade =
(parser->flags & F_UPGRADE || parser->method == HTTP_CONNECT);
/* Here we call the headers_complete callback. This is somewhat
* different than other callbacks because if the user returns 1, we
* will interpret that as saying that this message has no body. This
* is needed for the annoying case of receiving a response to a HEAD
* request.
*
* We'd like to use CALLBACK_NOTIFY_NOADVANCE() here but we cannot, so
* we have to simulate it by handling a change in errno below.
*/
size_t header_size = p - data + 1;
switch (settings->on_headers_complete(parser, nullptr, header_size)) {
case 0:
break;
case 1:
parser->flags |= F_SKIPBODY;
break;
default:
SET_ERRNO(HPE_CB_headers_complete);
RETURN(p - data); /* Error */
}
if (HTTP_PARSER_ERRNO(parser) != HPE_OK) {
RETURN(p - data);
}
goto reexecute_byte;
}
case s_headers_done:
{
NOOP_CHECK(ch != LF);
// we're done parsing headers, reset overflow counters
parser->nread = 0;
// (if we now move to s_body_*, then this is irrelevant)
data_or_header_data_start = p;
int hasBody = parser->flags & F_CHUNKED || parser->content_length > 0;
if (parser->upgrade && (parser->method == HTTP_CONNECT ||
(parser->flags & F_SKIPBODY) || !hasBody)) {
/* Exit, the rest of the message is in a different protocol. */
state = NEW_MESSAGE();
CALLBACK_NOTIFY(message_complete);
RETURN((p - data) + 1);
}
if (parser->flags & F_SKIPBODY) {
state = NEW_MESSAGE();
CALLBACK_NOTIFY(message_complete);
} else if (parser->flags & F_CHUNKED) {
/* chunked encoding - ignore Content-Length header */
state = s_chunk_size_start;
} else {
if (parser->content_length == 0) {
/* Content-Length header given but zero: Content-Length: 0\r\n */
state = NEW_MESSAGE();
CALLBACK_NOTIFY(message_complete);
} else if (parser->content_length > 0) {
/* Content-Length header given and non-zero */
state = s_body_identity;
} else {
unsigned short sc = parser->status_code;
if (parser->type == HTTP_REQUEST ||
((100 <= sc && sc <= 199) || sc == 204 || sc == 304)) {
/* Assume content-length 0 - read the next */
state = NEW_MESSAGE();
CALLBACK_NOTIFY(message_complete);
} else {
/* Read body until EOF */
state = s_body_identity_eof;
}
}
}
break;
}
case s_body_identity:
{
uint64_t to_read = MIN(parser->content_length, (data + len) - p);
assert(parser->content_length > 0);
/* The difference between advancing content_length and p is because
* the latter will automatically advance on the next loop iteration.
* Further, if content_length ends up at 0, we want to see the last
* byte again for our message complete callback.
*/
MARK(body);
parser->content_length -= to_read;
p += to_read - 1;
if (parser->content_length == 0) {
state = s_message_done;
/* Mimic CALLBACK_DATA_NOADVANCE() but with one extra byte.
*
* The alternative to doing this is to wait for the next byte to
* trigger the data callback, just as in every other case. The
* problem with this is that this makes it difficult for the test
* harness to distinguish between complete-on-EOF and
* complete-on-length. It's not clear that this distinction is
* important for applications, but let's keep it for now.
*/
_CALLBACK_DATA(body, p - body_mark + 1, p - data);
goto reexecute_byte;
}
break;
}
/* read until EOF */
case s_body_identity_eof:
MARK(body);
p = data + len - 1;
break;
case s_message_done:
state = NEW_MESSAGE();
parser->nread = 0;
data_or_header_data_start = p;
CALLBACK_NOTIFY(message_complete);
if (parser->upgrade) {
/* Exit, the rest of the message is in a different protocol. */
RETURN((p - data) + 1);
}
break;
case s_chunk_size_start:
{
assert(parser->flags & F_CHUNKED);
unhex_val = unhex[(unsigned char)ch];
if (unhex_val == -1) {
SET_ERRNO(HPE_INVALID_CHUNK_SIZE);
goto error;
}
parser->content_length = unhex_val;
state = s_chunk_size;
break;
}
case s_chunk_size:
{
assert(parser->flags & F_CHUNKED);
if (ch == CR) {
state = s_chunk_size_almost_done;
break;
}
unhex_val = unhex[(unsigned char)ch];
if (unhex_val == -1) {
if (ch == ';' || ch == ' ') {
state = s_chunk_parameters;
break;
}
SET_ERRNO(HPE_INVALID_CHUNK_SIZE);
goto error;
}
if (parser->content_length > (INT64_MAX - unhex_val) >> 4) {
/* overflow */
SET_ERRNO(HPE_HUGE_CHUNK_SIZE);
goto error;
}
parser->content_length *= 16;
parser->content_length += unhex_val;
break;
}
case s_chunk_parameters:
{
assert(parser->flags & F_CHUNKED);
/*
* just ignore this shit. TODO check for overflow
* TODO: It would be nice to pass this information to the
* on_chunk_header callback.
*/
if (ch == CR) {
state = s_chunk_size_almost_done;
break;
}
break;
}
case s_chunk_size_almost_done:
{
assert(parser->flags & F_CHUNKED);
STRICT_CHECK(ch != LF);
if (parser->content_length == 0) {
parser->flags |= F_TRAILING;
state = s_header_field_start;
CALLBACK_NOTIFY(chunk_header);
} else {
state = s_chunk_data;
CALLBACK_NOTIFY(chunk_header);
}
break;
}
case s_chunk_data:
{
uint64_t to_read = MIN(parser->content_length, (data + len) - p);
assert(parser->flags & F_CHUNKED);
assert(parser->content_length > 0);
/* See the explanation in s_body_identity for why the content
* length and data pointers are managed this way.
*/
MARK(body);
parser->content_length -= to_read;
p += to_read - 1;
if (parser->content_length == 0) {
state = s_chunk_data_almost_done;
}
break;
}
case s_chunk_data_almost_done:
assert(parser->flags & F_CHUNKED);
assert(parser->content_length == 0);
NOOP_CHECK(ch != CR);
state = s_chunk_data_done;
CALLBACK_DATA(body);
break;
case s_chunk_data_done:
assert(parser->flags & F_CHUNKED);
NOOP_CHECK(ch != LF);
state = s_chunk_size_start;
parser->nread = 0;
data_or_header_data_start = p;
CALLBACK_NOTIFY(chunk_complete);
break;
default:
assert(0 && "unhandled state");
SET_ERRNO(HPE_INVALID_INTERNAL_STATE);
goto error;
}
}
/* We can check for overflow here because in Proxygen, len <= ~8KB and so the
* worst thing that can happen is that we catch the overflow at 88KB rather
* than at 80KB.
* In case of chunk encoding, we count the overflow for every
* chunk separately.
* We zero the nread counter (and reset data_or_header_data_start) when we
* start parsing a new message or a new chunk.
*/
if (PARSING_HEADER(state)) {
parser->nread += p - data_or_header_data_start;
if (parser->nread > HTTP_MAX_HEADER_SIZE) {
SET_ERRNO(HPE_HEADER_OVERFLOW);
goto error;
}
}
/* Run callbacks for any marks that we have leftover after we ran out of
* bytes. There should be at most one of these set, so it's OK to invoke
* them in series (unset marks will not result in callbacks).
*
* We use the NOADVANCE() variety of callbacks here because 'p' has already
* overflowed 'data' and this allows us to correct for the off-by-one that
* we'd otherwise have (since CALLBACK_DATA() is meant to be run with a 'p'
* value that's in-bounds).
*/
assert(((header_field_mark ? 1 : 0) +
(header_value_mark ? 1 : 0) +
(url_mark ? 1 : 0) +
(reason_mark ? 1 : 0) +
(body_mark ? 1 : 0)) <= 1);
CALLBACK_DATA_NOADVANCE(header_field);
CALLBACK_DATA_NOADVANCE(header_value);
CALLBACK_DATA_NOADVANCE(url);
CALLBACK_DATA_NOADVANCE(reason);
CALLBACK_DATA_NOADVANCE(body);
RETURN(len);
error:
if (HTTP_PARSER_ERRNO(parser) == HPE_OK) {
SET_ERRNO(HPE_UNKNOWN);
}
RETURN(p - data);
}
const char * http_method_str (enum http_method m)
{
return method_strings[m];
}
void
http_parser_init (http_parser *parser, enum http_parser_type t)
{
parser->type = t;
parser->state = (t == HTTP_REQUEST ? s_pre_start_req : (t == HTTP_RESPONSE ? s_pre_start_res : s_pre_start_req_or_res));
parser->nread = 0;
parser->upgrade = 0;
parser->flags = 0;
parser->method = 0;
parser->http_major = 0;
parser->http_minor = 0;
parser->http_errno = HPE_OK;
}
const char *
http_errno_name(enum http_errno err) {
assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
return http_strerror_tab[err].name;
}
const char *
http_errno_description(enum http_errno err) {
assert(err < (sizeof(http_strerror_tab)/sizeof(http_strerror_tab[0])));
return http_strerror_tab[err].description;
}
static enum http_host_state
http_parse_host_char(enum http_host_state s, const char ch) {
switch(s) {
case s_http_userinfo:
case s_http_userinfo_start:
if (ch == '@') {
return s_http_host_start;
}
if (IS_USERINFO_CHAR(ch)) {
return s_http_userinfo;
}
break;
case s_http_host_start:
if (ch == '[') {
return s_http_host_v6_start;
}
if (IS_HOST_CHAR(ch)) {
return s_http_host;
}
break;
case s_http_host:
if (IS_HOST_CHAR(ch)) {
return s_http_host;
}
#if __cplusplus
[[fallthrough]];
#else /* __cplusplus */
__attribute__((fallthrough));
#endif /* __cplusplus */
case s_http_host_v6_end:
if (ch == ':') {
return s_http_host_port_start;
}
break;
case s_http_host_v6:
if (ch == ']') {
return s_http_host_v6_end;
}
#if __cplusplus
[[fallthrough]];
#else /* __cplusplus */
__attribute__((fallthrough));
#endif /* __cplusplus */
case s_http_host_v6_start:
if (IS_HEX(ch) || ch == ':' || ch == '.') {
return s_http_host_v6;
}
break;
case s_http_host_port:
case s_http_host_port_start:
if (IS_NUM(ch)) {
return s_http_host_port;
}
break;
default:
break;
}
return s_http_host_dead;
}
static int
http_parse_host(const char * buf, struct http_parser_url *u, int found_at) {
enum http_host_state s;
const char *p;
size_t buflen = u->field_data[UF_HOST].off + u->field_data[UF_HOST].len;
u->field_data[UF_HOST].len = 0;
s = found_at ? s_http_userinfo_start : s_http_host_start;
for (p = buf + u->field_data[UF_HOST].off; p < buf + buflen; p++) {
enum http_host_state new_s = http_parse_host_char(s, *p);
if (new_s == s_http_host_dead) {
return 1;
}
switch(new_s) {
case s_http_host:
if (s != s_http_host) {
u->field_data[UF_HOST].off = p - buf;
}
u->field_data[UF_HOST].len++;
break;
case s_http_host_v6:
if (s != s_http_host_v6) {
u->field_data[UF_HOST].off = p - buf;
}
u->field_data[UF_HOST].len++;
break;
case s_http_host_port:
if (s != s_http_host_port) {
u->field_data[UF_PORT].off = p - buf;
u->field_data[UF_PORT].len = 0;
u->field_set |= (1 << UF_PORT);
}
u->field_data[UF_PORT].len++;
break;
case s_http_userinfo:
if (s != s_http_userinfo) {
u->field_data[UF_USERINFO].off = p - buf ;
u->field_data[UF_USERINFO].len = 0;
u->field_set |= (1 << UF_USERINFO);
}
u->field_data[UF_USERINFO].len++;
break;
default:
break;
}
s = new_s;
}
/* Make sure we don't end somewhere unexpected */
switch (s) {
case s_http_host_start:
case s_http_host_v6_start:
case s_http_host_v6:
case s_http_host_port_start:
case s_http_userinfo:
case s_http_userinfo_start:
return 1;
default:
break;
}
return 0;
}
int
http_parser_parse_url(const char *buf, size_t buflen, int is_connect,
struct http_parser_url *u)
{
return http_parser_parse_url_options(buf, buflen, is_connect, u, 0);
}
int
http_parser_parse_url_options(const char *buf, size_t buflen, int is_connect,
struct http_parser_url *u, uint8_t options)
{
enum state s;
const char *p;
enum http_parser_url_fields uf, old_uf;
int found_at = 0;
u->port = u->field_set = 0;
s = is_connect ? s_req_server_start : s_req_spaces_before_url;
uf = old_uf = UF_MAX;
for (p = buf; p < buf + buflen; p++) {
s = parse_url_char(s, *p, options & F_PARSE_URL_OPTIONS_URL_STRICT);
/* Figure out the next field that we're operating on */
switch (s) {
case s_dead:
return 1;
/* Skip delimeters */
case s_req_schema_slash:
case s_req_schema_slash_slash:
case s_req_server_start:
case s_req_query_string_start:
case s_req_fragment_start:
continue;
case s_req_schema:
uf = UF_SCHEMA;
break;
case s_req_server_with_at:
found_at = 1;
#if __cplusplus
[[fallthrough]];
#else /* __cplusplus */
__attribute__((fallthrough));
#endif /* __cplusplus */
case s_req_server:
uf = UF_HOST;
break;
case s_req_path:
uf = UF_PATH;
break;
case s_req_query_string:
uf = UF_QUERY;
break;
case s_req_fragment:
uf = UF_FRAGMENT;
break;
default:
assert(false && "Unexpected state");
return 1;
}
/* Nothing's changed; soldier on */
if (uf == old_uf) {
u->field_data[uf].len++;
continue;
}
u->field_data[uf].off = p - buf;
u->field_data[uf].len = 1;
u->field_set |= (1 << uf);
old_uf = uf;
}
/* host must be present if there is a schema */
/* parsing http:///toto will fail */
if ((u->field_set & ((1 << UF_SCHEMA) | (1 << UF_HOST))) != 0) {
if (http_parse_host(buf, u, found_at) != 0) {
return 1;
}
}
/* CONNECT requests can only contain "hostname:port" */
if (is_connect && u->field_set != ((1 << UF_HOST)|(1 << UF_PORT))) {
return 1;
}
if (u->field_set & (1 << UF_PORT)) {
uint16_t off;
uint16_t len;
const char* portp;
const char* end;
unsigned long v;
off = u->field_data[UF_PORT].off;
len = u->field_data[UF_PORT].len;
end = buf + off + len;
/* NOTE: The characters are already validated and are in the [0-9] range */
assert(off + len <= buflen && "Port number overflow");
v = 0;
for (portp = buf + off; portp < end; portp++) {
v *= 10;
v += *portp - '0';
/* Ports have a max value of 2^16 */
if (v > 0xffff) {
return 1;
}
}
u->port = (uint16_t) v;
}
return 0;
}
void
http_parser_pause(http_parser *parser, int paused) {
/* Users should only be pausing/unpausing a parser that is not in an error
* state. In non-debug builds, there's not much that we can do about this
* other than ignore it.
*/
if (HTTP_PARSER_ERRNO(parser) == HPE_OK ||
HTTP_PARSER_ERRNO(parser) == HPE_PAUSED) {
SET_ERRNO((paused) ? HPE_PAUSED : HPE_OK);
} else {
assert(0 && "Attempting to pause parser in error state");
}
}
#if __cplusplus
}
#endif /* __cplusplus */
================================================
FILE: proxygen/external/http_parser/test.c
================================================
/* Copyright Joyent, Inc. and other Node contributors. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
* sell copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
* IN THE SOFTWARE.
*/
#include "http_parser.h"
#include
#include
#include
#include /* rand */
#include
#include
#undef TRUE
#define TRUE 1
#undef FALSE
#define FALSE 0
#define MAX_HEADERS 13
#define MAX_ELEMENT_SIZE 500
#define MAX_CHUNKS 16
#define MIN(a,b) ((a) < (b) ? (a) : (b))
static http_parser *parser;
static uint8_t parser_options = 0;
struct message {
const char *name; // for debugging purposes
const char *raw;
enum http_parser_type type;
enum http_method method;
int status_code;
char request_url[MAX_ELEMENT_SIZE];
char response_reason[MAX_ELEMENT_SIZE];
size_t response_reason_size;
char body[MAX_ELEMENT_SIZE];
size_t body_size;
int num_headers;
enum { NONE=0, FIELD, VALUE } last_header_element;
char headers [MAX_HEADERS][2][MAX_ELEMENT_SIZE];
int should_keep_alive;
int num_chunks;
int num_chunks_complete;
int chunk_lengths[MAX_CHUNKS];
const char *upgrade; // upgraded body
unsigned short http_major;
unsigned short http_minor;
int message_begin_cb_called;
int headers_complete_cb_called;
int message_complete_cb_called;
int message_complete_on_eof;
};
static int currently_parsing_eof;
static struct message messages[5];
static int num_messages;
static http_parser_settings *current_pause_parser;
/* * R E Q U E S T S * */
const struct message requests[] =
#define CURL_GET 0
{ {.name= "curl get"
,.type= HTTP_REQUEST
,.raw= "GET /test HTTP/1.1\r\n"
"User-Agent: curl/7.18.0 (i486-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1\r\n"
"Host: 0.0.0.0=5000\r\n"
"Accept: */*\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "/test"
,.num_headers= 3
,.headers=
{ { "User-Agent", "curl/7.18.0 (i486-pc-linux-gnu) libcurl/7.18.0 OpenSSL/0.9.8g zlib/1.2.3.3 libidn/1.1" }
, { "Host", "0.0.0.0=5000" }
, { "Accept", "*/*" }
}
,.body= ""
}
#define FIREFOX_GET 1
, {.name= "firefox get"
,.type= HTTP_REQUEST
,.raw= "GET /favicon.ico HTTP/1.1\r\n"
"Host: 0.0.0.0=5000\r\n"
"User-Agent: Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008061015 Firefox/3.0\r\n"
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n"
"Accept-Language: en-us,en;q=0.5\r\n"
"Accept-Encoding: gzip,deflate\r\n"
"Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n"
"Keep-Alive: 300\r\n"
"Connection: keep-alive\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "/favicon.ico"
,.num_headers= 8
,.headers=
{ { "Host", "0.0.0.0=5000" }
, { "User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9) Gecko/2008061015 Firefox/3.0" }
, { "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" }
, { "Accept-Language", "en-us,en;q=0.5" }
, { "Accept-Encoding", "gzip,deflate" }
, { "Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7" }
, { "Keep-Alive", "300" }
, { "Connection", "keep-alive" }
}
,.body= ""
}
#define DUMBFUCK 2
, {.name= "dumbfuck"
,.type= HTTP_REQUEST
,.raw= "GET /dumbfuck HTTP/1.1\r\n"
"aaaaaaaaaaaaa:++++++++++\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "/dumbfuck"
,.num_headers= 1
,.headers=
{ { "aaaaaaaaaaaaa", "++++++++++" }
}
,.body= ""
}
#define FRAGMENT_IN_URI 3
, {.name= "fragment in url"
,.type= HTTP_REQUEST
,.raw= "GET /forums/1/topics/2375?page=1#posts-17408 HTTP/1.1\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
/* XXX request url does include fragment? */
,.request_url= "/forums/1/topics/2375?page=1#posts-17408"
,.num_headers= 0
,.body= ""
}
#define GET_NO_HEADERS_NO_BODY 4
, {.name= "get no headers no body"
,.type= HTTP_REQUEST
,.raw= "GET /get_no_headers_no_body/world HTTP/1.1\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE /* would need Connection: close */
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "/get_no_headers_no_body/world"
,.num_headers= 0
,.body= ""
}
#define GET_ONE_HEADER_NO_BODY 5
, {.name= "get one header no body"
,.type= HTTP_REQUEST
,.raw= "GET /get_one_header_no_body HTTP/1.1\r\n"
"Accept: */*\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE /* would need Connection: close */
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "/get_one_header_no_body"
,.num_headers= 1
,.headers=
{ { "Accept" , "*/*" }
}
,.body= ""
}
#define GET_FUNKY_CONTENT_LENGTH 6
, {.name= "get funky content length body hello"
,.type= HTTP_REQUEST
,.raw= "GET /get_funky_content_length_body_hello HTTP/1.0\r\n"
"conTENT-Length: 5\r\n"
"\r\n"
"HELLO"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 0
,.method= HTTP_GET
,.request_url= "/get_funky_content_length_body_hello"
,.num_headers= 1
,.headers=
{ { "conTENT-Length" , "5" }
}
,.body= "HELLO"
}
#define POST_IDENTITY_BODY_WORLD 7
, {.name= "post identity body world"
,.type= HTTP_REQUEST
,.raw= "POST /post_identity_body_world?q=search#hey HTTP/1.1\r\n"
"Accept: */*\r\n"
"Transfer-Encoding: identity\r\n"
"Content-Length: 5\r\n"
"\r\n"
"World"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_POST
,.request_url= "/post_identity_body_world?q=search#hey"
,.num_headers= 3
,.headers=
{ { "Accept", "*/*" }
, { "Transfer-Encoding", "identity" }
, { "Content-Length", "5" }
}
,.body= "World"
}
#define POST_CHUNKED_ALL_YOUR_BASE 8
, {.name= "post - chunked body: all your base are belong to us"
,.type= HTTP_REQUEST
,.raw= "POST /post_chunked_all_your_base HTTP/1.1\r\n"
"Transfer-Encoding: chunked\r\n"
"\r\n"
"1e\r\nall your base are belong to us\r\n"
"0\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_POST
,.request_url= "/post_chunked_all_your_base"
,.num_headers= 1
,.headers=
{ { "Transfer-Encoding" , "chunked" }
}
,.body= "all your base are belong to us"
,.num_chunks= 1
,.num_chunks_complete= 2
,.chunk_lengths= { 0x1e }
}
#define TWO_CHUNKS_MULT_ZERO_END 9
, {.name= "two chunks ; triple zero ending"
,.type= HTTP_REQUEST
,.raw= "POST /two_chunks_mult_zero_end HTTP/1.1\r\n"
"Transfer-Encoding: chunked\r\n"
"\r\n"
"5\r\nhello\r\n"
"6\r\n world\r\n"
"000\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_POST
,.request_url= "/two_chunks_mult_zero_end"
,.num_headers= 1
,.headers=
{ { "Transfer-Encoding", "chunked" }
}
,.body= "hello world"
,.num_chunks= 2
,.num_chunks_complete= 3
,.chunk_lengths= { 5, 6 }
}
#define CHUNKED_W_TRAILING_HEADERS 10
, {.name= "chunked with trailing headers. blech."
,.type= HTTP_REQUEST
,.raw= "POST /chunked_w_trailing_headers HTTP/1.1\r\n"
"Transfer-Encoding: chunked\r\n"
"\r\n"
"5\r\nhello\r\n"
"6\r\n world\r\n"
"0\r\n"
"Vary: *\r\n"
"Content-Type: text/plain\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_POST
,.request_url= "/chunked_w_trailing_headers"
,.num_headers= 3
,.headers=
{ { "Transfer-Encoding", "chunked" }
, { "Vary", "*" }
, { "Content-Type", "text/plain" }
}
,.body= "hello world"
,.num_chunks= 2
,.num_chunks_complete= 3
,.chunk_lengths= { 5, 6 }
}
#define CHUNKED_W_BULLSHIT_AFTER_LENGTH 11
, {.name= "with bullshit after the length"
,.type= HTTP_REQUEST
,.raw= "POST /chunked_w_bullshit_after_length HTTP/1.1\r\n"
"Transfer-Encoding: chunked\r\n"
"\r\n"
"5; ihatew3;whatthefuck=aretheseparametersfor\r\nhello\r\n"
"6; blahblah; blah\r\n world\r\n"
"0\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_POST
,.request_url= "/chunked_w_bullshit_after_length"
,.num_headers= 1
,.headers=
{ { "Transfer-Encoding", "chunked" }
}
,.body= "hello world"
,.num_chunks= 2
,.num_chunks_complete= 3
,.chunk_lengths= { 5, 6 }
}
#define WITH_QUOTES 12
, {.name= "with quotes"
,.type= HTTP_REQUEST
,.raw= "GET /with_\"stupid\"_quotes?foo=\"bar\" HTTP/1.1\r\n\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "/with_\"stupid\"_quotes?foo=\"bar\""
,.num_headers= 0
,.headers= { }
,.body= ""
}
#define APACHEBENCH_GET 13
/* The server receiving this request SHOULD NOT wait for EOF
* to know that content-length == 0.
* How to represent this in a unit test? message_complete_on_eof
* Compare with NO_CONTENT_LENGTH_RESPONSE.
*/
, {.name = "apachebench get"
,.type= HTTP_REQUEST
,.raw= "GET /test HTTP/1.0\r\n"
"Host: 0.0.0.0:5000\r\n"
"User-Agent: ApacheBench/2.3\r\n"
"Accept: */*\r\n\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 0
,.method= HTTP_GET
,.request_url= "/test"
,.num_headers= 3
,.headers= { { "Host", "0.0.0.0:5000" }
, { "User-Agent", "ApacheBench/2.3" }
, { "Accept", "*/*" }
}
,.body= ""
}
#define QUERY_URL_WITH_QUESTION_MARK_GET 14
/* Some clients include '?' characters in query strings.
*/
, {.name = "query url with question mark"
,.type= HTTP_REQUEST
,.raw= "GET /test.cgi?foo=bar?baz HTTP/1.1\r\n\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "/test.cgi?foo=bar?baz"
,.num_headers= 0
,.headers= {}
,.body= ""
}
#define PREFIX_NEWLINE_GET 15
/* Some clients, especially after a POST in a keep-alive connection,
* will send an extra CRLF before the next request
*/
, {.name = "newline prefix get"
,.type= HTTP_REQUEST
,.raw= "\r\nGET /test HTTP/1.1\r\n\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "/test"
,.num_headers= 0
,.headers= { }
,.body= ""
}
#define UPGRADE_REQUEST 16
, {.name = "upgrade request"
,.type= HTTP_REQUEST
,.raw= "GET /demo HTTP/1.1\r\n"
"Host: example.com\r\n"
"Connection: Upgrade\r\n"
"Sec-WebSocket-Key2: 12998 5 Y3 1 .P00\r\n"
"Sec-WebSocket-Protocol: sample\r\n"
"Upgrade: WebSocket\r\n"
"Sec-WebSocket-Key1: 4 @1 46546xW%0l 1 5\r\n"
"Origin: http://example.com\r\n"
"\r\n"
"Hot diggity dogg"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "/demo"
,.num_headers= 7
,.upgrade="Hot diggity dogg"
,.headers= { { "Host", "example.com" }
, { "Connection", "Upgrade" }
, { "Sec-WebSocket-Key2", "12998 5 Y3 1 .P00" }
, { "Sec-WebSocket-Protocol", "sample" }
, { "Upgrade", "WebSocket" }
, { "Sec-WebSocket-Key1", "4 @1 46546xW%0l 1 5" }
, { "Origin", "http://example.com" }
}
,.body= ""
}
#define CONNECT_REQUEST 17
, {.name = "connect request"
,.type= HTTP_REQUEST
,.raw= "CONNECT 0-home0.netscape.com:443 HTTP/1.0\r\n"
"User-agent: Mozilla/1.1N\r\n"
"Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n"
"\r\n"
"some data\r\n"
"and yet even more data"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 0
,.method= HTTP_CONNECT
,.request_url= "0-home0.netscape.com:443"
,.num_headers= 2
,.upgrade="some data\r\nand yet even more data"
,.headers= { { "User-agent", "Mozilla/1.1N" }
, { "Proxy-authorization", "basic aGVsbG86d29ybGQ=" }
}
,.body= ""
}
#define REPORT_REQ 18
, {.name= "report request"
,.type= HTTP_REQUEST
,.raw= "REPORT /test HTTP/1.1\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_REPORT
,.request_url= "/test"
,.num_headers= 0
,.headers= {}
,.body= ""
}
#define NO_HTTP_VERSION 19
, {.name= "request with no http version"
,.type= HTTP_REQUEST
,.raw= "GET /\r\n"
"\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 0
,.http_minor= 9
,.method= HTTP_GET
,.request_url= "/"
,.num_headers= 0
,.headers= {}
,.body= ""
}
#define MSEARCH_REQ 20
, {.name= "m-search request"
,.type= HTTP_REQUEST
,.raw= "M-SEARCH * HTTP/1.1\r\n"
"HOST: 239.255.255.250:1900\r\n"
"MAN: \"ssdp:discover\"\r\n"
"ST: \"ssdp:all\"\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_MSEARCH
,.request_url= "*"
,.num_headers= 3
,.headers= { { "HOST", "239.255.255.250:1900" }
, { "MAN", "\"ssdp:discover\"" }
, { "ST", "\"ssdp:all\"" }
}
,.body= ""
}
#define LINE_FOLDING_IN_HEADER 20
, {.name= "line folding in header value"
,.type= HTTP_REQUEST
,.raw= "GET / HTTP/1.1\r\n"
"Line1: abc\r\n"
"\tdef\r\n"
" ghi\r\n"
"\t\tjkl\r\n"
" mno \r\n"
"\t \tqrs\r\n"
"Line2: \t line2\t\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "/"
,.num_headers= 2
,.headers= { { "Line1", "abc def ghi jkl mno qrs" }
, { "Line2", "line2\t" }
}
,.body= ""
}
#define QUERY_TERMINATED_HOST 21
, {.name= "host terminated by a query string"
,.type= HTTP_REQUEST
,.raw= "GET http://hypnotoad.org?hail=all HTTP/1.1\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "http://hypnotoad.org?hail=all"
,.num_headers= 0
,.headers= { }
,.body= ""
}
#define QUERY_TERMINATED_HOSTPORT 22
, {.name= "host:port terminated by a query string"
,.type= HTTP_REQUEST
,.raw= "GET http://hypnotoad.org:1234?hail=all HTTP/1.1\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "http://hypnotoad.org:1234?hail=all"
,.num_headers= 0
,.headers= { }
,.body= ""
}
#define SPACE_TERMINATED_HOSTPORT 23
, {.name= "host:port terminated by a space"
,.type= HTTP_REQUEST
,.raw= "GET http://hypnotoad.org:1234 HTTP/1.1\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "http://hypnotoad.org:1234"
,.num_headers= 0
,.headers= { }
,.body= ""
}
#if !HTTP_PARSER_STRICT_URL
#define UTF8_PATH_REQ 24
, {.name= "utf-8 path request"
,.type= HTTP_REQUEST
,.raw= "GET /δ¶/δt/pope?q=1#narf HTTP/1.1\r\n"
"Host: github.com\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_GET
,.request_url= "/δ¶/δt/pope?q=1#narf"
,.num_headers= 1
,.headers= { {"Host", "github.com" }
}
,.body= ""
}
#endif /* !HTTP_PARSER_STRICT_URL */
#if !HTTP_PARSER_STRICT_HOSTNAME
#define HOSTNAME_UNDERSCORE 25
, {.name = "hostname underscore"
,.type= HTTP_REQUEST
,.raw= "CONNECT home_0.netscape.com:443 HTTP/1.0\r\n"
"User-agent: Mozilla/1.1N\r\n"
"Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n"
"\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 0
,.method= HTTP_CONNECT
,.request_url= "home_0.netscape.com:443"
,.num_headers= 2
,.upgrade=""
,.headers= { { "User-agent", "Mozilla/1.1N" }
, { "Proxy-authorization", "basic aGVsbG86d29ybGQ=" }
}
,.body= ""
}
#endif /* !HTTP_PARSER_STRICT_HOSTNAME */
#define PATCH_REQ 26
, {.name = "PATCH request"
,.type= HTTP_REQUEST
,.raw= "PATCH /file.txt HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"Content-Type: application/example\r\n"
"If-Match: \"e0023aa4e\"\r\n"
"Content-Length: 10\r\n"
"\r\n"
"cccccccccc"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_PATCH
,.request_url= "/file.txt"
,.num_headers= 4
,.headers= { { "Host", "www.example.com" }
, { "Content-Type", "application/example" }
, { "If-Match", "\"e0023aa4e\"" }
, { "Content-Length", "10" }
}
,.body= "cccccccccc"
}
#define CONNECT_CAPS_REQUEST 27
, {.name = "connect caps request"
,.type= HTTP_REQUEST
,.raw= "CONNECT HOME0.NETSCAPE.COM:443 HTTP/1.0\r\n"
"User-agent: Mozilla/1.1N\r\n"
"Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n"
"\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 0
,.method= HTTP_CONNECT
,.request_url= "HOME0.NETSCAPE.COM:443"
,.num_headers= 2
,.upgrade=""
,.headers= { { "User-agent", "Mozilla/1.1N" }
, { "Proxy-authorization", "basic aGVsbG86d29ybGQ=" }
}
,.body= ""
}
#define IPV6_LITERAL_URI_GET 28
, {.name = "IPv6 literal URI GET"
,.type= HTTP_REQUEST
,.raw= "GET http://[2a03:2880:2050:1f01:face:b00c:0:9]/ HTTP/1.0\r\n"
"\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 0
,.method= HTTP_GET
,.request_url= "http://[2a03:2880:2050:1f01:face:b00c:0:9]/"
,.num_headers= 0
,.headers= {}
,.body= ""
}
#define IPV6_LITERAL_URI_CONNECT 29
, {.name = "IPv6 literal URI CONNECT"
,.type= HTTP_REQUEST
,.raw= "CONNECT [2a03:2880:2050:1f01:face:b00c:0:9]:443 HTTP/1.0\r\n"
"\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 0
,.method= HTTP_CONNECT
,.request_url= "[2a03:2880:2050:1f01:face:b00c:0:9]:443"
,.num_headers= 0
,.upgrade=""
,.headers= {}
,.body= ""
}
#define UPGRADE_POST_REQUEST 30
, {.name = "upgrade post request"
,.type= HTTP_REQUEST
,.raw= "POST /demo HTTP/1.1\r\n"
"Host: example.com\r\n"
"Connection: Upgrade\r\n"
"Upgrade: HTTP/2.0\r\n"
"Content-Length: 15\r\n"
"\r\n"
"sweet post body"
"Hot diggity dogg"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.method= HTTP_POST
,.request_url= "/demo"
,.num_headers= 4
,.upgrade="Hot diggity dogg"
,.headers= { { "Host", "example.com" }
, { "Connection", "Upgrade" }
, { "Upgrade", "HTTP/2.0" }
, { "Content-Length", "15" }
}
,.body= "sweet post body"
}
#define CONNECT_WITH_BODY_REQUEST 31
, {.name = "connect with body request"
,.type= HTTP_REQUEST
,.raw= "CONNECT foo.bar.com:443 HTTP/1.0\r\n"
"User-agent: Mozilla/1.1N\r\n"
"Proxy-authorization: basic aGVsbG86d29ybGQ=\r\n"
"Content-Length: 10\r\n"
"\r\n"
"blarfcicle"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 0
,.method= HTTP_CONNECT
,.request_url= "foo.bar.com:443"
,.num_headers= 3
,.upgrade="blarfcicle"
,.headers= { { "User-agent", "Mozilla/1.1N" }
, { "Proxy-authorization", "basic aGVsbG86d29ybGQ=" }
, { "Content-Length", "10" }
}
,.body= ""
}
, {.name= NULL } /* sentinel */
};
/* * R E S P O N S E S * */
const struct message responses[] =
#define GOOGLE_301 0
{ {.name= "google 301"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.1 301 Moved Permanently\r\n"
"Location: http://www.google.com/\r\n"
"Content-Type: text/html; charset=UTF-8\r\n"
"Date: Sun, 26 Apr 2009 11:11:49 GMT\r\n"
"Expires: Tue, 26 May 2009 11:11:49 GMT\r\n"
"X-$PrototypeBI-Version: 1.6.0.3\r\n" /* $ char in header field */
"Cache-Control: public, max-age=2592000\r\n"
"Server: gws\r\n"
"Content-Length: 219 \r\n"
"\r\n"
" \n"
"301 Moved \n"
"301 Moved \n"
"The document has moved\n"
"here .\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.status_code= 301
,.response_reason= "Moved Permanently"
,.num_headers= 8
,.headers=
{ { "Location", "http://www.google.com/" }
, { "Content-Type", "text/html; charset=UTF-8" }
, { "Date", "Sun, 26 Apr 2009 11:11:49 GMT" }
, { "Expires", "Tue, 26 May 2009 11:11:49 GMT" }
, { "X-$PrototypeBI-Version", "1.6.0.3" }
, { "Cache-Control", "public, max-age=2592000" }
, { "Server", "gws" }
, { "Content-Length", "219 " }
}
,.body= " \n"
"301 Moved \n"
"301 Moved \n"
"The document has moved\n"
"here .\r\n"
"\r\n"
}
#define NO_CONTENT_LENGTH_RESPONSE 1
/* The client should wait for the server's EOF. That is, when content-length
* is not specified, and "Connection: close", the end of body is specified
* by the EOF.
* Compare with APACHEBENCH_GET
*/
, {.name= "no content-length response"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.1 200 OK\r\n"
"Date: Tue, 04 Aug 2009 07:59:32 GMT\r\n"
"Server: Apache\r\n"
"X-Powered-By: Servlet/2.5 JSP/2.1\r\n"
"Content-Type: text/xml; charset=utf-8\r\n"
"Connection: close\r\n"
"\r\n"
"\n"
"\n"
" \n"
" \n"
" SOAP-ENV:Client \n"
" Client Error \n"
" \n"
" \n"
" "
,.should_keep_alive= FALSE
,.message_complete_on_eof= TRUE
,.http_major= 1
,.http_minor= 1
,.status_code= 200
,.response_reason= "OK"
,.num_headers= 5
,.headers=
{ { "Date", "Tue, 04 Aug 2009 07:59:32 GMT" }
, { "Server", "Apache" }
, { "X-Powered-By", "Servlet/2.5 JSP/2.1" }
, { "Content-Type", "text/xml; charset=utf-8" }
, { "Connection", "close" }
}
,.body= "\n"
"\n"
" \n"
" \n"
" SOAP-ENV:Client \n"
" Client Error \n"
" \n"
" \n"
" "
}
#define NO_HEADERS_NO_BODY_404 2
, {.name= "404 no headers no body"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.1 404 Not Found\r\n\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= TRUE
,.http_major= 1
,.http_minor= 1
,.status_code= 404
,.response_reason= "Not Found"
,.num_headers= 0 ,.headers= {}
,.body_size= 0
,.body= ""
}
#define NO_REASON_PHRASE 3
, {.name= "304 no response phrase"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.1 304\r\n\r\n"
,.should_keep_alive = TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.status_code= 304
,.response_reason= ""
,.num_headers= 0
,.headers= {}
,.body= ""
}
#define TRAILING_SPACE_ON_CHUNKED_BODY 4
, {.name="200 trailing space on chunked body"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.1 200 OK\r\n"
"Content-Type: text/plain\r\n"
"Transfer-Encoding: chunked\r\n"
"\r\n"
"25 \r\n"
"This is the data in the first chunk\r\n"
"\r\n"
"1C\r\n"
"and this is the second one\r\n"
"\r\n"
"0 \r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.status_code= 200
,.response_reason= "OK"
,.num_headers= 2
,.headers=
{ {"Content-Type", "text/plain" }
, {"Transfer-Encoding", "chunked" }
}
,.body_size = 37+28
,.body =
"This is the data in the first chunk\r\n"
"and this is the second one\r\n"
,.num_chunks= 2
,.num_chunks_complete= 3
,.chunk_lengths= { 0x25, 0x1c }
}
#define NO_CARRIAGE_RET 5
, {.name="no carriage ret"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.1 200 OK\n"
"Content-Type: text/html; charset=utf-8\n"
"Connection: close\n"
"\n"
"these headers are from http://news.ycombinator.com/"
,.should_keep_alive= FALSE
,.message_complete_on_eof= TRUE
,.http_major= 1
,.http_minor= 1
,.status_code= 200
,.response_reason= "OK"
,.num_headers= 2
,.headers=
{ {"Content-Type", "text/html; charset=utf-8" }
, {"Connection", "close" }
}
,.body= "these headers are from http://news.ycombinator.com/"
}
#define PROXY_CONNECTION 6
, {.name="proxy connection"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.1 200 OK\r\n"
"Content-Type: text/html; charset=UTF-8\r\n"
"Content-Length: 11\r\n"
"Proxy-Connection: close\r\n"
"Date: Thu, 31 Dec 2009 20:55:48 +0000\r\n"
"\r\n"
"hello world"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.status_code= 200
,.response_reason= "OK"
,.num_headers= 4
,.headers=
{ {"Content-Type", "text/html; charset=UTF-8" }
, {"Content-Length", "11" }
, {"Proxy-Connection", "close" }
, {"Date", "Thu, 31 Dec 2009 20:55:48 +0000"}
}
,.body= "hello world"
}
#define UNDERSTORE_HEADER_KEY 7
// shown by
// curl -o /dev/null -v "http://ad.doubleclick.net/pfadx/DARTSHELLCONFIGXML;dcmt=text/xml;"
, {.name="underscore header key"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.1 200 OK\r\n"
"Server: DCLK-AdSvr\r\n"
"Content-Type: text/xml\r\n"
"Content-Length: 0\r\n"
"DCLK_imp: v7;x;114750856;0-0;0;17820020;0/0;21603567/21621457/1;;~okv=;dcmt=text/xml;;~cs=o\r\n\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.status_code= 200
,.response_reason= "OK"
,.num_headers= 4
,.headers=
{ {"Server", "DCLK-AdSvr" }
, {"Content-Type", "text/xml" }
, {"Content-Length", "0" }
, {"DCLK_imp", "v7;x;114750856;0-0;0;17820020;0/0;21603567/21621457/1;;~okv=;dcmt=text/xml;;~cs=o" }
}
,.body= ""
}
#define BONJOUR_MADAME_FR 8
/* The client should not merge two headers fields when the first one doesn't
* have a value.
*/
, {.name= "bonjourmadame.fr"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.0 301 Moved Permanently\r\n"
"Date: Thu, 03 Jun 2010 09:56:32 GMT\r\n"
"Server: Apache/2.2.3 (Red Hat)\r\n"
"Cache-Control: public\r\n"
"Pragma: \r\n"
"Location: http://www.bonjourmadame.fr/\r\n"
"Vary: Accept-Encoding\r\n"
"Content-Length: 0\r\n"
"Content-Type: text/html; charset=UTF-8\r\n"
"Connection: keep-alive\r\n"
"\r\n"
,.should_keep_alive= TRUE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 0
,.status_code= 301
,.response_reason= "Moved Permanently"
,.num_headers= 9
,.headers=
{ { "Date", "Thu, 03 Jun 2010 09:56:32 GMT" }
, { "Server", "Apache/2.2.3 (Red Hat)" }
, { "Cache-Control", "public" }
, { "Pragma", "" }
, { "Location", "http://www.bonjourmadame.fr/" }
, { "Vary", "Accept-Encoding" }
, { "Content-Length", "0" }
, { "Content-Type", "text/html; charset=UTF-8" }
, { "Connection", "keep-alive" }
}
,.body= ""
}
#define RES_FIELD_UNDERSCORE 10
/* Should handle spaces in header fields */
, {.name= "field underscore"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.1 200 OK\r\n"
"Date: Tue, 28 Sep 2010 01:14:13 GMT\r\n"
"Server: Apache\r\n"
"Cache-Control: no-cache, must-revalidate\r\n"
"Expires: Mon, 26 Jul 1997 05:00:00 GMT\r\n"
".et-Cookie: PlaxoCS=1274804622353690521; path=/; domain=.plaxo.com\r\n"
"Vary: Accept-Encoding\r\n"
"_eep-Alive: timeout=45\r\n" /* semantic value ignored */
"_onnection: Keep-Alive\r\n" /* semantic value ignored */
"Transfer-Encoding: chunked\r\n"
"Content-Type: text/html\r\n"
"Connection: close\r\n"
"\r\n"
"0\r\n\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.status_code= 200
,.response_reason= "OK"
,.num_headers= 11
,.headers=
{ { "Date", "Tue, 28 Sep 2010 01:14:13 GMT" }
, { "Server", "Apache" }
, { "Cache-Control", "no-cache, must-revalidate" }
, { "Expires", "Mon, 26 Jul 1997 05:00:00 GMT" }
, { ".et-Cookie", "PlaxoCS=1274804622353690521; path=/; domain=.plaxo.com" }
, { "Vary", "Accept-Encoding" }
, { "_eep-Alive", "timeout=45" }
, { "_onnection", "Keep-Alive" }
, { "Transfer-Encoding", "chunked" }
, { "Content-Type", "text/html" }
, { "Connection", "close" }
}
,.body= ""
,.num_chunks= 0
,.num_chunks_complete= 1
,.chunk_lengths= {}
}
#define NON_ASCII_IN_STATUS_LINE 11
/* Should handle non-ASCII in status line */
, {.name= "non-ASCII in status line"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.1 500 Oriëntatieprobleem\r\n"
"Date: Fri, 5 Nov 2010 23:07:12 GMT+2\r\n"
"Content-Length: 0\r\n"
"Connection: close\r\n"
"\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 1
,.status_code= 500
,.response_reason= "Oriëntatieprobleem"
,.num_headers= 3
,.headers=
{ { "Date", "Fri, 5 Nov 2010 23:07:12 GMT+2" }
, { "Content-Length", "0" }
, { "Connection", "close" }
}
,.body= ""
}
, {.name= NULL } /* sentinel */
};
/* Test for execution of on_message_callback regardless of the request/response
* correctness */
/* Invalid response */
const struct message on_message_begin_cb_test[] =
{ {.name= "Invalid response"
,.type= HTTP_RESPONSE
,.raw= "ZZZZZ\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.body= ""
},
/* Invalid request */
{.name= "Invalid request"
,.type= HTTP_REQUEST
,.raw= "ZZZZZ\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.body= ""
},
/* Invalid unspecified request/response */
{.name= "Invalid 'both' 'request/response'"
,.type= HTTP_BOTH
,.raw= "ZZZZZ\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.body= ""
},
/* Valid 200 response */
{.name= "Simple valid 200 reponse"
,.type= HTTP_RESPONSE
,.raw= "HTTP/1.1 200 OK\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.body= ""
},
/* Valid request */
{.name= "Simple valid request"
,.type= HTTP_REQUEST
,.raw= "GET / HTTP/1.1\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.body= ""
},
/* Valid unspecified request */
{.name= "Simple valid 'both' request"
,.type= HTTP_BOTH
,.raw= "GET / HTTP/1.1\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.body= ""
},
/* Valid unspecified response */
{.name= "Simple valid 'both' response"
,.type= HTTP_BOTH
,.raw= "GET / HTTP/1.1\r\n"
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.body= ""
}
, {.name= NULL } /* sentinel */
};
int
request_url_cb (http_parser *p, const char *buf, size_t len)
{
assert(p == parser);
strncat(messages[num_messages].request_url, buf, len);
return 0;
}
int
header_field_cb (http_parser *p, const char *buf, size_t len)
{
assert(p == parser);
struct message *m = &messages[num_messages];
if (m->last_header_element != FIELD)
m->num_headers++;
strncat(m->headers[m->num_headers-1][0], buf, len);
m->last_header_element = FIELD;
return 0;
}
int
header_value_cb (http_parser *p, const char *buf, size_t len)
{
assert(p == parser);
struct message *m = &messages[num_messages];
const size_t cursize = strlen(m->headers[m->num_headers-1][1]);
if (cursize + len + 1 > MAX_ELEMENT_SIZE) {
len = MAX_ELEMENT_SIZE - (cursize + 1);
}
strncat(m->headers[m->num_headers-1][1], buf, len);
m->last_header_element = VALUE;
return 0;
}
int
body_cb (http_parser *p, const char *buf, size_t len)
{
assert(p == parser);
strncat(messages[num_messages].body, buf, len);
messages[num_messages].body_size += len;
// printf("body_cb: '%s'\n", requests[num_messages].body);
return 0;
}
int
count_body_cb (http_parser *p, const char *buf, size_t len)
{
assert(p == parser);
assert(buf);
messages[num_messages].body_size += len;
return 0;
}
int
message_begin_cb (http_parser *p)
{
assert(p == parser);
messages[num_messages].message_begin_cb_called++;
return 0;
}
int
headers_complete_cb (http_parser *p, const char *buf, size_t len)
{
assert(p == parser);
assert(!buf);
messages[num_messages].method = parser->method;
messages[num_messages].status_code = parser->status_code;
messages[num_messages].http_major = parser->http_major;
messages[num_messages].http_minor = parser->http_minor;
messages[num_messages].headers_complete_cb_called = TRUE;
return 0;
}
int
message_complete_cb (http_parser *p)
{
assert(p == parser);
messages[num_messages].message_complete_cb_called = TRUE;
messages[num_messages].message_complete_on_eof = currently_parsing_eof;
num_messages++;
return 0;
}
int
response_reason_cb (http_parser *p, const char *buf, size_t len)
{
assert(p == parser);
strncat(messages[num_messages].response_reason, buf, len);
return 0;
}
int
chunk_header_cb (http_parser *p)
{
assert(p == parser);
if (p->content_length == 0) {
// Terminating chunk. Don't increment num_chunks in this case.
return 0;
}
int chunk_idx = messages[num_messages].num_chunks;
++messages[num_messages].num_chunks;
if (chunk_idx < MAX_CHUNKS) {
messages[num_messages].chunk_lengths[chunk_idx] = p->content_length;
}
return 0;
}
int
chunk_complete_cb (http_parser *p)
{
assert(p == parser);
++messages[num_messages].num_chunks_complete;
return 0;
}
/* These dontcall_* callbacks exist so that we can verify that when we're
* paused, no additional callbacks are invoked */
int
dontcall_message_begin_cb (http_parser *p)
{
if (p) { } // gcc
fprintf(stderr, "\n\n*** on_message_begin() called on paused parser ***\n\n");
exit(1);
}
int
dontcall_header_field_cb (http_parser *p, const char *buf, size_t len)
{
if (p || buf || len) { } // gcc
fprintf(stderr, "\n\n*** on_header_field() called on paused parser ***\n\n");
exit(1);
}
int
dontcall_header_value_cb (http_parser *p, const char *buf, size_t len)
{
if (p || buf || len) { } // gcc
fprintf(stderr, "\n\n*** on_header_value() called on paused parser ***\n\n");
exit(1);
}
int
dontcall_request_url_cb (http_parser *p, const char *buf, size_t len)
{
if (p || buf || len) { } // gcc
fprintf(stderr, "\n\n*** on_request_url() called on paused parser ***\n\n");
exit(1);
}
int
dontcall_body_cb (http_parser *p, const char *buf, size_t len)
{
if (p || buf || len) { } // gcc
fprintf(stderr, "\n\n*** on_body_cb() called on paused parser ***\n\n");
exit(1);
}
int
dontcall_headers_complete_cb (http_parser *p, const char *buf, size_t len)
{
if (p || buf || len) { } // gcc
fprintf(stderr, "\n\n*** on_headers_complete() called on paused "
"parser ***\n\n");
exit(1);
}
int
dontcall_message_complete_cb (http_parser *p)
{
if (p) { } // gcc
fprintf(stderr, "\n\n*** on_message_complete() called on paused "
"parser ***\n\n");
exit(1);
}
int
dontcall_response_reason_cb (http_parser *p, const char *buf, size_t len)
{
if (p || buf || len) { } // gcc
fprintf(stderr, "\n\n*** on_reason() called on paused parser ***\n\n");
exit(1);
}
int
dontcall_chunk_header_cb (http_parser *p)
{
if (p) { } // gcc
fprintf(stderr, "\n\n*** on_chunk_header() called on paused parser ***\n\n");
exit(1);
}
int
dontcall_chunk_complete_cb (http_parser *p)
{
if (p) { } // gcc
fprintf(stderr, "\n\n*** on_chunk_complete() "
"called on paused parser ***\n\n");
exit(1);
}
static http_parser_settings settings_dontcall =
{.on_message_begin = dontcall_message_begin_cb
,.on_header_field = dontcall_header_field_cb
,.on_header_value = dontcall_header_value_cb
,.on_url = dontcall_request_url_cb
,.on_body = dontcall_body_cb
,.on_headers_complete = dontcall_headers_complete_cb
,.on_message_complete = dontcall_message_complete_cb
,.on_reason = dontcall_response_reason_cb
,.on_chunk_header = dontcall_chunk_header_cb
,.on_chunk_complete = dontcall_chunk_complete_cb
};
/* These pause_* callbacks always pause the parser and just invoke the regular
* callback that tracks content. Before returning, we overwrite the parser
* settings to point to the _dontcall variety so that we can verify that
* the pause actually did, you know, pause. */
int
pause_message_begin_cb (http_parser *p)
{
http_parser_pause(p, 1);
*current_pause_parser = settings_dontcall;
return message_begin_cb(p);
}
int
pause_header_field_cb (http_parser *p, const char *buf, size_t len)
{
http_parser_pause(p, 1);
*current_pause_parser = settings_dontcall;
return header_field_cb(p, buf, len);
}
int
pause_header_value_cb (http_parser *p, const char *buf, size_t len)
{
http_parser_pause(p, 1);
*current_pause_parser = settings_dontcall;
return header_value_cb(p, buf, len);
}
int
pause_request_url_cb (http_parser *p, const char *buf, size_t len)
{
http_parser_pause(p, 1);
*current_pause_parser = settings_dontcall;
return request_url_cb(p, buf, len);
}
int
pause_body_cb (http_parser *p, const char *buf, size_t len)
{
http_parser_pause(p, 1);
*current_pause_parser = settings_dontcall;
return body_cb(p, buf, len);
}
int
pause_headers_complete_cb (http_parser *p, const char *buf, size_t len)
{
http_parser_pause(p, 1);
*current_pause_parser = settings_dontcall;
return headers_complete_cb(p, buf, len);
}
int
pause_message_complete_cb (http_parser *p)
{
http_parser_pause(p, 1);
*current_pause_parser = settings_dontcall;
return message_complete_cb(p);
}
int
pause_response_reason_cb (http_parser *p, const char *buf, size_t len)
{
http_parser_pause(p, 1);
*current_pause_parser = settings_dontcall;
return response_reason_cb(p, buf, len);
}
int
pause_chunk_header_cb (http_parser *p)
{
http_parser_pause(p, 1);
*current_pause_parser = settings_dontcall;
return chunk_header_cb(p);
}
int
pause_chunk_complete_cb (http_parser *p)
{
http_parser_pause(p, 1);
*current_pause_parser = settings_dontcall;
return chunk_complete_cb(p);
}
int empty_cb (http_parser *p) { return 0; }
int empty_data_cb (http_parser *p, const char *buf, size_t len) { return 0; }
static http_parser_settings settings_pause =
{.on_message_begin = pause_message_begin_cb
,.on_header_field = pause_header_field_cb
,.on_header_value = pause_header_value_cb
,.on_url = pause_request_url_cb
,.on_body = pause_body_cb
,.on_headers_complete = pause_headers_complete_cb
,.on_message_complete = pause_message_complete_cb
,.on_reason = pause_response_reason_cb
,.on_chunk_header = pause_chunk_header_cb
,.on_chunk_complete = pause_chunk_complete_cb
};
static http_parser_settings settings =
{.on_message_begin = message_begin_cb
,.on_header_field = header_field_cb
,.on_header_value = header_value_cb
,.on_url = request_url_cb
,.on_body = body_cb
,.on_headers_complete = headers_complete_cb
,.on_message_complete = message_complete_cb
,.on_reason = response_reason_cb
,.on_chunk_header = chunk_header_cb
,.on_chunk_complete = chunk_complete_cb
};
static http_parser_settings settings_count_body =
{.on_message_begin = message_begin_cb
,.on_header_field = header_field_cb
,.on_header_value = header_value_cb
,.on_url = request_url_cb
,.on_body = count_body_cb
,.on_headers_complete = headers_complete_cb
,.on_message_complete = message_complete_cb
,.on_reason = response_reason_cb
,.on_chunk_header = chunk_header_cb
,.on_chunk_complete = chunk_complete_cb
};
static http_parser_settings settings_null =
{.on_message_begin = empty_cb
,.on_header_field = empty_data_cb
,.on_header_value = empty_data_cb
,.on_url = empty_data_cb
,.on_body = empty_data_cb
,.on_headers_complete = empty_data_cb
,.on_message_complete = empty_cb
,.on_reason = empty_data_cb
,.on_chunk_header = empty_cb
,.on_chunk_complete = empty_cb
};
void
parser_init (enum http_parser_type type)
{
num_messages = 0;
assert(parser == NULL);
parser = malloc(sizeof(http_parser));
http_parser_init(parser, type);
#if HTTP_PARSER_STRICT_URL
parser_options |= F_HTTP_PARSER_OPTIONS_URL_STRICT;
#endif
memset(&messages, 0, sizeof messages);
}
void
parser_free ()
{
assert(parser);
free(parser);
parser = NULL;
}
size_t parse (const char *buf, size_t len)
{
size_t nparsed;
currently_parsing_eof = (len == 0);
nparsed = http_parser_execute_options(
parser, &settings, parser_options, buf, len);
return nparsed;
}
size_t parse_count_body (const char *buf, size_t len)
{
size_t nparsed;
currently_parsing_eof = (len == 0);
nparsed = http_parser_execute_options(
parser, &settings_count_body, parser_options, buf, len);
return nparsed;
}
size_t parse_pause (const char *buf, size_t len)
{
size_t nparsed;
http_parser_settings s = settings_pause;
currently_parsing_eof = (len == 0);
current_pause_parser = &s;
nparsed = http_parser_execute_options(
parser, current_pause_parser, parser_options, buf, len);
return nparsed;
}
static inline int
check_str_eq (const struct message *m,
const char *prop,
const char *expected,
const char *found) {
if ((expected == NULL) != (found == NULL)) {
printf("\n*** Error: %s in '%s' ***\n\n", prop, m->name);
printf("expected %s\n", (expected == NULL) ? "NULL" : expected);
printf(" found %s\n", (found == NULL) ? "NULL" : found);
return 0;
}
if (expected != NULL && 0 != strcmp(expected, found)) {
printf("\n*** Error: %s in '%s' ***\n\n", prop, m->name);
printf("expected '%s'\n", expected);
printf(" found '%s'\n", found);
return 0;
}
return 1;
}
static inline int
check_num_eq (const struct message *m,
const char *prop,
int expected,
int found) {
if (expected != found) {
printf("\n*** Error: %s in '%s' ***\n\n", prop, m->name);
printf("expected %d\n", expected);
printf(" found %d\n", found);
return 0;
}
return 1;
}
#define MESSAGE_CHECK_STR_EQ(expected, found, prop) \
if (!check_str_eq(expected, #prop, expected->prop, found->prop)) return 0
#define MESSAGE_CHECK_NUM_EQ(expected, found, prop) \
if (!check_num_eq(expected, #prop, expected->prop, found->prop)) return 0
int
message_eq (int index, const struct message *expected)
{
int i;
struct message *m = &messages[index];
MESSAGE_CHECK_NUM_EQ(expected, m, http_major);
MESSAGE_CHECK_NUM_EQ(expected, m, http_minor);
if (expected->type == HTTP_REQUEST) {
MESSAGE_CHECK_NUM_EQ(expected, m, method);
} else {
MESSAGE_CHECK_NUM_EQ(expected, m, status_code);
MESSAGE_CHECK_STR_EQ(expected, m, response_reason);
}
MESSAGE_CHECK_NUM_EQ(expected, m, message_complete_on_eof);
assert(m->message_begin_cb_called);
assert(m->headers_complete_cb_called);
assert(m->message_complete_cb_called);
MESSAGE_CHECK_STR_EQ(expected, m, request_url);
if (expected->body_size) {
MESSAGE_CHECK_NUM_EQ(expected, m, body_size);
} else {
MESSAGE_CHECK_STR_EQ(expected, m, body);
}
MESSAGE_CHECK_NUM_EQ(expected, m, num_chunks);
MESSAGE_CHECK_NUM_EQ(expected, m, num_chunks_complete);
for (i = 0; i < m->num_chunks && i < MAX_CHUNKS; i++) {
MESSAGE_CHECK_NUM_EQ(expected, m, chunk_lengths[i]);
}
MESSAGE_CHECK_NUM_EQ(expected, m, num_headers);
int r;
for (i = 0; i < m->num_headers; i++) {
r = check_str_eq(expected, "header field", expected->headers[i][0], m->headers[i][0]);
if (!r) return 0;
r = check_str_eq(expected, "header value", expected->headers[i][1], m->headers[i][1]);
if (!r) return 0;
}
MESSAGE_CHECK_STR_EQ(expected, m, upgrade);
return 1;
}
/* Given a sequence of varargs messages, return the number of them that the
* parser should successfully parse, taking into account that upgraded
* messages prevent all subsequent messages from being parsed.
*/
size_t
count_parsed_messages(const size_t nmsgs, ...) {
size_t i;
va_list ap;
va_start(ap, nmsgs);
for (i = 0; i < nmsgs; i++) {
struct message *m = va_arg(ap, struct message *);
if (m->upgrade) {
va_end(ap);
return i + 1;
}
}
va_end(ap);
return nmsgs;
}
/* Given a sequence of bytes and the number of these that we were able to
* parse, verify that upgrade bodies are correct.
*/
void
upgrade_message_fix(char *body, const size_t nread, const size_t nmsgs, ...) {
va_list ap;
size_t i;
size_t off = 0;
va_start(ap, nmsgs);
for (i = 0; i < nmsgs; i++) {
struct message *m = va_arg(ap, struct message *);
off += strlen(m->raw);
if (m->upgrade) {
off -= strlen(m->upgrade);
/* Check the portion of the response after its specified upgrade */
if (!check_str_eq(m, "upgrade", body + off, body + nread)) {
exit(1);
}
/* Fix up the response so that message_eq() will verify the beginning
* of the upgrade */
*(body + nread + strlen(m->upgrade)) = '\0';
messages[num_messages -1 ].upgrade = body + nread;
va_end(ap);
return;
}
}
va_end(ap);
printf("\n\n*** Error: expected a message with upgrade ***\n");
exit(1);
}
static void
print_error (const char *raw, size_t error_location)
{
fprintf(stderr, "\n*** %s:%d -- %s ***\n\n",
"http_parser.c", HTTP_PARSER_ERRNO_LINE(parser),
http_errno_description(HTTP_PARSER_ERRNO(parser)));
int this_line = 0, char_len = 0;
size_t i, j, len = strlen(raw), error_location_line = 0;
for (i = 0; i < len; i++) {
if (i == error_location) this_line = 1;
switch (raw[i]) {
case '\r':
char_len = 2;
fprintf(stderr, "\\r");
break;
case '\n':
char_len = 2;
fprintf(stderr, "\\n\n");
if (this_line) goto print;
error_location_line = 0;
continue;
default:
char_len = 1;
fputc(raw[i], stderr);
break;
}
if (!this_line) error_location_line += char_len;
}
fprintf(stderr, "[eof]\n");
print:
for (j = 0; j < error_location_line; j++) {
fputc(' ', stderr);
}
fprintf(stderr, "^\n\nerror location: %u\n", (unsigned int)error_location);
}
struct url_test {
const char *name;
const char *url;
int is_connect;
struct http_parser_url u;
int rv;
};
const struct url_test url_tests[] =
{ {.name="proxy request"
,.url="http://hostname/"
,.is_connect=0
,.u=
{.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PATH)
,.port=0
,.field_data=
{{ 0, 4 } /* UF_SCHEMA */
,{ 7, 8 } /* UF_HOST */
,{ 0, 0 } /* UF_PORT */
,{ 15, 1 } /* UF_PATH */
,{ 0, 0 } /* UF_QUERY */
,{ 0, 0 } /* UF_FRAGMENT */
,{ 0, 0 } /* UF_USERINFO */
}
}
,.rv=0
}
, {.name="proxy request with port"
,.url="http://hostname:444/"
,.is_connect=0
,.u=
{.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PORT) | (1 << UF_PATH)
,.port=444
,.field_data=
{{ 0, 4 } /* UF_SCHEMA */
,{ 7, 8 } /* UF_HOST */
,{ 16, 3 } /* UF_PORT */
,{ 19, 1 } /* UF_PATH */
,{ 0, 0 } /* UF_QUERY */
,{ 0, 0 } /* UF_FRAGMENT */
,{ 0, 0 } /* UF_USERINFO */
}
}
,.rv=0
}
, {.name="CONNECT request"
,.url="hostname:443"
,.is_connect=1
,.u=
{.field_set=(1 << UF_HOST) | (1 << UF_PORT)
,.port=443
,.field_data=
{{ 0, 0 } /* UF_SCHEMA */
,{ 0, 8 } /* UF_HOST */
,{ 9, 3 } /* UF_PORT */
,{ 0, 0 } /* UF_PATH */
,{ 0, 0 } /* UF_QUERY */
,{ 0, 0 } /* UF_FRAGMENT */
,{ 0, 0 } /* UF_USERINFO */
}
}
,.rv=0
}
, {.name="CONNECT request but not connect"
,.url="hostname:443"
,.is_connect=0
,.rv=1
}
, {.name="proxy ipv6 request"
,.url="http://[1:2::3:4]/"
,.is_connect=0
,.u=
{.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PATH)
,.port=0
,.field_data=
{{ 0, 4 } /* UF_SCHEMA */
,{ 8, 8 } /* UF_HOST */
,{ 0, 0 } /* UF_PORT */
,{ 17, 1 } /* UF_PATH */
,{ 0, 0 } /* UF_QUERY */
,{ 0, 0 } /* UF_FRAGMENT */
,{ 0, 0 } /* UF_USERINFO */
}
}
,.rv=0
}
, {.name="proxy ipv6 request with port"
,.url="http://[1:2::3:4]:67/"
,.is_connect=0
,.u=
{.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PORT) | (1 << UF_PATH)
,.port=67
,.field_data=
{{ 0, 4 } /* UF_SCHEMA */
,{ 8, 8 } /* UF_HOST */
,{ 18, 2 } /* UF_PORT */
,{ 20, 1 } /* UF_PATH */
,{ 0, 0 } /* UF_QUERY */
,{ 0, 0 } /* UF_FRAGMENT */
,{ 0, 0 } /* UF_USERINFO */
}
}
,.rv=0
}
, {.name="CONNECT ipv6 address"
,.url="[1:2::3:4]:443"
,.is_connect=1
,.u=
{.field_set=(1 << UF_HOST) | (1 << UF_PORT)
,.port=443
,.field_data=
{{ 0, 0 } /* UF_SCHEMA */
,{ 1, 8 } /* UF_HOST */
,{ 11, 3 } /* UF_PORT */
,{ 0, 0 } /* UF_PATH */
,{ 0, 0 } /* UF_QUERY */
,{ 0, 0 } /* UF_FRAGMENT */
,{ 0, 0 } /* UF_USERINFO */
}
}
,.rv=0
}
, {.name="ipv4 in ipv6 address"
,.url="http://[2001:0000:0000:0000:0000:0000:1.9.1.1]/"
,.is_connect=0
,.u=
{.field_set=(1 << UF_SCHEMA) | (1 << UF_HOST) | (1 << UF_PATH)
,.port=0
,.field_data=
{{ 0, 4 } /* UF_SCHEMA */
,{ 8, 37 } /* UF_HOST */
,{ 0, 0 } /* UF_PORT */
,{ 46, 1 } /* UF_PATH */
,{ 0, 0 } /* UF_QUERY */
,{ 0, 0 } /* UF_FRAGMENT */
,{ 0, 0 } /* UF_USERINFO */
}
}
,.rv=0
}
, {.name="extra ? in query string"
,.url="http://a.tbcdn.cn/p/fp/2010c/??fp-header-min.css,fp-base-min.css,"
"fp-channel-min.css,fp-product-min.css,fp-mall-min.css,fp-category-min.css,"
"fp-sub-min.css,fp-gdp4p-min.css,fp-css3-min.css,fp-misc-min.css?t=20101022.css"
,.is_connect=0
,.u=
{.field_set=(1<field_set, u->port);
for (i = 0; i < UF_MAX; i++) {
if ((u->field_set & (1 << i)) == 0) {
printf("\tfield_data[%u]: unset\n", i);
continue;
}
printf("\tfield_data[%u]: off: %u len: %u part: \"%.*s\n\"",
i,
u->field_data[i].off,
u->field_data[i].len,
u->field_data[i].len,
url + u->field_data[i].off);
}
}
void
test_parse_url (void)
{
struct http_parser_url u;
const struct url_test *test;
unsigned int i;
int rv;
for (i = 0; i < (sizeof(url_tests) / sizeof(url_tests[0])); i++) {
test = &url_tests[i];
memset(&u, 0, sizeof(u));
#if HTTP_PARSER_STRICT_URL
rv = http_parser_parse_url_options(test->url,
strlen(test->url),
test->is_connect,
&u,
F_PARSE_URL_OPTIONS_URL_STRICT);
#else
rv = http_parser_parse_url(test->url,
strlen(test->url),
test->is_connect,
&u);
#endif
if (test->rv == 0) {
if (rv != 0) {
printf("\n*** http_parser_parse_url(\"%s\") \"%s\" test failed, "
"unexpected rv %d ***\n\n", test->url, test->name, rv);
abort();
}
if (memcmp(&u, &test->u, sizeof(u)) != 0) {
printf("\n*** http_parser_parse_url(\"%s\") \"%s\" failed ***\n",
test->url, test->name);
printf("target http_parser_url:\n");
dump_url(test->url, &test->u);
printf("result http_parser_url:\n");
dump_url(test->url, &u);
abort();
}
} else {
/* test->rv != 0 */
if (rv == 0) {
printf("\n*** http_parser_parse_url(\"%s\") \"%s\" test failed, "
"unexpected rv %d ***\n\n", test->url, test->name, rv);
abort();
}
}
}
}
void
test_message (const struct message *message)
{
size_t raw_len = strlen(message->raw);
size_t msg1len;
for (msg1len = 0; msg1len < raw_len; msg1len++) {
parser_init(message->type);
size_t read;
const char *msg1 = message->raw;
const char *msg2 = msg1 + msg1len;
size_t msg2len = raw_len - msg1len;
if (msg1len) {
read = parse(msg1, msg1len);
if (message->upgrade && parser->upgrade && num_messages > 0) {
messages[num_messages - 1].upgrade = msg1 + read;
goto test;
}
if (read != msg1len) {
print_error(msg1, read);
exit(1);
}
}
read = parse(msg2, msg2len);
if (message->upgrade && parser->upgrade) {
messages[num_messages - 1].upgrade = msg2 + read;
goto test;
}
if (read != msg2len) {
print_error(msg2, read);
exit(1);
}
read = parse(NULL, 0);
if (read != 0) {
print_error(message->raw, read);
exit(1);
}
test:
if (num_messages != 1) {
printf("\n*** num_messages != 1 after testing '%s' ***\n\n", message->name);
exit(1);
}
if(!message_eq(0, message)) exit(1);
parser_free();
}
}
void
test_message_count_body (const struct message *message)
{
parser_init(message->type);
size_t read;
size_t l = strlen(message->raw);
size_t i, toread;
size_t chunk = 4024;
for (i = 0; i < l; i+= chunk) {
toread = MIN(l-i, chunk);
read = parse_count_body(message->raw + i, toread);
if (read != toread) {
print_error(message->raw, read);
exit(1);
}
}
read = parse_count_body(NULL, 0);
if (read != 0) {
print_error(message->raw, read);
exit(1);
}
if (num_messages != 1) {
printf("\n*** num_messages != 1 after testing '%s' ***\n\n", message->name);
exit(1);
}
if(!message_eq(0, message)) exit(1);
parser_free();
}
void
test_simple (const char *buf, enum http_errno err_expected)
{
parser_init(HTTP_REQUEST);
size_t parsed;
int pass;
enum http_errno err;
parsed = parse(buf, strlen(buf));
pass = (parsed == strlen(buf));
err = HTTP_PARSER_ERRNO(parser);
parsed = parse(NULL, 0);
pass &= (parsed == 0);
parser_free();
/* In strict mode, allow us to pass with an unexpected HPE_STRICT as
* long as the caller isn't expecting success.
*/
#if HTTP_PARSER_STRICT_URL || HTTP_PARSER_STRICT_HOSTNAME
if (err_expected != err && err_expected != HPE_OK && err != HPE_STRICT) {
assert(pass != 0);
#else
if (err_expected != err) {
assert(pass == 0);
#endif
fprintf(stderr, "\n*** test_simple expected %s, but saw %s ***\n\n%s\n",
http_errno_name(err_expected), http_errno_name(err), buf);
exit(1);
}
(void)pass;
}
#if HTTP_PARSER_STRICT_URL
void
test_lax_in_strict_mode (const char *buf, enum http_errno err_expected)
{
parser_init(HTTP_REQUEST);
// clear strict flag
parser_options = 0;
size_t parsed;
int pass;
enum http_errno err;
parsed = parse(buf, strlen(buf));
pass = (parsed == strlen(buf));
err = HTTP_PARSER_ERRNO(parser);
parsed = parse(NULL, 0);
pass &= (parsed == 0);
parser_free();
/* In strict mode, allow us to pass with an unexpected HPE_STRICT as
* long as the caller isn't expecting success.
*/
if (err_expected != err && err_expected != HPE_OK && err != HPE_STRICT) {
assert(pass != 0);
fprintf(stderr, "\n*** test_simple expected %s, but saw %s ***\n\n%s\n",
http_errno_name(err_expected), http_errno_name(err), buf);
exit(1);
}
(void)pass;
}
#endif
void
test_no_overflow_parse_url (void)
{
int rv;
struct http_parser_url u;
rv = http_parser_parse_url("http://example.com:8001", 22, 0, &u);
if (rv != 0) {
fprintf(stderr,
"\n*** test_no_overflow_parse_url invalid return value=%d\n",
rv);
abort();
}
if (u.port != 800) {
fprintf(stderr,
"\n*** test_no_overflow_parse_url invalid port number=%d\n",
u.port);
abort();
}
}
void
test_header_overflow_error (int req)
{
http_parser parser;
http_parser_init(&parser, req ? HTTP_REQUEST : HTTP_RESPONSE);
size_t parsed;
const char *buf;
buf = req ? "GET / HTTP/1.1\r\n" : "HTTP/1.0 200 OK\r\n";
parsed = http_parser_execute_options(
&parser, &settings_null, parser_options, buf, strlen(buf));
assert(parsed == strlen(buf));
buf = "header-key: header-value\r\n";
size_t buflen = strlen(buf);
int i;
for (i = 0; i < 10000; i++) {
parsed = http_parser_execute_options(
&parser, &settings_null, parser_options, buf, buflen);
if (parsed != buflen) {
//fprintf(stderr, "error found on iter %d\n", i);
assert(HTTP_PARSER_ERRNO(&parser) == HPE_HEADER_OVERFLOW);
return;
}
}
fprintf(stderr, "\n*** Error expected but none in header overflow test ***\n");
exit(1);
}
void
test_no_overflow_long_body (int req, size_t length)
{
http_parser parser;
http_parser_init(&parser, req ? HTTP_REQUEST : HTTP_RESPONSE);
size_t parsed;
size_t i;
char buf1[3000];
size_t buf1len = snprintf(buf1, sizeof(buf1), "%s\r\nConnection: Keep-Alive\r\nContent-Length: %zu\r\n\r\n",
req ? "POST / HTTP/1.0" : "HTTP/1.0 200 OK", length);
parsed = http_parser_execute_options(
&parser, &settings_null, parser_options, buf1, buf1len);
if (parsed != buf1len)
goto err;
for (i = 0; i < length; i++) {
char foo = 'a';
parsed = http_parser_execute_options(
&parser, &settings_null, parser_options, &foo, 1);
if (parsed != 1)
goto err;
}
parsed = http_parser_execute_options(
&parser, &settings_null, parser_options, buf1, buf1len);
if (parsed != buf1len) goto err;
return;
err:
fprintf(stderr,
"\n*** error in test_no_overflow_long_body %s of length %zu ***\n",
req ? "REQUEST" : "RESPONSE",
length);
exit(1);
}
void
test_multiple3 (const struct message *r1, const struct message *r2, const struct message *r3)
{
int message_count = count_parsed_messages(3, r1, r2, r3);
char total[ strlen(r1->raw)
+ strlen(r2->raw)
+ strlen(r3->raw)
+ 1
];
total[0] = '\0';
strcat(total, r1->raw);
strcat(total, r2->raw);
strcat(total, r3->raw);
parser_init(r1->type);
size_t read;
read = parse(total, strlen(total));
if (parser->upgrade) {
upgrade_message_fix(total, read, 3, r1, r2, r3);
goto test;
}
if (read != strlen(total)) {
print_error(total, read);
exit(1);
}
read = parse(NULL, 0);
if (read != 0) {
print_error(total, read);
exit(1);
}
test:
if (message_count != num_messages) {
fprintf(stderr, "\n\n*** Parser didn't see 3 messages only %d *** \n", num_messages);
exit(1);
}
if (!message_eq(0, r1)) exit(1);
if (message_count > 1 && !message_eq(1, r2)) exit(1);
if (message_count > 2 && !message_eq(2, r3)) exit(1);
parser_free();
}
/* SCAN through every possible breaking to make sure the
* parser can handle getting the content in any chunks that
* might come from the socket
*/
void
test_scan (const struct message *r1, const struct message *r2, const struct message *r3)
{
char total[80*1024] = "\0";
char buf1[80*1024] = "\0";
char buf2[80*1024] = "\0";
char buf3[80*1024] = "\0";
strcat(total, r1->raw);
strcat(total, r2->raw);
strcat(total, r3->raw);
size_t read;
int total_len = strlen(total);
int total_ops = 2 * (total_len - 1) * (total_len - 2) / 2;
int ops = 0 ;
size_t buf1_len, buf2_len, buf3_len;
int message_count = count_parsed_messages(3, r1, r2, r3);
int i,j,type_both;
for (type_both = 0; type_both < 2; type_both ++ ) {
for (j = 2; j < total_len; j ++ ) {
for (i = 1; i < j; i ++ ) {
if (ops % 1000 == 0) {
printf("\b\b\b\b%3.0f%%", 100 * (float)ops /(float)total_ops);
fflush(stdout);
}
ops += 1;
parser_init(type_both ? HTTP_BOTH : r1->type);
buf1_len = i;
strncpy(buf1, total, buf1_len);
buf1[buf1_len] = 0;
buf2_len = j - i;
strncpy(buf2, total+i, buf2_len);
buf2[buf2_len] = 0;
buf3_len = total_len - j;
strncpy(buf3, total+j, buf3_len);
buf3[buf3_len] = 0;
read = parse(buf1, buf1_len);
if (parser->upgrade) goto test;
if (read != buf1_len) {
print_error(buf1, read);
goto error;
}
read += parse(buf2, buf2_len);
if (parser->upgrade) goto test;
if (read != buf1_len + buf2_len) {
print_error(buf2, read);
goto error;
}
read += parse(buf3, buf3_len);
if (parser->upgrade) goto test;
if (read != buf1_len + buf2_len + buf3_len) {
print_error(buf3, read);
goto error;
}
parse(NULL, 0);
test:
if (parser->upgrade) {
upgrade_message_fix(total, read, 3, r1, r2, r3);
}
if (message_count != num_messages) {
fprintf(stderr, "\n\nParser didn't see %d messages only %d\n",
message_count, num_messages);
goto error;
}
if (!message_eq(0, r1)) {
fprintf(stderr, "\n\nError matching messages[0] in test_scan.\n");
goto error;
}
if (message_count > 1 && !message_eq(1, r2)) {
fprintf(stderr, "\n\nError matching messages[1] in test_scan.\n");
goto error;
}
if (message_count > 2 && !message_eq(2, r3)) {
fprintf(stderr, "\n\nError matching messages[2] in test_scan.\n");
goto error;
}
parser_free();
}
}
}
puts("\b\b\b\b100%");
return;
error:
fprintf(stderr, "i=%d j=%d\n", i, j);
fprintf(stderr, "buf1 (%u) %s\n\n", (unsigned int)buf1_len, buf1);
fprintf(stderr, "buf2 (%u) %s\n\n", (unsigned int)buf2_len , buf2);
fprintf(stderr, "buf3 (%u) %s\n", (unsigned int)buf3_len, buf3);
exit(1);
}
// user required to free the result
// string terminated by \0
char *
create_large_chunked_message (int body_size_in_kb, const char* headers)
{
int i;
size_t wrote = 0;
size_t headers_len = strlen(headers);
size_t bufsize = headers_len + (5+1024+2)*body_size_in_kb + 6;
char * buf = malloc(bufsize);
memcpy(buf, headers, headers_len);
wrote += headers_len;
for (i = 0; i < body_size_in_kb; i++) {
// write 1kb chunk into the body.
memcpy(buf + wrote, "400\r\n", 5);
wrote += 5;
memset(buf + wrote, 'C', 1024);
wrote += 1024;
strcpy(buf + wrote, "\r\n");
wrote += 2;
}
memcpy(buf + wrote, "0\r\n\r\n", 6);
wrote += 6;
assert(wrote == bufsize);
return buf;
}
/* Verify that we can pause parsing at any of the bytes in the
* message and still get the result that we're expecting. */
void
test_message_pause (const struct message *msg)
{
char *buf = (char*) msg->raw;
size_t buflen = strlen(msg->raw);
size_t nread;
parser_init(msg->type);
do {
nread = parse_pause(buf, buflen);
// We can only set the upgrade buffer once we've gotten our message
// completion callback.
if (messages[0].message_complete_cb_called &&
msg->upgrade &&
parser->upgrade) {
messages[0].upgrade = buf + nread;
goto test;
}
if (nread < buflen) {
// Not much do to if we failed a strict-mode check
if (HTTP_PARSER_ERRNO(parser) == HPE_STRICT) {
parser_free();
return;
}
assert (HTTP_PARSER_ERRNO(parser) == HPE_PAUSED);
}
buf += nread;
buflen -= nread;
http_parser_pause(parser, 0);
} while (buflen > 0);
nread = parse_pause(NULL, 0);
assert (nread == 0);
test:
if (num_messages != 1) {
printf("\n*** num_messages != 1 after testing '%s' ***\n\n", msg->name);
exit(1);
}
if(!message_eq(0, msg)) exit(1);
parser_free();
}
/* Verify that on_message_begin callback is called regardless of correctness of
* the message */
void
test_on_message_begin_cb(const struct message *message)
{
parser_init(message->type);
assert(messages[num_messages].message_begin_cb_called == 0);
parse(message->raw, strlen(message->raw));
assert(messages[num_messages].message_begin_cb_called == 1);
parser_free();
}
int
main (void)
{
parser = NULL;
int i, j, k;
int request_count;
int response_count;
int on_message_begin_cb_count;
printf("sizeof(http_parser) = %u\n", (unsigned int)sizeof(http_parser));
for (request_count = 0; requests[request_count].name; request_count++);
for (response_count = 0; responses[response_count].name; response_count++);
for (on_message_begin_cb_count = 0;
on_message_begin_cb_test[on_message_begin_cb_count].name;
on_message_begin_cb_count++);
//// API
test_parse_url();
/// TESTING ON_MESSAGE_BEGIN CALLS
for (i = 0 ; i < on_message_begin_cb_count; i++) {
test_on_message_begin_cb(&on_message_begin_cb_test[i]);
}
//// OVERFLOW CONDITIONS
test_no_overflow_parse_url();
test_header_overflow_error(HTTP_REQUEST);
test_no_overflow_long_body(HTTP_REQUEST, 1000);
test_no_overflow_long_body(HTTP_REQUEST, 100000);
test_header_overflow_error(HTTP_RESPONSE);
test_no_overflow_long_body(HTTP_RESPONSE, 1000);
test_no_overflow_long_body(HTTP_RESPONSE, 100000);
//// RESPONSES
for (i = 0; i < response_count; i++) {
test_message(&responses[i]);
}
for (i = 0; i < response_count; i++) {
test_message_pause(&responses[i]);
}
for (i = 0; i < response_count; i++) {
if (!responses[i].should_keep_alive) continue;
for (j = 0; j < response_count; j++) {
if (!responses[j].should_keep_alive) continue;
for (k = 0; k < response_count; k++) {
test_multiple3(&responses[i], &responses[j], &responses[k]);
}
}
}
test_message_count_body(&responses[NO_HEADERS_NO_BODY_404]);
test_message_count_body(&responses[TRAILING_SPACE_ON_CHUNKED_BODY]);
// test very large chunked response
{
char * msg = create_large_chunked_message(31337,
"HTTP/1.0 200 OK\r\n"
"Transfer-Encoding: chunked\r\n"
"Content-Type: text/plain\r\n"
"\r\n");
struct message large_chunked =
{.name= "large chunked"
,.type= HTTP_RESPONSE
,.raw= msg
,.should_keep_alive= FALSE
,.message_complete_on_eof= FALSE
,.http_major= 1
,.http_minor= 0
,.status_code= 200
,.response_reason= "OK"
,.num_headers= 2
,.headers=
{ { "Transfer-Encoding", "chunked" }
, { "Content-Type", "text/plain" }
}
,.body_size= 31337*1024
,.num_chunks= 31337
,.num_chunks_complete= 31338
};
for (i = 0; i < MAX_CHUNKS; ++i) {
large_chunked.chunk_lengths[i] = 1024;
}
test_message_count_body(&large_chunked);
free(msg);
}
printf("response scan 1/2 ");
test_scan( &responses[TRAILING_SPACE_ON_CHUNKED_BODY]
, &responses[NO_REASON_PHRASE]
, &responses[NO_HEADERS_NO_BODY_404]
);
printf("response scan 2/2 ");
test_scan( &responses[BONJOUR_MADAME_FR]
, &responses[UNDERSTORE_HEADER_KEY]
, &responses[NO_CARRIAGE_RET]
);
puts("responses okay");
/// REQUESTS
test_simple("hello world", HPE_INVALID_METHOD);
test_simple("GET / HTP/1.1\r\n\r\n", HPE_INVALID_VERSION);
test_simple("ASDF / HTTP/1.1\r\n\r\n", HPE_INVALID_METHOD);
test_simple("PROPPATCHA / HTTP/1.1\r\n\r\n", HPE_INVALID_METHOD);
test_simple("GETA / HTTP/1.1\r\n\r\n", HPE_INVALID_METHOD);
// Well-formed but incomplete
test_simple("GET / HTTP/1.1\r\n"
"Content-Type: text/plain\r\n"
"Content-Length: 6\r\n"
"\r\n"
"fooba",
HPE_OK);
static const char *all_methods[] = {
"DELETE",
"GET",
"HEAD",
"POST",
"PUT",
//"CONNECT", //CONNECT can't be tested like other methods, it's a tunnel
"OPTIONS",
"TRACE",
"COPY",
"LOCK",
"MKCOL",
"MOVE",
"PROPFIND",
"PROPPATCH",
"UNLOCK",
"REPORT",
"MKACTIVITY",
"CHECKOUT",
"MERGE",
"M-SEARCH",
"NOTIFY",
"SUBSCRIBE",
"UNSUBSCRIBE",
"PATCH",
0 };
const char **this_method;
for (this_method = all_methods; *this_method; this_method++) {
char buf[200];
snprintf(buf, sizeof(buf), "%s / HTTP/1.1\r\n\r\n", *this_method);
test_simple(buf, HPE_OK);
}
static const char *bad_methods[] = {
"C******",
"M****",
0 };
for (this_method = bad_methods; *this_method; this_method++) {
char buf[200];
snprintf(buf, sizeof(buf), "%s / HTTP/1.1\r\n\r\n", *this_method);
test_simple(buf, HPE_UNKNOWN);
}
const char *dumbfuck2 =
"GET / HTTP/1.1\r\n"
"X-SSL-Bullshit: -----BEGIN CERTIFICATE-----\r\n"
"\tMIIFbTCCBFWgAwIBAgICH4cwDQYJKoZIhvcNAQEFBQAwcDELMAkGA1UEBhMCVUsx\r\n"
"\tETAPBgNVBAoTCGVTY2llbmNlMRIwEAYDVQQLEwlBdXRob3JpdHkxCzAJBgNVBAMT\r\n"
"\tAkNBMS0wKwYJKoZIhvcNAQkBFh5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMu\r\n"
"\tdWswHhcNMDYwNzI3MTQxMzI4WhcNMDcwNzI3MTQxMzI4WjBbMQswCQYDVQQGEwJV\r\n"
"\tSzERMA8GA1UEChMIZVNjaWVuY2UxEzARBgNVBAsTCk1hbmNoZXN0ZXIxCzAJBgNV\r\n"
"\tBAcTmrsogriqMWLAk1DMRcwFQYDVQQDEw5taWNoYWVsIHBhcmQYJKoZIhvcNAQEB\r\n"
"\tBQADggEPADCCAQoCggEBANPEQBgl1IaKdSS1TbhF3hEXSl72G9J+WC/1R64fAcEF\r\n"
"\tW51rEyFYiIeZGx/BVzwXbeBoNUK41OK65sxGuflMo5gLflbwJtHBRIEKAfVVp3YR\r\n"
"\tgW7cMA/s/XKgL1GEC7rQw8lIZT8RApukCGqOVHSi/F1SiFlPDxuDfmdiNzL31+sL\r\n"
"\t0iwHDdNkGjy5pyBSB8Y79dsSJtCW/iaLB0/n8Sj7HgvvZJ7x0fr+RQjYOUUfrePP\r\n"
"\tu2MSpFyf+9BbC/aXgaZuiCvSR+8Snv3xApQY+fULK/xY8h8Ua51iXoQ5jrgu2SqR\r\n"
"\twgA7BUi3G8LFzMBl8FRCDYGUDy7M6QaHXx1ZWIPWNKsCAwEAAaOCAiQwggIgMAwG\r\n"
"\tA1UdEwEB/wQCMAAwEQYJYIZIAYb4QgHTTPAQDAgWgMA4GA1UdDwEB/wQEAwID6DAs\r\n"
"\tBglghkgBhvhCAQ0EHxYdVUsgZS1TY2llbmNlIFVzZXIgQ2VydGlmaWNhdGUwHQYD\r\n"
"\tVR0OBBYEFDTt/sf9PeMaZDHkUIldrDYMNTBZMIGaBgNVHSMEgZIwgY+AFAI4qxGj\r\n"
"\tloCLDdMVKwiljjDastqooXSkcjBwMQswCQYDVQQGEwJVSzERMA8GA1UEChMIZVNj\r\n"
"\taWVuY2UxEjAQBgNVBAsTCUF1dGhvcml0eTELMAkGA1UEAxMCQ0ExLTArBgkqhkiG\r\n"
"\t9w0BCQEWHmNhLW9wZXJhdG9yQGdyaWQtc3VwcG9ydC5hYy51a4IBADApBgNVHRIE\r\n"
"\tIjAggR5jYS1vcGVyYXRvckBncmlkLXN1cHBvcnQuYWMudWswGQYDVR0gBBIwEDAO\r\n"
"\tBgwrBgEEAdkvAQEBAQYwPQYJYIZIAYb4QgEEBDAWLmh0dHA6Ly9jYS5ncmlkLXN1\r\n"
"\tcHBvcnQuYWMudmT4sopwqlBWsvcHViL2NybC9jYWNybC5jcmwwPQYJYIZIAYb4QgEDBDAWLmh0\r\n"
"\tdHA6Ly9jYS5ncmlkLXN1cHBvcnQuYWMudWsvcHViL2NybC9jYWNybC5jcmwwPwYD\r\n"
"\tVR0fBDgwNjA0oDKgMIYuaHR0cDovL2NhLmdyaWQt5hYy51ay9wdWIv\r\n"
"\tY3JsL2NhY3JsLmNybDANBgkqhkiG9w0BAQUFAAOCAQEAS/U4iiooBENGW/Hwmmd3\r\n"
"\tXCy6Zrt08YjKCzGNjorT98g8uGsqYjSxv/hmi0qlnlHs+k/3Iobc3LjS5AMYr5L8\r\n"
"\tUO7OSkgFFlLHQyC9JzPfmLCAugvzEbyv4Olnsr8hbxF1MbKZoQxUZtMVu29wjfXk\r\n"
"\thTeApBv7eaKCWpSp7MCbvgzm74izKhu3vlDk9w6qVrxePfGgpKPqfHiOoGhFnbTK\r\n"
"\twTC6o2xq5y0qZ03JonF7OJspEd3I5zKY3E+ov7/ZhW6DqT8UFvsAdjvQbXyhV8Eu\r\n"
"\tYhixw1aKEPzNjNowuIseVogKOLXxWI5vAi5HgXdS0/ES5gDGsABo4fqovUKlgop3\r\n"
"\tRA==\r\n"
"\t-----END CERTIFICATE-----\r\n"
"\r\n";
test_simple(dumbfuck2, HPE_OK);
const char *corrupted_header_name =
"GET / HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"X-Some-Header\r\033\065\325eep-Alive\r\n"
"Accept-Encoding: gzip\r\n"
"\r\n";
test_simple(corrupted_header_name, HPE_INVALID_HEADER_TOKEN);
const char *header_with_space =
"GET / HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"Foo Foo: some_value\r\n"
"Accept-Encoding: gzip\r\n"
"\r\n";
test_simple(header_with_space, HPE_INVALID_HEADER_TOKEN);
const char *header_with_curly_brace =
"GET / HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"Foo}Foo: some_value\r\n"
"Accept-Encoding: gzip\r\n"
"\r\n";
test_simple(header_with_curly_brace, HPE_INVALID_HEADER_TOKEN);
const char *header_with_quote =
"GET / HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"Foo\"Foo: some_value\r\n"
"Accept-Encoding: gzip\r\n"
"\r\n";
test_simple(header_with_quote, HPE_INVALID_HEADER_TOKEN);
const char *header_with_forward_slash =
"GET / HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"Foo/Foo: some_value\r\n"
"Accept-Encoding: gzip\r\n"
"\r\n";
test_simple(header_with_forward_slash, HPE_INVALID_HEADER_TOKEN);
const char *header_with_trailing_space =
"GET / HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"X-Some-Header : some_value\r\n"
"\r\n";
test_simple(header_with_trailing_space, HPE_INVALID_HEADER_TOKEN);
const char *bad_end_of_headers_1 =
"GET / HTTP/1.1\r\n"
"Host: www.example1.com\r\n"
"X-Some-Header: some_value"
"\r\n\r*";
test_simple(bad_end_of_headers_1, HPE_STRICT);
const char *bad_end_of_headers_2 =
"GET / HTTP/1.1\r\n"
"Host: www.example2.com\r\n"
"X-Some-Header: some_value"
"\n\r*";
test_simple(bad_end_of_headers_2, HPE_STRICT);
const char *empty_content_length_header =
"GET / HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"Content-Length:\r\n"
"Accept-Encoding: gzip\r\n"
"\r\n";
test_simple(empty_content_length_header, HPE_INVALID_CONTENT_LENGTH);
const char *empty_transfer_encoding_header =
"GET / HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"Transfer-Encoding:\r\n"
"Accept-Encoding: gzip\r\n"
"\r\n";
test_simple(empty_transfer_encoding_header, HPE_INVALID_TRANSFER_ENCODING);
const char *empty_upgrade_header =
"GET / HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"Upgrade:\r\n"
"Accept-Encoding: gzip\r\n"
"\r\n";
test_simple(empty_upgrade_header, HPE_INVALID_UPGRADE);
#if 0
// NOTE(Wed Nov 18 11:57:27 CET 2009) this seems okay. we just read body
// until EOF.
//
// no content-length
// error if there is a body without content length
const char *bad_get_no_headers_no_body = "GET /bad_get_no_headers_no_body/world HTTP/1.1\r\n"
"Accept: */*\r\n"
"\r\n"
"HELLO";
test_simple(bad_get_no_headers_no_body, 0);
#endif
/* TODO sending junk and large headers gets rejected */
/* check to make sure our predefined requests are okay */
for (i = 0; requests[i].name; i++) {
test_message(&requests[i]);
}
for (i = 0; i < request_count; i++) {
test_message_pause(&requests[i]);
}
for (i = 0; i < request_count; i++) {
if (!requests[i].should_keep_alive) continue;
for (j = 0; j < request_count; j++) {
if (!requests[j].should_keep_alive) continue;
for (k = 0; k < request_count; k++) {
test_multiple3(&requests[i], &requests[j], &requests[k]);
}
}
}
printf("request scan 1/4 ");
test_scan( &requests[GET_NO_HEADERS_NO_BODY]
, &requests[GET_ONE_HEADER_NO_BODY]
, &requests[GET_NO_HEADERS_NO_BODY]
);
printf("request scan 2/4 ");
test_scan( &requests[POST_CHUNKED_ALL_YOUR_BASE]
, &requests[POST_IDENTITY_BODY_WORLD]
, &requests[GET_FUNKY_CONTENT_LENGTH]
);
printf("request scan 3/4 ");
test_scan( &requests[TWO_CHUNKS_MULT_ZERO_END]
, &requests[CHUNKED_W_TRAILING_HEADERS]
, &requests[CHUNKED_W_BULLSHIT_AFTER_LENGTH]
);
printf("request scan 4/4 ");
test_scan( &requests[QUERY_URL_WITH_QUESTION_MARK_GET]
, &requests[PREFIX_NEWLINE_GET ]
, &requests[CONNECT_REQUEST]
);
#if HTTP_PARSER_STRICT_URL
const char *high_ascii_url =
"GET /high_ascii\xff HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"\r\n";
test_lax_in_strict_mode(high_ascii_url, HPE_OK);
test_simple(high_ascii_url, HPE_INVALID_PATH);
const char *ht_ff_url =
"GET /ht_ff\x09\x12 HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"\r\n";
test_lax_in_strict_mode(ht_ff_url, HPE_OK);
test_simple(ht_ff_url, HPE_INVALID_PATH);
const char *normal_url =
"GET /normal HTTP/1.1\r\n"
"Host: www.example.com\r\n"
"\r\n";
test_lax_in_strict_mode(normal_url, HPE_OK);
test_simple(normal_url, HPE_OK);
#endif
puts("requests okay");
return 0;
}
================================================
FILE: proxygen/fuzzers/CMakeLists.txt
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
function(proxygen_add_fuzzer)
if(NOT BUILD_FUZZERS)
return()
endif()
set(options)
set(one_value_args TARGET WORKING_DIRECTORY PREFIX)
set(multi_value_args SOURCES DEPENDS INCLUDES EXTRA_ARGS)
cmake_parse_arguments(PARSE_ARGV 0 PROXYGEN_FUZZ "${options}" "${one_value_args}" "${multi_value_args}")
# Remove any references to the main we have from gmock
# So we can use the one provided by the fuzzing engine
set(LIBGMOCK_LIBRARIES_NO_MAIN ${LIBGMOCK_LIBRARIES})
list(REMOVE_ITEM
LIBGMOCK_LIBRARIES_NO_MAIN
"${GTEST_BINARY_DIR}/${CMAKE_CFG_INTDIR}/googlemock/${CMAKE_STATIC_LIBRARY_PREFIX}gmock_main${CMAKE_STATIC_LIBRARY_SUFFIX}"
gmock_main
gmock_maind
GTest::gmock_main
)
if(NOT PROXYGEN_FUZZ_TARGET)
message(FATAL_ERROR "The TARGET parameter is mandatory.")
endif()
if(NOT PROXYGEN_FUZZ_SOURCES)
set(PROXYGEN_FUZZ_SOURCES "${PROXYGEN_FUZZ_TARGET}.cpp")
endif()
add_executable(${PROXYGEN_FUZZ_TARGET}
"${PROXYGEN_FUZZ_SOURCES}"
)
target_link_libraries(${PROXYGEN_FUZZ_TARGET} PRIVATE
"${PROXYGEN_FUZZ_DEPENDS}"
)
target_include_directories(${PROXYGEN_FUZZ_TARGET} PRIVATE
"${PROXYGEN_FUZZ_INCLUDES}"
)
target_compile_options(${PROXYGEN_FUZZ_TARGET} PRIVATE
"${_PROXYGEN_COMMON_COMPILE_OPTIONS}"
)
target_link_libraries(${PROXYGEN_FUZZ_TARGET} PRIVATE
${LIBGMOCK_LIBRARIES_NO_MAIN}
${LIB_FUZZING_ENGINE}
)
target_include_directories(${PROXYGEN_FUZZ_TARGET} SYSTEM PRIVATE
${LIBGMOCK_INCLUDE_DIR}
${LIBGTEST_INCLUDE_DIR}
)
target_compile_definitions(${PROXYGEN_FUZZ_TARGET} PRIVATE ${LIBGMOCK_DEFINES})
endfunction()
proxygen_add_fuzzer(TARGET ProxygenHTTP1xFuzzer
SOURCES
ProxygenHTTP1xFuzzer.cpp
DEPENDS
codectestutils
proxygen
)
================================================
FILE: proxygen/httpclient/CMakeLists.txt
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
if (BUILD_SAMPLES)
add_subdirectory(samples)
endif()
================================================
FILE: proxygen/httpclient/samples/CMakeLists.txt
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
add_subdirectory(curl)
add_subdirectory(H3Datagram)
add_subdirectory(httperf2)
================================================
FILE: proxygen/httpclient/samples/H3Datagram/CMakeLists.txt
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
add_executable(proxygen_h3datagram_client H3DatagramClient.cpp)
target_compile_options(
proxygen_h3datagram_client PRIVATE
${_PROXYGEN_COMMON_COMPILE_OPTIONS}
)
target_link_libraries(
proxygen_h3datagram_client
PUBLIC
proxygen
proxygencurl
)
install(
TARGETS proxygen_h3datagram_client
EXPORT proxygen-exports
DESTINATION bin
)
================================================
FILE: proxygen/httpclient/samples/H3Datagram/H3DatagramClient.cpp
================================================
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/
#include
#include
#include
#include
#include
#include
#include
#include
using namespace folly;
using namespace proxygen;
DEFINE_string(host, "::1", "Remote Server hostname/IP");
DEFINE_int32(port, 8888, "Remote Server port");
DEFINE_string(proxy_host, "::1", "Proxy hostname/IP");
DEFINE_int32(proxy_port, 6666, "Proxy port");
DEFINE_string(cert, "", "Certificate file path");
DEFINE_string(key, "", "Private key file path");
DEFINE_string(headers, "", "List of N=V headers separated by ,");
DEFINE_string(
masque_template,
"",
"URI template for RFC 9298 CONNECT-UDP (e.g., "
"https://proxy:4443/masque?h={target_host}&p={target_port}). "
"When set, uses RFC 9298 Extended CONNECT instead of legacy CONNECT-UDP.");
constexpr size_t kMaxReadBufferSize{1232};
namespace {
/*
* This class provides a sample client that speaks HTTP/3 Datagrams.
* It supports connecting to a remote UDP server through a proxy that supports
* masque connect-udp.
*
* The sample application logic implements the following.
* The datagram payload contains just an ASCII encoded integer number:
* - the client starts by sending a datagram with 0
* - on receiving an equally encoded datagram from the server, the client
* increment the received number by 1 and sends a new datagram
* - the client stops when reaching a maximum configured integer value
*/
class DatagramClient
: private folly::AsyncUDPSocket::ReadCallback
, private folly::AsyncTimeout {
public:
using folly::AsyncUDPSocket::ReadCallback::OnDataAvailableParams;
~DatagramClient() override = default;
explicit DatagramClient(folly::EventBase* evb, H3DatagramAsyncSocket& socket)
: folly::AsyncTimeout(evb), evb_(evb), socket_(socket) {
}
void start() {
CHECK(evb_->isInEventBaseThread());
try {
socket_.connect(SocketAddress(FLAGS_proxy_host, FLAGS_proxy_port));
} catch (const std::system_error& e) {
LOG(ERROR) << "Failed to connect to " << FLAGS_proxy_host << ":"
<< FLAGS_proxy_port << ": " << e.what();
return;
}
socket_.resumeRead(this);
sendPing();
}
void shutdown() {
CHECK(evb_->isInEventBaseThread());
socket_.pauseRead();
socket_.close();
closing_ = true;
}
void sendPing() {
if (n_ == 2000) {
shutdown();
return;
}
scheduleTimeout(1000);
writePing(folly::IOBuf::copyBuffer(folly::to(n_)));
}
virtual void writePing(std::unique_ptr buf) {
VLOG(2) << "Writing Datagram";
auto res =
socket_.write(SocketAddress(FLAGS_proxy_host, FLAGS_proxy_port), buf);
if (res < 0) {
LOG(ERROR) << "Failure to write: errno=" << errno;
}
}
void getReadBuffer(void** buf, size_t* len) noexcept override {
*buf = buf_.data();
*len = buf_.size();
}
void onDataAvailable(const folly::SocketAddress& client,
size_t len,
bool truncated,
OnDataAvailableParams) noexcept override {
++pongRecvd_;
VLOG(4) << "Read " << len << " bytes (trun:" << truncated << ") from "
<< client.describe() << " - " << std::string(buf_.data(), len);
auto datagramString = std::string(buf_.data(), len);
auto datagramInt = folly::tryTo(datagramString);
if (!datagramInt.hasValue()) {
VLOG(2) << "Received Datagram without Integer value. Stopping. len="
<< datagramString.length();
return;
}
VLOG(2) << "Received Datagram with Integer value (" << (int)*datagramInt
<< ")";
if (*datagramInt >= std::numeric_limits::max()) {
VLOG(2) << "Received Datagram with large Integer value. Stopping";
return;
}
VLOG(2) << "Sending Datagram with Integer value ("
<< (int)(*datagramInt + 1) << ")";
n_ = (int)(*datagramInt + 1);
scheduleTimeout(1000);
}
void onReadError(const folly::AsyncSocketException& ex) noexcept override {
LOG(ERROR) << ex.what();
}
void onReadClosed() noexcept override {
shutdown();
}
void timeoutExpired() noexcept override {
LOG(INFO) << "Timeout expired";
if (!closing_) {
sendPing();
}
}
private:
folly::EventBase* const evb_{nullptr};
H3DatagramAsyncSocket& socket_;
int pongRecvd_{0};
int n_{0};
std::array buf_;
bool closing_{false};
};
}; // namespace
int main(int argc, char* argv[]) {
#if FOLLY_HAVE_LIBGFLAGS
// Enable glog logging to stderr by default.
folly::gflags::SetCommandLineOptionWithMode(
"logtostderr", "1", folly::gflags::SET_FLAGS_DEFAULT);
#endif
const folly::Init init(&argc, &argv, false);
EventBase evb;
H3DatagramAsyncSocket::Options options;
options.mode_ = H3DatagramAsyncSocket::Mode::CLIENT;
options.txnTimeout_ = std::chrono::milliseconds(10000);
options.connectTimeout_ = std::chrono::milliseconds(500);
options.httpRequest_ = std::make_unique();
if (!FLAGS_masque_template.empty()) {
// RFC 9298 Extended CONNECT mode
auto target = proxygen::expandConnectUDPTemplate(
FLAGS_masque_template, FLAGS_host, FLAGS_port);
options.httpRequest_->setMethod(proxygen::HTTPMethod::CONNECT);
options.httpRequest_->setUpgradeProtocol("connect-udp");
options.httpRequest_->setSecure(true);
options.httpRequest_->setURL(target.path);
options.httpRequest_->getHeaders().set(proxygen::HTTP_HEADER_HOST,
target.authority);
options.httpRequest_->getHeaders().set("Capsule-Protocol", "?1");
options.rfcMode_ = true;
} else {
// Legacy CONNECT-UDP mode
options.httpRequest_->setMethod(proxygen::HTTPMethod::CONNECT_UDP);
options.httpRequest_->setURL(fmt::format("{}:{}", FLAGS_host, FLAGS_port));
options.httpRequest_->setMasque();
}
auto parsedHeaders = CurlService::CurlClient::parseHeaders(FLAGS_headers);
parsedHeaders.forEach(
[&options](const std::string& name, const std::string& value) {
options.httpRequest_->getHeaders().add(name, value);
});
options.certAndKey_ = std::make_pair(FLAGS_cert, FLAGS_key);
options.certVerifier_ = std::make_unique<
proxygen::InsecureVerifierDangerousDoNotUseInProduction>();
options.maxDatagramSize_ = kMaxReadBufferSize;
H3DatagramAsyncSocket datagramSocket(&evb, options);
DatagramClient client(&evb, datagramSocket);
client.start();
evb.loop();
return EXIT_SUCCESS;
}
================================================
FILE: proxygen/httpclient/samples/curl/CMakeLists.txt
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
# Manually maintained (generate_cmake.py skips this directory)
proxygen_add_library(proxygencurl
EXCLUDE_FROM_MONOLITH
SRCS
CurlClient.cpp
EXPORTED_DEPS
proxygen
)
add_executable(proxygen_curl CurlClientMain.cpp)
target_compile_options(
proxygen_curl PRIVATE
${_PROXYGEN_COMMON_COMPILE_OPTIONS}
)
target_link_libraries(
proxygen_curl
PUBLIC
proxygencurl
)
install(
TARGETS proxygen_curl
EXPORT proxygen-exports
DESTINATION bin
)
================================================
FILE: proxygen/httpclient/samples/curl/CurlClient.cpp
================================================
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/
#include "CurlClient.h"
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
using namespace folly;
using namespace proxygen;
using namespace std;
DECLARE_int32(recv_window);
namespace CurlService {
CurlClient::CurlClient(EventBase* evb,
HTTPMethod httpMethod,
URL url,
const proxygen::URL* proxy,
const HTTPHeaders& headers,
string inputFilename,
bool h2c,
unsigned short httpMajor,
unsigned short httpMinor)
: evb_(evb),
httpMethod_(httpMethod),
url_(std::move(url)),
inputFilename_(std::move(inputFilename)),
h2c_(h2c),
httpMajor_(httpMajor),
httpMinor_(httpMinor) {
if (proxy != nullptr) {
proxy_ = std::make_unique(proxy->getUrl());
}
outputStream_ = std::make_unique(std::cout.rdbuf());
headers.forEach([this](const string& header, const string& val) {
request_.getHeaders().add(header, val);
});
}
bool CurlClient::saveResponseToFile(const std::string& outputFilename) {
std::streambuf* buf;
if (outputFilename.empty()) {
return false;
}
uint16_t tries = 0;
while (tries < std::numeric_limits::max()) {
std::string suffix = (tries == 0) ? "" : folly::to("_", tries);
auto filename = folly::to(outputFilename, suffix);
struct stat statBuf;
if (stat(filename.c_str(), &statBuf) == -1) {
outputFile_ =
std::make_unique(filename, ios::out | ios::binary);
if (*outputFile_ && outputFile_->good()) {
buf = outputFile_->rdbuf();
outputStream_ = std::make_unique(buf);
return true;
}
}
tries++;
}
return false;
}
HTTPHeaders CurlClient::parseHeaders(const std::string& headersString) {
vector headersList;
HTTPHeaders headers;
folly::split(',', headersString, headersList);
for (const auto& headerPair : headersList) {
vector nv;
folly::split('=', headerPair, nv);
if (nv.size() > 0) {
if (nv[0].empty()) {
continue;
}
std::string value;
for (size_t i = 1; i < nv.size(); i++) {
value += folly::to(nv[i], '=');
}
if (nv.size() > 1) {
value.pop_back();
} // trim anything else
headers.add(nv[0], value);
}
}
return headers;
}
void CurlClient::initializeSsl(const string& caPath,
const string& nextProtos,
const string& certPath,
const string& keyPath) {
sslContext_ = std::make_shared();
sslContext_->setOptions(SSL_OP_NO_COMPRESSION);
folly::ssl::setCipherSuites(*sslContext_);
if (!caPath.empty()) {
sslContext_->loadTrustedCertificates(caPath.c_str());
}
if (!certPath.empty() && !keyPath.empty()) {
sslContext_->loadCertKeyPairFromFiles(certPath.c_str(), keyPath.c_str());
}
list nextProtoList;
folly::splitTo(
',', nextProtos, std::inserter(nextProtoList, nextProtoList.begin()));
sslContext_->setAdvertisedNextProtocols(nextProtoList);
h2c_ = false;
}
void CurlClient::sslHandshakeFollowup(HTTPUpstreamSession* session) noexcept {
auto* sslSocket = dynamic_cast(session->getTransport());
const unsigned char* nextProto = nullptr;
unsigned nextProtoLength = 0;
sslSocket->getSelectedNextProtocol(&nextProto, &nextProtoLength);
if (nextProto) {
VLOG(1) << "Client selected next protocol "
<< string((const char*)nextProto, nextProtoLength);
} else {
VLOG(1) << "Client did not select a next protocol";
}
// Note: This ssl session can be used by defining a member and setting
// something like sslSession_ = sslSocket->getSSLSession() and then
// passing it to the connector::connectSSL() method
}
void CurlClient::setFlowControlSettings(int32_t recvWindow) {
recvWindow_ = recvWindow;
}
void CurlClient::connectSuccess(HTTPUpstreamSession* session) {
if (url_.isSecure()) {
sslHandshakeFollowup(session);
}
session->setFlowControl(recvWindow_, recvWindow_, recvWindow_);
sendRequest(session->newTransaction(this));
session->closeWhenIdle();
}
void CurlClient::setupHeaders() {
request_.setMethod(httpMethod_);
request_.setHTTPVersion(httpMajor_, httpMinor_);
if (proxy_) {
request_.setURL(url_.getUrl());
} else {
request_.setURL(url_.makeRelativeURL());
}
request_.setSecure(url_.isSecure());
if (!request_.getHeaders().getNumberOfValues(HTTP_HEADER_USER_AGENT)) {
request_.getHeaders().add(HTTP_HEADER_USER_AGENT, "proxygen_curl");
}
if (!request_.getHeaders().getNumberOfValues(HTTP_HEADER_HOST)) {
request_.getHeaders().add(HTTP_HEADER_HOST, url_.getHostAndPort());
}
if (!request_.getHeaders().getNumberOfValues(HTTP_HEADER_ACCEPT)) {
request_.getHeaders().add("Accept", "*/*");
}
if (loggingEnabled_) {
request_.dumpMessage(4);
}
}
void CurlClient::sendRequest(HTTPTransaction* txn) {
LOG_IF(INFO, loggingEnabled_)
<< fmt::format("Sending request for {}", url_.getUrl());
txn_ = txn;
setupHeaders();
txnStartTime_ = std::chrono::steady_clock::now();
txn_->sendHeaders(request_);
if (httpMethod_ == HTTPMethod::POST) {
inputFile_ =
std::make_unique(inputFilename_, ios::in | ios::binary);
sendBodyFromFile();
} else {
txn_->sendEOM();
}
}
void CurlClient::sendBodyFromFile() {
const uint16_t kReadSize = 4096;
CHECK(inputFile_);
// Reading from the file by chunks
// Important note: It's pretty bad to call a blocking i/o function like
// ifstream::read() in an eventloop - but for the sake of this simple
// example, we'll do it.
// An alternative would be to put this into some folly::AsyncReader
// object.
while (inputFile_->good() && !egressPaused_) {
unique_ptr buf = IOBuf::createCombined(kReadSize);
inputFile_->read((char*)buf->writableData(), kReadSize);
buf->append(inputFile_->gcount());
txn_->sendBody(std::move(buf));
}
if (!egressPaused_) {
txn_->sendEOM();
}
}
void CurlClient::printMessageImpl(proxygen::HTTPMessage* msg,
const std::string& tag) {
if (!loggingEnabled_) {
return;
}
cout << tag;
msg->dumpMessage(10);
}
void CurlClient::connectError(const folly::AsyncSocketException& ex) {
LOG_IF(ERROR, loggingEnabled_)
<< "Coudln't connect to " << url_.getHostAndPort() << ":" << ex.what();
}
void CurlClient::setTransaction(HTTPTransaction*) noexcept {
}
void CurlClient::detachTransaction() noexcept {
}
void CurlClient::onHeadersComplete(unique_ptr msg) noexcept {
response_ = std::move(msg);
printMessageImpl(response_.get());
if (!headersLoggingEnabled_) {
return;
}
response_->describe(*outputStream_);
*outputStream_ << std::endl;
}
void CurlClient::onBody(std::unique_ptr chain) noexcept {
if (onBodyFunc_ && chain) {
onBodyFunc_.value()(request_, chain.get());
}
if (!loggingEnabled_) {
return;
}
CHECK(outputStream_);
if (chain) {
const IOBuf* p = chain.get();
do {
outputStream_->write((const char*)p->data(), p->length());
outputStream_->flush();
p = p->next();
} while (p != chain.get());
}
}
void CurlClient::onTrailers(std::unique_ptr) noexcept {
LOG_IF(INFO, loggingEnabled_) << "Discarding trailers";
}
void CurlClient::onEOM() noexcept {
LOG_IF(INFO, loggingEnabled_)
<< fmt::format("Got EOM for {}. Txn Time= {} ms",
url_.getUrl(),
std::chrono::duration_cast(
std::chrono::steady_clock::now() - txnStartTime_)
.count());
if (eomFunc_) {
eomFunc_.value()();
}
}
void CurlClient::onUpgrade(UpgradeProtocol) noexcept {
LOG_IF(INFO, loggingEnabled_) << "Discarding upgrade protocol";
}
void CurlClient::onError(const HTTPException& error) noexcept {
LOG_IF(ERROR, loggingEnabled_) << "An error occurred: " << error.what();
}
void CurlClient::onEgressPaused() noexcept {
VLOG_IF(1, loggingEnabled_) << "Egress paused";
egressPaused_ = true;
}
void CurlClient::onEgressResumed() noexcept {
VLOG_IF(1, loggingEnabled_) << "Egress resumed";
egressPaused_ = false;
if (inputFile_) {
sendBodyFromFile();
}
}
void CurlClient::onPushedTransaction(
proxygen::HTTPTransaction* pushedTxn) noexcept {
//
pushTxnHandlers_.emplace_back(std::make_unique(this));
pushedTxn->setHandler(pushTxnHandlers_.back().get());
// Add implementation of the push transaction reception here
}
const string& CurlClient::getServerName() const {
const string& res = request_.getHeaders().getSingleOrEmpty(HTTP_HEADER_HOST);
if (res.empty()) {
return url_.getHost();
}
return res;
}
// CurlPushHandler methods
void CurlClient::CurlPushHandler::setTransaction(
proxygen::HTTPTransaction* txn) noexcept {
LOG_IF(INFO, parent_->loggingEnabled_) << "Received pushed transaction";
pushedTxn_ = txn;
}
void CurlClient::CurlPushHandler::detachTransaction() noexcept {
LOG_IF(INFO, parent_->loggingEnabled_) << "Detached pushed transaction";
}
void CurlClient::CurlPushHandler::onHeadersComplete(
std::unique_ptr msg) noexcept {
if (!seenOnHeadersComplete_) {
seenOnHeadersComplete_ = true;
promise_ = std::move(msg);
parent_->printMessageImpl(promise_.get(), "[PP] ");
} else {
response_ = std::move(msg);
parent_->printMessageImpl(response_.get(), "[PR] ");
}
}
void CurlClient::CurlPushHandler::onBody(
std::unique_ptr chain) noexcept {
parent_->onBody(std::move(chain));
}
void CurlClient::CurlPushHandler::onEOM() noexcept {
LOG_IF(INFO, parent_->loggingEnabled_) << "Got PushTxn EOM";
}
void CurlClient::CurlPushHandler::onError(
const proxygen::HTTPException& error) noexcept {
parent_->onError(error);
}
} // namespace CurlService
================================================
FILE: proxygen/httpclient/samples/curl/CurlClient.h
================================================
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/
#pragma once
#include
#include
#include
#include
#include
#include
namespace CurlService {
class CurlClient
: public proxygen::HTTPConnector::Callback
, public proxygen::HTTPTransactionHandler {
class CurlPushHandler : public proxygen::HTTPTransactionHandler {
public:
explicit CurlPushHandler(CurlClient* parent) : parent_{parent} {
}
void setTransaction(proxygen::HTTPTransaction* /*txn*/) noexcept override;
void detachTransaction() noexcept override;
void onHeadersComplete(
std::unique_ptr /*msg*/) noexcept override;
void onBody(std::unique_ptr /*chain*/) noexcept override;
void onEOM() noexcept override;
void onError(const proxygen::HTTPException& /*error*/) noexcept override;
void onTrailers(
std::unique_ptr /*trailers*/) noexcept override {
}
void onUpgrade(proxygen::UpgradeProtocol /*protocol*/) noexcept override {
}
void onEgressResumed() noexcept override {
}
void onEgressPaused() noexcept override {
}
private:
// hack around the ambiguous API
bool seenOnHeadersComplete_{false};
// the pushed transaction
proxygen::HTTPTransaction* pushedTxn_{nullptr};
// information about the request
std::unique_ptr promise_;
std::unique_ptr response_;
CurlClient* parent_;
};
public:
CurlClient(folly::EventBase* evb,
proxygen::HTTPMethod httpMethod,
proxygen::URL url,
const proxygen::URL* proxy,
const proxygen::HTTPHeaders& headers,
std::string inputFilename,
bool h2c = false,
unsigned short httpMajor = 1,
unsigned short httpMinor = 1);
~CurlClient() override = default;
bool saveResponseToFile(const std::string& outputFilename);
static proxygen::HTTPHeaders parseHeaders(const std::string& headersString);
// initial SSL related structures
void initializeSsl(const std::string& caPath,
const std::string& nextProtos,
const std::string& certPath = "",
const std::string& keyPath = "");
void sslHandshakeFollowup(proxygen::HTTPUpstreamSession* session) noexcept;
// HTTPConnector methods
void connectSuccess(proxygen::HTTPUpstreamSession* session) override;
void connectError(const folly::AsyncSocketException& ex) override;
// HTTPTransactionHandler methods
void setTransaction(proxygen::HTTPTransaction* txn) noexcept override;
void detachTransaction() noexcept override;
void onHeadersComplete(
std::unique_ptr msg) noexcept override;
void onBody(std::unique_ptr chain) noexcept override;
void onTrailers(
std::unique_ptr trailers) noexcept override;
void onEOM() noexcept override;
void onUpgrade(proxygen::UpgradeProtocol protocol) noexcept override;
void onError(const proxygen::HTTPException& error) noexcept override;
void onEgressPaused() noexcept override;
void onEgressResumed() noexcept override;
void onPushedTransaction(
proxygen::HTTPTransaction* /* pushedTxn */) noexcept override;
void sendRequest(proxygen::HTTPTransaction* txn);
// Getters
folly::SSLContextPtr getSSLContext() {
return sslContext_;
}
const std::string& getServerName() const;
void setFlowControlSettings(int32_t recvWindow);
const proxygen::HTTPMessage* getResponse() const {
return response_.get();
}
void setLogging(bool enabled) {
loggingEnabled_ = enabled;
}
void setHeadersLogging(bool enabled) {
headersLoggingEnabled_ = enabled;
}
void setEOMFunc(std::function eomFunc) {
eomFunc_ = eomFunc;
}
void setOnBodyFunc(
const std::function& onBodyFunc) {
onBodyFunc_ = onBodyFunc;
}
protected:
void sendBodyFromFile();
void setupHeaders();
void printMessageImpl(proxygen::HTTPMessage* msg,
const std::string& tag = "");
proxygen::HTTPTransaction* txn_{nullptr};
folly::EventBase* evb_{nullptr};
proxygen::HTTPMethod httpMethod_;
proxygen::URL url_;
std::unique_ptr proxy_;
proxygen::HTTPMessage request_;
const std::string inputFilename_;
folly::SSLContextPtr sslContext_;
int32_t recvWindow_{0};
bool loggingEnabled_{true};
bool headersLoggingEnabled_{false};
bool h2c_{false};
unsigned short httpMajor_;
unsigned short httpMinor_;
bool egressPaused_{false};
std::unique_ptr inputFile_;
std::unique_ptr outputFile_;
std::unique_ptr outputStream_;
std::unique_ptr response_;
std::vector> pushTxnHandlers_;
std::chrono::time_point txnStartTime_;
folly::Optional> eomFunc_;
folly::Optional>
onBodyFunc_;
friend class CurlPushHandler;
};
} // namespace CurlService
================================================
FILE: proxygen/httpclient/samples/curl/CurlClientMain.cpp
================================================
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/
#include
#include
#include
#include
#include
#include
#include
#include
using namespace CurlService;
using namespace folly;
using namespace proxygen;
DEFINE_string(http_method,
"GET",
"HTTP method to use. GET or POST are supported");
DEFINE_string(url,
"https://github.com/facebook/proxygen",
"URL to perform the HTTP method against");
DEFINE_string(input_filename, "", "Filename to read from for POST requests");
DEFINE_int32(http_client_connect_timeout,
1000,
"connect timeout in milliseconds");
DEFINE_int32(http_client_request_timeout,
5000,
"request timeout in milliseconds");
DEFINE_string(ca_path,
"/etc/ssl/certs/ca-certificates.crt",
"Path to trusted CA file"); // default for Ubuntu 14.04
DEFINE_string(cert_path, "", "Path to client certificate file");
DEFINE_string(key_path, "", "Path to client private key file");
DEFINE_string(next_protos,
"h2,h2-14,spdy/3.1,spdy/3,http/1.1",
"Next protocol string for NPN/ALPN");
DEFINE_string(plaintext_proto, "", "plaintext protocol");
DEFINE_int32(recv_window, 65536, "Flow control receive window for h2/spdy");
DEFINE_bool(h2c, true, "Attempt HTTP/1.1 -> HTTP/2 upgrade");
DEFINE_string(headers, "", "List of N=V headers separated by ,");
DEFINE_string(proxy, "", "HTTP proxy URL");
DEFINE_bool(log_response,
true,
"Whether to log the response content to stderr");
int main(int argc, char* argv[]) {
#if FOLLY_HAVE_LIBGFLAGS
// Enable glog logging to stderr by default.
folly::gflags::SetCommandLineOptionWithMode(
"logtostderr", "1", folly::gflags::SET_FLAGS_DEFAULT);
#endif
auto _ = folly::Init(&argc, &argv, false);
EventBase evb;
URL url(FLAGS_url);
URL proxy(FLAGS_proxy);
auto httpMethod = stringToMethod(FLAGS_http_method);
if (!httpMethod) {
LOG(ERROR) << "Unsupported http_method=" << FLAGS_http_method;
return EXIT_FAILURE;
}
if (*httpMethod == HTTPMethod::POST) {
try {
File f(FLAGS_input_filename);
(void)f;
} catch (const std::system_error& se) {
LOG(ERROR) << "Couldn't open file for POST method";
LOG(ERROR) << se.what();
return EXIT_FAILURE;
}
}
HTTPHeaders headers = CurlClient::parseHeaders(FLAGS_headers);
CurlClient curlClient(&evb,
*httpMethod,
url,
FLAGS_proxy.empty() ? nullptr : &proxy,
headers,
FLAGS_input_filename,
FLAGS_h2c);
curlClient.setFlowControlSettings(FLAGS_recv_window);
curlClient.setLogging(FLAGS_log_response);
curlClient.setHeadersLogging(FLAGS_log_response);
SocketAddress addr;
if (!FLAGS_proxy.empty()) {
addr = SocketAddress(proxy.getHost(), proxy.getPort(), true);
} else {
addr = SocketAddress(url.getHost(), url.getPort(), true);
}
LOG(INFO) << "Trying to connect to " << addr;
HTTPConnector connector(
&curlClient,
WheelTimerInstance(
std::chrono::milliseconds(FLAGS_http_client_request_timeout), &evb));
if (!FLAGS_plaintext_proto.empty()) {
connector.setPlaintextProtocol(FLAGS_plaintext_proto);
}
static const SocketOptionMap opts{
{{.level = SOL_SOCKET, .optname = SO_REUSEADDR}, 1}};
if (url.isSecure()) {
curlClient.initializeSsl(
FLAGS_ca_path, FLAGS_next_protos, FLAGS_cert_path, FLAGS_key_path);
connector.connectSSL(
&evb,
addr,
curlClient.getSSLContext(),
nullptr,
std::chrono::milliseconds(FLAGS_http_client_connect_timeout),
opts,
folly::AsyncSocket::anyAddress(),
curlClient.getServerName());
} else {
connector.connect(
&evb,
addr,
std::chrono::milliseconds(FLAGS_http_client_connect_timeout),
opts);
}
evb.loop();
return EXIT_SUCCESS;
}
================================================
FILE: proxygen/httpclient/samples/httperf2/CMakeLists.txt
================================================
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
# Manually maintained (generate_cmake.py skips this directory)
proxygen_add_library(libhttperf2
EXCLUDE_FROM_MONOLITH
SRCS
HTTPerf2.cpp
Client.cpp
EXPORTED_DEPS
proxygenhqloggerhelper
proxygen
)
add_executable(proxygen_httperf2 Main.cpp)
target_link_libraries(
proxygen_httperf2 PUBLIC
libhttperf2
)
target_compile_options(
proxygen_httperf2 PRIVATE
${_PROXYGEN_COMMON_COMPILE_OPTIONS}
)
install(
TARGETS proxygen_httperf2
EXPORT proxygen-exports
DESTINATION bin
)
================================================
FILE: proxygen/httpclient/samples/httperf2/Client.cpp
================================================
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/
#include
#include
#include
#include
#include
#include
#include
#include
#include
using namespace folly;
using namespace proxygen;
using namespace std;
DEFINE_int32(max_outstanding_transactions,
6,
"Maximum allowed in-flight transactions per HTTP session");
DEFINE_int32(req_per_loop,
100,
"Maximum number of requests to send per event loop");
DEFINE_int32(connect_timeout, 120, "Connect timeout in ms");
DEFINE_int32(conn_flow_control,
proxygen::http2::kInitialWindow,
"Connection flow control");
DEFINE_int32(stream_flow_control,
proxygen::http2::kInitialWindow,
"Stream flow control");
DEFINE_string(congestion,
"cubic",
"Congestion control algorithm, cubic/newreno/bbr/none");
DEFINE_int32(max_receive_packet_size,
quic::kDefaultUDPReadBufferSize,
"Max UDP packet size Quic can receive");
DEFINE_bool(
can_ignore_pmtu,
false,
"We can ignore PTMU for the transport and use advertised recv buffer");
DEFINE_bool(udp_connect, true, "Whether or not to connect the udp socket");
DEFINE_bool(pacing, false, "Whether to enable pacing in Quic");
DEFINE_uint32(quic_batching_mode,
static_cast(quic::QuicBatchingMode::BATCHING_MODE_GSO),
"QUIC batching mode");
DEFINE_uint32(quic_batch_size,
quic::kDefaultQuicMaxBatchSize,
"Maximum number of packets that can be batchedi in Quic");
DEFINE_bool(early_data, false, "Enable Fizz early data");
DEFINE_uint32(quic_recv_batch_size,
16,
"Number of packets to receiver per loop.");
static std::atomic_bool exitAllClientsSoon{false};
void Client::exitAllSoon() {
exitAllClientsSoon = true;
}
Client::Client(EventBase* eventBase,
const WheelTimerInstance& transactionTimeouts,
HTTPerfStats& stats,
Optional bindAddr,
const SocketAddress& address,
HTTPMessage& request,
const std::string& requestData,
uint32_t n_requests,
FinishedCallback* callback,
const string& plaintextProto,
const string& serverName)
: eventBase_(eventBase),
stats_(stats),
requestsSent_(0),
bindAddr_(std::move(bindAddr)),
address_(address),
sslSession_(nullptr),
request_(request),
requestData_(requestData),
requests_(n_requests),
callback_(callback),
connector_(this, transactionTimeouts),
serverName_(serverName),
plaintextProto_(plaintextProto) {
CHECK_GT(requests_, 0);
connector_.setPlaintextProtocol(plaintextProto);
}
Client::~Client() {
if (isLoopCallbackScheduled()) {
cancelLoopCallback();
}
inDestructor_ = true;
collector_.stopCallbacks();
if (session_) {
// drain() might cause synchronous destruction, protect ourselves
// from it.
DelayedDestructionBase::DestructorGuard guard(session_);
VLOG(4) << "shutting down session";
session_->dropConnection("shutting down");
// The above will destroy session_ and kill all the transactions
// But we don't need any callbacks because we're toast
session_->setInfoCallback(nullptr);
session_ = nullptr;
}
}
void Client::start() {
connect();
}
void Client::exit() {
if (callback_ != nullptr) {
FinishedCallback* callback = callback_;
callback_ = nullptr;
callback->clientFinished(this);
}
}
void Client::setSSLParameters(const folly::SSLContextPtr& sslContext,
std::shared_ptr session) {
sslContext_ = sslContext;
sslSession_ = session;
if (sslSession_ != nullptr) {
shouldReuseSession_ = true;
}
}
void Client::setupFizzContext(std::shared_ptr pskCache,
bool pskKe,
const std::string& cert,
const std::string& key) {
fizzContext_ = std::make_shared();
fizzContext_->setSupportedAlpns({plaintextProto_});
if (FLAGS_early_data) {
fizzContext_->setSendEarlyData(true);
}
if (pskKe) {
fizzContext_->setSupportedPskModes({fizz::PskKeyExchangeMode::psk_ke});
}
fizzContext_->setPskCache(std::move(pskCache));
if (!cert.empty()) {
std::string certData;
std::string keyData;
folly::readFile(cert.c_str(), certData);
if (!key.empty()) {
folly::readFile(key.c_str(), keyData);
}
auto selfCert = fizz::openssl::CertUtils::makeSelfCert(certData, keyData);
auto certMgr = std::make_shared();
certMgr->addCert(std::move(selfCert));
fizzContext_->setClientCertManager(std::move(certMgr));
}
}
void Client::setQuicPskCache(std::shared_ptr quicPskCache) {
quicPskCache_ = std::move(quicPskCache);
}
void Client::setUseQuic(bool useQuic) {
useQuic_ = useQuic;
}
bool Client::supportsTickets() const {
return (sslContext_ &&
!(SSL_CTX_get_options(sslContext_->getSSLCtx()) & SSL_OP_NO_TICKET));
}
std::shared_ptr Client::extractSSLSession() {
auto session = std::move(sslSession_);
sslSession_ = nullptr;
return session;
}
static quic::TransportSettings createTransportSettings() {
quic::TransportSettings transportSettings;
transportSettings.attemptEarlyData = FLAGS_early_data;
transportSettings.pacingEnabled = FLAGS_pacing;
if (folly::to(FLAGS_max_receive_packet_size) <
quic::kDefaultUDPSendPacketLen) {
throw std::invalid_argument(
folly::to("max_receive_packet_size needs to be at least ",
quic::kDefaultUDPSendPacketLen));
}
transportSettings.maxRecvPacketSize = FLAGS_max_receive_packet_size;
transportSettings.canIgnorePathMTU = FLAGS_can_ignore_pmtu;
if (FLAGS_conn_flow_control != 0) {
transportSettings.advertisedInitialConnectionFlowControlWindow =
FLAGS_conn_flow_control;
}
// TODO FLAGS_stream_*
if (FLAGS_stream_flow_control != 0) {
transportSettings.advertisedInitialBidiLocalStreamFlowControlWindow =
FLAGS_stream_flow_control;
transportSettings.advertisedInitialBidiRemoteStreamFlowControlWindow =
FLAGS_stream_flow_control;
transportSettings.advertisedInitialUniStreamFlowControlWindow =
FLAGS_stream_flow_control;
}
auto ccType = quic::congestionControlStrToType(FLAGS_congestion);
if (!ccType) {
throw std::invalid_argument(folly::to(
"invalid congestion control ", FLAGS_congestion));
}
transportSettings.defaultCongestionController = *ccType;
transportSettings.batchingMode =
quic::getQuicBatchingMode(FLAGS_quic_batching_mode);
transportSettings.maxBatchSize = FLAGS_quic_batch_size;
transportSettings.shouldUseRecvmmsgForBatchRecv = true;
transportSettings.maxRecvBatchSize = FLAGS_quic_recv_batch_size;
transportSettings.connectUDP = FLAGS_udp_connect;
transportSettings.shouldDrain = false;
return transportSettings;
}
///////// Connection Setup
void Client::connect() {
connector_.reset();
start_ = std::chrono::steady_clock::now();
static const SocketOptionMap opts{
{{.level = SOL_SOCKET, .optname = SO_REUSEADDR}, 1}};
if (sslContext_) {
connector_.connectSSL(eventBase_,
address_,
sslContext_,
shouldReuseSession_ ? sslSession_ : nullptr,
std::chrono::milliseconds(FLAGS_connect_timeout),
opts);
} else if (fizzContext_ && !useQuic_) {
connector_.connectFizz(
eventBase_,
address_,
fizzContext_,
std::make_shared<
proxygen::InsecureVerifierDangerousDoNotUseInProduction>(),
std::chrono::milliseconds(FLAGS_connect_timeout),
std::chrono::milliseconds(0),
opts,
folly::AsyncSocket::anyAddress(),
serverName_);
} else if (fizzContext_ && useQuic_) {
hqConnector_ = std::make_unique(
this, std::chrono::milliseconds(FLAGS_connect_timeout));
hqConnector_->setTransportSettings(createTransportSettings());
hqConnector_->setQuicPskCache(quicPskCache_);
hqConnector_->connect(
eventBase_,
bindAddr_,
address_,
fizzContext_,
std::make_shared<
proxygen::InsecureVerifierDangerousDoNotUseInProduction>(),
std::chrono::milliseconds(FLAGS_connect_timeout),
folly::emptySocketOptionMap,
serverName_,
qlogger_);
} else {
connector_.connect(eventBase_,
address_,
std::chrono::milliseconds(FLAGS_connect_timeout),
opts);
}
}
///////// Request logic
bool Client::shouldExit() const {
return exitAllClientsSoon ||
(requestsSent_ >= requests_ && outstandingTransactions_ == 0);
}
void Client::runLoopCallback() noexcept {
if (session_) {
sendRequest();
} else if (shouldExit()) {
exit();
} else {
connect();
}
}
void Client::sendRequest() {
if (shouldExit()) {
// TODO: should track responses + errors before calling finished
exit();
return;
}
if (isConnecting()) {
// We are in the middle of the connect process. The request will
// happen once the connect succeeds.
return;
}
CHECK(session_);
uint32_t requestsThisLoop = 0;
while ((outstandingTransactions_ <
uint32_t(FLAGS_max_outstanding_transactions)) &&
(requestsSent_ < requests_) &&
(requestsThisLoop++ < uint32_t(FLAGS_req_per_loop))) {
auto handler = new TransactionHandler(this);
auto txn = session_->newTransaction(handler);
if (!txn) {
// The session doesn't support any more transactions
delete handler;
break;
}
outstandingTransactions_++;
requestsSent_++;
stats_.addRequest();
if (requestsSent_ == requests_ && useQuic_) {
// We need to drain before we send the last request
// in quic due to the way h1q works.
session_->drain();
}
txn->sendHeaders(request_);
if (!requestData_.empty()) {
auto data =
folly::IOBuf::wrapBuffer(requestData_.data(), requestData_.length());
txn->sendBody(std::move(data));
}
txn->sendEOM();
}
if ((outstandingTransactions_ <
uint32_t(FLAGS_max_outstanding_transactions)) &&
(requestsSent_ < requests_)) {
eventBase_->runInLoop(this);
}
}
void Client::connectSuccess(proxygen::HQUpstreamSession* session) {
CHECK(!session_);
const auto transport = session->getQuicSocket();
auto client =
CHECK_NOTNULL(dynamic_cast(transport));
if (client->isTLSResumed()) {
stats_.addResume();
} else {
stats_.addHandshake();
}
connectSuccessCommon(session);
}
void Client::connectError(const quic::QuicErrorCode&) {
connectErrorCommon();
}
void Client::connectSuccess(HTTPUpstreamSession* session) {
CHECK(!session_);
session->setByteEventTracker(nullptr);
if (sslContext_) {
auto sslSocket = dynamic_cast(session->getTransport());
const unsigned char* nextProto = nullptr;
unsigned nextProtoLength = 0;
sslSocket->getSelectedNextProtocol(&nextProto, &nextProtoLength);
if (nextProto) {
VLOG(4) << "Client selected next protocol "
<< string((const char*)nextProto, nextProtoLength);
} else {
VLOG(4) << "Client did not select a next protocol";
}
if (sslSocket->getSSLSessionReused()) {
stats_.addResume();
} else {
sslSession_ = sslSocket->getSSLSession();
stats_.addHandshake();
}
}
auto asyncSock =
session->getTransport()->getUnderlyingTransport();
if (asyncSock) {
if (asyncSock->getNetworkSocket().toFd() >= 0) {
// Set linger timeout to 0 to avoid sockets in time wait state.
struct linger optLinger = {.l_onoff = 1, .l_linger = 0};
asyncSock->setSockOpt(SOL_SOCKET, SO_LINGER, &optLinger);
}
}
connectSuccessCommon(session);
}
void Client::connectError(const AsyncSocketException& /*ex*/) {
connectErrorCommon();
}
void Client::connectSuccessCommon(HTTPSessionBase* session) {
CHECK(!session_);
session_ = session;
session_->setInfoCallback(&collector_);
session->setFlowControl(FLAGS_stream_flow_control,
FLAGS_stream_flow_control,
FLAGS_conn_flow_control);
// Don't artificially limit streams
session->setMaxConcurrentOutgoingStreams(
std::numeric_limits::max());
end_ = std::chrono::steady_clock::now();
stats_.addConnection(millisecondsBetween(end_, start_).count());
sendRequest();
}
void Client::connectErrorCommon() {
stats_.addConnectError();
requestsSent_++;
if (shouldExit()) {
exit();
} else {
eventBase_->runInLoop(this);
}
}
// HTTPTransaction::Handler callbacks
void Client::TransactionHandler::detachTransaction() noexcept {
DCHECK(!waitingForResponse_);
DCHECK(!inMessage_);
if (!parent_->inDestructor_) {
DCHECK_GT(parent_->outstandingTransactions_, 0);
parent_->outstandingTransactions_--;
VLOG(3) << __func__ << " requestsSent=" << parent_->requestsSent_
<< " requests=" << parent_->requests_
<< " outstanding=" << parent_->outstandingTransactions_;
if (!parent_->isLoopCallbackScheduled()) {
parent_->Client::eventBase_->runInLoop(parent_);
}
}
delete this;
}
void Client::TransactionHandler::onHeadersComplete(
unique_ptr