[ { "path": ".github/workflows/CompatHelper.yml", "content": "name: CompatHelper\n\non:\n schedule:\n - cron: '00 * * * *'\n issues:\n types: [opened, reopened]\n\njobs:\n build:\n runs-on: ${{ matrix.os }}\n strategy:\n matrix:\n julia-version: [1.5]\n julia-arch: [x86]\n os: [ubuntu-latest]\n steps:\n - uses: julia-actions/setup-julia@latest\n with:\n version: ${{ matrix.julia-version }}\n - name: Pkg.add(\"CompatHelper\")\n run: julia -e 'using Pkg; Pkg.add(\"CompatHelper\")'\n - name: CompatHelper.main()\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n run: julia -e 'using CompatHelper; CompatHelper.main()'\n" }, { "path": ".github/workflows/TagBot.yml", "content": "name: TagBot\non:\n issue_comment:\n types:\n - created\n workflow_dispatch:\njobs:\n TagBot:\n if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'\n runs-on: ubuntu-latest\n steps:\n - uses: JuliaRegistries/TagBot@v1\n with:\n token: ${{ secrets.GITHUB_TOKEN }}\n ssh: ${{ secrets.DOCUMENTER_KEY }}\n" }, { "path": ".github/workflows/ci.yml", "content": "name: CI\non:\n - push\n - pull_request\njobs:\n test:\n name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}\n runs-on: ${{ matrix.os }}\n strategy:\n fail-fast: false\n matrix:\n version:\n - '1.5'\n - 'nightly'\n os:\n - ubuntu-latest\n - macOS-latest\n - windows-latest\n arch:\n - x64\n steps:\n - uses: actions/checkout@v2\n - uses: julia-actions/setup-julia@v1\n with:\n version: ${{ matrix.version }}\n arch: ${{ matrix.arch }}\n - uses: actions/cache@v1\n env:\n cache-name: cache-artifacts\n with:\n path: ~/.julia/artifacts\n key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}\n restore-keys: |\n ${{ runner.os }}-test-${{ env.cache-name }}-\n ${{ runner.os }}-test-\n ${{ runner.os }}-\n - uses: julia-actions/julia-buildpkg@v1\n - uses: julia-actions/julia-runtest@v1\n - uses: julia-actions/julia-processcoverage@v1\n - uses: codecov/codecov-action@v1\n with:\n file: lcov.info\n docs:\n name: Documentation\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v2\n - uses: julia-actions/setup-julia@v1\n with:\n version: '1'\n - run: |\n julia --project=docs -e '\n using Pkg\n Pkg.develop(PackageSpec(path=pwd()))\n Pkg.instantiate()'\n - run: |\n julia --project=docs -e '\n using Documenter: doctest\n using NiLang\n doctest(NiLang)'\n - run: julia --project=docs docs/make.jl\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }}\n" }, { "path": ".gitignore", "content": "*.jl.*.cov\n*.jl.cov\n*.jl.mem\n.DS_Store\nManifest.toml\n/dev/\n/docs/build/\n/docs/site/\n/docs/src/examples/\n_local/\n*.swp\n.vscode/" }, { "path": "LICENSE", "content": "Copyright (c) 2019 JinGuo Liu, thautwarm\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [year] [fullname]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "Makefile", "content": "JL = julia --project\n\ndefault: init test\n\ninit:\n\t$(JL) -e 'using Pkg; Pkg.precompile()'\ninit-docs:\n\t$(JL) -e 'using Pkg; Pkg.activate(\"docs\"); Pkg.develop(path=\".\"), Pkg.precompile()'\n\nupdate:\n\t$(JL) -e 'using Pkg; Pkg.update(); Pkg.precompile()'\nupdate-docs:\n\t$(JL) -e 'using Pkg; Pkg.activate(\"docs\"); Pkg.update(); Pkg.precompile()'\n\ntest:\n\t$(JL) -e 'using Pkg; Pkg.test(\"GenericTensorNetworks\")'\n\ncoverage:\n\t$(JL) -e 'using Pkg; Pkg.test(\"GenericTensorNetworks\"; coverage=true)'\n\nserve:\n\t$(JL) -e 'using Pkg; Pkg.activate(\"docs\"); using LiveServer; servedocs(;skip_dirs=[\"docs/src/assets\", \"docs/src/generated\"], literate_dir=\"examples\")'\n\nclean:\n\trm -rf docs/build\n\tfind . -name \"*.cov\" -type f -print0 | xargs -0 /bin/rm -f\n\n.PHONY: init test coverage serve clean init-docs update update-docs" }, { "path": "Project.toml", "content": "name = \"NiLang\"\nuuid = \"ab4ef3a6-0b42-11ea-31f6-e34652774712\"\nauthors = [\"JinGuo Liu\", \"thautwarm\"]\nversion = \"0.9.4\"\n\n[deps]\nFixedPointNumbers = \"53c48c17-4a7d-5ca2-90c5-79b7896eea93\"\nLinearAlgebra = \"37e2e46d-f89d-539d-b4ee-838fcccc9c8e\"\nLogarithmicNumbers = \"aa2f6b4e-9042-5d33-9679-40d3a6b85899\"\nMLStyle = \"d8e11817-5142-5d16-987a-aa16d5891078\"\nNiLangCore = \"575d3204-02a4-11ea-3f62-238caa8bf11e\"\nReexport = \"189a3867-3050-52da-a836-e630ba90ab69\"\nSparseArrays = \"2f01184e-e22b-5df5-ae63-d93ebab69eaf\"\nTupleTools = \"9d95972d-f1c8-5527-a6e0-b4b365fa01f6\"\n\n[compat]\nFixedPointNumbers = \"0.6, 0.7, 0.8\"\nLogarithmicNumbers = \"0.4, 1.0\"\nMLStyle = \"0.4\"\nNiLangCore = \"0.10.1\"\nReexport = \"0.2, 1.0\"\nTupleTools = \"1.2\"\njulia = \"1.3\"\n\n[extras]\nDistributions = \"31c24e10-a181-5473-b8eb-7969acd0382f\"\nRandom = \"9a3f8284-a2c9-5f02-9a11-845980a1fd5c\"\nStatistics = \"10745b16-79ce-11e8-11f9-7d13ad32a3b2\"\nFiniteDifferences = \"26cc04aa-876d-5657-8c51-4c34ba976000\"\nTest = \"8dfed614-e22c-5e08-85e1-65c5234f0b40\"\n\n[targets]\ntest = [\"Test\", \"Random\", \"Statistics\", \"Distributions\", \"FiniteDifferences\"]\n" }, { "path": "README.md", "content": "\n\nNiLang.jl (逆lang), is a reversible domain-specific language (DSL) that allow a program to go back to the past.\n\n* Requires Julia version >= 1.3,\n\nNiLang features:\n\n* any program written in NiLang is differentiable,\n* a reversible language with abstraction and arrays,\n* complex values\n* reversible logarithmic number system\n\n![CI](https://github.com/GiggleLiu/NiLang.jl/workflows/CI/badge.svg)\n[![codecov](https://codecov.io/gh/GiggleLiu/NiLang.jl/branch/master/graph/badge.svg?token=th86D4USSX)](https://codecov.io/gh/GiggleLiu/NiLang.jl)\n\nThe main docs can be found here:\n[![](https://img.shields.io/badge/docs-stable-blue.svg)](https://giggleliu.github.io/NiLang.jl/stable/)\n[![](https://img.shields.io/badge/docs-dev-blue.svg)](https://giggleliu.github.io/NiLang.jl/dev/)\n\nThere are also some Pluto-based notebooks:\n* [tutorial](https://giggleliu.github.io/NiLang.jl/dev/notebooks/basic.html)\n* [documentation](https://giggleliu.github.io/NiLang.jl/dev/notebooks/documentation.html)\n* [Billiard ball model cellular automata](https://giggleliu.github.io/NiLang.jl/dev/notebooks/margolus.html)\n\n> The strangeness of reversible computing is mainly due to\n> our lack of experience with it.—Henry Baker, 1992\n\n## To Start\n```\npkg> add NiLang\n```\n\n## An example: Compute the norm of a vector\n```julia\njulia> using NiLang\n\njulia> @i function f(res, y, x)\n for i=1:length(x)\n y += x[i] ^ 2\n end\n res += sqrt(y)\n end\n\njulia> res_out, y_out, x_out = f(0.0, 0.0, [1, 2, 3.0])\n(3.7416573867739413, 14.0, [1.0, 2.0, 3.0])\n\njulia> (~f)(res_out, y_out, x_out) # automatically generated inverse program.\n(0.0, 0.0, [1.0, 2.0, 3.0])\n \njulia> ∂res, ∂y, ∂x = NiLang.AD.gradient(Val(1), f, (0.0, 0.0, [1, 2, 3.0])) \n # automatic differentiation, `Val(1)` means the first argument of `f` is the loss.\n(1.0, 0.1336306209562122, [0.2672612419124244, 0.5345224838248488, 0.8017837257372732])\n```\n\nThe performance of reversible programming automatic differentiation is much better than most traditional frameworks. Here is why, and how it works,\n\n![how it works](docs/src/asset/adprog.png)\n\n## Check our [paper](https://arxiv.org/abs/2003.04617)\n\n```bibtex\n@misc{Liu2020,\n title={Differentiate Everything with a Reversible Programming Language},\n author={Jin-Guo Liu and Taine Zhao},\n year={2020},\n eprint={2003.04617},\n archivePrefix={arXiv},\n primaryClass={cs.PL}\n}\n```\n" }, { "path": "benchmark/besselj_gpu.jl", "content": "using NiLang, NiLang.AD\nusing CuArrays, CUDAnative, GPUArrays\nusing BenchmarkTools\n\n@i @inline function :(-=)(CUDAnative.pow)(out!::GVar{T}, x::GVar{T}, n::GVar) where T\n value(out!) -= CUDAnative.pow(value(x), value(n))\n\n # grad x\n @routine @invcheckoff begin\n @zeros T anc1 anc2 anc3 jac1 jac2\n\n DEC(value(n))\n anc1 += CUDAnative.pow(value(x), value(n))\n INC(value(n))\n jac1 += anc1 * value(n)\n\n # get grad of n\n anc2 += log(value(x))\n anc3 += CUDAnative.pow(value(x), value(n))\n jac2 += anc3*anc2\n end\n grad(x) += grad(out!) * jac1\n grad(n) += grad(out!) * jac2\n ~@routine\nend\n\n@i @inline function :(-=)(CUDAnative.pow)(out!::GVar{T}, x::GVar, n) where T\n value(out!) -= CUDAnative.pow(value(x), n)\n @routine @invcheckoff begin\n anc1 ← zero(value(x))\n jac ← zero(value(x))\n\n DEC(value(n))\n anc1 += CUDAnative.pow(value(x), n)\n INC(value(n))\n jac += anc1 * n\n end\n grad(x) += grad(out!) * jac\n ~@routine\nend\n\n@i @inline function :(-=)(CUDAnative.pow)(out!::GVar{T}, x, n::GVar) where T\n value(out!) -= CUDAnative.pow(x, value(n))\n # get jac of n\n @routine @invcheckoff begin\n anc1 ← zero(x)\n anc2 ← zero(x)\n jac ← zero(x)\n\n anc1 += log(x)\n anc2 += CUDAnative.pow(x, value(n))\n jac += anc1*anc2\n end\n grad(n) += grad(out!) * jac\n ~@routine\nend\n\n\n# You need to replace all \"^\" operations in `ibessel` with `CUDAnative.pow`.\n# Please remember to turn invertiblity check off, because error handling is not supported in a cuda thread.\n# Function `i_dirtymul` and `i_factorial` are not changed.\n\n@i function ibesselj(out!, ν, z; atol=1e-8)\n @routine @invcheckoff begin\n k ← 0\n fact_nu ← zero(ν)\n halfz ← zero(z)\n halfz_power_nu ← zero(z)\n halfz_power_2 ← zero(z)\n out_anc ← zero(z)\n anc1 ← zero(z)\n anc2 ← zero(z)\n anc3 ← zero(z)\n anc4 ← zero(z)\n anc5 ← zero(z)\n\n halfz += z / 2\n halfz_power_nu += CUDAnative.pow(halfz, ν)\n halfz_power_2 += CUDAnative.pow(halfz, 2)\n i_factorial(fact_nu, ν)\n\n anc1 += halfz_power_nu/fact_nu\n out_anc += anc1\n @from k==0 while abs(unwrap(anc1)) > atol && abs(unwrap(anc4)) < atol\n INC(k)\n @routine begin\n anc5 += k\n anc5 += ν\n anc2 -= k * anc5\n anc3 += halfz_power_2 / anc2\n end\n i_dirtymul(anc1, anc3, anc4)\n out_anc += anc1\n ~@routine\n end\n end\n out! += out_anc\n ~@routine\nend\n\n# Define your reversible kernel function that calls the reversible bessel function\n\n@i function ibesselj_kernel(out!, ν, z, atol)\n i ← (blockIdx().x-1) * blockDim().x + threadIdx().x\n @inbounds ibesselj(out![i], ν, z[i]; atol=atol)\n @invcheckoff i → (blockIdx().x-1) * blockDim().x + threadIdx().x\nend\n\n# To launch this reversible kernel, you also need a reversible host function.\n\n@i function ibesselj(out!::CuVector, ν, z::CuVector; atol=1e-8)\n XY ← GPUArrays.thread_blocks_heuristic(length(out!))\n @cuda threads=XY.:1 blocks=XY.:2 ibesselj_kernel(out!, ν, z, atol)\n @invcheckoff XY → GPUArrays.thread_blocks_heuristic(length(out!))\nend\n\n# To test this function, we first define input parameters `a` and output `out!`\nN = 4096\nT = Float64\na = CuArray(ones(T, N))\nout! = CuArray(zeros(T, N))\n\n# We wrap the output with a randomly initialized gradient field, suppose we get the gradients from a virtual loss function.\n# Also, we need to initialize an empty gradient field for elements in input cuda tensor `a`.\nout! = ibesselj(out!, 2, GVar.(a))[1]\nout_g! = GVar.(out!, CuArray(ones(T, N)))\na_g = GVar.(a)\n\n# Call the inverse program, the multiple dispatch will drive you to the goal.\nprintln(\"Benchmarking NiLang on CUDA, N = $N, T = $T\")\ndisplay(@benchmark CuArrays.@sync (~ibesselj)($out_g!, 2, $a_g))\n" }, { "path": "benchmark/besselj_irreversible.jl", "content": "using Zygote\nusing ForwardDiff\nusing BenchmarkTools\n\nfunction besselj(ν, z; atol=1e-8)\n k = 0\n s = (z/2)^ν / factorial(ν)\n out = s\n while abs(s) > atol\n k += 1\n s *= (-1) / k / (k+ν) * (z/2)^2\n out += s\n end\n out\nend\n\nfunction grad_besselj_manual(ν, z; atol=1e-8)\n (besselj(ν-1, z; atol=atol) - besselj(ν+1, z); atol=atol)/2\nend\n\nprintln(\"Benchmarking Julia\")\ndisplay(@benchmark besselj(2, 1.0))\nprintln(\"Benchmarking Manual\")\ndisplay(@benchmark grad_besselj_manual(2, 1.0))\nprintln(\"Benchmarking Zygote\")\ndisplay(@benchmark Zygote.gradient(besselj, 2, 1.0))\nprintln(\"Benchmarking ForwardDiff\")\ndisplay(@benchmark ForwardDiff.derivative(x->besselj(2, x), 1.0))\n" }, { "path": "benchmark/besselj_reversible.jl", "content": "using NiLang, NiLang.AD\nusing BenchmarkTools\n\ninclude(\"../exmamples/besselj.jl\")\n\n# To test this function, we first define input parameters `a` and output `out!`\na = 1.0\nout! = 0.0\n\n# We wrap the output with a randomly initialized gradient field, suppose we get the gradients from a virtual loss function.\n# Also, we need to initialize an empty gradient field for elements in input cuda tensor `a`.\nout! = ibesselj(out!, 2, a)[1]\nout_g! = GVar(out!, 1.0)\na_g = GVar(a)\n\n# Call the inverse program, the multiple dispatch will drive you to the goal.\nprintln(\"Benchmarking NiLang\")\ndisplay(@benchmark ibesselj($out!, 2, $a))\nprintln(\"Benchmarking NiLang.AD\")\ndisplay(@benchmark (~ibesselj)($out_g!, 2, $a_g))\n" }, { "path": "benchmark/first_function.jl", "content": "t1 = time()\nusing NiLang\n\n@i function dot(x, y, z)\n for i=1:10\n x += y[i]' * z[i]\n end\nend\nt2 = time()\nprintln(\"costs $(t2-t1)s\")\n" }, { "path": "benchmark/stack.jl", "content": "" }, { "path": "docs/Project.toml", "content": "[deps]\nBenchmarkTools = \"6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf\"\nChainRules = \"082447d4-558c-5d27-93f4-14fc19e9eca2\"\nCompose = \"a81c6b42-2e10-5240-aca2-a61377ecd94b\"\nDelimitedFiles = \"8bb1440f-4735-579b-a4ab-409b98df4dab\"\nDocumenter = \"e30172f5-a6a5-5a46-863b-614d45cd2de4\"\nFixedPointNumbers = \"53c48c17-4a7d-5ca2-90c5-79b7896eea93\"\nForwardDiff = \"f6369f11-7733-5829-9624-2563aa707210\"\nKernelAbstractions = \"63c18a36-062a-441e-b654-da1e3ab1ce7c\"\nLinearAlgebra = \"37e2e46d-f89d-539d-b4ee-838fcccc9c8e\"\nLiterate = \"98b081ad-f1c9-55d3-8b20-4c87d4299306\"\nLiveServer = \"16fef848-5104-11e9-1b77-fb7a48bbb589\"\nLogarithmicNumbers = \"aa2f6b4e-9042-5d33-9679-40d3a6b85899\"\nNiLang = \"ab4ef3a6-0b42-11ea-31f6-e34652774712\"\nPlots = \"91a5bcdd-55d7-5caf-9e0b-520d859cae80\"\nRandom = \"9a3f8284-a2c9-5f02-9a11-845980a1fd5c\"\nReexport = \"189a3867-3050-52da-a836-e630ba90ab69\"\nSparseArrays = \"2f01184e-e22b-5df5-ae63-d93ebab69eaf\"\nTest = \"8dfed614-e22c-5e08-85e1-65c5234f0b40\"\nViznet = \"52a3aca4-6234-47fd-b74a-806bdf78ede9\"\nZygote = \"e88e6eb3-aa80-5325-afca-941959d7151f\"\n" }, { "path": "docs/make.jl", "content": "using Documenter, NiLang\nusing SparseArrays\n\nusing Literate\ntutorialpath = joinpath(@__DIR__, \"src/examples\")\nsourcepath = joinpath(dirname(@__DIR__), \"examples\")\nfor jlfile in [\"besselj.jl\", \"sparse.jl\", \"qr.jl\", \"port_zygote.jl\", \"port_chainrules.jl\", \"fib.jl\", \"unitary.jl\", \"nice.jl\", \"realnvp.jl\", \"boxmuller.jl\", \"lognumber.jl\", \"pyramid.jl\"]\n Literate.markdown(joinpath(sourcepath, jlfile), tutorialpath)\nend\n\n# # Pluto pages\n# import Pkg\n\n# Pkg.add([\n# Pkg.PackageSpec(url=\"https://github.com/GiggleLiu/PlutoUtils.jl\", rev=\"static-export\"),\n# Pkg.PackageSpec(url=\"https://github.com/fonsp/Pluto.jl\", rev=\"05e5b68\"),\n# ]);\n\nmakedocs(;\n modules=[NiLang],\n format=Documenter.HTML(),\n pages=[\n \"Home\" => \"index.md\",\n \"What and Why\" => \"why.md\",\n \"Tutorial\" => Any[\n \"tutorial.md\",\n \"examples/port_zygote.md\",\n \"examples/port_chainrules.md\"\n ],\n \"Examples\" => Any[\n \"examples/fib.md\",\n \"examples/pyramid.md\",\n \"examples/besselj.md\",\n \"examples/sparse.md\",\n \"examples/lognumber.md\",\n \"examples/unitary.md\",\n #\"examples/nice.md\",\n #\"examples/realnvp.md\",\n \"examples/qr.md\",\n \"examples/boxmuller.md\",\n ],\n \"API & Manual\" => Any[\n \"instructions.md\",\n \"extend.md\",\n \"api.md\",\n \"faq.md\",\n ]\n ],\n repo=\"https://github.com/GiggleLiu/NiLang.jl/blob/{commit}{path}#L{line}\",\n sitename=\"NiLang.jl\",\n authors=\"JinGuo Liu, thautwarm\",\n)\n\n# import PlutoUtils\n\n# PlutoUtils.Export.github_action(; notebook_dir=NiLang.project_relative_path(\"notebooks\"), offer_binder=false, export_dir=NiLang.project_relative_path(\"docs\", \"build\", \"notebooks\"), generate_default_index=false, project=NiLang.project_relative_path(\"docs\"))\n\n\ndeploydocs(;\n repo=\"github.com/GiggleLiu/NiLang.jl.git\",\n)\n" }, { "path": "docs/src/api.md", "content": "```@meta\nDocTestSetup = quote\n using NiLangCore, NiLang, NiLang.AD, Test\nend\n```\n\n# API Manual\n## Compiling Tools (Reexported from NiLangCore)\n```@autodocs\nModules = [NiLangCore]\nOrder = [:macro, :function, :type]\n```\n\n## Instructions\n```@autodocs\nModules = [NiLang]\nOrder = [:macro, :function, :type]\n```\n\n## Automatic Differentiation\n```@autodocs\nModules = [NiLang.AD]\nOrder = [:macro, :function, :type]\n```\n" }, { "path": "docs/src/extend.md", "content": "# How to extend\n\n## Extend `+=`, `-=` and `⊻=` for irreversible one-out functions\n\nIt directly works\n```julia\njulia> using SpecialFunctions, NiLang\n\njulia> x, y = 2.1, 1.0\n(2.1, 1.0)\n\njulia> @instr y += besselj0(x)\n2.1\n\njulia> x, y\n(2.1, 1.7492472503018073)\n\njulia> @instr ~(y += besselj0(x))\n2.1\n\njulia> x, y\n(2.1, 1.0)\n```\n\nHere the statement\n```julia\n@instr y += besselj0(x)\n```\n\nis mapped to\n```julia\n@instr y += besselj0(x)\n```\n\nHowever, doing this does not give you correct gradients.\nFor `y += scalar_out_function(x)`, one can bind the backward rules like\n\n```julia\njulia> using ChainRules, NiLang.AD\n\njulia> besselj0_back(x) = ChainRules.rrule(besselj0, x)[2](1.0)[2]\nbesselj0_back (generic function with 1 method)\n\njulia> primitive_grad(::typeof(besselj0), x::Real) = besselj0_back(x)\nprimitive_grad (generic function with 1 method)\n\njulia> xg, yg = GVar(x), GVar(y, 1.0)\n(GVar(2.1, 0.0), GVar(1.0, 1.0))\n\njulia> @instr yg -= besselj0(xg)\nGVar(2.1, -0.5682921357570385)\n\njulia> xg, yg\n(GVar(2.1, -0.5682921357570385), GVar(0.8333930196680097, 1.0))\n\njulia> @instr yg += besselj0(xg)\nGVar(2.1, 0.0)\n\njulia> xg, yg\n(GVar(2.1, 0.0), GVar(1.0, 1.0))\n\njulia> NiLang.AD.check_grad(PlusEq(besselj0), (1.0, 2.1); iloss=1)\ntrue\n\njulia> using BenchmarkTools\n\njulia> @benchmark PlusEq(besselj0)($yg, $xg)\nBenchmarkTools.Trial: \n memory estimate: 0 bytes\n allocs estimate: 0\n --------------\n minimum time: 451.523 ns (0.00% GC)\n median time: 459.431 ns (0.00% GC)\n mean time: 477.419 ns (0.00% GC)\n maximum time: 857.036 ns (0.00% GC)\n --------------\n samples: 10000\n evals/sample: 197\n```\n\nGood!\n\n## Reversible multi-in multi-out functions\n\nIt is easy to do, define two normal Julia functions reversible to each other,\nusing the macro `@dual` to tell the compiler they are reversible to each other.\n\nFor example, a pair of dual functions `ROT` (2D rotation) and `IROT` (inverse rotation) that already defined in NiLang.\n\n```julia\n\"\"\"\n ROT(a!, b!, θ) -> a!', b!', θ\n\"\"\"\n@inline function ROT(i::Real, j::Real, θ::Real)\n a, b = rot(i, j, θ)\n a, b, θ\nend\n\n\"\"\"\n IROT(a!, b!, θ) -> ROT(a!, b!, -θ)\n\"\"\"\n@inline function IROT(i::Real, j::Real, θ::Real)\n i, j, _ = ROT(i, j, -θ)\n i, j, θ\nend\n@dual ROT IROT\n```\n\nOne can easily check the reversibility by typing\n```julia\njulia> check_inv(ROT, (1.0, 2.0, 3.0))\ntrue\n```\n\nFor self-reversible functions, one can declare the reversibility for it like this\n```julia\n\"\"\"\n SWAP(a!, b!) -> b!, a!\n\"\"\"\n@inline function SWAP(a!::Real, b!::Real)\n b!, a!\nend\n@selfdual SWAP\n```\n\nTo bind gradients for this multi-in, multi-out function.\nThe general approach is *Binding the backward rule on its inverse*!\n\n```julia\n@i @inline function IROT(a!::GVar, b!::GVar, θ::GVar)\n IROT(a!.x, b!.x, θ.x)\n NEG(θ.x)\n θ.x -= π/2\n ROT(a!.g, b!.g, θ.x)\n θ.g += a!.x * a!.g\n θ.g += b!.x * b!.g\n θ.x += π/2\n NEG(θ.x)\n ROT(a!.g, b!.g, π/2)\nend\n\n@i @inline function IROT(a!::GVar, b!::GVar, θ::Real)\n IROT(a!.x, b!.x, θ)\n NEG(θ)\n θ -= π/2\n ROT(a!.g, b!.g, θ)\n θ += π/2\n NEG(θ)\n ROT(a!.g, b!.g, π/2)\nend\n\n@nograd IROT(a!::Real, b!::Real, θ::GVar)\n```\n\nWhen this inverse function is called, the backward rules are automatically applied.\n\nGood! This method can also be extended to linear algebra functions, however, the memory allocation overhead is high because one need to wrap each element with `GVar`.\n" }, { "path": "docs/src/faq.md", "content": "## Why reversibility check fails even though the program is reversible?\nDue to the fact that floating pointing numbers are not exactly reversible, sometimes the invertibility check might fail due to the rounding error.\n\nTo fix this issue, you may want to make the check less restrictive\n```julia\nNiLangCore.GLOBAL_ATOL[] = 1e-6 # default is 1e-8\n```\n\nOr just turn off the check in the program (only if you are sure the program is correct)\n```julia\n@routine @invcheckoff begin\n ...\nend\n```\nTurning off the check will make your program faster too!\n\n## What makes the gradient check fails?\n##### Finite difference error due to numeric instability\nThe `NiLang.AD.check_grad` function sometimes fail due to either the rounding error or the finite difference error, you may want to check the gradient manually with the `NiLang.AD.ng` function (numeric gradient).\n```julia\njulia> NiLang.AD.ng(jin, copy.((out,b,ma,jinzhi,spread,bili)), 6; iloss=1, δ=1e-4)\n-5449.643843214744\n\njulia> NiLang.AD.ng(jin, copy.((out,b,ma,jinzhi,spread,bili)), 5; iloss=1, δ=1e-4)\n4503-element Array{Float64,1}:\n -0.0023380584934784565\n -0.0021096593627589755\n -0.0019811886886600405\n ⋮\n -0.009526640951662557\n -0.006004695478623034\n 0.0\n```\n\nand \n```julia\njulia> NiLang.AD.gradient(Val(1), jin, copy.((out,b,ma,jinzhi,spread,bili)))[end]\n-5449.643116967733\n\njulia> NiLang.AD.gradient(Val(1), jin, copy.((out,b,ma,jinzhi,spread,bili)))[end-1]\n4503-element Array{Float64,1}:\n -0.0005285958114468947\n -0.00030225263725219137\n -0.00017545437275561654\n ⋮\n -0.010422627668532736\n -0.0069140339974312695\n 0.0\n```\n\nHere, we can see the `jin` function is numerically sensitive to perturbations, which makes the numeric gradient incorrect.\nThe above code is from https://github.com/HanLi123/NiLang/issues/3\n\n##### Allocating a non-constant ancilla\nAnother possibility is, a non-constant ancilla is allocated.\n\n```julia\njulia> @i function f1(z, y)\n x ← y # wrong!\n z += x\n x → y\n end\n\njulia> NiLang.AD.gradient(Val(1), f1, (0.0, 1.0))\n(1.0, 0.0)\n\njulia> @i function f2(z, y)\n x ← zero(y)\n x += y\n z += x\n x -= y\n x → zero(y)\n end\n\njulia> NiLang.AD.gradient(Val(1), f2, (0.0, 1.0))\n(1.0, 1.0)\n```\n`f1` will give incorrect gradient because when ancilla `x` is deallocated, its gradient field will also be discarded.\n" }, { "path": "docs/src/grammar.md", "content": "# NiLang Grammar\n\nTo define a reversible function one can use macro **@i** plus a function definition like bellow\n\n```julia\n\"\"\"\ndocstring...\n\"\"\"\n@i function f(args..., kwargs...) where {...}\n \nend\n```\n\nwhere the definition of **** are shown in the grammar bellow.\nThe following is a list of terminologies used in the definition of grammar\n\n* , symbols\n* , numbers\n* 0, empty statement\n* , native Julia expression\n* [ ], zero or one repetitions.\n\nHere, all $JuliaExpr$ should be pure, otherwise the reversibility is not guaranteed.\nDataview is a view of a data, it can be a bijective mapping of an object, an item of an array or a field of an object.\n\n\n```bnf\nStmts : 0 \n | Stmt\n | Stmts Stmt\n ;\n\nStmt : BlockStmt\n | IfStmt\n | WhileStmt\n | ForStmt\n | InstrStmt\n | RevStmt\n | AncillaStmt\n | TypecastStmt \n | @routine Stmt\n | @safe \n | CallStmt\n ;\n\n\nBlockStmt : 'begin' Stmts 'end';\n\nRevCond : '(' ',' ')';\n\nIfStmt : 'if' RevCond Stmts ['else' Stmts] 'end';\n\nWhileStmt : 'while' RevCond Stmts 'end';\n\nRange : ':' [':' ];\n\nForStmt : 'for' '=' Range Stmts 'end';\n\nKwArg : '=' ;\n\nKwArgs : [KwArgs ','] KwArg ;\n\nCallStmt : '(' [DataViews] [';' KwArgs] ')';\n\nConstant : | 'π';\n\nInstrBinOp : '+=' | '-=' | '⊻=';\n\nInstrTrailer : ['.'] '(' [DataViews] ')';\n\nInstrStmt : DataView InstrBinOp [InstrTrailer];\n\nRevStmt : '~' Stmt;\n\nAncillaStmt : '←' \n | '→' \n ;\n\nTypecastStmt : '(' '=>' ')' '(' ')';\n\n@routine : '@routine' Stmt;\n\n@safe : '@safe' ;\n\nDataViews : 0\n | DataView\n | DataViews ',' DataView\n | DataViews ',' DataView '...'\n ;\n\nDataView : DataView '[' ']'\n | DataView '.' \n | DataView '|>' \n | DataView '\\''\n | '-' DataView\n | Constant\n | \n ;\n```\n" }, { "path": "docs/src/index.md", "content": "# NiLang.jl\n\nNiLang is a reversible eDSL that can run backwards. The motation is to support source to source AD.\n\nCheck [our paper](https://arxiv.org/abs/2003.04617)!\n\nWelcome for discussion in [Julia slack](https://slackinvite.julialang.org/), **#autodiff** and **#reversible-commputing** channel.\n\n## Tutorials\n```@contents\nPages = [\n \"tutorial.md\",\n \"examples/port_zygote.md\",\n]\nDepth = 1\n```\n\nAlso see blog posts\n* [How to write a program differentiably](https://nextjournal.com/giggle/how-to-write-a-program-differentiably)\n* [Simulate a reversible Turing machine in 50 lines of code](https://nextjournal.com/giggle/rtm50)\n\n## Documentation\n\n## Examples\n```@contents\nPages = [\n \"examples/fib.md\",\n \"examples/besselj.md\",\n \"examples/sparse.md\",\n \"examples/lognumber.md\",\n \"examples/unitary.md\",\n \"examples/qr.md\",\n \"examples/nice.md\",\n \"examples/realnvp.md\",\n \"examples/boxmuller.md\",\n]\nDepth = 1\n```\n\n## Manual\n\n```@contents\nPages = [\n \"grammar.md\",\n \"instructions.md\",\n \"extend.md\",\n \"examples/sharedwrite.md\",\n \"api.md\",\n \"faq.md\",\n]\nDepth = 1\n```\n" }, { "path": "docs/src/instructions.md", "content": "# Instruction Reference\n\n## Instruction definitions\n\nThe Julia functions and symbols for instructions\n\n| instruction | translated | symbol |\n| ----------- | ---------- | ---- |\n| $y \\mathrel{+}= f(args...)$ | PlusEq(f)(args...) | $\\oplus$ |\n| $y \\mathrel{-}= f(args...)$ | MinusEq(f)(args...) | $\\ominus$ |\n| $y \\mathrel{\\veebar}= f(args...)$ | \\texttt{XorEq(f)(args...) | $\\odot$ |\n\nThe list of reversible instructions that implemented in NiLang\n\n| instruction | output |\n| ----------- | ---------- |\n| ${\\rm SWAP}(a, b)$ | $b, a$ |\n| ${\\rm ROT}(a, b, \\theta)$ | $a \\cos\\theta - b\\sin\\theta, b \\cos\\theta + a\\sin\\theta, \\theta$ |\n| ${\\rm IROT}(a, b, \\theta)$ | $a \\cos\\theta + b\\sin\\theta, b \\cos\\theta - a\\sin\\theta, \\theta$ |\n| $y \\mathrel{+}= a^\\wedge b$ | $y+a^b, a, b$ |\n| $y \\mathrel{+}= \\exp(x)$ | $y+e^x, x$ |\n| $y \\mathrel{+}= \\log(x)$ | $y+\\log x, x$ |\n| $y \\mathrel{+}= \\sin(x)$ | $y+\\sin x, x$ |\n| $y \\mathrel{+}= \\cos(x)$ | $y+\\cos x, x$ |\n| $y \\mathrel{+}= {\\rm abs}(x)$ | $y+ |x|, x$ |\n| $NEG(y)$ | $-y$ |\n\n\".\" is the broadcasting operations in Julia.\n\n## Jacobians and Hessians for Instructions\n\nSee my [blog post](https://giggleliu.github.io/2020/01/18/jacobians.html).\n" }, { "path": "docs/src/tutorial.md", "content": "# My first NiLang program\n\n## Basic Statements\n\n| Statement | Meaning |\n| :------------------------ | :----------------------------------------------------------- |\n| x ← val | allocate a new variable `x`, with an initial value `val` (a constant). |\n| x → val | deallocate variable `x` with content `val`. |\n| x += f(y) | a reversible instruction. |\n| x .+= f.(y) | instruction call with broadcasting. |\n| f(y) | a reversible function. |\n| f.(y) | function call with broadcasting. |\n| if (pre, post) ... end | if statement. |\n| @from post while pre ... end | while statement. |\n| for x=1:3 ... end | for statement. |\n| begin ... end | block statement. |\n| @safe ... | insert an irreversible statement. |\n| ~(...) | inverse a statement. |\n| @routine ... | record a routine in the **routine stack**. |\n| ~@routine | place the inverse of the routine on **routine stack** top. |\n\nThe condition expression in **if** and **while** statements are a bit hard to digest, please refer our paper [arXiv:2003.04617](https://arxiv.org/abs/2003.04617).\n\n## A reversible program\n\nOur first program is to compute a loss function defined as\n\n```math\n\\mathcal{L} = {\\vec z}^T(a\\vec{x} + \\vec{y}),\n```\n\nwhere $\\vec x$, $\\vec y$ and $\\vec{z}$ are column vectors, $a$ is a scalar.\n\n```julia\n@i function r_axpy!(a::T, x::AbstractVector{T}, y!::AbstractVector{T}) where T\n @safe @assert length(x) == length(y!)\n for i=1:length(x)\n y![i] += a * x[i]\n end\nend\n\n@i function r_loss(out!, a, x, y!, z)\n r_axpy!(a, x, y!)\n for i=1:length(z)\n \tout! += z[i] * y![i]\n end\nend\n```\n\nFunctions do not have return statements, they return input arguments instead.\nHence `r_loss` defines a 5 variable to 5 variable bijection.\nLet's check the reversibility\n```julia\njulia> out, a, x, y, z = 0.0, 2.0, randn(3), randn(3), randn(3)\n(0.0, 2.0, [0.9265845776642722, 0.8532458027149912, 0.6201064385679095],\n [1.1142808415540468, 0.5506163710455121, -1.9873779917908814],\n [1.1603953198942412, 0.5562855137395296, 1.9650050430758796])\n\njulia> out, a, x, y, z = r_loss(out, a, x, y, z)\n(3.2308283403544342, 2.0, [0.9265845776642722, 0.8532458027149912, 0.6201064385679095],\n [2.967449996882591, 2.2571079764754947, -0.7471651146550624],\n [1.1603953198942412, 0.5562855137395296, 1.9650050430758796])\n```\n\nWe find the contents in `out` and `y` are changed after calling the loss function.\nThen we call the inverse loss function `~r_loss`.\n\n```julia\njulia> out, a, x, y, z = (~r_loss)(out, a, x, y, z)\n(0.0, 2.0, [0.9265845776642722, 0.8532458027149912, 0.6201064385679095],\n [1.1142808415540466, 0.5506163710455123, -1.9873779917908814],\n [1.1603953198942412, 0.5562855137395296, 1.9650050430758796])\n```\n\nValues are restored. Here, instead of assigning variables one by one,\none can also use the macro `@instr`\n```julia\n@instr r_loss(out, a, x, y, z)\n```\n`@instr` macro is for executing a reversible statement.\n\n## My first reversible AD program\n\n```julia\njulia> using NiLang.AD: Grad\n\njulia> x, y, z = randn(3), randn(3), randn(3)\n([2.2683181471139906, -0.7374245775047469, 0.9568936661385092],\n [1.0275914704043452, 1.647972121962081, -0.8349079845797637],\n [1.4272076815911372, 0.5317755971532034, 0.4412421572457776])\n\njulia> Grad(r_loss)(0.0, 0.5, x, y, z; iloss=1)\n(GVar(0.0, 1.0), GVar(0.5, 3.2674385142974036),\n GVar{Float64,Float64}[GVar(2.2683181471139906, 0.7136038407955686), GVar(-0.7374245775047469, 0.2658877985766017), GVar(0.9568936661385092, 0.2206210786228888)],\n GVar{Float64,Float64}[GVar(2.1617505439613405, 1.4272076815911372), GVar(1.2792598332097076, 0.5317755971532034), GVar(-0.35646115151050906, 0.4412421572457776)],\n GVar{Float64,Float64}[GVar(1.4272076815911372, 3.295909617518336), GVar(0.5317755971532034, 0.9105475444573341), GVar(0.4412421572457776, 0.12198568155874556)])\n\njulia> gout, ga, gx, gy, gz = Grad(r_loss)(0.0, 0.5, x, y, z; iloss=1)\n(GVar(0.0, 1.0), GVar(0.5, 3.2674385142974036),\n GVar{Float64,Float64}[GVar(2.2683181471139906, 0.7136038407955686), GVar(-0.7374245775047469, 0.2658877985766017), GVar(0.9568936661385092, 0.2206210786228888)],\n GVar{Float64,Float64}[GVar(3.295909617518336, 1.4272076815911372), GVar(0.9105475444573341, 0.5317755971532034), GVar(0.12198568155874556, 0.4412421572457776)],\n GVar{Float64,Float64}[GVar(1.4272076815911372, 4.4300686910753315), GVar(0.5317755971532034, 0.5418352557049606), GVar(0.4412421572457776, 0.6004325146280002)])\n```\n\nThe results are a bit messy, since NiLang wraps each element with a gradient field automatically. We can take the gradient field using the `grad` function like\n\n```julia\njulia> grad(gout)\n1.0\n\njulia> grad(ga)\n3.2674385142974036\n\njulia> grad(gx)\n3-element Array{Float64,1}:\n 0.7136038407955686\n 0.2658877985766017\n 0.2206210786228888\n\njulia> grad(gy)\n3-element Array{Float64,1}:\n 1.4272076815911372\n 0.5317755971532034\n 0.4412421572457776\n\njulia> grad(gz)\n3-element Array{Float64,1}:\n 4.4300686910753315\n 0.5418352557049606\n 0.6004325146280002\n```\n" }, { "path": "docs/src/why.md", "content": "# What is Reversible Computing and why do we need it\n\n# What are reversible computing and reversible programming\nReversible computing is a computing paradigm that can deterministically undo a computational process, it requires a user not erasing any information during computations. It boomed during 1970-2005, however, but runs into a winter after that. It can do anything that a traditional computing device can do, with possible overheads in time and space. Reversible programing is often considered as the computing model designed for reversible computing, while it can also be executed on a irreversible device. The following book covers a lot about reversible programming.\n\n![Introduction to Reversible Computing](asset/revcomp.jpg)\n\n## Why reversible computing is the future of computing: from a physicist's perspective\n\nThe driving force of studying reversible computing is improving the energy efficiency of our computing devices. Energy efficiency of computing devices affect the value of [bitcoins](https://www.investopedia.com/news/do-bitcoin-mining-energy-costs-influence-its-price/), the battery size of a [spacecraft](https://ieeexplore.ieee.org/document/7945170) and artificial intelligence (AI) industry as we will cover bellow.\n\nAs is well know, the fundamental laws of physics are reversible. Have you ever had such a confusion that why our computing model is irreversible while our world is governed by reversible laws? This discrepency is due to the fact that the irreversibility is an emergent phenomenon of statistic physics,\nwe need a ideal heat bath that having an \"infinite size\" to create irreversibility. This is why the energy efficiency of traditional devices is getting harder and harder to improve, although they are still several orders above the Landauer's limit. The [Landauer's principle](https://en.wikipedia.org/wiki/Landauer%27s_principle) states that irreversible computing has a lower bound of energy cost ~``\\ln 2 k_b T``\n\n> Landauer's principle is a physical principle pertaining to the lower theoretical limit of energy consumption of computation. It holds that \"any logically irreversible manipulation of information, such as the erasure of a bit or the merging of two computation paths, must be accompanied by a corresponding entropy increase in non-information-bearing degrees of freedom of the information-processing apparatus or its environment\".Another way of phrasing Landauer's principle is that if an observer loses information about a physical system, the observer loses the ability to extract work from that system.\n\nMicroscopic systems that can be used to build up a reversible computing device are ubiquitous, like [fluxon](https://ieeexplore.ieee.org/abstract/document/8990955), cold atoms, [DNA](https://www.amazon.com/Feynman-Lectures-Computation-Frontiers-Physics/dp/0738202967) and quantum dots. Even the adiabatic CMOS (a reversible computing device utilizing CMOS technology) can potentially be orders more energy efficient than traditional CMOS, and it is [already useful in spacecrafts](https://www.osti.gov/servlets/purl/1377599). The detailed analysis of the energy-speed trade off in adiabatic CMOS can be found [here](https://www3.nd.edu/~lent/pdf/nd/AdiabaticCMOS_HanninenSniderLent2014.pdf).\n\nIn reversible programming, [automatically differentiating any program is directly achievable](https://arxiv.org/abs/2003.04617). Automatic differentiation is a building block of artificial intelligence, crunching this problem can potentially lead to the next boom of AI. Programs are built on top of basic instructions like \"+\", \"*\", \"/\", \"-\". We can use these basic instructions to write Bessel functions, singular value decompositions et. al. [Traditional autodiff frameworks](https://epubs.siam.org/doi/book/10.1137/1.9780898717761) keep track of intermediate states in a global stack and use them for back-propagation. However, doing this brings space overheads that linear to time, which can easily explode the memory. Reversible programming reverse the tape directly for you, while having flexible yet efficient time-space tradeoff algorithms to control the memory usage.\n\nI am optimistic about reversible computing also because we have so much room to improve in the energy perspective. Our computer computes one bit information at the energy cost ~``10^8 k_b T``, while in our body, DNA copy machine computes a bit information at an energy cost ~``10 k_b T``. To embrace the true artificial intelligence, we still have a long way to go.\n" }, { "path": "examples/Adam.jl", "content": "export Adam\n\nmutable struct Adam\n lr::AbstractFloat\n gclip::AbstractFloat\n beta1::AbstractFloat\n beta2::AbstractFloat\n eps::AbstractFloat\n t::Int\n fstm\n scndm\nend\n\nAdam(; lr=0.001, gclip=0, beta1=0.9, beta2=0.999, eps=1e-8)=Adam(lr, gclip, beta1, beta2, eps, 0, nothing, nothing)\n\nfunction update!(w, g, p::Adam)\n gclip!(g, p.gclip)\n if p.fstm===nothing; p.fstm=zero(w); p.scndm=zero(w); end\n p.t += 1\n lmul!(p.beta1, p.fstm)\n BLAS.axpy!(1-p.beta1, g, p.fstm)\n lmul!(p.beta2, p.scndm)\n BLAS.axpy!(1-p.beta2, g .* g, p.scndm)\n fstm_corrected = p.fstm / (1 - p.beta1 ^ p.t)\n scndm_corrected = p.scndm / (1 - p.beta2 ^ p.t)\n BLAS.axpy!(-p.lr, @.(fstm_corrected / (sqrt(scndm_corrected) + p.eps)), w)\nend\n\nfunction gclip!(g, gclip)\n if gclip == 0\n g\n else\n gnorm = vecnorm(g)\n if gnorm <= gclip\n g\n else\n BLAS.scale!(gclip/gnorm, g)\n end\n end\nend\n" }, { "path": "examples/CUDA/README.md", "content": "# Reversible programming on GPU\n\nSpecial Notes:\n* please use `@invcheckoff` to close all reversibility check in a kernel.\n* be careful about the race condition when automatic differentiating a CUDA program.\n\n## Suggested reading order\n1. `swap_gate.jl` simulates a quantum swap gate, its reversible counter part is here\nhttp://tutorials.yaoquantum.org/dev/generated/developer-guide/2.cuda-acceleration/\n2. `rotation_gate.jl` simulates a quantum rotation gate, obtaining the gradients on rotation angle would have race condition.\n" }, { "path": "examples/CUDA/rotation_gate.jl", "content": "using CUDA, GPUArrays\nusing NiLang, NiLang.AD\n\nconst RotGates = Union{Val{:Rz}, Val{:Rx}, Val{:Ry}}\n\n@i @inline function instruct!(state::CuVector, gate::RotGates, loc::Int, theta::Real)\n mask ← 1<<(loc-1)\n @cuda threads=256 blocks=ceil(Int, length(state)/256) rot_kernel(gate, state, mask, theta)\nend\n# @launchkernel CUDADevice() 256 length(out!) bessel_kernel(out!, v, z)\n\n@i @inline function rot_kernel(gate::Val{:Rz}, state, mask, θ)\n @invcheckoff b ← (blockIdx().x-1) * blockDim().x + threadIdx().x\n @invcheckoff if (b < length(state) && b & mask == 0, ~)\n ROT_INSTRUCT(gate, state[b+1], state[b⊻mask+1], θ)\n end\nend\n\n@i @inline function ROT_INSTRUCT(gate::Val{:Rz}, a::T, b, θ) where T\n # make sure `invcheck` is turned off!\n @routine @invcheckoff begin\n @zeros T anc1 anc2 anc3 anc4\n anc1 += θ*(0.5im)\n anc2 += CUDA.exp(anc1)\n end\n anc3 += a * anc2'\n anc4 += b * anc2\n NiLang.SWAP(a, anc3)\n NiLang.SWAP(b, anc4)\n anc3 -= a / anc2'\n anc4 -= b / anc2\n ~@routine\nend\n\nv = randn(ComplexF64, 128) |> CuArray\nv1 = instruct!(copy(v), Val(:Rz), 3, 0.5)[1]\n# we can not obtain the gradient for the race condition.\n\n\n# TODO: Rx and Ry gates, not finished!\n@i @inline function ROT_INSTRUCT(gate::Val{:Rx}, a, b, θ)\n ROT_INSTRUCT(Val(:Rz), a, b, π/2)\n ROT_INSTRUCT(Val(:Ry), a, b, θ)\n ROT_INSTRUCT(Val(:Rz), a, b, -π/2)\nend\n\n@i @inline function ROT_INSTRUCT(gate::Val{:Ry}, a, b, θ)\n divint(θ, 2)\n ROT(a, b, θ)\n mulint(θ, 2)\nend\n" }, { "path": "examples/CUDA/swap_gate.jl", "content": "using CUDA, GPUArrays\nusing NiLang, NiLang.AD\n\n\"\"\"\nA reversible swap kernel for GPU for SWAP gate in quantum computing.\nSee the irreversible version for comparison\n\nhttp://tutorials.yaoquantum.org/dev/generated/developer-guide/2.cuda-acceleration/\n\"\"\"\n@i @inline function swap_kernel(state::AbstractVector{T}, mask1, mask2) where T\n @invcheckoff b ← (blockIdx().x-1) * blockDim().x + threadIdx().x\n @invcheckoff if (b < length(state), ~)\n if (b&mask1==0 && b&mask2==mask2, ~)\n NiLang.SWAP(state[b+1], state[b ⊻ (mask1|mask2) + 1])\n end\n end\nend\n\n# TODO: support ::Type like argument.\n\"\"\"\nSWAP gate in quantum computing.\n\"\"\"\n@i function instruct!(state::CuVector, gate::Val{:SWAP}, locs::Tuple{Int,Int})\n mask1 ← 1 << (locs[1]-1)\n mask2 ← 1 << (locs[2]-1)\n @cuda threads=256 blocks=ceil(Int,length(state)/256) swap_kernel(state, mask1, mask2)\nend\n\nusing Test\n@testset \"swap gate\" begin\n v = cu(randn(128))\n v1 = instruct!(copy(v), Val(:SWAP), (3,4))[1]\n v2 = instruct!(copy(v1), Val(:SWAP), (3,4))[1]\n v3 = (~instruct!)(copy(v1), Val(:SWAP), (3,4))[1]\n @test !(v ≈ v1)\n @test v ≈ v2\n @test v ≈ v3\nend\n\n@i function loss(out!, state::CuVector)\n instruct!(state, Val(:SWAP), (3,4))\n out! += state[4]\nend\n\nloss(0.0, CuArray(randn(128)))\nGrad(loss)(Val(1), 0.0, CuArray(randn(128)))\n\n####################### A different loss ###############\n@i function loss(out!, state::CuVector, target::CuVector)\n instruct!(state, Val(:SWAP), (3,4))\n out! += state' * target\nend\n\n# requires defining a new primitive, we don't how to parallelize a CUDA program automatically yet.\nusing LinearAlgebra: Adjoint\nfunction (_::MinusEq{typeof(*)})(out!::GVar, x::Adjoint{<:Any, <:CuVector{<:GVar}}, y::CuVector{<:GVar})\n chfield(out!, value, value(out!)-(value.(x) * value.(y))[]),\n chfield.(parent(x), grad, grad.(parent(x)) .+ grad(out!)' .* conj.(value.(y)))',\n chfield.(y, grad, grad.(y) .+ grad(out!) .* conj.(value.(x')))\nend\n\nfunction (_::PlusEq{typeof(*)})(out!::GVar, x::Adjoint{<:Any, <:CuVector{<:GVar}}, y::CuVector{<:GVar})\n chfield(out!, value, value(out!)+(value.(x) * value.(y))[]),\n chfield.(parent(x), grad, grad.(parent(x)) .- grad(out!)' .* conj.(value.(y)))',\n chfield.(y, grad, grad.(y) .- grad(out!) .* conj.(value.(x')))\nend\n\nfunction (_::PlusEq{typeof(*)})(out!, x, y)\n out! += x * y\n out!, x, y\nend\n\nfunction (_::MinusEq{typeof(*)})(out!, x, y)\n out! -= x * y\n out!, x, y\nend\n\nloss(0.0, CuArray(randn(128)), CuArray(randn(128)))\nGrad(loss)(Val(1), 0.0, CuArray(randn(128)), CuArray(randn(128)))" }, { "path": "examples/README.md", "content": "# Examples\n\n1. Reversible CUDA programming: [CUDA/](CUDA/)\n2. Generate backward rules for Zygote: [port_zygote.jl](port_zygote.jl)\n3. Obtaining symbolics gradients: [Symbolics/](Symbolics/)\n4. Solving the graph embeding problem: [graph_embeding.jl](graph_embeding.jl) and [graph_embeding_zygote.jl](graph_embeding_zygote.jl)\n5. NICE network: [nice.jl](nice.jl)\n6. [Gaussian mixture model](https://github.com/JuliaReverse/NiGaussianMixture.jl)\n7. [Bundle Adjustment](https://github.com/JuliaReverse/NiBundleAdjustment.jl)\n" }, { "path": "examples/Symbolics/print_jacobians.jl", "content": "using NiLang, NiLang.AD\n\ninclude(\"symlib.jl\")\nNiLang.AD.isvar(sym::Basic) = true\nNiLang.AD.GVar(sym::Basic) = GVar(sym, zero(sym))\n\n# a patch for symbolic IROT\n@i @inline function NiLang.IROT(a!::GVar{<:Basic}, b!::GVar{<:Basic}, θ::GVar{<:Basic})\n IROT(a!.x, b!.x, θ.x)\n NEG(θ.x)\n θ.x -= Basic(π)/2\n ROT(a!.g, b!.g, θ.x)\n θ.g += a!.x * a!.g\n θ.g += b!.x * b!.g\n θ.x += Basic(π)/2\n NEG(θ.x)\n ROT(a!.g, b!.g, Basic(π)/2)\nend\n\nNiLang.INC(x::Basic) = x + one(x)\nNiLang.DEC(x::Basic) = x - one(x)\n@inline function NiLang.ROT(i::Basic, j::Basic, θ::Basic)\n a, b = rot(i, j, θ)\n a, b, θ\nend\n@inline function NiLang.IROT(i::Basic, j::Basic, θ::Basic)\n i, j, _ = ROT(i, j, -θ)\n i, j, θ\nend\nBase.sincos(x::Basic) = (sin(x), cos(x))\n\nfunction printall()\n syms = [Basic(:a), Basic(:b), Basic(:c)]\n\n for (subop, nargs) in [(identity, 2), (*, 3), (/, 3), (^, 3), (exp, 2), (log, 2), (sin, 2), (cos, 2)]\n for opm in [PlusEq, MinusEq]\n op = opm(subop)\n @show op\n printone(op, syms, nargs)\n end\n end\n for (op, nargs) in [(-, 1), (ROT, 3), (IROT, 3)]\n printone(op, syms, nargs)\n end\n # abs, conj\nend\n\n@i function jf1(op, x)\n op(x[1])\nend\n\n@i function jf2(op, x)\n op(x[1], x[2])\nend\n\n@i function jf3(op, x)\n op(x[1], x[2], x[3])\nend\n\n\"\"\"print the jacobian of one operator\"\"\"\nfunction printone(op, syms, n)\n if n==1\n jac = jacobian_repeat(jf1, op, syms[1:1]; iin=2, iout=2)\n elseif n==2\n jac = jacobian_repeat(jf2, op, syms[1:2]; iin=2, iout=2)\n elseif n==3\n jac = jacobian_repeat(jf3, op, syms[1:3]; iin=2, iout=2)\n end\n println(\"------ $op ------\")\n pretty_print_matrix(jac)\nend\n\nprintall()\n" }, { "path": "examples/Symbolics/symbolic_utils.jl", "content": "using NiLang, NiLang.AD\nusing SymbolicUtils\nusing SymbolicUtils: Term, Sym\nusing LinearAlgebra\n\nconst SymReal = Sym{Real}\nconst TermReal = Term{Real}\nconst SReals = Union{Term{Real}, Sym{Real}}\n\nimport NiLang: INC, DEC, ROT, IROT, FLIP\n@inline FLIP(b::Sym{Bool}) = !b\n\n@inline function INC(a!::SReals)\n a! + one(a!)\nend\n\n@inline function DEC(a!::SReals)\n a! - one(a!)\nend\n\n@inline function ROT(i::SReals, j::SReals, θ::SReals)\n a, b = rot(i, j, θ)\n a, b, θ\nend\n\n@inline function IROT(i::SReals, j::SReals, θ::SReals)\n i, j, _ = ROT(i, j, -θ)\n i, j, θ\nend\n\nNiLang.AD.GVar(x::SReals) = NiLang.AD.GVar(x, zero(x))\nBase.convert(::Type{SymReal}, x::Integer) = SymReal(Symbol(x))\nBase.convert(::Type{Term{Real}}, x::Integer) = TermReal(Symbol(x))\n\nBase.zero(x::Sym{T}) where T = zero(Sym{T})\nBase.one(x::Sym{T}) where T = one(Sym{T})\nBase.zero(::Type{<:Sym{T}}) where T = Sym{T}(Symbol(0))\nBase.zero(::Type{<:Term{T}}) where T = Term{T}(Symbol(0))\nBase.one(::Type{<:Sym{T}}) where T = Sym{T}(Symbol(1))\nBase.one(::Type{<:Term{T}}) where T = Term{T}(Symbol(1))\nBase.iszero(x::Sym{T}) where T = x === zero(x)\nBase.adjoint(x::SReals) = x\nSymbolicUtils.Term{T}(x::Sym{T}) where T = Term{T}(x.name)\n\nLinearAlgebra.dot(a::T, b::T) where T<:SReals = a * b\n\ninclude(\"sparse.jl\")\n\nusing BenchmarkTools, Random\nsyms = @syms a::Real b::Real c::Real d::Real e::Real f::Real g::Real\nBase.rand(r::Random.AbstractRNG, ::Type{SymReal}, i::Integer) = rand(r, syms, i)\nBase.rand(r::Random.AbstractRNG, ::Type{TermReal}, i::Integer) = rand(r, TermReal.(syms), i)\na = sprand(TermReal, 100, 100, 0.05);\nb = sprand(TermReal, 100, 100, 0.05);\n@benchmark SparseArrays.dot($a, $b)\n@benchmark idot(TermReal(Symbol(0)), $a, $b)\n@benchmark Grad(idot)(Val(1), TermReal(Symbol(0)), $a, $b)\nGVar(a)\n\ninclude(\"Symbolics/symlib.jl\")\nsyms = @vars a b c d e f g\nBase.rand(r::Random.AbstractRNG, ::Type{<:Basic}, i::Integer) = rand(r, syms, i)\na = sprand(Basic, 100, 100, 0.05);\nb = sprand(Basic, 100, 100, 0.05);\n@benchmark SparseArrays.dot($a, $b)\n@benchmark idot(Basic(0), $a, $b)\n@benchmark Grad(idot)(Val(1), Basic(0), $a, $b)\n" }, { "path": "examples/Symbolics/symlib.jl", "content": "using SymEngine\nusing SymEngine: BasicType\n\nsconj = SymFunction(\"conj\")\nBase.conj(x::Basic) = Basic(conj(SymEngine.BasicType(x)))\nBase.conj(x::BasicType) = real(x) - im * imag(x)\nBase.imag(x::BasicType{Val{:Constant}}) = Basic(0)\nBase.imag(x::BasicType{Val{:Symbol}}) = Basic(0)\n\npretty_print_number(x; lengthonly=false) = pretty_print_number(stdout, x; lengthonly=lengthonly)\nfunction pretty_print_number(io::IO, x; lengthonly=false)\n sx = string(x)\n lengthonly || print(io, sx)\n return length(sx)\nend\n\nfunction pretty_print_number(io::IO, x::AbstractFloat; lengthonly=false)\n closest_int = round(Int, x)\n if isapprox(x, closest_int, atol=1e-12)\n si = string(closest_int)\n lengthonly || print(io, si)\n return length(si)\n else\n sx = string(x)\n lengthonly || print(io, sx)\n return length(sx)\n end\nend\n\nfunction pretty_print_number(io::IO, x::Complex; atol::Real = 1e-12, lengthonly=false)\n l = 0\n if !isapprox(real(x), 0, atol=atol)\n l += pretty_print_number(io, real(x), lengthonly=lengthonly)\n end\n if !isapprox(imag(x), 0, atol=atol)\n if !isapprox(real(x), 0, atol=atol)\n lengthonly || print(imag(x) > 0 ? \"+\" : \"\")\n l += 1\n end\n l += pretty_print_number(io, imag(x), lengthonly=lengthonly)\n lengthonly || print(io, \"I\")\n l += 1\n else\n if isapprox(real(x), 0, atol=atol)\n lengthonly || print(io, \"0\")\n l += 1\n end\n end\n return l\nend\n\npretty_print_matrix(m) = pretty_print_matrix(stdout, m)\nfunction pretty_print_matrix(io::IO, m)\n minlen = maximum(pretty_print_number.(m, lengthonly=true))+1\n for i in 1:size(m,1)\n print(io, \"[\")\n for j in 1:size(m,2)\n l = pretty_print_number(m[i,j])\n print(\" \"^(minlen-l-(j==size(m,1))))\n end\n println(io, \"]\")\n end\nend\n" }, { "path": "examples/_sharedwrite.jl", "content": "# # The shared write problem on GPU\n\n# We will write a GPU version of `axpy!` function.\n\n# ## The main program\n\nusing NiLang, NiLang.AD\nusing CUDA\nusing KernelAbstractions\nCUDA.allowscalar(true)\n\n# so far, this example requires patch: https://github.com/JuliaGPU/KernelAbstractions.jl/pull/52\n\n@i @kernel function axpy_kernel(y!, α, x)\n ## invcheckoff to turn of `reversibility checker`\n ## GPU can not handle errors!\n @invcheckoff begin\n i ← @index(Global)\n y![i] += x[i] * α\n i → @index(Global)\n end\nend\n\n@i function cu_axpy!(y!::AbstractVector, α, x::AbstractVector)\n @launchkernel CUDADevice() 256 length(y!) axpy_kernel(y!, α, x)\nend\n\n@i function loss(out, y!, α, x)\n cu_axpy!(y!, α, x)\n ## Note: the following code is stupid scalar operations on CuArray,\n ## They are only for testing.\n for i=1:length(y!)\n out += y![i]\n end\nend\n\ny! = rand(100)\nx = rand(100)\ncuy! = y! |> CuArray\ncux = x |> CuArray\nα = 0.4\n\n# ## Check the correctness of results\n\nusing Test\ncu_axpy!(cuy!, α, cux)\n@test Array(cuy!) ≈ y! .+ α .* x\n(~cu_axpy!)(cuy!, α, cux)\n@test Array(cuy!) ≈ y!\n\n# Let's check the gradients\nlsout = 0.0\n@instr Grad(loss)(Val(1), lsout, cuy!, α, cux)\n\n# you will see a correct vector `[0.4, 0.4, 0.4 ...]`\ngrad.(cux)\n\n# you will see `0.0`.\ngrad(α)\n\n# ## Why some gradients not correct?\n# In the above example, `α` is a scalar, whereas a scalar is not allowed to change in a CUDA kernel.\n# What if we change `α` to a CuArray?\n\n# ## This one works: using a vector of `α`\n@i @kernel function axpy_kernel(y!, α, x)\n @invcheckoff begin\n i ← @index(Global)\n y![i] += x[i] * α[i]\n i → @index(Global)\n end\nend\n\ncuy! = y! |> CuArray\ncux = x |> CuArray\ncuβ = repeat([0.4], 100) |> CuArray\nlsout = 0.0\n@instr Grad(loss)(Val(1), lsout, cuy!, cuβ, cux)\n\n# You will see correct answer\ngrad.(cuβ)\n\n# ## This one has the shared write problem: using a vector of `α`, but shared read.\n@i @kernel function axpy_kernel(y!, α, x)\n @invcheckoff begin\n i ← @index(Global)\n y![i] += x[i] * α[i]\n i → @index(Global)\n end\nend\n\ncuy! = y! |> CuArray\ncux = x |> CuArray\ncuβ = repeat([0.4], 100) |> CuArray\nlsout = 0.0\ncuβ = [0.4] |> CuArray\n\n# Run the following will give you a happy error\n#\n# > ERROR: a exception was thrown during kernel execution.\n# > Run Julia on debug level 2 for device stack traces.\n\n# ```julia\n# @instr Grad(loss)(Val(1), lsout, cuy!, cuβ, cux)\n# ```\n\n# Because, shared write is not allowed. We need someone clever enough to solve this problem for us.\n\n# ## Conclusion\n# * Shared scalar: the gradient of a scalar will not be updated.\n# * Expanded vector: works properly, but costs more memory.\n# * Shared 1-element vector: error on shared write.\n" }, { "path": "examples/batched_tr.jl", "content": "using NiLang, NiLang.AD\nusing KernelAbstractions, CUDA, CUDAKernels\n\n@i @kernel function kernel_f(A, B::AbstractVector{TB}) where TB\n # turng off reversibility check, since GPU can not handle errors\n @invcheckoff begin\n # allocate\n batch ← @index(Global)\n s ← zero(TB)\n # computing\n for i in axes(A, 1)\n s += A[i, i, batch]\n end\n B[batch] += s\n # deallocate safely\n s → zero(TB)\n batch → @index(Global)\n end\nend\n\n@i function batched_tr!(A::CuArray{T, 3}, B::CuVector{T}) where T\n @launchkernel CUDADevice() 256 length(B) kernel_f(A, B)\nend\n\nA = CuArray(randn(ComplexF32, 10, 10, 100))\nB = CUDA.zeros(ComplexF32, 100)\nA_out, B_out = batched_tr!(A, B)\n# put random values in the gradient field of B\ngrad_B = CuArray(randn(ComplexF32, 100))\nA_with_g, B_with_g = (~batched_tr!)(GVar(A_out), GVar(B_out, grad_B))\n# will see nonzero gradients in complex diagonal parts of A\ngrad_A = grad(A_with_g |> Array)\n" }, { "path": "examples/besselj.jl", "content": "# # Bessel function\n# An Bessel function of the first kind of order ``\\nu`` can be computed using Taylor expansion\n\n# ```math\n# J_\\nu(z) = \\sum\\limits_{n=0}^{\\infty} \\frac{(z/2)^\\nu}{\\Gamma(k+1)\\Gamma(k+\\nu+1)} (-z^2/4)^{n}\n# ```\n\n# where ``\\Gamma(n) = (n-1)!`` is the Gamma function. One can compute the accumulated item iteratively as ``s_n = -\\frac{z^2}{4} s_{n-1}``.\n\nusing NiLang, NiLang.AD\nusing ForwardDiff: Dual\n\n# Since we need to use logarithmic numbers to handle the sequential mutiplication.\n# Let's first add patch about the conversion between `ULogarithmic` and `Dual` number.\nfunction Base.convert(::Type{Dual{T,V,N}}, x::ULogarithmic) where {T,V,N}\n\tDual{T,V,N}(exp(x.log))\nend\n\nfunction Base.exp(::Type{ULogarithmic{Dual{T,V,N}}}, d::Dual) where {T,V,N}\n invoke(Base.exp, Tuple{Type{ULogarithmic{T}}, T} where T<:Real, ULogarithmic{Dual{T,V,N}}, d)\nend\n\n@i function ibesselj(y!::T, ν, z::T; atol=1e-8) where T\n\tif z == 0\n\t\tif v == 0\n\t\t\tout! += 1\n\t\tend\n\telse\n\t\t@routine @invcheckoff begin\n\t\t\tk ← 0\n\t\t\t@ones ULogarithmic{T} lz halfz halfz_power_2 s\n\t\t\t@zeros T out_anc\n\t\t\tlz *= convert(z)\n\t\t\thalfz *= lz / 2\n\t\t\thalfz_power_2 *= halfz ^ 2\n\t\t\t## s *= (z/2)^ν/ factorial(ν)\n\t\t\ts *= halfz ^ ν\n\t\t\tfor i=1:ν\n\t\t\t\ts /= i\n\t\t\tend\n\t\t\tout_anc += convert(s)\n\t\t\t@from k==0 while s.log > -25 # upto precision e^-25\n\t\t\t\tk += 1\n\t\t\t\t## s *= 1 / k / (k+ν) * (z/2)^2\n\t\t\t\ts *= halfz_power_2 / (@const k*(k+ν))\n\t\t\t\tif k%2 == 0\n\t\t\t\t\tout_anc += convert(s)\n\t\t\t\telse\n\t\t\t\t\tout_anc -= convert(s)\n\t\t\t\tend\n\t\t\tend\n\t\tend\n\t\ty! += out_anc\n\t\t~@routine\n\tend\nend\n\n# To obtain gradients, one call **Grad(ibesselj)**\n\ny, x = 0.0, 1.0\nGrad(ibesselj)(Val(1), y, 2, x)\n\n# Here, **Grad(ibesselj)** is a callable instance of type **Grad{typeof(ibesselj)}}**.\n# The first parameter `Val(1)` indicates the first argument is the loss.\n\n# To obtain second order gradients, one can Feed dual numbers to this gradient function.\n_, hxy, _, hxx = Grad(ibesselj)(Val(1), Dual(y, zero(y)), 2, Dual(x, one(x)))\nprintln(\"The hessian dy^2/dx^2 is $(grad(hxx).partials[1])\")\n\n# Here, the gradient field is a Dual number, it has a field partials that stores the derivative with respect to `x`.\n# This is the Hessian that we need.\n\n# ## CUDA programming\n# The AD in NiLang avoids most heap allocation, so that it is able to execute on a GPU device\n# We suggest using [KernelAbstraction](https://github.com/JuliaGPU/KernelAbstractions.jl), it provides compatibility between CPU and GPU.\n# To execute the above function on GPU, we need only 11 lines of code.\n\n# ```julia\n# using CUDA, GPUArrays, KernelAbstractions\n#\n# @i @kernel function bessel_kernel(out!, v, z)\n# @invcheckoff i ← @index(Global)\n# ibesselj(out![i], v, z[i])\n# @invcheckoff i → @index(Global)\n# end\n# ```\n\n# We have a macro support to KernelAbstraction in NiLang.\n# So it is possible to launch directly like.\n# ```julia\n# @i function befunc(out!, v::Integer, z)\n# @launchkernel CUDADevice() 256 length(out!) bessel_kernel(out!, v, z)\n# end\n# ```\n\n# It is equivalent to call\n# ```julia\n# (~bessel_kernel)(CUDADevice(), 256)(out!, v, z; ndrange=length(out!))\n# ```\n# But it will execute the job eagerly for you.\n# We will consider better support in the future.\n\n# Except it is reversible\n# ```julia repl\n# julia> @code_reverse @launchkernel CUDA() 256 length(out!) bessel_kernel(out!, v, z)\n# :(#= REPL[4]:1 =# @launchkernel CUDA() 256 length(out!) (~bessel_kernel)(out!, v, z))\n# ```\n\n# To test this function, we first define input parameters `a` and output `out!`\n# ```julia\n# a = CuArray(rand(128))\n# out! = CuArray(zeros(128))\n# ```\n\n# We wrap the output with a randomly initialized gradient field, suppose we get the gradients from a virtual loss function.\n# Also, we need to initialize an empty gradient field for elements in input cuda tensor `a`.\n# ```julia\n# out! = ibesselj(out!, 2, GVar.(a))[1]\n# out_g! = GVar.(out!, CuArray(randn(128)))\n# ```\n\n# Call the inverse program, the multiple dispatch will drive you to the goal.\n# ```julia\n# (~ibesselj)(out_g!, 2, GVar.(a))\n# ```\n\n# You will get CUDA arrays with `GVar` elements as output, their gradient fields are what you want.\n# Cheers! Now you have a adjoint mode differentiable CUDA kernel.\n\n# ## Benchmark\n# We have different source to souce automatic differention implementations of the first type Bessel function ``J_2(1.0)`` benchmarked and show the results below.\n#\n#\n# | Package | Tangent/Adjoint | ``T_{\\rm min}``/ns | Space/KB |\n# | --------- | --------------- | ------------------- | --------- |\n# | Julia | - | 22 | 0 |\n# | NiLang | - | 59 | 0 |\n# | ForwardDiff | Tangent | 35 | 0 |\n# | Manual | Adjoint | 83 | 0 |\n# | NiLang.AD | Adjoint | 213 | 0 |\n# | NiLang.AD (GPU) | Adjoint | 1.4 | 0 |\n# | Zygote | Adjoint | 31201 | 13.47 |\n# | Tapenade | Adjoint | ? | ? |\n\n# Julia is the CPU time used for running the irreversible forward program, is the baseline of benchmarking.\n# NiLang is the reversible implementation, it is 2.7 times slower than its irreversible counterpart. Here, we have remove the reversibility check.\n# ForwardDiff gives the best performance because it is designed for functions with single input.\n# It is even faster than manually derived gradients\n# ```math\n# \\frac{\\partial J_{\\nu}(z)}{\\partial z} = \\frac{J_{\\nu-1} - J_{\\nu+1}}{2}\n# ```\n# NiLang.AD is the reversible differential programming implementation, it considers only the backward pass.\n# The benchmark of its GPU version is estimated on Nvidia Titan V by broadcasting the gradient function on CUDA array of size ``2^17`` and take average.\n# The Zygote benchmark considers both forward pass and backward pass.\n# Tapenade is not yet ready.\n" }, { "path": "examples/boxmuller.jl", "content": "# # Box-Muller method to Generate normal distribution\nusing NiLang\n\n# In this tutorial, we introduce using Box-Muller method to transform a uniform distribution to a normal distribution.\n# The transformation and inverse transformation of `Box-Muller` method could be found in\n# [this blog](https://mathworld.wolfram.com/Box-MullerTransformation.html)\n@i function boxmuller(x::T, y::T) where T\n @routine @invcheckoff begin\n @zeros T θ logx _2logx\n θ += 2π * y\n logx += log(x)\n _2logx += -2 * logx\n end\n\n ## store results\n z1 ← zero(T)\n z2 ← zero(T)\n z1 += _2logx ^ 0.5\n ROT(z1, z2, θ)\n ~@routine\n\n SWAP(x, z1)\n SWAP(y, z2)\n\n ## arithmetic uncomputing: recomputing the original values of `x` and `y` to deallocate z1 and z2\n @routine @invcheckoff begin\n @zeros T at sq _halfsq\n at += atan(y, x)\n if (y < 0, ~)\n at += T(2π)\n end\n sq += x ^ 2\n sq += y ^ 2\n _halfsq -= sq / 2\n end\n z1 -= exp(_halfsq)\n z2 -= at / (2π)\n @invcheckoff z1 → zero(T)\n @invcheckoff z2 → zero(T)\n ~@routine\nend\n\n# One may wonder why this implementation is so long,\n# should't NiLang generate the inverse for user?\n# The fact is, although Box-Muller is arithmetically reversible.\n# It is not finite precision reversible.\n# Hence we need to \"uncompute\" it manually,\n# this trick may introduce reversibility error.\n\nusing Plots\nN = 5000\nx = rand(2*N)\n\nPlots.histogram(x, bins = -3:0.1:3, label=\"uniform\",\n legendfontsize=16, xtickfontsize=16, ytickfontsize=16)\n\n# forward\n@instr boxmuller.(x[1:N], x[N+1:end])\nPlots.histogram(x, bins = -3:0.1:3, label=\"normal\",\n legendfontsize=16, xtickfontsize=16, ytickfontsize=16)\n\n# backward\n@instr (~boxmuller).(x[1:N], x[N+1:end])\nPlots.histogram(x, bins = -3:0.1:3, label=\"uniform\",\n legendfontsize=16, xtickfontsize=16, ytickfontsize=16)\n\n# ## Check the probability distribution function\nusing LinearAlgebra, Test\n\nnormalpdf(x) = sqrt(1/2π)*exp(-x^2/2)\n\n# obtain `log(abs(det(jacobians)))`\n@i function f(x::Vector)\n boxmuller(x[1], x[2])\nend\njac = NiLang.AD.jacobian(f, [0.5, 0.5], iin=1)\nladj = log(abs(det(jac)))\n\n# check if it matches the `log(p/q)`.\nz1, z2 = boxmuller(0.5, 0.5)\n@test ladj ≈ log(1.0 / (normalpdf(z1) * normalpdf(z2)))\n\n# ## To obtaining Jacobian - a simpler approach\n# We can define a function that exactly reversible from the instruction level,\n# but costs more space for storing output.\n@i function boxmuller2(x1::T, x2::T, z1::T, z2::T) where T\n @routine @invcheckoff begin\n @zeros T θ logx _2logx\n\n θ += 2π * x2\n logx += log(x1)\n _2logx += -2 * logx\n end\n\n ## store results\n z1 += _2logx ^ 0.5\n ROT(z1, z2, θ)\n ~@routine\nend\n\n# However, this is not a bijector from that maps `x` to `z`,\n# because computing the backward just erases the content in `z`.\n# However, this function can be used to obtain `log(abs(det(jacobians)))`\n@i function f2(x::Vector, z::Vector)\n boxmuller2(x[1], x[2], z[1], z[2])\nend\njac = NiLang.AD.jacobian(f2, [0.5, 0.5], [0.0, 0.0], iin=1, iout=2)\nladj = log(abs(det(jac)))\n\n# check if it matches the `log(p/q)`.\n_, _, z1, z2 = boxmuller2(0.5, 0.5, 0.0, 0.0)\n@test ladj ≈ log(1.0 / (normalpdf(z1) * normalpdf(z2)))\n" }, { "path": "examples/fft.jl", "content": "# https://rosettacode.org/wiki/Fast_Fourier_transform#Fortran\n# In place Cooley-Tukey FFT\nfunction fft!(x::AbstractVector{T}) where T\n N = length(x)\n @inbounds if N <= 1\n return x\n elseif N == 2\n t = x[2]\n oi = x[1]\n x[1] = oi + t\n x[2] = oi - t\n return x\n end\n \n # divide\n odd = x[1:2:N]\n even = x[2:2:N]\n \n # conquer\n fft!(odd)\n fft!(even)\n \n # combine\n @inbounds for i=1:N÷2\n t = exp(T(-2im*π*(i-1)/N)) * even[i]\n oi = odd[i]\n x[i] = oi + t\n x[i+N÷2] = oi - t\n end\n return x\nend\n\nusing NiLang\n@i function i_fft!(x::AbstractVector{T}) where T\n @routine @invcheckoff N ← length(x)\n @safe @assert N%2 == 0\n @invcheckoff @inbounds if N <= 1\n elseif N == 2\n HADAMARD(x[1].re, x[2].re)\n HADAMARD(x[1].im, x[2].im)\n else\n # devide and conquer\n i_fft!(x[1:2:N])\n i_fft!(x[2:2:N])\n\n x2 ← zeros(T, N)\n for i=1:N÷2\n x2[i] += x[2i-1]\n x2[i+N÷2] += x[2i]\n end\n for i=1:N\n SWAP(x[i], x2[i])\n end\n for i=1:N÷2\n x2[2i-1] -= x[i]\n x2[2i] -= x[i+N÷2]\n end\n # combine\n for i=1:N÷2\n @routine θ ← -2*π*(i-1)/N\n ROT(x[i+N÷2].re, x[i+N÷2].im, θ)\n HADAMARD(x[i].re, x[i+N÷2].re)\n HADAMARD(x[i].im, x[i+N÷2].im)\n ~@routine\n end\n x2 → zeros(T, N)\n end\n ~@routine\nend\n\nusing Test, FFTW\n@testset \"fft\" begin\n x = randn(ComplexF64, 64)\n @test fft!(copy(x)) ≈ FFTW.fft(x)\n @test i_fft!(copy(x)) .* sqrt(length(x)) ≈ FFTW.fft(x)\nend" }, { "path": "examples/fib.jl", "content": "# # Computing Fibonacci Numbers\n# The following is an example that everyone likes, computing Fibonacci number recursively.\nusing NiLang\n\n@i function rfib(out!, n::T) where T\n @routine begin\n n1 ← zero(T)\n n2 ← zero(T)\n n1 += n - 1\n n2 += n - 2\n end\n if (value(n) <= 2, ~)\n out! += 1\n else\n rfib(out!, n1)\n rfib(out!, n2)\n end\n ~@routine\nend\n\n# The time complexity of this recursive algorithm is exponential to input `n`. It is also possible to write a reversible linear time with for loops.\n# A slightly non-trivial task is computing the first Fibonacci number that greater or equal to a certain number `z`, where a `while` statement is required.\n\n@i function rfibn(n!, z)\n @safe @assert n! == 0\n out ← 0\n rfib(out, n!)\n @from n! == 0 while out < z\n ~rfib(out, n!)\n n! += 1\n rfib(out, n!)\n end\n ~rfib(out, n!)\n out → 0\nend\n\n# In this example, the postcondition `n!=0` in the `while` statement is false before entering the loop, and it becomes true in later iterations. In the reverse program, the `while` statement stops at `n==0`.\n# If executed correctly, a user will see the following result.\n\nrfib(0, 10)\n\n# compute which the first Fibonacci number greater than 100.\n\nrfibn(0, 100)\n\n# and uncompute\n\n(~rfibn)(rfibn(0, 100)...)\n\n# This example shows how an addition postcondition provided by the user can help to reverse a control flow without caching controls.\n" }, { "path": "examples/fixedlog.jl", "content": "using FixedPointNumbers, Test\n\n\"\"\"\nReference\n-------------------\n\n[1] C. S. Turner, \"A Fast Binary Logarithm Algorithm\", IEEE Signal\n Processing Mag., pp. 124,140, Sep. 2010.\n\"\"\"\nfunction log2fix(x::Fixed{T, P}) where {T, P}\n PREC = UInt(P)\n x.i == 0 && return typemin(T) # represents negative infinity\n\n y = zero(T)\n xi = x.i\n while xi < 1 << PREC\n xi <<= 1\n y -= T(1) << PREC\n end\n\n while xi >= 2 << PREC\n xi >>= 1\n y += T(1) << PREC\n end\n\n z = xi\n b = T(1) << (PREC - UInt(1))\n for i = 1:P\n temp = Base.widemul(z, z) >> PREC\n z = T(temp)\n if z >= T(2) << PREC\n z >>= 1\n y += b\n end\n b >>= 1\n end\n\n return Fixed{T,PREC}(y, nothing)\nend\n\n@test log2fix(Fixed{Int, 43}(2^1.24)) ≈ 1.24\n" }, { "path": "examples/lax_wendroff.jl", "content": "\"\"\"\nsolve the 1D linear advection equation\n```math\n∂q/∂t=−u∂q/∂x\n```\nin a periodic domain, where ``q`` is the quantity being advected,\n``t`` is time, ``x`` is the spatial coordinate and ``u`` is the velocity,\nwhich is constant with ``x``. \n\"\"\"\nfunction lax_wendroff!(nt::Int, c, q_init::AbstractVector{T}, q::AbstractVector{T}) where T\n nx = length(q)\n flux = zeros(T, nx-1) # Fluxes between boxes\n @inbounds for i=1:nx\n q[i] = q_init[i] # Initialize q\n end\n @inbounds for j=1:nt # Main loop in time\n for i=1:nx-1\n flux[i] = 0.5*c*(q[i]+q[i+1]+c*(q[i]-q[i+1]))\n end\n for i=2:nx-1\n q[i] += flux[i-1]-flux[i]\n end\n q[1] = q[nx-1]; q[nx] = q[2] # Treat boundary conditions\n end\n return q\nend\n\nusing Random\nRandom.seed!(2)\nq_init = randn(100)\nq = zeros(100)\n@show lax_wendroff!(2000, 1.0, q_init, zero(q_init))\nusing BenchmarkTools\n@benchmark lax_wendroff!(2000, 1.0, $q_init, x) setup=(x=zero(q_init))\n@time lax_wendroff!(2000, 1.0, q_init, q)\n\nusing NiLang\n@i function i_lax_wendroff!(nt::Int, c, q_init::AbstractVector{T}, q::AbstractVector{T},\n cache::AbstractMatrix{T}) where T\n nx ← length(q)\n @inbounds for i=1:nx\n q[i] += q_init[i] # Initialize q\n end\n @inbounds for j=1:nt # Main loop in time\n for i=1:nx-1\n @routine begin\n @zeros T anc1 anc2 anc3\n anc1 += 0.5 * c\n anc2 += q[i] - q[i+1]\n anc3 += q[i] + q[i+1]\n anc3 += c * anc2\n end\n cache[i,j] += anc1 * anc3\n ~@routine\n end\n for i=2:nx-1\n q[i] += cache[i-1,j]-cache[i,j]\n end\n # Treat boundary conditions\n cache[nx,j] += q[nx-1]\n SWAP(q[1], cache[nx,j])\n cache[nx+1,j] += q[2]\n SWAP(q[nx], cache[nx+1,j])\n end\n nx → length(q)\nend\nnt = 2000\ni_lax_wendroff!(nt, 1.0, q_init, zero(q_init), zeros(length(q_init)+1,nt))\n" }, { "path": "examples/lognumber.jl", "content": "# # Logarithmic number system\n\n# Computing basic functions like `power`, `exp` and `besselj` is not trivial for reversible programming.\n# There is no efficient constant memory algorithm using pure fixed point numbers only.\n# For example, to compute `x ^ n` reversiblly with fixed point numbers,\n# we need to allocate a vector of size $O(n)$.\n# With logarithmic numbers, the above computation is straight forward.\n\nusing LogarithmicNumbers\nusing NiLang, NiLang.AD\nusing FixedPointNumbers\n\n@i function i_power(y::T, x::T, n::Int) where T\n if !iszero(x)\n @routine begin\n lx ← one(ULogarithmic{T})\n ly ← one(ULogarithmic{T})\n ## convert `x` to a logarithmic number\n ## Here, `*=` is reversible for log numbers\n if x > 0\n lx *= convert(x)\n else\n lx *= convert(-x)\n end\n for i=1:n\n ly *= lx\n end\n end\n\n ## convert back to fixed point numbers\n y += convert(ly)\n if x < 0 && n%2 == 1\n NEG(y)\n end\n\n ~@routine\n end\nend\n\n# To check the function\ni_power(Fixed43(0.0), Fixed43(0.4), 3)\n\n# ## `exp` function as an example\n# The following example computes `exp(x)`.\n\n@i function i_exp(y!::T, x::T) where T<:Union{Fixed, GVar{<:Fixed}}\n @invcheckoff begin\n @routine begin\n s ← one(ULogarithmic{T})\n lx ← one(ULogarithmic{T})\n k ← 0\n end\n lx *= convert(x)\n y! += convert(s)\n @from k==0 while s.log > -20\n k += 1\n s *= lx / k\n y! += convert(s)\n end\n ~(@from k==0 while s.log > -20\n k += 1\n s *= x / k\n end)\n lx /= convert(x)\n ~@routine\n end\nend\n\nx = Fixed43(3.5)\n\n# We can check the reversibility\nout, _ = i_exp(Fixed43(0.0), x)\n@assert out ≈ exp(3.5)\n\n# Computing the gradients\n_, gx = NiLang.AD.gradient(Val(1), i_exp, (Fixed43(0.0), x))\n@assert gx ≈ exp(3.5)\n" }, { "path": "examples/nice.jl", "content": "# # NICE network\n# For the definition of this network and concepts of normalizing flow,\n# please refer this nice blog: https://lilianweng.github.io/lil-log/2018/10/13/flow-based-deep-generative-models.html,\n# and the pytorch notebook: https://github.com/GiggleLiu/marburg/blob/master/notebooks/nice.ipynb\n\nusing NiLang, NiLang.AD\nusing LinearAlgebra\nusing DelimitedFiles\nusing Plots\n\n# `include` the optimizer, you can find it under the `Adam.jl` file in the `examples/` folder.\ninclude(NiLang.project_relative_path(\"examples\", \"Adam.jl\"))\n\n\n# ## Model definition\n# First, define the single layer transformation and its behavior under `GVar` - the gradient wrapper.\nstruct NiceLayer{T}\n W1::Matrix{T}\n b1::Vector{T}\n W2::Matrix{T}\n b2::Vector{T}\n y1::Vector{T}\n y1a::Vector{T}\nend\n\n\"\"\"Apply a single NICE transformation.\"\"\"\n@i function nice_layer!(x::AbstractVector{T}, layer::NiceLayer{T},\n y!::AbstractVector{T}) where T\n @routine @invcheckoff begin\n i_affine!(layer.y1, layer.W1, layer.b1, x)\n @inbounds for i=1:length(layer.y1)\n if (layer.y1[i] > 0, ~)\n layer.y1a[i] += layer.y1[i]\n end\n end\n end\n i_affine!(y!, layer.W2, layer.b2, layer.y1a)\n ~@routine\n ## clean up accumulated rounding error, since this memory is reused.\n @safe layer.y1 .= zero(T)\nend\n\n# Here, in each layer, we use the information in `x` to update `y!`.\n# During computing, we use the `y1` and `y1a` fields of the network as ancilla space,\n# both of them can be uncomputed at the end of the function.\n# However, we need to erase small numbers to make sure the rounding error does not accumulate.\n\n# A nice network always transforms inputs reversibly.\n# We update one half of `x!` a time, so that input and output memory space do not clash.\nconst NiceNetwork{T} = Vector{NiceLayer{T}}\n\n\"\"\"Apply a the whole NICE network.\"\"\"\n@i function nice_network!(x!::AbstractVector{T}, network::NiceNetwork{T}) where T\n @invcheckoff for i=1:length(network)\n np ← length(x!)\n if (i%2==0, ~)\n @inbounds nice_layer!(x! |> subarray(np÷2+1:np), network[i], x! |> subarray(1:np÷2))\n else\n @inbounds nice_layer!(x! |> subarray(1:np÷2), network[i], x! |> subarray(np÷2+1:np))\n end\n np → length(x!)\n end\nend\n\nfunction random_nice_network(nparams::Int, nhidden::Int, nlayer::Int; scale=0.1)\n random_nice_network(Float64, nparams, nhidden, nlayer; scale=scale)\nend\n\nfunction random_nice_network(::Type{T}, nparams::Int, nhidden::Int, nlayer::Int; scale=0.1) where T\n nin = nparams÷2\n scale = T(scale)\n y1 = zeros(T, nhidden)\n NiceLayer{T}[NiceLayer(randn(T, nhidden, nin)*scale, randn(T, nhidden)*scale,\n randn(T, nin, nhidden)*scale, randn(T, nin)*scale, y1, zero(y1)) for _ = 1:nlayer]\nend\n\n# ## Parameter management\n\nnparameters(n::NiceLayer) = length(n.W1) + length(n.b1) + length(n.W2) + length(n.b2)\nnparameters(n::NiceNetwork) = sum(nparameters, n)\n\n\"\"\"collect parameters in the `layer` into a vector `out`.\"\"\"\nfunction collect_params!(out, layer::NiceLayer)\n a, b, c, d = length(layer.W1), length(layer.b1), length(layer.W2), length(layer.b2)\n out[1:a] .= vec(layer.W1)\n out[a+1:a+b] .= layer.b1\n out[a+b+1:a+b+c] .= vec(layer.W2)\n out[a+b+c+1:end] .= layer.b2\n return out\nend\n\n\"\"\"dispatch vectorized parameters `out` into the `layer`.\"\"\"\nfunction dispatch_params!(layer::NiceLayer, out)\n a, b, c, d = length(layer.W1), length(layer.b1), length(layer.W2), length(layer.b2)\n vec(layer.W1) .= out[1:a]\n layer.b1 .= out[a+1:a+b]\n vec(layer.W2) .= out[a+b+1:a+b+c]\n layer.b2 .= out[a+b+c+1:end]\n return layer\nend\n\nfunction collect_params(n::NiceNetwork{T}) where T\n out = zeros(T, nparameters(n))\n k = 0\n for layer in n\n np = nparameters(layer)\n collect_params!(view(out, k+1:k+np), layer)\n k += np\n end\n return out\nend\n\nfunction dispatch_params!(network::NiceNetwork, out)\n k = 0\n for layer in network\n np = nparameters(layer)\n dispatch_params!(layer, view(out, k+1:k+np))\n k += np\n end\n return network\nend\n\n# ## Loss function\n\n# To obtain the log-probability of a data.\n\n@i function logp!(out!::T, x!::AbstractVector{T}, network::NiceNetwork{T}) where T\n (~nice_network!)(x!, network)\n @invcheckoff for i = 1:length(x!)\n @routine begin\n xsq ← zero(T)\n @inbounds xsq += x![i]^2\n end\n out! -= 0.5 * xsq\n ~@routine\n end\nend\n\n# The negative-log-likelihood loss function\n\n@i function nice_nll!(out!::T, cum!::T, xs!::Matrix{T}, network::NiceNetwork{T}) where T\n @invcheckoff for i=1:size(xs!, 2)\n @inbounds logp!(cum!, xs! |> subarray(:,i), network)\n end\n out! -= cum!/(@const size(xs!, 2))\nend\n\n# ## Training\n\nfunction train(x_data, model; num_epochs = 800)\n num_vars = size(x_data, 1)\n params = collect_params(model)\n optimizer = Adam(; lr=0.01)\n for epoch = 1:num_epochs\n loss, a, b, c = nice_nll!(0.0, 0.0, copy(x_data), model)\n if epoch % 50 == 1\n println(\"epoch = $epoch, loss = $loss\")\n display(showmodel(x_data, model))\n end\n _, _, _, gmodel = (~nice_nll!)(GVar(loss, 1.0), GVar(a), GVar(b), GVar(c))\n g = grad.(collect_params(gmodel))\n update!(params, grad.(collect_params(gmodel)), optimizer)\n dispatch_params!(model, params)\n end\n return model\nend\n\nfunction showmodel(x_data, model; nsamples=2000)\n scatter(x_data[1,1:nsamples], x_data[2,1:nsamples]; xlims=(-5,5), ylims=(-5,5))\n zs = randn(2, nsamples)\n for i=1:nsamples\n nice_network!(view(zs, :, i), model)\n end\n scatter!(zs[1,:], zs[2,:])\nend\n\n# you can find the training data in `examples/` folder\nx_data = Matrix(readdlm(NiLang.project_relative_path(\"examples\", \"train.dat\"))')\n\nimport Random; Random.seed!(22)\nmodel = random_nice_network(Float64, size(x_data, 1), 10, 4; scale=0.1)\n\n# Before training, the distribution looks like\n# ![before](../asset/nice_before.png)\nmodel = train(x_data, model; num_epochs=800)\n\n# After training, the distribution looks like\n# ![before](../asset/nice_after.png)\n" }, { "path": "examples/nice_test.jl", "content": "# bijectivity check\nusing Test\ninclude(\"nice.jl\")\n\n@testset \"nice\" begin\n num_vars = 4\n model = random_nice_network(num_vars, 10, 3)\n z = randn(num_vars)\n x, _ = nice_network!(z, model)\n z_infer, _ = (~nice_network!)(x, model)\n @test z_infer ≈ z\n newparams = randn(nparameters(model))\n dispatch_params!(model, newparams)\n @test collect_params(model) ≈ newparams\n @test check_inv(logp!, (0.0, x, model))\nend\n\n@testset \"nice logp\" begin\n z1 = [0.5, 0.2]\n z2 = [-0.5, 1.2]\n model = random_nice_network(2, 10, 4)\n x1 = nice_network!(copy(z1), model)[1]\n x2 = nice_network!(copy(z2), model)[1]\n p1 = logp!(0.0, copy(x1), model)[1]\n p2 = logp!(0.0, copy(x2), model)[1]\n pz1 = exp(-sum(abs2, z1)/2)\n pz2 = exp(-sum(abs2, z2)/2)\n @test exp(p1 - p2) ≈ pz1/pz2\n @test nice_nll!(0.0, 0.0, hcat(x1, x2), model)[1] ≈ -log(pz1 * pz2)/2\n\n xs = hcat(x1, x2)\n gmodel = Grad(nice_nll!)(Val(1), 0.0, 0.0, copy(xs), model)[end]\n\n for i=1:10, j=1:4\n model[j].W2[i] -= 1e-4\n a = nice_nll!(0.0, 0.0, copy(xs), model)[1]\n model[j].W2[i] += 2e-4\n b = nice_nll!(0.0, 0.0, copy(xs), model)[1]\n model[j].W2[i] -= 1e-4\n ng = (b-a)/2e-4\n @test gmodel[j].W2[i].g ≈ ng\n end\n\n for i=1:10, j=1:4\n model[j].W1[i] -= 1e-4\n a = nice_nll!(0.0, 0.0, copy(xs), model)[1]\n model[j].W1[i] += 2e-4\n b = nice_nll!(0.0, 0.0, copy(xs), model)[1]\n model[j].W1[i] -= 1e-4\n ng = (b-a)/2e-4\n @test gmodel[j].W1[i].g ≈ ng\n end\nend\n\n\n" }, { "path": "examples/port_chainrules.jl", "content": "# # [How to port NiLang to ChainRules](@id port_chainrules)\n#\n# In [How to port NiLang to Zygote](@ref port_zygote) we showed the way to insert Nilang-based\n# gradient as Zygote's pullback/adjoint. Given that [ChainRules](https://github.com/JuliaDiff/ChainRules.jl)\n# is now the core of many AD packages including Zygote, extending `ChainRules.rrule` with Nilang\n# does the same job, except that it affects all ChainRules-based AD packages and not just Zygote.\n#\n# We'll use the same example as [How to port NiLang to Zygote](@ref port_zygote), so you might need\n# to restart your Julia to get a fresh environment.\n\nusing NiLang, NiLang.AD, Zygote, ChainRules\n\n# Let's start from the Julia native implementation of `norm2` function.\nfunction norm2(x::AbstractArray{T}) where T\n out = zero(T)\n for i=1:length(x)\n @inbounds out += x[i]^2\n end\n return out\nend\n\n# Zygote is able to generate correct dual function, i.e., gradients, but much slower than the primal\n# function `norm2`\nusing BenchmarkTools\nx = randn(1000);\noriginal_grad = norm2'(x)\n@benchmark norm2'($x) seconds=1\n\n# The primal function is\n@benchmark norm2($x) seconds=1\n\n# Then we have the reversible implementation\n@i function r_norm2(out::T, x::AbstractArray{T}) where T\n for i=1:length(x)\n @inbounds out += x[i]^2\n end\nend\n\n# The gradient generated by NiLang is much faster, which is comparable to the forward program\n@benchmark (~r_norm2)(GVar($(norm2(x)), 1.0), $(GVar(x))) seconds=1\n\n# By defining our custom `rrule` using Nilang's gradient implementation, `Zygote` automaticallly\n# gets boosted because it internally uses the available ChainRules ruleset.\n# Here we need to create a new symbol here because otherwise Zygote will still use the\n# previously generated slow implementation.\nnorm2_faster(x) = norm2(x)\nfunction ChainRules.rrule(::typeof(norm2_faster), x::AbstractArray{T}) where T\n out = norm2_faster(x)\n function pullback(ȳ)\n ChainRules.NoTangent(), grad((~r_norm2)(GVar(out, ȳ), GVar(x))[2])\n end\n out, pullback\nend\n@assert norm2_faster'(x) ≈ original_grad\n\n# See, much faster\n@benchmark norm2_faster'(x) seconds=1\n" }, { "path": "examples/port_zygote.jl", "content": "# # [How to port NiLang to Zygote](@id port_zygote)\n#\n# In this demo we'll show how to insert NiLang's gradient implementation to boost Zygote's gradient.\n# A similar demo for ChainRules can be found in [How to port NiLang to ChainRules](@ref port_chainrules).\n\nusing NiLang, NiLang.AD, Zygote\n\n# Let's start from the Julia native implementation of `norm2` function.\nfunction norm2(x::AbstractArray{T}) where T\n out = zero(T)\n for i=1:length(x)\n @inbounds out += x[i]^2\n end\n return out\nend\n\n# Zygote is able to generate correct dual function, i.e., gradients, but much slower than the primal\n# function `norm2`\nusing BenchmarkTools\nx = randn(1000);\noriginal_grad = norm2'(x)\n@benchmark norm2'($x) seconds=1\n\n# The primal function is\n@benchmark norm2($x) seconds=1\n\n# Then we have the reversible implementation\n@i function r_norm2(out::T, x::AbstractArray{T}) where T\n for i=1:length(x)\n @inbounds out += x[i]^2\n end\nend\n\n# The gradient generated by NiLang is much faster, which is comparable to the forward program\n@benchmark (~r_norm2)(GVar($(norm2(x)), 1.0), $(GVar(x))) seconds=1\n\n# to enjoy the speed of `NiLang` in `Zygote`, just bind the adjoint rule\nZygote.@adjoint function norm2(x::AbstractArray{T}) where T\n out = norm2(x)\n out, δy -> (grad((~r_norm2)(GVar(out, δy), GVar(x))[2]),)\nend\n@assert norm2'(x) ≈ original_grad\n\n# See, much faster\n@benchmark norm2'(x) seconds=1\n" }, { "path": "examples/pyramid.jl", "content": "# # Pyramid example\n#\n# This is the Pyramid example in the book \"Evaluate Derivatives\", Sec. 3.5.\n\nusing NiLang, NiLang.AD\n\n@i function pyramid!(y!, v!, x::AbstractVector{T}) where T\n @safe @assert size(v!,2) == size(v!,1) == length(x)\n @invcheckoff @inbounds for j=1:length(x)\n v![1,j] += x[j]\n end\n @invcheckoff @inbounds for i=1:size(v!,1)-1\n for j=1:size(v!,2)-i\n @routine begin\n @zeros T c s\n c += cos(v![i,j+1])\n s += sin(v![i,j])\n end\n v![i+1,j] += c * s\n ~@routine\n end\n end\n y! += v![end,1]\nend\n\nx = randn(20)\npyramid!(0.0, zeros(20, 20), x)\n\n# Let's benchmark the gradient of the pyramid function\nusing BenchmarkTools\n@benchmark gradient(Val(1), pyramid!, (0.0, zeros(20, 20), $x))\n" }, { "path": "examples/qr.jl", "content": "# # A simple QR decomposition\n\n# ## Functions used in this example\n\nusing NiLang, NiLang.AD, Test\n\n# ## The QR decomposition\n# Let us consider a naive implementation of QR decomposition from scratch.\n# This implementation is just a proof of principle which does not consider reorthogonalization and other practical issues.\n\n@i function qr(Q, R, A::Matrix{T}) where T\n @routine begin\n anc_norm ← zero(T)\n anc_dot ← zeros(T, size(A,2))\n ri ← zeros(T, size(A,1))\n end\n for col = 1:size(A, 1)\n ri .+= A[:,col]\n for precol = 1:col-1\n i_dot(anc_dot[precol], Q[:,precol], ri)\n R[precol,col] += anc_dot[precol]\n for row = 1:size(Q,1)\n ri[row] -=\n anc_dot[precol] * Q[row, precol]\n end\n end\n i_norm2(anc_norm, ri)\n\n R[col, col] += anc_norm^0.5\n for row = 1:size(Q,1)\n Q[row,col] += ri[row] / R[col, col]\n end\n\n ~begin\n ri .+= A[:,col]\n for precol = 1:col-1\n i_dot(anc_dot[precol], Q[:,precol], ri)\n for row = 1:size(Q,1)\n ri[row] -= anc_dot[precol] *\n Q[row, precol]\n end\n end\n i_norm2(anc_norm, ri)\n end\n end\n ~@routine\nend\n\n# Here, in order to avoid frequent uncomputing, we allocate ancillas `ri` and `anc_dot` as vectors.\n# The expression in `~` is used to uncompute `ri`, `anc_dot` and `anc_norm`.\n# `i_dot` and `i_norm2` are reversible functions to compute dot product and vector norm.\n# One can quickly check the correctness of the gradient function\n\nA = randn(4,4)\nq, r = zero(A), zero(A)\n@i function test1(out, q, r, A)\n qr(q, r, A)\n i_sum(out, q)\nend\n\n@test check_grad(test1, (0.0, q, r, A); iloss=1)\n\n# Here, the loss function `test1` is defined as the sum of the output unitary matrix `q`.\n# The `check_grad` function is a gradient checker function defined in module `NiLang.AD`.\n" }, { "path": "examples/realnvp.jl", "content": "# # RealNVP network\n# For the definition of this network and concepts of normalizing flow,\n# please refer this realnvp blog: https://lilianweng.github.io/lil-log/2018/10/13/flow-based-deep-generative-models.html,\n# and the pytorch notebook: https://github.com/GiggleLiu/marburg/blob/master/solutions/realnvp.ipynb\n\nusing NiLang, NiLang.AD\nusing LinearAlgebra\nusing DelimitedFiles\nusing Plots\n\n# `include` the optimizer, you can find it under the `Adam.jl` file in the `examples/` folder.\ninclude(NiLang.project_relative_path(\"examples\", \"Adam.jl\"))\n\n\n# ## Model definition\n# First, define the single layer transformation and its behavior under `GVar` - the gradient wrapper.\nstruct RealNVPLayer{T}\n ## transform network\n W1::Matrix{T}\n b1::Vector{T}\n W2::Matrix{T}\n b2::Vector{T}\n y1::Vector{T}\n y1a::Vector{T}\n\n ## scaling network\n sW1::Matrix{T}\n sb1::Vector{T}\n sW2::Matrix{T}\n sb2::Vector{T}\n sy1::Vector{T}\n sy1a::Vector{T}\nend\n\n\"\"\"collect parameters in the `layer` into a vector `out`.\"\"\"\nfunction collect_params!(out, layer::RealNVPLayer)\n k=0\n for field in [:W1, :b1, :W2, :b2, :sW1, :sb1, :sW2, :sb2]\n v = getfield(layer, field)\n nv = length(v)\n out[k+1:k+nv] .= vec(v)\n k += nv\n end\n return out\nend\n\n\"\"\"dispatch vectorized parameters `out` into the `layer`.\"\"\"\nfunction dispatch_params!(layer::RealNVPLayer, out)\n k=0\n for field in [:W1, :b1, :W2, :b2, :sW1, :sb1, :sW2, :sb2]\n v = getfield(layer, field)\n nv = length(v)\n vec(v) .= out[k+1:k+nv]\n k += nv\n end\n return out\nend\n\nfunction nparameters(n::RealNVPLayer)\n sum(x->length(getfield(n, x)), [:W1, :b1, :W2, :b2, :sW1, :sb1, :sW2, :sb2])\nend\n\n# Then, we define `network` and how to access the parameters.\nconst RealNVP{T} = Vector{RealNVPLayer{T}}\n\nnparameters(n::RealNVP) = sum(nparameters, n)\n\nfunction collect_params(n::RealNVP{T}) where T\n out = zeros(T, nparameters(n))\n k = 0\n for layer in n\n np = nparameters(layer)\n collect_params!(view(out, k+1:k+np), layer)\n k += np\n end\n return out\nend\n\nfunction dispatch_params!(network::RealNVP, out)\n k = 0\n for layer in network\n np = nparameters(layer)\n dispatch_params!(layer, view(out, k+1:k+np))\n k += np\n end\n return network\nend\n\nfunction random_realnvp(nparams::Int, nhidden::Int, nhidden_s::Int, nlayer::Int; scale=0.1)\n random_realnvp(Float64, nparams, nhidden, nhidden_s::Int, nlayer; scale=scale)\nend\n\nfunction random_realnvp(::Type{T}, nparams::Int, nhidden::Int, nhidden_s::Int, nlayer::Int; scale=0.1) where T\n nin = nparams÷2\n scale = T(scale)\n y1 = zeros(T, nhidden)\n sy1 = zeros(T, nhidden_s)\n RealNVPLayer{T}[RealNVPLayer(\n randn(T, nhidden, nin)*scale, randn(T, nhidden)*scale,\n randn(T, nin, nhidden)*scale, randn(T, nin)*scale, y1, zero(y1),\n randn(T, nhidden_s, nin)*scale, randn(T, nhidden_s)*scale,\n randn(T, nin, nhidden_s)*scale, randn(T, nin)*scale, sy1, zero(sy1),\n ) for _ = 1:nlayer]\nend\n\n\n# ## Loss function\n#\n# In each layer, we use the information in `x` to update `y!`.\n# During computing, we use to vector type ancillas `y1` and `y1a`,\n# both of them can be uncomputed at the end of the function.\n\n@i function onelayer!(x::AbstractVector{T}, layer::RealNVPLayer{T},\n y!::AbstractVector{T}, logjacobian!::T; islast) where T\n @routine @invcheckoff begin\n ## scale network\n scale ← zero(y!)\n ytemp2 ← zero(y!)\n i_affine!(layer.sy1, layer.sW1, layer.sb1, x)\n @inbounds for i=1:length(layer.sy1)\n if (layer.sy1[i] > 0, ~)\n layer.sy1a[i] += layer.sy1[i]\n end\n end\n i_affine!(scale, layer.sW2, layer.sb2, layer.sy1a)\n\n ## transform network\n i_affine!(layer.y1, layer.W1, layer.b1, x)\n ## relu\n @inbounds for i=1:length(layer.y1)\n if (layer.y1[i] > 0, ~)\n layer.y1a[i] += layer.y1[i]\n end\n end\n end\n ## inplace multiply exp of scale! -- dangerous\n @inbounds @invcheckoff for i=1:length(scale)\n @routine begin\n expscale ← zero(T)\n tanhscale ← zero(T)\n if (islast, ~)\n tanhscale += tanh(scale[i])\n else\n tanhscale += scale[i]\n end\n expscale += exp(tanhscale)\n end\n logjacobian! += tanhscale\n ## inplace multiply!!!\n temp ← zero(T)\n temp += y![i] * expscale\n SWAP(temp, y![i])\n temp -= y![i] / expscale\n temp → zero(T)\n ~@routine\n end\n\n ## affine the transform layer\n i_affine!(y!, layer.W2, layer.b2, layer.y1a)\n ~@routine\n ## clean up accumulated rounding error, since this memory is reused.\n @safe layer.y1 .= zero(T)\n @safe layer.sy1 .= zero(T)\nend\n\n# A realnvp network always transforms inputs reversibly.\n# We update one half of `x!` a time, so that input and output memory space do not clash.\n@i function realnvp!(x!::AbstractVector{T}, network::RealNVP{T}, logjacobian!) where T\n @invcheckoff for i=1:length(network)\n np ← length(x!)\n if (i%2==0, ~)\n @inbounds onelayer!(x! |> subarray(np÷2+1:np), network[i], x! |> subarray(1:np÷2), logjacobian!; islast=i==length(network))\n else\n @inbounds onelayer!(x! |> subarray(1:np÷2), network[i], x! |> subarray(np÷2+1:np), logjacobian!; islast=i==length(network))\n end\n np → length(x!)\n end\nend\n\n# How to obtain the log-probability of a data.\n\n@i function logp!(out!::T, x!::AbstractVector{T}, network::RealNVP{T}) where T\n (~realnvp!)(x!, network, out!)\n @invcheckoff for i = 1:length(x!)\n @routine begin\n xsq ← zero(T)\n @inbounds xsq += x![i]^2\n end\n out! -= 0.5 * xsq\n ~@routine\n end\nend\n\n# The negative-log-likelihood loss function\n\n@i function nll_loss!(out!::T, cum!::T, xs!::Matrix{T}, network::RealNVP{T}) where T\n @invcheckoff for i=1:size(xs!, 2)\n @inbounds logp!(cum!, xs! |> subarray(:,i), network)\n end\n out! -= cum!/(@const size(xs!, 2))\nend\n\n# ## Training\n\nfunction train(x_data, model; num_epochs = 800)\n num_vars = size(x_data, 1)\n params = collect_params(model)\n optimizer = Adam(; lr=0.01)\n for epoch = 1:num_epochs\n loss, a, b, c = nll_loss!(0.0, 0.0, copy(x_data), model)\n if epoch % 50 == 1\n println(\"epoch = $epoch, loss = $loss\")\n display(showmodel(x_data, model))\n end\n _, _, _, gmodel = (~nll_loss!)(GVar(loss, 1.0), GVar(a), GVar(b), GVar(c))\n g = grad.(collect_params(gmodel))\n update!(params, grad.(collect_params(gmodel)), optimizer)\n dispatch_params!(model, params)\n end\n return model\nend\n\nfunction showmodel(x_data, model; nsamples=2000)\n scatter(x_data[1,1:nsamples], x_data[2,1:nsamples]; xlims=(-5,5), ylims=(-5,5))\n zs = randn(2, nsamples)\n for i=1:nsamples\n realnvp!(view(zs, :, i), model, 0.0)\n end\n scatter!(zs[1,:], zs[2,:])\nend\n\n# you can find the training data in `examples/` folder\nx_data = Matrix(readdlm(NiLang.project_relative_path(\"examples\", \"train.dat\"))')\n\nimport Random; Random.seed!(22)\nmodel = random_realnvp(Float64, size(x_data, 1), 10, 10, 4; scale=0.1)\n\n# Before training, the distribution looks like\n# ![before](../asset/nice_before.png)\nmodel = train(x_data, model; num_epochs=800)\n\n# After training, the distribution looks like\n# ![before](../asset/realnvp_after.png)\n" }, { "path": "examples/sparse.jl", "content": "# # Sparse matrices\n#\n# Source to source automatic differentiation is useful in differentiating sparse matrices. It is a well-known problem that sparse matrix operations can not benefit directly from generic backward rules for dense matrices because general rules do not keep the sparse structure.\n# In the following, we will show that reversible AD can differentiate the Frobenius dot product between two sparse matrices with the state-of-the-art performance. Here, the Frobenius dot product is defined as \\texttt{trace(A'B)}.\n# Its native Julia (irreversible) implementation is `SparseArrays.dot`.\n#\n# The following is a reversible counterpart\n\nusing NiLang, NiLang.AD\nusing SparseArrays\n\n@i function idot(r::T, A::SparseMatrixCSC{T},B::SparseMatrixCSC{T}) where {T}\n @routine begin\n m, n ← size(A)\n branch_keeper ← zeros(Bool, 2*m)\n end\n @safe size(B) == (m,n) || throw(DimensionMismatch(\"matrices must have the same dimensions\"))\n @invcheckoff @inbounds for j = 1:n\n @routine begin\n ia1 ← A.colptr[j]\n ib1 ← B.colptr[j]\n ia2 ← A.colptr[j+1]\n ib2 ← B.colptr[j+1]\n ia ← ia1\n ib ← ib1\n end\n @inbounds for i=1:ia2-ia1+ib2-ib1-1\n ra ← A.rowval[ia]\n rb ← B.rowval[ib]\n if (ra == rb, ~)\n r += A.nzval[ia]' * B.nzval[ib]\n end\n ## b move -> true, a move -> false\n branch_keeper[i] ⊻= @const ia == ia2-1 || ra > rb\n ra → A.rowval[ia]\n rb → B.rowval[ib]\n if (branch_keeper[i], ~)\n INC(ib)\n else\n INC(ia)\n end\n end\n ~@inbounds for i=1:ia2-ia1+ib2-ib1-1\n ## b move -> true, a move -> false\n branch_keeper[i] ⊻= @const ia == ia2-1 || A.rowval[ia] > B.rowval[ib]\n if (branch_keeper[i], ~)\n INC(ib)\n else\n INC(ia)\n end\n end\n ~@routine\n end\n ~@routine\nend\n\n# Here, the key point is using a \\texttt{branch\\_keeper} vector to cache branch decisions.\n\n# The time used for a native implementation is\n\nusing BenchmarkTools\na = sprand(1000, 1000, 0.01);\nb = sprand(1000, 1000, 0.01);\n@benchmark SparseArrays.dot($a, $b)\n\n# To compute the gradients, we wrap each matrix element with `GVar`, and send them to the reversible backward pass\n\nout! = SparseArrays.dot(a, b)\n@benchmark (~idot)($(GVar(out!, 1.0)),\n $(GVar.(a)), $(GVar.(b)))\n\n# The time used for computing backward pass is approximately 1.6 times Julia's native forward pass.\n# Here, we have turned off the reversibility check off to achieve better performance.\n# By writing sparse matrix multiplication and other sparse matrix operations reversibly,\n# we will have a differentiable sparse matrix library with proper performance.\n\n# See my another blog post for [reversible sparse matrix multiplication](https://nextjournal.com/giggle/how-to-write-a-program-differentiably).\n" }, { "path": "examples/unitary.jl", "content": "# # Unitary matrix operations without allocation\n# A unitary matrix features uniform eigenvalues and reversibility. It is widely used as an approach to ease the gradient exploding and vanishing problem and the memory wall problem.\n# One of the simplest ways to parametrize a unitary matrix is representing a unitary matrix as a product of two-level unitary operations. A real unitary matrix of size $N$ can be parametrized compactly by $N(N-1)/2$ rotation operations\n#\n# ```math\n# {\\rm ROT}(a!, b!, \\theta) = \\left(\\begin{matrix}\n# \\cos(\\theta) & - \\sin(\\theta)\\\\\n# \\sin(\\theta) & \\cos(\\theta)\n# \\end{matrix}\\right)\n# \\left(\\begin{matrix}\n# a!\\\\\n# b!\n# \\end{matrix}\\right),\n# ```\n#\n# where $\\theta$ is the rotation angle, `a!` and `b!` are target registers.\n\nusing NiLang, NiLang.AD\n\n@i function umm!(x!, θ)\n @safe @assert length(θ) ==\n length(x!)*(length(x!)-1)/2\n k ← 0\n for j=1:length(x!)\n for i=length(x!)-1:-1:j\n k += 1\n ROT(x![i], x![i+1], θ[k])\n end\n end\n\n k → length(θ)\nend\n\n# Here, the ancilla `k` is deallocated manually by specifying its value, because we know the loop size is $N(N-1)/2$.\n# We define the test functions in order to check gradients.\n\n@i function isum(out!, x::AbstractArray)\n for i=1:length(x)\n out! += x[i]\n end\nend\n\n@i function test!(out!, x!::Vector, θ::Vector)\n umm!(x!, θ)\n isum(out!, x!)\nend\n\n# Let's print the program output\n\nout, x, θ = 0.0, randn(4), randn(6);\n@instr Grad(test!)(Val(1), out, x, θ)\nx\n\n# We can erease the gradient field by uncomputing the gradient function.\n# If you want, you can differentiate it twice to obtain Hessians.\n# However, we suggest using ForwardDifferentiation over our NiLang program, this is more efficient.\n\n@instr (~Grad(test!))(Val(1), out, x, θ)\nx\n\n# In the above testing code, `Grad(test)` attaches a gradient field to each element of `x`. `~Grad(test)` is the inverse program that erase the gradient fields.\n# Notably, this reversible implementation costs zero memory allocation, although it changes the target variables inplace.\n" }, { "path": "notebooks/README.md", "content": "# How to use notebooks\n\n1. Install Pluto notebook from [here](https://github.com/fonsp/Pluto.jl),\n2. Open this file in a Pluto notebook.\n" }, { "path": "notebooks/autodiff.jl", "content": "### A Pluto.jl notebook ###\n# v0.14.5\n\nusing Markdown\nusing InteractiveUtils\n\n# ╔═╡ f11023e5-8f7b-4f40-86d3-3407b61863d9\nbegin\n\tusing PlutoUI, Viznet, Compose, Plots\n\tfunction shrink(a, b, da, db)\n\t\td = b .- a\n\t\tr = sqrt(sum(abs2, d))\n\t\tunitd = d ./ r\n\t\ta .+ unitd .* da, b .- unitd .* db\n\tend\nend;\n\n# ╔═╡ ce44f8bd-692e-4eab-9ba4-055b25e40c81\nusing ForwardDiff: Dual\n\n# ╔═╡ 9a46597c-b1ee-4e3b-aed1-fd2874b6e77a\nusing BenchmarkTools\n\n# ╔═╡ ccd38f52-104d-434a-aea3-dd94e571374f\nusing NiLang\n\n# ╔═╡ f4230251-ba54-434a-b86b-f972c7389217\nusing MacroTools\n\n# ╔═╡ 69dc2685-b70f-4a81-af30-f02e0054bd52\nusing NiLang.AD\n\n# ╔═╡ 200f1848-0980-4185-919a-93ab2e7f788f\nusing SparseArrays\n\n# ╔═╡ 30c191c5-642b-4062-98f3-643d314a054d\nusing LinearAlgebra\n\n# ╔═╡ 864dbde7-b689-4165-a08e-6bbbd72190de\nusing Test\n\n# ╔═╡ a1ef579e-4b66-4042-944e-7e27c660095e\nmd\"\"\"\n```math\n\\newcommand{\\comment}[1]{{\\bf \\color{blue}{\\text{◂~ #1}}}}\n```\n\"\"\"\n\n# ╔═╡ 100b4293-fd1e-4b9c-a831-5b79bc2a5ebe\nbegin\n\t# left right layout\n\tfunction leftright(a, b; width=600)\n\t\tHTML(\"\"\"\n\n\t\t\t\n\n\n\t\n\t\n
$(html(a))$(html(b))
\n\"\"\")\n\tend\n\t\n\t# up down layout\n\tfunction updown(a, b; width=nothing)\n\t\tHTML(\"\"\"\n\n\t\n\n\n\t\n
$(html(a))
$(html(b))
\n\"\"\")\n\tend\n\t\n\tfunction highlight(str)\n\t\tHTML(\"\"\"$(str)\"\"\")\n\tend\nend;\n\n# ╔═╡ 9d11e058-a7d0-11eb-1d78-6592ff7a1b43\nmd\"# An introduction to automatic differentiation\n\n-- GiggleLiu\"\n\n# ╔═╡ b73157bf-1a77-47b8-8a06-8d6ec2045023\nhtml\"\"\n\n# ╔═╡ ec13e0a9-64ff-4f66-a5a6-5fef53428fa1\nmd\"\"\"\n* What is automatic differentiation (AD)?\n * A true history of AD\n * Forward mode AD\n * Reverse mode AD \n * primitves on tensors (including Jax, pytorch et al.)\n * primitves on elementary instructions (usually source code transformation based)\n * defined on a reversible program\n* Some applications in **scientific computing**\n * solving the graph embedding problem\n * inverse engineering a hamiltonian\n * obtaining maximum independent set (MIS) configurations\n * towards differentiating `expmv` ``\\comment{will be used in our emulator}``\n\"\"\"\n\n# ╔═╡ f8b0d1ce-99f7-4729-b46e-126da540cbbe\nmd\"\"\"\n## The true history of automatic differentiation\n\"\"\"\n\n# ╔═╡ 435ac19e-1c0c-4ee5-942d-f2a97c8c4d80\nmd\"\"\"\n* 1964 ~ Robert Edwin Wengert, A simple automatic derivative evaluation program. ``\\comment{first forward mode AD}``\n* 1970 ~ Seppo Linnainmaa, Taylor expansion of the accumulated rounding error. ``\\comment{first backward mode AD}``\n* 1986 ~ Rumelhart, D. E., Hinton, G. E., and Williams, R. J., Learning representations by back-propagating errors.\n* 1992 ~ Andreas Griewank, Achieving logarithmic growth of temporal and spatial complexity in reverse automatic differentiation. ``\\comment{foundation of source code transformation based AD.}``\n* 2000s ~ The boom of tensor based AD frameworks for machine learning.\n* 2018 ~ People re-invented AD as differential programming ([wiki](https://en.wikipedia.org/wiki/Differentiable_programming) and this [quora answer](https://www.quora.com/What-is-Differentiable-Programming).)\n![](https://qph.fs.quoracdn.net/main-qimg-fb2f8470f2120eb49c8142b08d9c4132)\n* 2020 ~ Me, Differentiate everything with a reversible embeded domain-specific language ``\\comment{AD based on reversible programming}``.\n\"\"\"\n\n# ╔═╡ 48ecd619-d01d-43ff-8b52-7c2566c3fa2b\nmd\"## Forward mode automatic differentiation\"\n\n# ╔═╡ 4878ce45-40ff-4fae-98e7-1be41e930e4d\nmd\"\"\"\nForward mode AD attaches a infitesimal number $\\epsilon$ to a variable, when applying a function $f$, it does the following transformation\n```math\n\\begin{align}\n f(x+g \\epsilon) = f(x) + f'(x) g\\epsilon + \\mathcal{O}(\\epsilon^2)\n\\end{align}\n```\n\nThe higher order infinitesimal is ignored. \n\n**In the program**, we can define a *dual number* with two fields, just like a complex number\n```\nf((x, g)) = (f(x), f'(x)*g)\n```\n\"\"\"\n\n# ╔═╡ b2c1936c-2c27-4fbb-8183-e38c5e858483\nres = sin(Dual(π/4, 2.0))\n\n# ╔═╡ 8be1b812-fcac-404f-98aa-0571cb990f34\nres === Dual(sin(π/4), cos(π/4)*2.0)\n\n# ╔═╡ 33e0c762-c75e-44aa-bfe2-bff92dd1ace8\nmd\"\nWe can apply this transformation consecutively, it reflects the chain rule.\n```math\n\\begin{align}\n\\frac{\\partial \\vec y_{i+1}}{\\partial x} &= \\boxed{\\frac{\\partial \\vec y_{i+1}}{\\partial \\vec y_i}}\\frac{\\partial \\vec y_i}{\\partial x}\\\\\n&\\text{local Jacobian}\n\\end{align}\n```\n\"\n\n# ╔═╡ c59c35ee-1907-4736-9893-e22c052150ca\nlet\n\tlb = textstyle(:math, fontsize(8), width=0.5, height=0.5)\n\ttb = textstyle(:default, fontsize(10), Compose.font(\"monospace\"))\n\ttb_big = textstyle(:default, fontsize(3.5), fill(\"white\"), Compose.font(\"monospace\"))\n\tnb = nodestyle(:circle, fill(\"white\"), Compose.stroke(\"black\"); r=0.08)\n\ttri = nodestyle(:triangle, Compose.stroke(\"transparent\"), fill(\"black\"); r=0.02)\n\teb = bondstyle(:default, linewidth(0.5mm))\n\tebr = bondstyle(:default, Compose.stroke(\"red\"), linewidth(0.5mm))\n\tebd = bondstyle(:default, linewidth(0.5mm), dashed=true)\n\teba = bondstyle(:default, linewidth(0.5mm), Compose.arrow(), Compose.stroke(\"red\"), Compose.fill(\"red\"))\n\t\t\n\tfunction arrow(x, y)\n\t\tmid = (x .+ y) ./ 2\n\t\tt = nodestyle(:triangle, fill(\"red\"), θ=π/2-atan((y .- x)...)-1π/6)\n\t\tebr >> (x, y)\n\t\tt >> mid\n\tend\n\t\n\tCompose.set_default_graphic_size(15cm, 5cm)\n\tx = (0.1, 0.5)\n\tfi0 = (0.35, 0.5)\n\tfi1 = (0.7, 0.5)\n\tfi2 = (1.0, 0.5)\n\timg = canvas() do\n\t\tnb >> fi0\n\t\tnb >> fi1\n\t\tlb >> (fi0 .- (0.05, 0.1), \"f_{i-1}\")\n\t\tlb >> (fi1 .- (0.02, 0.1), \"f_{i}\")\n\t\tlb >> (x, \"x\")\n\t\tlb >> ((fi1 .+ fi0) ./ 2 .- (0.02, 0.0), raw\"\\vec{y}_{i}\")\n\t\tlb >> ((fi1 .+ fi2) ./ 2 .- (0.05, 0.0), raw\"\\vec{y}_{i+1}\")\n\t\tlb >> ((fi1 .+ fi2) ./ 2 .- (0.05, 0.0), \"\\\\vec{y}_{i+1}\")\n\t\tlb >> (x .- (0.00, 0.25), raw\"\\color{red}{1}\")\n\t\tlb >> ((fi1 .+ fi0) ./ 2 .- (0.05, 0.45), raw\"\\color{red}{\\frac{\\partial \\vec{y}_{i}}{\\partial x}}\")\n\t\tlb >> ((fi1 .+ fi2) ./ 2 .- (0.08, 0.45), raw\"\\color{red}{\\frac{\\partial \\vec{y}_{i+1}}{\\partial x}}\")\n\t\tebd >> (x, fi0)\n\t\teb >> (fi0, fi1)\n\t\teb >> (fi1, fi2)\n\t\t#arrow((fi1 .+ fi0) ./ 2 .+ (0.08, -0.3), (fi1 .+ fi2) ./ 2 .+ (-0.08, -0.3))\n\t\tarrow((fi1 .+ fi0) ./ 2 .+ (0.08, -0.3), (fi1 .+ fi2) ./ 2 .+ (-0.08, -0.3))\n\tend\n\timg\nend\n\n# ╔═╡ 0ae13734-b826-4dbf-93d1-11044ce88bd4\nx_ = Dual(π/4, 1.0)\n\n# ╔═╡ 99187515-c8be-49c2-8d70-9c2998d9993c\nsin(x_)\n\n# ╔═╡ 78ca6b08-84c4-4e4d-8412-ae6c28bfafce\nmd\"when automatic comes in\"\n\n# ╔═╡ f12b25d8-7c78-4686-b46d-00b34e565605\nlet\n\tx = Dual(π/4, 1.0)\n\tz = Dual(1.1)\n\tfor i=1:10\n\t\tx = sin(x) * z\n\tend\n\tx\nend\n\n# ╔═╡ d90c3cc9-084d-4cf7-9db7-42cea043030b\nmd\"\"\"\n**Example:** Computing two gradients $\\frac{\\partial z\\sin x}{\\partial x}$ and $\\frac{\\partial \\sin^2x}{\\partial x}$ at one sweep\n\"\"\"\n\n# ╔═╡ 93c98cb2-18af-47df-afb3-8c5a34b4723c\nlet\n\tlb = textstyle(:math, fontsize(8), width=1.0, height=0.5)\n\ttb = textstyle(:default, fontsize(3.5), Compose.font(\"monospace\"))\n\ttb_big = textstyle(:default, fontsize(4.5), fill(\"white\"), Compose.font(\"monospace\"))\n\tnb = nodestyle(:circle, fill(\"black\"), Compose.stroke(\"transparent\"); r=0.05)\n\ttri = nodestyle(:triangle, Compose.stroke(\"transparent\"), fill(\"black\"); r=0.02)\n\teb = bondstyle(:default, linewidth(0.5mm))\n\t\n\tx_x = (0.1, 0.25)\n\tx_y = (0.9, 0.5)\n\tx_y2 = (0.9, 0.25)\n\tx_z = (0.3, 0.5)\n\tx_sin = (0.3, 0.25)\n\tx_mul = (0.5, 0.5)\n\tx_square = (0.5, 0.25)\n\t\n\tfunction arrow(x, y)\n\t\tmid = (x .+ y) ./ 2\n\t\tt = nodestyle(:triangle, θ=π/2-atan((y .- x)...)-1π/6)\n\t\teb >> (x, y)\n\t\tt >> mid\n\tend\n\n\timg = canvas() do\n\t\tnb >> x_sin\n\t\tnb >> x_mul\n\t\tnb >> x_square\n\t\ttb_big >> (x_sin, \"sin\")\n\t\ttb_big >> (x_mul .+ (0, 0.01), \"*\")\n\t\ttb_big >> (x_square, \"^2\")\n\t\tarrow(x_sin, x_mul)\n\t\tarrow(x_x, x_sin)\n\t\tarrow(x_mul, x_y)\n\t\tarrow(x_square, x_y2)\n\t\tarrow(x_z, x_mul)\n\t\tarrow(x_sin, x_square)\n\t\ttb >> ((x_x .+ x_sin) ./ 2 .- (0.02, 0.04), \"x+ϵˣ\")\n\t\ttb >> ((x_sin .+ x_mul) ./ 2 .- (0.08, 0.04), \"sin(x)+cos(x)*ϵˣ\")\n\t\ttb >> ((x_y .+ x_mul) ./ 2 .- (-0.04, 0.055), \"z*sin(x)\\n+z*cos(x)*ϵˣ\")\n\t\ttb >> ((x_y2 .+ x_square) ./ 2 .- (-0.04, 0.055), \"sin(x)^2\\n+2*sin(x)*cos(x)*ϵˣ\")\n\t\ttb >> ((x_z .+ x_mul) ./ 2 .- (0.05, 0.02), \"z\")\n\tend\n\t\n\tCompose.set_default_graphic_size(100mm, 100mm/2)\n\tCompose.compose(context(0, -0.15, 1, 2), img)\nend\n\n# ╔═╡ 2dc74e15-e2ea-4961-b43f-0ada1a73d80a\nmd\"so the gradients are $z\\cos x$ and $2\\sin x\\cos x$\"\n\n# ╔═╡ 7ee75a15-eaea-462a-92b6-293813d2d4d7\nmd\"\"\"\n**What if we want to compute gradients for multiple inputs?**\n\nThe computing time grows **linearly** as the number of variables that we want to differentiate. But does not grow significantly with the number of outputs.\n\"\"\"\n\n# ╔═╡ 02a25b73-7353-43b1-8738-e7ca472d0cc7\nmd\"\"\"\n## Reverse mode automatic differentiation\n\n\"\"\"\n\n# ╔═╡ 2afb984f-624e-4381-903f-ccc1d8a66a17\nmd\"On the other side, the back-propagation can differentiate **many inputs** with respect to a **single output** efficiently\"\n\n# ╔═╡ 7e5d5e69-90f2-4106-8edf-223c150a8168\nmd\"\"\"\n```math\n\\begin{align}\n \\frac{\\partial \\mathcal{L}}{\\partial \\vec y_i} = \\frac{\\partial \\mathcal{L}}{\\partial \\vec y_{i+1}}&\\boxed{\\frac{\\partial \\vec y_{i+1}}{\\partial \\vec y_i}}\\\\\n&\\text{local jacobian?}\n\\end{align}\n```\n\"\"\"\n\n# ╔═╡ 92d7a938-9463-4eee-8839-0b8c5f762c79\nlet\n\tlb = textstyle(:math, fontsize(8), width=0.5, height=0.5)\n\ttb = textstyle(:default, fontsize(10), Compose.font(\"monospace\"))\n\ttb_big = textstyle(:default, fontsize(3.5), fill(\"white\"), Compose.font(\"monospace\"))\n\tnb = nodestyle(:circle, fill(\"white\"), Compose.stroke(\"black\"); r=0.08)\n\ttri = nodestyle(:triangle, Compose.stroke(\"transparent\"), fill(\"black\"); r=0.02)\n\teb = bondstyle(:default, linewidth(0.5mm))\n\tebr = bondstyle(:default, Compose.stroke(\"red\"), linewidth(0.5mm))\n\tebd = bondstyle(:default, linewidth(0.5mm), dashed=true)\n\teba = bondstyle(:default, linewidth(0.5mm), Compose.arrow(), Compose.stroke(\"red\"), Compose.fill(\"red\"))\n\t\t\n\tfunction arrow(x, y)\n\t\tmid = (x .+ y) ./ 2\n\t\tt = nodestyle(:triangle, fill(\"red\"), θ=π/2-atan((y .- x)...)-1π/6)\n\t\tebr >> (x, y)\n\t\tt >> mid\n\tend\n\t\n\tCompose.set_default_graphic_size(15cm, 5cm)\n\tx = (0.1, 0.5)\n\tfi0 = (0.35, 0.5)\n\tfi1 = (0.7, 0.5)\n\tfi2 = (0.9, 0.5)\n\timg = canvas() do\n\t\tnb >> fi0\n\t\tnb >> fi1\n\t\tlb >> (fi0 .- (0.02, 0.1), \"f_{i}\")\n\t\tlb >> (fi1 .- (0.05, 0.1), \"f_{i+1}\")\n\t\tlb >> (fi2 .- (0.05, 0.0), raw\"\\mathcal{L}\")\n\t\tlb >> ((fi0 .+ x) ./ 2 .- (0.05, 0.0), raw\"\\vec{y}_{i}\")\n\t\tlb >> ((fi0 .+ fi1) ./ 2 .- (0.05, 0.0), raw\"\\vec{y}_{i+1}\")\n\t\tlb >> ((fi0 .+ fi1) ./ 2 .- (0.05, 0.0), \"\\\\vec{y}_{i+1}\")\n\t\tlb >> (fi2 .- (0.05, 0.25), raw\"\\color{red}{1}\")\n\t\tlb >> ((fi0 .+ x) ./ 2 .- (0.08, 0.45), raw\"\\color{red}{\\frac{\\partial \\mathcal{L}}{\\partial \\vec{y}_{i}}}\")\n\t\tlb >> ((fi0 .+ fi1) ./ 2 .- (0.08, 0.45), raw\"\\color{red}{\\frac{\\partial \\mathcal{L}}{\\partial \\vec{y}_{i+1}}}\")\n\t\tebd >> (fi1, fi2)\n\t\teb >> (fi0, fi1)\n\t\teb >> (x, fi0)\n\t\t#arrow((fi1 .+ fi0) ./ 2 .+ (0.08, -0.3), (fi1 .+ fi2) ./ 2 .+ (-0.08, -0.3))\n\t\tarrow( (fi0 .+ fi1) ./ 2 .+ (-0.08, -0.3), (fi0 .+ x) ./ 2 .+ (0.05, -0.3),)\n\tend\n\timg\nend\n\n# ╔═╡ 4b1a0b59-ddc6-4b2d-b5f5-d92084c31e46\nmd\"### How to visit local Jacobians in the reversed order? \"\n\n# ╔═╡ 81f16b8b-2f0b-4ba3-8c26-6669eabf48aa\nmd\"The naive approach is to store everything.\"\n\n# ╔═╡ fb6c3a48-550a-4d2e-a00b-a1e40d86b535\nmd\"\"\"\n**Example:** Computing the gradient $\\frac{\\partial z\\sin x}{\\partial x}$ and $\\frac{\\partial z\\sin x}{\\partial z}$ by back propagating cached local information.\n\"\"\"\n\n# ╔═╡ ab6fa4ac-29ed-4722-88ed-fa1caf2072f3\nlet\n\tlb = textstyle(:math, fontsize(10), width=1.0, height=0.5)\n\ttb = textstyle(:default, fontsize(3.5), Compose.font(\"monospace\"))\n\ttbc = textstyle(:default, fontsize(3.5), fill(\"red\"), Compose.font(\"monospace\"))\n\ttb_big = textstyle(:default, fontsize(4), fill(\"white\"), Compose.font(\"monospace\"))\n\tnb = nodestyle(:circle, fill(\"black\"), Compose.stroke(\"transparent\"); r=0.05)\n\ttri = nodestyle(:triangle, Compose.stroke(\"transparent\"), fill(\"black\"); r=0.02)\n\teb = bondstyle(:default, linewidth(0.5mm))\n\t\n\tx_x = (0.1, 0.2)\n\tx_y = (0.9, 0.5)\n\tx_z = (0.1, 0.7)\n\tx_sin = (0.3, 0.3)\n\tx_mul = (0.5, 0.5)\n\n\tfunction arrow(x, y)\n\t\tmid = (x .+ y) ./ 2\n\t\tt = nodestyle(:triangle, θ=π/2-atan((y .- x)...)-1π/6)\n\t\teb >> (x, y)\n\t\tt >> mid\n\tend\n\timg1 = canvas() do\n\t\tnb >> x_sin\n\t\tnb >> x_mul\n\t\ttb_big >> (x_sin, \"sin\")\n\t\ttb_big >> (x_mul .+ (0, 0.01), \"*\")\n\t\tarrow(x_sin, x_mul)\n\t\tarrow(x_x, x_sin)\n\t\tarrow(x_mul, x_y)\n\t\tarrow(x_z, x_mul)\n\t\ttb >> ((x_x .+ x_sin) ./ 2 .- (0.0, 0.1), \"x \\n push(Σ,x)\")\n\t\ttb >> ((x_sin .+ x_mul) ./ 2 .- (-0.15, 0.04), \"s = sin(x) \\n push(Σ,s)\")\n\t\ttb >> ((x_y .+ x_mul) ./ 2 .- (-0.05, 0.04), \"y = z*sin(x)\")\n\t\ttb >> ((x_z .+ x_mul) ./ 2 .- (0.05, 0.07), \"z\\n push(Σ,z)\")\n\tend\n\timg2 = canvas() do\n\t\tnb >> x_sin\n\t\tnb >> x_mul\n\t\ttb_big >> (x_sin, \"sin\")\n\t\ttb_big >> (x_mul .+ (0, 0.01), \"*\")\n\t\tarrow(x_mul, x_sin)\n\t\tarrow(x_sin, x_x)\n\t\tarrow(x_y, x_mul)\n\t\tarrow(x_mul, x_z)\n\t\ttb >> ((x_x .+ x_sin) ./ 2 .- (0.0, 0.1), \"x = pop(Σ)\\nx̄ = cos(x)*s̄\")\n\t\ttb >> ((x_sin .+ x_mul) ./ 2 .- (-0.12, 0.04), \"z = pop(Σ)\\ns̄ = z*ȳ\")\n\t\ttb >> ((x_y .+ x_mul) ./ 2 .- (-0.05, 0.06), \"y\\nȳ=1\")\n\t\ttb >> ((x_z .+ x_mul) ./ 2 .- (0.05, 0.07), \"s = pop(Σ)\\nz̄ = s*ȳ\")\n\tend\n\t\n\tCompose.set_default_graphic_size(150mm, 75mm/1.4)\n\tCompose.compose(context(), \n\t(context(0, -0.1, 0.5, 1.4), img1),\n\t(context(0.5, -0.1, 0.5, 1.4), img2)\n\t)\nend\n\n# ╔═╡ 8e72d934-e307-4505-ac82-c06734415df6\nmd\"Here, we use $\\overline y$ for $\\frac{\\partial \\mathcal{L}}{\\partial y}$, which is also called the adjoint.\"\n\n# ╔═╡ e6ff86a9-9f54-474b-8111-a59a25eda506\nmd\"### Primitives on different scales\"\n\n# ╔═╡ 9c1d9607-a634-4350-aacd-2d40984d647d\nmd\"We call the leaf nodes defining AD rules \\\"**primitives**\\\"\"\n\n# ╔═╡ 63db2fa2-50b2-4940-b8ee-0dc6e3966a57\nmd\"\n**Design Decision**\n\n* A: If we define primitives on **arrays**, we need tons of manually defined backward rules. (Jax, Pytorch, Zygote.jl, ReverseDiff.jl et al.)\n* B: If we define primitives on **scalar instructions**, we will have worse tensor performance. (Tapenade, Adept, NiLang et al.)\n\n*Note*: Here, implementing AD on scalars means specifically the **optimal checkpointing** approach, rather than a package like Jax, Zygote and ReverseDiff that having scalar support.\n\"\n\n# ╔═╡ 693167e7-e80c-401d-af89-55b5fae30848\nlet\n\tw, h = 0.22, 0.1\n\tlb = Compose.compose(context(), polygon([(-w, -h), (-w, h), (w, h), (w, -h)]), Compose.stroke(\"transparent\"))\n\tlb2 = Compose.compose(context(), polygon([(-w, -h), (-w, h), (w, h), (w, -h)]), Compose.stroke(\"transparent\"), fill(\"red\"))\n\ttb = Compose.compose(context(), Compose.text(0.0, 0.0, \"\"), fontsize(3), Compose.font(\"monospace\"))\n\ttb_big = textstyle(:default, fontsize(3), fill(\"white\"), Compose.font(\"monospace\"))\n\teb = bondstyle(:default, linewidth(0.5mm))\n\tar = bondstyle(:default, linewidth(0.3mm), Compose.arrow())\n\txprog = (0.25, 0.15)\n\txtensors = (0.25, 0.5)\n\tt1 = (0.5, 0.15)\n\tt2 = (0.5, 0.5)\n\tt3 = (0.5, 0.85)\n\txscalars2 = (0.25, 0.85)\n\t\n\tfunction box(loc, text; color=\"black\")\n\t\t(color==\"black\" ? lb : lb2) >> loc\n\t\ttb_big >> (loc, text)\n\tend\n\tCompose.set_default_graphic_size(10cm, 5cm)\n\tcanvas() do\n\t\tbox(xprog, \"Program\")\n\t\tar >> (xprog, xtensors .+ (0, -h-0.03))\n\t\t#ar >> (xprog, xscalars .+ (-w/2, -h-0.03))\n\t\tar >> (xtensors, xscalars2 .+ (0, -h-0.05))\n\t\tbox(xtensors, \"Functions on arrays\")\n\t\t#box(xscalars, \"Functions on Scalars\")\n\t\tbox(xscalars2, \"Finite instructions\"; color=\"red\")\n\t\ttb >> (t1, \"Neural networks\")\n\t\ttb >> (t2, \"matrix multiplication\")\n\t\ttb >> (t3, \"+, -, *\")\n\tend\nend\n\n# ╔═╡ 4cd70901-2142-4868-9a33-c46ca0d064ec\nhtml\"\"\"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
on tensorson finite instructions
meaningdefining backward rules manully for functions on tensorsdefining backward rules on a limited set of basic scalar operations, and generate gradient code using source code transformation
pros and cons\n
    \n
  1. Good tensor performance
    2. \n
  2. Mature machine learning ecosystem
    3. \n
  3. Need to define backward rules manually
    4. \n
\n		\n
    \n
  1. Reasonalbe scalar performance
    2. \n
  2. hard to utilize GPU kernels (except NiLang.jl) and BLAS
    3. \n
\n		\n
packagesJax
PyTorch		Tapenade
\nAdept
\nNiLang.jl\n
\n\"\"\"\n\n# ╔═╡ 89018a35-76f4-4f23-b15a-a600db046d6f\nmd\"## A book\"\n\n# ╔═╡ 1d219222-0778-4c37-9182-ed5ccbb3ef32\nleftright(html\"\"\"\n\n\"\"\", md\"**Evaluating derivatives: principles and techniques of algorithmic differentiation**\n\t\nBy: Griewank, Andreas, and Andrea Walther\n(2008)\")\n\n# ╔═╡ 4ff09f7c-aeac-48bd-9d58-8446137c3acd\nmd\"\"\"\n## The AD ecosystem in Julia\n\nPlease check JuliaDiff: [https://juliadiff.org/](https://juliadiff.org/)\n\nA short list:\n* Forward mode AD: ForwardDiff.jl\n* Reverse mode AD (tensor): ReverseDiff.jl/Zygote.jl\n* Reverse mode AD (scalar): NiLang.jl\n\nWarnings\n* The main authors of `Tracker`, `ReverseDiff` and `Zygote` are not maintaining them anymore.\n\"\"\"\n#=\n| | Rules | Favors Tensor? | Type |\n| ---- | ---- | --- | --- |\n| Zygote | C | ✓ | R |\n| ReverseDiff | D | ✓ | R |\n| Nabla | D→C | ✓ | R |\n| Tracker | D | ✓ | R |\n| Yota | C | ✓ | R |\n| NiLang | - | × | R |\n| Enzyme | - | × | R |\n| ForwardDiff | - | × | F |\n| Diffractor | ? | ? | ? |\n\n* R: reverse mode\n* F: forward mode\n* C: ChainRules\n* D: DiffRules\n\"\"\"\n=#\n\n# ╔═╡ ea44037b-9359-4fbd-990f-529d88d54351\nmd\"# Quick summary\n1. The history of AD is longer than many people have thought. People are most familar with *reverse mode AD with primitives implemented on tensors* that brings the boom of machine learning. There are also AD frameworks that can differentiate a general program directly, which does not require users defining AD rules manually.\n2. **Forward mode AD** propagate gradients forward, it has a computational overhead propotional to the number of input parameters.\n2. **Backward mode AD** propagate gradients backward, it has a computational overhead propotional to the number of output parameters.\n * primitives on **tensors** v.s. **scalars**\n * it is very expensive to reverse the program\n4. Julia has one of the most active AD community!\n\n#### Forward v.s. Backward\nwhen is forward mode AD more useful?\n\n* It is often combined with backward mode AD for obtaining Hessians (forward over backward).\n* Having <20 input parameters.\n\nwhen is backward mode AD more useful?\n* In most variational optimizations, especially when we are training a neural network with ~ 100M parameters.\n\"\n\n# ╔═╡ e731a8e3-6462-4a60-83e9-6ab7ddfff50e\nmd\"# How do AD libraries work?\"\n\n# ╔═╡ 685c2b28-b071-452c-a881-801128dcb6c3\nmd\"`ForwardDiff` is operator overloading based, many of its overheads can be optimized by Julia's JIT compiler.\"\n\n# ╔═╡ 177ddfc2-2cbe-4dba-9d05-2857633dd1ae\nmd\"# [Tapenade](http://tapenade.inria.fr:8080/tapenade/index.jsp)\n\n![](http://tapenade.inria.fr:8080/tapenade/tapenadelogo.gif)\"\n\n# ╔═╡ 6c2a3a93-385f-4758-9b6e-4cb594a8e856\nmd\"## Example 1: Bessel Example\"\n\n# ╔═╡ fb8168c2-8489-418b-909b-cede57b5ae64\nmd\"bessel.f90\"\n\n# ╔═╡ fdb39284-dbb1-49fa-9a1c-f360f9e6b765\nmd\"\"\"\n```fortran\nsubroutine besselj(res, v, z, atol)\n implicit none\n\tinteger, intent(in) :: v\n\treal*8, intent(in) :: z, atol\n\treal*8, intent(out) :: res\n\treal*8 :: s\n\tinteger :: k, i, factv\n k = 0\n factv = 1\n do i = 2,v\n factv = factv * i\n enddo\n\n s = (z/2.0)**v / factv\n res = s\n do while(abs(s) > atol)\n k = k + 1\n s = -s / k / (k+v) * ((z/2) ** 2)\n res = res + s\n enddo\nendsubroutine besselj\n```\n\"\"\"\n\n# ╔═╡ 60214f22-c8bb-4a32-a882-4e6c727b29a9\nmd\"\"\"\nbesselj_d.f90 (forward mode)\n```fortran\n! Generated by TAPENADE (INRIA, Ecuador team)\n! Tapenade 3.15 (master) - 15 Apr 2020 11:54\n!\n! Differentiation of besselj in forward (tangent) mode:\n! variations of useful results: res\n! with respect to varying inputs: z\n! RW status of diff variables: res:out z:in\nSUBROUTINE BESSELJ_D(res, resd, v, z, zd, atol)\n IMPLICIT NONE\n INTEGER, INTENT(IN) :: v\n REAL*8, INTENT(IN) :: z, atol\n REAL*8, INTENT(IN) :: zd\n REAL*8, INTENT(OUT) :: res\n REAL*8, INTENT(OUT) :: resd\n REAL*8 :: s\n REAL*8 :: sd\n INTEGER :: k, i, factv\n INTRINSIC ABS\n REAL*8 :: abs0\n REAL*8 :: pwx1\n REAL*8 :: pwx1d\n REAL*8 :: pwr1\n REAL*8 :: pwr1d\n INTEGER :: temp\n k = 0\n factv = 1\n DO i=2,v\n factv = factv*i\n END DO\n pwx1d = zd/2.0\n pwx1 = z/2.0\n IF (pwx1 .LE. 0.0 .AND. (v .EQ. 0.0 .OR. v .NE. INT(v))) THEN\n pwr1d = 0.0_8\n ELSE\n pwr1d = v*pwx1**(v-1)*pwx1d\n END IF\n pwr1 = pwx1**v\n sd = pwr1d/factv\n s = pwr1/factv\n resd = sd\n res = s\n DO WHILE (.true.)\n IF (s .GE. 0.) THEN\n abs0 = s\n ELSE\n abs0 = -s\n END IF\n IF (abs0 .GT. atol) THEN\n k = k + 1\n temp = k*(k+v)*(2*2)\n sd = -((z**2*sd+s*2*z*zd)/temp)\n s = -(s*(z*z)/temp)\n resd = resd + sd\n res = res + s\n ELSE\n EXIT\n END IF\n END DO\nEND SUBROUTINE BESSELJ_D\n```\n\nbesselj_b.f90 (backward mode)\n```fortran\n! Generated by TAPENADE (INRIA, Ecuador team)\n! Tapenade 3.15 (master) - 15 Apr 2020 11:54\n!\n! Differentiation of besselj in reverse (adjoint) mode:\n! gradient of useful results: res z\n! with respect to varying inputs: res z\n! RW status of diff variables: res:in-zero z:incr\nSUBROUTINE BESSELJ_B(res, resb, v, z, zb, atol)\n IMPLICIT NONE\n INTEGER, INTENT(IN) :: v\n REAL*8, INTENT(IN) :: z, atol\n REAL*8 :: zb\n REAL*8 :: res\n REAL*8 :: resb\n REAL*8 :: s\n REAL*8 :: sb\n INTEGER :: k, i, factv\n INTRINSIC ABS\n REAL*8 :: abs0\n REAL*8 :: tempb\n INTEGER :: ad_count\n INTEGER :: i0\n INTEGER :: branch\n k = 0\n factv = 1\n DO i=2,v\n factv = factv*i\n END DO\n s = (z/2.0)**v/factv\n ad_count = 1\n DO WHILE (.true.)\n IF (s .GE. 0.) THEN\n abs0 = s\n ELSE\n abs0 = -s\n END IF\n IF (abs0 .GT. atol) THEN\n CALL PUSHINTEGER4(k)\n k = k + 1\n CALL PUSHREAL8(s)\n s = -(s/k/(k+v)*(z/2)**2)\n ad_count = ad_count + 1\n ELSE\n GOTO 100\n END IF\n END DO\n CALL PUSHCONTROL1B(0)\n GOTO 110\n 100 CALL PUSHCONTROL1B(1)\n 110 DO i0=1,ad_count\n IF (i0 .EQ. 1) THEN\n CALL POPCONTROL1B(branch)\n IF (branch .EQ. 0) THEN\n sb = 0.0_8\n ELSE\n sb = 0.0_8\n END IF\n ELSE\n sb = sb + resb\n CALL POPREAL8(s)\n tempb = -(sb/(k*(k+v)*2**2))\n sb = z**2*tempb\n zb = zb + 2*z*s*tempb\n CALL POPINTEGER4(k)\n END IF\n END DO\n sb = sb + resb\n IF (.NOT.(z/2.0 .LE. 0.0 .AND. (v .EQ. 0.0 .OR. v .NE. INT(v)))) zb = &\n& zb + v*(z/2.0)**(v-1)*sb/(2.0*factv)\n resb = 0.0_8\nEND SUBROUTINE BESSELJ_B\n```\n\"\"\"\n\n# ╔═╡ 7a6dbe09-cb7f-405f-b9b5-b350ca170e5f\nmd\"## Example 2: Matrix multiplication\"\n\n# ╔═╡ 5dc4a849-76dd-4c4f-8828-755671839e5e\nmd\"\"\"\nmatmul_b.f90\n```fortran\n! Generated by TAPENADE (INRIA, Ecuador team)\n! Tapenade 3.16 (develop) - 9 Apr 2021 17:40\n!\n! Differentiation of mymatmul in reverse (adjoint) mode:\n! gradient of useful results: x y z\n! with respect to varying inputs: x y z\n! RW status of diff variables: x:incr y:incr z:in-out\nSUBROUTINE MYMATMUL_B(z, zb, x, xb, y, yb, m, n, o)\n IMPLICIT NONE\n INTEGER, INTENT(IN) :: m, n, o\n REAL*8, DIMENSION(:, :) :: z(m, n)\n REAL*8 :: zb(m, n)\n REAL*8, DIMENSION(:, :), INTENT(IN) :: x(m, o), y(o, n)\n REAL*8 :: xb(m, o), yb(o, n)\n REAL*8 :: temp\n REAL*8 :: tempb\n INTEGER :: i, j, k\n DO j=n,1,-1\n DO i=m,1,-1\n tempb = zb(i, j)\n zb(i, j) = 0.0_8\n DO k=o,1,-1\n xb(i, k) = xb(i, k) + y(k, j)*tempb\n yb(k, j) = yb(k, j) + x(i, k)*tempb\n END DO\n END DO\n END DO\nEND SUBROUTINE MYMATMUL_B\n```\n\"\"\"\n\n# ╔═╡ b053f11b-9ed7-47ff-ab32-0c70b87e71ed\nmd\"## Example 3: Pyramid\"\n\n# ╔═╡ 7b1aa6dd-647f-44cb-b580-b58e23e8b5a6\nhtml\"\"\"\n\n\"\"\"\n\n# ╔═╡ b96bac75-b4ad-45f7-aeec-cb6a387eebf0\nmd\"You will see a lot allocation\"\n\n# ╔═╡ 5fe022eb-6a17-466e-a6d0-d67e82af23cd\nmd\"pyramid.f90\"\n\n# ╔═╡ 92047e95-7eba-4021-9668-9bb4b92261d7\nmd\"\"\"\n```fortran\n! Differentiation of pyramid in reverse (adjoint) mode:\n! gradient of useful results: v x\n! with respect to varying inputs: v x\n! RW status of diff variables: v:in-out x:incr\nSUBROUTINE PYRAMID_B(v, vb, x, xb, n)\n IMPLICIT NONE\n INTEGER, INTENT(IN) :: n\n REAL*8 :: v(n, n)\n REAL*8 :: vb(n, n)\n REAL*8, INTENT(IN) :: x(n)\n REAL*8 :: xb(n)\n INTEGER :: i, j\n INTRINSIC SIN\n INTRINSIC COS\n INTEGER :: ad_to\n DO j=1,n\n v(1, j) = x(j)\n END DO\n DO i=1,n-1\n DO j=1,n-i\n CALL PUSHREAL8(v(i+1, j))\n v(i+1, j) = SIN(v(i, j))*COS(v(i, j+1))\n END DO\n CALL PUSHINTEGER4(j - 1)\n END DO\n DO i=n-1,1,-1\n CALL POPINTEGER4(ad_to)\n DO j=ad_to,1,-1\n CALL POPREAL8(v(i+1, j))\n vb(i, j) = vb(i, j) + COS(v(i, j))*COS(v(i, j+1))*vb(i+1, j)\n vb(i, j+1) = vb(i, j+1) - SIN(v(i, j+1))*SIN(v(i, j))*vb(i+1, j)\n vb(i+1, j) = 0.0_8\n END DO\n END DO\n DO j=n,1,-1\n xb(j) = xb(j) + vb(1, j)\n vb(1, j) = 0.0_8\n END DO\nEND SUBROUTINE PYRAMID_B\n```\n\"\"\"\n\n# ╔═╡ e2ae1084-8759-4f27-8ad1-43a88e434a3d\nmd\"## How does NiLang avoid too many allocation?\"\n\n# ╔═╡ edd3aea8-abdb-4e12-9ef9-12ac0fff835b\n@i function pyramid!(y!, v!, x::AbstractVector{T}) where T\n @safe @assert size(v!,2) == size(v!,1) == length(x)\n @inbounds for j=1:length(x)\n v![1,j] += x[j]\n end\n @invcheckoff @inbounds for i=1:size(v!,1)-1\n for j=1:size(v!,2)-i\n @routine begin\n @zeros T c s\n c += cos(v![i,j+1])\n s += sin(v![i,j])\n end\n v![i+1,j] += c * s\n ~@routine\n end\n end\n y! += v![end,1]\nend\n\n# ╔═╡ a2904efb-186c-449d-b1aa-caf530f88e91\n@i function power(x3, x)\n\t@routine begin\n\t\tx2 ← zero(x)\n\t\tx2 += x^2\n\tend\n\tx3 += x2 * x\n\t~@routine\nend\n\n# ╔═╡ 14faaf82-ad3e-4192-8d48-84adfa30442d\nex = NiLangCore.precom_ex(NiLang, :(for j=1:size(v!,2)-i\n @routine begin\n @zeros T c s\n c += cos(v![i,j+1])\n s += sin(v![i,j])\n end\n v![i+1,j] += c * s\n ~@routine\n\t\tend)) |> NiLangCore.rmlines\n\n# ╔═╡ 5d141b88-ec07-4a02-8eb3-37405e5c9f5d\nNiLangCore.dual_ex(NiLang, ex)\n\n# ╔═╡ 0907e683-f216-4cf6-a210-ae5181fdc487\nfunction pyramid0!(v!, x::AbstractVector{T}) where T\n @assert size(v!,2) == size(v!,1) == length(x)\n for j=1:length(x)\n v![1,j] = x[j]\n end\n @inbounds for i=1:size(v!,1)-1\n for j=1:size(v!,2)-i\n v![i+1,j] = cos(v![i,j+1]) * sin(v![i,j])\n end\n end\nend\n\n# ╔═╡ 0bbfa106-f465-4a7b-80a7-7732ba435822\nx = randn(20);\n\n# ╔═╡ 805c7072-98fa-4086-a69d-2e126c55af36\nlet\n\t@benchmark pyramid0!(v, x) seconds=1 setup=(x=randn(1000); v=zeros(1000, 1000))\nend\n\n# ╔═╡ 7e527024-c294-4c16-8626-9953588d9b6a\nlet\n\t@benchmark pyramid!(0.0, v, x) seconds=1 setup=(x=10*randn(1000); v=zeros(1000, 1000))\nend\n\n# ╔═╡ 3e59c65a-ceed-42ed-be64-a6964db016e7\npyramid!(0.0, zeros(20, 20), x)\n\n# ╔═╡ 29f85d05-99fd-4843-9be0-5663e681dad7\nhtml\"\"\"\n\"\"\"\n\n# ╔═╡ e7830e55-bd9e-4a8a-9239-4191a5f0b1d1\nlet\n\t@benchmark NiLang.AD.gradient(Val(1), pyramid!, (0.0, v, x)) seconds=1 setup=(x=randn(1000); v=zeros(1000, 1000))\nend\n\n# ╔═╡ de2cd247-ba68-4ba4-9784-27a743478635\nmd\"## NiLang's implementation\"\n\n# ╔═╡ dc929c23-7434-4848-847a-9fa696e84776\nmd\"\"\"\n```math\n\\begin{align}\n&v_{−1} &= & x_1 &=&1.5000\\\\\n&v_0 &= & x_2 &=&0.5000\\\\\n&v_1 &= & v_{−1}/v_0 &=&1.5000/0.5000 &= 3.0000\\\\\n&v_2 &= & \\sin(v1)&=& \\sin(3.0000) &= 0.1411\\\\\n&v_3 &= & \\exp(v0)&=& \\exp(0.5000) &= 1.6487\\\\\n&v_4 &= & v_1 − v_3 &=&3.0000 − 1.6487 &= 1.3513\\\\\n&v_5 &= & v_2 + v_4 &=&0.1411 + 1.3513 &= 1.4924\\\\\n&v_6 &= & v_5 ∗ v_4 &=&1.4924 ∗ 1.3513 &= 2.0167\\\\\n&y &= & v_6 &=&2.0167\n\\end{align}\n```\n\"\"\"\n\n# ╔═╡ 4f1df03f-c315-47b1-b181-749e1231594c\nhtml\"\"\"\n\n\"\"\"\n\n# ╔═╡ 7eccba6a-3ad5-440b-9c5d-392dc8dc7aba\n@i function example_linear(y::T, x1::T, x2::T) where T\n\t@routine begin\n\t\t@zeros T v1 v2 v3 v4 v5\n\t\tv1 += x1 / x2\n\t\tv2 += sin(v1)\n\t\tv3 += exp(x2)\n\t\tv4 += v1 - v3\n\t\tv5 += v2 + v4\n\tend\n\ty += v5 * v4\n\t~@routine\nend\n\n# ╔═╡ 4a858a3e-ce28-4642-b061-3975a3ed99ff\nmd\"NOTES:\n* a statement changes values inplace directly,\n* no return statement, returns the input arguments directly\n* `@routine ; ; ~@routine` is the Bennett's compute copy uncompute design pattern\n\"\n\n# ╔═╡ 674bb3bb-637b-44f2-bf6d-d1678da03fbd\nPlusEq(identity)(2, 3)\n\n# ╔═╡ 5a59d96f-b2f1-4564-82c7-7f0fe181afb8\nprettify(@macroexpand @i function f(y::T, x::T) where T\n\ty.re += x.re\nend)\n\n# ╔═╡ 55d2f8ee-4f77-4d44-b704-30643dbbab84\n@i function f3(y::T, x::T) where T\n\ty.re += x.re\nend\n\n# ╔═╡ 14951168-97c2-43ae-8d5e-5506408a2bb2\nf3(1+2im, 2+3im)\n\n# ╔═╡ 4f564581-6032-449c-8b15-3c741f44237a\nx5 = GVar(3+4.0im)\n\n# ╔═╡ a36516e8-76c1-4bff-8a12-3e1e621b857d\n~example_linear\n\n# ╔═╡ 402b861c-d363-4d23-b9e9-eb088f57b5c4\nexpre = NiLangCore.precom_ex(@__MODULE__, :(begin\n\t@routine begin\n\t\t@zeros T v1 v2 v3 v4 v5\n\t\tv1 += x1 / x2\n\t\tv2 += sin(v1)\n\t\tv3 += exp(x2)\n\t\tv4 += v1 - v3\n\t\tv5 += v2 + v4\n\tend\n\ty += v5 * v4\n\t~@routine\nend), NiLangCore.PreInfo(Symbol[])) |> NiLangCore.rmlines\n\n# ╔═╡ 63975a80-1b41-4f55-91a1-4a316ad7bf26\nexample_linear(0.0, 1.5, 0.5)\n\n# ╔═╡ 6f688f88-432a-42b2-a2db-19d6bb282e0a\nNiLangCore.dual_ex(@__MODULE__, expre)\n\n# ╔═╡ fb46db14-f7e0-4f01-9096-02334c62942d\n(~example_linear)(example_linear(0.0, 1.5, 0.5)...)\n\n# ╔═╡ b2c3db3d-c250-4daa-8453-3c9a2734aede\nmd\"**How to get gradients?**\"\n\n# ╔═╡ 9a986264-5ba7-4697-a00d-711f8efe29f0\nlet\n\ty, x1, x2 = 0.0, 1.5, 0.5\n\t# compute\n\t(y_out, x1_out, x2_out) = example_linear(y, x1, x2)\n\t\n\t# wrap elements with GVar\n\ty_out_with_g = GVar(y_out, 1.0)\n\tx1_out_with_g = GVar(x1_out, 0.0)\n\tx2_out_with_g = GVar(x2_out, 0.0)\n\t\n\t# uncompute\n\t(y_with_g, x1_with_g, x2_with_g) = (~example_linear)(y_out_with_g, x1_out_with_g, x2_out_with_g)\n\t\n\t# get gradients\n\tgrad(y_with_g), grad(x1_with_g), grad(x2_with_g)\nend\n\n# ╔═╡ 560cf3e9-0c14-4497-85b9-f07045eea32a\nwith_terminal() do\n\tdump(GVar)\nend\n\n# ╔═╡ 8ab79efc-e8d0-4c6f-81df-a89008142bb7\ngvar1 = GVar(1.5, 0.0)\n\n# ╔═╡ 0eec318c-2c09-4dd6-9187-9c0273d29915\ngrad(gvar1)\n\n# ╔═╡ 1f0ef29c-0ad5-4d97-aeed-5ff44e86577a\ngvar2 = GVar(1.0, 2.0)\n\n# ╔═╡ 603d8fc2-5e7b-4d55-92b6-208b25ea6569\ngrad(gvar2)\n\n# ╔═╡ 2b3c765e-b505-4f07-9bcb-3c8cc47364ad\nmd\"To differentiate operation `y += exp(x)`, we bind the backward rule on its inverse `y -= exp(x)`, i.e. `MinusEq(exp)` in the program.\"\n\n# ╔═╡ e0f266da-7e65-4398-bfd4-a6c0b54e626b\nMinusEq(exp)(gvar2, gvar1)\n\n# ╔═╡ e1d35886-79d0-40a5-bd33-1c4e5f4a0a9a\nmd\"\"\"\n```math\n\\left(\\begin{matrix}\\overline y& \\overline x\\end{matrix}\\right) \\rightarrow \\left(\\begin{matrix}\\overline y& \\overline x\\end{matrix}\\right)\\left(\\begin{matrix}\n1 & \\exp(x) \\\\\n0 & 1\n\\end{matrix}\\right) = \\left(\\begin{matrix}\\overline y& \\overline x + \\exp(x) \\overline y\\end{matrix}\\right)\n```\n\"\"\"\n\n# ╔═╡ b63a30b0-c75b-4998-a2b2-0b79574cab81\nexp(1.5) * 2\n\n# ╔═╡ 139bf020-c4a8-45c8-96fa-aeebc7ddaedc\nmd\"*one line version*\"\n\n# ╔═╡ 8967c0f0-89f8-4893-b11b-253333d1a823\nNiLang.AD.gradient(example_linear, (0.0, 1.5, 0.5); iloss=1)\n\n# ╔═╡ f2540450-5a07-4fb8-93fb-a6d48dd36a56\nmd\"## Control Flows\"\n\n# ╔═╡ 3acb2cfd-fa29-4a2b-8f23-f5aaf474edd0\n(@code_julia for i=1:10\n\tx += y\nend) |> NiLangCore.rmlines\n\n# ╔═╡ aa1547f2-5edd-4b7e-b93e-bdfc4e4fc6d5\nmd\"\"\"# Memory Management\"\"\"\n\n# ╔═╡ 6e76a107-4f51-4e32-b133-7b6e04d7d107\nmd\"The true reverse mode autodiff has to handle the memory wall problem.\"\n\n# ╔═╡ 999f7a8f-d72e-4ccd-8cbf-b5bbb7db1842\nmd\"\"\"\n## Checkpointing\n\"\"\"\n\n# ╔═╡ 32772c2a-6b80-4779-963c-06974ff0d832\nhtml\"\"\"\n\n\"\"\"\n\n# ╔═╡ 41642bd5-1321-490a-95ad-4c1d6363456f\nmd\"\n* red arrow: back propagation\n* black dot: cached\n* white dot: not cached\n\"\n\n# ╔═╡ 2a553e32-05ef-4c2d-aba7-41185c6035d4\nmd\"Most time efficient (checkpoint every step)\"\n\n# ╔═╡ ab8345ce-e038-4d6b-9e1f-57e4f33bb67b\nhtml\"\"\"\n\n\"\"\"\n\n# ╔═╡ bb9c9a4c-601a-4708-9b2d-04d1583938f2\nmd\"Most space efficient (only checkpoint the first step)\"\n\n# ╔═╡ b9917e94-c33d-423f-a478-3252bacc2494\nhtml\"\"\"\n\n\"\"\"\n\n# ╔═╡ 4978f404-11ff-41b8-a673-f2d051b1f526\nmd\"Restricting the number of checkpoints, is evenly checkpointed program optimal?\"\n\n# ╔═╡ 73bd2e3b-902f-461b-860f-246257608ecd\nhtml\"\"\"\n\n\"\"\"\n\n# ╔═╡ 4dd47dc8-6dfa-47a4-a088-689b4b870762\nmd\"## Optimal checkpointing\"\n\n# ╔═╡ ecd975d2-9374-4f40-80ac-2cceda11e7fb\nmd\"\"\"\n1992 ~ Andreas Griewank, Achieving logarithmic growth of temporal and spatial complexity in reverse automatic differentiation.\n\nJulia implementation: [TreeverseAlgorithm.jl](https://github.com/GiggleLiu/TreeverseAlgorithm.jl)\n\"\"\"\n\n# ╔═╡ 832cc81d-a49d-46e7-9d2b-d8bde9bb1273\nhtml\"\"\"\n\n\"\"\"\n\n# ╔═╡ 2192a1de-1042-4b13-a313-b67de489124c\nmd\"\"\"\n1. Devide the program into ``\\delta`` segments, each segment having size $\\eta(\\delta, \\tau) = \\frac{(\\delta+\\tau)!}{\\delta! \\tau!}$, where ``\\delta=1,...,d`` and ``\\tau=t-1``.\n2. Cache the first state of each segment,\n3. Compute gradients in the last segment,\n4. Deallocate last checkpoint,\n5. Devide the second last segments into two parts.\n6. Recursively apply treeverse (Step 2-5).\n\"\"\"\n\n# ╔═╡ 01c709c7-806c-4389-bbb2-4081e64426d9\nmd\"total number of steps ``T = \\eta(d, t)``, both ``t`` and ``d`` can be logarithmic\"\n\n# ╔═╡ b1e0cf83-4337-4044-a7d1-5fca8ae79268\nmd\"## An example\"\n\n# ╔═╡ 71f4b476-027d-4c8f-b561-1ee418bc9e61\nhtml\"\"\"\n\n\"\"\"\n\n# ╔═╡ 042013cf-9cd2-409d-827f-a311a2f8ce62\nmd\"\"\"\n* black dot: current step,\n* gray dot: checkpointed state,\n* empty dot: state deallocated in current step,\n* red square: gradient computed.\n\"\"\"\n\n# ╔═╡ 82593cd0-1403-4597-8370-919c80494479\nmd\"# Program is not always linear!\"\n\n# ╔═╡ f58720b5-2bcb-4950-b453-bd59f648c66a\nmd\"You think your program is like\"\n\n# ╔═╡ 4576d791-6af7-4ba5-9b80-fe99c0bb2e88\nlet\n\tCompose.set_default_graphic_size(15cm, 3cm)\n\tnb = nodestyle(:circle, r=0.01)\n\teb = compose(context(), bondstyle(:default, r=0.1), Compose.arrow(), linewidth(0.2mm))\n\tloc(i) = (i/11, 0.5)\n\teloc(i) = (loc(i-1) .- (-0.02, 0.0), loc(i) .- (0.025, 0.0))\n\tcanvas() do\n\t\tfor i=1:10\n\t\t\tnb >> loc(i)\n\t\t\ti == 1 || eb >> eloc(i)\n\t\tend\n\tend\nend\n\n# ╔═╡ 6e9d17f1-b17d-4e8d-82a3-921558a20c0f\nmd\"or a DAG (directed acyclic graph)\"\n\n# ╔═╡ f18d89f5-1129-43e0-8b4a-5c1fcd618eab\nlet\n\tCompose.set_default_graphic_size(15cm, 3cm)\n\tnb = nodestyle(:circle, r=0.01)\n\teb = compose(context(), bondstyle(:default, r=0.1), Compose.arrow(), linewidth(0.2mm))\n\tloc(i) = (i/11, 0.2)\n\tloc2(i) = (i/11, 0.7)\n\teloc(i, j) = shrink(loc(i), loc(j), 0.02, 0.025)\n\teloc2(i, j) = shrink(loc2(i), loc2(j), 0.02, 0.025)\n\teloc12(i, j) = shrink(loc(i), loc2(j), 0.1, 0.15)\n\teloc21(i, j) = shrink(loc2(i), loc(j), 0.05, 0.1)\n\tcanvas() do\n\t\tfor i=1:10\n\t\t\tnb >> loc(i)\n\t\t\ti == 1 || eb >> eloc(i-1,i)\n\t\tend\n\t\tfor i=2:5\n\t\t\tnb >> loc2(i)\n\t\t\ti == 2 || eb >> eloc2(i-1, i)\n\t\tend\n\t\teb >> eloc12(2,2)\n\t\teb >> eloc12(4,5)\n\t\teb >> eloc21(5,7)\n\tend\nend\n\n# ╔═╡ 2912c7ed-75e3-4dfd-9c40-92115cc08194\nmd\"The truth is\"\n\n# ╔═╡ 5d1517c0-562b-40db-bec2-32b5494de1b8\nlet\n\tCompose.set_default_graphic_size(15cm, 3cm)\n\tnb = nodestyle(:circle, r=0.01)\n\ttb = textstyle(:default)\n\teb = compose(context(), bondstyle(:default, r=0.1), Compose.arrow(), linewidth(0.2mm))\n\teb2 = compose(context(), bondstyle(:dcurve, r=0.8), Compose.arrow(), linewidth(0.2mm))\n\tloc(i) = (i/11, 0.2)\n\tloc2(i) = (i/11, 0.7)\n\teloc(i, j) = shrink(loc(i), loc(j), 0.02, 0.025)\n\teloc2(i, j) = shrink(loc2(j), loc2(i), 0.02, 0.025)\n\teloc12(i, j) = shrink(loc2(j), loc(i), 0.1, 0.15)\n\teloc21(i, j) = shrink(loc(j), loc2(i), 0.05, 0.1)\n\tcanvas() do\n\t\tfor i=1:10\n\t\t\tnb >> loc(i)\n\t\t\ti == 1 || eb >> eloc(i-1,i)\n\t\tend\n\t\tfor i=2:5\n\t\t\tnb >> loc2(i)\n\t\t\ti == 2 || eb >> eloc2(i-1, i)\n\t\tend\n\t\teb >> eloc12(2,2)\n\t\teb >> eloc12(4,5)\n\t\ttb >> ((0.3, 0.45), \"× n\")\n\t\t\n\t\tfor i=7:8\n\t\t\tnb >> loc2(i)\n\t\t\ti == 7 || eb >> eloc2(i-1, i)\n\t\tend\n\t\teb >> eloc12(7,7)\n\t\teb >> eloc12(8,8)\n\t\ttb >> ((0.68, 0.45), \"× ∞\")\n\t\t\n\t\teb2 >> (loc(6) .+ (0.0, 0.1), loc(9) .+ (0, 0.15))\n\tend\nend\n\n# ╔═╡ ae096ad2-3ae9-4440-a959-0d7d9a174f1d\nmd\"## Example 3: Sparse matrix multiplication\"\n\n# ╔═╡ 8148bc1f-ef99-40a4-a5ce-0a42643f703d\nmd\"original implementation: [https://github.com/JuliaLang/julia/blob/master/stdlib/SparseArrays/src/linalg.jl](https://github.com/JuliaLang/julia/blob/master/stdlib/SparseArrays/src/linalg.jl)\n\"\n\n# ╔═╡ bd86c5c2-16be-4cfd-ba7a-a0e2544d82d1\n@i function mul!(C::StridedVecOrMat{T}, A::SparseMatrixCSC{T}, B::StridedVecOrMat{T}, α::Number) where T\n @safe A.n == size(B, 1) || throw(DimensionMismatch())\n @safe A.m == size(C, 1) || throw(DimensionMismatch())\n @safe size(B, 2) == size(C, 2) || throw(DimensionMismatch())\n @invcheckoff for k = 1:size(C, 2)\n @inbounds for col = 1:A.n\n @routine begin\n αxj ← zero(T)\n αxj += α*B[col,k]\n end\n for j = A.colptr[col]:(A.colptr[col + 1] - 1)\n C[A.rowval[j], k] += A.nzval[j]*αxj\n end\n ~@routine\n end\n end\nend\n\n# ╔═╡ 11557d6b-3a1e-416d-874f-b8d217976f76\nmd\"## Example 4: How to differentiate QR\"\n\n# ╔═╡ 48a10ea2-5d32-4a55-b8c0-f6a5e82eace9\nmd\"original implementation: [https://github.com/JuliaLang/julia/blob/master/stdlib/LinearAlgebra/src/qr.jl](https://github.com/JuliaLang/julia/blob/master/stdlib/LinearAlgebra/src/qr.jl)\n\"\n\n# ╔═╡ fafc1b0f-6469-4b6c-a00d-5272a45fc69b\nmd\"See also\"\n\n# ╔═╡ ad6cff7b-5cbf-4ab1-94f7-d21cbc171000\nleftright(html\"\", md\"**Matrix computations**\n\t\nGolub, Gene H., and Charles F. Van Loan (2013)\")\n\n# ╔═╡ 4d373cf6-9b39-44bc-8f13-220933fc8f5c\nfunction qrfactPivotedUnblocked!(A::AbstractMatrix)\n m, n = size(A)\n piv = Vector(UnitRange{BlasInt}(1,n))\n τ = Vector{eltype(A)}(undef, min(m,n))\n for j = 1:min(m,n)\n\n # Find column with maximum norm in trailing submatrix\n jm = indmaxcolumn(view(A, j:m, j:n)) + j - 1\n\n if jm != j\n # Flip elements in pivoting vector\n tmpp = piv[jm]\n piv[jm] = piv[j]\n piv[j] = tmpp\n\n # Update matrix with\n for i = 1:m\n tmp = A[i,jm]\n A[i,jm] = A[i,j]\n A[i,j] = tmp\n end\n end\n\n # Compute reflector of columns j\n x = view(A, j:m, j)\n τj = LinearAlgebra.reflector!(x)\n τ[j] = τj\n\n # Update trailing submatrix with reflector\n LinearAlgebra.reflectorApply!(x, τj, view(A, j:m, j+1:n))\n end\n return LinearAlgebra.QRPivoted{eltype(A), typeof(A)}(A, τ, piv)\nend\n\n# ╔═╡ 293a68ca-e02f-47b3-85ed-aeeb8995f3ec\nstruct Reflector{T,RT,VT<:AbstractVector{T}}\n ξ::T\n normu::RT\n sqnormu::RT\n r::T\n y::VT\nend\n\n# ╔═╡ fa5716f9-8bff-4295-812b-691ccdc12832\nstruct QRPivotedRes{T,RT,VT}\n factors::Matrix{T}\n τ::Vector{T}\n jpvt::Vector{Int}\n reflectors::Vector{Reflector{T,RT,VT}}\n vAs::Vector{Vector{T}}\n jms::Vector{Int}\nend\n\n# ╔═╡ 8324f365-fd12-4ca3-8ca6-657e5917f946\n# Elementary reflection similar to LAPACK. The reflector is not Hermitian but\n# ensures that tridiagonalization of Hermitian matrices become real. See lawn72\n@i function reflector!(R::Reflector{T,RT}, x::AbstractVector{T}) where {T,RT}\n n ← length(x)\n @inbounds @invcheckoff if n != 0\n @zeros T ξ1\n @zeros RT normu sqnormu\n ξ1 += x[1]\n sqnormu += abs2(ξ1)\n for i = 2:n\n sqnormu += abs2(x[i])\n end\n if !iszero(sqnormu)\n normu += sqrt(sqnormu)\n if real(ξ1) < 0\n NEG(normu)\n end\n ξ1 += normu\n R.y[1] -= normu\n for i = 2:n\n R.y[i] += x[i] / ξ1\n end\n R.r += ξ1/normu\n end\n SWAP(R.ξ, ξ1)\n SWAP(R.normu, normu)\n SWAP(R.sqnormu, sqnormu)\n end\nend\n\n# ╔═╡ 70fb10ea-9229-46ef-8ba3-b1d3874b7929\n# apply reflector from left\n@i function reflectorApply!(vA::AbstractVector{T}, x::AbstractVector, τ::Number, A::StridedMatrix{T}) where T\n (m, n) ← size(A)\n if length(x) != m || length(vA) != n\n @safe throw(DimensionMismatch(\"reflector has length ($(length(x)), $(length(vA))), which must match the first dimension of matrix A, ($m, $n)\"))\n end\n @inbounds @invcheckoff if m != 0\n for j = 1:n\n # dot\n @zeros T vAj vAj_τ\n vAj += A[1, j]\n for i = 2:m\n vAj += x[i]'*A[i, j]\n end\n vAj_τ += τ' * vAj\n # ger\n A[1, j] -= vAj_τ\n for i = 2:m\n A[i, j] -= x[i]*vAj_τ\n end\n vAj_τ -= τ' * vAj\n SWAP(vA[j], vAj)\n end\n end\nend\n\n# ╔═╡ 51504ba4-4711-48b7-aab9-d4f26c009659\nfunction alloc(::typeof(reflector!), x::AbstractVector{T}) where T\n\tRT = real(T)\n\tReflector(zero(T), zero(RT), zero(RT), zero(T), zero(x))\nend\n\n# ╔═╡ f267e315-3c19-4345-8fba-641bb0ea515b\n@i function qr_pivoted!(res::QRPivotedRes, A::StridedMatrix{T}) where T\n m, n ← size(A)\n @invcheckoff @inbounds for j = 1:min(m,n)\n # Find column with maximum norm in trailing submatrix\n jm ← LinearAlgebra.indmaxcolumn(NiLang.value.(view(A, j:m, j:n))) + j - 1\n\n if jm != j\n # Flip elements in pivoting vector\n SWAP(res.jpvt[jm], res.jpvt[j])\n\n # Update matrix with\n for i = 1:m\n SWAP(A[i, jm], A[i, j])\n end\n end\n\n # Compute reflector of columns j\n R ← alloc(reflector!, A |> subarray(j:m, j))\n vA ← zeros(T, n-j)\n reflector!(R, A |> subarray(j:m, j))\n # Update trailing submatrix with reflector\n reflectorApply!(vA, R.y, R.r, A |> subarray(j:m, j+1:n))\n for i=1:length(R.y)\n SWAP(R.y[i], A[j+i-1, j])\n end\n PUSH!(res.reflectors, R)\n PUSH!(res.vAs, vA)\n PUSH!(res.jms, jm)\n R → _zero(Reflector{T,real(T),Vector{T}})\n vA → zeros(T, 0)\n jm → 0\n end\n @inbounds for i=1:length(res.reflectors)\n res.τ[i] += res.reflectors[i].r\n end\n res.factors += A\nend\n\n# ╔═╡ a07b93b1-742b-41d4-bd0f-bc899de55338\nfunction alloc_qr(A::AbstractMatrix{T}) where T\n\t(m, n) = size(A)\n\tτ = zeros(T, min(m,n))\n\tjpvt = collect(1:n)\n\treflectors = Reflector{T,real(T),Vector{T}}[]\n\tvAs = Vector{T}[]\n\tjms = Int[]\n\tQRPivotedRes(zero(A), τ, jpvt, reflectors, vAs, jms)\nend\n\n# ╔═╡ 5f207f59-b9f4-477f-b79f-0aee743bdb8e\nA = randn(ComplexF64, 20, 20);\n\n# ╔═╡ f88517d6-b87d-45ba-bf3f-67074fa51fca\n@test qr_pivoted!(alloc_qr(A), copy(A))[1].factors ≈ LinearAlgebra.qrfactPivotedUnblocked!(copy(A)).factors\n\n# ╔═╡ 45aef837-9b2c-49b2-b815-e4d60f103f58\nlet\n\t@testset \"qr pivoted gradient\" begin\n\t\t# rank deficient initial matrix\n\t\tn = 50\n\t\tU = LinearAlgebra.qr(randn(n, n)).Q\n\t\tΣ = Diagonal((x=randn(n); x[n÷2+1:end] .= 0; x))\n\t\tA = U*Σ*U'\n\t\tres = alloc_qr(A)\n\t\t@test rank(A) == n ÷ 2\n\t\tqrres = qr_pivoted!(deepcopy(res), copy(A))[1]\n\t\t@test count(x->(x>1e-12), sum(abs2, QRPivoted(qrres.factors, qrres.τ, qrres.jpvt).R, dims=2)) == n ÷ 2\n\n\t\t@i function loss(y, qrres, A)\n\t\t\tqr_pivoted!(qrres, A)\n\t\t\ty += abs(qrres.factors[1])\n\t\tend\n\t\tnrloss(A) = loss(0.0, deepcopy(res), A)[1]\n\t\tngA = zero(A)\n\t\tδ = 1e-5\n\t\tfor j=1:size(A, 2)\n\t\t\tfor i=1:size(A, 1)\n\t\t\t\tA_ = copy(A)\n\t\t\t\tA_[i,j] -= δ/2\n\t\t\t\tl1 = nrloss(copy(A_))\n\t\t\t\tA_[i,j] += δ\n\t\t\t\tl2 = nrloss(A_)\n\t\t\t\tngA[i,j] = (l2-l1)/δ\n\t\t\tend\n\t\tend\n\t\tgA = NiLang.AD.gradient(loss, (0.0, res, A); iloss=1)[3]\n\t\t@test real.(gA) ≈ ngA\n\tend\nend\n\n# ╔═╡ Cell order:\n# ╟─a1ef579e-4b66-4042-944e-7e27c660095e\n# ╟─100b4293-fd1e-4b9c-a831-5b79bc2a5ebe\n# ╟─f11023e5-8f7b-4f40-86d3-3407b61863d9\n# ╟─9d11e058-a7d0-11eb-1d78-6592ff7a1b43\n# ╟─b73157bf-1a77-47b8-8a06-8d6ec2045023\n# ╟─ec13e0a9-64ff-4f66-a5a6-5fef53428fa1\n# ╟─f8b0d1ce-99f7-4729-b46e-126da540cbbe\n# ╟─435ac19e-1c0c-4ee5-942d-f2a97c8c4d80\n# ╟─48ecd619-d01d-43ff-8b52-7c2566c3fa2b\n# ╟─4878ce45-40ff-4fae-98e7-1be41e930e4d\n# ╠═ce44f8bd-692e-4eab-9ba4-055b25e40c81\n# ╠═b2c1936c-2c27-4fbb-8183-e38c5e858483\n# ╠═8be1b812-fcac-404f-98aa-0571cb990f34\n# ╟─33e0c762-c75e-44aa-bfe2-bff92dd1ace8\n# ╟─c59c35ee-1907-4736-9893-e22c052150ca\n# ╠═0ae13734-b826-4dbf-93d1-11044ce88bd4\n# ╠═99187515-c8be-49c2-8d70-9c2998d9993c\n# ╟─78ca6b08-84c4-4e4d-8412-ae6c28bfafce\n# ╠═f12b25d8-7c78-4686-b46d-00b34e565605\n# ╟─d90c3cc9-084d-4cf7-9db7-42cea043030b\n# ╟─93c98cb2-18af-47df-afb3-8c5a34b4723c\n# ╟─2dc74e15-e2ea-4961-b43f-0ada1a73d80a\n# ╟─7ee75a15-eaea-462a-92b6-293813d2d4d7\n# ╟─02a25b73-7353-43b1-8738-e7ca472d0cc7\n# ╟─2afb984f-624e-4381-903f-ccc1d8a66a17\n# ╟─7e5d5e69-90f2-4106-8edf-223c150a8168\n# ╟─92d7a938-9463-4eee-8839-0b8c5f762c79\n# ╟─4b1a0b59-ddc6-4b2d-b5f5-d92084c31e46\n# ╟─81f16b8b-2f0b-4ba3-8c26-6669eabf48aa\n# ╟─fb6c3a48-550a-4d2e-a00b-a1e40d86b535\n# ╟─ab6fa4ac-29ed-4722-88ed-fa1caf2072f3\n# ╟─8e72d934-e307-4505-ac82-c06734415df6\n# ╟─e6ff86a9-9f54-474b-8111-a59a25eda506\n# ╟─9c1d9607-a634-4350-aacd-2d40984d647d\n# ╟─63db2fa2-50b2-4940-b8ee-0dc6e3966a57\n# ╟─693167e7-e80c-401d-af89-55b5fae30848\n# ╟─4cd70901-2142-4868-9a33-c46ca0d064ec\n# ╟─89018a35-76f4-4f23-b15a-a600db046d6f\n# ╟─1d219222-0778-4c37-9182-ed5ccbb3ef32\n# ╟─4ff09f7c-aeac-48bd-9d58-8446137c3acd\n# ╟─ea44037b-9359-4fbd-990f-529d88d54351\n# ╟─e731a8e3-6462-4a60-83e9-6ab7ddfff50e\n# ╟─685c2b28-b071-452c-a881-801128dcb6c3\n# ╟─177ddfc2-2cbe-4dba-9d05-2857633dd1ae\n# ╟─6c2a3a93-385f-4758-9b6e-4cb594a8e856\n# ╟─fb8168c2-8489-418b-909b-cede57b5ae64\n# ╟─fdb39284-dbb1-49fa-9a1c-f360f9e6b765\n# ╟─60214f22-c8bb-4a32-a882-4e6c727b29a9\n# ╟─7a6dbe09-cb7f-405f-b9b5-b350ca170e5f\n# ╟─5dc4a849-76dd-4c4f-8828-755671839e5e\n# ╟─b053f11b-9ed7-47ff-ab32-0c70b87e71ed\n# ╟─7b1aa6dd-647f-44cb-b580-b58e23e8b5a6\n# ╟─b96bac75-b4ad-45f7-aeec-cb6a387eebf0\n# ╟─5fe022eb-6a17-466e-a6d0-d67e82af23cd\n# ╟─92047e95-7eba-4021-9668-9bb4b92261d7\n# ╟─e2ae1084-8759-4f27-8ad1-43a88e434a3d\n# ╠═edd3aea8-abdb-4e12-9ef9-12ac0fff835b\n# ╠═a2904efb-186c-449d-b1aa-caf530f88e91\n# ╠═14faaf82-ad3e-4192-8d48-84adfa30442d\n# ╠═5d141b88-ec07-4a02-8eb3-37405e5c9f5d\n# ╠═0907e683-f216-4cf6-a210-ae5181fdc487\n# ╠═805c7072-98fa-4086-a69d-2e126c55af36\n# ╠═7e527024-c294-4c16-8626-9953588d9b6a\n# ╠═0bbfa106-f465-4a7b-80a7-7732ba435822\n# ╠═3e59c65a-ceed-42ed-be64-a6964db016e7\n# ╟─29f85d05-99fd-4843-9be0-5663e681dad7\n# ╠═9a46597c-b1ee-4e3b-aed1-fd2874b6e77a\n# ╠═e7830e55-bd9e-4a8a-9239-4191a5f0b1d1\n# ╟─de2cd247-ba68-4ba4-9784-27a743478635\n# ╟─dc929c23-7434-4848-847a-9fa696e84776\n# ╟─4f1df03f-c315-47b1-b181-749e1231594c\n# ╠═ccd38f52-104d-434a-aea3-dd94e571374f\n# ╠═7eccba6a-3ad5-440b-9c5d-392dc8dc7aba\n# ╠═f4230251-ba54-434a-b86b-f972c7389217\n# ╟─4a858a3e-ce28-4642-b061-3975a3ed99ff\n# ╠═674bb3bb-637b-44f2-bf6d-d1678da03fbd\n# ╠═5a59d96f-b2f1-4564-82c7-7f0fe181afb8\n# ╠═55d2f8ee-4f77-4d44-b704-30643dbbab84\n# ╠═14951168-97c2-43ae-8d5e-5506408a2bb2\n# ╠═4f564581-6032-449c-8b15-3c741f44237a\n# ╠═a36516e8-76c1-4bff-8a12-3e1e621b857d\n# ╠═402b861c-d363-4d23-b9e9-eb088f57b5c4\n# ╠═63975a80-1b41-4f55-91a1-4a316ad7bf26\n# ╠═6f688f88-432a-42b2-a2db-19d6bb282e0a\n# ╠═fb46db14-f7e0-4f01-9096-02334c62942d\n# ╟─b2c3db3d-c250-4daa-8453-3c9a2734aede\n# ╠═69dc2685-b70f-4a81-af30-f02e0054bd52\n# ╠═9a986264-5ba7-4697-a00d-711f8efe29f0\n# ╠═560cf3e9-0c14-4497-85b9-f07045eea32a\n# ╠═8ab79efc-e8d0-4c6f-81df-a89008142bb7\n# ╠═0eec318c-2c09-4dd6-9187-9c0273d29915\n# ╠═1f0ef29c-0ad5-4d97-aeed-5ff44e86577a\n# ╠═603d8fc2-5e7b-4d55-92b6-208b25ea6569\n# ╟─2b3c765e-b505-4f07-9bcb-3c8cc47364ad\n# ╠═e0f266da-7e65-4398-bfd4-a6c0b54e626b\n# ╟─e1d35886-79d0-40a5-bd33-1c4e5f4a0a9a\n# ╠═b63a30b0-c75b-4998-a2b2-0b79574cab81\n# ╟─139bf020-c4a8-45c8-96fa-aeebc7ddaedc\n# ╠═8967c0f0-89f8-4893-b11b-253333d1a823\n# ╟─f2540450-5a07-4fb8-93fb-a6d48dd36a56\n# ╠═3acb2cfd-fa29-4a2b-8f23-f5aaf474edd0\n# ╟─aa1547f2-5edd-4b7e-b93e-bdfc4e4fc6d5\n# ╟─6e76a107-4f51-4e32-b133-7b6e04d7d107\n# ╟─999f7a8f-d72e-4ccd-8cbf-b5bbb7db1842\n# ╟─32772c2a-6b80-4779-963c-06974ff0d832\n# ╟─41642bd5-1321-490a-95ad-4c1d6363456f\n# ╟─2a553e32-05ef-4c2d-aba7-41185c6035d4\n# ╟─ab8345ce-e038-4d6b-9e1f-57e4f33bb67b\n# ╟─bb9c9a4c-601a-4708-9b2d-04d1583938f2\n# ╟─b9917e94-c33d-423f-a478-3252bacc2494\n# ╟─4978f404-11ff-41b8-a673-f2d051b1f526\n# ╟─73bd2e3b-902f-461b-860f-246257608ecd\n# ╟─4dd47dc8-6dfa-47a4-a088-689b4b870762\n# ╟─ecd975d2-9374-4f40-80ac-2cceda11e7fb\n# ╟─832cc81d-a49d-46e7-9d2b-d8bde9bb1273\n# ╟─2192a1de-1042-4b13-a313-b67de489124c\n# ╟─01c709c7-806c-4389-bbb2-4081e64426d9\n# ╟─b1e0cf83-4337-4044-a7d1-5fca8ae79268\n# ╟─71f4b476-027d-4c8f-b561-1ee418bc9e61\n# ╟─042013cf-9cd2-409d-827f-a311a2f8ce62\n# ╟─82593cd0-1403-4597-8370-919c80494479\n# ╟─f58720b5-2bcb-4950-b453-bd59f648c66a\n# ╟─4576d791-6af7-4ba5-9b80-fe99c0bb2e88\n# ╟─6e9d17f1-b17d-4e8d-82a3-921558a20c0f\n# ╟─f18d89f5-1129-43e0-8b4a-5c1fcd618eab\n# ╟─2912c7ed-75e3-4dfd-9c40-92115cc08194\n# ╟─5d1517c0-562b-40db-bec2-32b5494de1b8\n# ╟─ae096ad2-3ae9-4440-a959-0d7d9a174f1d\n# ╟─8148bc1f-ef99-40a4-a5ce-0a42643f703d\n# ╠═200f1848-0980-4185-919a-93ab2e7f788f\n# ╠═bd86c5c2-16be-4cfd-ba7a-a0e2544d82d1\n# ╟─11557d6b-3a1e-416d-874f-b8d217976f76\n# ╟─48a10ea2-5d32-4a55-b8c0-f6a5e82eace9\n# ╟─fafc1b0f-6469-4b6c-a00d-5272a45fc69b\n# ╟─ad6cff7b-5cbf-4ab1-94f7-d21cbc171000\n# ╠═30c191c5-642b-4062-98f3-643d314a054d\n# ╠═fa5716f9-8bff-4295-812b-691ccdc12832\n# ╠═f267e315-3c19-4345-8fba-641bb0ea515b\n# ╠═4d373cf6-9b39-44bc-8f13-220933fc8f5c\n# ╠═293a68ca-e02f-47b3-85ed-aeeb8995f3ec\n# ╠═8324f365-fd12-4ca3-8ca6-657e5917f946\n# ╠═70fb10ea-9229-46ef-8ba3-b1d3874b7929\n# ╠═51504ba4-4711-48b7-aab9-d4f26c009659\n# ╠═a07b93b1-742b-41d4-bd0f-bc899de55338\n# ╠═864dbde7-b689-4165-a08e-6bbbd72190de\n# ╠═5f207f59-b9f4-477f-b79f-0aee743bdb8e\n# ╠═f88517d6-b87d-45ba-bf3f-67074fa51fca\n# ╠═45aef837-9b2c-49b2-b815-e4d60f103f58\n" }, { "path": "notebooks/basic.jl", "content": "### A Pluto.jl notebook ###\n# v0.14.5\n\nusing Markdown\nusing InteractiveUtils\n\n# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).\nmacro bind(def, element)\n quote\n local el = $(esc(element))\n global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing\n el\n end\nend\n\n# ╔═╡ 1ef174fa-16f0-11eb-328a-afc201effd2f\nusing Pkg, Printf\n\n# ╔═╡ 55cfdab8-d792-11ea-271f-e7383e19997c\nusing PlutoUI;\n\n# ╔═╡ 9e509f80-d485-11ea-0044-c5b7e750aacb\nusing NiLang\n\n# ╔═╡ 37ed073a-d492-11ea-156f-1fb155128d0f\nusing Zygote, BenchmarkTools\n\n# ╔═╡ 4d75f302-d492-11ea-31b9-bbbdb43f344e\nusing NiLang.AD\n\n# ╔═╡ 627ea2fb-6530-4ea0-98ee-66be3db54411\nhtml\"\"\"\n\n\"\"\"\n\n# ╔═╡ 94b2b962-e02a-11ea-09a5-81b3226891ed\nmd\"\"\"# 连猩猩都能懂的可逆编程\n### (Reversible programming made simple)\n[https://github.com/JuliaReverse/NiLangTutorial/](https://github.com/JuliaReverse/NiLangTutorial/)\n\n$(html\"
\")\n\n**Jinguo Liu** (github: [GiggleLiu](https://github.com/GiggleLiu/))\n\n*Postdoc, Institute of physics, Chinese academy of sciences* (when doing this project)\n\n*Consultant, QuEra Computing* (current)\n\n*Postdoc, Havard* (soon)\n\"\"\"\n\n# ╔═╡ a5ee60c8-e02a-11ea-3512-7f481e499f23\nmd\"\"\"\n# Table of Contents\n1. Reversible programming basics\n2. Differentiate everything with a reversible programming language\n4. Real world applications and benchmarks\n\"\"\"\n\n# ╔═╡ a11c4b60-d77d-11ea-1afe-1f2ab9621f42\nmd\"\"\"\n## In this talk,\nWe use the reversible eDSL [NiLang](https://github.com/GiggleLiu/NiLang.jl) is a [Julia](https://julialang.org/) as our reversible programming tool.\n\nA package that can differentiate everything.\n\n![NiLang](https://raw.githubusercontent.com/GiggleLiu/NiLang.jl/master/docs/src/asset/logo3.png)\n\nAuthors:\n[GiggleLiu](https://github.com/GiggleLiu), [Taine Zhao](https://github.com/thautwarm)\n\"\"\"\n\n# ╔═╡ e54a1be6-d485-11ea-0262-034c56e0fda8\nmd\"\"\"\n## Sec I. Reversible programming basic\n\n### Reversible function definition\n\nA reversible function `f` is defined as\n```julia\n(~f)(f(x, y, z...)...) == (x, y, z...)\n```\n\"\"\"\n\n# ╔═╡ d1628f08-ddfb-11ea-241a-c7e6c1a22212\nmd\"\"\"\n## Example 1: reversible adder\n```math\n\\begin{align}\nf &: x, y → x+y, y\\\\\n{\\small \\mathrel{\\sim}}f &: x, y → x-y, y\n\\end{align}\n```\n\"\"\"\n\n# ╔═╡ 278ac6b6-e02c-11ea-1354-cd7ecd1099be\nmd\"The reversible macro `@i` defines two functions, the function itself and its inverse.\"\n\n# ╔═╡ a28d38be-d486-11ea-2c40-a377b74a05c1\n@i function reversible_plus(x, y)\n\tx += y\nend\n\n# ╔═╡ e93f0bf6-d487-11ea-1baa-21d51ddb4a20\nreversible_plus(2.0, 3.0)\n\n# ╔═╡ fc932606-d487-11ea-303e-75ca8b7a02f6\n(~reversible_plus)(5.0, 3.0)\n\n# ╔═╡ e3d2b23a-ddfb-11ea-0f5e-e72ed299bb45\nmd\"## The difference to a regular programming language\"\n\n# ╔═╡ a961e048-ddf2-11ea-0262-6d19eb82b36b\nmd\"**Comment 1**: The return statement is not allowed, a reversible function returns input arguments directly.\"\n\n# ╔═╡ 2d22f504-ddf1-11ea-28ec-5de6f4ee79bb\nmd\"**Comment 2**: Every operation is reversible. `+=` is considered as reversible for integers and floating point numbers in NiLang, although for floating point numbers, there are *rounding errors*.\"\n\n# ╔═╡ 7d08ac24-e143-11ea-2085-539fd9e35889\nmd\"### A case where `+=` is not reversible\"\n\n# ╔═╡ 9fcdd77c-e0df-11ea-09e6-49a2861137e5\nlet\n\tx, y = 1e-20, 1e20\n\tx += y\n\tx -= y\n\t(x, y)\nend\n\n# ╔═╡ 0a1a8594-ddfc-11ea-119a-1997c86cd91b\nmd\"\"\"\n## Use this function\n\"\"\"\n\n# ╔═╡ 0b4edb1a-ddf0-11ea-220c-91f2df7452e7\n@i function reversible_plus2(x, y)\n\treversible_plus(x, y) # equivalent to `reversible_plus(x, y)`\n\treversible_plus(x, y)\nend\n\n# ╔═╡ f875ecd6-ddef-11ea-22a1-619809d15b37\nmd\"**Comment**: Inside a reversible function definition, a statement changes a variable *inplace*\"\n\n# ╔═╡ e7557bee-e0cc-11ea-1788-411e759b4766\nreversible_plus2(2.0, 3.0)\n\n# ╔═╡ cd7b2a2e-ddf5-11ea-04c4-f7583bbb5a53\nmd\"A statement can be **uncalled** with `~`\"\n\n# ╔═╡ bc98a824-ddf5-11ea-1a6a-1f795452d3d0\n@i function do_nothing(x, y)\n\treversible_plus(x, y)\n\t~reversible_plus(x, y) # uncall the expression\nend\n\n# ╔═╡ 05f8b91c-e0cd-11ea-09e3-f3c5c0e07e63\ndo_nothing(2.0, 3.0)\n\n# ╔═╡ ac302844-e07b-11ea-35dd-e3e06054401b\nmd\"## Example 2: Compute $x^5$\"\n\n# ╔═╡ b722e098-e07b-11ea-3483-01360fb6954e\n@i function naive_power5(y, x::T) where T\n\ty = one(T) # error 1: `=` is not reversible\n\tfor i=1:5\n\t\ty *= x # error 2: `*=` is not reversible\n\tend\nend\n\n# ╔═╡ bf8b722c-dfa4-11ea-196a-719802bc23c5\nmd\"\"\"\n## Compute $x^5$ reversibly\n\"\"\"\n\n# ╔═╡ 330edc28-dfac-11ea-35a5-3144c4afbfcf\nmd\"note: `*=` is not reversible for usual number systems\"\n\n# ╔═╡ 0a679e04-dfa7-11ea-0288-a1fa490c4387\n@i function power5(x5, x4, x3, x2, x1, x)\n\tx1 += x\n\tx2 += x1 * x\n\tx3 += x2 * x\n\tx4 += x3 * x\n\tx5 += x4 * x\nend\n\n# ╔═╡ cc32cae8-dfab-11ea-0d0b-c70ea8de720a\npower5(0.0, 0.0, 0.0, 0.0, 0.0, 2.0)\n\n# ╔═╡ b4240c16-dfac-11ea-3a40-33c54436e3a3\nmd\"# Don't make me so many input arguments!\"\n\n# ╔═╡ ade52358-dfac-11ea-2dd3-d3a691e7a8a2\n@i function power5_twoinputs(x5, x::T) where T\n\tx1 ← zero(T)\n\tx2 ← zero(T)\n\tx3 ← zero(T)\n\tx4 ← zero(T)\n\tx1 += x\n\tx2 += x1 * x\n\tx3 += x2 * x\n\tx4 += x3 * x\n\t\n\tx5 += x4 * x\n\t\n\tx4 -= x3 * x\n\tx3 -= x2 * x\n\tx2 -= x1 * x\n\tx1 -= x\n\tx4 → zero(T)\n\tx3 → zero(T)\n\tx2 → zero(T)\n\tx1 → zero(T)\nend\n\n# ╔═╡ d86e2e5e-dfab-11ea-0053-6d52f1164bc5\npower5_twoinputs(0.0, 2.0)\n\n# ╔═╡ 7951b9ec-e030-11ea-32ee-b1de49378186\nmd\"\"\"\n**Comment**:\n`n ← zero(T)` is the variable allocation operation. It means\n```\nif n is defined\n\terror\nelse\n\tn = zero(T)\nend\n```\nIts inverse is `n → zero(T)`. It means\n```\n@assert n == zero(T)\ndeallocate(n)\n```\n\"\"\"\n\n# ╔═╡ 6bc97f5e-dfad-11ea-0c43-e30b6620e6e8\nmd\"# Shorter: compute-copy-uncompute\"\n\n# ╔═╡ 80d24e9e-dfad-11ea-1dae-49568d534f10\n@i function power5_twoinputs_shorter(x5, x::T) where T\n\t@routine begin # compute\n\t\t@zeros T x1 x2 x3 x4\n\t\tx1 += x\n\t\tx2 += x1 * x\n\t\tx3 += x2 * x\n\t\tx4 += x3 * x\n\tend\n\t\n\tx5 += x4 * x # copy\n\t\n\t~@routine # uncompute\nend\n\n# ╔═╡ a8092b18-dfad-11ea-0989-474f37d05f73\npower5_twoinputs_shorter(0.0, 2.0)\n\n# ╔═╡ 43f0c2fc-e030-11ea-25d9-b323e6496a35\nmd\"\"\"**Comment**:\n```\n@routine statement\n~@routine\n```\n\nis equivalent to\n```\nstatement\n~(statement)\n```\nThis is the famous `compute-copy-uncompute` design pattern in reversible computing. Check this [reference](https://epubs.siam.org/doi/10.1137/0219046).\n\"\"\"\n\n# ╔═╡ b4ad5830-dfad-11ea-0057-055dda8cc9be\nmd\"# How to compute x^1000?\"\n\n# ╔═╡ cf576d38-dfad-11ea-2682-7bd540db44a5\n@i function power1000(x1000, x::T) where T\n\t@routine begin\n\t\txs ← zeros(T, 1000)\n\t\txs[1] += 1\n\t\tfor i=2:1000\n\t\t\txs[i] += xs[i-1] * x\n\t\tend\n\tend\n\t\n\tx1000 += xs[1000] * x\n\t\n\t~@routine\nend\n\n# ╔═╡ 35fff53c-dfae-11ea-3602-918a17d5a5fa\npower1000(0.0, 1.001)\n\n# ╔═╡ 9b9b5328-e030-11ea-1d00-f3341572734a\nhtml\"\"\"\n
For loop
\n
\n
\n\t
Forward
\n\t
\n\tfor i=start:step:stop\n\t\t# do something\n\tend\n\t
\n
\n
\n\t
Reverse
\n\t
\n\tfor i=stop:-step:start\n\t\t# undo something\n\tend\n\t\n\t
\n
\n
\n\"\"\"\n\n# ╔═╡ f3b87892-e080-11ea-353d-8d81c52cf9ac\nmd\"### Sometimes, a for loop can break down\"\n\n# ╔═╡ b27a3974-e030-11ea-0bcd-7f7035d55165\n@i function power1000_bad(x1000, x::T) where T\n\t@routine begin\n\t\txs ← zeros(T, 1000)\n\t\txs[1] += 1\n\t\tfor i=2:length(xs)\n\t\t\txs[i] += xs[i-1] * x\n\t\t\tPUSH!(xs, @const zero(T))\n\t\tend\n\tend\n\t\n\tx1000 += xs[1000] * x\n\t\n\t~@routine\nend\n\n# ╔═╡ e5d47096-e030-11ea-1e87-5b9b1dbecfe0\npower1000_bad(0.0, 1.001)\n\n# ╔═╡ 9c62289a-dfae-11ea-0fe0-b1cb80a87704\nmd\"# Don't allocate for me!\"\n\n# ╔═╡ 88838bce-dfaf-11ea-1a72-7d15629cfcb0\nmd\"\"\"\nMultipling two unsigned logarithmic numbers `x = exp(lx)` and `y = exp(ly)`\n```\nx * y = exp(lx) * exp(ly) = exp(lx + ly)\n```\n\"\"\"\n\n# ╔═╡ a593f970-dfae-11ea-2d79-876030850dee\n@i function power1000_noalloc(x1000, x::T) where T\n\tif x!= 0\n\t\t@routine begin\n\t\t\tabsx ← zero(T)\n\t\t\tlx ← one(ULogarithmic{T})\n\t\t\tlx1000 ← one(ULogarithmic{T})\n\t\t\tabsx += abs(x)\n\t\t\tlx *= convert(absx)\n\t\t\tfor i=1:1000\n\t\t\t\tlx1000 *= lx\n\t\t\tend\n\t\tend\n\t\tx1000 += convert(lx1000)\n\t\t~@routine\n\tend\nend\n\n# ╔═╡ f448548e-dfaf-11ea-05c0-d5d177683445\npower1000_noalloc(0.0, 1.001)\n\n# ╔═╡ 65cd13ca-e031-11ea-3fc6-977792eb5f8c\nhtml\"\"\"\n
If statement
\n
\n
\n\t
Forward
\n\t
\n\tif (precondition, postcondition)\n\t\t# do A\n\telse\n\t\t# do B\n\tend\n\t
\n
\n
\n\t
Reverse
\n\t
\n\tif (postcondition, precondition)\n\t\t# undo A\n\telse\n\t\t# undo B\n\tend\n\t\n\t
\n
\n
\n\"\"\"\n\n# ╔═╡ 53c02100-e08f-11ea-1f5d-8b2311b095d2\nmd\"![](https://user-images.githubusercontent.com/6257240/116341762-78a31080-a7af-11eb-8376-d2ba0bf2b454.png)\"\n\n# ╔═╡ 75751b24-e0b8-11ea-2b37-9d138121345c\nmd\"### You should not do\"\n\n# ╔═╡ 76b84de4-e031-11ea-0bcf-39b86a6b4552\n@i function break_if(x)\n\tif x%2 == 1\n\t\tx += 1\n\telse\n\t\tx -= 1\n\tend\nend\n\n# ╔═╡ b1984d24-e031-11ea-3b13-3bd0119a2bcb\nbreak_if(3)\n\n# ╔═╡ 7f163d82-e0b8-11ea-2fe7-332bb4dee586\nmd\"### You should do\"\n\n# ╔═╡ ddc6329e-e031-11ea-0e6e-e7332fa26e22\n@i function happy_if(x)\n\tif (x%2 == 1, x%2 == 0)\n\t\tx += 1\n\telse\n\t\tx -= 1\n\tend\nend\n\n# ╔═╡ f3d5e1b0-e031-11ea-1a90-7bed88e28bad\nhappy_if(3)\n\n# ╔═╡ ab67419a-dfae-11ea-27ba-09321303ad62\nmd\"\"\"# Wrap up\n\n1. reversible arithmetic instructions `+=` and `-=`, besides, we have `SWAP`, `NEG`, `INC` and `ROT` et. al.\n2. inverse statement `~`\n3. there is no \"`=`\" operation in reversible computing, use \"`←`\" to allocate a new variable, and use \"`→`\" to deallocate an pre-emptied variable.\n4. compute-uncompute macro `@routine` and `~@routine`\n5. reversible control flow: `for` loop and `if` statement, the `while` statement is also available.\n6. logarithmic number is reversible under `*=` and `/=`\n\"\"\"\n\n# ╔═╡ d5c2efbc-d779-11ea-11ad-1f5873b95628\nmd\"\"\"\n![yeah](https://media1.tenor.com/images/40147f2eac14c0a7f18c34ecba73fa34/tenor.gif?itemid=7805520)\n\"\"\"\n# ![yeah](https://pic.chinesefontdesign.com/uploads/2017/03/chinesefontdesign.com_2017-03-07_08-19-24.gif)\n\n# ╔═╡ 30af9642-e084-11ea-1f92-b52abfddcf06\nmd\"# Sec II. Automatic differentiation in NiLang\n\n### References\n* Nextjournal [https://nextjournal.com/giggle/reverse-checkpointing](https://nextjournal.com/giggle/reverse-checkpointing)\n\n* arXiv: 2003.04617\n\"\n\n# ╔═╡ db1fab1c-e084-11ea-0bf0-b1fbe9e74b3f\nhtml\"\"\"

Automatic differentiation?

\"\"\"\n\n# ╔═╡ e1370f80-e0bc-11ea-2a90-d50cc762cbcb\nmd\"When we start learning AD, we start by learning the backward rules of the matrix multiplication\"\n\n# ╔═╡ 3098411c-e0bc-11ea-2754-eb0afbd663de\nfunction mymul!(out::AbstractMatrix, A::AbstractMatrix, B::AbstractMatrix)\n\t@assert size(A, 2) == size(B, 1) && size(out) == (size(A, 1), size(B, 2))\n\tfor k=1:size(B, 2)\n\t\tfor j=1:size(B, 1)\n\t\t\tfor i=size(A, 1)\n\t\t\t\t@inbounds out[i, k] += A[i, j] * B[j, k]\n\t\t\tend\n\t\tend\n\tend\n\treturn out\nend\n\n# ╔═╡ 3d0150ee-e0bd-11ea-0a5a-339465b496dc\nmd\"Then, we learning how to use chain rule to chain different utilities.\"\n\n# ╔═╡ 8016ff94-e0bc-11ea-3b9e-4f0676587edf\nmd\"##### But wait! Why don't we start from the backward rules of `+` and `*`, then use the chain rule to derive the backward rule for matrix multiplication?\"\n\n# ╔═╡ 99108ace-e0bc-11ea-2744-d1b18db50ae1\nmd\"# They are different\"\n\n# ╔═╡ b2337f26-e0bb-11ea-3da0-9507c35101ae\nmd\"\"\"\n### Domain-specific autodiff (DS-AD)\n* **Tensor**Flow\n* **PyTorch**\n* **Jax**\n* **Flux (Zygote backended)**\n\n### General Purposed autodiff (GP-AD)\n* **Tapenade**\n* **NiLang**\n\"\"\"\n\n# ╔═╡ 48db515c-e084-11ea-2eec-018b8545fa34\nmd\"## Traditional AD uses checkpointing\nCheckpoint every 100 steps. Blue and yellow objects are computing and re-computing. Here states 1 and state 101 are cached. Blue objects are computing, and yellow ones are re-computing. The state 100 is the desired state.\n\"\n\n# ╔═╡ f531f556-e083-11ea-2f7e-77e110d6c53a\nmd\"![](https://nextjournal.com/data/Qmes4v3ic2VrYQt6W9mWu4p6W53Gd1DmbDcYCuafbwTe7Y?filename=checkpointing.png&content-type=image/png)\"\n\n# ╔═╡ 62643fbc-e084-11ea-1b1f-39b87ff32b9e\nmd\"## Reverse Computing\nReversible computing approach to free up memories (a) when no operations are reversible. (b) when all operations are reversible. Blue and yellow diamonds are reversible operations executed in forward and backward directions, red cubics are garbage variables.\n\"\n\n# ╔═╡ 0bf54b08-e084-11ea-3d11-7be65f3ec022\nmd\"![](https://nextjournal.com/data/QmPsgm4Z4mqVw2h2eC3RkGf96xTQp13KE9rdzmPeUe5KWN?filename=reversecomputing.png&content-type=image/png)\"\n\n# ╔═╡ 15f7c60a-e08e-11ea-31ea-a5cd055644db\nmd\"## Difference Explained\"\n\n# ╔═╡ 55a3a260-d48e-11ea-06e2-1b7bd7bba6f5\nmd\"\"\"\n![adprog](https://github.com/GiggleLiu/NiLang.jl/raw/master/docs/src/asset/adprog.png)\n\"\"\"\n\n# ╔═╡ 38014ad0-e08e-11ea-1905-198038ab7e5f\nmd\"# Obtaining the gradient of norm in Zygote\"\n\n# ╔═╡ 2e6fe4da-d79d-11ea-1e90-f5215190395c\nmd\"**Obtaining the gradient of the norm function**\"\n\n# ╔═╡ 6560c28c-e08e-11ea-1094-d333b88071ce\nfunction regular_norm(x::AbstractArray{T}) where T\n\tres = zero(T)\n\tfor i=1:length(x)\n\t\t@inbounds res += x[i]^2\n\tend\n\treturn sqrt(res)\nend\n\n# ╔═╡ 744dd3c6-d492-11ea-0ed5-0fe02f99db1f\n@benchmark Zygote.gradient($regular_norm, $(randn(1000))) seconds=1\n\n# ╔═╡ f72246f8-e08e-11ea-3aa0-53f47a64f3e9\nmd\"## The reversible counterpart\"\n\n# ╔═╡ f025e454-e08e-11ea-20d6-d139b9a6b301\n@i function reversible_norm(res, y, x::AbstractArray{T}) where {T}\n\tfor i=1:length(x)\n\t\t@inbounds y += x[i]^2\n\tend\n\tres += sqrt(y)\nend\n\n# ╔═╡ 8fedd65a-e08e-11ea-27f4-03bf9ed65875\nlet x = randn(1000)\n\t@assert Zygote.gradient(regular_norm, x)[1] ≈ NiLang.AD.gradient(reversible_norm, (0.0, 0.0, x), iloss=1)[3]\nend\n\n# ╔═╡ 8ad60dc0-d492-11ea-2cb3-1750b39ddf86\n@benchmark NiLang.AD.gradient($reversible_norm, (0.0, 0.0, $(randn(1000))), iloss=1)\n\n# ╔═╡ 7bab4614-d77e-11ea-037c-8d1f432fc3b8\nmd\"\"\"\n![yeah](https://media1.tenor.com/images/40147f2eac14c0a7f18c34ecba73fa34/tenor.gif?itemid=7805520)\n\"\"\"\n# ![yeah](https://pic.chinesefontdesign.com/uploads/2017/03/chinesefontdesign.com_2017-03-07_08-19-24.gif)\n\n\n# ╔═╡ fcca27ba-d4a4-11ea-213a-c3e2305869f1\n#**1. The bundle adjustment jacobian benchmark**\n#$(LocalResource(\"ba-origin.png\"))\n#![ba](https://github.com/JuliaReverse/NiBundleAdjustment.jl/raw/master/benchmarks/adbench.png)\n\n#**2. The Gaussian mixture model benchmark**\n#$(LocalResource(\"gmm-origin.png\"))\n#![gmm](https://github.com/JuliaReverse/NiGaussianMixture.jl/raw/master/benchmarks/adbench.png)\n\nmd\"\"\"\n# Sec III. Applications in real world and benchmarks\n\"\"\"\n\n# ╔═╡ 519dc834-e092-11ea-2151-57ef23810b84\nmd\"\"\"\n## 1. Bundle Adjustment (Jacobian)\n![bundle adjustment](https://encrypted-tbn0.gstatic.com/images?q=tbn%3AANd9GcRgpGSCWRjHDDaIYQX5ejhMvyKY_GFhynVoQg&usqp=CAU)\n\n*Srajer, Filip, Zuzana Kukelova, and Andrew Fitzgibbon. \"A benchmark of selected algorithmic differentiation tools on some problems in computer vision and machine learning.\" Optimization Methods and Software 33.4-6 (2018): 889-906.*\n\n### Benchmarks\n**Devices**\n* CPU: Intel(R) Xeon(R) Gold 6230 CPU @ 2.10GHz\n* GPU: Nvidia Titan V. \n\n**Github Repos** \n* [https://github.com/microsoft/ADBench](https://github.com/microsoft/ADBench)\n* [https://github.com/JuliaReverse/NiBundleAdjustment.jl](https://github.com/JuliaReverse/NiBundleAdjustment.jl)\n\"\"\"\n\n# ╔═╡ c89108f0-e092-11ea-0fe2-efad85008b28\nhtml\"\"\"\n
\n\"\"\"\n\n# ╔═╡ 2ec4c700-e093-11ea-06ff-47d2c21a068f\nmd\"\"\"##### NiLang.AD and Tapenade\n![](https://user-images.githubusercontent.com/6257240/116341804-907a9480-a7af-11eb-934f-7eb94803f5f2.png)\"\"\"\n\n# ╔═╡ 474aa228-e092-11ea-042b-bdfaeb99f16f\nmd\"\"\"\n## 2. Gaussian Mixture Model (Gradient)\n![gmm](https://prateekvjoshi.files.wordpress.com/2013/06/multimodal.jpg)\n\"\"\"\n\n# ╔═╡ 2baaff10-d56c-11ea-2a23-bfa3a7ae2e4b\nmd\"\"\"\n### Benchmarks\n*Srajer, Filip, Zuzana Kukelova, and Andrew Fitzgibbon. \"A benchmark of selected algorithmic differentiation tools on some problems in computer vision and machine learning.\" Optimization Methods and Software 33.4-6 (2018): 889-906.*\n\n**Devices**\n* CPU: Intel(R) Xeon(R) Gold 6230 CPU @ 2.10GHz\n\n**Github Repos** \n* [https://github.com/microsoft/ADBench](https://github.com/microsoft/ADBench)\n* [https://github.com/JuliaReverse/NiGaussianMixture.jl](https://github.com/JuliaReverse/NiGaussianMixture.jl)\n\"\"\"\n\n# ╔═╡ 102fbf2e-d56b-11ea-189d-c78d56c0a924\nhtml\"\"\"\n
Results from the original benchmark
\n\n\"\"\"\n\n# ╔═╡ cc0d5622-d788-11ea-19cd-3bf6864d9263\nmd\"\"\"##### Including NiLang.AD\n![](https://github.com/JuliaReverse/NiLangTutorial/blob/master/notebooks/asset/benchmarks_gmm.png?raw=true)\"\"\"\n\n# ╔═╡ a1646ef0-e091-11ea-00f1-e7c246e191ff\nmd\"## 3. Solve the memory wall problem in machine learning\"\n\n# ╔═╡ b18b3ae8-e091-11ea-24a1-e968b70b217c\nhtml\"\"\"\nLearning a ring distribution with NICE network, before and after training\n\n\n\n\n
References
\n\n\"\"\"\n\n# ╔═╡ bf3774de-e091-11ea-3372-ef56452158e6\nmd\"\"\"\n## 4. Solve the spinglass ground state configuration\nObtaining the optimal configuration of a spinglass problem on a $28 \\times 28$ square lattice.\n\n![](https://user-images.githubusercontent.com/6257240/116342088-067efb80-a7b0-11eb-935e-0b5e29010a22.png)\n\n##### References\nJin-Guo Liu, Lei Wang, Pan Zhang, **arXiv 2008.06888**\n\"\"\"\n\n# ╔═╡ c8e4f7a6-e091-11ea-24a3-4399635a41a5\nmd\"\"\"\n## 5. Optimizing problems in finance\nGradient based optimization of Sharpe rate.\n\n\n600x acceleration comparing with using pure Zygote.\n\n##### References\n* Han Li's Github repo: [https://github.com/HanLi123/NiLang](https://github.com/HanLi123/NiLang) and his Zhihu blog [猴子掷骰子](https://zhuanlan.zhihu.com/c_1092471228488634368).\n\"\"\"\n\n# ╔═╡ bc872296-e09f-11ea-143b-9bfd5e52b14f\nmd\"\"\"## 6. Accelerate the performance critical part of variational mean field\n\n[https://github.com/quantumlang/NiLangTest/pull/1](https://github.com/quantumlang/NiLangTest/pull/1)\n\n600x acceleration comparing with using pure Zygote.\n\"\"\"\n\n# ╔═╡ e7b21fce-e091-11ea-180c-7b42e00598a9\nmd\"\"\"# Thank you!\nSpecial thanks to my collaborator **Taine Zhao** and (ex-)advisor **Lei Wang**.\n\nQuEra computing (a quantum computing company located in Boston) is hiring people.\n\"\"\"\n\n# ╔═╡ 7c79975c-d789-11ea-30b1-67ff05418cdb\nmd\"\"\"\n![yeah](https://media1.tenor.com/images/40147f2eac14c0a7f18c34ecba73fa34/tenor.gif?itemid=7805520)\n\"\"\"\n# ![yeah](https://pic.chinesefontdesign.com/uploads/2017/03/chinesefontdesign.com_2017-03-07_08-19-24.gif)\n\n# ╔═╡ 5f1c3f6c-d48b-11ea-3eb0-357fd3ece4fc\nmd\"\"\"\n## Sec IV. More about number systems\n\n \n* Integers are reversible under (`+=`, `-=`).\n* Floating point number system is **irreversible** under (`+=`, `-=`) and (`*=`, `/=`).\n* [Fixedpoint number system](https://github.com/JuliaMath/FixedPointNumbers.jl) are reversible under (`+=`, `-=`)\n* [Logarithmic number system](https://github.com/cjdoris/LogarithmicNumbers.jl) is reversible under (`*=`, `/=`)\n\"\"\"\n\n# ╔═╡ 11ddebfe-d488-11ea-223a-e9403f6ec8de\nmd\"\"\"\n##### Example 1: Affine transformation with rounding error\n\n```julia\ny = A * x + b\n```\n\"\"\"\n\n# ╔═╡ 030e592e-d488-11ea-060d-97a3bb6353b7\n@i function reversible_affine!(y!::AbstractVector{T}, W::AbstractMatrix{T}, b::AbstractVector{T}, x::AbstractVector{T}) where T\n @safe @assert size(W) == (length(y!), length(x)) && length(b) == length(y!)\n for j=1:size(W, 2)\n for i=1:size(W, 1)\n @inbounds y![i] += W[i,j]*x[j]\n end\n end\n for i=1:size(W, 1)\n @inbounds y![i] += b[i]\n end\nend\n\n# ╔═╡ c8d26856-d48a-11ea-3cd3-1124cd172f3a\nbegin\n\tW = randn(10, 10)\n\tb = randn(10)\n\tx = randn(10)\nend;\n\n# ╔═╡ 37c4394e-d489-11ea-174c-b13bdddbe741\nyout, Wout, bout, xout = reversible_affine!(zeros(10), W, b, x)\n\n# ╔═╡ fef54688-d48a-11ea-340b-295b88d21382\n# should be restored to 0, but not!\nyin, Win, bin, xin = (~reversible_affine!)(yout, Wout, bout, xout)\n\n# ╔═╡ 259a2852-d48c-11ea-0f01-b9634850e09d\nmd\"\"\"\n### Reversible arithmetic functions\n\nComputing basic functions like `power`, `exp` and `besselj` is not trivial for reversible programming.\nThere is no efficient constant memory algorithm using pure fixed point numbers only.\n\"\"\"\n\n# ╔═╡ f06fb004-d79f-11ea-0d60-8151019bf8c7\nmd\"\"\"\n##### Example 2: Computing power function\nTo compute `x ^ n` reversiblly with fixed point numbers,\nwe need to either allocate a vector of size $O(n)$ or suffer from polynomial time overhead. It does not show the advantage to checkpointing.\n\"\"\"\n\n# ╔═╡ 26a8a42c-d7a1-11ea-24a3-45bc6e0674ea\n@i function i_power_cache(y!::T, x::T, n::Int) where T\n @routine @invcheckoff begin\n cache ← zeros(T, n) # allocate a buffer of size n\n\t\tcache[1] += x\n for i=2:n\n cache[i] += cache[i-1] * x\n end\n end\n\n y! += cache[n]\n\n ~@routine # uncompute cache\nend\n\n# ╔═╡ 399552c4-d7a1-11ea-36bb-ad5ca42043cb\n# To check the function\ni_power_cache(Fixed43(0.0), Fixed43(0.99), 100)\n\n# ╔═╡ 4bb19760-d7bf-11ea-12ed-4d9e4efb3482\nmd\"\"\"\n##### Example 3: reversible thinker, the logarithmic number approach\n\nWith **logarithmic numbers**, we can still utilize reversibility. Fixed point numbers and logarithmic numbers can be converted via \"a fast binary logarithm algorithm\".\n\n##### References\n* [1] C. S. Turner, \"A Fast Binary Logarithm Algorithm\", IEEE Signal Processing Mag., pp. 124,140, Sep. 2010.\n\"\"\"\n\n# ╔═╡ 5a8ba8f4-d493-11ea-1839-8ba81f86799d\n@i function i_power_lognumber(y::T, x::T, n::Int) where T\n @routine @invcheckoff begin\n lx ← one(ULogarithmic{T})\n ly ← one(ULogarithmic{T})\n ## convert `x` to a logarithmic number\n ## Here, `*=` is reversible for log numbers\n lx *= convert(x)\n for i=1:n\n ly *= lx\n end\n end\n\n ## convert back to fixed point numbers\n y += convert(ly)\n\n ~@routine\nend\n\n# ╔═╡ a625a922-d493-11ea-1fe9-bdd4a694cde0\n# To check the function\ni_power_lognumber(Fixed43(0.0), Fixed43(0.99), 100)\n\n# ╔═╡ 4fd20ed2-d7a2-11ea-206e-13799234913f\nmd\"**Less allocation, better speed**\"\n\n# ╔═╡ 692dfb44-d7a1-11ea-00da-af6550bc0622\n@benchmark i_power_cache(Fixed43(0.0), Fixed43(0.99), 100)\n\n# ╔═╡ 7e4ee09c-d7a1-11ea-0e56-c1921012bc30\n@benchmark i_power_lognumber(Fixed43(0.0), Fixed43(0.99), 100)\n\n# ╔═╡ 4c209bbe-d7b1-11ea-0628-33eb8d664f5b\nmd\"\"\"##### Example 4: The first kind Bessel function computed with Taylor expansion\n```math\nJ_\\nu(z) = \\sum\\limits_{n=0}^{\\infty} \\frac{(z/2)^\\nu}{\\Gamma(k+1)\\Gamma(k+\\nu+1)} (-z^2/4)^{n}\n```\n\n\n\"\"\"\n\n# ╔═╡ fd44a3d4-d7a4-11ea-24ea-09456ff2c53d\n@i function ibesselj(y!::T, ν, z::T; atol=1e-8) where T\n\tif z == 0\n\t\tif ν == 0\n\t\t\tout! += 1\n\t\tend\n\telse\n\t\t@routine @invcheckoff begin\n\t\t\tk ← 0\n\t\t\t@ones ULogarithmic{T} lz halfz halfz_power_2 s\n\t\t\t@zeros T out_anc\n\t\t\tlz *= convert(z)\n\t\t\thalfz *= lz / 2\n\t\t\thalfz_power_2 *= halfz ^ 2\n\t\t\t# s *= (z/2)^ν/ factorial(ν)\n\t\t\ts *= halfz ^ ν\n\t\t\tfor i=1:ν\n\t\t\t\ts /= i\n\t\t\tend\n\t\t\tout_anc += convert(s)\n\t\t\twhile (s.log > -25, k!=0) # upto precision e^-25\n\t\t\t\tk += 1\n\t\t\t\t# s *= 1 / k / (k+ν) * (z/2)^2\n\t\t\t\t@routine begin\n\t\t\t\t\t@zeros Int kkv kv\n\t\t\t\t\tkv += k+ ν\n\t\t\t\t\tkkv += kv*k\n\t\t\t\tend\n\t\t\t\ts *= halfz_power_2 / kkv\n\t\t\t\tif k%2 == 0\n\t\t\t\t\tout_anc += convert(s)\n\t\t\t\telse\n\t\t\t\t\tout_anc -= convert(s)\n\t\t\t\tend\n\t\t\t\t~@routine\n\t\t\tend\n\t\tend\n\t\ty! += out_anc\n\t\t~@routine\n\tend\nend\n\n# ╔═╡ 84272664-d7b7-11ea-2e37-dffd2023d8d6\nmd\"z = $(@bind z Slider(0:0.01:10; default=1.0))\"\n\n# ╔═╡ 900e2ea4-d7b8-11ea-3511-6f12d95e638a\nbegin\n\ty = ibesselj(Fixed43(0.0), 2, Fixed43(z))[1]\n\tgz = NiLang.AD.gradient(Val(1), ibesselj, (Fixed43(0.0), 2, Fixed43(z)))[3]\nend;\n\n# ╔═╡ d76be888-d7b4-11ea-2989-2174682ead76\nlet\n\tmd\"\"\"\n| ``z`` | ``y`` | ``\\partial y/\\partial z`` |\n| ---- | ----- | -------- |\n| $(@sprintf \"%.5f\" z) | $(@sprintf \"%.5f\" y) | $(@sprintf \"%.5f\" gz) |\n\"\"\"\nend\n\n# ╔═╡ 85c9edcc-d789-11ea-14c8-71697cd6a047\nmd\"\"\"\n![yeah](https://media1.tenor.com/images/40147f2eac14c0a7f18c34ecba73fa34/tenor.gif?itemid=7805520)\n\"\"\"\n# ![yeah](https://pic.chinesefontdesign.com/uploads/2017/03/chinesefontdesign.com_2017-03-07_08-19-24.gif)\n\n# ╔═╡ Cell order:\n# ╟─1ef174fa-16f0-11eb-328a-afc201effd2f\n# ╟─627ea2fb-6530-4ea0-98ee-66be3db54411\n# ╟─94b2b962-e02a-11ea-09a5-81b3226891ed\n# ╟─a5ee60c8-e02a-11ea-3512-7f481e499f23\n# ╟─a11c4b60-d77d-11ea-1afe-1f2ab9621f42\n# ╟─e54a1be6-d485-11ea-0262-034c56e0fda8\n# ╟─55cfdab8-d792-11ea-271f-e7383e19997c\n# ╟─d1628f08-ddfb-11ea-241a-c7e6c1a22212\n# ╠═9e509f80-d485-11ea-0044-c5b7e750aacb\n# ╟─278ac6b6-e02c-11ea-1354-cd7ecd1099be\n# ╠═a28d38be-d486-11ea-2c40-a377b74a05c1\n# ╠═e93f0bf6-d487-11ea-1baa-21d51ddb4a20\n# ╠═fc932606-d487-11ea-303e-75ca8b7a02f6\n# ╟─e3d2b23a-ddfb-11ea-0f5e-e72ed299bb45\n# ╟─a961e048-ddf2-11ea-0262-6d19eb82b36b\n# ╟─2d22f504-ddf1-11ea-28ec-5de6f4ee79bb\n# ╟─7d08ac24-e143-11ea-2085-539fd9e35889\n# ╠═9fcdd77c-e0df-11ea-09e6-49a2861137e5\n# ╟─0a1a8594-ddfc-11ea-119a-1997c86cd91b\n# ╠═0b4edb1a-ddf0-11ea-220c-91f2df7452e7\n# ╟─f875ecd6-ddef-11ea-22a1-619809d15b37\n# ╠═e7557bee-e0cc-11ea-1788-411e759b4766\n# ╟─cd7b2a2e-ddf5-11ea-04c4-f7583bbb5a53\n# ╠═bc98a824-ddf5-11ea-1a6a-1f795452d3d0\n# ╠═05f8b91c-e0cd-11ea-09e3-f3c5c0e07e63\n# ╟─ac302844-e07b-11ea-35dd-e3e06054401b\n# ╠═b722e098-e07b-11ea-3483-01360fb6954e\n# ╟─bf8b722c-dfa4-11ea-196a-719802bc23c5\n# ╟─330edc28-dfac-11ea-35a5-3144c4afbfcf\n# ╠═0a679e04-dfa7-11ea-0288-a1fa490c4387\n# ╠═cc32cae8-dfab-11ea-0d0b-c70ea8de720a\n# ╟─b4240c16-dfac-11ea-3a40-33c54436e3a3\n# ╠═ade52358-dfac-11ea-2dd3-d3a691e7a8a2\n# ╠═d86e2e5e-dfab-11ea-0053-6d52f1164bc5\n# ╟─7951b9ec-e030-11ea-32ee-b1de49378186\n# ╟─6bc97f5e-dfad-11ea-0c43-e30b6620e6e8\n# ╠═80d24e9e-dfad-11ea-1dae-49568d534f10\n# ╠═a8092b18-dfad-11ea-0989-474f37d05f73\n# ╟─43f0c2fc-e030-11ea-25d9-b323e6496a35\n# ╟─b4ad5830-dfad-11ea-0057-055dda8cc9be\n# ╠═cf576d38-dfad-11ea-2682-7bd540db44a5\n# ╠═35fff53c-dfae-11ea-3602-918a17d5a5fa\n# ╟─9b9b5328-e030-11ea-1d00-f3341572734a\n# ╟─f3b87892-e080-11ea-353d-8d81c52cf9ac\n# ╠═b27a3974-e030-11ea-0bcd-7f7035d55165\n# ╠═e5d47096-e030-11ea-1e87-5b9b1dbecfe0\n# ╟─9c62289a-dfae-11ea-0fe0-b1cb80a87704\n# ╟─88838bce-dfaf-11ea-1a72-7d15629cfcb0\n# ╠═a593f970-dfae-11ea-2d79-876030850dee\n# ╠═f448548e-dfaf-11ea-05c0-d5d177683445\n# ╟─65cd13ca-e031-11ea-3fc6-977792eb5f8c\n# ╟─53c02100-e08f-11ea-1f5d-8b2311b095d2\n# ╟─75751b24-e0b8-11ea-2b37-9d138121345c\n# ╠═76b84de4-e031-11ea-0bcf-39b86a6b4552\n# ╠═b1984d24-e031-11ea-3b13-3bd0119a2bcb\n# ╟─7f163d82-e0b8-11ea-2fe7-332bb4dee586\n# ╠═ddc6329e-e031-11ea-0e6e-e7332fa26e22\n# ╠═f3d5e1b0-e031-11ea-1a90-7bed88e28bad\n# ╟─ab67419a-dfae-11ea-27ba-09321303ad62\n# ╟─d5c2efbc-d779-11ea-11ad-1f5873b95628\n# ╟─30af9642-e084-11ea-1f92-b52abfddcf06\n# ╟─db1fab1c-e084-11ea-0bf0-b1fbe9e74b3f\n# ╟─e1370f80-e0bc-11ea-2a90-d50cc762cbcb\n# ╠═3098411c-e0bc-11ea-2754-eb0afbd663de\n# ╟─3d0150ee-e0bd-11ea-0a5a-339465b496dc\n# ╟─8016ff94-e0bc-11ea-3b9e-4f0676587edf\n# ╟─99108ace-e0bc-11ea-2744-d1b18db50ae1\n# ╟─b2337f26-e0bb-11ea-3da0-9507c35101ae\n# ╟─48db515c-e084-11ea-2eec-018b8545fa34\n# ╟─f531f556-e083-11ea-2f7e-77e110d6c53a\n# ╟─62643fbc-e084-11ea-1b1f-39b87ff32b9e\n# ╟─0bf54b08-e084-11ea-3d11-7be65f3ec022\n# ╟─15f7c60a-e08e-11ea-31ea-a5cd055644db\n# ╟─55a3a260-d48e-11ea-06e2-1b7bd7bba6f5\n# ╟─38014ad0-e08e-11ea-1905-198038ab7e5f\n# ╟─2e6fe4da-d79d-11ea-1e90-f5215190395c\n# ╠═6560c28c-e08e-11ea-1094-d333b88071ce\n# ╠═37ed073a-d492-11ea-156f-1fb155128d0f\n# ╠═744dd3c6-d492-11ea-0ed5-0fe02f99db1f\n# ╟─f72246f8-e08e-11ea-3aa0-53f47a64f3e9\n# ╠═f025e454-e08e-11ea-20d6-d139b9a6b301\n# ╠═4d75f302-d492-11ea-31b9-bbbdb43f344e\n# ╠═8fedd65a-e08e-11ea-27f4-03bf9ed65875\n# ╠═8ad60dc0-d492-11ea-2cb3-1750b39ddf86\n# ╟─7bab4614-d77e-11ea-037c-8d1f432fc3b8\n# ╟─fcca27ba-d4a4-11ea-213a-c3e2305869f1\n# ╟─519dc834-e092-11ea-2151-57ef23810b84\n# ╟─c89108f0-e092-11ea-0fe2-efad85008b28\n# ╟─2ec4c700-e093-11ea-06ff-47d2c21a068f\n# ╟─474aa228-e092-11ea-042b-bdfaeb99f16f\n# ╟─2baaff10-d56c-11ea-2a23-bfa3a7ae2e4b\n# ╟─102fbf2e-d56b-11ea-189d-c78d56c0a924\n# ╟─cc0d5622-d788-11ea-19cd-3bf6864d9263\n# ╟─a1646ef0-e091-11ea-00f1-e7c246e191ff\n# ╟─b18b3ae8-e091-11ea-24a1-e968b70b217c\n# ╟─bf3774de-e091-11ea-3372-ef56452158e6\n# ╟─c8e4f7a6-e091-11ea-24a3-4399635a41a5\n# ╟─bc872296-e09f-11ea-143b-9bfd5e52b14f\n# ╟─e7b21fce-e091-11ea-180c-7b42e00598a9\n# ╟─7c79975c-d789-11ea-30b1-67ff05418cdb\n# ╟─5f1c3f6c-d48b-11ea-3eb0-357fd3ece4fc\n# ╟─11ddebfe-d488-11ea-223a-e9403f6ec8de\n# ╠═030e592e-d488-11ea-060d-97a3bb6353b7\n# ╠═c8d26856-d48a-11ea-3cd3-1124cd172f3a\n# ╠═37c4394e-d489-11ea-174c-b13bdddbe741\n# ╠═fef54688-d48a-11ea-340b-295b88d21382\n# ╟─259a2852-d48c-11ea-0f01-b9634850e09d\n# ╟─f06fb004-d79f-11ea-0d60-8151019bf8c7\n# ╠═26a8a42c-d7a1-11ea-24a3-45bc6e0674ea\n# ╠═399552c4-d7a1-11ea-36bb-ad5ca42043cb\n# ╟─4bb19760-d7bf-11ea-12ed-4d9e4efb3482\n# ╠═5a8ba8f4-d493-11ea-1839-8ba81f86799d\n# ╠═a625a922-d493-11ea-1fe9-bdd4a694cde0\n# ╟─4fd20ed2-d7a2-11ea-206e-13799234913f\n# ╠═692dfb44-d7a1-11ea-00da-af6550bc0622\n# ╠═7e4ee09c-d7a1-11ea-0e56-c1921012bc30\n# ╟─4c209bbe-d7b1-11ea-0628-33eb8d664f5b\n# ╠═fd44a3d4-d7a4-11ea-24ea-09456ff2c53d\n# ╟─84272664-d7b7-11ea-2e37-dffd2023d8d6\n# ╠═900e2ea4-d7b8-11ea-3511-6f12d95e638a\n# ╟─d76be888-d7b4-11ea-2989-2174682ead76\n# ╟─85c9edcc-d789-11ea-14c8-71697cd6a047\n" }, { "path": "notebooks/documentation.jl", "content": "### A Pluto.jl notebook ###\n# v0.14.5\n\nusing Markdown\nusing InteractiveUtils\n\n# ╔═╡ d941d6c2-55bf-11eb-0002-35c7474e4050\nusing NiLang, Test\n\n# ╔═╡ 2061b434-0ad1-46eb-a0c7-1a5f432bfa62\nbegin\n\ttwocol(left, right; llabel=\"forward\", rlabel=\"backward\") = HTML(\"\n\n\t\n\t\n\t\n\t\n\n\n\n\n
$llabel$rlabel
\n$(html(left))\n\n$(html(right))\n
\n\")\n\texample(str) = HTML(\"\"\"
$str
\"\"\")\n\ttitle1(str) = HTML(\"\"\"

'_'))>$str

\"\"\")\n\ttitle2(str) = HTML(\"\"\"

'_'))>$str

\"\"\")\n\ttitleref(str) = HTML(\"\"\"'_'))\">$str\"\"\")\n\tusing PlutoUI: TableOfContents\n\tusing Pkg\n\tpkgversion(m::Module) = Pkg.TOML.parsefile(NiLang.project_relative_path(\"Project.toml\"))[\"version\"]\n\thightlight(str) = HTML(\"$str\")\nend;\n\n# ╔═╡ 8c2c4fa6-172f-4dde-a279-5d0aecfdbe46\nmodule M\nusing NiLang\n\n# define two functions\nfunction new_forward(x)\n\tif x > 0\n\t\treturn x * 2\n\telseif x < 0\n\t\treturn x / 2\n\tend\nend\n\nfunction new_backward(x)\n\tif x > 0\n\t\treturn x / 2\n\telseif x < 0\n\t\treturn x * 2\n\tend\nend\n\n# declare them as reversible to each other\n@dual new_forward new_backward\n\n# The following is need only when your function is differentiable\nusing NiLang.AD: GVar\nfunction new_backward(x::GVar)\n\tif x.x > 0\n\t\tGVar(new_backward(x.x), x.g * 2)\n\telseif x.x < 0\n\t\tGVar(new_backward(x.x), x.g / 2)\n\tend\nend\n\nfunction new_forward(x::GVar)\n\tif x.x > 0\n\t\tGVar(new_forward(x.x), x.g / 2)\n\telseif x.x < 0\n\t\tGVar(new_forward(x.x), x.g * 2)\n\tend\nend\nend\n\n# ╔═╡ 3199a048-7b39-40f8-8183-6a54cccd91b6\nusing BenchmarkTools\n\n# ╔═╡ 0e1ba158-a6bc-401c-9ba7-ed78020ad068\nusing Base.Threads\n\n# ╔═╡ a4e76427-f051-4b29-915a-fdfce3a299bb\nhtml\"\"\"\n\n\n\"\"\"\n\n# ╔═╡ c2c7b4d4-f8c9-4ebf-8da2-0103f03136e7\nmd\"# NiLang's (v$(pkgversion(NiLang))) Documentation\n\nNiLang is a embeded domain specific language (eDSL) in Julia, so one need to install [Julia](https://julialang.org/) first. Before reading this documentation, you need to know basic Julia grammar, and how to install and use packages.\nAlso, it might be good to read the [README](https://github.com/GiggleLiu/NiLang.jl) first. In this tutorial, we focus on\n\n* NiLang grammar and design patterns,\n* Automatic differentiation based on reversible programming.\n\"\n\n# ╔═╡ 12f07cc7-979c-43c3-9dc9-36ea1463c1f6\nmd\"The symbols used in this notebook\"\n\n# ╔═╡ 611b577f-4722-42bf-8f8e-aeb2fb30be71\nmd\"\"\"\n| symbol | meaning | how to type |\n| ------- | --------- | ----------- |\n| ← | allocate | \\leftarrow + TAB |\n| → | deallocate | \\rightarrow + TAB |\n| ↔ | exchange | \\leftrightarrow + TAB |\n| ∅ | empty variable | \\emptyset + TAB |\n| ~ | inverse | ~ |\n\"\"\"\n\n# ╔═╡ 605872cf-f3fd-462e-a2b1-7d1c5ae45efd\ntitle1(\"Getting started\")\n\n# ╔═╡ fb3dee44-5fa9-4773-8b7f-a83c44358545\nmd\"\nAfter installing NiLang in a Julia REPL by typing `]add NiLang`, one can use NiLang and use the macro `@i` to define a reversible function ``f_1: (x, y) → (x+5y, y)``.\"\n\n# ╔═╡ 70088425-6779-4a2d-ba6d-b0a34c8e93a6\n@i @inline function f1(x, y; constant)\n\tx += y * constant\nend\n\n# ╔═╡ af738f89-3214-429c-9c7d-18a6ea0d9401\nf1(1.0, 2.0; constant=5.0) # call\n\n# ╔═╡ 48d7ebc1-5def-4a57-9ec1-3fc370a4543f\n (~f1)(11.0, 2.0; constant = 5.0) # uncall\n\n# ╔═╡ f0e94247-f615-472b-8218-3fa287b38aa1\nmd\"A NiLang function defines a ``\\mathbb{R}^n\\rightarrow\\mathbb{R}^n`` (notice inputs and outputs have the same shapes) mapping, it can take both keyword and positional arguments, where positional arguments can only be used as constants. There is no `return` statement because this function returns input non-keyword variables automatically, it is forbiden to write `return` statement inside a NiLang function.\nOne can aslo put macros like `@inline` after NiLang's `@i` macro. NiLang's macro will render the body of the function first and pass it to other macros.\n\"\n\n# ╔═╡ 2581aa33-1dc5-40b1-aa9f-6a11cc750c93\nmd\"`x += y * constant` is an instruction that defines a bijective mapping ``\\mathbb{R}^3\\rightarrow\\mathbb{R}^3``. All NiLang instructions change the variable inplace. Here, we accumulate the result to `x` rather than using `y *= constant` to modify the variable directly. This is because in a regular number system, one can easily use the zero element as the eraser to erase all information, which will cause irreversibility.\"\n\n# ╔═╡ 60575978-081a-4bca-a3ed-2b51cd6abc92\nmd\"One can differentiate a NiLang function with the `NiLang.gradient(f, args; iloss, kwargs...)` where `args` and `kwargs` are positional and keyword arguments for `f`, and `iloss` is the index of the loss variable.\"\n\n# ╔═╡ f98305cb-4ba2-404a-a5c3-65510e059504\nNiLang.AD.gradient(f1, (1.0, 2.0); iloss=1, constant=5.0)\n\n# ╔═╡ e8cd6667-597f-458b-8465-1822e09a7891\nmd\"Here, we specify the first variable as the one that stores the loss. We get\n```math\n\\begin{cases}\n\\frac{\\partial x+5y}{\\partial x}=1\\\\\n\\frac{\\partial x+5y}{\\partial y}=5\\\\\n\\end{cases}\n```\n\"\n\n# ╔═╡ 20145d75-004a-4c2f-b7ff-c400ca846d42\nlet\n\tcontent = md\"\"\"The above macro generates two functions, one is `f` and another is `~f` (or `Inv(f)`). The `x += y * constant` is translated to function `(x, y, constant) = PlusEq(*)(x, y, constant)`, where the function `PlusEq(*)` is bijective.\n\n```julia\njulia> using MacroTools\n\njulia> MacroTools.prettify(@macroexpand @i function f(x, y; constant)\n x += y * constant\n end)\nquote\n $(Expr(:meta, :doc))\n function $(Expr(:where, :(f(x, y; constant))))\n hare = wrap_tuple(((PlusEq)(*))(x, y, constant))\n x = hare[1]\n y = hare[2]\n constant = hare[3]\n (x, y)\n end\n if (NiLangCore)._typeof(f) != _typeof(~f)\n function $(Expr(:where, :((newt::_typeof(~f))(x, y; constant))))\n boar = wrap_tuple(((MinusEq)(*))(x, y, constant))\n x = boar[1]\n y = boar[2]\n constant = boar[3]\n (x, y)\n end\n end\nend\n```\n\"\"\"\n\tHTML(\"
How does the compiler work?$(html(content))
\")\nend\n\n# ╔═╡ c682a17f-600f-4034-bfe3-a851ab645c10\ntitle1(\"Instructions and operands\")\n\n# ╔═╡ 5239dfe2-ea6d-4e07-a1b1-90954fe8ddc9\nmd\"\"\"\nThe basic form of an instruction is `y ⊙= f(args...)`, where `⊙` can be `+`, `-`, `*`, `/` or `⊻`, where `*=` and `/=` are only reversible in the logarithmic number systems. See section $(titleref(\"Integers, floating-point numbers, fixed-point numbers and logarithmic numbers\")) for details.\n\nA function/instruction can be used in NiLang only if its reverse is defined. A NiLang function is differentiable if the function body is composed of differentiable instructions, differentiable NiLang functions and NiLang control flows. A list of differentiable NiLang instructions are\n\n| instruction | output |\n| ----------- | ---------- |\n| ``{\\rm FLIP}(y)`` | ``\\sim y`` |\n| ``{\\rm NEG}(y)`` | ``-y`` |\n| ``{\\rm INC}(y)`` | ``y+1`` |\n| ``{\\rm DEC}(y)`` | ``y-1`` |\n| ``{\\rm INV}(y)`` | ``y ^ {-1}`` |\n| ``{\\rm HADAMARD}(x, y)`` | ``\\frac{1}{\\sqrt{2}}(x+y), \\frac{1}{\\sqrt{2}}(x-y)``\n| ``{\\rm SWAP}(a, b)`` | ``b, a`` |\n| ``{\\rm ROT}(a, b, \\theta)`` | ``a \\cos\\theta - b\\sin\\theta, b \\cos\\theta + a\\sin\\theta, \\theta`` |\n| ``{\\rm IROT}(a, b, \\theta)`` | ``a \\cos\\theta + b\\sin\\theta, b \\cos\\theta - a\\sin\\theta, \\theta`` |\n| ``y \\mathrel{\\{+,-\\}}= f_{+-}(args...)`` | ``y\\{+, -\\}f_{+-}(args...), args...`` |\n| ``y \\mathrel{\\{*, /\\}}= f_{*/}(args...)`` | ``y\\{*, /\\}f_{*/}(args...), args...`` |\n\nFunctions ``f_{+-} ∈ \\rm \\{identity, +, -, *, /, ^\\wedge, abs, abs2, sqrt, exp, log, sin, sinh, asin, cos, cosh,`` ``\\rm acos, tan, tanh, atan, sincos, convert\\}`` and ``f_{*/}∈\\rm \\{identity, +, -, *, /, ^\\wedge, convert\\}.`` \nFunctions `FLIP`, `NEG`, `INV`, `HADAMARD`, `SWAP` and `y ⊻= f_{⊻}(args...)` are self-reversible (or reflexive). {`ROT`, `IROT`} and {`INC`, `DEC`}, {`y += f_{+-}(args...)`, `y -= f_{+-}(args...)`} and {`y *= f_{*/}(args...)`, `y /= f_{*/}(args...)`} are pair-wise reversible.\n\nFor Jacobians and Hessians defined on these instructions, please check this [blog post](https://giggleliu.github.io/2020/01/18/jacobians.html).\nThis set of instructions is extensible, see section $(titleref(\"Extending the instruction set\")) for an example.\n\"\"\"\n\n# ╔═╡ e4d86a5a-e820-4a70-8a87-08bac416291b\nmd\"The operands of an instruction can be a composition of the following expressions\"\n\n# ╔═╡ c307b6a4-906d-4be7-9fd7-57c942aded51\nmd\"\"\"\n\n| expression | meaning |\n| ---------- | ------- |\n| x | change a variable |\n| x.field | change a field |\n| x.:1 | change a tuple element |\n| x' | change the adjoint |\n| -x | change the inverse |\n| x[i] | change an array/dict element |\n| (x, y, z) | change multiple variables |\n| @fields x | change the fields of an object |\n| A{Float64}(x, y) | wrap, update fields and unwrap |\n| x \\|> subarray(:,i) | change the view of an array |\n| x \\|> f | change a field map or a bijective mapping |\n| @const 3 | target is a constant that can not be changed |\n| @skip! f(x) | an expression that can not be assigned back |\n\"\"\"\n\n# ╔═╡ 47b502d4-e8af-4d58-9067-9700784ea435\nmd\"In the following example, one modifies the negated real part of a complex number in a vector inside a tuple directly.\"\n\n# ╔═╡ faecc0a7-55d7-42a1-8e9f-7e30143eef9c\n@i function dataview_func1(x::Vector, y::Tuple, θ)\n\t(-x[2].re, y.:1) += sincos(θ)\nend\n\n# ╔═╡ 4cbe69d6-b68f-4bda-a0dd-209f9ee54f18\ndataview_func1([1+2.0im, 3+4im], (1.0,2.0), π/6)\n\n# ╔═╡ 7dc82f28-77bf-40da-b520-800ed1bc80c9\nmd\"\"\"One can check the real part of the second element of `x` is decreased by `sin(π/6)`, while the first element of the tuple is increased by `cos(π/6)`. A NiLang instruction is different to a regular Julia instruction in that $(hightlight(\"A NiLang instruction changes variables inplace\")), eventhough `ComplexF64`, `-x` and `tuple` are all considered as immutable in Julia.\"\"\"\n\n# ╔═╡ 9f5f9de3-9558-4c18-9d98-b77d19b570ec\nexample(\"Complex valued log\")\n\n# ╔═╡ 6dfcfa19-f78f-4dac-89f7-d3c5dbe17987\n@i function complex_log(y!::Complex{T}, x::Complex{T}) where T\n n ← zero(T)\n n += abs(x)\n\n y!.re += log(n)\n y!.im += angle(x)\n\n n -= abs(x)\n n → zero(T)\nend; @test complex_log(0.0im, 3.0+2.0im)[1] ≈ log(3.0+2.0im)\n\n# ╔═╡ dad1c6c0-d61b-4f9f-a71e-e683fe143aaa\ntitle2(\"Broadcasting\")\n\n# ╔═╡ 3e4a3916-8fe4-4262-bcd3-3014822717a3\nmd\"The interfaces is similar to julia's native broadcast, but expanded to native NiLang loops.\"\n\n# ╔═╡ 34906208-e6f1-4a67-860a-a7b056a86dde\n@i function complex_log_broadcast!(ys!, xs)\n\tcomplex_log.(ys!, xs)\nend; @test (x=[1.0+2im, 3.0+4im, 4.0+5im]; complex_log_broadcast!(zeros(ComplexF64, 3), x)[1] ≈ log.(x))\n\n# ╔═╡ 4601df35-679f-465d-9191-c18748b2fd83\ntitle2(\"Avoid shared read-write\")\n\n# ╔═╡ 4fc72b9d-19a2-40f1-a4a8-5e97d3d5e529\nmd\"Shared read-write is not allowed\"\n\n# ╔═╡ a0fde16f-8454-4f5c-a29c-a9e415c0c311\n# `y -= y` effectively clears the content in `y`, this is why shared read-write is so dangerous.\n@test_throws LoadError macroexpand(NiLang, :(@i function shared_readwrite_error(y)\n\ty -= y\nend))\n\n# ╔═╡ 633ff8f3-8d93-4f73-bec2-c42070e6ece9\nmd\"NiLang is more restrictive, it forbids shared read too. This is in purpose, shared read will become shared write to the gradient field in the back-propagation of gradients - the main goal of NiLang.\"\n\n# ╔═╡ 57f2d890-b5a0-47b7-9e3d-af4d03b10605\n# `shared read is also forbidden`\n@test_throws LoadError macroexpand(NiLang, :(@i function shared_read_error(x, y)\n\tx -= y * y # should be written as `x -= y^2`\nend))\n\n# ╔═╡ 34063cd0-171e-46ce-80dd-52a341fa50a1\nmd\"The correct way of avoiding shared read is renaming one of the variable.\"\n\n# ╔═╡ 5d5d01db-8ff9-434c-8771-1fec6393e1fb\n@i function avoid_shared_read(x, y)\n\ttmp ← zero(y)\n\ttmp += y\n\tx -= y * tmp\n\ttmp -= y\n\ttmp → zero(y)\nend\n\n# ╔═╡ 10d85a50-f2f9-403e-8f6c-baef61cf702a\navoid_shared_read(0.0, 3.0)\n\n# ╔═╡ b52648bf-a28a-48af-8912-31729d943ce0\nmd\"Shared read-write issue is more tricky when one uses NiLang to write kernel function in a parallel program (multi-threading, MPI and CUDA). See $(titleref(\\\"Multi-threading and CUDA\\\")) for details.\"\n\n# ╔═╡ f45db10f-a836-40f3-9d8d-054ea6540e87\ntitle2(\"Protect constant variables\")\n\n# ╔═╡ 1903563e-ccc2-44d9-9dbe-e5dede275b3c\nmd\"\"\"\nTo achieve the goal of \"everything is mutable\", NiLang assigns the output value back to the input variable after a call. Sometime it can cause issues for special variables like function, type parameters and constants (including the results generated from a function call). One can use `@const` (assert a variable is a constant) or `@skip!` (skip assigning back) to avoid such complications.\n\"\"\"\n\n# ╔═╡ 583f2585-15a3-47c6-a70e-e2f002754028\nmd\"When using a function (e.g. `exp` as shown bellow) as an variable, one should be careful about the scope issue.\"\n\n# ╔═╡ 60e6ff80-3593-4ae4-a273-914847f692db\n@i function func_arg(y, f, x)\n\ty += f(x)\nend\n\n# ╔═╡ 9e5cfd68-b58d-4d83-aae2-447e5f805c97\n@i function use_func_arg(y, x)\n\tfunc_arg(y, exp, x)\nend\n\n# ╔═╡ dc85a942-cf52-4405-ad03-32a768e1b6e7\n@test_throws UndefVarError use_func_arg(0.0, 3.0)\n\n# ╔═╡ 5cdd346b-10a5-485c-ba78-4c0b3cb0e02f\nmd\"We see an error, but why calling `f2g` causes an error? If one check the generated code with `macroexpand`, one will see the `exp` is assigned in the local scope. The compiler takes it as a local variable and compaints that `exp` is not defined.\"\n\n# ╔═╡ af9287b7-6131-46f6-beb8-6885e55e1975\nmacroexpand(NiLang, :(@i function use_func_arg(y, x)\n\tf2g(y, exp, x)\nend)) |> NiLangCore.rmlines\n\n# ╔═╡ e20eeabf-1c80-431e-8cfc-4d1b79c52b5a\n@i function avoid_assignback(y, x)\n\tfunc_arg(y, (@const exp), x)\nend; @test avoid_assignback(0.0, 3.0)[1] == exp(3)\n\n# ╔═╡ 90d30eea-53de-48a0-9700-ff35681fdf38\nmd\"Type parameter can not be assigned back too.\"\n\n# ╔═╡ 390f58a5-6f5f-4d3a-bb16-ba04e43a07e7\n@test_throws ErrorException Core.eval(NiLang, :(@i function type_arg(t::Type{T}, x) where T\n\tx += one(T)\nend))\n\n# ╔═╡ 2b57443e-a516-434b-be86-80616a98e2f5\n@i function avoid_type_assignback(t::Type{T}, x) where T\n\tx += one(@skip! T)\nend; @test avoid_type_assignback(Float64, 0.0)[2] == 1.0\n\n# ╔═╡ fc2e27f9-b7ba-44cc-a953-6745548ad733\nmd\"A function call that returning a constant should also be decorated with the `@const` or `@skip!` macro.\"\n\n# ╔═╡ fc744931-360b-4478-9f77-c50f048de243\n@test_throws LoadError macroexpand(NiLang, :(@i function funccall_arg(y, x::Matrix) where T\n\ty += size(x, 1) * size(x, 2)\nend))\n\n# ╔═╡ 9a152b36-f377-44da-9700-ca9e05e365ff\n@i function avoid_funccall_assignback(y, x::Matrix) where T\n\ty += (@const size(x, 1)) * (@const size(x, 2))\nend; @test avoid_funccall_assignback(0, randn(3,4))[1] == 12\n\n# ╔═╡ b6dcd18c-606f-4340-b2ec-163e8bad03f5\ntitle1(\"Variable manipulation\")\n\n# ╔═╡ a1a29f34-f8a9-4e9f-9afe-7d0096771440\ntitle2(\"Allocate and deallocate a variable\")\n\n# ╔═╡ 90bd6ad4-3dd8-4e7c-b445-aed1d248a2ec\nmd\"\"\"\nOne can allocate a new variable `x` like `x ← constant` and deallocate a variable with known value with `x → known value`. They are reversible to each other with the following relation.\n\n$(\ntwocol(md\"\n```julia\nx ← constant\n```\", md\"\n```julia\nx → constant\n```\n\")\n)\n\nFor example\n\"\"\"\n\n# ╔═╡ c0259e48-1973-486c-a828-1fcd3e4331c6\n@i function alloc_func1()\n\ttmp ← 1\n\t# some code that uses `tmp` for computing and restores it to `1`\n\ttmp → 1\nend\n\n# ╔═╡ 8bbffa31-04a6-49ca-b36f-4d4140d75992\nmd\"allocate multiple variable of the same type at one time\"\n\n# ╔═╡ a6f18c34-80ee-4b52-9ff8-f3c1b1d80f90\n@i function power12(y, x::T) where T\n\t@zeros T a b c # three variable of type `T`\n\ta += x^2\n\tb += a^2\n\tc += b*a\n\ty += c^2\n\tc -= b*a\n\tb -= a^2\n\ta -= x^2\n\t@safe @show a b c x y\n\t~@zeros T a b c\nend\n\n# ╔═╡ a694132b-4f52-467f-8bc4-dc32fe2812db\n@test power12(0, 2)[1] == 4096\n\n# ╔═╡ 8c2c82f2-1240-4f2f-830e-ee8021c1a41a\nmd\"One can copy and push a value into a stack and use it later. It inverse operation will pop out a variable and assert its value.\"\n\n# ╔═╡ 6203cf10-f8cc-4fb9-b814-7552b68c01dc\ntwocol(md\"\n```julia\nstack[end+1] ← variable\n```\", md\"\n```julia\nstack[end] → variable\n```\n\")\n\n# ╔═╡ f97a6bab-b9f9-4b95-98a9-381c51397526\n@i function stack_push_and_pop!(stack, x, y, z)\n\tz += y\n\tstack[end+1] ← x # copy a variable into a stack\n\tstack[end+1] ← y \n\tstack[end] → z # pop a variable from a stack, `z` must have the same value as the variable.\nend\n\n# ╔═╡ 2a2970f4-ab01-486b-89a2-6ff96f734018\nmd\"A less recommended approach is using the global stacks in NiLang, since NiLang is an eDSL, it can not guarantee the access order. Available global stacks are `FLOAT64_STACK`, `COMPLEXF64_STACK`, `INT64_STACK` and their 32 bit counter parts, as well as a `BOOL_STACK`.\"\n\n# ╔═╡ 0b80d9be-53d7-4bf3-a558-659607af4709\n@i function stack_push_and_pop!(x, y)\n\tGLOBAL_STACK[end+1] ← x # copy a variable into a stack\n\tFLOAT64_STACK[end+1] ← y \nend\n\n# ╔═╡ 4ca48a2e-43da-457a-8e9f-6476097e4d7b\nlet\n\tstack = FastStack{Float64}(1000) # a preallocated stack of size 1000\n\tstack_push_and_pop!(stack, 5.0, 1.0, 0.0) # you will get a stack of size 1\n\t@test length(stack) == 1\nend\n\n# ╔═╡ 92362fda-bae2-4e35-bfe4-dcaea853d50b\nlet\n\tNiLang.empty_global_stacks!() # empty stacks\n\tstack_push_and_pop!(5.0, 1.0)\n\t@test length(GLOBAL_STACK) == 1\n\t@test length(FLOAT64_STACK) == 1\nend\n\n# ╔═╡ db9e7940-39f1-4ccf-ac70-146a521daa6e\nmd\"one can also allocate and deallocate on dicts\"\n\n# ╔═╡ 93936612-1447-4114-b864-aba43adef4bd\nmd\"\"\"\nThe forward pass of dictionary allocation adds an entry to the dictionary (cause key error if the key already exists), the backward pass. The backward pass checks the variable in the dictionary is consistent with the asserted variable, and delete the key.\n\n$(\ntwocol(md\"\n```julia\ndict[key] ← variable\n```\", md\"\n```julia\ndict[key] → variable\n```\n\")\n)\n\"\"\"\n\n# ╔═╡ b2add70c-c5d6-4e0f-a153-43e21a197181\n@i function dict_assign(dict::Dict)\n\tvar1 ← 3.14\n\n\t# copy a new variable to a dict\n\tdict[\"data\"] ← var1\n\t\n\t# deallocate the original variables\n\tvar1 → dict[\"data\"]\nend\n\n# ╔═╡ 5c03d5a5-99f0-4efd-9a32-ce6d7c2b266c\ndict_assign(Dict())\n\n# ╔═╡ 2349e3ea-3053-42a4-b9d9-f97a76e4abd7\ntitle2(\"Exchange two variables\")\n\n# ╔═╡ 269f18ee-3cd8-466a-a522-7c624503e31b\nlet \n\texpr = twocol(md\"\n```julia\nvar1 ↔ var2\n```\", md\"\n```julia\nvar1 ↔ var2\n```\n\")\nmd\"\"\"One can exchange two variables is using `↔`.\n$expr\n\"\"\"\nend\n\n# ╔═╡ 89139719-c478-4066-9452-f9893f36d561\n@i function exchange_func1(x, y)\n\tx ↔ y\nend\n\n# ╔═╡ d620c5ee-7d9c-4d3f-9e87-0c828dfab9ca\nexchange_func1(3, 5)\n\n# ╔═╡ 255d01b9-a873-4e63-9298-9d8f073348b0\nmd\"One can also make a \\\"link\\\" by exchanging a variable with an empty variable such as `var::∅` and `stack[end+1]`. The forward pass push a variable to the stack and deallocate variable. The backward pass pops a variable and asserts its value.\"\n\n# ╔═╡ f6cf1729-766c-4ed7-b004-c8c8ec6c7e07\nlet \n\texpr = twocol(md\"\n```julia\nstack[end+1] ↔ var2\nvar1::∅ ↔ var2\n```\", md\"\n```julia\nstack[end] ↔ var2::∅\nvar1 ↔ var2::∅\n```\n\")\nend\n\n\n# ╔═╡ 3645d672-423f-4ac8-805f-0452793fee5a\n@i function exchange_func2(x, y)\n\tanc ← 0.0\n\tanc += x * y\n\tanc ↔ z::∅ # declare `z` as an empty variable\n\t# after exchange, `anc` is empty and deallocated automatically.\n\tz -= x * y\n\tz → 0.0\nend\n\n# ╔═╡ c2a0024e-11dd-4ef7-8346-4374d98cafc0\nexchange_func2(3, 4)\n\n# ╔═╡ b20004e9-3c73-4dfb-8fd5-f377786fd53b\nmd\"When exchanging with a stack top + 1, it means push and deallocate.\"\n\n# ╔═╡ 5c1952b1-5016-4c87-b23c-8e6a235bf8cd\n@i function stack_exchange(stack, y, x)\n\tstack[end+1] ↔ y # push a variable into a stack and deallocate `y`\n\ty ← 1.2 # since `y` is deallocated, you can assign any value to it\n\tstack[end] ↔ anc::∅ # pop a variable to `anc`\n\tanc ↔ x # exchange `anc` and x\n\tstack[end+1] ↔ anc # push `anc` back to stack\nend\n\n# ╔═╡ 8e4470ee-01da-4547-b091-c4f65cd729b0\nlet\n\tstack = FastStack{Float64}(1000) # a preallocated stack\n\tstack_exchange(stack, 2.0, 3.0) # you will get a stack of size 2\n\t@test length(stack) == 1 && stack.data[1] == 3.0\nend\n\n# ╔═╡ 0863bd06-cc70-4dde-b3b2-0a466805a356\nmd\"\"\"$(title2(\"Integers, floating-point numbers, fixed-point numbers and logarithmic numbers\"))\n\nA fixed-point zero with 43 fraction bits can be declared as `x ← Fixed43(0.0)` or `x ← zero(Fixed43)`, a logarithmic one can be declared as `x ← ULogarithmic(1.0)`, `x ← one(ULogarithmic{Float64})` or `x ← ULogarithmic(Fixed43(1.0))`. A complex number zero can be defined as `x ← Complex(0.0, 0.0)`.\n\n\n| Number Type | += | *= | ⊻= | Source |\n| ---- | ---- | ---- | ---- | ---- |\n| boolean | - | - | ✓ | JuliaLang |\n| integer | ✓ | × | ✓ | JuliaLang |\n| floating-point number | ✓ (rounding error) | × | - | JuliaLang |\n| fixed-point number | ✓ | × | - | [FixedPointNumbers.jl](https://github.com/JuliaMath/FixedPointNumbers.jl)\n| logarithmic number | × | ✓ | - | [LogarithmicNumbers.jl](https://github.com/cjdoris/LogarithmicNumbers.jl)\n\n* `✓`: an operation has its reverse when operating on a number type.\n* `×`: an operation does not have a reverse when operating on a number type.\n* `-`: an operation does not apply on a number type.\n\nThe `+=` operation is not regoriously reversible on floating point numbers, but we ignore the rounding errors in NiLang and use the reversibility check to detect the potential too-large rounding errors. Whether logarithmic number has rounding errors depends on its content type. If it uses floating point numbers as storage, then yes, otherwise if it uses fixed point number as the content type, then no.\n\nOne can use the `y ⊙= convert(x)` statement to convert `x` to the target type `typeof(y)` and accumulate it to `y`. Here `⊙=` can be one of `+=`, `*=` and `⊻=` that has its reverse on type `typeof(y)`.\n\"\"\"\n\n# ╔═╡ a0bae195-04e1-4642-9e14-fe4691e0906b\nmd\"Fixed point numbers and Floating point numbers are reversible under the `+=` operation\"\n\n# ╔═╡ 20d6e8a0-2cf5-48ad-9549-60506b42b970\n@i function fixed_pluseq(y1::T, x::T) where T<:Union{Fixed43, AbstractFloat}\n\ty1 += x\nend; @test fixed_pluseq(Fixed43(0.5), Fixed43(0.6))[1] === Fixed43(1.1) && fixed_pluseq(0.5, 0.6)[1] === 1.1\n\n# ╔═╡ 7dee5748-ed73-4e13-aa80-7a50efbc8449\n@i function fixed_muleq(y1::T, x::T) where T<:Union{Fixed43, AbstractFloat}\n\ty1 *= x\nend; @test_throws MethodError fixed_muleq(Fixed43(1.0), Fixed43(2.0))\n\n# ╔═╡ 4c719e9b-641e-404e-9ab7-59e89135f3ba\nmd\"Logarithmic numbers are reversible under the `*=` operation\"\n\n# ╔═╡ 77947e00-42c3-4c9e-b62a-b4b29489db43\n@i function ulog_pluseq(y1::ULogarithmic{T}, x::ULogarithmic{T}) where T\n\ty1 += x\nend; @test_throws MethodError ulog_pluseq(ULogarithmic(1.0), ULogarithmic(3.0))\n\n# ╔═╡ 2614127d-34fb-4c3d-b678-42693f3c9341\n@i function ulog_muleq(y1::ULogarithmic{T}, x::ULogarithmic{T}) where T\n\ty1 *= x\nend; @test ulog_muleq(ULogarithmic(1.0), ULogarithmic(3.0))[1] === ULogarithmic(3.0)\n\n# ╔═╡ 8f169235-3bd1-4cc4-a083-79736d306ad5\nexample(\"computing x ^ 3 with logarithmic numbers\")\n\n# ╔═╡ dfc9d305-5bce-4555-bfa3-d8d61fe4ca09\n@i function power3(y::ULogarithmic{T}, x::T) where T\n\tfor i=1:3\n\t\ty *= convert(x)\n\tend\nend; @test power3(ULogarithmic(1.0), 3.0)[1] ≈ 27\n\n# ╔═╡ f6bfa015-c101-45e8-995c-2bb6a3b7dc7d\ntitle2(\"Types and arrays\")\n\n# ╔═╡ 8651d7ec-6bcd-4dbe-a062-c4bde32e5e91\nmd\"\nA Julia type can be accessed in NiLang if its default constructor is not overloaded, because NiLang requires the default constructor to \\\"modify\\\" a field of a immutable type.\n\"\n\n# ╔═╡ edaa9fdb-3af8-4554-a701-0e3bff2107a5\nmd\"It is also possbile to extract the fields directly.\"\n\n# ╔═╡ 7551a880-340e-4e3f-815b-188e73f7eb9a\n@i function complex_add(y::Complex{T}, x::Complex{T}) where T\n ((a, b), (c, d))::∅ ↔ ((@fields x), (@fields y))\n\ta += c\n\tb += d\n ((a, b), (c, d)) ↔ ((@fields x), (@fields y))::∅\nend\n\n# ╔═╡ 0489e51b-781f-4441-bb7f-ff3bd2e848ad\n@test complex_add(1+2im, 3+4im) == (1+2im, 4+6im)\n\n# ╔═╡ 7b0d30d6-39ff-4f6e-b13c-0ddbfcb576e5\nmd\"Type cast is also possible\"\n\n# ╔═╡ 042297d8-6ab3-4ae6-b6e7-3b1ab2d5553b\n@i function add4(a, b, c, d)\n\tcomplex_add(Complex{}(a, b), Complex{}(c, d)) # do not omit `{}`\nend\n\n# ╔═╡ 57d65a36-bfa8-4dc2-8e11-d87fa1324122\n@test add4(1, 2, 3, 4) == (1, 2, 4, 6)\n\n# ╔═╡ c21d81c3-981f-4472-ad61-d1661bfe5c4e\nexample(\"Implementing \\\"axpy\\\" function\")\n\n# ╔═╡ 99d6fe7b-d704-48f3-b115-2b3159a78068\nmd\"`axpy` function is defined as ``\\vec y += a \\vec x``. One can modify the `Array` directly\"\n\n# ╔═╡ 1950ff70-54eb-4ece-a26d-a23fd0e90f5a\n@i function arrayaxpy!(y!::Vector{T}, a::T, x::Vector{T}) where T\n for i=1:length(x)\n\t\ty![i] += a * x[i]\n\tend\nend; @test arrayaxpy!(zeros(10), 2.0, collect(1.0:10.0))[1] ≈ collect(2.0:2.0:20.0)\n\n# ╔═╡ 21458f81-9007-46f8-92e0-7a17c60beb36\nmd\"To modify an element of a `Tuple`, we need to use a different style to avoid confusion with array\"\n\n# ╔═╡ 7813f4ce-6e98-45f3-94a8-7f5981129f2b\n@i function tupleaxpy!(y!::NTuple{N,T}, a::T, x::NTuple{N,T}) where {N, T}\n for i=1:length(x)\n\t\t(y! |> tget(i)) += a * (x |> tget(i))\n\tend\nend; @test tupleaxpy!((0,0,0), 2, (1,2,3))[1] == (2, 4, 6)\n\n# ╔═╡ 59ec7cb7-6011-456d-9f57-a55bb8ea51a0\nmd\"Here `data |> tget(i)` represents the `i`th field of the tuple (note it is not allowed to write `data.:i`).\"\n\n# ╔═╡ aacf63a2-9708-40db-8928-049621a7bbc4\nmd\"## Control flows\"\n\n# ╔═╡ ad0097e7-c8ad-457a-82a9-18b998a9e9fb\nmd\"\"\"\n#### If statement\n\nThe condition expression in `if` statement contains two parts, one is precondition and another is postcondition.\n\n$(\ntwocol(md\"\n```julia\nif (precondition[, postcondition])\n\t...\nend\n```\n\", md\"\n```julia\nif (postcondition[, precondition])\n\t~(...)\nend\n```\n\")\n)\n\nwhere `...` are statements and `~(...)` are the backward execution of them.\n\"\"\"\n\n# ╔═╡ 94cd1345-3132-4882-86fe-d2429f610d1d\nmd\"\"\"If no postcondition is provided, it means the precondition is the same as the postcondition. It is translated to `if (cond, cond) ... end`. `elseif` is also supported.\"\"\"\n\n# ╔═╡ 4a558bd3-6e42-4c61-bd23-888b7f33ae25\n@i function f7a(x)\n\tif x > 1\n\t\tx -= 1\n\tend\nend; @test_throws InvertibilityError f7a(1.2)\n\n# ╔═╡ 004f727a-e0c8-49cb-8858-dfdf4d3ac57a\n@i function f7b(x, branch_keeper)\n\tbranch_keeper ⊻= x > 1\n\tif (x > 1, branch_keeper)\n\t\tx += 1\n\tend\nend; @test f7b(1.2, false)[1] == 2.2\n\n# ╔═╡ 4c03cde9-b643-40ff-b275-f1795f88949e\ntitle2(\"For statement\")\n\n# ╔═╡ fae7c74e-d25e-4c1e-ac97-199e6dae3365\nmd\"\"\"\nThe reversible `for` statement is similar to its irreversible counterpart.\n\n$(\ntwocol(md\"\n```julia\nfor iter = start:step:stop\n\t...\nend\n```\n\", md\"\n```julia\nfor iter = stop:-step:start\n\t~(...)\nend\n```\n\")\n)\n\"\"\"\n\n# ╔═╡ 2f2b24ea-66d0-4b3b-a460-53b6b3f28ef0\nmd\"The iterator length should not be changed during the iteration.\"\n\n# ╔═╡ e9dbb64a-27b9-443a-b917-69d55c290235\n@i function f7c(x, y)\n\tfor i=1:length(x)\n\t\tPOP!(x, y[i])\n\tend\nend; @test_throws InvertibilityError f7c([1,2,3], [0,0,0])\n\n# ╔═╡ 0d56ce96-81a5-4102-acbc-7d88f80adcb3\nmd\"There is an `InvertibilityError` because the length of the `x` has been changed. The inverse execution will give incorrect result.\"\n\n# ╔═╡ 95c41bd1-e50a-42e8-93c3-3b754a458c13\ntitle2(\"While statement\")\n\n# ╔═╡ b8629aeb-6c9a-44ed-87a1-9ab22d9485ed\nmd\"\"\"\nThe reversible `while` statement starts with the `@from` macro.\n\n$(\ntwocol(md\"\n```julia\n@from condition1 while condition2\n\t...\nend\n```\n\", md\"\n```julia\n@from !(condition2) while !(condition1)\n ~(...)\nend\n```\n\")\n)\n\"\"\"\n\n# ╔═╡ 3b211406-041f-4b41-acae-3958e4a37224\nmd\"where the `condition1` in the forward pass is a condition that holds before entering the loop body, but broken at the first iteration, while `condition2` is just a normal while condition. In the backward pass, `!(condition1)` becomes the criteria to break the loop.\"\n\n# ╔═╡ 62522772-cb59-4d13-acdd-d5067b223910\n@i function f7d(x, i)\n\t@from i==0 while i<10\n\t\ti += 1\n\t\tx += 1\n\tend\nend\n\n# ╔═╡ 2ba68a0f-6e36-4ea2-a91d-6af43741bad1\nf7d(1, 0)\n\n# ╔═╡ 75cebaf1-38de-475f-892e-346fd2b46f6f\n@test (~f7d)(11, 10) == (1, 0)\n\n# ╔═╡ 72dcf2fe-eb48-4dee-8121-efafc87637e3\ntitle2(\"Compute-copy-uncompute statement\")\n\n# ╔═╡ 84321198-93d4-4d22-8c0f-a5a10b884e1f\nmd\"The *compute-copy-uncompute* statement is a widely used design pattern in reversible programming. \nWe compute the forward pass for the result, then we copy the result to the output variable, and run the backward pass to erase intermediate results.\n\n\n\nFor example, to compute `y = x * exp(k)`, we might write the following code\"\n\n# ╔═╡ 32244789-afbf-4215-97cd-15483f438eee\n@i function f7e(y, x, k)\n\texpk ← zero(k)\n\texpk += exp(k)\n\ty += x * expk\n\t# uncompute the ancilla and deallocate it\n\texpk -= exp(k)\n\texpk → zero(k)\nend; @test f7e(0.0, 2.0, 3.0)[1] ≈ 2.0 * exp(3.0)\n\n# ╔═╡ 4b7f0baf-0316-4da7-9ded-50c064ddbaa3\nmd\"It is equivalent to the following statement that generates the backward pass automatically for you.\"\n\n# ╔═╡ 0e02952c-7589-4606-b006-16a9f3e52ae1\n@i function f7f(y, x, k)\n\t# record the forward pass\n\t@routine begin\n\t\texpk ← zero(k)\n\t\texpk += exp(k)\n\tend\n\ty += x * expk\n\t# reverse execute the recorded the program\n\t~@routine\nend; @test f7f(0.0, 2.0, 3.0)[1] ≈ 2.0 * exp(3.0)\n\n# ╔═╡ f0904d3f-1bf1-459c-9959-b53c0f774e3f\nexample(\"Computing Fibonacci numbers\")\n\n# ╔═╡ 19bb2af5-2a67-453d-82b0-7d3059b1fa47\nmd\"The sequence of Fibonacci numbers are: 1, 1, 2, 3, 5, 8\"\n\n# ╔═╡ 5b5858bf-63ac-4e31-a516-055a9cd18ffe\n@i function rfib(out!, n::T) where T\n @routine begin\n\t\tn1 ← zero(T)\n\t\tn2 ← zero(T)\n n1 += n - 1\n n2 += n - 2\n end\n if (value(n) <= 2, ~)\n out! += 1\n else\n rfib(out!, n1)\n rfib(out!, n2)\n end\n ~@routine\nend\n\n# ╔═╡ 95060588-f24b-4eeb-9b0b-ed7159962a3c\n@test rfib(0, 6)[1] == 8\n\n# ╔═╡ c4cd9f88-9cd6-4364-b016-78f90aba6a66\ntitle1(\"Extending the instruction set\")\n# They are translated to `y += f(args...)` is translated to `PlusEq(f)(y, args...)`, `y -= f(args...)` is translated to `MinusEq(f)(y, args...)`, `y *= f(args...)` is translated to `MulEq(f)(y, args...)`, `y /= f(args...)` is translated to `DivEq(f)(y, args...)` and `y ⊻= f(args...)` is translated to `XorEq(f)(y, args...)`.\n\n# ╔═╡ f6049c78-7468-47ce-a4a5-84fab34d115a\ntitle2(\"How to create a new elementary reversible function\")\n\n# ╔═╡ b0f73825-bbb1-448c-b491-bf634fdd398a\nmd\"To define a pair of elementary functions that **reverse to each other**,\n1. declare two functions `f` and `g` that each of them defines a mapping ``\\mathbb{R}^n \\rightarrow \\mathbb{R}^n``\n2. use `@dual f g` to tell NiLang they are reversible to each other.\n3. if you want to make `f` and `g` differentiable, you can specify backward rules on these two function by defining two mappings on ``\\mathbb{G}^n\\rightarrow \\mathbb{G}^n``, where ``\\mathbb{G}`` is a 2-tuple of ``\\mathbb{R}`` (or `NiLang.AD.GVar`) in NiLang. It is similar to `ForwardDiff.Dual` (check [ForwardDiff](https://github.com/JuliaDiff/ForwardDiff.jl)) but defined for the backward pass.\n\nTo define a **self-reversible** elementary function\n1. declare a functions `f` that defines a mapping ``\\mathbb{R}^n \\rightarrow \\mathbb{R}^n``\n2. use `@selfdual f` \n3. define the backward rule on `f` to make it differentiable.\n\"\n\n# ╔═╡ 648cdcd6-f4f5-461f-a525-4b350cae9eb0\nexample(\"defining a new elementary function\")\n\n# ╔═╡ d6b1abd6-749d-4591-99e8-64aaa9199ab5\nmd\"\"\"\nOne can use the invertibility checker to check if the function is really reversible (under a certain tolerance `NiLangCore.GLOBAL_ATOL[]` = $(NiLangCore.GLOBAL_ATOL[])).\n\"\"\"\n\n# ╔═╡ f502b8c1-9b80-4e67-80e8-a64ddb88fb0f\n@test NiLang.check_inv(M.new_forward, (3.0,))\n\n# ╔═╡ 0bce342e-9a8e-4005-8b88-82da2d2c7163\nmd\"\"\"\nTo check of the gradients are properly defined, one can use `NiLang.AD.check_grad`\n\"\"\"\n\n# ╔═╡ fd9cf757-2698-4886-9f0a-c6c23ff0d331\n@test NiLang.AD.check_grad(M.new_forward, (3.0,); iloss=1)\n\n# ╔═╡ dda6652a-d063-4511-8041-e869bb88ca26\n@test NiLang.AD.check_grad(M.new_backward, (3.0,); iloss=1)\n\n# ╔═╡ a7d47e83-7f44-49d0-a43d-e01316fc6eba\ntitle1(\"Performance Tips\")\n\n# ╔═╡ eca3efef-f35b-4623-8af8-0b830a55566d\nmd\"The following trick still work in NiLang\n* Removing boundary check with `@inbounds` (it works on FastStack),\n* Add `@inline` before short functions,\n* Add `@simd` before a for loop,\n\nOther tricks like type stability are introduced in the [Julia documentation](https://docs.julialang.org/en/v1/manual/performance-tips/).\n\"\n\n# ╔═╡ 45985244-adbf-4d6d-9732-a963cca62212\ntitle2(\"Remove reversibility check\")\n\n# ╔═╡ 83d7e75f-7273-4c6a-bec1-a2180ebc3fb9\nmd\"This can be done by putting an `@invcheckoff` before a code block.\"\n\n# ╔═╡ 7fb05c65-f47c-430a-b588-c2f9bade40a9\nexample(\"computing the exp function by Taylor expansion\")\n\n# ╔═╡ 14c0caa1-51ea-448c-a7dc-d06e34dd0895\nmd\"Note: this is not a clever implementation. There is an approach of defining it without allocation.\"\n\n# ╔═╡ 457d07bb-e999-413e-8f29-58714670296f\n@i function exp_with_reversibility_check(y::T, x::T) where T\n\t@routine begin\n\t\tN ← 1_000\n\t\tanc ← zeros(T, N)\n\t\tanc[1] += 1\n\t\tanc_y ← T(2.0)\n\t\tfor i=2:N\n\t\t\t@routine begin\n\t\t\t\ttemp ← zero(T)\n\t\t\t\ttemp += x * anc[i-1]\n\t\t\tend\n\t\t\tanc[i] += temp / i\n\t\t\tanc_y += anc[i]\n\t\t\t~@routine\n\t\tend\n\tend\n\ty += anc_y\n\t~@routine\nend\n\n# ╔═╡ ac53eac0-1a59-4407-8bf6-3d8b966a9bff\n@benchmark exp_with_reversibility_check(0.0, 1.0) seconds=0.3\n\n# ╔═╡ 85c8ac7b-54f5-47dc-bd50-e78ffd6cf1cf\n@i function exp_without_reversibility_check(y::T, x::T) where T\n\t@routine @invcheckoff begin\n\t\tN ← 1_000\n\t\tanc ← zeros(T, N)\n\t\tanc[1] += 1\n\t\tanc_y ← T(2.0)\n\t\tfor i=2:N\n\t\t\t@routine begin\n\t\t\t\ttemp ← zero(T)\n\t\t\t\ttemp += x * anc[i-1]\n\t\t\tend\n\t\t\tanc[i] += temp / i\n\t\t\tanc_y += anc[i]\n\t\t\t~@routine\n\t\tend\n\tend\n\ty += anc_y\n\t~@routine\nend\n\n# ╔═╡ 95c55847-0591-4f7f-b9a1-aa974ccfef69\n@benchmark exp_without_reversibility_check(0.0, 1.0) seconds=0.3\n\n# ╔═╡ 91f8cfc6-e261-4945-8506-eed8caa607c2\ntitle1(\"Multi-threading and CUDA\")\n\n# ╔═╡ c82b3b5c-c4e2-4bf6-b4ec-0d05ba9a669b\n@i function multi_thread_exp(y::Vector, x::Vector)\n\t# check the size of `x` and `y`. `@assert` is not a valid statement in NiLang, so one should decorate it with `@safe` to tell the compiler, doing this is safe, do not check this statement.\n\t@safe @assert length(x) == length(y)\n\t@threads for i=1:length(y)\n\t\ty[i] += exp(x[i])\n\tend\nend; @test multi_thread_exp(zeros(3), [1.0, 2.0, 3.0])[1] ≈ [exp(1.0), exp(2.0), exp(3.0)]\n\n# ╔═╡ 32d75270-60d7-4326-a4ff-8674d0fbd491\nmd\"With [CUDA](https://github.com/JuliaGPU/CUDA.jl), one can also define parallel reversile and differentiable GPU device functions. Use the broadcasting version `y += x` as an example.\"\n\n# ╔═╡ 4b8834c1-8bb3-49f2-ae9e-1dbb8832d7f0\n@i function addkernel(target, source)\n @invcheckoff begin\n\t\t@routine b ← (blockIdx().x-1) * blockDim().x + threadIdx().x\n\t\tif (b <= length(target), ~)\n\t\t\t@inbounds target[b] += source[b]\n\t\tend\n\t\t~@routine\n\tend\nend\n\n# ╔═╡ d8f0ae56-e643-48a1-86ee-1cd907ecb662\nmd\"One can launch the kernel function in NiLang with `@cuda`\"\n\n# ╔═╡ e72b9dc2-dfac-4631-b114-01ec14297427\nmd\"\"\"\n```julia\nusing CUDA\n\n@i function :(+=)(identity)(target::CuArray, source::CuArray)\n @safe @assert length(target) == length(source)\n @cuda threads=256 blocks=ceil(Int,length(target)/256) addkernel(target, source)\nend\n```\n\"\"\"\n\n# ╔═╡ 8c93a773-edc0-4ec2-88ef-1b58b7deddc5\ntitle2(\"Shared read-write in parallel computing and autodiff\")\n\n# ╔═╡ 16d08950-0575-4a4b-afc8-11ddca3198c7\nmd\"The parallel code may suffer from the shared read issue when computing gradients.\nLet's take a look at a parallel code that computes the loss ``\\mathcal{L} = \\sum x \\vec z``.\"\n\n# ╔═╡ 7c594d19-59fc-433a-bffa-c63bad46869e\n@i function shared_read(loss::Real, y::Vector, x::Real, z::Vector)\n\t@safe @assert length(z) == length(y)\n\t@threads for i=1:length(y)\n\t\ty[i] += x * z[i]\n\tend\n\tfor i=1:length(y)\n\t\tloss += y[i]\n\tend\nend; @test shared_read(0.0, zeros(3), 2.0, [1.0, 2.0, 3.0])[1] ≈ 12\n\n# ╔═╡ 345b344d-afda-4ce1-a0e7-ce6063a69206\nmd\"However, when computing the gradients, the gradient on `x` will not be computed correctly.\"\n\n# ╔═╡ c55eb045-daca-42f9-a357-095edef24644\nlet\n\tz = randn(100)\n\t_, gy, gx, gz = NiLang.AD.gradient(shared_read, (0.0, zeros(100), 2.0, z); iloss=1)\n\t@test gx ≈ sum(z)\nend\n\n# ╔═╡ c6903b65-a8b8-4aef-8c43-c822077b9d0e\nmd\"The error is expected. Because the variable `x` is shared by multiple threads, when updating the gradient field of `x` in the backward pass, all threads will try to update the same gradient field, this is famous [race condition](https://en.wikipedia.org/wiki/Race_condition) in parallel computing.\"\n\n# ╔═╡ f80353d6-0dfe-4b0a-a1af-655d344473bf\ntitle1(\"Resources\")\n\n# ╔═╡ 4ca276fb-859d-4c5a-81c3-8e4b28922fa4\ntitle2(\"Help and Discussion\")\n\n# ╔═╡ 3d020209-b8dd-4605-9329-78a985f0a6a3\nmd\"\"\"\n`reversible-computing` channel of [Julia slack](https://julialang.org/slack/) and [Julia Zulip](https://julialang.zulipchat.com/register/).\n\"\"\"\n\n# ╔═╡ c7ec3496-79ea-4956-976c-b88dd22207c7\ntitle2(\"Learning\")\n\n# ╔═╡ 8530a9d1-5a27-4a1d-883f-4b033a6f8fe4\nmd\"\"\"\n1. Reversible computing book: Perumalla, Kalyan S. Introduction to reversible computing. CRC Press, 2013.\n2. Our paper: Liu, Jin-Guo, and Taine Zhao. \"Differentiate Everything with a Reversible Programming Language.\" arXiv:2003.04617 (2020).\n\"\"\"\n\n# ╔═╡ 7ce31932-0447-4445-99aa-7ebced7d0bad\nTableOfContents()\n\n# ╔═╡ Cell order:\n# ╟─2061b434-0ad1-46eb-a0c7-1a5f432bfa62\n# ╟─a4e76427-f051-4b29-915a-fdfce3a299bb\n# ╟─c2c7b4d4-f8c9-4ebf-8da2-0103f03136e7\n# ╟─12f07cc7-979c-43c3-9dc9-36ea1463c1f6\n# ╟─611b577f-4722-42bf-8f8e-aeb2fb30be71\n# ╟─605872cf-f3fd-462e-a2b1-7d1c5ae45efd\n# ╟─fb3dee44-5fa9-4773-8b7f-a83c44358545\n# ╠═d941d6c2-55bf-11eb-0002-35c7474e4050\n# ╠═70088425-6779-4a2d-ba6d-b0a34c8e93a6\n# ╠═af738f89-3214-429c-9c7d-18a6ea0d9401\n# ╠═48d7ebc1-5def-4a57-9ec1-3fc370a4543f\n# ╟─f0e94247-f615-472b-8218-3fa287b38aa1\n# ╟─2581aa33-1dc5-40b1-aa9f-6a11cc750c93\n# ╟─60575978-081a-4bca-a3ed-2b51cd6abc92\n# ╠═f98305cb-4ba2-404a-a5c3-65510e059504\n# ╟─e8cd6667-597f-458b-8465-1822e09a7891\n# ╟─20145d75-004a-4c2f-b7ff-c400ca846d42\n# ╟─c682a17f-600f-4034-bfe3-a851ab645c10\n# ╟─5239dfe2-ea6d-4e07-a1b1-90954fe8ddc9\n# ╟─e4d86a5a-e820-4a70-8a87-08bac416291b\n# ╟─c307b6a4-906d-4be7-9fd7-57c942aded51\n# ╟─47b502d4-e8af-4d58-9067-9700784ea435\n# ╠═faecc0a7-55d7-42a1-8e9f-7e30143eef9c\n# ╠═4cbe69d6-b68f-4bda-a0dd-209f9ee54f18\n# ╟─7dc82f28-77bf-40da-b520-800ed1bc80c9\n# ╟─9f5f9de3-9558-4c18-9d98-b77d19b570ec\n# ╠═6dfcfa19-f78f-4dac-89f7-d3c5dbe17987\n# ╟─dad1c6c0-d61b-4f9f-a71e-e683fe143aaa\n# ╟─3e4a3916-8fe4-4262-bcd3-3014822717a3\n# ╠═34906208-e6f1-4a67-860a-a7b056a86dde\n# ╟─4601df35-679f-465d-9191-c18748b2fd83\n# ╟─4fc72b9d-19a2-40f1-a4a8-5e97d3d5e529\n# ╠═a0fde16f-8454-4f5c-a29c-a9e415c0c311\n# ╟─633ff8f3-8d93-4f73-bec2-c42070e6ece9\n# ╠═57f2d890-b5a0-47b7-9e3d-af4d03b10605\n# ╟─34063cd0-171e-46ce-80dd-52a341fa50a1\n# ╠═5d5d01db-8ff9-434c-8771-1fec6393e1fb\n# ╠═10d85a50-f2f9-403e-8f6c-baef61cf702a\n# ╟─b52648bf-a28a-48af-8912-31729d943ce0\n# ╟─f45db10f-a836-40f3-9d8d-054ea6540e87\n# ╟─1903563e-ccc2-44d9-9dbe-e5dede275b3c\n# ╟─583f2585-15a3-47c6-a70e-e2f002754028\n# ╠═60e6ff80-3593-4ae4-a273-914847f692db\n# ╠═9e5cfd68-b58d-4d83-aae2-447e5f805c97\n# ╠═dc85a942-cf52-4405-ad03-32a768e1b6e7\n# ╟─5cdd346b-10a5-485c-ba78-4c0b3cb0e02f\n# ╠═af9287b7-6131-46f6-beb8-6885e55e1975\n# ╠═e20eeabf-1c80-431e-8cfc-4d1b79c52b5a\n# ╟─90d30eea-53de-48a0-9700-ff35681fdf38\n# ╠═390f58a5-6f5f-4d3a-bb16-ba04e43a07e7\n# ╠═2b57443e-a516-434b-be86-80616a98e2f5\n# ╟─fc2e27f9-b7ba-44cc-a953-6745548ad733\n# ╠═fc744931-360b-4478-9f77-c50f048de243\n# ╠═9a152b36-f377-44da-9700-ca9e05e365ff\n# ╟─b6dcd18c-606f-4340-b2ec-163e8bad03f5\n# ╟─a1a29f34-f8a9-4e9f-9afe-7d0096771440\n# ╟─90bd6ad4-3dd8-4e7c-b445-aed1d248a2ec\n# ╠═c0259e48-1973-486c-a828-1fcd3e4331c6\n# ╟─8bbffa31-04a6-49ca-b36f-4d4140d75992\n# ╠═a6f18c34-80ee-4b52-9ff8-f3c1b1d80f90\n# ╠═a694132b-4f52-467f-8bc4-dc32fe2812db\n# ╟─8c2c82f2-1240-4f2f-830e-ee8021c1a41a\n# ╟─6203cf10-f8cc-4fb9-b814-7552b68c01dc\n# ╠═f97a6bab-b9f9-4b95-98a9-381c51397526\n# ╠═4ca48a2e-43da-457a-8e9f-6476097e4d7b\n# ╟─2a2970f4-ab01-486b-89a2-6ff96f734018\n# ╠═0b80d9be-53d7-4bf3-a558-659607af4709\n# ╠═92362fda-bae2-4e35-bfe4-dcaea853d50b\n# ╟─db9e7940-39f1-4ccf-ac70-146a521daa6e\n# ╟─93936612-1447-4114-b864-aba43adef4bd\n# ╠═b2add70c-c5d6-4e0f-a153-43e21a197181\n# ╠═5c03d5a5-99f0-4efd-9a32-ce6d7c2b266c\n# ╟─2349e3ea-3053-42a4-b9d9-f97a76e4abd7\n# ╟─269f18ee-3cd8-466a-a522-7c624503e31b\n# ╠═89139719-c478-4066-9452-f9893f36d561\n# ╠═d620c5ee-7d9c-4d3f-9e87-0c828dfab9ca\n# ╟─255d01b9-a873-4e63-9298-9d8f073348b0\n# ╟─f6cf1729-766c-4ed7-b004-c8c8ec6c7e07\n# ╠═3645d672-423f-4ac8-805f-0452793fee5a\n# ╠═c2a0024e-11dd-4ef7-8346-4374d98cafc0\n# ╟─b20004e9-3c73-4dfb-8fd5-f377786fd53b\n# ╠═5c1952b1-5016-4c87-b23c-8e6a235bf8cd\n# ╠═8e4470ee-01da-4547-b091-c4f65cd729b0\n# ╟─0863bd06-cc70-4dde-b3b2-0a466805a356\n# ╟─a0bae195-04e1-4642-9e14-fe4691e0906b\n# ╠═20d6e8a0-2cf5-48ad-9549-60506b42b970\n# ╠═7dee5748-ed73-4e13-aa80-7a50efbc8449\n# ╟─4c719e9b-641e-404e-9ab7-59e89135f3ba\n# ╠═77947e00-42c3-4c9e-b62a-b4b29489db43\n# ╠═2614127d-34fb-4c3d-b678-42693f3c9341\n# ╟─8f169235-3bd1-4cc4-a083-79736d306ad5\n# ╠═dfc9d305-5bce-4555-bfa3-d8d61fe4ca09\n# ╟─f6bfa015-c101-45e8-995c-2bb6a3b7dc7d\n# ╟─8651d7ec-6bcd-4dbe-a062-c4bde32e5e91\n# ╟─edaa9fdb-3af8-4554-a701-0e3bff2107a5\n# ╠═7551a880-340e-4e3f-815b-188e73f7eb9a\n# ╠═0489e51b-781f-4441-bb7f-ff3bd2e848ad\n# ╟─7b0d30d6-39ff-4f6e-b13c-0ddbfcb576e5\n# ╠═042297d8-6ab3-4ae6-b6e7-3b1ab2d5553b\n# ╠═57d65a36-bfa8-4dc2-8e11-d87fa1324122\n# ╟─c21d81c3-981f-4472-ad61-d1661bfe5c4e\n# ╟─99d6fe7b-d704-48f3-b115-2b3159a78068\n# ╠═1950ff70-54eb-4ece-a26d-a23fd0e90f5a\n# ╟─21458f81-9007-46f8-92e0-7a17c60beb36\n# ╠═7813f4ce-6e98-45f3-94a8-7f5981129f2b\n# ╟─59ec7cb7-6011-456d-9f57-a55bb8ea51a0\n# ╟─aacf63a2-9708-40db-8928-049621a7bbc4\n# ╟─ad0097e7-c8ad-457a-82a9-18b998a9e9fb\n# ╟─94cd1345-3132-4882-86fe-d2429f610d1d\n# ╠═4a558bd3-6e42-4c61-bd23-888b7f33ae25\n# ╠═004f727a-e0c8-49cb-8858-dfdf4d3ac57a\n# ╟─4c03cde9-b643-40ff-b275-f1795f88949e\n# ╟─fae7c74e-d25e-4c1e-ac97-199e6dae3365\n# ╟─2f2b24ea-66d0-4b3b-a460-53b6b3f28ef0\n# ╠═e9dbb64a-27b9-443a-b917-69d55c290235\n# ╟─0d56ce96-81a5-4102-acbc-7d88f80adcb3\n# ╟─95c41bd1-e50a-42e8-93c3-3b754a458c13\n# ╟─b8629aeb-6c9a-44ed-87a1-9ab22d9485ed\n# ╟─3b211406-041f-4b41-acae-3958e4a37224\n# ╠═62522772-cb59-4d13-acdd-d5067b223910\n# ╠═2ba68a0f-6e36-4ea2-a91d-6af43741bad1\n# ╠═75cebaf1-38de-475f-892e-346fd2b46f6f\n# ╟─72dcf2fe-eb48-4dee-8121-efafc87637e3\n# ╟─84321198-93d4-4d22-8c0f-a5a10b884e1f\n# ╠═32244789-afbf-4215-97cd-15483f438eee\n# ╟─4b7f0baf-0316-4da7-9ded-50c064ddbaa3\n# ╠═0e02952c-7589-4606-b006-16a9f3e52ae1\n# ╟─f0904d3f-1bf1-459c-9959-b53c0f774e3f\n# ╟─19bb2af5-2a67-453d-82b0-7d3059b1fa47\n# ╠═5b5858bf-63ac-4e31-a516-055a9cd18ffe\n# ╠═95060588-f24b-4eeb-9b0b-ed7159962a3c\n# ╟─c4cd9f88-9cd6-4364-b016-78f90aba6a66\n# ╟─f6049c78-7468-47ce-a4a5-84fab34d115a\n# ╟─b0f73825-bbb1-448c-b491-bf634fdd398a\n# ╟─648cdcd6-f4f5-461f-a525-4b350cae9eb0\n# ╠═8c2c4fa6-172f-4dde-a279-5d0aecfdbe46\n# ╟─d6b1abd6-749d-4591-99e8-64aaa9199ab5\n# ╠═f502b8c1-9b80-4e67-80e8-a64ddb88fb0f\n# ╟─0bce342e-9a8e-4005-8b88-82da2d2c7163\n# ╠═fd9cf757-2698-4886-9f0a-c6c23ff0d331\n# ╠═dda6652a-d063-4511-8041-e869bb88ca26\n# ╟─a7d47e83-7f44-49d0-a43d-e01316fc6eba\n# ╟─eca3efef-f35b-4623-8af8-0b830a55566d\n# ╟─45985244-adbf-4d6d-9732-a963cca62212\n# ╟─83d7e75f-7273-4c6a-bec1-a2180ebc3fb9\n# ╟─7fb05c65-f47c-430a-b588-c2f9bade40a9\n# ╟─14c0caa1-51ea-448c-a7dc-d06e34dd0895\n# ╠═457d07bb-e999-413e-8f29-58714670296f\n# ╠═3199a048-7b39-40f8-8183-6a54cccd91b6\n# ╠═ac53eac0-1a59-4407-8bf6-3d8b966a9bff\n# ╠═85c8ac7b-54f5-47dc-bd50-e78ffd6cf1cf\n# ╠═95c55847-0591-4f7f-b9a1-aa974ccfef69\n# ╟─91f8cfc6-e261-4945-8506-eed8caa607c2\n# ╠═0e1ba158-a6bc-401c-9ba7-ed78020ad068\n# ╠═c82b3b5c-c4e2-4bf6-b4ec-0d05ba9a669b\n# ╟─32d75270-60d7-4326-a4ff-8674d0fbd491\n# ╠═4b8834c1-8bb3-49f2-ae9e-1dbb8832d7f0\n# ╟─d8f0ae56-e643-48a1-86ee-1cd907ecb662\n# ╟─e72b9dc2-dfac-4631-b114-01ec14297427\n# ╟─8c93a773-edc0-4ec2-88ef-1b58b7deddc5\n# ╟─16d08950-0575-4a4b-afc8-11ddca3198c7\n# ╠═7c594d19-59fc-433a-bffa-c63bad46869e\n# ╟─345b344d-afda-4ce1-a0e7-ce6063a69206\n# ╠═c55eb045-daca-42f9-a357-095edef24644\n# ╟─c6903b65-a8b8-4aef-8c43-c822077b9d0e\n# ╟─f80353d6-0dfe-4b0a-a1af-655d344473bf\n# ╟─4ca276fb-859d-4c5a-81c3-8e4b28922fa4\n# ╟─3d020209-b8dd-4605-9329-78a985f0a6a3\n# ╟─c7ec3496-79ea-4956-976c-b88dd22207c7\n# ╟─8530a9d1-5a27-4a1d-883f-4b033a6f8fe4\n# ╟─7ce31932-0447-4445-99aa-7ebced7d0bad\n" }, { "path": "notebooks/feynman.jl", "content": "### A Pluto.jl notebook ###\n# v0.15.0\n\nusing Markdown\nusing InteractiveUtils\n\n# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).\nmacro bind(def, element)\n quote\n local el = $(esc(element))\n global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing\n el\n end\nend\n\n# ╔═╡ f3e235e7-76b9-4c39-bc70-038539838ff4\nbegin\n\tusing Revise, Viznet, Compose, PlutoUI, Random, TikzPictures\n\tfunction leftright(a, b; width=600, leftcellwidth=0.5)\n\t\tHTML(\"\"\"\n\n\t\t\t\n\n\n\t\n\t\n
$(html(a))$(html(b))
\n\"\"\")\n\tend\n\t\n\t# up down layout\n\tfunction updown(a, b; width=nothing)\n\t\tHTML(\"\"\"\n\n\t\n\n\n\t\n
$(html(a))
$(html(b))
\n\"\"\")\n\tend\n\t\n\tfunction highlight(str)\n\t\tHTML(\"\"\"$(str)\"\"\")\n\tend\nend;\n\n# ╔═╡ e81de385-0070-49a9-a889-8fcf9d9e2951\nusing Plots; gr();\n\n# ╔═╡ db9a97b1-f76d-4f51-96c6-0159469c5adb\nusing NiLang\n\n# ╔═╡ e20e2d2e-4b28-4e32-8d80-ce029928a094\nhtml\"\"\"\n\n\n\n\"\"\"\n\n# ╔═╡ 8308df59-3faa-4abf-8f05-119bbae48f64\nlet\n\tgithub = html\"\"\"\n GiggleLiu\n\"\"\"\n\tmd\"# Feynman's Lectures on computing, Chap. 5\n\n-- Jinguo Liu ($github)\n\"\nend\n\n# ╔═╡ a3532a83-9fd3-4d24-b1bb-b52457317e51\nhtml\"\"\"A postdoc in Mikhail Lukin's group, department of physics

\n Harvard university

\n Quera Computing\n\"\"\"\n\n# ╔═╡ 15657e4b-848e-43ad-a99f-37143d11705e\nmd\"# Table of contents\n1. Irreversible computing requires dissipating heat to the environment\n * The relation between reversibility and energy cost\n * Compressing boxes\n * An eingine driven by information\n * The relation between computing speed and energy cost\n2. several reversible computing models\n * Copy machine\n * Magnetic dipole\n * DNA copying is a type of copy machine\n * General reversible computing\n * Billiard ball model\n\"\n\n# ╔═╡ f9675365-36aa-430c-b747-3bc4f602e6fb\nmd\"## Information erasure requires dissipating heat to the environment!\"\n\n# ╔═╡ 94c5eaa1-432c-4553-829e-f78d97f3c0ca\nmd\"\"\"\n* *Information*: uncertainty of system, can be quantified by entropy,\n* *Information erasure*: decrease of information entropy,\n* *Knowledge*: the complement of information, the more knowledge, the less uncertainty.\n\"\"\"\n\n# ╔═╡ bef6978d-e654-4364-b5eb-e9608cf68464\nmd\"*setup*: a collection of boxes, with same size ``V``, immersed to a heat bath of temperature of ``T``. Each having a jiggling particle inside.\"\n\n# ╔═╡ 046f7559-4af9-4982-b5c3-335add0911d7\nhtml\"\"\"\n
\n\"\"\"\n\n# ╔═╡ 0a039bfa-571e-4fad-b73c-1324d08777fc\nhtml\"\"\"\n
\n\"\"\"\n\n# ╔═╡ bf7abacc-5b0a-4623-b2c5-af60183ad4b0\nmd\"\"\"\nA piston attached to each box.\n\"\"\"\n\n# ╔═╡ 3f1e4d7a-32a7-4c7e-92dd-465bac925e63\nmd\"\"\"\n*goal*: Compress the boxes from size $V$ to size $V/2$, the process is isothermal.\n\"\"\"\n\n# ╔═╡ 95a21058-0b07-4859-af68-8ca5b48b2a77\nmd\"## Case 1: Ideal Gas\"\n\n# ╔═╡ f68bcfb6-97ce-48d1-b0b8-e8466d4ac879\nmd\"\"\"\nCase 1: we know nothing about the system. The gas does work\n```math\n\\begin{align}\n&pV = N \\underbrace{k}_{\\substack{\\text{Boltzman constant}\\\\\\sim 1.38 × 10-23 m^2 {\\rm kg} s^{-2} K^{-1}}} T\\\\\n&W_{\\rm gas} = \\int_{V}^{V/2} p dV = -NkT\\log 2\n\\end{align}\n```\n\"\"\"\n\n# ╔═╡ 2fe7c298-4c5d-464c-980b-6cd9a537ac1e\nlet\n\timg = TikzPicture(L\"\"\"\n\t\\draw (-1.05, -1.05) rectangle (1.1,1.1);\n\t\\foreach[evaluate={\\a=rand; \\b=rand;\\c=rand;\\d=rand;}] \\x in {1,...,20}{\n\t\t\\fill (\\a, \\b) circle [radius=0.05];\n\t\t\\draw[->,thick] (\\a, \\b) -- (\\a+\\c*0.2, \\b+\\d*0.2);\n\t}\n\\node[draw, single arrow, minimum height=10mm, minimum width=3mm,\n single arrow head extend=2mm, rotate=90] at (0.0,1.3) {presure};\n\\node[draw, single arrow, minimum height=10mm, minimum width=3mm,\n single arrow head extend=2mm, rotate=-90] at (0.0,-1.3) {presure};\n\\node[draw, single arrow, minimum height=10mm, minimum width=3mm,\n single arrow head extend=2mm, rotate=0] at (1.3,0.0) {presure};\n\\node[draw, single arrow, minimum height=10mm, minimum width=3mm,\n single arrow head extend=2mm, rotate=180] at (-1.3,0.0) {presure};\n\"\"\"; options=\"scale=2.0\", preamble=raw\"\\usetikzlibrary{shapes.arrows}\")\n\tleftright(img, md\"the microscopic explaination of the presure\n\t\t\n``kT \\sim \\text{average kinetic energy}``\")\nend\n\n# ╔═╡ 49dab78a-7bd9-4faa-8a30-9af8a96e0c5b\nmd\"## Case 2: With prior knowledge\"\n\n# ╔═╡ 3d4ba750-8d62-48ac-bf96-691397689ddc\nmd\"\"\"\nCase 2: we know one bit knowledge about each box:\n* 1: the atom is in the right half\n* 0: the atom is in the left half\n\n```math\nW_{\\rm gas} = 0\n```\n\"\"\"\n\n# ╔═╡ 7aa7b0ee-beeb-4a3e-abf1-aa71e916f4cd\nmd\"\"\"\n## Compare\nCase 1:\n\n* erase information (the left-right information of the atom),\n* consumes energy\n\nCase 2:\n* do not erase information,\n* does not consume energy\n\"\"\"\n\n# ╔═╡ f4cb9212-181f-4338-b858-1d99c7f415e9\nmd\"Erasing each bit information comes along with $kT \\log 2$ heat dissipation!!\"\n\n# ╔═╡ c1bbaec8-4fb9-4ab8-a30d-06a286597de0\nmd\"\"\"\n## Maxwell's demon\n\nThe *Second Law of Thermodynamics* states that the state of entropy of the entire universe, as an isolated system, will always increase over time. The second law also states that the changes in the entropy in the universe can never be negative.\n\n![](https://user-images.githubusercontent.com/6257240/124372430-b14fe200-dc57-11eb-9e8d-75385e2c621b.png)\n\"\"\"\n\n# ╔═╡ 6d7a07ff-be1b-4902-8a6d-7d9257c1157f\nmd\"\"\"\nBefore observing: (s, t), number of possible configurations ``2^{|s|+|t|}``\n\nAfter observing: (s, t=s), number of possible configurations ``2^{|s|}``\n\"\"\"\n\n# ╔═╡ 0577d67f-648f-407c-8abf-507d086445bd\nmd\"\"\"\n## Proving from the quantum setup\n\"\"\"\n\n# ╔═╡ eb10e436-bcce-4d81-891e-15158219fe80\nmd\"Reeb (2014)\"\n\n# ╔═╡ 7c5a30fd-95f9-4bb8-b34f-b10b0f2a27f2\nmd\"\"\"\n1. the process involves a “system” ``S`` and a “reservoir” ``R``, both described by Hilbert spaces,\n2. the reservoir ``R`` is initially in a thermal state, ``\\rho_R = e^{−\\beta H}/{\\rm tr}[e^{−\\beta H}]`` , where H is a Hermitian operator on ``R`` (“Hamiltonian”) and ``\\beta \\in [−\\infty, +\\infty]`` is the “inverse temperature”,\n3. the system ``S`` and the reservoir ``R`` are initially uncorrelated, ``\\rho_{SR} = \\rho_S \\otimes \\rho_R``, \n4. the process itself proceeds by unitary evolution, ``\\rho_{SR}'=U\\rho_{SR}U^\\dagger``.\n\"\"\"\n\n# ╔═╡ abee1bee-ed01-4b05-a848-3aeb695a24ba\nmd\"\"\"\n![](https://user-images.githubusercontent.com/6257240/124468410-0e868900-dd67-11eb-91b4-b9ab92f21152.png)\n\"\"\"\n\n# ╔═╡ b05538cc-de01-4b1e-a602-feb780cddf4a\nmd\"\"\"\nMain result: ``\\Delta > \\Delta S``, because\n```math\n\\begin{align}\n[S(\\rho_S') - S(\\rho_S)] + [S(\\rho_R')-S(\\rho_R)] &=[S(\\rho_S') + S(\\rho_R')-S(\\rho_{SR})] \\ldots (3)\\\\\n&=[S(\\rho_S') + S(\\rho_R')-S(\\rho_{SR}')] \\ldots (4)\\\\\n&=I(S': R') \\geq 0\n\\end{align}\n```\n\"\"\"\n\n# ╔═╡ 42c398ab-bb45-423f-b030-404e7582df5a\nmd\"\"\"\n## An information driven car\n\"\"\"\n\n# ╔═╡ 48081dd4-2bf4-43a1-899c-0303b4fcedd3\nmd\"\"\"\n![](https://user-images.githubusercontent.com/6257240/124372207-2bcc3200-dc57-11eb-840e-1bf2c85abf9b.png)\"\"\"\n\n# ╔═╡ c6ef8479-639b-45c1-9b48-a5d2c233d3b8\nmd\"1. set up the initial state to ``0``, contacting with a heat bath of temperature ``T``,\n2. place a piston at the half way of the box\n3. the environment warm up the box\n4. the particle isothermally push the piston outwards\"\n\n# ╔═╡ 6f01cdc2-6ce9-41da-b279-b047c9779405\nmd\"## An example of reversible computing: Copy machine\"\n\n# ╔═╡ 876ad6cf-84c1-4e34-89de-6f9273ba3479\nmd\"*setup*: A copier (state known) and a model (state unknown),\nboth being double well potentials. In figure\n* ``x`` axis is the parameter,\n* ``y`` axis is the energy.\n\"\n\n# ╔═╡ 9ca8912d-5fc5-4066-adb8-ad02f75c2cbe\nmd\"``0`` state, left well\n\n``1`` state, right well\"\n\n# ╔═╡ 9aab5751-e9e0-46c0-8e66-4b98258fed08\nmd\"\"\"\n![](https://user-images.githubusercontent.com/6257240/124372454-cc225680-dc57-11eb-8526-ed397ce10583.png)\n\"\"\"\n\n# ╔═╡ a29af398-ff22-44cb-a5aa-0b0409312be9\nmd\"There is a tilt force when we make two double wells close\"\n\n# ╔═╡ c3db622f-e9ff-4d99-afb6-9db65c6cae7a\nmd\"\n*goal*: set the state of copier to the same state as the model.\n\"\n\n# ╔═╡ 12cbf4b7-9b55-423c-bf59-5cb18e167afd\nmd\"*procedure*\"\n\n# ╔═╡ 89a5ff44-1b04-4bd8-a40a-83382a027fb3\nmd\"Step 1: lowering the copier's potential barrier.\"\n\n# ╔═╡ 51e7b853-8640-4415-a9a4-8c0e06ad916a\nmd\"Step 2: bring the model close to copier (above illustration).\"\n\n# ╔═╡ a8fe838e-727d-4068-887d-17b1bf99f90b\nmd\"Step 3: raise the copier's potential barrier.\"\n\n# ╔═╡ c7cd75cb-4c64-4704-b839-c5a556f89be7\nmd\"Step 4: take the model away\"\n\n# ╔═╡ ec14fba6-0cb9-483f-b3ea-cc4c5e83c965\nmd\"## Magnetic dipole\n\n[ref](https://en.wikipedia.org/wiki/Magnetic_dipole)\"\n\n# ╔═╡ f1abc5c1-2c34-422a-86c4-5ad8e7df8b7e\nmd\"\"\"\n*setup*: two magnetic dipoles pointing to the same direction.\n\"\"\"\n\n# ╔═╡ 8ad4e7c0-c496-4d29-ac09-e6525b1b4c0f\nmd\"\"\"\n![](https://user-images.githubusercontent.com/6257240/124498319-23c0df00-dd8a-11eb-9fca-51a87fde6ec0.png)\n\"\"\"\n\n# ╔═╡ 757e2d78-c5ee-4b40-bfd6-1b39af338d9d\nmd\"\"\"\n```math\n\\text{potential energy} \\approx \\sin^2 \\phi\n```\n\"\"\"\n\n# ╔═╡ cbea35c7-c3c8-48e7-bb47-d5e193aee2c4\nmd\"\"\"\nstate ``0``: ← ←\n\nstate ``1``: → →\n\"\"\"\n\n# ╔═╡ 81013954-1c48-4c05-82c9-49b4bfafda95\nlet\n\tx = 0:1000\n\ty = map(x->sin(x/1000*2π)^2, x)\n\tPlots.plot(x./1000, y, xlabel=\"ϕ/2π\", ylabel=\"potential energy\")\nend\n\n# ╔═╡ 1924eff7-1423-4e90-8005-43113d9deb3d\nmd\"\"\"\nStep 1: Introduce a vertical magnetic field ``B``, we have\n```math\n\\text{potential energy (magnetic field)} = -B \\sin \\phi\n```\n\"\"\"\n\n# ╔═╡ c7edbc15-cd59-45fd-a0dc-c48aadb1c096\nmd\"B = $(@bind B Slider(0:0.01:2; show_value=true))\"\n\n# ╔═╡ bf2c9da7-8c45-409f-82a3-979cd63ea993\nlet\n\tx = 0:1000\n\ty = map(x) do x\n\t\tϕ = x/1000*2π\n\t\tsin(ϕ)^2 - B * sin(ϕ)\n\tend\n\tPlots.plot(x./1000, y, xlabel=\"ϕ/2π\", ylabel=\"potential energy\")\nend\n\n# ╔═╡ 1c32a491-ac85-4132-82fd-9b846a8485df\nmd\"\"\"\ncopier state is ``\\uparrow \\uparrow``\n\"\"\"\n\n# ╔═╡ 6cbf202f-34e4-42b6-a7a7-5d766bfdfc37\nmd\"\"\"\nStep 2: bring the model (assume it is in state 1) close to the copier\n\n```math\n\\text{potential energy (model)} = -b \\cos \\phi\n```\n\"\"\"\n\n# ╔═╡ d40b318f-bff2-4d0b-b2a6-d00933ac7567\nmd\"b = $(@bind b Slider(-0.5:0.01:0.5; default=0.0,show_value=true))\"\n\n# ╔═╡ 54f53a7b-74e8-433b-94fd-9fa7192dfca5\nlet\n\tx = 0:1000\n\ty = map(x) do x\n\t\tϕ = x/1000*2π\n\t\tsin(ϕ)^2 - B * sin(ϕ) - b*cos(ϕ)\n\tend\n\tPlots.plot(x./1000, y, xlabel=\"ϕ/2π\", ylabel=\"potential energy\")\nend\n\n# ╔═╡ b0dcad96-e439-4e09-9e92-8cad7ede79af\nmd\"\"\"\n# Last time\n* Erase information -> make the system into a more certain state -> a decrease of entropy in the system -> requires dissipating heat to the heat bath (Landauer's principle), and this can be proved regorously from the quantum perspective.\n* Introduce a type of reversible computing model: copy machine\n * Magnetic dipole\n * Protein synthesis\n\"\"\"\n\n# ╔═╡ 0a15a2cf-2e7a-4bd7-ac78-0803fc3d5c73\nmd\"\"\"\n# This time\n* The relation between energy and speed in reversible computing.\n* General reversible gates and reversible programming,\n * The billiard ball model\n * Reversible control flows\n* Several reversible computing architectures\n\"\"\"\n\n# ╔═╡ ffbc5616-d2d9-4ce4-996f-d1a743bb89b3\nmd\"## Protein synthesis （Brownian computer）\"\n\n# ╔═╡ 2602d857-4a21-478a-97a2-58a177666f52\nmd\"\"\"\n*setup*: we only consider the first stage of protein synthesis, copying information from DNA to m-RNA. A DNA strand is immersed to a biological soup with lots of triphostrates such as ATP, CTP, GTP and UTP.\n\nA DNA strand is made up of alternating phosphate and pentose\nsugar groups. To each sugar group is attached one of four bases, A (adenine),\nT (thymine), C (cytosine) and G (guanine)\n\"\"\"\n\n# ╔═╡ cf27e340-578a-440d-8d4a-e5a2277d5205\nmd\"\"\"\n![](https://user-images.githubusercontent.com/6257240/122641081-f957fc00-d0d0-11eb-9c7b-180e11f9bc33.png)\n\"\"\"\n\n# ╔═╡ aa53fd68-5acd-488d-a096-5ce39759f481\nmd\"lowering potential: enzyme\"\n\n# ╔═╡ cb9a9ef0-c0dc-487c-8008-0f73f9910ef8\nmd\"\"\"\nkey point: chemecal reaction is reversible, the direction to evolve depends on the relative concentrations of pyrophosphates and triphosphates\nin the soup\n\"\"\"\n\n# ╔═╡ f7e0478d-1839-4684-9265-ee990fe9da45\nmd\"## Speed and energy in a Brownian computer\"\n\n# ╔═╡ 751e32d6-2582-4b1d-9558-124b1ef54f81\nmd\"\"\"\n![](https://user-images.githubusercontent.com/6257240/124506870-8c17bc80-dd9a-11eb-9144-4116cf00f1c2.png)\n\"\"\"\n\n# ╔═╡ 5f18987d-a69e-4db9-96d3-426ed298d9b8\nmd\"\"\"\nThermal fluctuation helps overcome the barrier\n```math\n\\text{forward rate} = C X e^{-(A-E_1)/kT}\n```\n```math\n\\text{backward rate} = C X e^{-(A-E_2)/kT}\n```\n\n``C`` is a factor that carries information about the thermal fluctuations in the\nenvironment\n\n``X`` is a factor depends on a variety of molecular properties of the particular\nsubstance\n\"\"\"\n\n# ╔═╡ 66495c77-3bbc-4731-b9e1-db11bbc24283\nmd\"*analysis*:\nThe rate of forward/backward computing is \n```math\nr = \\frac{\\text{forward rate}}{\\text{backward rate}} = e^{(E_1-E_2)/kT}\n```\nThe minimum free energy/step we need to pay is ``kT \\log r = (E_1-E_2)``\n\"\n\n# ╔═╡ 84b867a3-804e-4e7e-a56c-0ffc1f4e6683\nmd\"\"\"\n## Speed v.s. energy efficiency - the entropy perspective\n\"\"\"\n\n# ╔═╡ 4642d311-ef0b-4c29-901d-b5398a3ca7b6\nmd\"\"\"\n![](https://user-images.githubusercontent.com/6257240/124663469-200b8600-de78-11eb-8501-8ce97ea140c5.png)\n\"\"\"\n\n# ╔═╡ 96c3d50b-8a79-4de0-b7e0-c63c3b769b74\nmd\"The ratio of the forward rate and backward rate\"\n\n# ╔═╡ 066aa825-81e4-404d-bf5a-6a9431969702\nmd\"``r = n_2/n_1`` (determines the speed)\"\n\n# ╔═╡ 6e99ed64-a896-450e-8bab-845e0fe971ae\nmd\"``kT\\log r = \\underbrace{(S_2 - S_1) T}_{\\text{the cost of free energy}}``\"\n\n# ╔═╡ 7f803113-653a-4dfd-93f0-83babb253b32\nmd\"$F = E - TS$\"\n\n# ╔═╡ 7eb29d49-05f5-47e9-b4f5-4f31c5cd37ce\nmd\"## A more concrete example about the energy efficiency and speed\"\n\n# ╔═╡ aefdec07-dcef-4e00-bcb0-4747250cdd9b\nmd\"Charging up the capacitor to store signal energy in adiabatic CMOS.\"\n\n# ╔═╡ cbe4abaf-46f9-4726-97ae-cf3c378abaaf\nmd\"\"\"\n![](https://user-images.githubusercontent.com/6257240/122668453-287c7500-d186-11eb-962f-cc478be1dafe.png)\n\"\"\"\n\n# ╔═╡ 4f0c81f5-ce5f-4f73-a528-9feff4a7fc14\nmd\"Case 1: do it fast\n```math\nE = CU^2\n```\n\"\n\n# ╔═╡ 8249b820-8fb1-45d4-a95c-9c81e62e8216\nlet\n\tx = 0:1000\n\ty = map(x->abs(x-500)<250 ? 1.0 : 0.0, x)\n\tPlots.plot(x./1000, y, xlabel=\"time\", ylabel=\"voltage\")\nend\n\n# ╔═╡ 6503b377-b2d5-48be-90a4-97947afb4e5f\nmd\"Case 1: do it slow\n```math\nE = \\frac{CU^2}{2}\n```\n\"\n\n# ╔═╡ 53a571dd-cac7-432a-869c-b93a8fe05e17\nlet\n\tx = 0:1000\n\ty = map(x->abs(x-500)<200 ? 1.0 : (abs(x-500) < 400 ? 1/200 * (x < 500 ? abs(x-100) : abs(900-x)) : 0.0), x)\n\tPlots.plot(x./1000, y, xlabel=\"time\", ylabel=\"voltage\")\nend\n\n# ╔═╡ 2e5c7f59-dd35-4846-815a-b92eabeee089\nmd\"Conclusion: fast ramping dissipates heat to the environment.\"\n\n# ╔═╡ 7b326477-43b6-4a6e-8862-12e8b70e1ad9\nmd\"## Universal reversible gate set\"\n\n# ╔═╡ 47cd7560-a29e-4b55-bef5-28daa1cdb834\nmd\"Toffoli gate is universal\"\n\n# ╔═╡ 59ab4431-ea4d-4707-9a42-d50eafa40b56\nmd\"truth table ``(A, B, C) \\mapsto (A, B, C')``\n\n| A | B | C | C' |\n| --- | --- | --- | --- |\n| 0 | 0 | 0 | 0 |\n| 0 | 0 | 1 | 1 |\n| 0 | 1 | 0 | 0 |\n| 0 | 1 | 1 | 1 |\n| 1 | 0 | 0 | 0 |\n| 1 | 0 | 1 | 1 |\n| 1 | 1 | 0 | 1 |\n| 1 | 1 | 1 | 0 |\n\"\n\n# ╔═╡ 3630b412-beeb-455a-a4b8-1e1d50860266\nmd\"\n```julia\nif A && B\n\tC = ¬C\nend\n```\n\"\n\n# ╔═╡ ba6347d6-4ad0-403b-824f-dcf290a7c002\nmd\"proved by constructing a NAND gate (a classical univeral gate)\"\n\n# ╔═╡ a2f4975d-eeee-4a2d-97dd-dd0cfd29d665\nTikzPicture(L\"\"\"\n\\draw (0,0) rectangle (2,3);\n\\draw (-0.5,0.5) -- (0.0, 0.5);\n\\node at (-0.8, 0.5) {1};\n\\draw (-0.5,1.5) -- (0.0, 1.5);\n\\node at (-0.8, 1.5) {B};\n\\draw (-0.5,2.5) -- (0.0, 2.5);\n\\node at (-0.8, 2.5) {A};\n\\draw (2.5,0.5) -- (2.0, 0.5);\n\\node at (3.1, 0.5) {$\\overline{A\\land B}$};\n\\draw (2.5,1.5) -- (2.0, 1.5);\n\\node at (2.8, 1.5) {B};\n\\draw (2.5,2.5) -- (2.0, 2.5);\n\\node at (2.8, 2.5) {A};\n\\node at (1.0, 1.5) {Toffoli};\n\"\"\")\n\n# ╔═╡ 32d411e9-b01d-4ad2-b4aa-2f091034e6c0\nmd\"Fredkin gate is universal\"\n\n# ╔═╡ e5b83421-dd94-43ad-84eb-ca558bff6a2d\nmd\"truth table ``(A, B, C) \\mapsto (A, B', C')``\n\n| A | B | C | B' | C' |\n| --- | --- | --- | --- | --- |\n| 0 | 0 | 0 | 0 | 0 |\n| 0 | 0 | 1 | 0 | 1 |\n| 0 | 1 | 0 | 1 | 0 |\n| 0 | 1 | 1 | 1 | 1 |\n| 1 | 0 | 0 | 0 | 0 |\n| 1 | 0 | 1 | 1 | 0 |\n| 1 | 1 | 0 | 0 | 1 |\n| 1 | 1 | 1 | 1 | 1 |\n\"\n\n# ╔═╡ 118642ad-1aad-4f91-8da0-55a417b67750\nmd\"\n```julia\nif A\n\tB, C = C, B\nend\n```\n\"\n\n# ╔═╡ 45171ecc-9d34-4ab6-a00b-ec9c9afc33f8\nmd\"prove by constructing an AND gate and NOT gate\"\n\n# ╔═╡ 6cd60f7d-d7ce-4189-a2dd-e47ce6825741\nlet\n\timg1 = TikzPicture(L\"\"\"\n\\draw (0,0) rectangle (2,3);\n\\draw (-0.5,0.5) -- (0.0, 0.5);\n\\node at (-0.8, 0.5) {$C$};\n\\draw (-0.5,1.5) -- (0.0, 1.5);\n\\node at (-0.8, 1.5) {$0$};\n\\draw (-0.5,2.5) -- (0.0, 2.5);\n\\node at (-0.8, 2.5) {$A$};\n\\draw (2.5,0.5) -- (2.0, 0.5);\n\\node at (3.1, 0.5) {$\\overline{A}\\land C$};\n\\draw (2.5,1.5) -- (2.0, 1.5);\n\\node at (3.1, 1.5) {$A\\land C$};\n\\draw (2.5,2.5) -- (2.0, 2.5);\n\\node at (2.8, 2.5) {$A$};\n\\node at (1.0, 1.5) {Fredkin};\n\"\"\")\n\timg2 = TikzPicture(L\"\"\"\n\\draw (0,0) rectangle (2,3);\n\\draw (-0.5,0.5) -- (0.0, 0.5);\n\\node at (-0.8, 0.5) {$1$};\n\\draw (-0.5,1.5) -- (0.0, 1.5);\n\\node at (-0.8, 1.5) {$0$};\n\\draw (-0.5,2.5) -- (0.0, 2.5);\n\\node at (-0.8, 2.5) {$A$};\n\\draw (2.5,0.5) -- (2.0, 0.5);\n\\node at (2.8, 0.5) {$\\overline{A}$};\n\\draw (2.5,1.5) -- (2.0, 1.5);\n\\node at (2.8, 1.5) {$A$};\n\\draw (2.5,2.5) -- (2.0, 2.5);\n\\node at (2.8, 2.5) {$A$};\n\\node at (1.0, 1.5) {Fredkin};\n\"\"\")\n\tleftright(img1, img2)\nend\n\n# ╔═╡ ff3fc929-f448-41be-8f60-65de33dff36a\nmd\"\"\"\n## Billiard Ball Computer\n\"\"\"\n\n# ╔═╡ 5ec2649e-9988-4f38-896a-64ef6ed91d82\nmd\"*setup*: Billiard balls in 2D space\"\n\n# ╔═╡ aa8475c3-c68b-4200-8634-ace33f525417\nmd\"The collision Gate\"\n\n# ╔═╡ aa22f905-b69d-405e-b09a-a765d60f6079\nmd\"\"\"\n![](https://user-images.githubusercontent.com/6257240/124666172-a2497980-de7b-11eb-9221-378f0453d41d.png)\n\"\"\"\n\n# ╔═╡ 2dcbaac0-2fad-4292-ad31-8188a60876da\nmd\"Four redirection gates\"\n\n# ╔═╡ 6bad4f5f-806f-480a-ae16-2582761ce5e3\nmd\"\"\"\n![](https://user-images.githubusercontent.com/6257240/124666268-cc02a080-de7b-11eb-9017-c62d9d251521.png)\n\"\"\"\n\n# ╔═╡ e200dde3-9033-45b5-bfe0-2d03753b2c11\nmd\"*Define some other useful gates*:\"\n\n# ╔═╡ d7a4b342-ef0a-44f9-b88d-bbb04483e8b3\nlet\n\timg1 = md\"\"\"\n![](https://user-images.githubusercontent.com/6257240/124670709-62d25b80-de82-11eb-9c1b-25377e16c43a.png)\n\"\"\"\n\timg2 = md\"\"\"\n![](https://user-images.githubusercontent.com/6257240/124670724-6960d300-de82-11eb-8ebb-4747e0d43fff.png)\n\"\"\"\n\tleftright(img1, img2)\nend\n\n# ╔═╡ 908f19a2-6d32-4776-95eb-b249a8155ddc\nmd\"\"\"prove by mapping it to a Fredkin gate\"\"\"\n\n# ╔═╡ 05fc1fae-b378-4c39-b060-74ca635745ec\nmd\"\"\"![](https://user-images.githubusercontent.com/6257240/124670762-78478580-de82-11eb-8061-409e0db1388c.png)\"\"\"\n\n# ╔═╡ d0573bf9-0fd6-4512-bc13-17aa23a3265b\nmd\"\"\"\n## General reversible computing\n\"\"\"\n\n# ╔═╡ d8998d5f-65b2-4850-aef9-f19ecc192eca\nmd\"\"\"*Problem 5.4*: A related problem concerns how to get \"if' clauses to work. What\nif, after having followed an \"if... then ... \" command, the machine starts to\nreverse? How can the machine get back to the original condition that dictated\nwhich way the \"if' branched? Of course, a set of initial conditions can result in\na single \"if' output (\"if x = 2, 3, 4 or 6.159 let F= d\"), so this condition may not be uniquely specified. Here is a nice way to analyze things. Simply bring in a\nnew variable at each branch, and assign a unique value to this variable for each\nchoice at a branch point. You might like to work this through in detail.\"\"\"\n\n# ╔═╡ 6b4c180c-9a12-4e3d-9336-1431e7c5875a\nTikzPicture(L\"\"\"\n\\draw [black, thick,->] (0, 0) -- (1, 0);\n\\draw [black, thick,->,dashed] (1, 0) .. controls (1.5, 0.5) .. (2, 0);\n\\node at (1.5, 0.5) {goto $\\ldots$};\n\\draw [black, thick,->] (2, 0) -- (3, 0);\n \n\\def\\x{4};\n\\draw [black, thick,<-] (\\x, 0) -- (\\x+1, 0);\n\\draw [black, thick,<-,dashed] (\\x+1, 0) .. controls (\\x+1.5, 0.5) .. (\\x+2, 0);\n\\draw [black, thick,<-,dashed] (\\x+1, 0) -- (\\x+2, 0);\n\\node at (\\x+1.5, 0.5) {comefrom?};\n\\draw [black, thick,<-] (\\x+2, 0) -- (\\x+3, 0);\n \n\\node at (1.5, -0.2) {call};\n\\node at (\\x+1.5, -0.2) {uncall};\n\"\"\", options=\"scale=2.0\", preamble=\"\")\n\n# ╔═╡ 0eb66cc9-93c0-4f07-b31b-a9bf9000260e\nmd\"Reversible branching statement\"\n\n# ╔═╡ 85bf9f92-30f1-4e05-8d07-d8e481f20ccb\nTikzPicture(L\"\"\"\n\\node [test] (pre) {precondition};\n\\node [proc, it] (st1) [right=of pre] {statements 1};\n\\node [proc, it] (st2) {statements 2};\t\n\\node [test] (post1) [right=of st1] {postcondition};\n\\node [test] (post2) [right=of st2] {postcondition};\n\\node [proc,red] (err1) [above=of post1] {invertibility error};\n\\node [proc,red] (err2) [below=of post2] {invertibility error};\n\\draw [->,black] (pre.east) -- (st1) node[midway,above] {T};\n\\draw [->,black] (pre.south) |- (st2) node[midway,below] {F};\n\\draw [->,black] (-2.5, 0.0) -- (pre.west);\n\\draw [->,black] (st1) -- (post1);\n\\draw [->,black] (st2) -- (post2);\n\\draw [->,red] (post1) -- (err1) node[midway,right] {F};\n\\draw [->,red] (post2) -- (err2) node[midway,right] {T};\n\\draw [->,black] (post1.east) -- (12, 0) node[midway,above] {T};\n\\draw [black] (post2.east) -| (11, 0) node[midway,right] {F};\n\"\"\", options=raw\" font=\\sffamily\\small,\n >={Triangle[]},\n */.tip={Circle[]},\n start chain=going below,\n node distance=18mm and 40mm,\n every join/.style={norm},\n base/.style={draw, on chain, on grid, align=center, minimum height=4ex, inner color=black!50!gray!10, outer color=black!50!gray!15},\n proc/.style={base, rectangle, text width=8em},\n test/.style={base, diamond, text centered, aspect=2.6,inner sep=-0ex},\n norm/.style={->, draw, black},\n it/.style={font={\\sffamily\\small\\itshape}}\", preamble=raw\"\\usetikzlibrary{shapes.geometric,arrows.meta,chains,positioning,quotes}\")\n\n# ╔═╡ fbba0a91-9f48-4d91-90e7-f6a7df3227f9\nmd\"## Bennett's compute copy uncompute scheme\"\n\n# ╔═╡ 7fc81b9f-73ed-4780-9204-ddf39467e58f\nmd\"*setup*: we have a long linear program. We use the reversible embeded domain specific language in Julia\"\n\n# ╔═╡ d08ae188-937f-474b-92d7-cb8eeda063fe\nhtml\"\"\"\"\"\"\n\n# ╔═╡ 7ee8cfc9-26b2-4fe4-8263-1f4d2f7c276d\nmd\"Initially written by GiggleLiu and Taine Zhao (The author of MLStyle)\"\n\n# ╔═╡ 1669f5e3-efe1-4b79-a2b6-11ed7476a2a1\nmd\"the program of finding the maximum number\"\n\n# ╔═╡ 5000f4c3-5416-4e53-88ae-e30d8d09827e\n@i function i_find_maximum_v1(s₂, s₃, s₄, x₁, x₂, x₃, x₄) where T\n\ts₂ += max(x₁, x₂) # step 1\n\ts₃ += max(s₂, x₃) # step 2\n\ts₄ += max(s₃, x₄) # step 3\nend\n\n# ╔═╡ 2413c061-89de-403f-8011-e458f5a9859d\ni_find_maximum_v1(0, 0, 0, 3, 2, 8, 1)\n\n# ╔═╡ d4704779-9261-478b-bbf6-551220783e12\nmd\"the basic building block of compute-copy-uncompute\"\n\n# ╔═╡ 4be065b5-0841-4d54-b9ab-d6770d4d9d94\n@i function i_find_maximum_v2(s₄, x₁, x₂, x₃, x₄) where T\n\t# compute\n\ts₂ ← 0 # variable on the working tape\n\ts₃ ← 0\n\ts₂ += max(x₁, x₂) # step 1\n\ts₃ += max(s₂, x₃) # step 2\n\t\n\t# copy\n\ts₄ += max(s₃, x₄) # step 3\n\t\n\t# uncompute\n\ts₃ -= max(s₂, x₃) # step 4\n\ts₂ -= max(x₁, x₂) # step 5\n\ts₂ → 0\n\ts₃ → 0\nend\n\n# ╔═╡ e850e53d-cf61-4fc7-9cb3-e318ae957f0b\ni_find_maximum_v2(0, 3, 2, 8, 1)\n\n# ╔═╡ a267ea5f-8bd5-4ee0-9c8d-47e2d3b81692\nTikzPicture(L\"\"\"\n\\def\\r{0.15};\n\\foreach \\x in {1,4}{\n\t\\fill[fill=black] (\\x, 0) circle [radius=\\r];\n\t\\node[white] at (\\x, 0) {$s_{\\x}$};\n}\n\\foreach \\x in {2,3}{\n\t\\draw (\\x, 0) circle [radius=\\r];\n\t\\node[black] at (\\x, 0) {$s_{\\x}$};\n}\n\\fill[fill=white] (5.5, 0) circle [radius=\\r];\n\\foreach \\x in {1,...,3}{\n\t\\draw [black, thick, ->] (\\x+\\r, \\r) .. controls (\\x+0.5, 0.3) .. (\\x+1-\\r, \\r);\n\t\\node at (\\x+0.5, 0.4) {\\x};\n\t}\n\\foreach[evaluate={\\y=int(6-\\x)}] \\x in {1,...,2}{\n\t\\draw [red, thick, <-] (\\x+\\r, -\\r) .. controls (\\x+0.5, -0.3) .. (\\x+1-\\r, -\\r);\n\t\\node at (\\x+0.5, -0.4) {\\y};\n\t}\n\"\"\"\n, options=\"scale=1.8\", preamble=\"\")\n\n# ╔═╡ c02520a3-3375-4d83-a0dc-1aeac2aa7d5f\nmd\"Recursively apply Bennett's time space tradeoff scheme\"\n\n# ╔═╡ 2e18fc92-4185-493b-9ce8-cca63dad7d2d\nTikzPicture(L\"\"\"\n\\def\\r{0.15};\n\\def\\n{10};\n\\foreach \\x in {1,4,7,10}{\n \\fill[fill=black] (\\x, 0) circle [radius=\\r];\n \\node[white] at (\\x, 0) {$s_{\\x}$};\n}\n\\foreach \\x in {2,3,5,6,8,9}{\n \\draw (\\x, 0) circle [radius=\\r];\n \\node[black] at (\\x, 0) {$s_{\\x}$};\n}\n\\fill[fill=white] (\\n+0.5, 0) circle [radius=\\r];\n\\foreach \\x/\\t in {1/1,2/2,3/3,4/6,5/7,6/8,7/11,8/12,9/13}{\n \\draw [black, thick, ->] (\\x+\\r, \\r) .. controls (\\x+0.5, 0.3) .. (\\x+1-\\r, \\r);\n \\node[black] at (\\x+0.5, 0.4) {\\t};\n }\n\\foreach \\x/\\t in {1/5,2/4,4/10,5/9,7/15,8/14}{\n \\draw [black, thick, <-] (\\x+\\r, -\\r) .. controls (\\x+0.5, -0.3) .. (\\x+1-\\r, -\\r);\n \\node[black] at (\\x+0.5, -0.4) {\\t};\n}\n\"\"\"\n, options=\"scale=2.0\", preamble=\"\")\n\n# ╔═╡ acc7b185-e4df-4aca-aa42-554215065384\nTikzPicture(L\"\"\"\n\\def\\r{0.15};\n\\def\\n{10};\n\\foreach \\x in {1,4,7,10}{\n \\fill[fill=black] (\\x, 0) circle [radius=\\r];\n \\node[white] at (\\x, 0) {$s_{\\x}$};\n}\n\n\\fill[fill=white] (\\n+0.5, 0) circle [radius=\\r];\n\\foreach \\x in {1,4,7}{\n \\draw [black, thick, ->] (\\x+\\r, \\r) .. controls (\\x+1.5, 0.6) .. (\\x+3-\\r, \\r);\n }\n\\foreach \\x in {1,4}{\n \\draw [black, thick, <-] (\\x+\\r, -\\r) .. controls (\\x+1.5, -0.6) .. (\\x+3-\\r, -\\r);\n}\n\"\"\"\n, options=\"scale=2.0\", preamble=\"\")\n\n# ╔═╡ 00c9e973-7e06-4483-bf4c-be7374707118\nmd\"\n* Space complexity: ``O(S \\log T)``\n* Time complexity: ``O(T^{1+\\epsilon})``\"\n\n# ╔═╡ 83ff3fc3-bcd8-4235-a42f-1d75c7d6aa5b\nmd\"## Computing architectures\"\n\n# ╔═╡ b308e270-6b40-4946-ac92-c705823f2c1e\nlet\n\ttxt1 = md\"Traditional irreversible computer\n\t\n$E \\sim 10^8 kT$\"\n\timg1 = html\"\"\"\"\"\"\n\ttxt2 = md\"DNA copying is a living copy machine\n\n$E \\sim 100k T$\"\n\timg2 = html\"\"\"\n\n\"\"\"\n\ttxt3 = md\"\"\"\nAdiabatic CMOS [Athas, 1994]\n\n$E \\sim 10^6 kT$\n\"\"\"\n\timg3 = html\"\"\"\n\n\"\"\"\n\ttxt4 = md\"\"\"Adiabatic superconducting devices [Takeuchi, 2014]\n\n$E \\sim kT$\n\"\"\"\n\timg4 = html\"\"\"\n\n\"\"\"\n\tupdown(leftright(updown(img1, txt1), updown(img2, txt2)), leftright(updown(img3, txt3), updown(img4, txt4)))\nend\n\n# ╔═╡ e483b3d4-d01c-4a98-8e68-e8120a7d95a7\nmd\"## More\n\n\n![](https://user-images.githubusercontent.com/6257240/123520518-22ebc700-d67f-11eb-8af1-a452605cc1d8.png)\n\n*Youtube*: Michael P. Frank: Fundamental Physics of Reversible Computing — An Introduction, Part 1\n\"\n\n# ╔═╡ 74017e78-0f02-41bb-a160-5f2d26c18268\nmd\"\"\"\n![](https://user-images.githubusercontent.com/6257240/125467165-d3ec7c24-18cb-48d6-99b6-708326789bf9.png)\n\n\t\nKenichi Morita, How can we construct reversible Turing machines in a very simple reversible cellular automaton? Video can be found in this conference page: [https://reversible-computation-2021.github.io/program/](https://reversible-computation-2021.github.io/program/)\n\"\"\"\n\n# ╔═╡ 0b3735c2-695c-4225-843e-16ca17aac0eb\nmd\"\"\"## Take home message\n\n1. Irreversible computing -> information erasure -> disspate heat (``kT\\log 2`` per bit)\n2. Reversible computing -> requires operations being adiabatic -> slow\n2. Reversible programming suffers from **polynomial time overhead and logarithmic space overhead** when differentiating a irreversible linear program\n3. Brownian computer\n * mRNA copy\n * Magnetic dopile\n4. General reversible computer\n * Billiard ball model\n * Reversible cellular automata\n * Adiabatic CMOS\n\n6. **How to find this notebook?** In NiLang's Github repo, file: `notebooks/feynman.jl`\n\"\"\"\n\n# ╔═╡ d7942b37-f821-494a-8f18-5f267aa3457a\nmd\"\"\"\n### References\n* Reeb, David, and Michael M. Wolf. \"An improved Landauer principle with finite-size corrections.\" (2014).\n* Athas, William C., and L. J. Svensson. \"Reversible logic issues in adiabatic CMOS.\" Proceedings Workshop on Physics and Computation. (1994).\n* Takeuchi, N., Y. Yamanashi, and N. Yoshikawa. \"Reversible logic gate using adiabatic superconducting devices.\" (2014)\n* Griewank, Andreas. \"Achieving logarithmic growth of temporal and spatial complexity in reverse automatic differentiation.\" Optimization Methods and software 1.1 (1992): 35-54.\n* Ming Li, John Tromp, Paul Vitanyi. \"Reversible Simulation of Irreversible Computation by Pebble Games\" (1997)\n\"\"\"\n\n# ╔═╡ 00000000-0000-0000-0000-000000000001\nPLUTO_PROJECT_TOML_CONTENTS = \"\"\"\n[deps]\nCompose = \"a81c6b42-2e10-5240-aca2-a61377ecd94b\"\nNiLang = \"ab4ef3a6-0b42-11ea-31f6-e34652774712\"\nPlots = \"91a5bcdd-55d7-5caf-9e0b-520d859cae80\"\nPlutoUI = \"7f904dfe-b85e-4ff6-b463-dae2292396a8\"\nRandom = \"9a3f8284-a2c9-5f02-9a11-845980a1fd5c\"\nRevise = \"295af30f-e4ad-537b-8983-00126c2a3abe\"\nTikzPictures = \"37f6aa50-8035-52d0-81c2-5a1d08754b2d\"\nViznet = \"52a3aca4-6234-47fd-b74a-806bdf78ede9\"\n\n[compat]\nCompose = \"~0.9.2\"\nNiLang = \"~0.9.1\"\nPlots = \"~1.18.0\"\nPlutoUI = \"~0.7.9\"\nRevise = \"~3.1.17\"\nTikzPictures = \"~3.3.3\"\nViznet = \"~0.3.3\"\n\"\"\"\n\n# ╔═╡ 00000000-0000-0000-0000-000000000002\nPLUTO_MANIFEST_TOML_CONTENTS = \"\"\"\n# This file is machine-generated - editing it directly is not advised\n\n[[Adapt]]\ndeps = [\"LinearAlgebra\"]\ngit-tree-sha1 = \"84918055d15b3114ede17ac6a7182f68870c16f7\"\nuuid = \"79e6a3ab-5dfb-504d-930d-738a2a938a0e\"\nversion = \"3.3.1\"\n\n[[ArgTools]]\nuuid = \"0dad84c5-d112-42e6-8d28-ef12dabb789f\"\n\n[[Artifacts]]\nuuid = \"56f22d72-fd6d-98f1-02f0-08ddc0907c33\"\n\n[[Base64]]\nuuid = \"2a0f44e3-6c83-55bd-87e4-b1978d98bd5f\"\n\n[[Bzip2_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"c3598e525718abcc440f69cc6d5f60dda0a1b61e\"\nuuid = \"6e34b625-4abd-537c-b88f-471c36dfa7a0\"\nversion = \"1.0.6+5\"\n\n[[Cairo_jll]]\ndeps = [\"Artifacts\", \"Bzip2_jll\", \"Fontconfig_jll\", \"FreeType2_jll\", \"Glib_jll\", \"JLLWrappers\", \"LZO_jll\", \"Libdl\", \"Pixman_jll\", \"Pkg\", \"Xorg_libXext_jll\", \"Xorg_libXrender_jll\", \"Zlib_jll\", \"libpng_jll\"]\ngit-tree-sha1 = \"e2f47f6d8337369411569fd45ae5753ca10394c6\"\nuuid = \"83423d85-b0ee-5818-9007-b63ccbeb887a\"\nversion = \"1.16.0+6\"\n\n[[CodeTracking]]\ndeps = [\"InteractiveUtils\", \"UUIDs\"]\ngit-tree-sha1 = \"8ad457cfeb0bca98732c97958ef81000a543e73e\"\nuuid = \"da1fd8a2-8d9e-5ec2-8556-3022fb5608a2\"\nversion = \"1.0.5\"\n\n[[ColorSchemes]]\ndeps = [\"ColorTypes\", \"Colors\", \"FixedPointNumbers\", \"Random\", \"StaticArrays\"]\ngit-tree-sha1 = \"c8fd01e4b736013bc61b704871d20503b33ea402\"\nuuid = \"35d6a980-a343-548e-a6ea-1d62b119f2f4\"\nversion = \"3.12.1\"\n\n[[ColorTypes]]\ndeps = [\"FixedPointNumbers\", \"Random\"]\ngit-tree-sha1 = \"32a2b8af383f11cbb65803883837a149d10dfe8a\"\nuuid = \"3da002f7-5984-5a60-b8a6-cbb66c0b333f\"\nversion = \"0.10.12\"\n\n[[Colors]]\ndeps = [\"ColorTypes\", \"FixedPointNumbers\", \"Reexport\"]\ngit-tree-sha1 = \"417b0ed7b8b838aa6ca0a87aadf1bb9eb111ce40\"\nuuid = \"5ae59095-9a9b-59fe-a467-6f913c188581\"\nversion = \"0.12.8\"\n\n[[Compat]]\ndeps = [\"Base64\", \"Dates\", \"DelimitedFiles\", \"Distributed\", \"InteractiveUtils\", \"LibGit2\", \"Libdl\", \"LinearAlgebra\", \"Markdown\", \"Mmap\", \"Pkg\", \"Printf\", \"REPL\", \"Random\", \"SHA\", \"Serialization\", \"SharedArrays\", \"Sockets\", \"SparseArrays\", \"Statistics\", \"Test\", \"UUIDs\", \"Unicode\"]\ngit-tree-sha1 = \"dc7dedc2c2aa9faf59a55c622760a25cbefbe941\"\nuuid = \"34da2185-b29b-5c13-b0c7-acf172513d20\"\nversion = \"3.31.0\"\n\n[[CompilerSupportLibraries_jll]]\ndeps = [\"Artifacts\", \"Libdl\"]\nuuid = \"e66e0078-7015-5450-92f7-15fbd957f2ae\"\n\n[[Compose]]\ndeps = [\"Base64\", \"Colors\", \"DataStructures\", \"Dates\", \"IterTools\", \"JSON\", \"LinearAlgebra\", \"Measures\", \"Printf\", \"Random\", \"Requires\", \"Statistics\", \"UUIDs\"]\ngit-tree-sha1 = \"c6461fc7c35a4bb8d00905df7adafcff1fe3a6bc\"\nuuid = \"a81c6b42-2e10-5240-aca2-a61377ecd94b\"\nversion = \"0.9.2\"\n\n[[Contour]]\ndeps = [\"StaticArrays\"]\ngit-tree-sha1 = \"9f02045d934dc030edad45944ea80dbd1f0ebea7\"\nuuid = \"d38c429a-6771-53c6-b99e-75d170b6e991\"\nversion = \"0.5.7\"\n\n[[DataAPI]]\ngit-tree-sha1 = \"ee400abb2298bd13bfc3df1c412ed228061a2385\"\nuuid = \"9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a\"\nversion = \"1.7.0\"\n\n[[DataStructures]]\ndeps = [\"Compat\", \"InteractiveUtils\", \"OrderedCollections\"]\ngit-tree-sha1 = \"4437b64df1e0adccc3e5d1adbc3ac741095e4677\"\nuuid = \"864edb3b-99cc-5e75-8d2d-829cb0a9cfe8\"\nversion = \"0.18.9\"\n\n[[DataValueInterfaces]]\ngit-tree-sha1 = \"bfc1187b79289637fa0ef6d4436ebdfe6905cbd6\"\nuuid = \"e2d170a0-9d28-54be-80f0-106bbe20a464\"\nversion = \"1.0.0\"\n\n[[Dates]]\ndeps = [\"Printf\"]\nuuid = \"ade2ca70-3891-5945-98fb-dc099432e06a\"\n\n[[DelimitedFiles]]\ndeps = [\"Mmap\"]\nuuid = \"8bb1440f-4735-579b-a4ab-409b98df4dab\"\n\n[[Dierckx]]\ndeps = [\"Dierckx_jll\"]\ngit-tree-sha1 = \"5fefbe52e9a6e55b8f87cb89352d469bd3a3a090\"\nuuid = \"39dd38d3-220a-591b-8e3c-4c3a8c710a94\"\nversion = \"0.5.1\"\n\n[[Dierckx_jll]]\ndeps = [\"CompilerSupportLibraries_jll\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"a580560f526f6fc6973e8bad2b036514a4e3b013\"\nuuid = \"cd4c43a9-7502-52ba-aa6d-59fb2a88580b\"\nversion = \"0.0.1+0\"\n\n[[Distributed]]\ndeps = [\"Random\", \"Serialization\", \"Sockets\"]\nuuid = \"8ba89e20-285c-5b6f-9357-94700520ee1b\"\n\n[[Downloads]]\ndeps = [\"ArgTools\", \"LibCURL\", \"NetworkOptions\"]\nuuid = \"f43a241f-c20a-4ad4-852c-f6b1247861c6\"\n\n[[EarCut_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"92d8f9f208637e8d2d28c664051a00569c01493d\"\nuuid = \"5ae413db-bbd1-5e63-b57d-d24a61df00f5\"\nversion = \"2.1.5+1\"\n\n[[Expat_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"b3bfd02e98aedfa5cf885665493c5598c350cd2f\"\nuuid = \"2e619515-83b5-522b-bb60-26c02a35a201\"\nversion = \"2.2.10+0\"\n\n[[FFMPEG]]\ndeps = [\"FFMPEG_jll\"]\ngit-tree-sha1 = \"b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8\"\nuuid = \"c87230d0-a227-11e9-1b43-d7ebe4e7570a\"\nversion = \"0.4.1\"\n\n[[FFMPEG_jll]]\ndeps = [\"Artifacts\", \"Bzip2_jll\", \"FreeType2_jll\", \"FriBidi_jll\", \"JLLWrappers\", \"LAME_jll\", \"LibVPX_jll\", \"Libdl\", \"Ogg_jll\", \"OpenSSL_jll\", \"Opus_jll\", \"Pkg\", \"Zlib_jll\", \"libass_jll\", \"libfdk_aac_jll\", \"libvorbis_jll\", \"x264_jll\", \"x265_jll\"]\ngit-tree-sha1 = \"3cc57ad0a213808473eafef4845a74766242e05f\"\nuuid = \"b22a6f82-2f65-5046-a5b2-351ab43fb4e5\"\nversion = \"4.3.1+4\"\n\n[[FileWatching]]\nuuid = \"7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee\"\n\n[[FixedPointNumbers]]\ndeps = [\"Statistics\"]\ngit-tree-sha1 = \"335bfdceacc84c5cdf16aadc768aa5ddfc5383cc\"\nuuid = \"53c48c17-4a7d-5ca2-90c5-79b7896eea93\"\nversion = \"0.8.4\"\n\n[[Fontconfig_jll]]\ndeps = [\"Artifacts\", \"Bzip2_jll\", \"Expat_jll\", \"FreeType2_jll\", \"JLLWrappers\", \"Libdl\", \"Libuuid_jll\", \"Pkg\", \"Zlib_jll\"]\ngit-tree-sha1 = \"35895cf184ceaab11fd778b4590144034a167a2f\"\nuuid = \"a3f928ae-7b40-5064-980b-68af3947d34b\"\nversion = \"2.13.1+14\"\n\n[[Formatting]]\ndeps = [\"Printf\"]\ngit-tree-sha1 = \"8339d61043228fdd3eb658d86c926cb282ae72a8\"\nuuid = \"59287772-0a20-5a39-b81b-1366585eb4c0\"\nversion = \"0.4.2\"\n\n[[FreeType2_jll]]\ndeps = [\"Artifacts\", \"Bzip2_jll\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Zlib_jll\"]\ngit-tree-sha1 = \"cbd58c9deb1d304f5a245a0b7eb841a2560cfec6\"\nuuid = \"d7e528f0-a631-5988-bf34-fe36492bcfd7\"\nversion = \"2.10.1+5\"\n\n[[FriBidi_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91\"\nuuid = \"559328eb-81f9-559d-9380-de523a88c83c\"\nversion = \"1.0.10+0\"\n\n[[GLFW_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Libglvnd_jll\", \"Pkg\", \"Xorg_libXcursor_jll\", \"Xorg_libXi_jll\", \"Xorg_libXinerama_jll\", \"Xorg_libXrandr_jll\"]\ngit-tree-sha1 = \"dba1e8614e98949abfa60480b13653813d8f0157\"\nuuid = \"0656b61e-2033-5cc2-a64a-77c0f6c09b89\"\nversion = \"3.3.5+0\"\n\n[[GR]]\ndeps = [\"Base64\", \"DelimitedFiles\", \"GR_jll\", \"HTTP\", \"JSON\", \"Libdl\", \"LinearAlgebra\", \"Pkg\", \"Printf\", \"Random\", \"Serialization\", \"Sockets\", \"Test\", \"UUIDs\"]\ngit-tree-sha1 = \"b83e3125048a9c3158cbb7ca423790c7b1b57bea\"\nuuid = \"28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71\"\nversion = \"0.57.5\"\n\n[[GR_jll]]\ndeps = [\"Artifacts\", \"Bzip2_jll\", \"Cairo_jll\", \"FFMPEG_jll\", \"Fontconfig_jll\", \"GLFW_jll\", \"JLLWrappers\", \"JpegTurbo_jll\", \"Libdl\", \"Libtiff_jll\", \"Pixman_jll\", \"Pkg\", \"Qt5Base_jll\", \"Zlib_jll\", \"libpng_jll\"]\ngit-tree-sha1 = \"e14907859a1d3aee73a019e7b3c98e9e7b8b5b3e\"\nuuid = \"d2c73de3-f751-5644-a686-071e5b155ba9\"\nversion = \"0.57.3+0\"\n\n[[GeometryBasics]]\ndeps = [\"EarCut_jll\", \"IterTools\", \"LinearAlgebra\", \"StaticArrays\", \"StructArrays\", \"Tables\"]\ngit-tree-sha1 = \"15ff9a14b9e1218958d3530cc288cf31465d9ae2\"\nuuid = \"5c1252a2-5f33-56bf-86c9-59e7332b4326\"\nversion = \"0.3.13\"\n\n[[Gettext_jll]]\ndeps = [\"Artifacts\", \"CompilerSupportLibraries_jll\", \"JLLWrappers\", \"Libdl\", \"Libiconv_jll\", \"Pkg\", \"XML2_jll\"]\ngit-tree-sha1 = \"9b02998aba7bf074d14de89f9d37ca24a1a0b046\"\nuuid = \"78b55507-aeef-58d4-861c-77aaff3498b1\"\nversion = \"0.21.0+0\"\n\n[[Glib_jll]]\ndeps = [\"Artifacts\", \"Gettext_jll\", \"JLLWrappers\", \"Libdl\", \"Libffi_jll\", \"Libiconv_jll\", \"Libmount_jll\", \"PCRE_jll\", \"Pkg\", \"Zlib_jll\"]\ngit-tree-sha1 = \"47ce50b742921377301e15005c96e979574e130b\"\nuuid = \"7746bdde-850d-59dc-9ae8-88ece973131d\"\nversion = \"2.68.1+0\"\n\n[[Grisu]]\ngit-tree-sha1 = \"53bb909d1151e57e2484c3d1b53e19552b887fb2\"\nuuid = \"42e2da0e-8278-4e71-bc24-59509adca0fe\"\nversion = \"1.0.2\"\n\n[[HTTP]]\ndeps = [\"Base64\", \"Dates\", \"IniFile\", \"Logging\", \"MbedTLS\", \"NetworkOptions\", \"Sockets\", \"URIs\"]\ngit-tree-sha1 = \"c6a1fff2fd4b1da29d3dccaffb1e1001244d844e\"\nuuid = \"cd3eb016-35fb-5094-929b-558a96fad6f3\"\nversion = \"0.9.12\"\n\n[[IniFile]]\ndeps = [\"Test\"]\ngit-tree-sha1 = \"098e4d2c533924c921f9f9847274f2ad89e018b8\"\nuuid = \"83e8ac13-25f8-5344-8a64-a9f2b223428f\"\nversion = \"0.5.0\"\n\n[[InteractiveUtils]]\ndeps = [\"Markdown\"]\nuuid = \"b77e0a4c-d291-57a0-90e8-8db25a27a240\"\n\n[[IterTools]]\ngit-tree-sha1 = \"05110a2ab1fc5f932622ffea2a003221f4782c18\"\nuuid = \"c8e1da08-722c-5040-9ed9-7db0dc04731e\"\nversion = \"1.3.0\"\n\n[[IteratorInterfaceExtensions]]\ngit-tree-sha1 = \"a3f24677c21f5bbe9d2a714f95dcd58337fb2856\"\nuuid = \"82899510-4779-5014-852e-03e436cf321d\"\nversion = \"1.0.0\"\n\n[[JLLWrappers]]\ndeps = [\"Preferences\"]\ngit-tree-sha1 = \"642a199af8b68253517b80bd3bfd17eb4e84df6e\"\nuuid = \"692b3bcd-3c85-4b1f-b108-f13ce0eb3210\"\nversion = \"1.3.0\"\n\n[[JSON]]\ndeps = [\"Dates\", \"Mmap\", \"Parsers\", \"Unicode\"]\ngit-tree-sha1 = \"81690084b6198a2e1da36fcfda16eeca9f9f24e4\"\nuuid = \"682c06a0-de6a-54ab-a142-c8b1cf79cde6\"\nversion = \"0.21.1\"\n\n[[JpegTurbo_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"d735490ac75c5cb9f1b00d8b5509c11984dc6943\"\nuuid = \"aacddb02-875f-59d6-b918-886e6ef4fbf8\"\nversion = \"2.1.0+0\"\n\n[[JuliaInterpreter]]\ndeps = [\"CodeTracking\", \"InteractiveUtils\", \"Random\", \"UUIDs\"]\ngit-tree-sha1 = \"31c2eee64c1eee6e8e3f30d5a03d4b5b7086ab29\"\nuuid = \"aa1ae85d-cabe-5617-a682-6adf51b2e16a\"\nversion = \"0.8.18\"\n\n[[LAME_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"f6250b16881adf048549549fba48b1161acdac8c\"\nuuid = \"c1c5ebd0-6772-5130-a774-d5fcae4a789d\"\nversion = \"3.100.1+0\"\n\n[[LZO_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"e5b909bcf985c5e2605737d2ce278ed791b89be6\"\nuuid = \"dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac\"\nversion = \"2.10.1+0\"\n\n[[LaTeXStrings]]\ngit-tree-sha1 = \"c7f1c695e06c01b95a67f0cd1d34994f3e7db104\"\nuuid = \"b964fa9f-0449-5b57-a5c2-d3ea65f4040f\"\nversion = \"1.2.1\"\n\n[[Latexify]]\ndeps = [\"Formatting\", \"InteractiveUtils\", \"LaTeXStrings\", \"MacroTools\", \"Markdown\", \"Printf\", \"Requires\"]\ngit-tree-sha1 = \"a4b12a1bd2ebade87891ab7e36fdbce582301a92\"\nuuid = \"23fbe1c1-3f47-55db-b15f-69d7ec21a316\"\nversion = \"0.15.6\"\n\n[[LibCURL]]\ndeps = [\"LibCURL_jll\", \"MozillaCACerts_jll\"]\nuuid = \"b27032c2-a3e7-50c8-80cd-2d36dbcbfd21\"\n\n[[LibCURL_jll]]\ndeps = [\"Artifacts\", \"LibSSH2_jll\", \"Libdl\", \"MbedTLS_jll\", \"Zlib_jll\", \"nghttp2_jll\"]\nuuid = \"deac9b47-8bc7-5906-a0fe-35ac56dc84c0\"\n\n[[LibGit2]]\ndeps = [\"Base64\", \"NetworkOptions\", \"Printf\", \"SHA\"]\nuuid = \"76f85450-5226-5b5a-8eaa-529ad045b433\"\n\n[[LibSSH2_jll]]\ndeps = [\"Artifacts\", \"Libdl\", \"MbedTLS_jll\"]\nuuid = \"29816b5a-b9ab-546f-933c-edad1886dfa8\"\n\n[[LibVPX_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"12ee7e23fa4d18361e7c2cde8f8337d4c3101bc7\"\nuuid = \"dd192d2f-8180-539f-9fb4-cc70b1dcf69a\"\nversion = \"1.10.0+0\"\n\n[[Libdl]]\nuuid = \"8f399da3-3557-5675-b5ff-fb832c97cbdb\"\n\n[[Libffi_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"761a393aeccd6aa92ec3515e428c26bf99575b3b\"\nuuid = \"e9f186c6-92d2-5b65-8a66-fee21dc1b490\"\nversion = \"3.2.2+0\"\n\n[[Libgcrypt_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Libgpg_error_jll\", \"Pkg\"]\ngit-tree-sha1 = \"64613c82a59c120435c067c2b809fc61cf5166ae\"\nuuid = \"d4300ac3-e22c-5743-9152-c294e39db1e4\"\nversion = \"1.8.7+0\"\n\n[[Libglvnd_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libX11_jll\", \"Xorg_libXext_jll\"]\ngit-tree-sha1 = \"7739f837d6447403596a75d19ed01fd08d6f56bf\"\nuuid = \"7e76a0d4-f3c7-5321-8279-8d96eeed0f29\"\nversion = \"1.3.0+3\"\n\n[[Libgpg_error_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"c333716e46366857753e273ce6a69ee0945a6db9\"\nuuid = \"7add5ba3-2f88-524e-9cd5-f83b8a55f7b8\"\nversion = \"1.42.0+0\"\n\n[[Libiconv_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"42b62845d70a619f063a7da093d995ec8e15e778\"\nuuid = \"94ce4f54-9a6c-5748-9c1c-f9c7231a4531\"\nversion = \"1.16.1+1\"\n\n[[Libmount_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"9c30530bf0effd46e15e0fdcf2b8636e78cbbd73\"\nuuid = \"4b2f31a3-9ecc-558c-b454-b3730dcb73e9\"\nversion = \"2.35.0+0\"\n\n[[Libtiff_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"JpegTurbo_jll\", \"Libdl\", \"Pkg\", \"Zlib_jll\", \"Zstd_jll\"]\ngit-tree-sha1 = \"340e257aada13f95f98ee352d316c3bed37c8ab9\"\nuuid = \"89763e89-9b03-5906-acba-b20f662cd828\"\nversion = \"4.3.0+0\"\n\n[[Libuuid_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"7f3efec06033682db852f8b3bc3c1d2b0a0ab066\"\nuuid = \"38a345b3-de98-5d2b-a5d3-14cd9215e700\"\nversion = \"2.36.0+0\"\n\n[[LinearAlgebra]]\ndeps = [\"Libdl\"]\nuuid = \"37e2e46d-f89d-539d-b4ee-838fcccc9c8e\"\n\n[[LittleCMS_jll]]\ndeps = [\"JpegTurbo_jll\", \"Libdl\", \"Libtiff_jll\", \"Pkg\"]\ngit-tree-sha1 = \"e6ea89d915cdad8d264f7f9158c6664f879edcde\"\nuuid = \"d3a379c0-f9a3-5b72-a4c0-6bf4d2e8af0f\"\nversion = \"2.9.0+0\"\n\n[[LogarithmicNumbers]]\ndeps = [\"Random\", \"Requires\"]\ngit-tree-sha1 = \"d88b70111754e3660f80d3596a343ce42bf5ee84\"\nuuid = \"aa2f6b4e-9042-5d33-9679-40d3a6b85899\"\nversion = \"0.4.2\"\n\n[[Logging]]\nuuid = \"56ddb016-857b-54e1-b83d-db4d58db5568\"\n\n[[LoweredCodeUtils]]\ndeps = [\"JuliaInterpreter\"]\ngit-tree-sha1 = \"4bfb8b57df913f3b28a6bd3bdbebe9a50538e689\"\nuuid = \"6f1432cf-f94c-5a45-995e-cdbf5db27b0b\"\nversion = \"2.1.0\"\n\n[[MacroTools]]\ndeps = [\"Markdown\", \"Random\"]\ngit-tree-sha1 = \"6a8a2a625ab0dea913aba95c11370589e0239ff0\"\nuuid = \"1914dd2f-81c6-5fcd-8719-6d5c9610ff09\"\nversion = \"0.5.6\"\n\n[[Markdown]]\ndeps = [\"Base64\"]\nuuid = \"d6f4376e-aef5-505a-96c1-9c027394607a\"\n\n[[MatchCore]]\ngit-tree-sha1 = \"90af9fe333f8c9851f952dfa7f335185c94567c0\"\nuuid = \"5dd3f0b1-72a9-48ad-ae6e-79f673da005f\"\nversion = \"0.1.1\"\n\n[[MbedTLS]]\ndeps = [\"Dates\", \"MbedTLS_jll\", \"Random\", \"Sockets\"]\ngit-tree-sha1 = \"1c38e51c3d08ef2278062ebceade0e46cefc96fe\"\nuuid = \"739be429-bea8-5141-9913-cc70e7f3736d\"\nversion = \"1.0.3\"\n\n[[MbedTLS_jll]]\ndeps = [\"Artifacts\", \"Libdl\"]\nuuid = \"c8ffd9c3-330d-5841-b78e-0817d7145fa1\"\n\n[[Measures]]\ngit-tree-sha1 = \"e498ddeee6f9fdb4551ce855a46f54dbd900245f\"\nuuid = \"442fdcdd-2543-5da2-b0f3-8c86c306513e\"\nversion = \"0.3.1\"\n\n[[Missings]]\ndeps = [\"DataAPI\"]\ngit-tree-sha1 = \"4ea90bd5d3985ae1f9a908bd4500ae88921c5ce7\"\nuuid = \"e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28\"\nversion = \"1.0.0\"\n\n[[Mmap]]\nuuid = \"a63ad114-7e13-5084-954f-fe012c677804\"\n\n[[MozillaCACerts_jll]]\nuuid = \"14a3606d-f60d-562e-9121-12d972cd8159\"\n\n[[NaNMath]]\ngit-tree-sha1 = \"bfe47e760d60b82b66b61d2d44128b62e3a369fb\"\nuuid = \"77ba4419-2d1f-58cd-9bb1-8ffee604a2e3\"\nversion = \"0.3.5\"\n\n[[NetworkOptions]]\nuuid = \"ca575930-c2e3-43a9-ace4-1e988b2c1908\"\n\n[[NiLang]]\ndeps = [\"FixedPointNumbers\", \"LinearAlgebra\", \"LogarithmicNumbers\", \"MatchCore\", \"NiLangCore\", \"Reexport\", \"SparseArrays\", \"TupleTools\"]\ngit-tree-sha1 = \"3fe439482d8c08a15f929ae7278a6c7f737672d5\"\nuuid = \"ab4ef3a6-0b42-11ea-31f6-e34652774712\"\nversion = \"0.9.1\"\n\n[[NiLangCore]]\ndeps = [\"MatchCore\", \"TupleTools\"]\ngit-tree-sha1 = \"239f97ea947531cfe7a596746e31c8429c7169b9\"\nuuid = \"575d3204-02a4-11ea-3f62-238caa8bf11e\"\nversion = \"0.10.3\"\n\n[[Ogg_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f\"\nuuid = \"e7412a2a-1a6e-54c0-be00-318e2571c051\"\nversion = \"1.3.5+0\"\n\n[[OpenJpeg_jll]]\ndeps = [\"Libdl\", \"Libtiff_jll\", \"LittleCMS_jll\", \"Pkg\", \"libpng_jll\"]\ngit-tree-sha1 = \"e330ffff1c6a593fa44cc40c29900bee82026406\"\nuuid = \"643b3616-a352-519d-856d-80112ee9badc\"\nversion = \"2.3.1+0\"\n\n[[OpenSSL_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"15003dcb7d8db3c6c857fda14891a539a8f2705a\"\nuuid = \"458c3c95-2e84-50aa-8efc-19380b2a3a95\"\nversion = \"1.1.10+0\"\n\n[[Opus_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"51a08fb14ec28da2ec7a927c4337e4332c2a4720\"\nuuid = \"91d4177d-7536-5919-b921-800302f37372\"\nversion = \"1.3.2+0\"\n\n[[OrderedCollections]]\ngit-tree-sha1 = \"85f8e6578bf1f9ee0d11e7bb1b1456435479d47c\"\nuuid = \"bac558e1-5e72-5ebc-8fee-abe8a469f55d\"\nversion = \"1.4.1\"\n\n[[PCRE_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"b2a7af664e098055a7529ad1a900ded962bca488\"\nuuid = \"2f80f16e-611a-54ab-bc61-aa92de5b98fc\"\nversion = \"8.44.0+0\"\n\n[[Parsers]]\ndeps = [\"Dates\"]\ngit-tree-sha1 = \"c8abc88faa3f7a3950832ac5d6e690881590d6dc\"\nuuid = \"69de0a69-1ddd-5017-9359-2bf0b02dc9f0\"\nversion = \"1.1.0\"\n\n[[Pixman_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"b4f5d02549a10e20780a24fce72bea96b6329e29\"\nuuid = \"30392449-352a-5448-841d-b1acce4e97dc\"\nversion = \"0.40.1+0\"\n\n[[Pkg]]\ndeps = [\"Artifacts\", \"Dates\", \"Downloads\", \"LibGit2\", \"Libdl\", \"Logging\", \"Markdown\", \"Printf\", \"REPL\", \"Random\", \"SHA\", \"Serialization\", \"TOML\", \"Tar\", \"UUIDs\", \"p7zip_jll\"]\nuuid = \"44cfe95a-1eb2-52ea-b672-e2afdf69b78f\"\n\n[[PlotThemes]]\ndeps = [\"PlotUtils\", \"Requires\", \"Statistics\"]\ngit-tree-sha1 = \"a3a964ce9dc7898193536002a6dd892b1b5a6f1d\"\nuuid = \"ccf2f8ad-2431-5c83-bf29-c5338b663b6a\"\nversion = \"2.0.1\"\n\n[[PlotUtils]]\ndeps = [\"ColorSchemes\", \"Colors\", \"Dates\", \"Printf\", \"Random\", \"Reexport\", \"Statistics\"]\ngit-tree-sha1 = \"501c20a63a34ac1d015d5304da0e645f42d91c9f\"\nuuid = \"995b91a9-d308-5afd-9ec6-746e21dbc043\"\nversion = \"1.0.11\"\n\n[[Plots]]\ndeps = [\"Base64\", \"Contour\", \"Dates\", \"FFMPEG\", \"FixedPointNumbers\", \"GR\", \"GeometryBasics\", \"JSON\", \"Latexify\", \"LinearAlgebra\", \"Measures\", \"NaNMath\", \"PlotThemes\", \"PlotUtils\", \"Printf\", \"REPL\", \"Random\", \"RecipesBase\", \"RecipesPipeline\", \"Reexport\", \"Requires\", \"Scratch\", \"Showoff\", \"SparseArrays\", \"Statistics\", \"StatsBase\", \"UUIDs\"]\ngit-tree-sha1 = \"9f126950870ef24ce75cdd841f4b7cf34affc6d2\"\nuuid = \"91a5bcdd-55d7-5caf-9e0b-520d859cae80\"\nversion = \"1.18.0\"\n\n[[PlutoUI]]\ndeps = [\"Base64\", \"Dates\", \"InteractiveUtils\", \"JSON\", \"Logging\", \"Markdown\", \"Random\", \"Reexport\", \"Suppressor\"]\ngit-tree-sha1 = \"44e225d5837e2a2345e69a1d1e01ac2443ff9fcb\"\nuuid = \"7f904dfe-b85e-4ff6-b463-dae2292396a8\"\nversion = \"0.7.9\"\n\n[[Poppler_jll]]\ndeps = [\"Artifacts\", \"Cairo_jll\", \"Fontconfig_jll\", \"Glib_jll\", \"JLLWrappers\", \"JpegTurbo_jll\", \"Libdl\", \"Libtiff_jll\", \"OpenJpeg_jll\", \"Pkg\", \"libpng_jll\"]\ngit-tree-sha1 = \"e11443687ac151ac6ef6699eb75f964bed8e1faa\"\nuuid = \"9c32591e-4766-534b-9725-b71a8799265b\"\nversion = \"0.87.0+2\"\n\n[[Preferences]]\ndeps = [\"TOML\"]\ngit-tree-sha1 = \"00cfd92944ca9c760982747e9a1d0d5d86ab1e5a\"\nuuid = \"21216c6a-2e73-6563-6e65-726566657250\"\nversion = \"1.2.2\"\n\n[[Printf]]\ndeps = [\"Unicode\"]\nuuid = \"de0858da-6303-5e67-8744-51eddeeeb8d7\"\n\n[[Qt5Base_jll]]\ndeps = [\"Artifacts\", \"CompilerSupportLibraries_jll\", \"Fontconfig_jll\", \"Glib_jll\", \"JLLWrappers\", \"Libdl\", \"Libglvnd_jll\", \"OpenSSL_jll\", \"Pkg\", \"Xorg_libXext_jll\", \"Xorg_libxcb_jll\", \"Xorg_xcb_util_image_jll\", \"Xorg_xcb_util_keysyms_jll\", \"Xorg_xcb_util_renderutil_jll\", \"Xorg_xcb_util_wm_jll\", \"Zlib_jll\", \"xkbcommon_jll\"]\ngit-tree-sha1 = \"ad368663a5e20dbb8d6dc2fddeefe4dae0781ae8\"\nuuid = \"ea2cea3b-5b76-57ae-a6ef-0a8af62496e1\"\nversion = \"5.15.3+0\"\n\n[[REPL]]\ndeps = [\"InteractiveUtils\", \"Markdown\", \"Sockets\", \"Unicode\"]\nuuid = \"3fa0cd96-eef1-5676-8a61-b3b8758bbffb\"\n\n[[Random]]\ndeps = [\"Serialization\"]\nuuid = \"9a3f8284-a2c9-5f02-9a11-845980a1fd5c\"\n\n[[RecipesBase]]\ngit-tree-sha1 = \"b3fb709f3c97bfc6e948be68beeecb55a0b340ae\"\nuuid = \"3cdcf5f2-1ef4-517c-9805-6587b60abb01\"\nversion = \"1.1.1\"\n\n[[RecipesPipeline]]\ndeps = [\"Dates\", \"NaNMath\", \"PlotUtils\", \"RecipesBase\"]\ngit-tree-sha1 = \"2a7a2469ed5d94a98dea0e85c46fa653d76be0cd\"\nuuid = \"01d81517-befc-4cb6-b9ec-a95719d0359c\"\nversion = \"0.3.4\"\n\n[[Reexport]]\ngit-tree-sha1 = \"5f6c21241f0f655da3952fd60aa18477cf96c220\"\nuuid = \"189a3867-3050-52da-a836-e630ba90ab69\"\nversion = \"1.1.0\"\n\n[[Requires]]\ndeps = [\"UUIDs\"]\ngit-tree-sha1 = \"4036a3bd08ac7e968e27c203d45f5fff15020621\"\nuuid = \"ae029012-a4dd-5104-9daa-d747884805df\"\nversion = \"1.1.3\"\n\n[[Revise]]\ndeps = [\"CodeTracking\", \"Distributed\", \"FileWatching\", \"JuliaInterpreter\", \"LibGit2\", \"LoweredCodeUtils\", \"OrderedCollections\", \"Pkg\", \"REPL\", \"Requires\", \"UUIDs\", \"Unicode\"]\ngit-tree-sha1 = \"410bbe13d9a7816e862ed72ac119bda7fb988c08\"\nuuid = \"295af30f-e4ad-537b-8983-00126c2a3abe\"\nversion = \"3.1.17\"\n\n[[SHA]]\nuuid = \"ea8e919c-243c-51af-8825-aaa63cd721ce\"\n\n[[Scratch]]\ndeps = [\"Dates\"]\ngit-tree-sha1 = \"0b4b7f1393cff97c33891da2a0bf69c6ed241fda\"\nuuid = \"6c6a2e73-6563-6170-7368-637461726353\"\nversion = \"1.1.0\"\n\n[[Serialization]]\nuuid = \"9e88b42a-f829-5b0c-bbe9-9e923198166b\"\n\n[[SharedArrays]]\ndeps = [\"Distributed\", \"Mmap\", \"Random\", \"Serialization\"]\nuuid = \"1a1011a3-84de-559e-8e89-a11a2f7dc383\"\n\n[[Showoff]]\ndeps = [\"Dates\", \"Grisu\"]\ngit-tree-sha1 = \"91eddf657aca81df9ae6ceb20b959ae5653ad1de\"\nuuid = \"992d4aef-0814-514b-bc4d-f2e9a6c4116f\"\nversion = \"1.0.3\"\n\n[[Sockets]]\nuuid = \"6462fe0b-24de-5631-8697-dd941f90decc\"\n\n[[SortingAlgorithms]]\ndeps = [\"DataStructures\"]\ngit-tree-sha1 = \"2ec1962eba973f383239da22e75218565c390a96\"\nuuid = \"a2af1166-a08f-5f64-846c-94a0d3cef48c\"\nversion = \"1.0.0\"\n\n[[SparseArrays]]\ndeps = [\"LinearAlgebra\", \"Random\"]\nuuid = \"2f01184e-e22b-5df5-ae63-d93ebab69eaf\"\n\n[[StaticArrays]]\ndeps = [\"LinearAlgebra\", \"Random\", \"Statistics\"]\ngit-tree-sha1 = \"896d55218776ab8f23fb7b222a5a4a946d4aafc2\"\nuuid = \"90137ffa-7385-5640-81b9-e52037218182\"\nversion = \"1.2.5\"\n\n[[Statistics]]\ndeps = [\"LinearAlgebra\", \"SparseArrays\"]\nuuid = \"10745b16-79ce-11e8-11f9-7d13ad32a3b2\"\n\n[[StatsAPI]]\ngit-tree-sha1 = \"1958272568dc176a1d881acb797beb909c785510\"\nuuid = \"82ae8749-77ed-4fe6-ae5f-f523153014b0\"\nversion = \"1.0.0\"\n\n[[StatsBase]]\ndeps = [\"DataAPI\", \"DataStructures\", \"LinearAlgebra\", \"Missings\", \"Printf\", \"Random\", \"SortingAlgorithms\", \"SparseArrays\", \"Statistics\", \"StatsAPI\"]\ngit-tree-sha1 = \"2f6792d523d7448bbe2fec99eca9218f06cc746d\"\nuuid = \"2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91\"\nversion = \"0.33.8\"\n\n[[StructArrays]]\ndeps = [\"Adapt\", \"DataAPI\", \"StaticArrays\", \"Tables\"]\ngit-tree-sha1 = \"000e168f5cc9aded17b6999a560b7c11dda69095\"\nuuid = \"09ab397b-f2b6-538f-b94a-2f83cf4a842a\"\nversion = \"0.6.0\"\n\n[[Suppressor]]\ngit-tree-sha1 = \"a819d77f31f83e5792a76081eee1ea6342ab8787\"\nuuid = \"fd094767-a336-5f1f-9728-57cf17d0bbfb\"\nversion = \"0.2.0\"\n\n[[TOML]]\ndeps = [\"Dates\"]\nuuid = \"fa267f1f-6049-4f14-aa54-33bafae1ed76\"\n\n[[TableTraits]]\ndeps = [\"IteratorInterfaceExtensions\"]\ngit-tree-sha1 = \"c06b2f539df1c6efa794486abfb6ed2022561a39\"\nuuid = \"3783bdb8-4a98-5b6b-af9a-565f29a5fe9c\"\nversion = \"1.0.1\"\n\n[[Tables]]\ndeps = [\"DataAPI\", \"DataValueInterfaces\", \"IteratorInterfaceExtensions\", \"LinearAlgebra\", \"TableTraits\", \"Test\"]\ngit-tree-sha1 = \"8ed4a3ea724dac32670b062be3ef1c1de6773ae8\"\nuuid = \"bd369af6-aec1-5ad0-b16a-f7cc5008161c\"\nversion = \"1.4.4\"\n\n[[Tar]]\ndeps = [\"ArgTools\", \"SHA\"]\nuuid = \"a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e\"\n\n[[Test]]\ndeps = [\"InteractiveUtils\", \"Logging\", \"Random\", \"Serialization\"]\nuuid = \"8dfed614-e22c-5e08-85e1-65c5234f0b40\"\n\n[[TikzPictures]]\ndeps = [\"LaTeXStrings\", \"Poppler_jll\", \"Requires\"]\ngit-tree-sha1 = \"06b36e2baa9b97814ef1993207b71e2e23e9efb5\"\nuuid = \"37f6aa50-8035-52d0-81c2-5a1d08754b2d\"\nversion = \"3.3.3\"\n\n[[TupleTools]]\ngit-tree-sha1 = \"3c712976c47707ff893cf6ba4354aa14db1d8938\"\nuuid = \"9d95972d-f1c8-5527-a6e0-b4b365fa01f6\"\nversion = \"1.3.0\"\n\n[[URIs]]\ngit-tree-sha1 = \"97bbe755a53fe859669cd907f2d96aee8d2c1355\"\nuuid = \"5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4\"\nversion = \"1.3.0\"\n\n[[UUIDs]]\ndeps = [\"Random\", \"SHA\"]\nuuid = \"cf7118a7-6976-5b1a-9a39-7adc72f591a4\"\n\n[[Unicode]]\nuuid = \"4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5\"\n\n[[Viznet]]\ndeps = [\"Compose\", \"Dierckx\"]\ngit-tree-sha1 = \"7a022ae6ac8b153d47617ed8c196ce60645689f1\"\nuuid = \"52a3aca4-6234-47fd-b74a-806bdf78ede9\"\nversion = \"0.3.3\"\n\n[[Wayland_jll]]\ndeps = [\"Artifacts\", \"Expat_jll\", \"JLLWrappers\", \"Libdl\", \"Libffi_jll\", \"Pkg\", \"XML2_jll\"]\ngit-tree-sha1 = \"3e61f0b86f90dacb0bc0e73a0c5a83f6a8636e23\"\nuuid = \"a2964d1f-97da-50d4-b82a-358c7fce9d89\"\nversion = \"1.19.0+0\"\n\n[[Wayland_protocols_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Wayland_jll\"]\ngit-tree-sha1 = \"2839f1c1296940218e35df0bbb220f2a79686670\"\nuuid = \"2381bf8a-dfd0-557d-9999-79630e7b1b91\"\nversion = \"1.18.0+4\"\n\n[[XML2_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Libiconv_jll\", \"Pkg\", \"Zlib_jll\"]\ngit-tree-sha1 = \"1acf5bdf07aa0907e0a37d3718bb88d4b687b74a\"\nuuid = \"02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a\"\nversion = \"2.9.12+0\"\n\n[[XSLT_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Libgcrypt_jll\", \"Libgpg_error_jll\", \"Libiconv_jll\", \"Pkg\", \"XML2_jll\", \"Zlib_jll\"]\ngit-tree-sha1 = \"91844873c4085240b95e795f692c4cec4d805f8a\"\nuuid = \"aed1982a-8fda-507f-9586-7b0439959a61\"\nversion = \"1.1.34+0\"\n\n[[Xorg_libX11_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libxcb_jll\", \"Xorg_xtrans_jll\"]\ngit-tree-sha1 = \"5be649d550f3f4b95308bf0183b82e2582876527\"\nuuid = \"4f6342f7-b3d2-589e-9d20-edeb45f2b2bc\"\nversion = \"1.6.9+4\"\n\n[[Xorg_libXau_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"4e490d5c960c314f33885790ed410ff3a94ce67e\"\nuuid = \"0c0b7dd1-d40b-584c-a123-a41640f87eec\"\nversion = \"1.0.9+4\"\n\n[[Xorg_libXcursor_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libXfixes_jll\", \"Xorg_libXrender_jll\"]\ngit-tree-sha1 = \"12e0eb3bc634fa2080c1c37fccf56f7c22989afd\"\nuuid = \"935fb764-8cf2-53bf-bb30-45bb1f8bf724\"\nversion = \"1.2.0+4\"\n\n[[Xorg_libXdmcp_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"4fe47bd2247248125c428978740e18a681372dd4\"\nuuid = \"a3789734-cfe1-5b06-b2d0-1dd0d9d62d05\"\nversion = \"1.1.3+4\"\n\n[[Xorg_libXext_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libX11_jll\"]\ngit-tree-sha1 = \"b7c0aa8c376b31e4852b360222848637f481f8c3\"\nuuid = \"1082639a-0dae-5f34-9b06-72781eeb8cb3\"\nversion = \"1.3.4+4\"\n\n[[Xorg_libXfixes_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libX11_jll\"]\ngit-tree-sha1 = \"0e0dc7431e7a0587559f9294aeec269471c991a4\"\nuuid = \"d091e8ba-531a-589c-9de9-94069b037ed8\"\nversion = \"5.0.3+4\"\n\n[[Xorg_libXi_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libXext_jll\", \"Xorg_libXfixes_jll\"]\ngit-tree-sha1 = \"89b52bc2160aadc84d707093930ef0bffa641246\"\nuuid = \"a51aa0fd-4e3c-5386-b890-e753decda492\"\nversion = \"1.7.10+4\"\n\n[[Xorg_libXinerama_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libXext_jll\"]\ngit-tree-sha1 = \"26be8b1c342929259317d8b9f7b53bf2bb73b123\"\nuuid = \"d1454406-59df-5ea1-beac-c340f2130bc3\"\nversion = \"1.1.4+4\"\n\n[[Xorg_libXrandr_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libXext_jll\", \"Xorg_libXrender_jll\"]\ngit-tree-sha1 = \"34cea83cb726fb58f325887bf0612c6b3fb17631\"\nuuid = \"ec84b674-ba8e-5d96-8ba1-2a689ba10484\"\nversion = \"1.5.2+4\"\n\n[[Xorg_libXrender_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libX11_jll\"]\ngit-tree-sha1 = \"19560f30fd49f4d4efbe7002a1037f8c43d43b96\"\nuuid = \"ea2f1a96-1ddc-540d-b46f-429655e07cfa\"\nversion = \"0.9.10+4\"\n\n[[Xorg_libpthread_stubs_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"6783737e45d3c59a4a4c4091f5f88cdcf0908cbb\"\nuuid = \"14d82f49-176c-5ed1-bb49-ad3f5cbd8c74\"\nversion = \"0.1.0+3\"\n\n[[Xorg_libxcb_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"XSLT_jll\", \"Xorg_libXau_jll\", \"Xorg_libXdmcp_jll\", \"Xorg_libpthread_stubs_jll\"]\ngit-tree-sha1 = \"daf17f441228e7a3833846cd048892861cff16d6\"\nuuid = \"c7cfdc94-dc32-55de-ac96-5a1b8d977c5b\"\nversion = \"1.13.0+3\"\n\n[[Xorg_libxkbfile_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libX11_jll\"]\ngit-tree-sha1 = \"926af861744212db0eb001d9e40b5d16292080b2\"\nuuid = \"cc61e674-0454-545c-8b26-ed2c68acab7a\"\nversion = \"1.1.0+4\"\n\n[[Xorg_xcb_util_image_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_xcb_util_jll\"]\ngit-tree-sha1 = \"0fab0a40349ba1cba2c1da699243396ff8e94b97\"\nuuid = \"12413925-8142-5f55-bb0e-6d7ca50bb09b\"\nversion = \"0.4.0+1\"\n\n[[Xorg_xcb_util_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libxcb_jll\"]\ngit-tree-sha1 = \"e7fd7b2881fa2eaa72717420894d3938177862d1\"\nuuid = \"2def613f-5ad1-5310-b15b-b15d46f528f5\"\nversion = \"0.4.0+1\"\n\n[[Xorg_xcb_util_keysyms_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_xcb_util_jll\"]\ngit-tree-sha1 = \"d1151e2c45a544f32441a567d1690e701ec89b00\"\nuuid = \"975044d2-76e6-5fbe-bf08-97ce7c6574c7\"\nversion = \"0.4.0+1\"\n\n[[Xorg_xcb_util_renderutil_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_xcb_util_jll\"]\ngit-tree-sha1 = \"dfd7a8f38d4613b6a575253b3174dd991ca6183e\"\nuuid = \"0d47668e-0667-5a69-a72c-f761630bfb7e\"\nversion = \"0.3.9+1\"\n\n[[Xorg_xcb_util_wm_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_xcb_util_jll\"]\ngit-tree-sha1 = \"e78d10aab01a4a154142c5006ed44fd9e8e31b67\"\nuuid = \"c22f9ab0-d5fe-5066-847c-f4bb1cd4e361\"\nversion = \"0.4.1+1\"\n\n[[Xorg_xkbcomp_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libxkbfile_jll\"]\ngit-tree-sha1 = \"4bcbf660f6c2e714f87e960a171b119d06ee163b\"\nuuid = \"35661453-b289-5fab-8a00-3d9160c6a3a4\"\nversion = \"1.4.2+4\"\n\n[[Xorg_xkeyboard_config_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_xkbcomp_jll\"]\ngit-tree-sha1 = \"5c8424f8a67c3f2209646d4425f3d415fee5931d\"\nuuid = \"33bec58e-1273-512f-9401-5d533626f822\"\nversion = \"2.27.0+4\"\n\n[[Xorg_xtrans_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"79c31e7844f6ecf779705fbc12146eb190b7d845\"\nuuid = \"c5fb5394-a638-5e4d-96e5-b29de1b5cf10\"\nversion = \"1.4.0+3\"\n\n[[Zlib_jll]]\ndeps = [\"Libdl\"]\nuuid = \"83775a58-1f1d-513f-b197-d71354ab007a\"\n\n[[Zstd_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"cc4bf3fdde8b7e3e9fa0351bdeedba1cf3b7f6e6\"\nuuid = \"3161d3a3-bdf6-5164-811a-617609db77b4\"\nversion = \"1.5.0+0\"\n\n[[libass_jll]]\ndeps = [\"Artifacts\", \"Bzip2_jll\", \"FreeType2_jll\", \"FriBidi_jll\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Zlib_jll\"]\ngit-tree-sha1 = \"acc685bcf777b2202a904cdcb49ad34c2fa1880c\"\nuuid = \"0ac62f75-1d6f-5e53-bd7c-93b484bb37c0\"\nversion = \"0.14.0+4\"\n\n[[libfdk_aac_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"7a5780a0d9c6864184b3a2eeeb833a0c871f00ab\"\nuuid = \"f638f0a6-7fb0-5443-88ba-1cc74229b280\"\nversion = \"0.1.6+4\"\n\n[[libpng_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Zlib_jll\"]\ngit-tree-sha1 = \"94d180a6d2b5e55e447e2d27a29ed04fe79eb30c\"\nuuid = \"b53b4c65-9356-5827-b1ea-8c7a1a84506f\"\nversion = \"1.6.38+0\"\n\n[[libvorbis_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Ogg_jll\", \"Pkg\"]\ngit-tree-sha1 = \"c45f4e40e7aafe9d086379e5578947ec8b95a8fb\"\nuuid = \"f27f6e37-5d2b-51aa-960f-b287f2bc3b7a\"\nversion = \"1.3.7+0\"\n\n[[nghttp2_jll]]\ndeps = [\"Artifacts\", \"Libdl\"]\nuuid = \"8e850ede-7688-5339-a07c-302acd2aaf8d\"\n\n[[p7zip_jll]]\ndeps = [\"Artifacts\", \"Libdl\"]\nuuid = \"3f19e933-33d8-53b3-aaab-bd5110c3b7a0\"\n\n[[x264_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"d713c1ce4deac133e3334ee12f4adff07f81778f\"\nuuid = \"1270edf5-f2f9-52d2-97e9-ab00b5d0237a\"\nversion = \"2020.7.14+2\"\n\n[[x265_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"487da2f8f2f0c8ee0e83f39d13037d6bbf0a45ab\"\nuuid = \"dfaa095f-4041-5dcd-9319-2fabd8486b76\"\nversion = \"3.0.0+3\"\n\n[[xkbcommon_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Wayland_jll\", \"Wayland_protocols_jll\", \"Xorg_libxcb_jll\", \"Xorg_xkeyboard_config_jll\"]\ngit-tree-sha1 = \"ece2350174195bb31de1a63bea3a41ae1aa593b6\"\nuuid = \"d8fb68d0-12a3-5cfd-a85a-d49703b185fd\"\nversion = \"0.9.1+5\"\n\"\"\"\n\n# ╔═╡ Cell order:\n# ╟─e20e2d2e-4b28-4e32-8d80-ce029928a094\n# ╟─f3e235e7-76b9-4c39-bc70-038539838ff4\n# ╟─8308df59-3faa-4abf-8f05-119bbae48f64\n# ╟─a3532a83-9fd3-4d24-b1bb-b52457317e51\n# ╟─15657e4b-848e-43ad-a99f-37143d11705e\n# ╟─f9675365-36aa-430c-b747-3bc4f602e6fb\n# ╟─94c5eaa1-432c-4553-829e-f78d97f3c0ca\n# ╟─bef6978d-e654-4364-b5eb-e9608cf68464\n# ╟─046f7559-4af9-4982-b5c3-335add0911d7\n# ╟─0a039bfa-571e-4fad-b73c-1324d08777fc\n# ╟─bf7abacc-5b0a-4623-b2c5-af60183ad4b0\n# ╟─3f1e4d7a-32a7-4c7e-92dd-465bac925e63\n# ╟─95a21058-0b07-4859-af68-8ca5b48b2a77\n# ╟─f68bcfb6-97ce-48d1-b0b8-e8466d4ac879\n# ╟─2fe7c298-4c5d-464c-980b-6cd9a537ac1e\n# ╟─49dab78a-7bd9-4faa-8a30-9af8a96e0c5b\n# ╟─3d4ba750-8d62-48ac-bf96-691397689ddc\n# ╟─7aa7b0ee-beeb-4a3e-abf1-aa71e916f4cd\n# ╟─f4cb9212-181f-4338-b858-1d99c7f415e9\n# ╟─c1bbaec8-4fb9-4ab8-a30d-06a286597de0\n# ╟─6d7a07ff-be1b-4902-8a6d-7d9257c1157f\n# ╟─0577d67f-648f-407c-8abf-507d086445bd\n# ╟─eb10e436-bcce-4d81-891e-15158219fe80\n# ╟─7c5a30fd-95f9-4bb8-b34f-b10b0f2a27f2\n# ╟─abee1bee-ed01-4b05-a848-3aeb695a24ba\n# ╟─b05538cc-de01-4b1e-a602-feb780cddf4a\n# ╟─42c398ab-bb45-423f-b030-404e7582df5a\n# ╟─48081dd4-2bf4-43a1-899c-0303b4fcedd3\n# ╟─c6ef8479-639b-45c1-9b48-a5d2c233d3b8\n# ╟─6f01cdc2-6ce9-41da-b279-b047c9779405\n# ╟─876ad6cf-84c1-4e34-89de-6f9273ba3479\n# ╟─9ca8912d-5fc5-4066-adb8-ad02f75c2cbe\n# ╟─9aab5751-e9e0-46c0-8e66-4b98258fed08\n# ╟─a29af398-ff22-44cb-a5aa-0b0409312be9\n# ╟─c3db622f-e9ff-4d99-afb6-9db65c6cae7a\n# ╟─12cbf4b7-9b55-423c-bf59-5cb18e167afd\n# ╟─89a5ff44-1b04-4bd8-a40a-83382a027fb3\n# ╟─51e7b853-8640-4415-a9a4-8c0e06ad916a\n# ╟─a8fe838e-727d-4068-887d-17b1bf99f90b\n# ╟─c7cd75cb-4c64-4704-b839-c5a556f89be7\n# ╟─ec14fba6-0cb9-483f-b3ea-cc4c5e83c965\n# ╟─f1abc5c1-2c34-422a-86c4-5ad8e7df8b7e\n# ╟─8ad4e7c0-c496-4d29-ac09-e6525b1b4c0f\n# ╟─757e2d78-c5ee-4b40-bfd6-1b39af338d9d\n# ╟─cbea35c7-c3c8-48e7-bb47-d5e193aee2c4\n# ╟─81013954-1c48-4c05-82c9-49b4bfafda95\n# ╟─1924eff7-1423-4e90-8005-43113d9deb3d\n# ╟─c7edbc15-cd59-45fd-a0dc-c48aadb1c096\n# ╟─bf2c9da7-8c45-409f-82a3-979cd63ea993\n# ╟─1c32a491-ac85-4132-82fd-9b846a8485df\n# ╟─6cbf202f-34e4-42b6-a7a7-5d766bfdfc37\n# ╟─d40b318f-bff2-4d0b-b2a6-d00933ac7567\n# ╟─54f53a7b-74e8-433b-94fd-9fa7192dfca5\n# ╟─b0dcad96-e439-4e09-9e92-8cad7ede79af\n# ╟─0a15a2cf-2e7a-4bd7-ac78-0803fc3d5c73\n# ╟─ffbc5616-d2d9-4ce4-996f-d1a743bb89b3\n# ╟─2602d857-4a21-478a-97a2-58a177666f52\n# ╟─cf27e340-578a-440d-8d4a-e5a2277d5205\n# ╟─aa53fd68-5acd-488d-a096-5ce39759f481\n# ╟─cb9a9ef0-c0dc-487c-8008-0f73f9910ef8\n# ╟─f7e0478d-1839-4684-9265-ee990fe9da45\n# ╟─751e32d6-2582-4b1d-9558-124b1ef54f81\n# ╟─5f18987d-a69e-4db9-96d3-426ed298d9b8\n# ╟─66495c77-3bbc-4731-b9e1-db11bbc24283\n# ╟─84b867a3-804e-4e7e-a56c-0ffc1f4e6683\n# ╟─4642d311-ef0b-4c29-901d-b5398a3ca7b6\n# ╟─96c3d50b-8a79-4de0-b7e0-c63c3b769b74\n# ╟─066aa825-81e4-404d-bf5a-6a9431969702\n# ╟─6e99ed64-a896-450e-8bab-845e0fe971ae\n# ╟─7f803113-653a-4dfd-93f0-83babb253b32\n# ╟─7eb29d49-05f5-47e9-b4f5-4f31c5cd37ce\n# ╟─aefdec07-dcef-4e00-bcb0-4747250cdd9b\n# ╟─cbe4abaf-46f9-4726-97ae-cf3c378abaaf\n# ╟─4f0c81f5-ce5f-4f73-a528-9feff4a7fc14\n# ╠═e81de385-0070-49a9-a889-8fcf9d9e2951\n# ╟─8249b820-8fb1-45d4-a95c-9c81e62e8216\n# ╟─6503b377-b2d5-48be-90a4-97947afb4e5f\n# ╟─53a571dd-cac7-432a-869c-b93a8fe05e17\n# ╟─2e5c7f59-dd35-4846-815a-b92eabeee089\n# ╟─7b326477-43b6-4a6e-8862-12e8b70e1ad9\n# ╟─47cd7560-a29e-4b55-bef5-28daa1cdb834\n# ╟─59ab4431-ea4d-4707-9a42-d50eafa40b56\n# ╟─3630b412-beeb-455a-a4b8-1e1d50860266\n# ╟─ba6347d6-4ad0-403b-824f-dcf290a7c002\n# ╟─a2f4975d-eeee-4a2d-97dd-dd0cfd29d665\n# ╟─32d411e9-b01d-4ad2-b4aa-2f091034e6c0\n# ╟─e5b83421-dd94-43ad-84eb-ca558bff6a2d\n# ╟─118642ad-1aad-4f91-8da0-55a417b67750\n# ╟─45171ecc-9d34-4ab6-a00b-ec9c9afc33f8\n# ╟─6cd60f7d-d7ce-4189-a2dd-e47ce6825741\n# ╟─ff3fc929-f448-41be-8f60-65de33dff36a\n# ╟─5ec2649e-9988-4f38-896a-64ef6ed91d82\n# ╟─aa8475c3-c68b-4200-8634-ace33f525417\n# ╟─aa22f905-b69d-405e-b09a-a765d60f6079\n# ╟─2dcbaac0-2fad-4292-ad31-8188a60876da\n# ╟─6bad4f5f-806f-480a-ae16-2582761ce5e3\n# ╟─e200dde3-9033-45b5-bfe0-2d03753b2c11\n# ╟─d7a4b342-ef0a-44f9-b88d-bbb04483e8b3\n# ╟─908f19a2-6d32-4776-95eb-b249a8155ddc\n# ╟─05fc1fae-b378-4c39-b060-74ca635745ec\n# ╟─d0573bf9-0fd6-4512-bc13-17aa23a3265b\n# ╟─d8998d5f-65b2-4850-aef9-f19ecc192eca\n# ╟─6b4c180c-9a12-4e3d-9336-1431e7c5875a\n# ╟─0eb66cc9-93c0-4f07-b31b-a9bf9000260e\n# ╟─85bf9f92-30f1-4e05-8d07-d8e481f20ccb\n# ╟─fbba0a91-9f48-4d91-90e7-f6a7df3227f9\n# ╟─7fc81b9f-73ed-4780-9204-ddf39467e58f\n# ╟─d08ae188-937f-474b-92d7-cb8eeda063fe\n# ╟─7ee8cfc9-26b2-4fe4-8263-1f4d2f7c276d\n# ╠═db9a97b1-f76d-4f51-96c6-0159469c5adb\n# ╟─1669f5e3-efe1-4b79-a2b6-11ed7476a2a1\n# ╠═5000f4c3-5416-4e53-88ae-e30d8d09827e\n# ╠═2413c061-89de-403f-8011-e458f5a9859d\n# ╟─d4704779-9261-478b-bbf6-551220783e12\n# ╠═4be065b5-0841-4d54-b9ab-d6770d4d9d94\n# ╠═e850e53d-cf61-4fc7-9cb3-e318ae957f0b\n# ╟─a267ea5f-8bd5-4ee0-9c8d-47e2d3b81692\n# ╟─c02520a3-3375-4d83-a0dc-1aeac2aa7d5f\n# ╟─2e18fc92-4185-493b-9ce8-cca63dad7d2d\n# ╟─acc7b185-e4df-4aca-aa42-554215065384\n# ╟─00c9e973-7e06-4483-bf4c-be7374707118\n# ╟─83ff3fc3-bcd8-4235-a42f-1d75c7d6aa5b\n# ╟─b308e270-6b40-4946-ac92-c705823f2c1e\n# ╟─e483b3d4-d01c-4a98-8e68-e8120a7d95a7\n# ╟─74017e78-0f02-41bb-a160-5f2d26c18268\n# ╟─0b3735c2-695c-4225-843e-16ca17aac0eb\n# ╟─d7942b37-f821-494a-8f18-5f267aa3457a\n# ╟─00000000-0000-0000-0000-000000000001\n# ╟─00000000-0000-0000-0000-000000000002\n" }, { "path": "notebooks/margolus.jl", "content": "### A Pluto.jl notebook ###\n# v0.12.21\n\nusing Markdown\nusing InteractiveUtils\n\n# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).\nmacro bind(def, element)\n quote\n local el = $(esc(element))\n global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing\n el\n end\nend\n\n# ╔═╡ 845b7d0a-7ca2-11eb-2683-cd370811bf68\nusing NiLang\n\n# ╔═╡ 88912bea-7ca2-11eb-1cde-db111d594c20\nusing Viznet, PlutoUI, Compose\n\n# ╔═╡ f42405a8-7ca2-11eb-133b-df4f1ce9bb19\nhtml\"\"\"\"\"\"\n\n# ╔═╡ 66e972c2-7ca2-11eb-0a7a-a9305dd20511\nmd\"# BBMCA - NiLang implementation\"\n\n# ╔═╡ c8722cea-7ca5-11eb-3e7d-0f39322ea40a\nmd\"Check [Physics-like models of computation](https://www.sciencedirect.com/science/article/abs/pii/0167278984902525) (Norman Margolus, 1984) for theories about Billiard ball celluar automata (BBMCA).\"\n\n# ╔═╡ 845d1458-7ca2-11eb-0e69-11a10b9894a8\n@i function load_and_clear!(x, config, i, j)\n\tm, n ← size(config)\n\tx ⊻= config[i,j] # to make it faster, should put `@inbounds` before it\n\tconfig[i,j] ⊻= x\n\tx ⊻= config[i,mod1(j+1, n)] << 1\n\tconfig[i,mod1(j+1, n)] ⊻= x >> 1\n\tx ⊻= config[mod1(i+1, m),j] << 2\n\tconfig[mod1(i+1, m),j] ⊻= x >> 2\n\tx ⊻= config[mod1(i+1, m),mod1(j+1, n)] << 3\n\tconfig[mod1(i+1, m),mod1(j+1, n)] ⊻= x >> 3\nend\n\n# ╔═╡ 84680136-7ca2-11eb-1df5-ffa0b4b9d126\n@i function margolus_rule(y, x)\n\t# remove reversibility check to make it run faster\n @invcheckoff if x==6\n y ⊻= 9\n elseif x==9\n y ⊻= 6\n elseif x==4\n y ⊻= 2\n elseif x==2\n y ⊻= 4\n elseif x==1\n y ⊻= 8\n elseif x==8\n y ⊻= 1\n else\n y ⊻= x\n end\nend\n\n# ╔═╡ 845c1292-7ca2-11eb-3e56-996aa5229b4e\n@i function update_bbmca!(config, iseven)\n\t# computing offsets, and borrow some ancillas from system\n\t@routine begin\n\t\toffset ← 1\n\t\tm, n ← size(config)\n\t\tif !iseven\n\t\t\toffset += 1\n\t\tend\n\tend\n\tfor j=offset:2:n\n\t\tfor i=offset:2:m\n\t\t\tx ← 0\n\t\t\ty ← 0\n\t\t\t# load block to `x` and clean up original data\n\t\t\tload_and_clear!(x, config, i, j)\n\t\t\t# compute new config to `y`\n\t\t\tmargolus_rule(y, x)\n\t\t\t# clean up `x` with the following observation:\n\t\t\t# applying margolus rule twice restores the configuration\n\t\t\tmargolus_rule(x, y)\n\t\t\t# store `y` to block\n\t\t\t(~load_and_clear!)(y, config, i, j)\n\t\t\t# ancillas `x` and `y` are returned to the pool automatically\n\t\tend\n\tend\n\t# uncompute `offset`\n\t~@routine\nend\n\n# ╔═╡ 34006d60-7ca3-11eb-3c51-c9758394b838\nmd\"# Visualization\"\n\n# ╔═╡ 846e21da-7ca2-11eb-05a3-51e22ed04147\nfunction showconfig(ba::AbstractMatrix)\n m, n = size(ba, 1), size(ba, 2)\n lt = Viznet.SquareLattice(n, m)\n brush1 = nodestyle(:square, fill(\"black\"), stroke(\"#888888\"), linewidth(unit(lt)*mm); r=unit(lt)/2.2)\n brush0 = nodestyle(:square, fill(\"white\"), stroke(\"#888888\"), linewidth(unit(lt)*mm); r=unit(lt)/2.2)\n canvas() do\n for i=1:m, j=1:n\n (ba[i, j] == 1 ? brush1 : brush0) >> lt[j,i]\n end\n end\nend\n\n# ╔═╡ 846e8170-7ca2-11eb-351e-eb45c629f6a6\n@bind btn Clock(0.1)\n\n# ╔═╡ 84730f04-7ca2-11eb-3b13-3fd82a9e2109\n# initial configuration\nconfig = let\n\tx=zeros(Int, 10, 10)\n\tx[1,1] = 1\n\tx\nend;\n\n# ╔═╡ 84763e9c-7ca2-11eb-356c-c391966cdc98\n# parity - Note: BBMCA is a two state CA\nbbmca_parity = Ref(true)\n\n# ╔═╡ 847711e6-7ca2-11eb-326a-15f6bfc05347\nlet\n\tbtn\n\t# update\n\tupdate_bbmca!(config, bbmca_parity[])\n\t# change parity\n\tbbmca_parity[] = !(bbmca_parity[])\n\t# visualize\n\tCompose.set_default_graphic_size(10cm, 10cm)\n\tshowconfig(config)\nend\n\n# ╔═╡ Cell order:\n# ╟─f42405a8-7ca2-11eb-133b-df4f1ce9bb19\n# ╟─66e972c2-7ca2-11eb-0a7a-a9305dd20511\n# ╟─c8722cea-7ca5-11eb-3e7d-0f39322ea40a\n# ╠═845b7d0a-7ca2-11eb-2683-cd370811bf68\n# ╠═845c1292-7ca2-11eb-3e56-996aa5229b4e\n# ╠═845d1458-7ca2-11eb-0e69-11a10b9894a8\n# ╠═84680136-7ca2-11eb-1df5-ffa0b4b9d126\n# ╟─34006d60-7ca3-11eb-3c51-c9758394b838\n# ╠═88912bea-7ca2-11eb-1cde-db111d594c20\n# ╠═846e21da-7ca2-11eb-05a3-51e22ed04147\n# ╟─846e8170-7ca2-11eb-351e-eb45c629f6a6\n# ╠═84730f04-7ca2-11eb-3b13-3fd82a9e2109\n# ╠═84763e9c-7ca2-11eb-356c-c391966cdc98\n# ╠═847711e6-7ca2-11eb-326a-15f6bfc05347\n" }, { "path": "notebooks/reversibleprog.jl", "content": "### A Pluto.jl notebook ###\n# v0.15.0\n\nusing Markdown\nusing InteractiveUtils\n\n# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).\nmacro bind(def, element)\n quote\n local el = $(esc(element))\n global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing\n el\n end\nend\n\n# ╔═╡ f3e235e7-76b9-4c39-bc70-038539838ff4\nbegin\n\tusing Revise, Viznet, Compose, PlutoUI, Random, TikzPictures\n\tfunction leftright(a, b; width=600, leftcellwidth=0.5)\n\t\tHTML(\"\"\"\n\n\t\t\t\n\n\n\t\n\t\n
$(html(a))$(html(b))
\n\"\"\")\n\tend\n\t\n\t# up down layout\n\tfunction updown(a, b; width=nothing)\n\t\tHTML(\"\"\"\n\n\t\n\n\n\t\n
$(html(a))
$(html(b))
\n\"\"\")\n\tend\n\t\n\tfunction highlight(str)\n\t\tHTML(\"\"\"$(str)\"\"\")\n\tend\nend;\n\n# ╔═╡ 3b0fd2b5-5c6d-4d56-9e48-cda1493b4c72\nusing NiLang\n\n# ╔═╡ a7810352-7967-460d-abd7-361a324c20a9\nusing ForwardDiff: Dual\n\n# ╔═╡ a56445b5-e530-4035-9ac6-a2d196a6276a\nusing NiLang.AD: GVar\n\n# ╔═╡ c6bd40af-50ed-4cee-8043-60b2bac05058\nusing NiLang: bennett\n\n# ╔═╡ 873ef2c2-653e-425e-9732-b1ed19f7a0b7\nusing TreeverseAlgorithm\n\n# ╔═╡ 141e21c0-1bdf-4e6b-b76d-129567a1180f\nusing LinearAlgebra\n\n# ╔═╡ ac35b26c-0585-4d2a-8fbf-bda9b141d6af\nusing Test\n\n# ╔═╡ d4726239-81af-4792-8472-c680508449c6\nusing BenchmarkTools\n\n# ╔═╡ e20e2d2e-4b28-4e32-8d80-ce029928a094\nhtml\"\"\"\n\n\n\n\"\"\"\n\n# ╔═╡ 8308df59-3faa-4abf-8f05-119bbae48f64\nlet\n\tgithub = html\"\"\"\n GiggleLiu\n\"\"\"\n\tmd\"# Pebble games - Time and space to differentiate a program\n\n-- Jinguo Liu ($github)\n\"\nend\n\n# ╔═╡ a3532a83-9fd3-4d24-b1bb-b52457317e51\nhtml\"\"\"A postdoc in Mikhail Lukin's group, department of physics

\n Harvard university

\n Quera Computing\n\"\"\"\n\n# ╔═╡ 15657e4b-848e-43ad-a99f-37143d11705e\nmd\"# Table of contents\n1. An introduction to reversible computing\n2. A reversible eDSL NiLang\n3. Automatic differentiating a reversible computing language\n\"\n\n# ╔═╡ 34a6b7f4-7d72-485d-86dc-f4b1ba6174eb\nmd\"## Let's start from physics\"\n\n# ╔═╡ 67d1b500-964e-4668-a7d0-ed93886446ca\nlet\n\timg1 = html\"\"\"\n\n\"\"\"\n\timg2 = html\"\"\"\n\"\"\"\n\tleftright(updown(img2, html\"
electromagnetic force
\"), updown(img1, html\"
friction
\"))\nend\n\n# ╔═╡ 673992ed-6963-400a-a69b-d65d26c4f443\nmd\"Both can be explained by reversible quantum dynamics.\"\n\n# ╔═╡ 6078758e-b392-4bdb-a1e7-44b135ce900e\nmd\"\"\"\n## How come our programming style is irreversible?\n\"\"\"\n\n# ╔═╡ 41e06e2a-b482-4e0f-8569-fee2ffd8aaaf\nfunction find_maximum(x::AbstractVector)\n\t@assert !isempty(x) # error handling\n\tm = x[1] # assignment (a)\n\tfor i=2:length(x)\n\t\tm = max(m, x[i]) # assignment (b)\n\tend\n\treturn m # function return\nend\n\n# ╔═╡ dcf53d46-e259-4101-8530-9621094ee586\nTikzPicture(L\"\"\"\n\\draw [black, thick,->] (0, 0) -- (1, 0);\n\\draw [black, thick,->,dashed] (1, 0) .. controls (1.5, 0.5) .. (2, 0);\n\\node at (1.5, 0.5) {goto $\\ldots$};\n\\draw [black, thick,->] (2, 0) -- (3, 0);\n \n\\def\\x{4};\n\\draw [black, thick,<-] (\\x, 0) -- (\\x+1, 0);\n\\draw [black, thick,<-,dashed] (\\x+1, 0) .. controls (\\x+1.5, 0.5) .. (\\x+2, 0);\n\\draw [black, thick,<-,dashed] (\\x+1, 0) -- (\\x+2, 0);\n\\node at (\\x+1.5, 0.5) {comefrom?};\n\\draw [black, thick,<-] (\\x+2, 0) -- (\\x+3, 0);\n \n\\node at (1.5, -0.2) {call};\n\\node at (\\x+1.5, -0.2) {uncall};\n\"\"\", options=\"scale=2.0\", preamble=\"\")\n\n# ╔═╡ 6a88d26c-c895-4852-ab4f-37297b848731\nmd\"\"\"\n* **Information**: the uncertainty, quantified of *information entropy*.\n* **Information erasure**: make the system more certain, e.g.\n```julia\nm = max(m, x[i])\n```\nquantified by the decrease of information entropy.\n\"\"\"\n\n# ╔═╡ f9675365-36aa-430c-b747-3bc4f602e6fb\nmd\"## Information erasure requires dissipating heat to the environment!\"\n\n# ╔═╡ 46eb4ba9-dce6-4711-9c4d-3f16de6240de\nleftright(html\"\"\"\n\"\"\", md\"Feynman, Richard P.\n\n**Feynman Lectures on Computation**\n\t\n(2018)\")\n\n# ╔═╡ 046f7559-4af9-4982-b5c3-335add0911d7\nhtml\"\"\"\n
\n\"\"\"\n\n# ╔═╡ 0a039bfa-571e-4fad-b73c-1324d08777fc\nhtml\"\"\"\n
\n\"\"\"\n\n# ╔═╡ 3f1e4d7a-32a7-4c7e-92dd-465bac925e63\nmd\"\"\"\nCompress the boxes from size $V$ to size $V/2$, the process is isothermal.\n\"\"\"\n\n# ╔═╡ f68bcfb6-97ce-48d1-b0b8-e8466d4ac879\nmd\"\"\"\nCase 1: we know nothing about the system. The gas does work\n```math\n\\begin{align}\n&pV = N k T\\\\\n&W_{\\rm gas} = \\int_{V}^{V/2} p dV = -NkT\\log 2\n\\end{align}\n```\n\"\"\"\n\n# ╔═╡ 3d4ba750-8d62-48ac-bf96-691397689ddc\nmd\"\"\"\nCase 2: we know one bit knowledge about each box:\n* 1: the atom is in the right half\n* 0: the atom is in the left half\n\n```math\nW_{\\rm gas} = 0\n```\n\"\"\"\n\n# ╔═╡ f4cb9212-181f-4338-b858-1d99c7f415e9\nmd\"Erasing each bit information comes along with $kT \\log 2$ heat dissipation!!\"\n\n# ╔═╡ 31bde262-6352-4be0-b5cc-1781e3df2268\nmd\"Later people proved it from the microscopic picture. [Reeb, 2014]\"\n\n# ╔═╡ 83ff3fc3-bcd8-4235-a42f-1d75c7d6aa5b\nmd\"## Computing architectures\"\n\n# ╔═╡ b308e270-6b40-4946-ac92-c705823f2c1e\nlet\n\ttxt1 = md\"Traditional irreversible computer\n\t\n$E \\sim 10^8 kT$\"\n\timg1 = html\"\"\"\"\"\"\n\ttxt2 = md\"DNA copying is a living copy machine\n\n$E \\sim 100k T$\"\n\timg2 = html\"\"\"\n\n\"\"\"\n\ttxt3 = md\"\"\"\nAdiabatic CMOS [Athas, 1994]\n\n$E \\sim 10^6 kT$\n\"\"\"\n\timg3 = html\"\"\"\n\n\"\"\"\n\ttxt4 = md\"\"\"Adiabatic superconducting devices [Takeuchi, 2014]\n\n$E \\sim kT$\n\"\"\"\n\timg4 = html\"\"\"\n\n\"\"\"\n\tupdown(leftright(updown(img1, txt1), updown(img2, txt2)), leftright(updown(img3, txt3), updown(img4, txt4)))\nend\n\n# ╔═╡ e483b3d4-d01c-4a98-8e68-e8120a7d95a7\nmd\"# Summary\n* An isolated system is reversible,\n* Our programs are not reversible,\n * Need a heatbath\n * Dissipate heat to heat bath: ``kT \\log 2``/bit (Landauer's principle),\n\n\n![](https://user-images.githubusercontent.com/6257240/123520518-22ebc700-d67f-11eb-8af1-a452605cc1d8.png)\n\n*Youtube*: Michael P. Frank: Fundamental Physics of Reversible Computing — An Introduction, Part 1\n\n*Loophole*: need to take algorithmic overheads into consideration!\n\"\n\n# ╔═╡ 20c34526-c7c4-11eb-21fa-d706fd684a4c\nmd\"# A short introduction to the reversible programming\"\n\n# ╔═╡ 3f96abdf-fb5f-4d79-a288-e20b8c1f55d1\nhtml\"\"\"\"\"\"\n\n# ╔═╡ 5d51231a-8bf0-4414-9a39-cea264df84f2\nmd\"Initially written by Jinguo Liu and Taine Zhao (The author of MLStyle)\"\n\n# ╔═╡ e10e0be8-b26e-4719-92dc-8ca46af0b4b5\nmd\"## Feature 1. one function for two\"\n\n# ╔═╡ b1a9946b-82b4-4954-8bb9-5df035eaefe4\nmd\"Example: an identity mapping ``(x, y) \\mapsto (x,y)`` \"\n\n# ╔═╡ 00342a51-36d8-4fdd-aab7-ee02e2122c49\n@i function f1(x1, x2)\n\t# will return inputs automatically for you\nend\n\n# ╔═╡ 40c1c48d-0e5a-4a47-b7e4-8f7666281249\nf1(2, 3)\n\n# ╔═╡ 59df1f80-9be1-4b26-b263-ca0c7a0b9ab7\n(~f1)(2, 3)\n\n# ╔═╡ 8320d326-c1ab-4807-befb-13dda3480bf5\nmd\"## Feature 2. every instruction is reversible, every object is ''mutable''\"\n\n# ╔═╡ 93238608-3b86-49f1-ad60-9360e12cff1c\nmd\"General design patterns\n* `y += f(x)`\n* `y -= f(x)`\n* `y ⊻= f(x)`\n\nThere are also instructions like `SWAP`, `ROT`.\"\n\n# ╔═╡ f657c3fb-e140-4c76-8065-54f1cb6d05eb\nmd\"Example: mutating fields of complex numbers\"\n\n# ╔═╡ 629a2549-745c-48a2-9bbc-a8f5fb046d11\n@i function f2(x1::Complex, x2::Complex)\n\tx2 += exp(x1) # accumulative form\n\tSWAP(x1.im, x2.im) # other primitive functions\n\tf1(x1, x2)\t\t\t# other reversible functions\nend\n\n# ╔═╡ 640e0029-7931-4afd-bdf9-fed317efbd8e\nmd\" $(@bind expand_f2 CheckBox()) macroexpand\"\n\n# ╔═╡ 6930345b-6e93-4b35-8d4f-91ad49141fa1\nif expand_f2\n\tmacroexpand(NiLang, :(@i function f2(x1::Complex, x2::Complex)\n\t\tx2 += exp(x1)\n\t\tSWAP(x1.im, x2.im)\n\tend)) |> NiLangCore.rmlines\nend\n\n# ╔═╡ 090522bf-0ff2-4022-8460-aec6e37e936a\nf2(1.0+2im, 2.0+4.9im)\n\n# ╔═╡ 88c30609-2f42-405e-a14c-dfab44aef23b\n(~f2)(f2(1.0+2im, 2.0+4.9im)...)\n\n# ╔═╡ 2dc665a9-b131-4fef-acde-db346eb0f48b\nmd\"## Feature 3. One can reverse the control flows too\"\n\n# ╔═╡ 4e479f48-42cd-476d-8604-08ecbb503a90\nmd\"\"\"\n#### Reversible `if` statement\n\"\"\"\n\n# ╔═╡ dc41e99a-f598-4bf6-9f76-ecdb04f5f40c\nleftright(md\"\n```julia\nif (precondition[, postcondition])\n\t...\nend\n```\n\", md\"\n```julia\nif (postcondition[, precondition])\n\t~(...)\nend\n```\n\")\n\n# ╔═╡ 97e0bae1-69ac-4cbf-b9d9-6b38180edd78\nTikzPicture(L\"\"\"\n\\node [test] (pre) {precondition};\n\\node [proc, it] (st1) [right=of pre] {statements 1};\n\\node [proc, it] (st2) {statements 2};\t\n\\node [test] (post1) [right=of st1] {postcondition};\n\\node [test] (post2) [right=of st2] {postcondition};\n\\node [proc,red] (err1) [above=of post1] {invertibility error};\n\\node [proc,red] (err2) [below=of post2] {invertibility error};\n\\draw [->,black] (pre.east) -- (st1) node[midway,above] {T};\n\\draw [->,black] (pre.south) |- (st2) node[midway,below] {F};\n\\draw [->,black] (-2.5, 0.0) -- (pre.west);\n\\draw [->,black] (st1) -- (post1);\n\\draw [->,black] (st2) -- (post2);\n\\draw [->,red] (post1) -- (err1) node[midway,right] {F};\n\\draw [->,red] (post2) -- (err2) node[midway,right] {T};\n\\draw [->,black] (post1.east) -- (12, 0) node[midway,above] {T};\n\\draw [black] (post2.east) -| (11, 0) node[midway,right] {F};\n\"\"\", options=raw\" font=\\sffamily\\small,\n >={Triangle[]},\n */.tip={Circle[]},\n start chain=going below,\n node distance=18mm and 40mm,\n every join/.style={norm},\n base/.style={draw, on chain, on grid, align=center, minimum height=4ex, inner color=black!50!gray!10, outer color=black!50!gray!15},\n proc/.style={base, rectangle, text width=8em},\n test/.style={base, diamond, text centered, aspect=2.6,inner sep=-0ex},\n norm/.style={->, draw, black},\n it/.style={font={\\sffamily\\small\\itshape}}\", preamble=raw\"\\usetikzlibrary{shapes.geometric,arrows.meta,chains,positioning,quotes}\")\n\n# ╔═╡ 355ba831-6be0-456a-8f94-36acd2365f17\nmd\"Example: obtaining the absolute value ``x \\mapsto |x|``\"\n\n# ╔═╡ 003c3e68-600e-4688-832b-5e061572b128\n@i function abs_incorrect(x)\n\tif x < 0\n\t\tNEG(x)\n\tend\nend\n\n# ╔═╡ 1fb196f9-0f0a-42dc-b094-077cdf18d13d\nabs_incorrect(-3)\n\n# ╔═╡ aa9a679e-63bc-4951-b6f4-65316e212bc8\n@i function abs_correct(x, sgn)\n\tif (x < 0, sgn)\n\t\tNEG(x)\n\t\tsgn ⊻= true\n\tend\nend\n\n# ╔═╡ e6fd3c2d-cadd-40d7-a575-b1e68c45ee13\nabs_correct(-3, false)\n\n# ╔═╡ 02b7e1b4-4622-4e68-966e-ff79817557d1\nmd\"#### Reversible `while` statement\"\n\n# ╔═╡ 364fd613-0ebd-4b45-a3fd-f9baa8c487e3\nleftright(md\"\n```julia\n@from condition1 while condition2\n\t...\nend\n```\n\", md\"\n```julia\n@from !(condition2) while !(condition1)\n\t~(...)\nend\n```\n\")\n\n# ╔═╡ 75d8283a-b331-4648-84a8-489e168e33f9\nTikzPicture(L\"\"\"\n\\node [test] (c1) {condition 1};\n\\node [test] (c2) [right=of c1] {condition 2};\n\\node [test] (c3) [right=of c2] {condition 1};\n\\node [proc, it] (st1) [above=of c2] {statements};\n\\node [proc,red] (err1) [below=of c1] {invertibility error};\n\\node [proc,red] (err2) [right=of c3] {invertibility error};\n\\draw [->,black] (c2) -- (st1) node[midway,right] {T};\n\\draw [->,black] (st1) -| (c3);\n\\draw [->,black] (-2.5, 0.0) -- (c1.west);\n\\draw [->,black] (c1) -- (c2) node[midway,above] {T};\n\\draw [->,black] (c3) -- (c2) node[midway,above] {F};\n\\draw [->,red] (c1) -- (err1) node[midway,right] {F};\n\\draw [->,red] (c3) -- (err2) node[midway,above] {T};\n\\draw [->,black] (c2.south) |- (11, -2) node[midway,below] {F};\n\"\"\", options=raw\" font=\\sffamily\\small,\n >={Triangle[]},\n */.tip={Circle[]},\n start chain=going below,\n node distance=18mm and 40mm,\n every join/.style={norm},\n base/.style={draw, on chain, on grid, align=center, minimum height=4ex, inner color=black!50!gray!10, outer color=black!50!gray!15},\n proc/.style={base, rectangle, text width=8em},\n test/.style={base, diamond, text centered, aspect=2.6,inner sep=-0ex},\n norm/.style={->, draw, black},\n it/.style={font={\\sffamily\\small\\itshape}}\", preamble=raw\"\\usetikzlibrary{shapes.geometric,arrows.meta,chains,positioning,quotes}\")\n\n# ╔═╡ 2207c2fb-4a52-4766-8dd3-03872744aa74\nmd\"example: computing Fibonacci numbers\"\n\n# ╔═╡ 288331c3-2dfb-4941-985f-554be409c0ab\n@i function fib(y, n)\n @invcheckoff if (n >= 1, ~)\n counter ← 0\n counter += n\n @from counter==n while counter > 1\n\t\t\tcounter -= 1\n fib(y, counter)\n counter -= 1\n end\n counter -= n % 2\n counter → 0\n end\n y += 1\nend\n\n# ╔═╡ 12a30359-d6ff-4113-bab5-b198e908cf1a\nfib(0, 10)\n\n# ╔═╡ 23ea88b4-6b89-462a-92da-0e8bdf5c73b5\n(~fib)(89, 10)\n\n# ╔═╡ 4bfdabd6-b7ff-40fb-b567-52910acb5a07\nmd\"\"\"\n#### Reversible `for` statement\n\n$(\nleftright(md\"\n```julia\nfor iter = start:step:stop\n\t...\nend\n```\n\", md\"\n```julia\nfor iter = stop:-step:start\n\t~(...)\nend\n```\n\")\n)\n\"\"\"\n\n# ╔═╡ 2603147b-e7a2-4cae-b88f-2cfebe16bacb\nmd\"## Feature 4. storage access should also be reversible\"\n\n# ╔═╡ 12d49e2e-cc6e-48d6-b11a-e7c311453bfc\nmd\"\"\"\n$(\nleftright(updown(md\"\n```julia\nvar ← zero(T)\n```\", md\"borrow some memory from system and allocate it to variable var of type T.\"), updown(md\"\n```julia\nvar → zero(T)\n```\n\", md\"return the zero cleared variable to system.\"))\n)\n\"\"\"\n\n# ╔═╡ 10312dc7-e861-4c89-b2fd-672cfe8850bf\nmd\"\"\"\n$(\nleftright(updown(md\"\n```julia\ndict[key] ← variable\n```\", md\"create a new entry, asserting `key` does not exist\"), updown(md\"\n```julia\ndict[key] → variable\n```\n\", md\"asserting the value of an existing key, and delete it.\"))\n)\n\"\"\"\n\n# ╔═╡ 03f58f2a-24b6-4235-9102-71a19b9679ac\nmd\"Example: implementing `y += log(x)` for complex number.\n\n```math\n\\log(z) = \\log(|z|) + i {\\rm Arg}(z)\n```\"\n\n# ╔═╡ 22a5853e-4f9a-4da0-bc03-84a6b0061cfe\n@i function clog_v1(y::Complex{T}, squaren::T, n::T, x::Complex{T}) where T\n\tsquaren += x.re^2\n\tsquaren += x.im^2\n\tn += sqrt(squaren)\n y.re += log(n)\n\ty.im += atan(x.im, x.re)\nend\n\n# ╔═╡ 1ecdc4d2-b3ca-4f5c-a454-0f0bc51b6ec2\n@test clog_v1(0.0im, 0.0, 0.0, 3.0im)[1] ≈ log(3.0im)\n\n# ╔═╡ 927ea209-2ccd-48d5-b69e-0a3c735bb496\nmd\"\"\"Bennett, Charles H. \"Logical reversibility of computation.\" (1973).\"\"\"\n\n# ╔═╡ 962b204c-8195-4938-944c-b7c4a52e70bd\nTikzPicture(L\"\"\"\n\\def\\r{0.15};\n\\foreach \\x in {1,...,5}{\n\t\\fill[fill=black] (\\x, 0) circle [radius=\\r];\n\t\\node[white] at (\\x, 0) {$s_{\\x}$};\n}\n\\fill[fill=white] (5.5, 0) circle [radius=\\r];\n\\foreach \\x in {1,...,4}{\n\t\\draw [black, thick, ->] (\\x+\\r, \\r) .. controls (\\x+0.5, 0.3) .. (\\x+1-\\r, \\r);\n\t\\node at (\\x+0.5, 0.4) {\\x};\n\t}\n\\foreach[evaluate={\\y=int(8-\\x)}] \\x in {1,...,3}{\n\t\\draw [red, thick, <-] (\\x+\\r, -\\r) .. controls (\\x+0.5, -0.3) .. (\\x+1-\\r, -\\r);\n\t\\node at (\\x+0.5, -0.4) {\\y};\n\t}\n\"\"\"\n, options=\"scale=2.0\", preamble=\"\")\n\n# ╔═╡ af58a0f8-e3fd-465f-b1ae-6fbd94123c91\n@i function clog_v2(y::Complex{T}, x::Complex{T}) where T\n\t######### compute ########\n\tn ← zero(T)\n\tsquaren ← zero(T)\n\tsquaren += x.re^2\n\tsquaren += x.im^2\n\tn += sqrt(squaren)\n\n\t########## copy ##########\n\t\n y.re += log(n)\n\ty.im += atan(x.im, x.re)\n\t\n\t####### uncompute ########\n\tn -= sqrt(squaren)\n\tsquaren -= x.im^2\n\tsquaren -= x.re^2\n\tn → zero(T)\n\tsquaren → zero(T)\nend\n\n# ╔═╡ 4f993061-c6c3-4acb-aef3-8453e7b83997\n@test clog_v2(0.0im, 3.0im)[1] ≈ log(3.0im)\n\n# ╔═╡ 6e5cf9bb-7cab-4da1-8831-541e0ee3bde8\n@i @inline function clog_v3(y::Complex{T}, x::Complex{T}) where T\n\t# @invcheckoff turns of reversibility check and accelerate code\n @routine @invcheckoff begin\n @zeros T squaren n\n\t\tsquaren += x.re^2\n\t\tsquaren += x.im^2\n\t\tn += sqrt(squaren)\n end\n y.re += log(n)\n y.im += atan(x.im, x.re)\n ~@routine\nend\n\n# ╔═╡ 320e4114-f0da-4106-90ab-a9f7b0ef0099\n@test clog_v3(0.0im, 3.0im)[1] ≈ log(3.0im)\n\n# ╔═╡ 2d944e8d-e19d-48be-ab7f-c3e54e9f43ef\nmd\"# III. Automatic differentiation in NiLang\"\n\n# ╔═╡ 4f53fa8e-ea9b-461f-8199-7bbe2a3ef544\nmd\"## Scalar or tensor\"\n\n# ╔═╡ 56ea4f5f-ea88-46d6-beed-b9a7afed315d\nmd\"Differentiating matrix vector multiplication\"\n\n# ╔═╡ 124a9ecd-0bda-4823-9507-92efcf449d9c\nmd\"\"\"\n```math\ny = A x\n```\n\"\"\"\n\n# ╔═╡ 6819498c-7a46-48fa-9eec-38341dca72f9\nlet\n\ttl = md\"tensor level view\"\n\ttl2 = md\"\n```julia\ny = A * x\n```\n\"\n\tsl = md\"scalar level view\"\n\tsl2 = md\"\n```julia\nfor j=1:n\n\tfor i=1:m\n\t\ty[i] += A[i,j] * x[j]\n\tend\nend\n```\"\n\tleftright(updown(tl, tl2, width=300), updown(sl, sl2, width=300))\nend\n\n# ╔═╡ 400f79cf-9260-4195-9582-0e8c486ddb5a\nhtml\"\"\"implementing AD on scalars\n
    \n
  • limited primitive function
    • \n
  • hard to utilize BLAS
    • \n
  • harder to manage memory caching
    • \n
\n\"\"\"\n\n# ╔═╡ 2dad3acd-332c-46b9-8f84-21c076bdef41\nmd\"## Forward mode autodiff and reverse mode autodiff\"\n\n# ╔═╡ a674bde5-70e1-4d21-aedf-8977f8039c36\nmd\"\nA program: ``\\vec p \\mapsto \\vec q``, containing the following forward/backward instruction.\n\n```math\n\\begin{cases}\n\\vec y = f(\\vec x) \\\\\n\\vec x = f^{-1}(\\vec y)\n\\end{cases}\n```\n\"\n\n# ╔═╡ bf696784-23d4-42b2-8ee1-bfec11ff8d78\nlet\n\tfd = md\"\"\"\n```math\n\\begin{align}\n \\frac{d \\vec x}{d p_i} = \\underbrace{\\frac{d \\vec y}{d \\vec x}}_{\\text{local jacobian}}\\frac{d \\vec x}{d p_i}\n\\end{align}\n```\n\n\nForwardDiff: ``(\\vec x, \\frac{d\\vec x}{dp_i}) \\mapsto (y, \\frac{d\\vec y}{dp_i})``\n\"\"\"\n\n\tbd = md\"\"\"\n```math\n\\begin{align*}\n \\frac{d q_j}{d \\vec x} &\\mathrel{+}= \\frac{\\partial q_j}{\\partial \\vec y}\\underbrace{\\frac{d \\vec y}{d \\vec x}}_{\\text{local jacobian}}\n\\end{align*}\n```\nNiLang: ``(\\vec y, \\frac{d\\mathcal{L}}{d\\vec y}) \\mapsto (\\vec x, \\frac{d\\mathcal{L}}{d\\vec x})``\n\"\"\"\n\tleftright(fd, bd)\nend\n\n# ╔═╡ 9193fcbb-ec4f-41b4-8fca-be8e183dea31\ng_forwarddiff = sin(Dual(π/3, 1.0))\n\n# ╔═╡ 3fadf1d4-8fa2-4c02-aa21-b7969b465536\n# note: y += sin(x) is translate to `PlusEq(sin)(y, x)` in NiLang.\ng_nilang = MinusEq(sin)(GVar(sin(π/3), 1.0), GVar(π/3, 0.0))\n\n# ╔═╡ 12e933ca-13f3-414c-926a-f1bb1bbe66cf\n@test g_forwarddiff.partials[1] ≈ g_nilang[2].g\n\n# ╔═╡ 4403c183-5eeb-4fd0-87a4-4ad29e1f4dc2\nmd\"## Differentiating complex valued log\"\n\n# ╔═╡ 4c3f9b91-4f27-4fb8-a9db-1ddbfd62dbdd\n@i function real_of_clog(loss::Real, y::Complex, x::Complex)\n\tclog_v3(y, x)\n\tloss += y.re\nend\n\n# ╔═╡ af71e2d7-a600-46fd-9a46-1b9d4607f06d\n@test let\n\t# forward pass to compute results\n\tloss_out, y_out, x_out = real_of_clog(0.0, 0.0im, 2+3.0im)\n\n\t# backward pass to compute inputs from results, using element type `GVar`\n\tgloss_out = GVar(loss_out, 1.0)\n\tgy_out = GVar(y_out)\n\tgx_out = GVar(x_out)\n\tgloss_out, gy_out, gx_out = (~real_of_clog)(gloss_out, gy_out, gx_out)\n\n\t# forward diff\n\tdloss_out = Dual(loss_out, 0.0)\n\tdy_out = Complex(Dual(0.0, 0.0), Dual(0.0, 0.0))\n\tdx_out = Complex(Dual(2.0, 1.0), Dual(3.0, 0.0))\n\tdloss_out, dy_out, dx_out = real_of_clog(dloss_out, dy_out, dx_out)\n\t\n\tgx_out.re.g ≈ dloss_out.partials[1]\nend\n\n# ╔═╡ 1d3c7324-6828-47aa-b30e-bcac0e052213\nmd\"A shortcut\"\n\n# ╔═╡ 2993fe1f-042b-4c85-85b8-bcc7ed449a54\nNiLang.AD.gradient(real_of_clog, (0.0, 0.0im, 2+3.0im); iloss=1)\n\n# ╔═╡ 048d482e-3e5b-496f-95d7-17589a5f6f11\nmd\"# Overheads matters!\"\n\n# ╔═╡ e22bbe66-56f5-40be-b464-1f8651e6a6ac\nmd\"\"\"\n* Case 1: Intrinsically irreversible linear program,\n* Case 2: A linear algebra function: QR decomposition\n\"\"\"\n\n# ╔═╡ 28767d73-47a8-4f4b-b3dd-146c4ae3e038\nmd\"## Case 1: differentiating a linear program\"\n\n# ╔═╡ ea907d51-d4a9-48ca-90a5-bd91309ccfad\nmd\"Imagine we have a very long linear program that intrinsically irreversible\"\n\n# ╔═╡ 3aa99be5-6747-4163-9bb6-ed8cd5ce19f6\nTikzPicture(L\"\"\"\n\\def\\r{0.15};\n\\def\\n{10};\n\\foreach \\x in {\\n}{\n \\fill[fill=black] (\\x, 0) circle [radius=\\r];\n \\node[white] at (\\x, 0) {$s_{\\x}$};\n}\n\\foreach \\x in {1,...,9}{\n \\draw (\\x, 0) circle [radius=\\r];\n \\node[black] at (\\x, 0) {$s_{\\x}$};\n}\n\\fill[fill=white] (\\n+0.5, 0) circle [radius=\\r];\n\\foreach \\x/\\t in {1/1,2/2,3/3,4/4,5/5,6/6,7/7,8/8,9/9}{\n \\draw [black, thick, ->] (\\x+\\r, \\r) .. controls (\\x+0.5, 0.3) .. (\\x+1-\\r, \\r);\n \\node[black] at (\\x+0.5, 0.4) {\\t};\n }\n\"\"\"\n, options=\"scale=2.0\", preamble=\"\")\n\n\n# ╔═╡ 2f0263f5-2ace-4d4b-8d71-cee26c03122e\nmd\"The accumulative version\"\n\n# ╔═╡ 03a21468-c2fe-4df7-ae8a-38b28a0efe2f\nTikzPicture(L\"\"\"\n\\def\\r{0.15};\n\\def\\n{10};\n\\foreach \\x in {1,...,\\n}{\n \\fill[fill=black] (\\x, 0) circle [radius=\\r];\n \\node[white] at (\\x, 0) {$s_{\\x}$};\n}\n\\fill[fill=white] (\\n+0.5, 0) circle [radius=\\r];\n\\foreach \\x/\\t in {1/1,2/2,3/3,4/4,5/5,6/6,7/7,8/8,9/9}{\n \\draw [black, thick, ->] (\\x+\\r, \\r) .. controls (\\x+0.5, 0.3) .. (\\x+1-\\r, \\r);\n \\node[black] at (\\x+0.5, 0.4) {\\t};\n }\n\"\"\"\n, options=\"scale=2.0\", preamble=\"\")\n\n\n# ╔═╡ f326d8e5-7117-4eed-b30d-0f64e5974426\nmd\"With uncomputing\"\n\n# ╔═╡ b9af3db6-5725-4190-b96c-f3fa41f07c93\nTikzPicture(L\"\"\"\n\\def\\r{0.15};\n\\def\\n{10};\n\\foreach \\x in {1,4,7,10}{\n \\fill[fill=black] (\\x, 0) circle [radius=\\r];\n \\node[white] at (\\x, 0) {$s_{\\x}$};\n}\n\\foreach \\x in {2,3,5,6,8,9}{\n \\draw (\\x, 0) circle [radius=\\r];\n \\node[black] at (\\x, 0) {$s_{\\x}$};\n}\n\\fill[fill=white] (\\n+0.5, 0) circle [radius=\\r];\n\\foreach \\x/\\t in {1/1,2/2,3/3,4/6,5/7,6/8,7/11,8/12,9/13}{\n \\draw [black, thick, ->] (\\x+\\r, \\r) .. controls (\\x+0.5, 0.3) .. (\\x+1-\\r, \\r);\n \\node[black] at (\\x+0.5, 0.4) {\\t};\n }\n\\foreach \\x/\\t in {1/5,2/4,4/10,5/9,7/15,8/14}{\n \\draw [black, thick, <-] (\\x+\\r, -\\r) .. controls (\\x+0.5, -0.3) .. (\\x+1-\\r, -\\r);\n \\node[black] at (\\x+0.5, -0.4) {\\t};\n}\n\"\"\"\n, options=\"scale=2.0\", preamble=\"\")\n\n# ╔═╡ 47cf7e85-c49f-4618-9689-e0de789625f6\nmd\"With uncomputing: the coarser grain\"\n\n# ╔═╡ 2fcf48fd-bedc-41d9-a433-68efd5dd0d20\nTikzPicture(L\"\"\"\n\\def\\r{0.15};\n\\def\\n{10};\n\\foreach \\x in {1,4,7,10}{\n \\fill[fill=black] (\\x, 0) circle [radius=\\r];\n \\node[white] at (\\x, 0) {$s_{\\x}$};\n}\n\n\\fill[fill=white] (\\n+0.5, 0) circle [radius=\\r];\n\\foreach \\x in {1,4,7}{\n \\draw [black, thick, ->] (\\x+\\r, \\r) .. controls (\\x+1.5, 0.6) .. (\\x+3-\\r, \\r);\n }\n\\foreach \\x in {1,4}{\n \\draw [black, thick, <-] (\\x+\\r, -\\r) .. controls (\\x+1.5, -0.6) .. (\\x+3-\\r, -\\r);\n}\n\"\"\"\n, options=\"scale=2.0\", preamble=\"\")\n\n# ╔═╡ 5060f5bb-8430-42aa-b61d-c88249edb323\nmd\"## Pebble game\"\n\n# ╔═╡ 55dd53ed-6420-4426-95ae-feb47bf50f22\nmd\"The optimal time-space tradeoff corresponds to the optimal solution to the pebble game.\"\n\n# ╔═╡ c62a9f94-457f-496b-bee0-bb0db02aca5d\nTikzPicture(L\"\"\"\n\\def\\y{0}\n\\node at (4, \\y-1) {initial configuration};\n\\foreach \\x in {0,...,16}{\n\t\\draw (0.5*\\x-0.25, 0.5*\\y-0.25) rectangle (0.5*\\x+0.25, 0.5*\\y+0.25);\n\t\\ifnum \\x > 0\n\t\t\\node at (0.5*\\x, 0.5*\\y) {\\x};\n\t\\fi\n}\n\\fill (0, 0) ellipse (0.2 and 0.15);\n\\def\\dx{11}\n\\foreach \\a/\\b in {-0.1/0.3, 0.2/0.5, -0.5/0.4, 0.1/0.24, 0.6/0.1, -0.3/-0.3}\n\t\\fill (\\dx+\\a, \\y+\\b) ellipse (0.2 and 0.15);\n\\node at (11, -1) {free pool of pebbles};\n\\node at (13, -1) {};\n\\node (goal) at (9, \\y+1) {goal};\n\\draw[<-,thick] (8, \\y+0.3) .. controls (8.2, \\y+0.7) .. (goal);\n\"\"\", options=\"scale=1.0\")\n\n# ╔═╡ bccadcb5-6d9f-4a70-b7c4-74e0e3d5f8c8\nTikzPicture(L\"\"\"\n\\def\\y{0}\n\\node at (4, \\y-1) {put rule (if and only if the previous grid is occupied)};\n\\foreach \\x in {0,...,16}\n\t\\draw (0.5*\\x-0.25, 0.5*\\y-0.25) rectangle (0.5*\\x+0.25, 0.5*\\y+0.25);\n\\fill (0, 0) ellipse (0.2 and 0.15);\n\\fill (2, 0) ellipse (0.2 and 0.15);\n\\draw[dashed] (2.5, 0) ellipse (0.2 and 0.15);\n\\def\\dx{11}\n\\foreach \\a/\\b in {-0.1/0.3, 0.2/0.5, -0.5/0.4, 0.1/0.24, 0.6/0.1, -0.3/-0.3}\n\t\\fill (\\dx+\\a, \\y+\\b) ellipse (0.2 and 0.15);\n\\node at (13, -1) {};\n\\draw[<-,thick] (2.5, \\y+0.3) .. controls (2.8, \\y+1) and (8.0, \\y+1) .. (10, 0.5);\n\"\"\", options=\"scale=1.0\")\n\n# ╔═╡ b308ecb0-070e-4ad8-8009-dc60e75bbe01\nTikzPicture(L\"\"\"\n\\def\\y{0}\n\\node at (4, \\y-1) {remove rule (if and only if the previous grid is occupied)};\n\\foreach \\x in {0,...,16}\n\t\\draw (0.5*\\x-0.25, 0.5*\\y-0.25) rectangle (0.5*\\x+0.25, 0.5*\\y+0.25);\n\\fill (0, 0) ellipse (0.2 and 0.15);\n\\fill (2, 0) ellipse (0.2 and 0.15);\n\\fill (2.5, 0) ellipse (0.2 and 0.15);\n\\def\\dx{11}\n\\foreach \\a/\\b in {-0.1/0.3, 0.2/0.5, -0.5/0.4, 0.1/0.24, 0.6/0.1, -0.3/-0.3}\n\t\\fill (\\dx+\\a, \\y+\\b) ellipse (0.2 and 0.15);\n\\node at (13, -1) {};\n\\draw[->,thick] (2.5, \\y+0.3) .. controls (2.8, \\y+1) and (8.0, \\y+1) .. (10, 0.5);\n\"\"\", options=\"scale=1.0\")\n\n# ╔═╡ 9123e669-19c7-47c1-a924-0c618b4a9c1f\nmd\"\nSpace complexity: ``O(\\log(T)S)``\n\nTime complexity: ``O(T^{1+\\epsilon})``\n\"\n\n# ╔═╡ 83d0c5fc-9cb7-4e37-bfdb-ed630b94d9b4\nmd\"\nThe recursive Bennett's time-space tradeoff scheme is probably optimal for a reversible program. (Li 1997)\n\"\n\n# ╔═╡ 3d2feca5-43d3-4a46-ba1c-849c5ceeb676\nmd\"nstep = $(@bind nstep Slider(2:20000; show_value=true, default=10000))\"\n\n# ╔═╡ 7ca616d1-caed-40cf-b768-b07af81a654d\nmd\"k = $(@bind bennett_k Slider(2:100; show_value=true, default=2))\"\n\n# ╔═╡ a54d5fc4-643c-47d2-be97-4626a060c9b4\nmd\"Optimal checkpointing is recursive, Griewank (1992). \nJulia Implementation: [https://github.com/GiggleLiu/TreeverseAlgorithm.jl](https://github.com/GiggleLiu/TreeverseAlgorithm.jl)\"\n\n# ╔═╡ 8b556fbe-275f-4e7b-94a0-7434ce81ad8b\nmd\"Time complexity is ``O(T\\log(T))``\"\n\n# ╔═╡ 8e9c55f6-8175-4cb4-8798-91107c4d16ee\nmd\"Space complexity is ``O(S\\log(T))``\"\n\n# ╔═╡ 972d889c-c48c-470a-b710-aba9ecaacdaa\nmd\"nstep = $(@bind treeverse_nstep Slider(2:20000; show_value=true, default=10000))\"\n\n# ╔═╡ 54b3a283-7d42-431f-9ecb-48f37198409a\nmd\"number of checkpoints = $(@bind treeverse_δ Slider(2:100; show_value=true, default=2))\"\n\n# ╔═╡ 590f56f7-5654-4493-9103-02a0fce6e945\nlet\n\tlogger = TreeverseLog()\n\ttreeverse(identity, (x,gy)->0, 0; δ=treeverse_δ, N=treeverse_nstep, logger=logger)\n\tlogger\nend\n\n# ╔═╡ 11964529-8e88-4743-a725-57fc5c525649\nhtml\"\"\"\n\n\"\"\"\n\n# ╔═╡ 45448141-214d-4e08-978e-5d1d25f763cd\nmd\"## Case 2: differentiating linear algebra functions\"\n\n# ╔═╡ dfe3cc09-6e27-4166-9a4e-2dd22f1e08a2\nmd\"The definition of QR factorization\n\n```math\nA = QR\n```\n\"\n\n# ╔═╡ 3d07e6d1-2964-4718-b7bc-114d82389aa4\nmd\"We implement Householder QR\"\n\n# ╔═╡ c00cd648-d685-4a17-9ddf-39c06dd5f066\nmd\"\"\"\n$Q = H_1H_2 \\ldots H_n$\n\"\"\"\n\n# ╔═╡ 5390c3ba-1507-4db9-9972-8295cbe493bc\nmd\"\"\"\n```math\n\\begin{align}\n&H = 1-\\beta vv^T\n\\end{align}\n```\n\"\"\"\n\n# ╔═╡ 33a0bda1-c943-4792-bc3d-fbf1adf16d0a\nlet\n\timg = TikzPicture(L\"\"\"\n\\draw[->,thick] (0, 0) -- (1, 1);\n\\draw[->,thick] (0, 0) -- ({sqrt(2)}, 0);\n\\draw[thick,dashed] (0, 0) -- (1.5, {1.5*tan(22.5)});\n\\node at (1.5, 0) {$e$};\n\\node at (1.1, 1.1) {$x$};\n\\node at (1.4, {1.6*tan(22.5)}) {$v$};\n\\node at (2.6, 0.5) {$v = x-\\|x\\|_2 e$};\n\"\"\", options=\"scale=2.0\", preamble=\"\")\n\tHTML(\"\"\"
$(html(img))
\"\"\")\nend\n\n# ╔═╡ a2e81e79-3f85-4eef-8639-f455f9165a25\nmd\"Apply reflector, step $(@bind house_step NumberField(0:4))\"\n\n# ╔═╡ 0a04c470-8f5a-4b56-b2e2-5b32e3b944f3\nlet\n\tnum(i, j) = let\n\t\tsym = j>=i || house_step < j ? raw\"\\times\" : raw\"0\"\n\t\tif i>=house_step && j>=house_step\n\t\t\tsym = \"\\\\color{red}{$sym}\"\n\t\tend\n\t\tsym\n\tend\n\telements = join([join([num(i,j) for j=1:5], \" & \") for i=1:5], raw\"\\\\\\\\\")\n\tdiag(i) = if i == house_step\n\t\traw\"\\boldsymbol{\\times}\"\n\telse\n\t\traw\"\\times\"\n\tend\n\tMarkdown.parse(\"\"\"\n```math\n\\\\begin{align}\n$(join([\"H_$i\" for i in house_step:-1:1], \"\")) A = \\\\left(\\\\begin{matrix}\n$elements\\\\end{matrix}\\\\right)\n\\\\end{align}\n```\n\"\"\")\nend\n\n# ╔═╡ dced36eb-3b84-4d08-8268-0ffb831e39b5\nmd\"the following code is adapted from the Julia standard library\"\n\n# ╔═╡ 0dc268fa-2b71-4915-b73b-3f1de9fbc157\nstruct Reflector{T,RT,VT<:AbstractVector{T}}\n ξ::T\n normu::RT\n sqnormu::RT\n r::T\n y::VT # reflector vector\nend\n\n# ╔═╡ 871f62a3-2c9c-445e-b1dc-4cba363fa604\n# compute \"Householder\" reflector\n@i function reflector!(R::Reflector{T,RT}, x::AbstractVector{T}) where {T,RT}\n @inbounds @invcheckoff if length(x) != 0\n R.ξ += x[1]\n R.sqnormu += abs2(R.ξ)\n for i = 2:length(x)\n R.sqnormu += abs2(x[i])\n end\n if !iszero(R.sqnormu)\n R.normu += sqrt(R.sqnormu)\n if real(R.ξ) < 0\n NEG(R.normu)\n end\n R.ξ += R.normu\n R.y[1] -= R.normu\n for i = 2:length(x)\n R.y[i] += x[i] / R.ξ\n end\n R.r += R.ξ/R.normu\n end\n end\nend\n\n# ╔═╡ 6a50f7ac-b5a5-444d-9042-81b82cb66aec\n# apply reflector from left\n@i function reflectorApply!(vA::AbstractVector{T}, x::AbstractVector, τ::Number, A::StridedMatrix{T}) where T\n (m, n) ← size(A)\n @safe if length(x) != m || length(vA) != n\n throw(DimensionMismatch(\"reflector has length ($(length(x)), $(length(vA))), which must match the first dimension of matrix A, ($m, $n)\"))\n end\n @inbounds @invcheckoff if m != 0\n for j = 1:n\n # dot\n @routine @zeros T vAj vAj_τ\n vAj += A[1, j]\n for i = 2:m\n vAj += x[i]'*A[i, j]\n end\n @routine vAj_τ += τ' * vAj # `vAj_τ` can be uncomputed easily\n # ger\n A[1, j] -= vAj_τ\n for i = 2:m\n A[i, j] -= x[i]*vAj_τ\n end\n ~@routine\n SWAP(vA[j], vAj)\n ~@routine\n end\n end\n (m, n) → size(A)\nend\n\n# ╔═╡ bace7924-3aff-463f-9351-cc59191b469a\nstruct QRPivotedRes{T,RT,VT}\n factors::Matrix{T} # resulting matrix\n τ::Vector{T}\n jpvt::Vector{Int} # pivot vector\n reflectors::Vector{Reflector{T,RT,VT}} # ~ half size of A (overhead)\n vAs::Vector{Vector{T}} \t\t\t # ~ half size of A (overhead)\n jms::Vector{Int}\nend\n\n# ╔═╡ b12bcfbb-c7f2-4dc9-821a-e59365bf6fa4\nbegin\n\t_norm(v) = sqrt(sum(x->abs2(NiLang.value(x)), v))\n\tfunction indmaxcolumn(A::AbstractMatrix)\n\t\tmm = _norm(view(A, :, 1))\n\t\tii = 1\n\t\tfor i = 2:size(A, 2)\n\t\t\tmi = _norm(view(A, :, i))\n\t\t\tif abs(mi) > mm\n\t\t\t\tmm = mi\n\t\t\t\tii = i\n\t\t\tend\n\t\tend\n\t\treturn ii\n\tend\nend;\n\n# ╔═╡ 12d52dc8-da0b-46dc-9251-eb0cc9a39a7e\nbegin\n\tfunction alloc_qr(A::AbstractMatrix{T}) where T\n\t\t(m, n) = size(A)\n\t\tτ = zeros(T, min(m,n))\n\t\tjpvt = collect(1:n)\n\t\treflectors = Reflector{T,real(T),Vector{T}}[]\n\t\tvAs = Vector{T}[]\n\t\tjms = Int[]\n\t\tQRPivotedRes(zero(A), τ, jpvt, reflectors, vAs, jms)\n\tend\n\tfunction alloc_reflector(x::AbstractVector{T}) where T\n\t\tRT = real(T)\n\t\tReflector(zero(T), zero(RT), zero(RT), zero(T), zero(x))\n\tend\nend\n\n# ╔═╡ 923a5e2a-cc91-4404-913a-6e8012fa5834\n@i function qr_pivoted!(res::QRPivotedRes, A::StridedMatrix{T}) where T\n m, n ← size(A)\n res.factors += A\n @inbounds @invcheckoff for j = 1:min(m,n)\n # Find column with maximum norm in trailing submatrix\n jm ← indmaxcolumn(view(res.factors, j:m, j:n)) + j - 1\n\n\t\t# pivot columns\n if jm != j\n # Flip elements in pivoting vector\n SWAP(res.jpvt[jm], res.jpvt[j])\n # Update matrix\n\t\t\tSWAP.(res.factors |> subarray(:, jm), res.factors |> subarray(:, j))\n end\n\n # Compute reflector of columns j\n R ← alloc_reflector(res.factors |> subarray(j:m, j))\n vA ← zeros(T, n-j)\n reflector!(R, res.factors |> subarray(j:m, j))\n # Update trailing submatrix with reflector\n reflectorApply!(vA, R.y, R.r, res.factors |> subarray(j:m, j+1:n))\n for i=1:length(R.y)\n SWAP(R.y[i], res.factors[j+i-1, j])\n end\n\t\tres.reflectors[end+1] ↔ R # stack push\n\t\tres.vAs[end+1] ↔ vA\n\t\tres.jms[end+1] ↔ jm\n end\n @inbounds for i=1:length(res.reflectors)\n res.τ[i] += res.reflectors[i].r\n end\n m, n → size(A)\nend\n\n# ╔═╡ f8616bf4-02e6-4d66-a0f6-d35db701e82c\n@testset \"qr\" begin\n for A in [randn(5, 5), randn(6, 4), randn(4, 6)]\n res = alloc_qr(A)\n res, = qr_pivoted!(res, copy(A))\n res2 = LinearAlgebra.qrfactPivotedUnblocked!(copy(A))\n @test res.factors ≈ res2.factors\n @test res.τ ≈ res2.τ\n @test res.jpvt ≈ res2.jpvt\n end\nend\n\n# ╔═╡ bee4ab06-4c60-4bec-89d0-b3c9a512f7a6\nmd\"## Comparing with manual AD\n\n\"\n\n# ╔═╡ 717f51fb-bd0a-4bb4-a5fc-1f1cf5d56ed8\nhtml\"\"\"\n
    \n
  • slower,
    • \n
  • an extra space overhead of the size of input matrix,
    • \n
  • stabler, e.g. can handle rank deficient matrices
    • \n
  • works consistently for complex numbers.
    • \n
\n\"\"\"\n\n# ╔═╡ db6b6481-7e67-4418-b907-13b38c77bac7\nmd\"## Performance\"\n\n# ╔═╡ a0be4807-52ed-4626-8009-97a79e36e2f1\nlet # Note: approximately 2x slower than the BLAS version\n\tRandom.seed!(3)\n\tA = randn(ComplexF64, 200, 200)\n\t@benchmark LinearAlgebra.qrfactPivotedUnblocked!(copy($A))\nend\n\n# ╔═╡ 52dec342-d3de-4a88-8acb-0aa186bcc086\nlet\n\tRandom.seed!(3)\n\tA = randn(ComplexF64, 200, 200)\n\t@benchmark qr_pivoted!(alloc_qr($A), copy($A))\nend\n\n# ╔═╡ 280c9363-9b49-44f1-a240-b7f205ffc56b\n@benchmark let\n\tres = alloc_qr(A)\n\tres, A = qr_pivoted!(res, A)\n\t(~qr_pivoted!)(GVar(res), GVar(A))\nend setup=(Random.seed!(3); A = randn(ComplexF64, 200, 200))\n\n# ╔═╡ 0b3735c2-695c-4225-843e-16ca17aac0eb\nmd\"\"\"## Take home message\n\n1. Comparing to irreversible computing, reversible computing is more **energy efficient**.\n2. Reversible programming suffers from **polynomial time overhead and logarithmic space overhead** when differentiating a irreversible linear program. The overhead is much less when writting linear algebra functions.\n3. Reversible embedded domain specific language NiLang.jl:\n$(html\"
\")\n2. It is easy to balance time and space when differentiating a program in a reversible programming language.\n4. It is not always more reliable to differentiate a program by deriving the backward rule manually.\n\n5. TODOs\n - Is it possible to port `LoopVectorization` to NiLang to write blas level reversible program?\n6. **How to find this notebook?** In NiLang's Github repo, file: `notebooks/reversibleprog.jl`\n\"\"\"\n\n# ╔═╡ d7942b37-f821-494a-8f18-5f267aa3457a\nmd\"\"\"\n### References\n* Reeb, David, and Michael M. Wolf. \"An improved Landauer principle with finite-size corrections.\" (2014).\n* Athas, William C., and L. J. Svensson. \"Reversible logic issues in adiabatic CMOS.\" Proceedings Workshop on Physics and Computation. (1994).\n* Takeuchi, N., Y. Yamanashi, and N. Yoshikawa. \"Reversible logic gate using adiabatic superconducting devices.\" (2014)\n* Griewank, Andreas. \"Achieving logarithmic growth of temporal and spatial complexity in reverse automatic differentiation.\" Optimization Methods and software 1.1 (1992): 35-54.\n* Ming Li, John Tromp, Paul Vitanyi. \"Reversible Simulation of Irreversible Computation by Pebble Games\" (1997)\n\"\"\"\n\n# ╔═╡ 865f049f-54d0-4d21-860e-062262edcb58\nmd\"# Removed\"\n\n# ╔═╡ c3b730a4-d5b4-471e-bd06-30ace6e8b8fe\nlet\n\tnodes_list = [[0], [0,1], [0,1,2], [0,2], [0,2,3], [0,2,3,4], [0,2,4], [0,1,2,4], [0,1,4], [0,4], [0,4,5], [0,4,5,6], [0,4,6], [0,4,6,7], [0,4,6,7,8], [0,4,6,8], [0,4,5,6,8], [0,4,5,8], [0,4,8], [0,1,4,8], [0,1,2,4,8], [0,2,4,8], [0,2,3,4,8], [0,2,3,8], [0,2,8], [0,1,2,8], [0,1,8], [0,8],[0,8,9], [0,8,9,10], [0,8,10], [0,8,10,11], [0,8,10,11,12], [0,8,10,12], [0,8,9,10,12], [0,8,9,12], [0,8,12], [0,8,12,13], [0,8,12,13,14], [0,8,12,14], [0,8,12,14,15], [0,8,12,14,15,16]]\n\ts = join([raw\"\"\"\n\t\\def\\y{\"\"\"*string(j-1)*raw\"\"\"}\n\t\\node at (-1, -0.7*\\y) {step \\y};\n\t\\foreach \\x in {0,...,16}{\n\t\t\\draw (0.5*\\x-0.25, -0.7*\\y-0.25) rectangle (0.5*\\x+0.25, -0.7*\\y+0.25);\n\t}\n\t\\foreach \\x in {\"\"\"*join(nodes, \",\")*raw\"\"\"}{\n\t\t\\fill (0.5*\\x, -0.7*\\y) ellipse (0.2 and 0.15);\n\t}\n\"\"\" for (j, nodes) in enumerate(nodes_list)], \"\\n\")\n\tTikzPicture(LaTeXString(s), options=\"scale=1.0\")\nend\n\n# ╔═╡ 98f42f60-7870-4813-b0c1-728285c25f01\nmd\"## Finding maximum, the reversible programming implementation\"\n\n# ╔═╡ c3c63865-f538-4d93-bef3-6b9c69cb177f\nmd\"The naive implementation with linear space overhead\"\n\n# ╔═╡ 66225c05-165e-4051-bb5e-4cfba579fd5b\n@i function i_find_maximum(m, y::AbstractVector, x::AbstractVector) where T\n\t@safe @assert !isempty(x) && length(y) == length(x) # error handling\n\ty[1] += x[1]\n\tfor i=2:length(x)\n\t\tif x[i] > y[i-1]\n\t\t\ty[i] += x[i]\n\t\telse\n\t\t\ty[i] += y[i-1]\n\t\tend\n\tend\n\tm += y[end]\nend\n\n# ╔═╡ 14291e8d-001f-4e26-b094-addb970cf530\nx = randn(17)\n\n# ╔═╡ bf670e51-06f8-4094-949c-ca2d02fd0d01\nfind_maximum(x)\n\n# ╔═╡ 416e53e4-6123-430f-b433-4334b7e85298\ni_find_maximum(0.0, zero(x), x)\n\n# ╔═╡ edd5f8df-9abd-4254-abf6-33ae31d88a8d\nstruct FindMaxState{T}\n\tm::T\n\tstep::Int\nend\n\n# ╔═╡ c0ff9103-2aa8-45fd-bea5-1903c53196de\nlet\n\tx, y = FindMaxState(5.0, 1), FindMaxState(2.0, 3)\n\t@instr x += y\n\tx, y\nend\n\n# ╔═╡ e17e70cb-2d82-4a08-9f6e-e50dcaa325e3\n@i function i_find_maximum_step(t, s, x)\n\tt.step += s.step + 1\n\tif x[t.step] > s.m\n\t\tt.m += x[t.step] # everything is mutable in NiLang\n\telse\n\t\tt.m += s.m\n\tend\nend\n\n# ╔═╡ ddc662d7-6901-4885-9d2c-1876a2c9d2ff\nlet\n\tRandom.seed!(3)\n\tx = randn(nstep)\n\tlogger = NiLang.BennettLog()\n\toutput = NiLang.bennett(i_find_maximum_step, FindMaxState(0.0, 0), FindMaxState(x[1], 1), x; k=bennett_k, N=length(x)-1, logger=logger)[2]\n\tText(\"output = $(output.m)\\n\\n$logger\")\nend\n\n# ╔═╡ d2001eb2-45cb-4f07-aa99-dd84996359b7\nmd\"## The connection to automatic differentiation\"\n\n# ╔═╡ fa3b6d6a-a55d-4097-8ad2-7dafb5f01d8c\nmd\"\"\"\n* put rule: Only if there exists a pebble in grid $i$, you can move a pebble from your own pool to the grid $i+1$,\n* take rule: you can take a pebble from the board any time,\n* doodle rule: you can doodle grid $i$ only it when this grid has a pebble in it and grid $i+1$ is doodled,\n* end rule: doodle all grids.\n\"\"\"\n\n# ╔═╡ dabb4656-4aed-4168-8659-c0472528c41d\nmd\"\"\"\n## Optimal time\n\"\"\"\n\n# ╔═╡ 0367be06-4185-4add-a04b-f696c5a43638\nTikzPicture(L\"\"\"\n\\foreach[evaluate={\\j=int(16-\\y)}] \\y in {0,...,16}{\n\t\\node at (-1, 0.7*\\y) {step \\j};\n\t\\foreach \\x in {0,...,16}{\n\t\t\\draw (0.5*\\x-0.25, 0.7*\\y-0.25) rectangle (0.5*\\x+0.25, 0.7*\\y+0.25);\n\t}\n\t\\foreach \\x in {0,...,\\j}{\n\t\t\\fill (0.5*\\x, 0.7*\\y) ellipse (0.2 and 0.15);\n\t}\n}\n\"\"\", options=\"scale=1.0\")\n\n# ╔═╡ c79e7651-975b-407c-8c8c-d0c5653ec570\nmd\"\nSpace complexity: ``O(TS)``\n\nTime complexity: ``O(T)``\n\"\n\n# ╔═╡ 3fed55d7-dbed-4fc9-8410-2633d5200db6\nmd\"\"\"\n## Limited space\n\"\"\"\n\n# ╔═╡ 663180d2-8fe1-4996-a194-d38120ae05fd\nmd\"The recursive Bennett's time-space tradeoff\"\n\n# ╔═╡ 39884e8a-bc83-4ff4-85bc-cfaccb4674f2\nhtml\"\"\"\n\n\"\"\"\n\n# ╔═╡ c24e7391-a187-4a7e-aab7-93027f7db965\nmd\"The space optimal solution for 16 grids requires recursive Bennett's algorithm\"\n\n# ╔═╡ 12734842-d66f-4bfc-a9ad-01adeb2450e0\n# stepfunc: step function\n# state: state dictionary, initial value should contain entry `state[base]`\n# k: compute `k` chunks forward and `k-1` chunks backward\n# base: starting point\n# len: number of steps to compute\n@i function bennett_alg!(stepfunc, state::Dict{Int,T}, k::Int, base, len, args...; kwargs...) where T\n if len == 1 \t\t# lowest level\n state[base+1] ← _zero(state[base])\n stepfunc(state[base+1], state[base], args...; kwargs...)\n else\n\t\t@safe @assert len % k == 0\n @routine begin # compute block size\n chunksize ← 0\n\t\t\tstart ← 0\n\t\t\tchunksize += len ÷ k\n\t\t\tstart += base\n\t\t\tfor j=1:k-1\n\t\t\t\tbennett_alg!(stepfunc, state, k, start, chunksize, args...; kwargs...)\n\t\t\t\tstart += chunksize\n\t\t\tend\n\t\tend\n\t\tbennett_alg!(stepfunc, state, k, start, chunksize, args...; kwargs...)\n ~@routine\n end\nend\n\n# ╔═╡ 357fd442-6e6e-4b14-b2d0-efd3fa775d0b\nbennett_alg!(i_find_maximum_step, Dict(0=>FindMaxState(x[1], 1)), 2, 0, length(x)-1, x)[2]\n\n# ╔═╡ 1bca53ef-3438-4db5-9900-9fee71936a62\n@i function loss(result, state, x)\n\tnstep ← length(x)-1\n\tbennett_alg!((@skip! i_find_maximum_step), state, 2, 0, nstep, x)\n\tresult += state[nstep].m\n\tnstep → length(x)-1\nend\n\n# ╔═╡ 4e273d1f-a00e-49ca-9f8d-a0f6930550fb\nlet\n\t@testset \"qr pivoted gradient\" begin\n\t\tRandom.seed!(3)\n\t\tA = randn(ComplexF64, 5, 5)\n\t\tres = alloc_qr(A)\n\t\tres, = qr_pivoted!(res, copy(A))\n\t\tres2 = LinearAlgebra.qrfactPivotedUnblocked!(copy(A))\n\t\t@test res.factors ≈ res2.factors\n\t\t@test res.τ ≈ res2.τ\n\t\t@test res.jpvt ≈ res2.jpvt\n\n\t\t# rank deficient initial matrix\n\t\tn = 50\n\t\tU = LinearAlgebra.qr(randn(n, n)).Q\n\t\tΣ = Diagonal((x=randn(n); x[n÷2+1:end] .= 0; x))\n\t\tA = U*Σ*U'\n\t\tres = alloc_qr(A)\n\t\t@test rank(A) == n ÷ 2\n\t\tqrres = qr_pivoted!(deepcopy(res), copy(A))[1]\n\t\t@test count(x->(x>1e-12), sum(abs2, QRPivoted(qrres.factors, qrres.τ, qrres.jpvt).R, dims=2)) == n ÷ 2\n\n\t\t#A = randn(ComplexF64, n, n)\n\t\t@i function loss(y, qrres, A)\n\t\t\tqr_pivoted!(qrres, A)\n\t\t\ty += abs(qrres.factors[1])\n\t\tend\n\t\tnrloss(A) = loss(0.0, deepcopy(res), A)[1]\n\t\tngA = zero(A)\n\t\tδ = 1e-5\n\t\tfor j=1:size(A, 2)\n\t\t\tfor i=1:size(A, 1)\n\t\t\t\tA_ = copy(A)\n\t\t\t\tA_[i,j] -= δ/2\n\t\t\t\tl1 = nrloss(copy(A_))\n\t\t\t\tA_[i,j] += δ\n\t\t\t\tl2 = nrloss(A_)\n\t\t\t\tngA[i,j] = (l2-l1)/δ\n\t\t\tend\n\t\tend\n\t\tgA = NiLang.AD.gradient(loss, (0.0, res, A); iloss=1)[3]\n\t\t@test real.(gA) ≈ ngA\n\tend\nend\n\n# ╔═╡ 50f7070d-9f6f-4025-9be3-13812c3000eb\nlet\n\tx = [1.0, 3.0, 2.0, 1.3, -1.0]\n\tNiLang.gradient(loss, (0.0, Dict(0=>FindMaxState(x[1], 1)), x); iloss=1)\nend\n\n# ╔═╡ 00000000-0000-0000-0000-000000000001\nPLUTO_PROJECT_TOML_CONTENTS = \"\"\"\n[deps]\nBenchmarkTools = \"6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf\"\nCompose = \"a81c6b42-2e10-5240-aca2-a61377ecd94b\"\nForwardDiff = \"f6369f11-7733-5829-9624-2563aa707210\"\nLinearAlgebra = \"37e2e46d-f89d-539d-b4ee-838fcccc9c8e\"\nNiLang = \"ab4ef3a6-0b42-11ea-31f6-e34652774712\"\nPlutoUI = \"7f904dfe-b85e-4ff6-b463-dae2292396a8\"\nRandom = \"9a3f8284-a2c9-5f02-9a11-845980a1fd5c\"\nRevise = \"295af30f-e4ad-537b-8983-00126c2a3abe\"\nTest = \"8dfed614-e22c-5e08-85e1-65c5234f0b40\"\nTikzPictures = \"37f6aa50-8035-52d0-81c2-5a1d08754b2d\"\nTreeverseAlgorithm = \"e1c63c57-2fea-45bf-a8bf-df3ea6afb545\"\nViznet = \"52a3aca4-6234-47fd-b74a-806bdf78ede9\"\n\n[compat]\nBenchmarkTools = \"~1.0.0\"\nCompose = \"~0.9.2\"\nForwardDiff = \"~0.10.18\"\nNiLang = \"~0.9.1\"\nPlutoUI = \"~0.7.9\"\nRevise = \"~3.1.17\"\nTikzPictures = \"~3.3.3\"\nTreeverseAlgorithm = \"~0.1.0\"\nViznet = \"~0.3.3\"\n\"\"\"\n\n# ╔═╡ 00000000-0000-0000-0000-000000000002\nPLUTO_MANIFEST_TOML_CONTENTS = \"\"\"\n# This file is machine-generated - editing it directly is not advised\n\n[[ArgTools]]\nuuid = \"0dad84c5-d112-42e6-8d28-ef12dabb789f\"\n\n[[Artifacts]]\nuuid = \"56f22d72-fd6d-98f1-02f0-08ddc0907c33\"\n\n[[Base64]]\nuuid = \"2a0f44e3-6c83-55bd-87e4-b1978d98bd5f\"\n\n[[BenchmarkTools]]\ndeps = [\"JSON\", \"Logging\", \"Printf\", \"Statistics\", \"UUIDs\"]\ngit-tree-sha1 = \"01ca3823217f474243cc2c8e6e1d1f45956fe872\"\nuuid = \"6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf\"\nversion = \"1.0.0\"\n\n[[Bzip2_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"19a35467a82e236ff51bc17a3a44b69ef35185a2\"\nuuid = \"6e34b625-4abd-537c-b88f-471c36dfa7a0\"\nversion = \"1.0.8+0\"\n\n[[Cairo_jll]]\ndeps = [\"Artifacts\", \"Bzip2_jll\", \"Fontconfig_jll\", \"FreeType2_jll\", \"Glib_jll\", \"JLLWrappers\", \"LZO_jll\", \"Libdl\", \"Pixman_jll\", \"Pkg\", \"Xorg_libXext_jll\", \"Xorg_libXrender_jll\", \"Zlib_jll\", \"libpng_jll\"]\ngit-tree-sha1 = \"f2202b55d816427cd385a9a4f3ffb226bee80f99\"\nuuid = \"83423d85-b0ee-5818-9007-b63ccbeb887a\"\nversion = \"1.16.1+0\"\n\n[[ChainRulesCore]]\ndeps = [\"Compat\", \"LinearAlgebra\", \"SparseArrays\"]\ngit-tree-sha1 = \"be770c08881f7bb928dfd86d1ba83798f76cf62a\"\nuuid = \"d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4\"\nversion = \"0.10.9\"\n\n[[CodeTracking]]\ndeps = [\"InteractiveUtils\", \"UUIDs\"]\ngit-tree-sha1 = \"8ad457cfeb0bca98732c97958ef81000a543e73e\"\nuuid = \"da1fd8a2-8d9e-5ec2-8556-3022fb5608a2\"\nversion = \"1.0.5\"\n\n[[ColorTypes]]\ndeps = [\"FixedPointNumbers\", \"Random\"]\ngit-tree-sha1 = \"024fe24d83e4a5bf5fc80501a314ce0d1aa35597\"\nuuid = \"3da002f7-5984-5a60-b8a6-cbb66c0b333f\"\nversion = \"0.11.0\"\n\n[[Colors]]\ndeps = [\"ColorTypes\", \"FixedPointNumbers\", \"Reexport\"]\ngit-tree-sha1 = \"417b0ed7b8b838aa6ca0a87aadf1bb9eb111ce40\"\nuuid = \"5ae59095-9a9b-59fe-a467-6f913c188581\"\nversion = \"0.12.8\"\n\n[[CommonSubexpressions]]\ndeps = [\"MacroTools\", \"Test\"]\ngit-tree-sha1 = \"7b8a93dba8af7e3b42fecabf646260105ac373f7\"\nuuid = \"bbf7d656-a473-5ed7-a52c-81e309532950\"\nversion = \"0.3.0\"\n\n[[Compat]]\ndeps = [\"Base64\", \"Dates\", \"DelimitedFiles\", \"Distributed\", \"InteractiveUtils\", \"LibGit2\", \"Libdl\", \"LinearAlgebra\", \"Markdown\", \"Mmap\", \"Pkg\", \"Printf\", \"REPL\", \"Random\", \"SHA\", \"Serialization\", \"SharedArrays\", \"Sockets\", \"SparseArrays\", \"Statistics\", \"Test\", \"UUIDs\", \"Unicode\"]\ngit-tree-sha1 = \"dc7dedc2c2aa9faf59a55c622760a25cbefbe941\"\nuuid = \"34da2185-b29b-5c13-b0c7-acf172513d20\"\nversion = \"3.31.0\"\n\n[[CompilerSupportLibraries_jll]]\ndeps = [\"Artifacts\", \"Libdl\"]\nuuid = \"e66e0078-7015-5450-92f7-15fbd957f2ae\"\n\n[[Compose]]\ndeps = [\"Base64\", \"Colors\", \"DataStructures\", \"Dates\", \"IterTools\", \"JSON\", \"LinearAlgebra\", \"Measures\", \"Printf\", \"Random\", \"Requires\", \"Statistics\", \"UUIDs\"]\ngit-tree-sha1 = \"c6461fc7c35a4bb8d00905df7adafcff1fe3a6bc\"\nuuid = \"a81c6b42-2e10-5240-aca2-a61377ecd94b\"\nversion = \"0.9.2\"\n\n[[DataStructures]]\ndeps = [\"Compat\", \"InteractiveUtils\", \"OrderedCollections\"]\ngit-tree-sha1 = \"4437b64df1e0adccc3e5d1adbc3ac741095e4677\"\nuuid = \"864edb3b-99cc-5e75-8d2d-829cb0a9cfe8\"\nversion = \"0.18.9\"\n\n[[Dates]]\ndeps = [\"Printf\"]\nuuid = \"ade2ca70-3891-5945-98fb-dc099432e06a\"\n\n[[DelimitedFiles]]\ndeps = [\"Mmap\"]\nuuid = \"8bb1440f-4735-579b-a4ab-409b98df4dab\"\n\n[[Dierckx]]\ndeps = [\"Dierckx_jll\"]\ngit-tree-sha1 = \"5fefbe52e9a6e55b8f87cb89352d469bd3a3a090\"\nuuid = \"39dd38d3-220a-591b-8e3c-4c3a8c710a94\"\nversion = \"0.5.1\"\n\n[[Dierckx_jll]]\ndeps = [\"CompilerSupportLibraries_jll\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"a580560f526f6fc6973e8bad2b036514a4e3b013\"\nuuid = \"cd4c43a9-7502-52ba-aa6d-59fb2a88580b\"\nversion = \"0.0.1+0\"\n\n[[DiffResults]]\ndeps = [\"StaticArrays\"]\ngit-tree-sha1 = \"c18e98cba888c6c25d1c3b048e4b3380ca956805\"\nuuid = \"163ba53b-c6d8-5494-b064-1a9d43ac40c5\"\nversion = \"1.0.3\"\n\n[[DiffRules]]\ndeps = [\"NaNMath\", \"Random\", \"SpecialFunctions\"]\ngit-tree-sha1 = \"214c3fcac57755cfda163d91c58893a8723f93e9\"\nuuid = \"b552c78f-8df3-52c6-915a-8e097449b14b\"\nversion = \"1.0.2\"\n\n[[Distributed]]\ndeps = [\"Random\", \"Serialization\", \"Sockets\"]\nuuid = \"8ba89e20-285c-5b6f-9357-94700520ee1b\"\n\n[[DocStringExtensions]]\ndeps = [\"LibGit2\"]\ngit-tree-sha1 = \"a32185f5428d3986f47c2ab78b1f216d5e6cc96f\"\nuuid = \"ffbed154-4ef7-542d-bbb7-c09d3a79fcae\"\nversion = \"0.8.5\"\n\n[[Downloads]]\ndeps = [\"ArgTools\", \"LibCURL\", \"NetworkOptions\"]\nuuid = \"f43a241f-c20a-4ad4-852c-f6b1247861c6\"\n\n[[Expat_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"b3bfd02e98aedfa5cf885665493c5598c350cd2f\"\nuuid = \"2e619515-83b5-522b-bb60-26c02a35a201\"\nversion = \"2.2.10+0\"\n\n[[FileWatching]]\nuuid = \"7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee\"\n\n[[FixedPointNumbers]]\ndeps = [\"Statistics\"]\ngit-tree-sha1 = \"335bfdceacc84c5cdf16aadc768aa5ddfc5383cc\"\nuuid = \"53c48c17-4a7d-5ca2-90c5-79b7896eea93\"\nversion = \"0.8.4\"\n\n[[Fontconfig_jll]]\ndeps = [\"Artifacts\", \"Bzip2_jll\", \"Expat_jll\", \"FreeType2_jll\", \"JLLWrappers\", \"Libdl\", \"Libuuid_jll\", \"Pkg\", \"Zlib_jll\"]\ngit-tree-sha1 = \"21efd19106a55620a188615da6d3d06cd7f6ee03\"\nuuid = \"a3f928ae-7b40-5064-980b-68af3947d34b\"\nversion = \"2.13.93+0\"\n\n[[ForwardDiff]]\ndeps = [\"CommonSubexpressions\", \"DiffResults\", \"DiffRules\", \"LinearAlgebra\", \"NaNMath\", \"Printf\", \"Random\", \"SpecialFunctions\", \"StaticArrays\"]\ngit-tree-sha1 = \"e2af66012e08966366a43251e1fd421522908be6\"\nuuid = \"f6369f11-7733-5829-9624-2563aa707210\"\nversion = \"0.10.18\"\n\n[[FreeType2_jll]]\ndeps = [\"Artifacts\", \"Bzip2_jll\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Zlib_jll\"]\ngit-tree-sha1 = \"87eb71354d8ec1a96d4a7636bd57a7347dde3ef9\"\nuuid = \"d7e528f0-a631-5988-bf34-fe36492bcfd7\"\nversion = \"2.10.4+0\"\n\n[[Gettext_jll]]\ndeps = [\"Artifacts\", \"CompilerSupportLibraries_jll\", \"JLLWrappers\", \"Libdl\", \"Libiconv_jll\", \"Pkg\", \"XML2_jll\"]\ngit-tree-sha1 = \"9b02998aba7bf074d14de89f9d37ca24a1a0b046\"\nuuid = \"78b55507-aeef-58d4-861c-77aaff3498b1\"\nversion = \"0.21.0+0\"\n\n[[Glib_jll]]\ndeps = [\"Artifacts\", \"Gettext_jll\", \"JLLWrappers\", \"Libdl\", \"Libffi_jll\", \"Libiconv_jll\", \"Libmount_jll\", \"PCRE_jll\", \"Pkg\", \"Zlib_jll\"]\ngit-tree-sha1 = \"47ce50b742921377301e15005c96e979574e130b\"\nuuid = \"7746bdde-850d-59dc-9ae8-88ece973131d\"\nversion = \"2.68.1+0\"\n\n[[InteractiveUtils]]\ndeps = [\"Markdown\"]\nuuid = \"b77e0a4c-d291-57a0-90e8-8db25a27a240\"\n\n[[IterTools]]\ngit-tree-sha1 = \"05110a2ab1fc5f932622ffea2a003221f4782c18\"\nuuid = \"c8e1da08-722c-5040-9ed9-7db0dc04731e\"\nversion = \"1.3.0\"\n\n[[JLLWrappers]]\ndeps = [\"Preferences\"]\ngit-tree-sha1 = \"642a199af8b68253517b80bd3bfd17eb4e84df6e\"\nuuid = \"692b3bcd-3c85-4b1f-b108-f13ce0eb3210\"\nversion = \"1.3.0\"\n\n[[JSON]]\ndeps = [\"Dates\", \"Mmap\", \"Parsers\", \"Unicode\"]\ngit-tree-sha1 = \"81690084b6198a2e1da36fcfda16eeca9f9f24e4\"\nuuid = \"682c06a0-de6a-54ab-a142-c8b1cf79cde6\"\nversion = \"0.21.1\"\n\n[[JpegTurbo_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"d735490ac75c5cb9f1b00d8b5509c11984dc6943\"\nuuid = \"aacddb02-875f-59d6-b918-886e6ef4fbf8\"\nversion = \"2.1.0+0\"\n\n[[JuliaInterpreter]]\ndeps = [\"CodeTracking\", \"InteractiveUtils\", \"Random\", \"UUIDs\"]\ngit-tree-sha1 = \"31c2eee64c1eee6e8e3f30d5a03d4b5b7086ab29\"\nuuid = \"aa1ae85d-cabe-5617-a682-6adf51b2e16a\"\nversion = \"0.8.18\"\n\n[[LZO_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"e5b909bcf985c5e2605737d2ce278ed791b89be6\"\nuuid = \"dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac\"\nversion = \"2.10.1+0\"\n\n[[LaTeXStrings]]\ngit-tree-sha1 = \"c7f1c695e06c01b95a67f0cd1d34994f3e7db104\"\nuuid = \"b964fa9f-0449-5b57-a5c2-d3ea65f4040f\"\nversion = \"1.2.1\"\n\n[[LibCURL]]\ndeps = [\"LibCURL_jll\", \"MozillaCACerts_jll\"]\nuuid = \"b27032c2-a3e7-50c8-80cd-2d36dbcbfd21\"\n\n[[LibCURL_jll]]\ndeps = [\"Artifacts\", \"LibSSH2_jll\", \"Libdl\", \"MbedTLS_jll\", \"Zlib_jll\", \"nghttp2_jll\"]\nuuid = \"deac9b47-8bc7-5906-a0fe-35ac56dc84c0\"\n\n[[LibGit2]]\ndeps = [\"Base64\", \"NetworkOptions\", \"Printf\", \"SHA\"]\nuuid = \"76f85450-5226-5b5a-8eaa-529ad045b433\"\n\n[[LibSSH2_jll]]\ndeps = [\"Artifacts\", \"Libdl\", \"MbedTLS_jll\"]\nuuid = \"29816b5a-b9ab-546f-933c-edad1886dfa8\"\n\n[[Libdl]]\nuuid = \"8f399da3-3557-5675-b5ff-fb832c97cbdb\"\n\n[[Libffi_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"761a393aeccd6aa92ec3515e428c26bf99575b3b\"\nuuid = \"e9f186c6-92d2-5b65-8a66-fee21dc1b490\"\nversion = \"3.2.2+0\"\n\n[[Libgcrypt_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Libgpg_error_jll\", \"Pkg\"]\ngit-tree-sha1 = \"64613c82a59c120435c067c2b809fc61cf5166ae\"\nuuid = \"d4300ac3-e22c-5743-9152-c294e39db1e4\"\nversion = \"1.8.7+0\"\n\n[[Libgpg_error_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"c333716e46366857753e273ce6a69ee0945a6db9\"\nuuid = \"7add5ba3-2f88-524e-9cd5-f83b8a55f7b8\"\nversion = \"1.42.0+0\"\n\n[[Libiconv_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"42b62845d70a619f063a7da093d995ec8e15e778\"\nuuid = \"94ce4f54-9a6c-5748-9c1c-f9c7231a4531\"\nversion = \"1.16.1+1\"\n\n[[Libmount_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"9c30530bf0effd46e15e0fdcf2b8636e78cbbd73\"\nuuid = \"4b2f31a3-9ecc-558c-b454-b3730dcb73e9\"\nversion = \"2.35.0+0\"\n\n[[Libtiff_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"JpegTurbo_jll\", \"Libdl\", \"Pkg\", \"Zlib_jll\", \"Zstd_jll\"]\ngit-tree-sha1 = \"340e257aada13f95f98ee352d316c3bed37c8ab9\"\nuuid = \"89763e89-9b03-5906-acba-b20f662cd828\"\nversion = \"4.3.0+0\"\n\n[[Libuuid_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"7f3efec06033682db852f8b3bc3c1d2b0a0ab066\"\nuuid = \"38a345b3-de98-5d2b-a5d3-14cd9215e700\"\nversion = \"2.36.0+0\"\n\n[[LinearAlgebra]]\ndeps = [\"Libdl\"]\nuuid = \"37e2e46d-f89d-539d-b4ee-838fcccc9c8e\"\n\n[[LittleCMS_jll]]\ndeps = [\"JpegTurbo_jll\", \"Libdl\", \"Libtiff_jll\", \"Pkg\"]\ngit-tree-sha1 = \"e6ea89d915cdad8d264f7f9158c6664f879edcde\"\nuuid = \"d3a379c0-f9a3-5b72-a4c0-6bf4d2e8af0f\"\nversion = \"2.9.0+0\"\n\n[[LogExpFunctions]]\ndeps = [\"DocStringExtensions\", \"LinearAlgebra\"]\ngit-tree-sha1 = \"1ba664552f1ef15325e68dc4c05c3ef8c2d5d885\"\nuuid = \"2ab3a3ac-af41-5b50-aa03-7779005ae688\"\nversion = \"0.2.4\"\n\n[[LogarithmicNumbers]]\ndeps = [\"Random\", \"Requires\"]\ngit-tree-sha1 = \"d88b70111754e3660f80d3596a343ce42bf5ee84\"\nuuid = \"aa2f6b4e-9042-5d33-9679-40d3a6b85899\"\nversion = \"0.4.2\"\n\n[[Logging]]\nuuid = \"56ddb016-857b-54e1-b83d-db4d58db5568\"\n\n[[LoweredCodeUtils]]\ndeps = [\"JuliaInterpreter\"]\ngit-tree-sha1 = \"4bfb8b57df913f3b28a6bd3bdbebe9a50538e689\"\nuuid = \"6f1432cf-f94c-5a45-995e-cdbf5db27b0b\"\nversion = \"2.1.0\"\n\n[[MacroTools]]\ndeps = [\"Markdown\", \"Random\"]\ngit-tree-sha1 = \"6a8a2a625ab0dea913aba95c11370589e0239ff0\"\nuuid = \"1914dd2f-81c6-5fcd-8719-6d5c9610ff09\"\nversion = \"0.5.6\"\n\n[[Markdown]]\ndeps = [\"Base64\"]\nuuid = \"d6f4376e-aef5-505a-96c1-9c027394607a\"\n\n[[MatchCore]]\ngit-tree-sha1 = \"90af9fe333f8c9851f952dfa7f335185c94567c0\"\nuuid = \"5dd3f0b1-72a9-48ad-ae6e-79f673da005f\"\nversion = \"0.1.1\"\n\n[[MbedTLS_jll]]\ndeps = [\"Artifacts\", \"Libdl\"]\nuuid = \"c8ffd9c3-330d-5841-b78e-0817d7145fa1\"\n\n[[Measures]]\ngit-tree-sha1 = \"e498ddeee6f9fdb4551ce855a46f54dbd900245f\"\nuuid = \"442fdcdd-2543-5da2-b0f3-8c86c306513e\"\nversion = \"0.3.1\"\n\n[[Mmap]]\nuuid = \"a63ad114-7e13-5084-954f-fe012c677804\"\n\n[[MozillaCACerts_jll]]\nuuid = \"14a3606d-f60d-562e-9121-12d972cd8159\"\n\n[[NaNMath]]\ngit-tree-sha1 = \"bfe47e760d60b82b66b61d2d44128b62e3a369fb\"\nuuid = \"77ba4419-2d1f-58cd-9bb1-8ffee604a2e3\"\nversion = \"0.3.5\"\n\n[[NetworkOptions]]\nuuid = \"ca575930-c2e3-43a9-ace4-1e988b2c1908\"\n\n[[NiLang]]\ndeps = [\"FixedPointNumbers\", \"LinearAlgebra\", \"LogarithmicNumbers\", \"MatchCore\", \"NiLangCore\", \"Reexport\", \"SparseArrays\", \"TupleTools\"]\ngit-tree-sha1 = \"3fe439482d8c08a15f929ae7278a6c7f737672d5\"\nuuid = \"ab4ef3a6-0b42-11ea-31f6-e34652774712\"\nversion = \"0.9.1\"\n\n[[NiLangCore]]\ndeps = [\"MatchCore\", \"TupleTools\"]\ngit-tree-sha1 = \"239f97ea947531cfe7a596746e31c8429c7169b9\"\nuuid = \"575d3204-02a4-11ea-3f62-238caa8bf11e\"\nversion = \"0.10.3\"\n\n[[OpenJpeg_jll]]\ndeps = [\"Libdl\", \"Libtiff_jll\", \"LittleCMS_jll\", \"Pkg\", \"libpng_jll\"]\ngit-tree-sha1 = \"e330ffff1c6a593fa44cc40c29900bee82026406\"\nuuid = \"643b3616-a352-519d-856d-80112ee9badc\"\nversion = \"2.3.1+0\"\n\n[[OpenSpecFun_jll]]\ndeps = [\"Artifacts\", \"CompilerSupportLibraries_jll\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"13652491f6856acfd2db29360e1bbcd4565d04f1\"\nuuid = \"efe28fd5-8261-553b-a9e1-b2916fc3738e\"\nversion = \"0.5.5+0\"\n\n[[OrderedCollections]]\ngit-tree-sha1 = \"85f8e6578bf1f9ee0d11e7bb1b1456435479d47c\"\nuuid = \"bac558e1-5e72-5ebc-8fee-abe8a469f55d\"\nversion = \"1.4.1\"\n\n[[PCRE_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"b2a7af664e098055a7529ad1a900ded962bca488\"\nuuid = \"2f80f16e-611a-54ab-bc61-aa92de5b98fc\"\nversion = \"8.44.0+0\"\n\n[[Parsers]]\ndeps = [\"Dates\"]\ngit-tree-sha1 = \"c8abc88faa3f7a3950832ac5d6e690881590d6dc\"\nuuid = \"69de0a69-1ddd-5017-9359-2bf0b02dc9f0\"\nversion = \"1.1.0\"\n\n[[Pixman_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"b4f5d02549a10e20780a24fce72bea96b6329e29\"\nuuid = \"30392449-352a-5448-841d-b1acce4e97dc\"\nversion = \"0.40.1+0\"\n\n[[Pkg]]\ndeps = [\"Artifacts\", \"Dates\", \"Downloads\", \"LibGit2\", \"Libdl\", \"Logging\", \"Markdown\", \"Printf\", \"REPL\", \"Random\", \"SHA\", \"Serialization\", \"TOML\", \"Tar\", \"UUIDs\", \"p7zip_jll\"]\nuuid = \"44cfe95a-1eb2-52ea-b672-e2afdf69b78f\"\n\n[[PlutoUI]]\ndeps = [\"Base64\", \"Dates\", \"InteractiveUtils\", \"JSON\", \"Logging\", \"Markdown\", \"Random\", \"Reexport\", \"Suppressor\"]\ngit-tree-sha1 = \"44e225d5837e2a2345e69a1d1e01ac2443ff9fcb\"\nuuid = \"7f904dfe-b85e-4ff6-b463-dae2292396a8\"\nversion = \"0.7.9\"\n\n[[Poppler_jll]]\ndeps = [\"Artifacts\", \"Cairo_jll\", \"Fontconfig_jll\", \"Glib_jll\", \"JLLWrappers\", \"JpegTurbo_jll\", \"Libdl\", \"Libtiff_jll\", \"OpenJpeg_jll\", \"Pkg\", \"libpng_jll\"]\ngit-tree-sha1 = \"e11443687ac151ac6ef6699eb75f964bed8e1faa\"\nuuid = \"9c32591e-4766-534b-9725-b71a8799265b\"\nversion = \"0.87.0+2\"\n\n[[Preferences]]\ndeps = [\"TOML\"]\ngit-tree-sha1 = \"00cfd92944ca9c760982747e9a1d0d5d86ab1e5a\"\nuuid = \"21216c6a-2e73-6563-6e65-726566657250\"\nversion = \"1.2.2\"\n\n[[Printf]]\ndeps = [\"Unicode\"]\nuuid = \"de0858da-6303-5e67-8744-51eddeeeb8d7\"\n\n[[REPL]]\ndeps = [\"InteractiveUtils\", \"Markdown\", \"Sockets\", \"Unicode\"]\nuuid = \"3fa0cd96-eef1-5676-8a61-b3b8758bbffb\"\n\n[[Random]]\ndeps = [\"Serialization\"]\nuuid = \"9a3f8284-a2c9-5f02-9a11-845980a1fd5c\"\n\n[[Reexport]]\ngit-tree-sha1 = \"5f6c21241f0f655da3952fd60aa18477cf96c220\"\nuuid = \"189a3867-3050-52da-a836-e630ba90ab69\"\nversion = \"1.1.0\"\n\n[[Requires]]\ndeps = [\"UUIDs\"]\ngit-tree-sha1 = \"4036a3bd08ac7e968e27c203d45f5fff15020621\"\nuuid = \"ae029012-a4dd-5104-9daa-d747884805df\"\nversion = \"1.1.3\"\n\n[[Revise]]\ndeps = [\"CodeTracking\", \"Distributed\", \"FileWatching\", \"JuliaInterpreter\", \"LibGit2\", \"LoweredCodeUtils\", \"OrderedCollections\", \"Pkg\", \"REPL\", \"Requires\", \"UUIDs\", \"Unicode\"]\ngit-tree-sha1 = \"410bbe13d9a7816e862ed72ac119bda7fb988c08\"\nuuid = \"295af30f-e4ad-537b-8983-00126c2a3abe\"\nversion = \"3.1.17\"\n\n[[SHA]]\nuuid = \"ea8e919c-243c-51af-8825-aaa63cd721ce\"\n\n[[Serialization]]\nuuid = \"9e88b42a-f829-5b0c-bbe9-9e923198166b\"\n\n[[SharedArrays]]\ndeps = [\"Distributed\", \"Mmap\", \"Random\", \"Serialization\"]\nuuid = \"1a1011a3-84de-559e-8e89-a11a2f7dc383\"\n\n[[Sockets]]\nuuid = \"6462fe0b-24de-5631-8697-dd941f90decc\"\n\n[[SparseArrays]]\ndeps = [\"LinearAlgebra\", \"Random\"]\nuuid = \"2f01184e-e22b-5df5-ae63-d93ebab69eaf\"\n\n[[SpecialFunctions]]\ndeps = [\"ChainRulesCore\", \"LogExpFunctions\", \"OpenSpecFun_jll\"]\ngit-tree-sha1 = \"a50550fa3164a8c46747e62063b4d774ac1bcf49\"\nuuid = \"276daf66-3868-5448-9aa4-cd146d93841b\"\nversion = \"1.5.1\"\n\n[[StaticArrays]]\ndeps = [\"LinearAlgebra\", \"Random\", \"Statistics\"]\ngit-tree-sha1 = \"745914ebcd610da69f3cb6bf76cb7bb83dcb8c9a\"\nuuid = \"90137ffa-7385-5640-81b9-e52037218182\"\nversion = \"1.2.4\"\n\n[[Statistics]]\ndeps = [\"LinearAlgebra\", \"SparseArrays\"]\nuuid = \"10745b16-79ce-11e8-11f9-7d13ad32a3b2\"\n\n[[Suppressor]]\ngit-tree-sha1 = \"a819d77f31f83e5792a76081eee1ea6342ab8787\"\nuuid = \"fd094767-a336-5f1f-9728-57cf17d0bbfb\"\nversion = \"0.2.0\"\n\n[[TOML]]\ndeps = [\"Dates\"]\nuuid = \"fa267f1f-6049-4f14-aa54-33bafae1ed76\"\n\n[[Tar]]\ndeps = [\"ArgTools\", \"SHA\"]\nuuid = \"a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e\"\n\n[[Test]]\ndeps = [\"InteractiveUtils\", \"Logging\", \"Random\", \"Serialization\"]\nuuid = \"8dfed614-e22c-5e08-85e1-65c5234f0b40\"\n\n[[TikzPictures]]\ndeps = [\"LaTeXStrings\", \"Poppler_jll\", \"Requires\"]\ngit-tree-sha1 = \"06b36e2baa9b97814ef1993207b71e2e23e9efb5\"\nuuid = \"37f6aa50-8035-52d0-81c2-5a1d08754b2d\"\nversion = \"3.3.3\"\n\n[[TreeverseAlgorithm]]\ndeps = [\"Requires\"]\ngit-tree-sha1 = \"4292bc608573c2047fd12b0a611787e77f5595ba\"\nuuid = \"e1c63c57-2fea-45bf-a8bf-df3ea6afb545\"\nversion = \"0.1.0\"\n\n[[TupleTools]]\ngit-tree-sha1 = \"62a7a6cd5a608ff71cecfdb612e67a0897836069\"\nuuid = \"9d95972d-f1c8-5527-a6e0-b4b365fa01f6\"\nversion = \"1.2.0\"\n\n[[UUIDs]]\ndeps = [\"Random\", \"SHA\"]\nuuid = \"cf7118a7-6976-5b1a-9a39-7adc72f591a4\"\n\n[[Unicode]]\nuuid = \"4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5\"\n\n[[Viznet]]\ndeps = [\"Compose\", \"Dierckx\"]\ngit-tree-sha1 = \"7a022ae6ac8b153d47617ed8c196ce60645689f1\"\nuuid = \"52a3aca4-6234-47fd-b74a-806bdf78ede9\"\nversion = \"0.3.3\"\n\n[[XML2_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Libiconv_jll\", \"Pkg\", \"Zlib_jll\"]\ngit-tree-sha1 = \"1acf5bdf07aa0907e0a37d3718bb88d4b687b74a\"\nuuid = \"02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a\"\nversion = \"2.9.12+0\"\n\n[[XSLT_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Libgcrypt_jll\", \"Libgpg_error_jll\", \"Libiconv_jll\", \"Pkg\", \"XML2_jll\", \"Zlib_jll\"]\ngit-tree-sha1 = \"91844873c4085240b95e795f692c4cec4d805f8a\"\nuuid = \"aed1982a-8fda-507f-9586-7b0439959a61\"\nversion = \"1.1.34+0\"\n\n[[Xorg_libX11_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libxcb_jll\", \"Xorg_xtrans_jll\"]\ngit-tree-sha1 = \"5be649d550f3f4b95308bf0183b82e2582876527\"\nuuid = \"4f6342f7-b3d2-589e-9d20-edeb45f2b2bc\"\nversion = \"1.6.9+4\"\n\n[[Xorg_libXau_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"4e490d5c960c314f33885790ed410ff3a94ce67e\"\nuuid = \"0c0b7dd1-d40b-584c-a123-a41640f87eec\"\nversion = \"1.0.9+4\"\n\n[[Xorg_libXdmcp_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"4fe47bd2247248125c428978740e18a681372dd4\"\nuuid = \"a3789734-cfe1-5b06-b2d0-1dd0d9d62d05\"\nversion = \"1.1.3+4\"\n\n[[Xorg_libXext_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libX11_jll\"]\ngit-tree-sha1 = \"b7c0aa8c376b31e4852b360222848637f481f8c3\"\nuuid = \"1082639a-0dae-5f34-9b06-72781eeb8cb3\"\nversion = \"1.3.4+4\"\n\n[[Xorg_libXrender_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Xorg_libX11_jll\"]\ngit-tree-sha1 = \"19560f30fd49f4d4efbe7002a1037f8c43d43b96\"\nuuid = \"ea2f1a96-1ddc-540d-b46f-429655e07cfa\"\nversion = \"0.9.10+4\"\n\n[[Xorg_libpthread_stubs_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"6783737e45d3c59a4a4c4091f5f88cdcf0908cbb\"\nuuid = \"14d82f49-176c-5ed1-bb49-ad3f5cbd8c74\"\nversion = \"0.1.0+3\"\n\n[[Xorg_libxcb_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"XSLT_jll\", \"Xorg_libXau_jll\", \"Xorg_libXdmcp_jll\", \"Xorg_libpthread_stubs_jll\"]\ngit-tree-sha1 = \"daf17f441228e7a3833846cd048892861cff16d6\"\nuuid = \"c7cfdc94-dc32-55de-ac96-5a1b8d977c5b\"\nversion = \"1.13.0+3\"\n\n[[Xorg_xtrans_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"79c31e7844f6ecf779705fbc12146eb190b7d845\"\nuuid = \"c5fb5394-a638-5e4d-96e5-b29de1b5cf10\"\nversion = \"1.4.0+3\"\n\n[[Zlib_jll]]\ndeps = [\"Libdl\"]\nuuid = \"83775a58-1f1d-513f-b197-d71354ab007a\"\n\n[[Zstd_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\"]\ngit-tree-sha1 = \"cc4bf3fdde8b7e3e9fa0351bdeedba1cf3b7f6e6\"\nuuid = \"3161d3a3-bdf6-5164-811a-617609db77b4\"\nversion = \"1.5.0+0\"\n\n[[libpng_jll]]\ndeps = [\"Artifacts\", \"JLLWrappers\", \"Libdl\", \"Pkg\", \"Zlib_jll\"]\ngit-tree-sha1 = \"94d180a6d2b5e55e447e2d27a29ed04fe79eb30c\"\nuuid = \"b53b4c65-9356-5827-b1ea-8c7a1a84506f\"\nversion = \"1.6.38+0\"\n\n[[nghttp2_jll]]\ndeps = [\"Artifacts\", \"Libdl\"]\nuuid = \"8e850ede-7688-5339-a07c-302acd2aaf8d\"\n\n[[p7zip_jll]]\ndeps = [\"Artifacts\", \"Libdl\"]\nuuid = \"3f19e933-33d8-53b3-aaab-bd5110c3b7a0\"\n\"\"\"\n\n# ╔═╡ Cell order:\n# ╟─e20e2d2e-4b28-4e32-8d80-ce029928a094\n# ╟─f3e235e7-76b9-4c39-bc70-038539838ff4\n# ╟─8308df59-3faa-4abf-8f05-119bbae48f64\n# ╟─a3532a83-9fd3-4d24-b1bb-b52457317e51\n# ╟─15657e4b-848e-43ad-a99f-37143d11705e\n# ╟─34a6b7f4-7d72-485d-86dc-f4b1ba6174eb\n# ╟─67d1b500-964e-4668-a7d0-ed93886446ca\n# ╟─673992ed-6963-400a-a69b-d65d26c4f443\n# ╟─6078758e-b392-4bdb-a1e7-44b135ce900e\n# ╠═41e06e2a-b482-4e0f-8569-fee2ffd8aaaf\n# ╟─dcf53d46-e259-4101-8530-9621094ee586\n# ╟─6a88d26c-c895-4852-ab4f-37297b848731\n# ╟─f9675365-36aa-430c-b747-3bc4f602e6fb\n# ╟─46eb4ba9-dce6-4711-9c4d-3f16de6240de\n# ╟─046f7559-4af9-4982-b5c3-335add0911d7\n# ╟─0a039bfa-571e-4fad-b73c-1324d08777fc\n# ╟─3f1e4d7a-32a7-4c7e-92dd-465bac925e63\n# ╟─f68bcfb6-97ce-48d1-b0b8-e8466d4ac879\n# ╟─3d4ba750-8d62-48ac-bf96-691397689ddc\n# ╟─f4cb9212-181f-4338-b858-1d99c7f415e9\n# ╟─31bde262-6352-4be0-b5cc-1781e3df2268\n# ╟─83ff3fc3-bcd8-4235-a42f-1d75c7d6aa5b\n# ╟─b308e270-6b40-4946-ac92-c705823f2c1e\n# ╟─e483b3d4-d01c-4a98-8e68-e8120a7d95a7\n# ╟─20c34526-c7c4-11eb-21fa-d706fd684a4c\n# ╟─3f96abdf-fb5f-4d79-a288-e20b8c1f55d1\n# ╟─5d51231a-8bf0-4414-9a39-cea264df84f2\n# ╠═3b0fd2b5-5c6d-4d56-9e48-cda1493b4c72\n# ╟─e10e0be8-b26e-4719-92dc-8ca46af0b4b5\n# ╟─b1a9946b-82b4-4954-8bb9-5df035eaefe4\n# ╠═00342a51-36d8-4fdd-aab7-ee02e2122c49\n# ╠═40c1c48d-0e5a-4a47-b7e4-8f7666281249\n# ╠═59df1f80-9be1-4b26-b263-ca0c7a0b9ab7\n# ╟─8320d326-c1ab-4807-befb-13dda3480bf5\n# ╟─93238608-3b86-49f1-ad60-9360e12cff1c\n# ╟─f657c3fb-e140-4c76-8065-54f1cb6d05eb\n# ╠═629a2549-745c-48a2-9bbc-a8f5fb046d11\n# ╟─640e0029-7931-4afd-bdf9-fed317efbd8e\n# ╟─6930345b-6e93-4b35-8d4f-91ad49141fa1\n# ╠═090522bf-0ff2-4022-8460-aec6e37e936a\n# ╠═88c30609-2f42-405e-a14c-dfab44aef23b\n# ╟─2dc665a9-b131-4fef-acde-db346eb0f48b\n# ╟─4e479f48-42cd-476d-8604-08ecbb503a90\n# ╟─dc41e99a-f598-4bf6-9f76-ecdb04f5f40c\n# ╟─97e0bae1-69ac-4cbf-b9d9-6b38180edd78\n# ╟─355ba831-6be0-456a-8f94-36acd2365f17\n# ╠═003c3e68-600e-4688-832b-5e061572b128\n# ╠═1fb196f9-0f0a-42dc-b094-077cdf18d13d\n# ╠═aa9a679e-63bc-4951-b6f4-65316e212bc8\n# ╠═e6fd3c2d-cadd-40d7-a575-b1e68c45ee13\n# ╟─02b7e1b4-4622-4e68-966e-ff79817557d1\n# ╟─364fd613-0ebd-4b45-a3fd-f9baa8c487e3\n# ╟─75d8283a-b331-4648-84a8-489e168e33f9\n# ╟─2207c2fb-4a52-4766-8dd3-03872744aa74\n# ╠═288331c3-2dfb-4941-985f-554be409c0ab\n# ╠═12a30359-d6ff-4113-bab5-b198e908cf1a\n# ╠═23ea88b4-6b89-462a-92da-0e8bdf5c73b5\n# ╟─4bfdabd6-b7ff-40fb-b567-52910acb5a07\n# ╟─2603147b-e7a2-4cae-b88f-2cfebe16bacb\n# ╟─12d49e2e-cc6e-48d6-b11a-e7c311453bfc\n# ╟─10312dc7-e861-4c89-b2fd-672cfe8850bf\n# ╟─03f58f2a-24b6-4235-9102-71a19b9679ac\n# ╠═22a5853e-4f9a-4da0-bc03-84a6b0061cfe\n# ╠═1ecdc4d2-b3ca-4f5c-a454-0f0bc51b6ec2\n# ╟─927ea209-2ccd-48d5-b69e-0a3c735bb496\n# ╟─962b204c-8195-4938-944c-b7c4a52e70bd\n# ╠═af58a0f8-e3fd-465f-b1ae-6fbd94123c91\n# ╠═4f993061-c6c3-4acb-aef3-8453e7b83997\n# ╠═6e5cf9bb-7cab-4da1-8831-541e0ee3bde8\n# ╠═320e4114-f0da-4106-90ab-a9f7b0ef0099\n# ╟─2d944e8d-e19d-48be-ab7f-c3e54e9f43ef\n# ╟─4f53fa8e-ea9b-461f-8199-7bbe2a3ef544\n# ╟─56ea4f5f-ea88-46d6-beed-b9a7afed315d\n# ╟─124a9ecd-0bda-4823-9507-92efcf449d9c\n# ╟─6819498c-7a46-48fa-9eec-38341dca72f9\n# ╟─400f79cf-9260-4195-9582-0e8c486ddb5a\n# ╟─2dad3acd-332c-46b9-8f84-21c076bdef41\n# ╟─a674bde5-70e1-4d21-aedf-8977f8039c36\n# ╟─bf696784-23d4-42b2-8ee1-bfec11ff8d78\n# ╠═a7810352-7967-460d-abd7-361a324c20a9\n# ╠═9193fcbb-ec4f-41b4-8fca-be8e183dea31\n# ╠═a56445b5-e530-4035-9ac6-a2d196a6276a\n# ╠═3fadf1d4-8fa2-4c02-aa21-b7969b465536\n# ╠═12e933ca-13f3-414c-926a-f1bb1bbe66cf\n# ╟─4403c183-5eeb-4fd0-87a4-4ad29e1f4dc2\n# ╠═4c3f9b91-4f27-4fb8-a9db-1ddbfd62dbdd\n# ╠═af71e2d7-a600-46fd-9a46-1b9d4607f06d\n# ╟─1d3c7324-6828-47aa-b30e-bcac0e052213\n# ╠═2993fe1f-042b-4c85-85b8-bcc7ed449a54\n# ╟─048d482e-3e5b-496f-95d7-17589a5f6f11\n# ╟─e22bbe66-56f5-40be-b464-1f8651e6a6ac\n# ╟─28767d73-47a8-4f4b-b3dd-146c4ae3e038\n# ╟─ea907d51-d4a9-48ca-90a5-bd91309ccfad\n# ╟─3aa99be5-6747-4163-9bb6-ed8cd5ce19f6\n# ╟─2f0263f5-2ace-4d4b-8d71-cee26c03122e\n# ╟─03a21468-c2fe-4df7-ae8a-38b28a0efe2f\n# ╟─f326d8e5-7117-4eed-b30d-0f64e5974426\n# ╟─b9af3db6-5725-4190-b96c-f3fa41f07c93\n# ╟─47cf7e85-c49f-4618-9689-e0de789625f6\n# ╟─2fcf48fd-bedc-41d9-a433-68efd5dd0d20\n# ╟─5060f5bb-8430-42aa-b61d-c88249edb323\n# ╟─55dd53ed-6420-4426-95ae-feb47bf50f22\n# ╟─c62a9f94-457f-496b-bee0-bb0db02aca5d\n# ╟─bccadcb5-6d9f-4a70-b7c4-74e0e3d5f8c8\n# ╟─b308ecb0-070e-4ad8-8009-dc60e75bbe01\n# ╟─9123e669-19c7-47c1-a924-0c618b4a9c1f\n# ╟─83d0c5fc-9cb7-4e37-bfdb-ed630b94d9b4\n# ╠═c6bd40af-50ed-4cee-8043-60b2bac05058\n# ╟─3d2feca5-43d3-4a46-ba1c-849c5ceeb676\n# ╟─7ca616d1-caed-40cf-b768-b07af81a654d\n# ╠═ddc662d7-6901-4885-9d2c-1876a2c9d2ff\n# ╟─a54d5fc4-643c-47d2-be97-4626a060c9b4\n# ╟─8b556fbe-275f-4e7b-94a0-7434ce81ad8b\n# ╟─8e9c55f6-8175-4cb4-8798-91107c4d16ee\n# ╠═873ef2c2-653e-425e-9732-b1ed19f7a0b7\n# ╟─972d889c-c48c-470a-b710-aba9ecaacdaa\n# ╟─54b3a283-7d42-431f-9ecb-48f37198409a\n# ╠═590f56f7-5654-4493-9103-02a0fce6e945\n# ╟─11964529-8e88-4743-a725-57fc5c525649\n# ╟─45448141-214d-4e08-978e-5d1d25f763cd\n# ╟─dfe3cc09-6e27-4166-9a4e-2dd22f1e08a2\n# ╟─3d07e6d1-2964-4718-b7bc-114d82389aa4\n# ╟─c00cd648-d685-4a17-9ddf-39c06dd5f066\n# ╟─5390c3ba-1507-4db9-9972-8295cbe493bc\n# ╟─33a0bda1-c943-4792-bc3d-fbf1adf16d0a\n# ╟─a2e81e79-3f85-4eef-8639-f455f9165a25\n# ╟─0a04c470-8f5a-4b56-b2e2-5b32e3b944f3\n# ╟─dced36eb-3b84-4d08-8268-0ffb831e39b5\n# ╠═141e21c0-1bdf-4e6b-b76d-129567a1180f\n# ╠═0dc268fa-2b71-4915-b73b-3f1de9fbc157\n# ╠═871f62a3-2c9c-445e-b1dc-4cba363fa604\n# ╠═6a50f7ac-b5a5-444d-9042-81b82cb66aec\n# ╠═bace7924-3aff-463f-9351-cc59191b469a\n# ╠═923a5e2a-cc91-4404-913a-6e8012fa5834\n# ╟─b12bcfbb-c7f2-4dc9-821a-e59365bf6fa4\n# ╠═12d52dc8-da0b-46dc-9251-eb0cc9a39a7e\n# ╠═ac35b26c-0585-4d2a-8fbf-bda9b141d6af\n# ╠═f8616bf4-02e6-4d66-a0f6-d35db701e82c\n# ╠═4e273d1f-a00e-49ca-9f8d-a0f6930550fb\n# ╟─bee4ab06-4c60-4bec-89d0-b3c9a512f7a6\n# ╟─717f51fb-bd0a-4bb4-a5fc-1f1cf5d56ed8\n# ╟─db6b6481-7e67-4418-b907-13b38c77bac7\n# ╠═d4726239-81af-4792-8472-c680508449c6\n# ╠═a0be4807-52ed-4626-8009-97a79e36e2f1\n# ╠═52dec342-d3de-4a88-8acb-0aa186bcc086\n# ╠═280c9363-9b49-44f1-a240-b7f205ffc56b\n# ╟─0b3735c2-695c-4225-843e-16ca17aac0eb\n# ╟─d7942b37-f821-494a-8f18-5f267aa3457a\n# ╟─865f049f-54d0-4d21-860e-062262edcb58\n# ╟─c3b730a4-d5b4-471e-bd06-30ace6e8b8fe\n# ╟─98f42f60-7870-4813-b0c1-728285c25f01\n# ╟─c3c63865-f538-4d93-bef3-6b9c69cb177f\n# ╠═66225c05-165e-4051-bb5e-4cfba579fd5b\n# ╠═14291e8d-001f-4e26-b094-addb970cf530\n# ╠═bf670e51-06f8-4094-949c-ca2d02fd0d01\n# ╠═416e53e4-6123-430f-b433-4334b7e85298\n# ╠═edd5f8df-9abd-4254-abf6-33ae31d88a8d\n# ╠═c0ff9103-2aa8-45fd-bea5-1903c53196de\n# ╠═e17e70cb-2d82-4a08-9f6e-e50dcaa325e3\n# ╠═357fd442-6e6e-4b14-b2d0-efd3fa775d0b\n# ╟─d2001eb2-45cb-4f07-aa99-dd84996359b7\n# ╠═1bca53ef-3438-4db5-9900-9fee71936a62\n# ╠═50f7070d-9f6f-4025-9be3-13812c3000eb\n# ╟─fa3b6d6a-a55d-4097-8ad2-7dafb5f01d8c\n# ╟─dabb4656-4aed-4168-8659-c0472528c41d\n# ╟─0367be06-4185-4add-a04b-f696c5a43638\n# ╟─c79e7651-975b-407c-8c8c-d0c5653ec570\n# ╟─3fed55d7-dbed-4fc9-8410-2633d5200db6\n# ╟─663180d2-8fe1-4996-a194-d38120ae05fd\n# ╟─39884e8a-bc83-4ff4-85bc-cfaccb4674f2\n# ╟─c24e7391-a187-4a7e-aab7-93027f7db965\n# ╠═12734842-d66f-4bfc-a9ad-01adeb2450e0\n# ╟─00000000-0000-0000-0000-000000000001\n# ╟─00000000-0000-0000-0000-000000000002\n" }, { "path": "src/NiLang.jl", "content": "module NiLang\n\nusing Reexport\n@reexport using NiLangCore\nimport NiLangCore: invtype\n\nusing FixedPointNumbers: Q20f43, Fixed\nimport NiLangCore: empty_global_stacks!, loaddata\nexport Fixed43\nconst Fixed43 = Q20f43\n\ninclude(\"utils.jl\")\ninclude(\"wrappers.jl\")\ninclude(\"vars.jl\")\ninclude(\"instructs.jl\")\ninclude(\"ulog.jl\")\ninclude(\"complex.jl\")\ninclude(\"autobcast.jl\")\ninclude(\"macros.jl\")\n\ninclude(\"autodiff/autodiff.jl\")\n\ninclude(\"stdlib/stdlib.jl\")\n\ninclude(\"deprecations.jl\")\n\nexport AD\n\nproject_relative_path(xs...) = normpath(joinpath(dirname(dirname(pathof(@__MODULE__))), xs...))\n\nend # module\n" }, { "path": "src/autobcast.jl", "content": "export AutoBcast\n\n\"\"\"\n AutoBcast{T,N} <: IWrapper{T}\n\nA vectorized variable.\n\"\"\"\nstruct AutoBcast{T,N} <: IWrapper{T} x::Vector{T} end\nAutoBcast(x::Vector{T}) where {T} = AutoBcast{T, length(x)}(x)\nAutoBcast(x::AutoBcast{T,N}) where {T,N} = x # to avoid ambiguity error\nAutoBcast{T,N}(x::AutoBcast{T,N}) where {T,N} = x\nvalue(x::AutoBcast) = x.x\nNiLangCore.chfield(x::AutoBcast, ::typeof(value), xval) = chfield(x, Val(:x), xval)\nBase.zero(x::AutoBcast) = AutoBcast(zero(x.x))\nBase.zero(::Type{AutoBcast{T,N}}) where {T,N} = AutoBcast{T,N}(zeros(T, N))\nBase.length(ab::AutoBcast{T,N}) where {T, N} = N\n\nfor F1 in [:(Base.:-), :INC, :FLIP, :DEC]\n @eval function $F1(a!::AutoBcast)\n @instr @invcheckoff @inbounds for i=1:length(a!)\n $F1(a!.x[i])\n end\n a!\n end\nend\n\nfor F2 in [:SWAP, :((inf::PlusEq)), :((inf::MinusEq)), :((inf::XorEq))]\n F2 != :SWAP && @eval function $F2(a::AutoBcast, b::Real)\n @instr @invcheckoff @inbounds for i=1:length(a)\n $F2(a.x[i], b)\n end\n a, b\n end\n @eval function $F2(a::AutoBcast, b::AutoBcast)\n @instr @invcheckoff @inbounds for i=1:length(a)\n $F2(a.x[i], b.x[i])\n end\n a, b\n end\nend\n\nfor F3 in [:ROT, :IROT, :((inf::PlusEq)), :((inf::MinusEq)), :((inf::XorEq))]\n if !(F3 in [:ROT, :IROT])\n @eval function $F3(a::AutoBcast, b::Real, c::Real)\n @instr @invcheckoff @inbounds for i=1:length(a)\n $F3(a.x[i], b, c)\n end\n a, b, c\n end\n @eval function $F3(a::AutoBcast, b::Real, c::AutoBcast)\n @instr @invcheckoff for i=1:length(a)\n $F3(a.x[i], b, c.x[i])\n end\n a, b, c\n end\n end\n @eval function $F3(a::AutoBcast, b::AutoBcast, c::Real)\n @instr @invcheckoff @inbounds for i=1:length(a)\n $F3(a.x[i], b.x[i], c)\n end\n a, b, c\n end\n @eval function $F3(a::AutoBcast, b::AutoBcast, c::AutoBcast)\n @instr @invcheckoff @inbounds for i=1:length(a)\n $F3(a.x[i], b.x[i], c.x[i])\n end\n a, b, c\n end\nend\n\n(f::PlusEq{typeof(identity)})(x::T, y::T) where T<:AutoBcast = invoke(f, Tuple{T,T} where T, x, y)\n(f::MinusEq{typeof(identity)})(x::T, y::T) where T<:AutoBcast = invoke(f, Tuple{T,T} where T, x, y)" }, { "path": "src/autodiff/autodiff.jl", "content": "module AD\n\nusing ..NiLang\nusing NiLangCore\nusing MLStyle, TupleTools\n\nimport ..NiLang: ROT, IROT, SWAP,\n chfield, value, NoGrad, INC, DEC, HADAMARD,\n AddConst, SubConst, NEG, INV\nusing NiLangCore: default_constructor\n\nexport GVar, grad, Loss, NoGrad, @nograd\n\ninclude(\"vars.jl\")\ninclude(\"stack.jl\")\ninclude(\"gradfunc.jl\")\ninclude(\"checks.jl\")\n\ninclude(\"instructs.jl\")\ninclude(\"ulog.jl\")\ninclude(\"jacobian.jl\")\ninclude(\"hessian_backback.jl\")\ninclude(\"complex.jl\")\n\nend\n" }, { "path": "src/autodiff/checks.jl", "content": "export check_grad, nparams\nexport gradient_numeric\n\nusing FixedPointNumbers: Fixed\n\n@nospecialize\nisvar(x) = nparams(x) != 0\n\nnparams(model) = nparams(NiLangCore.type2tuple(model))\nnparams(x::AbstractArray{<:AbstractFloat}) = length(x)\nnparams(x::AbstractArray{<:GVar}) = length(x)\nnparams(x::AbstractArray) = sum(nparams, x)\nnparams(x::Fixed) = 1\nfunction nparams(x::Union{Tuple,NamedTuple})\n res = 0\n for xi in x\n res += nparams(xi)\n end\n res\nend\nnparams(x::AbstractFloat) = 1\nnparams(x::GVar) = 1\n\nfunction tset(vfunc::Function, tp::Tuple, iloss)\n map(i->i===iloss ? vfunc(tp[i]) : tp[i], (1:length(tp)...,))\nend\nfunction tset(value, tp::Tuple, iloss)\n map(i->i===iloss ? value : tp[i], (1:length(tp)...,))\nend\n\nfunction update_var(args, iarg, i::Int, val)\n args[iarg][i] += val\n args\nend\n\nfunction update_var(args, iarg, ::Nothing, val)\n tset(x->chfield(x, value, value(x) + val), args, iarg)\nend\n\nfunction ng_single(::Type{T}, f, args, kwargs, iarg, i, iloss, δ) where T\n args = update_var(args, iarg, i, T(δ/2))\n @instr f(args...; kwargs...)\n pos = value(args[iloss])\n @instr (~f)(args...; kwargs...)\n args = update_var(args, iarg, i, -T(δ))\n @instr f(args...; kwargs...)\n neg = value(args[iloss])\n @instr (~f)(args...; kwargs...)\n args = update_var(args, iarg, i, T(δ/2))\n (pos - neg)/δ\nend\n\nfunction ng_single(::Type{T}, f, args, kwargs, iarg, i, iloss, δ) where T<:Complex\n res = zero(T)\n for dd = [δ, im*δ]\n args = update_var(args, iarg, i, dd/2)\n @instr f(args...; kwargs...)\n pos = value(args[iloss])\n @instr (~f)(args...; kwargs...)\n args = update_var(args, iarg, i, -dd)\n @instr f(args...; kwargs...)\n neg = value(args[iloss])\n @instr (~f)(args...; kwargs...)\n args = update_var(args, iarg, i, dd/2)\n if dd == δ\n res += (pos - neg)/δ\n else\n res += im*(pos - neg)/δ\n end\n end\n res\nend\n\nfunction ng(f, args, iarg; iloss::Int, δ=1e-5, kwargs...)\n x = args[iarg]\n T = eltype(x)\n if x isa AbstractArray\n res = zero(x)\n for i = 1:length(x)\n res[i] = ng_single(T, f, args, kwargs, iarg, i, iloss, δ)\n end\n return res\n else\n ng_single(T, f, args, kwargs, iarg, nothing, iloss, δ)\n end\nend\n\n\"\"\"\n gradient_numeric(f, args...; iloss, kwargs...)\n\nNumeric differentiating f(args..., kwargs...).\n\"\"\"\nfunction gradient_numeric(f, args; iloss::Int, kwargs...)\n map(1:length(args)) do iarg\n if isvar(args[iarg])\n ng(f, args, iarg; iloss=iloss, kwargs...)\n else\n 0\n end\n end\nend\n\n\"\"\"\n check_grad(f, args; atol::Real=1e-8, verbose::Bool=false, iloss::Int, kwargs...)\n\nReturn true if the gradient of `f(args..., kwargs...)` is reversible.\n\"\"\"\nfunction check_grad(f, args; atol::Real=1e-4, verbose::Bool=false, iloss::Int, kwargs...)\n vars = ((iarg for iarg in 1:length(args) if isvar(args[iarg]))...,)\n initial_vars = deepcopy(vars)\n ngs = gradient_numeric(f, args; kwargs..., iloss=iloss)\n gs = gradient(f, args; kwargs..., iloss=iloss)\n verbose && @show ngs\n verbose && @show gs\n if !all(isapprox.(ngs, gs, atol=atol))\n verbose && println(\"gradient not match: $ngs v.s. $gs\")\n return false\n end\n\n if !world_similar(initial_vars, vars, atol=atol, verbose=verbose)\n verbose && println(\"world changed during obtaining gradient.\")\n return false\n end\n return true\nend\n\n@specialize\n" }, { "path": "src/autodiff/complex.jl", "content": "@i @inline function :(+=)(angle)(r!::T, x::Complex{T}) where T<:GVar\n r! += atan(x.im, x.re)\nend\n\n@i @inline function :(+=)(abs2)(y!::T, a::Complex{T}) where T<:GVar\n y! += a.re^2\n y! += a.im^2\nend\n\n@i @inline function :(+=)(abs)(y!::T, a::Complex{T}) where T<:GVar\n @routine @invcheckoff begin\n y2 ← zero(y!)\n y2 += abs2(a)\n end\n y! += sqrt(y2)\n ~@routine\nend\n\nBase.zero(x::Complex{T}) where T<:GVar = Complex(zero(T), zero(T))\nBase.zero(::Type{Complex{T}}) where T<:GVar = Complex(zero(T), zero(T))\nBase.one(x::Complex{T}) where T<:GVar = Complex(one(T), zero(T))\nBase.one(::Type{Complex{T}}) where T<:GVar = Complex(one(T), zero(T))\n" }, { "path": "src/autodiff/gradfunc.jl", "content": "export Grad, NGrad, Hessian, gradient\n\n\"\"\"\n NGrad{N,FT} <: Function\n\nObtain gradients `Grad(f)(Val(i), args..., kwargs...)`, where `i` is the index of loss in `args`. `Grad` object calls forward first, and then backward.\n\n!!! note\n `Val(1)` is specially optimized, so putting the loss as the first parameter can avoid potential overhead.\n```\n\"\"\"\nstruct NGrad{N,FT} <: Function\n f::FT\nend\nfunction NGrad{N}(f::FT) where {N,FT}\n NGrad{N,FT}(f)\nend\n\nconst Grad{FT} = NGrad{1,FT}\nconst Hessian{FT} = NGrad{2,FT}\n\nBase.show_function(io::IO, b::NGrad{N}, compact::Bool) where {N} = print(io, \"$(b.f)\"*\"'\"^N)\nBase.show_function(io::IO, ::MIME\"text/plain\", b::NGrad{N}, compact::Bool) where {N} = print(io, b)\nBase.display(bf::NGrad) = print(bf)\n(_::Type{Inv{NGrad{N}}})(f::NGrad{M}) where {M, N} = NGrad{M-N}(f.f)\n(_::Type{Inv{NGrad{M}}})(f::NGrad{M}) where {M} = f.f\n\n\n@i function (g::Grad)(il::Val{iloss}, args...; kwargs...) where iloss\n protectf(g).f(args...; kwargs...)\n GVar.(args)\n INC(args |> tget(iloss) |> grad)\n (~protectf(g).f)(args...; kwargs...)\nend\n\n@i function (g::Grad)(il::Val{1}, x, ys...; kwargs...)\n protectf(g).f(x, ys...; kwargs...)\n GVar(x)\n INC(x |> grad)\n GVar.(ys)\n (~protectf(g).f)(x, ys...; kwargs...)\nend\n\n@i function (g::Grad)(args...; iloss::Int, kwargs...)\n protectf(g).f(args...; kwargs...)\n GVar.(args)\n INC(args |> tget(iloss) |> grad)\n (~protectf(g).f)(args...; kwargs...)\nend\n\n@generated function gradient(::Val{iloss}, f, args::NTuple{N,Any}; kwargs...) where {iloss,N}\n newres = gensym()\n newargs = Any[:(GVar($newres[$i])) for i=1:N]\n newargs[iloss] = :(GVar($newres[$iloss], one($newres[$iloss])))\n quote\n $newres = f(args...; kwargs...)\n grad((~f)($(newargs...); kwargs...))\n end\nend\n\ngradient(f, args; iloss::Int, kwargs...) = gradient(Val(iloss), f, args; kwargs...)\n" }, { "path": "src/autodiff/hessian_backback.jl", "content": "export hessian_backback\n\n@i function backback(f, args...; index::Int, iloss::Int, kwargs...)\n # forward\n Grad(f)(args...; kwargs..., iloss=iloss)\n\n for i = 1:length(args)\n GVar(args |> tget(i) |> grad)\n GVar(args |> tget(i) |> value)\n end\n (args |> tget(index) |> grad |> grad) += 1\n # backward#2\n (~Grad(f))(args...; kwargs..., iloss=iloss)\nend\n\n\"\"\"\n hessian_backback(f, args; iloss::Int, kwargs...)\n\nObtain the Hessian matrix of `f(args..., kwargs...)` by back propagating adjoint program.\n\"\"\"\nfunction hessian_backback(f, args; iloss::Int, kwargs...)\n N = length(args)\n hmat = zeros(N, N)\n for i=1:N\n if !(args[i] isa Integer || args[i] isa AbstractVector)\n res = backback(f, args...; kwargs..., index=i, iloss=iloss)\n hmat[:,i] .= map(x->grad(value(x)), res[2:end])\n end\n end\n hmat\nend\n\nfunction hessian_numeric(f, args; iloss::Int, η=1e-5, kwargs...)\n narg = length(args)\n res = zeros(narg, narg)\n largs = [args...]\n for i = 1:narg\n if nparams(args[i]) == 1\n @instr (largs[i] |> value) += η/2\n gpos = gradient(f, (largs...,); iloss=iloss, kwargs...)\n @instr (largs[i] |> value) -= η\n gneg = gradient(f, (largs...,); iloss=iloss, kwargs...)\n @instr (largs[i] |> value) += η/2\n res[:,i] .= (gpos .- gneg)./η\n end\n end\n return res\nend\n\nfunction local_hessian_numeric(f, args; kwargs...)\n nargs = length(args)\n hes = zeros(nargs,nargs,nargs)\n for j=1:nargs\n if nparams(args[j]) == 1\n hes[:,:,j] .= hessian_numeric(f, args; kwargs..., iloss=j)\n end\n end\n mask = BitArray(nparams.(args) .== 1)\n hes[mask, mask, mask]\nend\n" }, { "path": "src/autodiff/instructs.jl", "content": "# unary\n@i @inline function NEG(a!::GVar)\n NEG(a!.x)\n NEG(a!.g)\nend\n\nfunction INV(x!::GVar{T}) where T\n x2 = x!.x ^ 2\n GVar(INV(x!.x), -x!.g * x2)\nend\n\n@i @inline function DEC(a!::GVar)\n DEC(a!.x)\nend\n\n# +-\n@i @inline function :(-=)(identity)(a!::GVar, b::GVar)\n a!.x -= b.x\n b.g += a!.g\nend\n\n# inv\n@eval @i @inline function :(-=)(inv)(out!::GVar{T}, y::GVar) where T\n out!.x -= inv(y.x)\n @routine @invcheckoff begin\n @zeros T a1\n a1 += y.x ^ 2\n end\n y.g -= out!.g / a1\n ~@routine\nend\n\n# +- (triple)\n@i @inline function :(-=)(+)(out!::GVar, x::GVar, y::GVar)\n out!.x -= x.x + y.x\n x.g += out! |> grad\n y.g += out! |> grad\nend\n\n@i @inline function :(-=)(+)(out!::GVar, x::GVar, y::Real)\n out!.x -= (x |> value) + (y |> value)\n x.g += out! |> grad\nend\n\n@i @inline function :(-=)(+)(out!::GVar, x::Real, y::GVar)\n out!.x -= (x |> value) + (y |> value)\n y.g += out! |> grad\nend\n\n@i @inline function :(-=)(-)(out!::GVar, x::GVar, y::GVar)\n out!.x -= x.x - y.x\n x.g += out! |> grad\n y.g -= out! |> grad\nend\n\n@i @inline function :(-=)(-)(out!::GVar, x::Real, y::GVar)\n out!.x -= (x |> value) - y.x\n y.g -= out!.g\nend\n\n@i @inline function :(-=)(-)(out!::GVar, x::GVar, y::Real)\n out!.x -= x.x - (y |> value)\n x.g += out! |> grad\nend\n\n# NOTE: it will error on `SWAP(a!::GVar, b)` or `SWAP(a!, b:GVar)`\n@i @inline function SWAP(a!::GVar, b!::GVar)\n SWAP(a! |> value, b! |> value)\n SWAP(a!.g, b!.g)\nend\n\n# */\n@i @inline function :(-=)(*)(out!::GVar, x::GVar, y::GVar)\n out!.x -= x.x * y.x\n x.g += out!.g * y.x\n y.g += x.x * out!.g\nend\n\n@i @inline function :(-=)(*)(out!::GVar, x::Real, y::GVar)\n out!.x -= (x |> value) * y.x\n y.g += (x |> value) * out!.g\nend\n\n@i @inline function :(-=)(*)(out!::GVar, x::GVar, y::Real)\n out!.x -= x.x * (y |> value)\n x.g += out!.g * (y |> value)\nend\n\nfor DIV in [:/, :÷]\n @eval @i @inline function :(-=)($DIV)(out!::GVar{T}, x::GVar, y::GVar) where T\n out!.x -= $DIV(x.x, y.x)\n @routine @invcheckoff begin\n a1 ← zero(out! |> grad)\n a2 ← zero(out! |> grad)\n a1 += x.x * out!.g\n a2 += $DIV(a1, y.x)\n end\n x.g += $DIV(out!.g, y.x)\n y.g -= $DIV(a2, y.x)\n ~@routine\nend\n\n@eval @i @inline function :(-=)($DIV)(out!::GVar{T}, x::Real, y::GVar) where T\n out!.x -= $DIV(x, y.x)\n @routine @invcheckoff begin\n a1 ← zero(out!.g)\n a2 ← zero(out!.g)\n a1 += x * out!.g\n a2 += $DIV(a1, y.x)\n end\n y.g -= $DIV(a2, y.x)\n ~@routine\nend\n\n@eval @i @inline function :(-=)($DIV)(out!::GVar, x::GVar, y::Real)\n out!.x -= $DIV(x.x, y)\n x.g += $DIV(out!.g, y)\nend\nend\n\n@i @inline function :(-=)(^)(out!::GVar{T}, x::GVar, n::GVar) where T\n # grad x\n @routine @invcheckoff begin\n @zeros T anc1 anc2 anc3 jac1 jac2 nx_1\n nx_1 += n.x - 1\n anc1 += x.x ^ nx_1\n jac1 += anc1 * n.x\n\n # get grad of n\n anc2 += log(x.x)\n anc3 += anc1 * x.x\n jac2 += anc3 * anc2\n end\n out!.x -= anc1 * x.x\n x.g += out!.g * jac1\n n.g += out!.g * jac2\n ~@routine\nend\n\n@i @inline function :(-=)(^)(out!::GVar{T}, x::GVar, n::Real) where T\n @routine @invcheckoff begin\n anc1 ← zero(x.x)\n jac ← zero(x.x)\n\tnx_1 ← zero(n)\n\n\tnx_1 += n - 1\n anc1 += x.x ^ nx_1\n jac += anc1 * n\n end\n out!.x -= anc1 * x.x\n x.g += out!.g * jac\n ~@routine\nend\n\n@i @inline function :(-=)(^)(out!::GVar{T}, x::Real, n::GVar) where T\n # get jac of n\n @routine @invcheckoff begin\n anc1 ← zero(x)\n anc2 ← zero(x)\n jac ← zero(x)\n\n anc1 += log(x)\n anc2 += x ^ n.x\n jac += anc1*anc2\n end\n out!.x -= anc2\n n.g += out!.g * jac\n ~@routine\nend\n\nfor (OP, F) in [(:min, :<), (:max, :>)]\n@eval @i @inline function :(-=)($OP)(out!::GVar{T}, x::GVar, y::GVar) where T\n if $F(x, y)\n out!.x -= x.x\n x.g += out!.g\n else\n out!.x -= y.x\n y.g += out!.g\n end\nend\n\n@eval @i @inline function :(-=)($OP)(out!::GVar{T}, x::GVar, y::Real) where T\n if $F(x, y)\n out!.x -= x.x\n x.g += out!.g\n else\n out!.x -= y.x\n end\nend\n\n@eval @i @inline function :(-=)($OP)(out!::GVar{T}, x::Real, y::GVar) where T\n if $F(x, y)\n out!.x -= x.x\n else\n out!.x -= y.x\n y.g += out!.g\n end\nend\nend\n\n@i @inline function :(-=)(atan)(out!::GVar{T}, y::GVar, x::GVar) where T\n out!.x -= atan(y.x, x.x)\n @routine @invcheckoff begin\n @zeros T xy2 jac_x jac_y\n xy2 += abs2(x.x)\n xy2 += abs2(y.x)\n jac_y += x.x / xy2\n jac_x += (-y.x) / xy2\n end\n y.g += out!.g * jac_y\n x.g += out!.g * jac_x\n ~@routine\nend\n@i @inline function :(-=)(atan)(out!::GVar{T}, y::Real, x::GVar) where T\n out!.x -= atan(y, x.x)\n @routine @invcheckoff begin\n @zeros T xy2 jac_x\n xy2 += abs2(x.x)\n xy2 += abs2(y)\n jac_x += (-y) / xy2\n end\n x.g += out!.g * jac_x\n ~@routine\nend\n@i @inline function :(-=)(atan)(out!::GVar{T}, y::GVar, x::Real) where T\n out!.x -= atan(y.x, x)\n @routine @invcheckoff begin\n @zeros T xy2 jac_y\n xy2 += abs2(x)\n xy2 += abs2(y.x)\n jac_y += x / xy2\n end\n y.g += out!.g * jac_y\n ~@routine\nend\n\n@i @inline function :(-=)(atan)(out!::GVar{T}, x::GVar) where T\n out!.x -= atan(x.x)\n @routine @invcheckoff begin\n xy2 ← one(T)\n xy2 += abs2(x.x)\n end\n x.g += out!.g / xy2\n ~@routine\nend\n\n@i @inline function :(-=)(abs)(out!::GVar, x::GVar{T}) where T\n out!.x -= abs(x.x)\n if (x > 0, ~)\n x.g += out!.g\n else\n x.g -= out!.g\n end\nend\n\n@i @inline function :(-=)(abs2)(out!::GVar, x::GVar{T}) where T\n out!.x -= abs2(x.x)\n x.g += out!.g * x.x\n x.g += out!.g * x.x\nend\n\nfor op in [:*, :/, :^, :+, :-, :atan, :max, :min]\n @eval @nograd :(-=)($op)(out!::GVar, x::Real, y::Real)\n @eval @nograd :(-=)($op)(out!::Real, x::Real, y::GVar)\n @eval @nograd :(-=)($op)(out!::Real, x::GVar, y::GVar)\n @eval @nograd :(-=)($op)(out!::Real, x::GVar, y::Real)\nend\n\n@i @inline function :(-=)(sqrt)(out!::GVar, x::GVar{T}) where T\n if x.x != 0\n @routine @invcheckoff begin\n @zeros T anc1 anc2\n anc1 += sqrt(x.x)\n anc2 += 2 * anc1\n end\n out!.x -= anc1\n x.g += out!.g / anc2\n ~@routine\n end\nend\n\n@i @inline function :(-=)(exp)(out!::GVar, x::GVar{T}) where T\n @routine @invcheckoff begin\n anc1 ← zero(T)\n anc1 += exp(x.x)\n end\n out!.x -= anc1\n x.g += out!.g * anc1\n ~@routine\nend\n\n@i @inline function :(-=)(log)(out!::GVar, x::GVar{T}) where T\n out!.x -= log(x.x)\n x.g += out!.g / x.x\nend\n\n@i @inline function :(-=)(sin)(out!::GVar, x::GVar{T}) where T\n @routine @invcheckoff begin\n @zeros T s c\n (s, c) += sincos(x.x)\n end\n out!.x -= s\n x.g += out!.g * c\n ~@routine\nend\n\n@i @inline function :(-=)(sinh)(out!::GVar, x::GVar{T}) where T\n out!.x -= sinh(x.x)\n @routine @invcheckoff begin\n anc1 ← zero(x.x)\n anc1 += cosh(x.x)\n end\n x.g += out!.g * anc1\n ~@routine\nend\n\n@i @inline function (:-=)(asin)(out!::GVar, x::GVar{T}) where T\n out!.x -= asin(x.x)\n @routine @invcheckoff begin\n @zeros T sqrt_1_x2 x2\n x2 += x.x ^ 2\n sqrt_1_x2 += sqrt(x2 |> NEG |> AddConst(1))\n end\n x.g += out!.g / sqrt_1_x2\n ~@routine\nend\n\n@i @inline function (:-=)(cos)(out!::GVar, x::GVar{T}) where T\n @routine @invcheckoff begin\n @zeros T s c\n (s, c) += sincos(x.x)\n end\n out!.x -= c\n x.g -= out!.g * s\n ~@routine\nend\n\n@i @inline function :(-=)(cosh)(out!::GVar, x::GVar{T}) where T\n out!.x -= cosh(x.x)\n @routine @invcheckoff begin\n anc1 ← zero(x.x)\n anc1 += sinh(x.x)\n end\n x.g += out!.g * anc1\n ~@routine\nend\n\n@i @inline function :(-=)(acos)(out!::GVar, x::GVar{T}) where T\n out!.x -= acos(x.x)\n @routine @invcheckoff begin\n @zeros T sqrt_1_x2 x2\n x2 += x.x ^ 2\n sqrt_1_x2 += sqrt(x2 |> NEG |> AddConst(1))\n end\n x.g -= out!.g / sqrt_1_x2\n ~@routine\nend\n\n@i @inline function :(-=)(tan)(out!::GVar, x::GVar{T}) where T\n @routine @invcheckoff begin\n anc1 ← zero(x.x)\n anc2 ← one(x.x)\n anc1 += tan(x.x)\n anc2 += anc1^2\n end\n out!.x -= anc1\n x.g += out!.g * anc2\n ~@routine\nend\n\n@i @inline function :(-=)(tanh)(out!::GVar, x::GVar{T}) where T\n @routine @invcheckoff begin\n anc1 ← zero(x.x)\n anc2 ← one(x.x)\n anc1 += tanh(x.x)\n anc2 -= anc1^2\n end\n out!.x -= anc1\n x.g += out!.g * anc2\n ~@routine\nend\n\n@i @inline function :(-=)(sincos)(out!::Tuple{T1,T1}, x::GVar{T}) where {T1<:GVar, T}\n @routine @invcheckoff begin\n s ← zero(T)\n c ← zero(T)\n (s, c) += sincos(x.x)\n end\n (out! .|> value) -= (s, c)\n x.g += (out!.:1 |> grad) * c\n x.g -= (out!.:2 |> grad) * s\n ~@routine\nend\n\nfor op in [:sqrt, :exp, :log, :sin, :cos, :tanh, :abs, :abs2, :identity, :inv]\n @eval @nograd :(-=)($op)(out!::Real, x::GVar)\n @eval @nograd :(-=)($op)(out!::GVar, x::Real)\nend\n\n@nograd :(-=)(sincos)(out!::Tuple{<:Real,<:Real}, x::GVar)\n@nograd :(-=)(sincos)(out!::Tuple{<:GVar,<:GVar}, x::Real)\n\n@i @inline function IROT(a!::GVar, b!::GVar, θ::GVar)\n IROT(a!.x, b!.x, θ.x)\n NEG(θ |> value)\n θ.x -= π/2\n ROT(a!.g, b!.g, θ.x)\n θ.g += a!.x * a!.g\n θ.g += b!.x * b!.g\n θ.x += π/2\n NEG(θ |> value)\n ROT(a!.g, b!.g, π/2)\nend\n\n@i @inline function IROT(a!::GVar, b!::GVar, θ::Real)\n IROT(a!.x, b!.x, θ)\n NEG(θ)\n θ -= π/2\n ROT(a!.g, b!.g, θ)\n θ += π/2\n NEG(θ)\n ROT(a!.g, b!.g, π/2)\nend\n\n@nograd IROT(a!::Real, b!::Real, θ::GVar)\n\nexport primitive_grad\nfunction primitive_grad end\n\n@i @inline function (mf::MinusEq)(out!::GVar, args...; kwargs...)\n out!.x -= mf.f((args .|> value)...; kwargs...)\n (args .|> grad) .+= (@skip! ntuple(x->out!.g, length(args))) .* (@skip! primitive_grad(mf.f, (args .|> value)...; kwargs...)) # unsafe statement, error on recursive gradient\nend\n\n@i @inline function (mf::MinusEq)(out!::GVar, x::GVar; kwargs...)\n out!.x -= mf.f(x |> value; kwargs...)\n x.g += (@skip! out!.g) * (@skip! primitive_grad(mf.f, x.x; kwargs...)) # unsafe statement\nend\n\n@i @inline function :(-=)(convert)(out!::GVar{Tx, Tg}, y::GVar) where {Tx, Tg}\n out!.x -= convert(y.x)\n y.g += convert(out!.g)\nend\n\n@i @inline function HADAMARD(x::GVar, y::GVar)\n HADAMARD(x.x, y.x)\n HADAMARD(x.g, y.g)\nend\n\n@i @inline function (f::AddConst)(y::GVar)\n y.x += f.x\nend\n\n# more data views\nfor (DT, OP, NOP) in [(:AddConst, :+, :-, :add), (:SubConst, :-, :+)]\n @eval chfield(x::GVar, ac::$DT, xval::GVar) = GVar($NOP(xval.x, ac.x), xval.g)\nend\n\n#chfield(x::T, ::typeof(INV), xval::T) where T<:GVar = GVar(INV(xval.x), -xval.g*(xval.x^2))\n#chfield(x::T, ::typeof(NEG), xval::T) where T<:GVar = GVar(-xval.x, -xval.g)\n\nfor F in [:INV, :NEG, :FLIP, :INC, :DEC]\n @eval NiLangCore.chfield(x::T, ::typeof($F), xval::T) where T<:GVar = (~$F)(xval)\nend\n" }, { "path": "src/autodiff/jacobian.jl", "content": "export jacobian, jacobian_repeat\n\nwrap_tuple(x, args) = length(args) == 1 ? (x,) : x\n\n\"\"\"\n jacobian_repeat(f, args...; iin::Int, iout::Int=iin, kwargs...)\n\nGet the Jacobian matrix for function `f(args..., kwargs...)` using repeated computing gradients for each output.\nOne can use key word arguments `iin` and `iout` to specify the input and output tensor.\n\"\"\"\nfunction jacobian_repeat(f, args...; iin::Int, iout::Int=iin, kwargs...)\n _check_input(args, iin, iout)\n N = length(args[iout])\n res = zeros(eltype(args[iin]), length(args[iin]), N)\n xargs = wrap_tuple(f(args...; kwargs...), args)\n for i = 1:N\n gxargs = GVar.(xargs)\n @inbounds gxargs[iout][i] = GVar(value(gxargs[iout][i]), one(eltype(xargs[iout])))\n @inbounds res[:,i] .= vec(grad.(wrap_tuple((~f)(gxargs...; kwargs...), gxargs)[iin]))\n end\n return res\nend\n\n_copy(x) = x\n_copy(x::AbstractArray) = copy(x)\n\n\"\"\"\n jacobian(f, args...; iin::Int, iout::Int=iin, kwargs...)\n\nGet the Jacobian matrix for function `f(args..., kwargs...)` using vectorized variables in the gradient field.\nOne can use key word arguments `iin` and `iout` to specify the input and output tensor.\n\"\"\"\nfunction jacobian(f, args...; iin::Int, iout::Int=iin, kwargs...)\n _check_input(args, iin, iout)\n args = wrap_tuple(f(args...; kwargs...), args)\n ABT = AutoBcast{eltype(args[iout]), length(args[iout])}\n _args = map(i-> i==iout ? wrap_jacobian(ABT, args[i]) : wrap_bcastgrad(ABT, args[i]), 1:length(args))\n _args = wrap_tuple((~f)(_args...; kwargs...), args)\n out = zeros(eltype(args[iin]), length(args[iin]), length(args[iout]))\n for i=1:length(args[iin])\n @inbounds out[i,:] .= grad(_args[iin][i]).x\n end\n out\nend\n\nfunction wrap_jacobian(::Type{AutoBcast{T,N}}, outarray::AbstractArray{T}) where {T,N}\n map(k->GVar(outarray[k], AutoBcast{T,N}(onehot(T, N, k))), LinearIndices(outarray))\nend\nfunction wrap_bcastgrad(::Type{AutoBcast{T,N}}, x::XT) where {T,N,XT}\n GVar(x, zero(AutoBcast{XT,N}))\nend\nfunction wrap_bcastgrad(::Type{AutoBcast{T,N}}, x::Union{Integer, Function}) where {T,N}\n x\nend\nfunction wrap_bcastgrad(::Type{AutoBcast{T,N}}, x::NoGrad) where {T,N}\n (~NoGrad)(x)\nend\nfunction wrap_bcastgrad(::Type{AutoBcast{T,N}}, x::Union{Tuple,AbstractArray}) where {T,N}\n wrap_bcastgrad.(AutoBcast{T,N}, x)\nend\n\nfunction onehot(::Type{T}, N::Int, k::Int) where T\n res = zeros(T, N)\n res[k] = one(T)\n res\nend\n\nfunction _check_input(args, iin, iout)\n if !(args[iin] isa AbstractArray && args[iout] isa AbstractArray)\n throw(ArgumentError(\"argument at position $iin and $iout are not arrays.\"))\n elseif (eltype(args[iin]) != eltype(args[iout]))\n throw(ArgumentError(\"argument at position $iin and $iout do not have the same type.\"))\n end\nend\n" }, { "path": "src/autodiff/stack.jl", "content": "# This is a patch for loading a data to GVar correctly.\nimport NiLangCore\n\nNiLangCore.loaddata(::Type{GT}, x::T) where {T, GT<:GVar{T}} = convert(GT, x)\nfunction NiLangCore.loaddata(t::Type{VT}, x::AbstractVector) where {T, VT<:AbstractVector{T}}\n convert.(T, x)\nend\n\nfunction NiLangCore.loaddata(t::VT, x::AbstractVector) where {T, VT<:AbstractVector{T}}\n convert(VT, NiLangCore.loaddata.(t, x))\nend\n\nfunction NiLangCore.loaddata(::Type{T}, x::XT) where {N, T<:Tuple{N}, XT<:Tuple{N}}\n ntuple(i=>NiLangCore.loaddata.(T.parameters[i], [i]), N)\nend\n" }, { "path": "src/autodiff/ulog.jl", "content": "@i function (:-=)(gaussian_log)(y!::GVar{T}, x::GVar{T}) where T\n\ty!.x -= gaussian_log(x.x)\n\t@routine @invcheckoff begin\n\t\texp_x ← zero(x)\n\t\tjac ← zero(x)\n\t\texp_x += exp(-x)\n\tend\n\tx.g += y!.g * (exp_x |> AddConst(1) |> INV)\n\t~@routine\nend\n\n@i function (:-=)(gaussian_nlog)(y!::GVar{T}, x::GVar{T}) where T\n\ty!.x -= gaussian_nlog(x.x)\n\t@routine @invcheckoff begin\n\t\texp_x ← zero(x)\n\t\texp_x += exp(-x)\n\tend\n\tx.g -= y!.g * (exp_x |> SubConst(1) |> INV)\n\t~@routine\nend\n\n@i function :(-=)(convert)(out!::GVar{Tx, Tg}, y::ULogarithmic) where {Tx, Tg}\n\tout! -= exp(y.log)\nend\n" }, { "path": "src/autodiff/vars.jl", "content": "######## GVar, a bundle that records gradient\n\"\"\"\n GVar{T,GT} <: IWrapper{T}\n GVar(x)\n\nAdd gradient information to variable `x`, where `x` can be a real number or a general structure.\nIf it is a non-integer real number, it will wrap the element with a gradient field,\notherwise it will propagate into the type and wrap the elements with `GVar`.\nRuning a program backward will update the gradient fields of `GVar`s. The following is a toy using case.\n\n### Example\n\n```jldoctest; setup=:(using NiLang)\njulia> using NiLang.AD: GVar, grad\n\njulia> struct A{T}\n x::T\n end\n\njulia> GVar(A(2.0+3im), A(3.0+3im))\nA{Complex{GVar{Float64, Float64}}}(GVar(2.0, 3.0) + GVar(3.0, 3.0)*im)\n\njulia> @i function f(a::A, b::A)\n a.x += log(b.x)\n end\n\njulia> outputs = f(A(2.0+3im), A(2.0-1im)) # forward pass\n(A{ComplexF64}(2.8047189562170503 + 2.536352390999194im), A{ComplexF64}(2.0 - 1.0im))\n\njulia> outputs_with_gradients = (GVar(outputs[1], A(3.0+3im)), GVar(outputs[2])) # wrap `GVar`\n(A{Complex{GVar{Float64, Float64}}}(GVar(2.8047189562170503, 3.0) + GVar(2.536352390999194, 3.0)*im), A{Complex{GVar{Float64, Float64}}}(GVar(2.0, 0.0) - GVar(1.0, -0.0)*im))\n\njulia> inputs_with_gradients = (~f)(outputs_with_gradients...) # backward pass\n(A{Complex{GVar{Float64, Float64}}}(GVar(2.0, 3.0) + GVar(3.0, 3.0)*im), A{Complex{GVar{Float64, Float64}}}(GVar(2.0, 1.8) - GVar(1.0, -0.6000000000000002)*im))\n\njulia> grad(inputs_with_gradients)\n(A{ComplexF64}(3.0 + 3.0im), A{ComplexF64}(1.8 + 0.6000000000000002im))\n```\n\nThe outputs of `~f` are gradients for input variables, one can use `grad` to take the gradient fields recursively.\n\"\"\"\nstruct GVar{T,GT} <: IWrapper{T}\n x::T\n g::GT\n function GVar{T,GT}(x::T, g::GT) where {T,GT}\n new{T,GT}(x, g)\n end\n function GVar(x::T, g::T) where T<:Real\n new{T,T}(x, g)\n end\n function GVar{T,GT}(x::T2) where {T,T2,GT}\n new{T,GT}(T(x), zero(GT))\n end\n function GVar(x::T, g::GT) where {T,GT}\n new{T,GT}(x, g)\n end\nend\n\n# `GVar` and `~GVar` on composite types\n@generated function GVar(x::Type{T}) where T\n ps = GVar.(T.parameters)\n if length(ps) == 0\n :($(getfield(T.name.module, nameof(T))))\n else\n :($(getfield(T.name.module, nameof(T))){$(ps...)})\n end\nend\n@generated function GVar(x::Type{T}, y::Type{T}) where T\n :($(getfield(T.name.module, nameof(T))){$(GVar.(T.parameters, T.parameters)...)})\nend\n@generated function (_::Type{Inv{GVar}})(x::Type{T}) where T\n :($(getfield(T.name.module, nameof(T))){$((~GVar).(T.parameters)...)})\nend\n# `GVar` and `~GVar` on composite vars\n@generated function GVar(x::T) where T\n Expr(:new, GVar(T), [:(GVar(x.$NAME)) for NAME in fieldnames(T)]...)\nend\n@generated function GVar(x::T, g::T) where T\n Expr(:new, GVar(T, T), [:(GVar(x.$NAME, g.$NAME)) for NAME in fieldnames(T)]...)\nend\n@generated function (_::Type{Inv{GVar}})(x::T) where T\n Expr(:new, (~GVar)(T), [:((~GVar)(x.$NAME)) for NAME in fieldnames(T)]...)\nend\n\nfor T in [:Real]\n ## differentiable elementary types\n @eval GVar(::Type{ET}) where ET<:$T = GVar{ET,ET}\n @eval GVar(::Type{ET}, ::Type{ET}) where ET<:$T = GVar{ET,ET}\n @eval (_::Type{Inv{GVar}})(::Type{GVar{ET,GT}}) where {ET<:$T,GT} = ET\n\n ## differentiable elementary vars\n @eval GVar(x::$T) = GVar(x, zero(x))\n @eval @inline function (_::Type{Inv{GVar}})(x::GVar{<:$T})\n @invcheck x.g zero(x.x)\n x.x\n end\nend\n\nfor T in [:Integer, :Bool, :Function, :String, :Char, :Nothing]\n ## non-differentiable elementary types\n @eval GVar(::Type{ET}) where ET<:$T = ET\n @eval GVar(::Type{ET}, ::Type{ET}) where ET<:$T = GVar{ET,ET}\n @eval (_::Type{Inv{GVar}})(::Type{ET}) where ET<:$T = ET\n\n ## non-differentiable elementary vars\n @eval GVar(x::$T) = x\n @eval (_::Type{Inv{GVar}})(x::$T) = x\nend\n\nfor T in [:Tuple, :AbstractArray]\n ## broadcastable elementary types\n @eval GVar(x::$T) = GVar.(x)\n @eval GVar(x::$T, y::$T) = GVar.(x, y)\n @eval (_::Type{Inv{GVar}})(x::$T) = (~GVar).(x)\nend\n\n# no gradient wrapper\nGVar(x::NoGrad) = (~NoGrad)(x)\n\n# define on complex numbers to fix ambiguity errors\nGVar(x::Complex) = Complex(GVar(x.re), GVar(x.im))\nGVar(x::Complex, y::Complex) = Complex(GVar(x.re, y.re), GVar(x.im, y.im))\n(_::Type{Inv{GVar}})(x::Complex) = Complex((~GVar)(x.re), (~GVar)(x.im))\n\nBase.copy(b::GVar) = GVar(b.x, copy(b.g))\nBase.zero(x::GVar) = GVar(Base.zero(x.x), Base.zero(x.g))\nBase.zero(::Type{<:GVar{T,GT}}) where {T,GT} = GVar(zero(T), zero(GT))\nBase.one(x::GVar) = GVar(Base.one(x.x), Base.zero(x.g))\nBase.one(::Type{<:GVar{T}}) where T = GVar(one(T))\nBase.adjoint(b::GVar) = GVar(b.x', b.g')\nBase.:-(b::GVar) = GVar(-b.x, -b.g)\nBase.isapprox(x::GVar, y::GVar; kwargs...) = isapprox(x.x, y.x; kwargs...) && isapprox(x.g, y.g; kwargs...)\n\n# define kernel and field views\n\"\"\"\n grad(var)\n\nGet the gradient field of `var`.\n\"\"\"\n@fieldview grad(gv::GVar) = gv.g\n@fieldview value(gv::GVar) = gv.x\n# TODO: fix the problem causing this patch, the field type can not change?!\nchfield(x::GVar, ::typeof(value), xval::GVar) = GVar(xval, x.g)\n\n@generated function grad(x::T) where T\n isprimitivetype(T) && throw(\"not supported type to obtain gradients: $T.\")\n Expr(:new, typegrad(T), [:(grad(x.$NAME)) for NAME in fieldnames(T)]...)\nend\ntypegrad(x) = x\n@generated function typegrad(x::Type{T}) where T\n if isprimitivetype(T)\n T\n else\n ps = typegrad.(T.parameters)\n if length(ps) == 0\n :($(getfield(T.name.module, nameof(T))))\n else\n :($(getfield(T.name.module, nameof(T))){$(ps...)})\n end\n end\nend\ntypegrad(::Type{GVar{ET,GT}}) where {ET,GT} = ET\ngrad(gv::T) where T<:Real = zero(T)\ngrad(gv::AbstractArray{T}) where T = grad.(gv)\ngrad(gv::Function) = 0\ngrad(gv::String) = \"\"\ngrad(t::Tuple) = grad.(t)\nchfield(x::T, ::typeof(grad), g::T) where T = (@invcheck g zero(g); x)\nchfield(x::GVar, ::typeof(grad), g::GVar) = GVar(x.x, g)\n#chfield(x::GVar, ::typeof(-), val::GVar) = GVar(-val.x, -val.g)\nchfield(x::Complex{<:GVar}, ::typeof(grad), g::Complex) = Complex(GVar(value(x.re), g.re), GVar(value(x.im), g.im))\n\n# NOTE: superwarning: check value only to make ancilla gradient descardable.\nNiLangCore.deanc(x::GVar{T}, val::GVar{T}) where T = NiLangCore.deanc(value(x), value(val))\nfunction deanc(x::T, val::T) where {T<:AbstractArray}\n x === val || deanc.(x, val)\nend\n\n# constructors and deconstructors\nBase.iszero(x::GVar) = iszero(x.x)\n\n## variable mapping\nfunction (_::Type{Inv{GVar}})(x::GVar{<:GVar,<:GVar})\n Partial{:x}(x)\nend\n\nBase.show(io::IO, gv::GVar) = print(io, \"GVar($(gv.x), $(gv.g))\")\nBase.show(io::IO, ::MIME\"plain/text\", gv::GVar) = Base.show(io, gv)\n\n# used in log number iszero function.\nBase.isfinite(x::GVar) = isfinite(x.x)\n# interfaces\n\n_replace_opmx_callable(ex) = @match ex begin\n :(:+=($f)) => :(PlusEq($f))\n :(:-=($f)) => :(MinusEq($f))\n :(:*=($f)) => :(MulEq($f))\n :(:/=($f)) => :(DivEq($f))\n :(:⊻=($f)) => :(XorEq($f))\n _ => ex\nend\n\n\"\"\"\n @nograd f(args...)\n\nMark `f(args...)` as having no gradients.\n\"\"\"\nmacro nograd(ex)\n @match ex begin\n :($f($(args...))) => begin\n f2 = _replace_opmx_callable(f)\n newargs = []\n for arg in args\n push!(newargs, @match arg begin\n :($x::GVar) => :($x.x)\n :($x::VecGVar) => :($x.x)\n :($x::GVar{$tp}) => :($x.x)\n _ => NiLangCore.get_argname(arg)\n end\n )\n end\n esc(quote\n @i function $f($(args...))\n $f2($(newargs...))\n end\n end)\n end\n _ => error(\"expect `f(args...)`, got $ex\")\n end\nend\n\n# ULogarithmic\n_content(x::ULogarithmic) = x.log\nNiLang.AD.GVar(x::ULogarithmic) = exp(ULogarithmic, GVar(_content(x), zero(_content(x))))\n(_::Type{Inv{GVar}})(x::ULogarithmic{GVar{TE}}) where TE = exp(ULogarithmic{TE}, (~GVar)(_content(x)))\n\nBase.one(x::ULogarithmic{GVar{T,GT}}) where {T, GT} = one(ULogarithmic{GVar{T,GT}})\nBase.one(::Type{ULogarithmic{GVar{T,GT}}}) where {T,GT} = exp(ULogarithmic, GVar(zero(T), zero(GT)))\nBase.zero(x::ULogarithmic{GVar{T,GT}}) where {T,GT} =zero(ULogarithmic{GVar{T,GT}})\nBase.zero(::Type{ULogarithmic{GVar{T,T}}}) where T = exp(ULogarithmic, GVar(zero(T), zero(T)))\n\n# the patch for dicts\nfunction GVar(d::Dict)\n Dict([(k=>GVar(v)) for (k, v) in d])\nend\n\nfunction (_::Type{Inv{GVar}})(d::Dict)\n Dict([(k=>(~GVar)(v)) for (k, v) in d])\nend\n\nfunction grad(d::Dict)\n Dict([(k=>grad(v)) for (k, v) in d])\nend\n" }, { "path": "src/complex.jl", "content": "export CONJ\nNiLangCore.chfield(x::Complex, ::typeof(real), r) = chfield(x, Val{:re}(), r)\nNiLangCore.chfield(x::Complex, ::typeof(imag), r) = chfield(x, Val{:im}(), r)\n\n@i @inline function NEG(y!::Complex)\n NEG(y!.re)\n NEG(y!.im)\nend\n\n@i @inline function CONJ(y!::Complex{T}) where T\n NEG(y!.im)\nend\n\n@i @inline function :(+=)(angle)(r!::Real, x::Complex)\n r! += atan(x.im, x.re)\nend\n\n@i @inline function :(+=)(identity)(y!::Complex, a::Complex)\n y!.re += a.re\n y!.im += a.im\nend\n\n@inline function SWAP(a!::Complex, b!::Complex)\n b!, a!\nend\n\n@i @inline function :(+=)(abs2)(y!::Real, a::Complex)\n y! += a.re^2\n y! += a.im^2\nend\n\n@i @inline function :(+=)(abs)(y!::Real, a::Complex)\n @routine @invcheckoff begin\n y2 ← zero(y!)\n y2 += abs2(a)\n end\n y! += sqrt(y2)\n ~@routine\nend\n\n@i @inline function :(+=)(*)(y!::Complex{T}, a::Complex, b::Complex) where T\n @routine @invcheckoff begin\n @zeros T rere imim reim imre\n rere += a.re * b.re\n imim += a.im * b.im\n reim += a.re * b.im\n imre += a.im * b.re\n end\n y!.re += rere - imim\n y!.im += reim + imre\n ~@routine\nend\n\n@i @inline function :(+=)(*)(y!::Complex, a::Real, b::Complex)\n y!.re += a * b.re\n y!.im += a * b.im\nend\n\n@i @inline function :(+=)(*)(y!::Complex, a::Complex, b::Real)\n y!.re += a.re * b\n y!.im += a.im * b\nend\n\nfor OP in [:+, :-]\n @eval @i @inline function :(+=)($OP)(y!::Complex, a::Complex, b::Complex)\n y!.re += $OP(a.re, b.re)\n y!.im += $OP(a.im, b.im)\n end\n\n @eval @i @inline function :(+=)($OP)(y!::Complex, a::Complex, b::Real)\n y!.re += $OP(a.re, b)\n end\n\n @eval @i @inline function :(+=)($OP)(y!::Complex, a::Real, b::Complex)\n y!.re += $OP(a, b.re)\n end\nend\n\n@i @inline function :(+=)(/)(y!::Complex, a::Complex, b::Complex{T}) where T\n @routine @invcheckoff begin\n b2 ← zero(T)\n ab ← zero(y!)\n b2 += abs2(b)\n CONJ(b)\n ab += a * b\n end\n y! += ab / b2\n ~@routine\nend\n\n@i @inline function :(+=)(/)(y!::Complex, a::Complex, b::Real)\n y!.re += a.re / b\n y!.im += a.im / b\nend\n\n@i @inline function :(+=)(/)(y!::Complex, a::Real, b::Complex{T}) where T\n @routine @invcheckoff begin\n b2 ← zero(T)\n ab ← zero(y!)\n b2 += abs2(b)\n CONJ(b)\n ab += a * b\n end\n y! += ab / b2\n ~@routine\nend\n\n@i @inline function :(+=)(inv)(y!::Complex, b::Complex{T}) where T\n @routine @invcheckoff begin\n b2 ← zero(real(T))\n b2 += abs2(b)\n end\n y! += b' / b2\n ~@routine\nend\n\n@i @inline function :(+=)(exp)(y!::Complex, x::Complex{T}) where T\n @routine @invcheckoff begin\n @zeros T s c expn\n z ← zero(y!)\n (s, c) += sincos(x.im)\n SWAP(z.re, c)\n SWAP(z.im, s)\n expn += exp(x.re)\n end\n y! += expn * z\n ~@routine\nend\n\n@i @inline function :(+=)(log)(y!::Complex, x::Complex{T}) where T\n @routine @invcheckoff begin\n n ← zero(T)\n n += abs(x)\n end\n y!.re += log(n)\n y!.im += angle(x)\n ~@routine\nend\n\n@i @inline function :(+=)(^)(y!::Complex, a::Complex{T}, b::Real) where T\n @routine @invcheckoff begin\n @zeros T r θ s c absy bθ\n r += abs(a)\n θ += angle(a)\n bθ += θ * b\n (s, c) += sincos(bθ)\n absy += r ^ b\n end\n y!.re += absy * c\n y!.im += absy * s\n ~@routine\nend\n\n@i @inline function :(+=)(complex)(y!::Complex, a::Real, b::Real)\n y!.re += a\n y!.im += b\nend\n\nfor OP in [:*, :/, :+, :-, :^]\n @eval @i @inline function :(+=)($OP)(y!::Complex, a::Real, b::Real)\n y!.re += $OP(a, b)\n end\nend\n\nfor OP in [:identity, :cos, :sin, :log, :exp]\n @eval @i @inline function :(+=)($OP)(y!::Complex, a::Real)\n y!.re += $OP(a)\n end\nend\n\n@i @inline function HADAMARD(x::Complex, y::Complex)\n HADAMARD(x.re, y.re)\n HADAMARD(x.im, y.im)\nend\n" }, { "path": "src/deprecations.jl", "content": "@deprecate simple_hessian hessian_backback\n@deprecate hessian_repeat hessian_backback\n@deprecate ngradient gradient_numeric\n@deprecate nhessian hessian_numeric\n@deprecate NEG Base.:-\n@deprecate ipush! PUSH!\n@deprecate ipop! POP!\n" }, { "path": "src/instructs.jl", "content": "export SWAP, FLIP\nexport ROT, IROT\nexport INC, DEC, NEG, INV, AddConst, SubConst\nexport HADAMARD\nexport PUSH!, POP!, COPYPOP!, COPYPUSH!\n\n\"\"\"\n NoGrad{T} <: IWrapper{T}\n NoGrad(x)\n\nA `NoGrad(x)` is equivalent to `GVar^{-1}(x)`, which cancels the `GVar` wrapper.\n\"\"\"\nstruct NoGrad{T} <: IWrapper{T}\n x::T\nend\nNoGrad(x::NoGrad{T}) where T = x # to avoid ambiguity error\nNoGrad{T}(x::NoGrad{T}) where T = x # to avoid ambiguity error\n(_::Type{Inv{NoGrad}})(x) = x.x\n@fieldview value(x::NoGrad) = x.x\n\nconst NullType{T} = Union{NoGrad{T}, Partial{T}}\n\nNEG(a!) = -(a!)\n@selfdual NEG\n@selfdual -\n\nINV(a!) = inv(a!)\n@selfdual INV\n\n@inline FLIP(b::Bool) = !b\n@selfdual FLIP\n\n\"\"\"\n INC(a!) -> a! + 1\n\"\"\"\n@inline function INC(a!::Number)\n a! + one(a!)\nend\n\n\"\"\"\n DEC(a!) -> a! - 1\n\"\"\"\n@inline function DEC(a!::Number)\n a! - one(a!)\nend\n@dual INC DEC\n\n\n\"\"\"\n SWAP(a!, b!) -> b!, a!\n\"\"\"\n@inline function SWAP(a!::T, b!::T) where T\n b!, a!\nend\n@selfdual SWAP\n\n\"\"\"\n ROT(a!, b!, θ) -> a!', b!', θ\n\n```math\n\\\\begin{align}\n {\\\\rm ROT}(a!, b!, \\\\theta) = \\\\begin{bmatrix}\n \\\\cos(\\\\theta) & - \\\\sin(\\\\theta)\\\\\\\\\n \\\\sin(\\\\theta) & \\\\cos(\\\\theta)\n \\\\end{bmatrix}\n \\\\begin{bmatrix}\n a!\\\\\\\\\n b!\n \\\\end{bmatrix},\n\\\\end{align}\n```\n\"\"\"\n@inline function ROT(i::Real, j::Real, θ::Real)\n a, b = rot(i, j, θ)\n a, b, θ\nend\n\n\"\"\"\n IROT(a!, b!, θ) -> ROT(a!, b!, -θ)\n\"\"\"\n@inline function IROT(i::Real, j::Real, θ::Real)\n i, j, _ = ROT(i, j, -θ)\n i, j, θ\nend\n@dual ROT IROT\n\n\"\"\"\n HADAMARD(x::Real, y::Real)\n\nHadamard transformation that returns `(x + y)/√2, (x - y)/√2`\n\"\"\"\nfunction HADAMARD(x::Real, y::Real)\n sqrt(0.5) * (x + y), sqrt(0.5) * (x - y)\nend\n\n@selfdual HADAMARD\n\n# more data views\nfor (DT, OP, NOP) in [(:AddConst, :+, :-), (:SubConst, :-, :+)]\n @eval struct $DT{T}\n x::T\n end\n\n @eval function (f::$DT)(y::Real)\n $OP(y, f.x)\n end\n\n @eval NiLangCore.chfield(x::T, ac::$DT, xval::T) where T<:Real = $NOP(xval, ac.x)\nend\n\nfor F1 in [:(Base.:-), :NEG, :(ac::AddConst), :(sc::SubConst)]\n @eval @inline function $F1(a!::NullType)\n @instr $F1(a! |> value)\n a!\n end\nend\n\nfor (OP, F, f) in [(:(PlusEq{typeof(identity)}), :(PlusEq(identity)), :+), (:(MinusEq{typeof(identity)}), :(MinusEq(identity)), :-)]\n @eval @inline @generated function (::$OP)(x::T, y::T) where T\n if isprimitivetype(T)\n Expr(:tuple, Expr(:call, $f, :x, :y), :y)\n else\n res = gensym(\"results\")\n computes = Any[:($($F)(x.$field, y.$field)) for field in fieldnames(T)]\n comp = Expr(:(=), res, Expr(:tuple, computes...))\n res1 = Expr(:new, T, [:($res[$i][1]) for i=1:length(computes)]...)\n res2 = Expr(:new, T, [:($res[$i][2]) for i=1:length(computes)]...)\n quote\n $comp\n ($res1, $res2)\n end\n end\n end\n @eval (f::$OP)(x::T, y::T) where T<:Tuple = invoke(f, Tuple{T,T} where T, x, y)\n @eval (f::$OP)(x::T, y::T) where T<:Real = $f(x, y), y\nend\n\nfor F2 in [:SWAP, :HADAMARD, :((inf::PlusEq)), :((inf::MinusEq)), :((inf::XorEq))]\n @eval @inline function $F2(a::NullType, b::Real)\n @instr $(NiLangCore.get_argname(F2))(a |> value, b)\n a, b\n end\n @eval @inline function $F2(a::NullType, b::NullType)\n @instr $(NiLangCore.get_argname(F2))(a |> value, b |> value)\n a, b\n end\n @eval @inline function $F2(a::Real, b::NullType)\n @instr $(NiLangCore.get_argname(F2))(a, b |> value)\n a, b\n end\nend\n\nfunction type_except(::Type{TT}, ::Type{T2}) where {TT, T2}\n N = length(TT.parameters)\n setdiff(Base.Iterators.product(zip(TT.parameters, repeat([T2], N))...), [ntuple(x->T2, N)])\nend\n\nfor F3 in [:ROT, :IROT, :((inf::PlusEq)), :((inf::MinusEq)), :((inf::XorEq))]\n PS = (:a, :b, :c)\n for PTS in type_except(Tuple{NullType, NullType, NullType}, Real)\n params = map((P,PT)->PT <: NullType ? :($P |> value) : P, PS, PTS)\n params_ts = map((P,PT)->:($P::$PT), PS, PTS)\n @eval @inline function $F3($(params_ts...))\n @instr $F3($(params...))\n ($(PS...),)\n end\n end\nend\n\n# patch for fixed point numbers\nfunction (f::PlusEq{typeof(/)})(out!::T, x::Integer, y::Integer) where T<:Fixed\n out!+T(x)/y, x, y\nend\n\nfunction (f::MinusEq{typeof(/)})(out!::T, x::Integer, y::Integer) where T<:Fixed\n out!-T(x)/y, x, y\nend\n\nfor F in [:exp, :log, :sin, :sinh, :asin, :cos, :cosh, :acos, :tan, :tanh, :atan]\n @eval Base.$F(x::Fixed43) = Fixed43($F(Float64(x)))\n @eval (f::PlusEq{typeof($F)})(out!::Fixed43, x::Real) = out! + Fixed43($F(x)), x\n @eval (f::MinusEq{typeof($F)})(out!::Fixed43, x::Real) = out! - Fixed43($F(x)), x\nend\n\nBase.:^(x::Integer, y::Fixed43) = Fixed43(x^(Float64(y)))\nBase.:^(x::Fixed43, y::Fixed43) = Fixed43(x^(Float64(y)))\nBase.:^(x::T, y::Fixed43) where T<:AbstractFloat = x^(T(y))\n\nfunction (::PlusEq{typeof(convert)})(out!::T, y) where T<:Real\n out! + convert(T, y), y\nend\n\nfunction (::MinusEq{typeof(convert)})(out!::T, y) where T<:Real\n out! - convert(T, y), y\nend\n\nBase.:~(ac::AddConst) = SubConst(ac.x)\nBase.:~(ac::SubConst) = AddConst(ac.x)\n@dualtype AddConst SubConst\n\nfor F in [:INV, :NEG, :FLIP, :INC, :DEC]\n @eval NiLangCore.chfield(x::T, ::typeof($F), xval::T) where T<:Real = (~$F)(xval)\nend\n\n#### The following functions are not safe!\n@i @inline function PUSH!(x::T) where T\n PUSH!((@skip! GLOBAL_STACK), x)\nend\n\n@i @inline function POP!(x::T) where T\n POP!((@skip! GLOBAL_STACK), x)\nend\n\n@i @inline function COPYPUSH!(x)\n COPYPUSH!((@skip! GLOBAL_STACK), x)\nend\n\n@i @inline function COPYPOP!(x)\n COPYPOP!((@skip! GLOBAL_STACK), x)\nend\n\n# reversibility turned off, in principle, we can not deallocate `GVar{T}` to `T`\n@i @inline function PUSH!(st, x::T) where T\n @invcheckoff st[end+1] ↔ x\n @invcheckoff x ← _zero(T)\nend\n\n@i @inline function POP!(st, x::T) where T\n @invcheckoff x → _zero(T)\n @invcheckoff st[end] ↔ (x::T)::∅\nend\n\n@i @inline function COPYPUSH!(st, x)\n @invcheckoff st[end+1] ← x\nend\n\n@i @inline function COPYPOP!(st, x)\n @invcheckoff st[end] → x\nend\n\n# accumulation on arrays: initially for Bennett algorithm\n# TODO: also define it for composite types. or maybe a macro for it.\n@i function :(+=)(identity)(target::AbstractArray, source::AbstractArray)\n @safe @assert length(target) == length(source)\n @inbounds for i=1:length(target)\n target[i] += source[i]\n end\nend\n" }, { "path": "src/macros.jl", "content": "using MLStyle, NiLang\nexport alloc, @auto_alloc, @auto_expand\n\n\"\"\"\n alloc(f, args...)\n\nallocate function output space (the first argument), where `args` only contains the last `N-1` arguments.\n\"\"\"\nfunction alloc end\n\nmacro auto_alloc(ex)\n esc(auto_alloc(ex))\nend\n\nfunction auto_alloc(ex)\n @match ex begin\n :($f($out, $(args...))) => begin\n Expr(:block, :($out ← $alloc($f, $(args...))), ex)\n end\n :($out = $f($(args...))) => begin\n if length(args) == 0\n error(\"number of arguments must be >= 1.\")\n else\n Expr(:block, :($out ← $alloc($f, $(args...))), :($out += $f($(args...))))\n end\n end\n _ => error(\"can not allocate automatically for expression: `$ex`\")\n end\nend\n\nfor OPM in [:PlusEq, :MinusEq]\n for OP in [:+, :-, :*, :/, :^]\n @eval alloc(::$OPM{typeof($OP)}, x::T1, ::T2) where {T1<:Number,T2<:Number} = zero(promote_type(T1, T2))\n end\n for OP in [:sin, :cos, :tan, :asin, :atan, :acos, :sinh, :cosh, :tanh, :identity, :sqrt, :exp, :log]\n @eval alloc(::$OPM{typeof($OP)}, x::T) where T<:Number = zero(T)\n end\n for OP in [:abs, :abs2]\n @eval alloc(::$OPM{typeof($OP)}, x::T) where T<:Number = zero(real(T))\n end\n @eval alloc(::$OPM{typeof(sincos)}, x::T) where T<:Number = (zero(T), zero(T))\nend\n\nfunction auto_expand(ex)\n res = Expr[]\n auto_expand!(copy(ex), res)\n Expr(:block, res..., NiLangCore.dual_body(@__MODULE__, res[1:end-1])...)\nend\n\nfunction auto_expand!(ex, exprs, sym=nothing, addnew=true)\n @match ex begin\n :($f($(args...))) => begin\n for (i, arg) in enumerate(args)\n @match arg begin\n :($_{$(_...)}($(_...))) => begin\n auto_expand!(arg, exprs, nothing, false)\n end\n :($f2($(vs...))) => begin\n sym2 = gensym()\n auto_expand!(:(PlusEq($f2)($sym2, $(vs...))), exprs, sym2, true)\n args[i] = sym2\n end\n _ => nothing\n end\n end\n if sym !== nothing\n push!(exprs, :($sym ← $alloc($f, $(args[2:end]...))))\n end\n if addnew\n push!(exprs, :($f($(args...))))\n end\n end\n :($a += $b) || :($a -= $b) || :($a *= $b) || :($a /= $b) || :($a ⊻= $b) => begin\n auto_expand!(NiLangCore.to_standard_format(ex), exprs, sym, addnew)\n end\n _ => error(\"Can only expand an expression like `f(args...)`, got $(ex)!\")\n end\nend\n\nmacro auto_expand(ex)\n esc(auto_expand(ex))\nend\n" }, { "path": "src/stdlib/base.jl", "content": "export i_sqdistance, i_dirtymul, i_factorial\n\n\"\"\"\n i_sqdistance(dist!, x1, x2)\n\nSquared distance between two points `x1` and `x2`.\n\"\"\"\n@i function i_sqdistance(dist!, x1::AbstractVector{T}, x2::AbstractVector) where T\n @inbounds for i=1:length(x1)\n x1[i] -= x2[i]\n dist! += x1[i] ^ 2\n x1[i] += x2[i]\n end\nend\n\n\"\"\"\n i_dirtymul(out!, x, anc!)\n\n\"dirty\" reversible multiplication that computes `out! *= x` approximately for floating point numbers,\nthe `anc!` is anticipated as a number ~0.\n\"\"\"\n@i @inline function i_dirtymul(out!, x, anc!)\n anc! += out! * x\n out! -= anc! / x\n SWAP(out!, anc!)\nend\n\n@i @inline function i_dirtymul(out!::Int, x::Int, anc!::Int)\n anc! += out! * x\n out! -= anc! ÷ x\n SWAP(out!, anc!)\nend\n\n\"\"\"\n i_factorial(out!, n)\n\nCompute the factorial `out! = factorial(n)`.\n\"\"\"\n@i function i_factorial(out!::Int, n::Int)\n INC(out!)\n @invcheckoff for i=1:n\n i_dirtymul(out!, i, 0)\n end\nend\n" }, { "path": "src/stdlib/bennett.jl", "content": "export bennett, bennett!\n\nfunction direct_emulate(step, x0::T, args...; N::Int, kwargs...) where T\n xpre = copy(x0)\n local x\n for i=1:N\n x = _zero(xpre)\n res = step(x, xpre, args...; kwargs...)\n xpre = res[1]\n args = res[3:end]\n end\n return xpre\nend\n\nstruct BennettLog\n fcalls::Vector{NTuple{3,Any}} # depth, function index f_i := s_{i-1} -> s_{i}, length should be `(2k-1)^n` and function\n peak_mem::Base.RefValue{Int} # should be `n*(k-1)+2`\n depth::Base.RefValue{Int}\nend\nBennettLog() = BennettLog(NTuple{3,Any}[], Ref(0), Ref(0))\n\n# hacking the reversible program\nfunction logfcall(l::BennettLog, i, f)\n push!(l.fcalls, (l.depth[], i, f))\n l, i, f\nend\nfunction ilogfcall(l::BennettLog, i, f)\n push!(l.fcalls, (l.depth[], i, ~f))\n l, i, f\nend\n\n@dual logfcall ilogfcall\n\nBase.show(io::IO, ::MIME\"text/plain\", logger::BennettLog) = Base.show(io, logger)\nfunction Base.show(io::IO, logger::BennettLog)\n nreverse = count(x->x[3] isa Inv, logger.fcalls)\n print(io, \"\"\"Bennett log\n| peak memory usage = $(logger.peak_mem[])\n| number of function forward/backward calls = $(length(logger.fcalls)-nreverse)/$nreverse\"\"\")\nend\n\n\"\"\"\n bennett(step, y, x, args...; k, N, logger=BennettLog(), kwargs...)\n\n* `step` is a reversible step function,\n* `y` is the output state,\n* `x` is the input state,\n* `k` is the number of steps in each Bennett's recursion,\n* `N` is the total number of steps,\n* `logger=BennettLog()` is the logging of Bennett's algorithm,\n* `args...` and `kwargs...` are additional arguments for steps.\n\"\"\"\n@i function bennett(step, y::T, x::T, args...; k::Int, N::Int, logger=BennettLog(), kwargs...) where T\n state ← Dict{Int, T}()\n state[1] ← _zero(x)\n state[1] += x\n bennett!((@skip! step), state, k, 1, N, args...; do_uncomputing=true, logger=logger, kwargs...)\n SWAP(y, state[N+1])\n state[1] -= x\n state[1] → _zero(x)\n state[N+1] → _zero(x)\n state → Dict{Int, T}()\nend\n\n\"\"\"\n bennett!(step, state::Dict, args...; k, N, logger=BennettLog(), do_uncomputing=false, kwargs...)\n\n* `step` is a reversible step function,\n* `state` is the dictionary state, with `state[1]` the input state, the return value is stored in `state[N+1]`,\n* `k` is the number of steps in each Bennett's recursion,\n* `N` is the total number of steps,\n* `logger=BennettLog()` is the logging of Bennett's algorithm,\n* `args...` and `kwargs...` are additional arguments for steps.\n\"\"\"\n@i function bennett!(step, state::Dict{Int,T}, args...; k::Int, N::Int, logger=BennettLog(), do_uncomputing=false, kwargs...) where T\n bennett!(step, state, k, 1, N, args...; logger=logger, do_uncomputing=do_uncomputing, kwargs...)\nend\n\n@i function bennett!(step, state::Dict{Int,T}, k::Int, base, len, args...; logger, do_uncomputing, kwargs...) where T\n @safe logger !== nothing && (logger.depth[] += 1)\n @invcheckoff if len == 1\n state[base+1] ← _zero(state[base])\n @safe logger !== nothing && (logger.peak_mem[] = max(logger.peak_mem[], length(state)))\n getf(step, base)(state[base+1], state[base], args...; kwargs...)\n if logger !== nothing\n logfcall(logger, (@const base+1), (@const getf(step, base)))\n end\n else\n @routine begin\n @zeros Int nstep n\n n += ceil((@skip! Int), (@const len / k))\n nstep += ceil((@skip! Int), (@const len / n))\n end\n for j=1:nstep\n bennett!(step, state, k, (@const base+n*(j-1)), (@const min(n,len-n*(j-1))), args...; logger=logger, do_uncomputing=true, kwargs...)\n end\n if do_uncomputing\n for j=nstep-1:-1:1\n ~bennett!(step, state, k, (@const base+n*(j-1)), n, args...; logger=logger, do_uncomputing=true, kwargs...)\n end\n end\n ~@routine\n end\nend\n\ngetf(f, i::Int) = f\ngetf(f::AbstractArray, i::Int) = f[i]\n" }, { "path": "src/stdlib/blas.jl", "content": "export i_sum, i_mul!, i_dot, i_axpy!, i_umm!, i_norm2\n\n\"\"\"\n i_sum(out!, x)\n\nget the sum of `x`.\n\"\"\"\n@i function i_sum(out!, x::AbstractArray)\n\t@invcheckoff for i=1:length(x)\n\t\t@inbounds out! += x[i]\n\tend\nend\n\n@i function i_sum(out!, f, x::AbstractArray)\n\t@invcheckoff for i=1:length(x)\n\t\t@inbounds out! += f(x[i])\n\tend\nend\n\n\"\"\"\n i_mul!(out!, x, y)\n\ncompute `x * y` (`x` and `y` are matrices, and store results in `out!`.\n\"\"\"\n@i function i_mul!(out!::AbstractMatrix{T}, x::AbstractMatrix{T}, y::AbstractMatrix{T}) where T\n\t@safe size(x, 2) == size(y, 1) || throw(DimensionMismatch())\n\t@invcheckoff @inbounds for k=1:size(y,2)\n\t for j=1:size(x,2)\n\t\t for i=1:size(x,1)\n\t\t\t\tout![i,k] += x[i,j] * y[j,k]\n\t\t\tend\n\t\tend\n\tend\nend\n\n@i function i_mul!(out!::AbstractVector{T}, x::AbstractMatrix, y::AbstractVector) where T\n\t@safe size(x, 2) == size(y, 1) || throw(DimensionMismatch())\n\t@invcheckoff @inbounds for j=1:size(x,2)\n @routine begin\n yj ← zero(T)\n yj += y[j]\n end\n\t\tfor i=1:size(x,1)\n\t\t\tout![i] += x[i,j] * yj\n\t\tend\n ~@routine\n\tend\nend\n\n@i function i_dot(out!, x, y)\n @safe @assert length(x) == length(y)\n @invcheckoff @inbounds for i=1:length(x)\n out! += x[i]' * y[i]\n end\nend\n\n\"\"\"\n i_norm2(out!, x)\n\nget the squared norm of `x`.\n\"\"\"\n@i function i_norm2(out!, x)\n @invcheckoff @inbounds for i=1:length(x)\n out! += abs2(x[i])\n end\nend\n\n\"\"\"\n i_axpy!(a, x, y!)\n\ncompute `y! += a * x`, where `x` and `y` are vectors.\n\"\"\"\n@i function i_axpy!(a, X, Y)\n @safe @assert length(X) == length(Y)\n @invcheckoff @inbounds for i=1:length(Y)\n Y[i] += a * X[i]\n end\nend\n\n\"\"\"\n i_umm!(x!, θ)\n\nCompute unitary matrix multiplication on `x`, where the unitary matrix is parameterized by (N+1)*N/2 `θ`s.\n\"\"\"\n@i function i_umm!(x!::AbstractArray, θ)\n @routine begin\n M ← size(x!, 1)\n N ← size(x!, 2)\n end\n k ← 0\n @safe @assert length(θ) == M*(M-1)/2\n for l = 1:N\n for j=1:M\n for i=M-1:-1:j\n INC(k)\n ROT(x![i,l], x![i+1,l], θ[k])\n end\n end\n end\n\n k → length(θ)\n ~@routine\nend\n" }, { "path": "src/stdlib/linalg.jl", "content": "export i_inv!, i_affine!\n\n\"\"\"\n i_inv!(out!, A)\n\nGet the inverse of `A`.\n\n```note!!!\nthis function is implemented as a primitive.\n```\n\"\"\"\n@i function i_inv!(out!::AbstractMatrix{T}, A::AbstractMatrix{T}) where T\n @invcheckoff invA ← inv(A)\n out! .+= invA\n @invcheckoff invA → inv(A)\nend\n\n@i function i_inv!(out!::AbstractMatrix{T}, A::AbstractMatrix{T}) where T<:GVar\n @routine @invcheckoff begin\n invA ← inv(value.(A))\n gA ← -transpose(invA) * grad(out!) * transpose(invA)\n end\n for i=1:length(out!)\n (out![i] |> value) -= invA[i]\n end\n for i=1:length(A)\n (A[i] |> grad) -= gA[i]\n end\n ~@routine\nend\n\n@i function :(-=)(det)(out!::T, A::AbstractMatrix{T}) where T<:GVar\n @routine @invcheckoff begin\n vA ← value.(A)\n detA ← det(vA)\n gA ← detA * grad(out!) * transpose(inv(vA))\n end\n (out! |> value) -= detA\n for i=1:length(A)\n (A[i] |> grad) += gA[i]\n end\n ~@routine\nend\n\n@i function :(-=)(logdet)(out!::T, A::AbstractMatrix{T}) where T<:GVar\n @routine @invcheckoff begin\n gA ← grad(out!) * transpose(inv(value.(A)))\n end\n (out! |> value) -= det(A |> grad)\n for i=1:length(A)\n (A[i] |> grad) += gA[i]\n end\n ~@routine\nend\n\n\"\"\"\n i_affine!(y!, W, b, x)\n\n`affine!` transformation `y! += W*x + b`.\n\"\"\"\n@i function i_affine!(y!::AbstractVector{T}, W::AbstractMatrix{T}, b::AbstractVector{T}, x::AbstractVector{T}) where T\n @safe @assert size(W) == (length(y!), length(x)) && length(b) == length(y!)\n @invcheckoff for j=1:size(W, 2)\n for i=1:size(W, 1)\n @inbounds y![i] += W[i,j]*x[j]\n end\n end\n @invcheckoff for i=1:size(W, 1)\n @inbounds y![i] += b[i]\n end\nend\n" }, { "path": "src/stdlib/mapreduce.jl", "content": "export i_mapfoldl, i_filter!, i_map!\n\n\"\"\"\n i_mapfoldl(map, fold, out!, iter)\n\nReversible `mapfoldl` function, `map` can be irreversible, but `fold` should be reversible.\n\"\"\"\n@i function i_mapfoldl(map, fold, out!::T, iter) where T\n anc ← zero(T)\n for i=1:length(iter)\n anc += map(iter[i])\n fold(out!, anc)\n anc -= map(iter[i])\n end\n anc → zero(T)\nend\n\n\"\"\"\n i_filter!(f, out!, iter)\n\nReversible `filter` function, `out!` is an emptied vector.\n\"\"\"\n@i function i_filter!(f, out!::AbstractVector, x::AbstractVector{T}) where T\n @invcheckoff @inbounds for i = 1:length(x)\n if (f(x[i]), ~)\n COPYPUSH!(out!, x[i])\n end\n end\nend\n" }, { "path": "src/stdlib/nnlib.jl", "content": "export i_softmax_crossentropy, i_relu, i_logsumexp\n\nfunction (_::PlusEq{typeof(argmax)})(out!, x::AbstractArray)\n out! += argmax(x)\n out!, x\nend\n\nfunction (_::MinusEq{typeof(argmax)})(out!, x::AbstractArray)\n out! -= argmax(x)\n out!, x\nend\n\n\n\"\"\"\n i_softmax_crossentropy(x, p, imax, xmax, Z, out)\n\nSoftmax-Cross entropy function.\n\"\"\"\n@i function i_softmax_crossentropy(x, p, imax, xmax, Z, out::T) where T\n # subtract maximum\n imax += argmax(x) # trade off space of xmax to time\n xmax += x[imax]\n # accumulate exp(x) to Z, and finally get logZ\n for i=1:length(x)\n x[i] -= xmax\n Z += Base.exp(x[i])\n end\n @routine begin\n yi ← zero(T)\n logZ ← zero(T)\n logZ += log(Z)\n end\n for i=1:length(x)\n yi += logZ\n yi -= x[i]\n out += yi * p[i]\n yi += x[i]\n yi -= logZ\n end\n ~@routine\nend\n\n\"\"\"\n i_relu(out!, x)\n\nReLU in machine learning.\n\"\"\"\n@i function i_relu(out!, x)\n @invcheckoff if (x > 0, ~)\n out! += x\n end\nend\n\n\"\"\"\n i_logsumexp(logout!, out!, xs!, inds!, x)\n\nCompute `logout! = log(sum(exp(x)))`.\n\n# Arguments\n\n * `out!`, output,\n * `logout!`, logged output,\n * `xs!`, an empty vector to cache the ascending values (same type as `x`),\n * `inds!`, an empty vector to cache the ascending indices (integer type),\n * `x`, input vector.\n\"\"\"\n@i function i_logsumexp(logout!, out!, xs!, inds!, x::AbstractArray{T}) where T\n \ti_ascending!(xs!, inds!, x)\n @routine begin\n mx ← zero(T)\n mx += xs![end]\n end\n\t@invcheckoff @inbounds for i=1:length(x)\n\t\tx[i] -= mx\n\t\tout! += exp(x[i])\n\t\tx[i] += mx\n\tend\n \tlogout! += log(out!)\n\tlogout! += mx\n ~@routine\nend\n\n\n" }, { "path": "src/stdlib/sorting.jl", "content": "export i_ascending!\n\n\"\"\"\n\ti_ascending!(xs!, inds!, arr)\n\nFind the ascending sequence in `arr` and store the results into `xs!`, indices are stored in `inds!`.\nThis function can be used to get the maximum value and maximum indices.\n\"\"\"\n@i function i_ascending!(xs!::AbstractVector{T}, inds!, arr::AbstractArray{T}) where T\n\t@invcheckoff if (length(arr) > 0, ~)\n\t\ty ← zero(T)\n\t\ty += arr[1]\n\t\txs![end+1] ↔ y\n\t\tanc ← 1\n\t\tinds![end+1] ↔ anc\n\t\t@inbounds for i = 2:length(arr)\n\t\t\tif (arr[i] > xs![end], i==inds![end])\n\t\t\t\tind ← i\n\t\t\t\tx ← zero(T)\n\t\t\t\tx += arr[i]\n\t\t\t\txs![end+1] ↔ x\n\t\t\t\tinds![end+1] ↔ ind\n\t\t\tend\n\t\tend\n\tend\nend\n" }, { "path": "src/stdlib/sparse.jl", "content": "using SparseArrays\n\n@i function i_mul!(C::StridedVecOrMat, A::AbstractSparseMatrix, B::StridedVector{T}, α::Number, β::Number) where T\n @safe size(A, 2) == size(B, 1) || throw(DimensionMismatch())\n @safe size(A, 1) == size(C, 1) || throw(DimensionMismatch())\n @safe size(B, 2) == size(C, 2) || throw(DimensionMismatch())\n @routine begin\n nzv ← nonzeros(A)\n rv ← rowvals(A)\n end\n if (β != 1, ~)\n @safe error(\"only β = 1 is supported, got β = $(β).\")\n end\n # Here, we close the reversibility check inside the loop to increase performance\n @invcheckoff for k = 1:size(C, 2)\n @inbounds for col = 1:size(A, 2)\n @routine begin\n αxj ← zero(T)\n αxj += B[col,k] * α\n end\n for j = SparseArrays.getcolptr(A)[col]:(SparseArrays.getcolptr(A)[col + 1] - 1)\n C[rv[j], k] += nzv[j]*αxj\n end\n ~@routine\n end\n end\n ~@routine\nend\n\n\n@i function i_dot(r::T, A::SparseMatrixCSC{T},B::SparseMatrixCSC{T}) where {T}\n @routine @invcheckoff begin\n (m, n) ← size(A)\n branch_keeper ← zeros(Bool, 2*m)\n end\n @safe size(B) == (m,n) || throw(DimensionMismatch(\"matrices must have the same dimensions\"))\n @invcheckoff @inbounds for j = 1:n\n @routine begin\n ia1 ← A.colptr[j]\n ib1 ← B.colptr[j]\n ia2 ← A.colptr[j+1]\n ib2 ← B.colptr[j+1]\n ia ← ia1\n ib ← ib1\n end\n @inbounds for i=1:ia2-ia1+ib2-ib1-1\n ra ← A.rowval[ia]\n rb ← B.rowval[ib]\n if (ra == rb, ~)\n r += A.nzval[ia]' * B.nzval[ib]\n end\n ## b move -> true, a move -> false\n branch_keeper[i] ⊻= @const ia == ia2-1 || (ib != ib2-1 && ra > rb)\n ra → A.rowval[ia]\n rb → B.rowval[ib]\n if (branch_keeper[i], ~)\n INC(ib)\n else\n INC(ia)\n end\n end\n ~@inbounds for i=1:ia2-ia1+ib2-ib1-1\n ## b move -> true, a move -> false\n branch_keeper[i] ⊻= @const ia == ia2-1 || (ib != ib2-1 && A.rowval[ia] > B.rowval[ib])\n if (branch_keeper[i], ~)\n INC(ib)\n else\n INC(ia)\n end\n end\n ~@routine\n end\n ~@routine\nend\n" }, { "path": "src/stdlib/statistics.jl", "content": "export i_mean_sum, i_var_mean_sum, i_normal_logpdf, i_cor_cov\nexport VarianceInfo\n\n\"\"\"\n i_mean_sum(out!, sum!, x)\n\nget the `mean` and `sum` of `x`.\n\"\"\"\n@i function i_mean_sum(out!, sum!, x)\n for i=1:length(x)\n sum! += x[i]\n end\n out! += sum!/(@const length(x))\nend\n\nstruct VarianceInfo{T}\n variance::T\n variance_accumulated::T\n mean::T\n sum::T\nend\n\nfunction VarianceInfo(::Type{T}) where T\n VarianceInfo(zero(T), zero(T), zero(T), zero(T))\nend\n\n\"\"\"\n i_var_mean_sum(varinfo, sqv)\n i_var_mean_sum(var!, varsum!, mean!, sum!, v)\n\nCompute the variance, the accumulated variance, mean and sum.\n`varinfo` is the `VarianceInfo` object to store outputs.\n\"\"\"\n@i function i_var_mean_sum(varinfo::VarianceInfo{T}, v::AbstractVector{T}) where T\n i_var_mean_sum(varinfo.variance, varinfo.variance_accumulated, varinfo.mean, varinfo.sum, v)\nend\n\n@i function i_var_mean_sum(var!, varsum!, mean!, sum!, v::AbstractVector{T}) where T\n i_mean_sum(mean!, sum!, v)\n for i=1:length(v)\n @routine @invcheckoff begin\n x ← zero(T)\n x += v[i] - mean!\n end\n varsum! += x ^ 2\n ~@routine\n end\n var! += varsum! / (@const length(v)-1)\n end\n\n\"\"\"\n i_normal_logpdf(out, x, μ, σ)\n\nget the pdf of `Normal(μ, σ)` at point `x`.\n\"\"\"\n@i function i_normal_logpdf(out, x::T, μ, σ) where T\n @routine @invcheckoff begin\n @zeros T anc1 anc2 anc3\n anc1 += x\n anc1 -= μ\n anc2 += anc1 / σ # (x- μ)/σ\n anc3 += anc2^2 # (x-μ)^2/σ^2\n end\n\n out -= anc3 * 0.5 # -(x-μ)^2/2σ^2\n out -= log(σ) # -(x-μ)^2/2σ^2 - log(σ)\n out -= log(2π)/2 # -(x-μ)^2/2σ^2 - log(σ) - log(2π)/2\n\n ~@routine\nend\n\n\"\"\"\n i_cor_cov(rho!,cov!,a,b)\n\nget Pearson correlation and covariance of two vectors `a` and `b` \n\n\"\"\"\n@i function i_cor_cov(rho!::T, cov!::T, a::AbstractVector{T}, b::AbstractVector{T}) where T\n @safe @assert length(a) == length(b)\n @routine @invcheckoff begin\n @zeros T std1 std2\n info1 ← _zero(VarianceInfo{T})\n i_var_mean_sum(info1, a)\n std1 += sqrt(info1.variance)\n info2 ← _zero(VarianceInfo{T})\n i_var_mean_sum(info2, b)\n std2 += sqrt(info2.variance)\n @zeros T anc5 anc6 anc7\n @inbounds for i=1:length(b)\n @routine begin\n @zeros T anc3 anc4\n anc3 += a[i] - info1.mean\n anc4 += b[i] - info2.mean\n end\n anc5 += anc3 * anc4\n ~@routine\n end\n anc6 += std1 * std2\n anc7 += anc6 * (@const length(b)-1)\n end\n cov! += anc5 / (@const length(b)-1)\n rho! += anc5 / anc7 \n ~@routine\nend\n" }, { "path": "src/stdlib/stdlib.jl", "content": "using .NiLang.AD\nusing LinearAlgebra\ninclude(\"base.jl\")\ninclude(\"blas.jl\")\ninclude(\"linalg.jl\")\ninclude(\"statistics.jl\")\ninclude(\"nnlib.jl\")\ninclude(\"sparse.jl\")\ninclude(\"mapreduce.jl\")\ninclude(\"sorting.jl\")\ninclude(\"bennett.jl\")\n" }, { "path": "src/ulog.jl", "content": "using LogarithmicNumbers\nexport gaussian_log, gaussian_nlog\nexport ULogarithmic\n\n@i @inline function (:*=(identity))(x::ULogarithmic, y::ULogarithmic)\n x.log += y.log\nend\n\n@i @inline function (:*=(identity))(x::ULogarithmic, y::Real)\n x.log += log(y)\nend\n\nfor (OP1, OP2, OP3) in [(:*, :+, :(+=)), (:/, :-, :(-=))]\n\t@eval @i @inline function (:*=($OP1))(out!::ULogarithmic, x::ULogarithmic, y::ULogarithmic)\n\t out!.log += $OP2(x.log, y.log)\n\tend\n\n\t@eval @i @inline function (:*=($OP1))(out!::ULogarithmic, x::Real, y::Real)\n\t out!.log += log(x)\n\t\t$(Expr(OP3, :(out!.log), :(log(y))))\n\tend\n\n\t@eval @i @inline function (:*=($OP1))(out!::ULogarithmic, x::ULogarithmic, y::Real)\n\t out!.log += x.log\n\t\t$(Expr(OP3, :(out!.log), :(log(y))))\n\tend\n\n\t@eval @i @inline function (:*=($OP1))(out!::ULogarithmic, x::Real, y::ULogarithmic)\n\t out!.log += log(x)\n\t\t$(Expr(OP3, :(out!.log), :(y.log)))\n\tend\nend\n\n@i @inline function (:*=(^))(out!::ULogarithmic, x::ULogarithmic, y::Real)\n out!.log += x.log * y\nend\n\ngaussian_log(x) = log1p(exp(x))\ngaussian_nlog(x) = log1p(-exp(x))\n\n@i function (:*=)(+)(out!::ULogarithmic{T}, x::ULogarithmic{T}, y::ULogarithmic{T}) where {T}\n\t@invcheckoff if (x.log == y.log, ~)\n\t\tout!.log += x.log\n\t\tout!.log += log(2)\n\telseif (x.log ≥ y.log, ~)\n\t\tout!.log += x.log\n\t\ty.log -= x.log\n\t\tout!.log += gaussian_log(y.log)\n\t\ty.log += x.log\n\telse\n\t\tout!.log += y.log\n\t\tx.log -= y.log\n\t\tout!.log += gaussian_log(x.log)\n\t\tx.log += y.log\n\tend\nend\n\n@i function (:*=)(-)(out!::ULogarithmic{T}, x::ULogarithmic{T}, y::ULogarithmic{T}) where {T}\n\t@safe @assert x.log ≥ y.log\n\t@invcheckoff if (!iszero(x), ~)\n\t\tout!.log += x.log\n\t\ty.log -= x.log\n\t\tout!.log += gaussian_nlog(y.log)\n\t\ty.log += x.log\n\tend\nend\n\n@i function :(*=)(convert)(out!::ULogarithmic{T}, y::ULogarithmic) where T\n out!.log += convert((@skip! T), y.log)\nend\n\n@i function :(*=)(convert)(out!::ULogarithmic{T}, y::T) where T<:Real\n out!.log += log(y)\nend\n\nfunction (f::PlusEq)(out!::ULogarithmic{T}, args...) where T\n throw(MethodError(f, (out!, args...)))\nend\n\nfunction (f::MinusEq)(out!::ULogarithmic{T}, args...) where T\n throw(MethodError(f, (out!, args...)))\nend\n\nBase.convert(::Type{T}, x::ULogarithmic{T}) where {T<:Fixed} = exp(x.log)\n\nfunction NiLangCore.deanc(x::T, v::T) where T<:ULogarithmic\n x === v || NiLangCore.deanc(x.log, v.log)\nend\n" }, { "path": "src/utils.jl", "content": "export rot, plshift, prshift, arshift\n\n\"\"\"\n rot(a, b, θ)\n\nrotate variables `a` and `b` by an angle `θ`\n\"\"\"\nfunction rot(a, b, θ)\n s, c = sincos(θ)\n a*c-b*s, a*s+b*c\nend\n\n\"\"\"\n plshift(x, n)\n\nperiodic left shift.\n\"\"\"\nplshift(x, n) = (x << n) | (x >> (sizeof(x)*8-n))\n\n\"\"\"\n plshift(x, n)\n\nperiodic right shift.\n\"\"\"\nprshift(x, n) = (x >> n) | (x << (sizeof(x)*8-n))\n\n\"\"\"\n arshift(x, n)\n\nright shift, sign extending.\n\"\"\"\narshift(x::T, n) where T = (x >> n) | (x & (T(1) << (sizeof(x)*8-1)))\n" }, { "path": "src/vars.jl", "content": "# variable manipulation\nexport @zeros, @ones\n\n\"\"\"\nCreate zeros of specific type.\n\n```julia\njulia> @i function f(x)\n @zeros Float64 a b c\n # do something\n end\n```\n\"\"\"\nmacro zeros(T, args...)\n esc(Expr(:block, map(x->:($x ← zero($T)), args)...))\nend\n\nmacro ones(T, args...)\n esc(Expr(:block, map(x->:($x ← one($T)), args)...))\nend\n\nfunction NiLangCore.chfield(a::AbstractArray, ::typeof(vec), val)\n reshape(val, size(a)...)\nend" }, { "path": "src/wrappers.jl", "content": "export IWrapper, Partial, unwrap, value\n\n\"\"\"\n value(x)\n\nGet the `value` from a wrapper instance.\n\"\"\"\nvalue(x) = x\nNiLangCore.chfield(x::T, ::typeof(value), y::T) where T = y\n\n\"\"\"\n IWrapper{T} <: Real\n\nIWrapper{T} is a wrapper of for data of type T.\nIt will forward `>, <, >=, <=, ≈` operations.\n\"\"\"\nabstract type IWrapper{T} <: Real end\nNiLangCore.chfield(x, ::Type{T}, v) where {T<:IWrapper} = (~T)(v)\nBase.eps(::Type{<:IWrapper{T}}) where T = Base.eps(T)\n\n\"\"\"\n unwrap(x)\n\nUnwrap a wrapper instance (recursively) to get the content value.\n\"\"\"\nunwrap(x::IWrapper) = unwrap(value(x))\nunwrap(x) = x\n\nfor op in [:>, :<, :>=, :<=, :isless, :(==), :≈]\n @eval Base.$op(a::IWrapper, b::IWrapper) = $op(unwrap(a), unwrap(b))\n @eval Base.$op(a::IWrapper, b::Real) = $op(unwrap(a), b)\n @eval Base.$op(a::IWrapper, b::AbstractFloat) = $op(unwrap(a), b)\n @eval Base.$op(a::Real, b::IWrapper) = $op(a, unwrap(b))\n @eval Base.$op(a::AbstractFloat, b::IWrapper) = $op(a, unwrap(b))\nend\n\n\"\"\"\nPartial{FIELD, T, T2} <: IWrapper{T2}\n\nTake a field `FIELD` without dropping information.\nThis operation can be undone by calling `~Partial{FIELD}`.\n\"\"\"\nstruct Partial{FIELD, T, T2} <: IWrapper{T2}\n x::T\n function Partial{FIELD,T,T2}(x::T) where {T,T2,FIELD}\n new{FIELD,T,T2}(x)\n end\n function Partial{FIELD,T,T2}(x::T) where {T<:Complex,T2,FIELD}\n new{FIELD,T,T2}(x)\n end\nend\nPartial{FIELD}(x::T) where {T,FIELD} = Partial{FIELD,T,typeof(getfield(x,FIELD))}(x)\nPartial{FIELD}(x::T) where {T<:Complex,FIELD} = Partial{FIELD,T,typeof(getfield(x,FIELD))}(x)\n\n@generated function (_::Type{Inv{Partial{FIELD}}})(x::Partial{FIELD}) where {FIELD}\n :(x.x)\nend\n\nfunction NiLangCore.chfield(hd::Partial{FIELD}, ::typeof(value), val) where FIELD\n chfield(hd, Val(:x), chfield(hd.x, Val(FIELD), val))\nend\n\n@generated function value(hv::Partial{FIELD}) where FIELD\n :(hv.x.$FIELD)\nend\n\nfunction Base.zero(x::T) where T<:Partial\n zero(T)\nend\n\nfunction Base.zero(x::Type{<:Partial{FIELD,T}}) where {FIELD, T}\n Partial{FIELD}(Base.zero(T))\nend\nBase.show(io::IO, gv::Partial{FIELD}) where FIELD = print(io, \"$(gv.x).$FIELD\")\nBase.show(io::IO, ::MIME\"plain/text\", gv::Partial) = Base.show(io, gv)" }, { "path": "test/autobcast.jl", "content": "using NiLang\nusing Test\n\n@testset \"auto bcast\" begin\n a = AutoBcast([1.0, 2.0, 3.0])\n @instr NEG(a)\n @test a.x == [-1.0,-2.0,-3.0]\n a = AutoBcast([1.0, 2.0, 3.0])\n @instr INC(a)\n @test a.x == [2.0,3.0,4.0]\n @instr DEC(a)\n @test a.x == [1.0,2.0,3.0]\n\n a = AutoBcast([false, true, true])\n @instr FLIP(a)\n @test a.x == [true, false, false]\n\n a = AutoBcast([1.0, 2.0, 3.0])\n b = AutoBcast([1.0, 2.0, 4.0])\n @instr a += b\n @test a.x == [2,4,7.0]\n @test b.x == [1,2,4.0]\n @instr SWAP(a, b)\n @test b.x == [2,4,7.0]\n @test a.x == [1,2,4.0]\n\n a = AutoBcast([1.0, 2.0, 3.0])\n b = 2.0\n @instr a += b\n @test a.x == [3,4,5.0]\n @test b == 2.0\n\n a = AutoBcast([1.0, 2.0, 3.0])\n b = AutoBcast([1.0, 2.0, 4.0])\n c = AutoBcast([1.0, 2.0, 1.0])\n @instr a += b * c\n @test a.x == [2,6,7.0]\n @test b.x == [1,2,4.0]\n @test c.x == [1,2,1.0]\n\n a = AutoBcast([1.0, 2.0, 3.0])\n b = 2.0\n c = AutoBcast([1.0, 2.0, 1.0])\n @instr a += b * c\n @test a.x == [3,6,5.0]\n @test b == 2.0\n @test c.x == [1,2,1.0]\n\n a = AutoBcast([1.0, 2.0, 3.0])\n b = AutoBcast([1.0, 2.0, 4.0])\n c = 3.0\n @instr a += b * c\n @test a.x == [4,8,15.0]\n @test b.x == [1,2,4.0]\n @test c == 3.0\n\n a = AutoBcast([1.0, 2.0, 3.0])\n b = 2.0\n c = 3.0\n @instr a += b * c\n @test a.x == [7,8,9.0]\n @test b == 2.0\n @test c == 3.0\n\n @test zero(AutoBcast{Int,3}) == AutoBcast([0, 0, 0])\nend\n" }, { "path": "test/autodiff/autodiff.jl", "content": "using Test, NiLang, NiLang.AD\n\ninclude(\"vars.jl\")\ninclude(\"stack.jl\")\ninclude(\"gradfunc.jl\")\n\ninclude(\"instructs.jl\")\ninclude(\"ulog.jl\")\ninclude(\"complex.jl\")\ninclude(\"manual.jl\")\ninclude(\"jacobian.jl\")\ninclude(\"hessian_backback.jl\")\n" }, { "path": "test/autodiff/complex.jl", "content": "using Test, NiLang, NiLang.AD\n\n@testset \"complex GVar\" begin\n a = 1.0+ 2im\n @test GVar(a) == Complex(GVar(1.0), GVar(2.0))\n @test GVar(a, a) == Complex(GVar(1.0, 1.0), GVar(2.0, 2.0))\n\n gx = GVar(1.0 + 1.0im)\n gx2 = chfield(gx, grad, 1.0+0.0im)\n @test gx2 == Complex(GVar(1.0, 1.0), GVar(1.0, 0.0))\nend\n\n@i function fr(f, loss, args...; il)\n f(args...)\n loss += (args |> tget(il)).re\nend\n@i function fi(f, loss, args...; il)\n f(args...)\n loss += (args |> tget(il)).im\nend\nfunction ccheck_grad(f, args; verbose=true, iloss=1)\n check_grad(fr, (f, 0.0, args...); verbose=verbose, iloss=2, il=1) &&\n check_grad(fi, (f, 0.0, args...); verbose=verbose, iloss=2, il=1)\nend\n\n@testset \"check grad\" begin\n x = 1.0 - 4.0im\n y = 2.0 - 2.3im\n z = 3.0 + 1.0im\n r = 4.0\n for opm in [PlusEq, MinusEq]\n @test check_inv(opm(complex), (1+2.0im, 2.0, 3.0); verbose=true)\n @test ccheck_grad(opm(complex), (1+2.0im, 2.0, 3.0); verbose=true, iloss=1)\n for (subop, args) in [\n (opm(identity), (x,y)), (opm(+), (x, y, z)),\n (opm(-), (x, y, z)), (opm(*), (x, y, z)),\n (opm(/), (x, y, z)), (opm(^), (x, y, r)),\n (opm(exp), (x, y)), (opm(log), (x, y)),\n (opm(inv), (x, y))\n ]\n @test ccheck_grad(subop, args; verbose=true, iloss=1)\n r1 = subop(args...)\n r2 = [(opm == (PlusEq) ? Base.:+ : Base.:-)(args[1], subop.f(args[2:end]...)), args[2:end]...]\n @test all(r1 .≈ r2)\n end\n\n for (subop, args) in [\n (opm(angle), (r, y)), (opm(abs), (r, y)), (opm(abs), (r, 0.0im)),\n (opm(abs2), (r, y))\n ]\n @show subop, args\n r1 = [subop(args...)...]\n r2 = [(opm == (PlusEq) ? Base.:+ : Base.:-)(args[1], subop.f(args[2:end]...)), args[2:end]...]\n @test r1 ≈ r2\n @test check_grad(subop, args; verbose=true, iloss=1)\n end\n end\n for op in [NEG]\n @test check_inv(op, (x,); verbose=true)\n @test ccheck_grad(op, (x,); verbose=true, iloss=1)\n end\nend\n" }, { "path": "test/autodiff/gradfunc.jl", "content": "using Test, NiLang, NiLang.AD\n\nconst add = PlusEq(identity)\n\n@testset \"NGrad\" begin\n @test NGrad{3}(exp) isa NGrad{3,typeof(exp)}\nend\n\n@testset \"instr\" begin\n x, y = 3.0, 4.0\n @instr Grad(add)(x, y; iloss=1)\n @test grad(x) == 1.0\n @test grad(y) == 1.0\n @test check_inv(Grad(add), (3.0, 4.0); verbose=true, atol=1e-5, iloss=1)\n x, y = 3.0, 4.0\n @test check_grad(add, (x, y); iloss=1)\n\n x, y = 3.0, 4.0\n Grad(add)(x, NoGrad(y); iloss=1)\n @test grad(y) === 0.0\n\n @test check_inv(PlusEq(*), (0.4, 0.4, 0.5))\n @test MinusEq(*)(GVar(0.0, 1.0), GVar(0.4), GVar(0.6)) == (GVar(-0.24, 1.0), GVar(0.4, 0.6), GVar(0.6, 0.4))\n @test check_grad(PlusEq(*), (0.4, 0.4, 0.5); iloss=1)\n @test check_grad(MinusEq(*), (0.4, 0.4, 0.5); iloss=1)\nend\n\n@testset \"i\" begin\n @i function test1(a, b, out)\n a += b\n out += a * b\n end\n\n @i function tt(a, b)\n out ← 0.0\n test1(a, b, out)\n (~test1)(a, b, out)\n a += b\n out → 0.0\n end\n\n # compute (a+b)*b -> out\n x = 3.0\n y = 4.0\n out = 0.0\n @test check_grad(test1, (x, y, out); iloss=3)\n @test check_grad(tt, (x, y); iloss=1)\nend\n\n\n@testset \"broadcast\" begin\n # compute (a+b)*b -> out\n @i function test1(a, b)\n a .+= b\n end\n @i function test2(a, b, out, loss)\n a .+= b\n out .+= (a .* b)\n loss += out[1]\n end\n\n x = [3, 1.0]\n y = [4, 2.0]\n out = [0.0, 1.0]\n loss = 0.0\n # gradients\n @test check_grad(test2, (x, y, out, loss); iloss=4)\nend\n\n@testset \"broadcast 2\" begin\n # compute (a+b)*b -> out\n @i function test1(a, b)\n a += b\n end\n @i function test2(a, b, out)\n a += b\n out += (a * b)\n end\n\n # gradients\n a = 1.0\n b = 1.3\n c = 1.9\n @test check_grad(test2, (a,b,c); iloss=3)\n\n x = GVar([3, 1.0])\n y = GVar([4, 2.0])\n lout = GVar.([0.0, 1.0], [0.0, 2.0])\n @instr (~test2).(x, y, lout)\n @test grad.(lout) == [0,2.0]\n @test grad.(x) == [0, 4.0]\n @test grad.(y) == [0, 6.0]\nend\n\n@testset \"function call function\" begin\n # compute (a+b)*b -> out\n @i function test1(a, b)\n a += b\n end\n\n @i function test2(a, b, out)\n test1(a, out)\n (~test1)(a, out)\n out += (a * b)\n end\n\n a = 1.0\n b = 1.3\n c = 1.9\n @test check_grad(test2, (a,b,c); iloss=3)\nend\n\n@testset \"neg sign\" begin\n @i function test(out, x, y)\n out += x * (-y)\n end\n @test check_grad(test, (0.1, 2.0, -2.5); verbose=true, iloss=1)\nend\n\n@testset \"i\" begin\n @i function test1(a::T, b, out) where T<:Number\n add(a, b)\n out += a * b\n end\n @test isreversible(Grad(test1), Tuple{Number, Any,Any})\n @test isreversible(~Grad(test1), Tuple{Number, Any,Any})\n @test Grad(~test1) != ~(Grad(test1)) # this is not true\nend\n\n@testset \"gradient\" begin\n @test gradient((PlusEq(*)), (0.0, 2.0, 3.0); iloss=1) == (1.0, 3.0, 2.0)\nend\n" }, { "path": "test/autodiff/hessian_backback.jl", "content": "using NiLang, NiLang.AD, Test\nusing NiLang.AD: hessian_numeric\n\n@testset \"hessian\" begin\n h1 = hessian_backback(PlusEq(*), (0.0, 2.0, 3.0); iloss=1)\n h2 = hessian_numeric(PlusEq(*), (0.0, 2.0, 3.0); iloss=1)\n @test h1 ≈ h2\n\n @i function test(a,b,c,d)\n a += b*c\n a += b^d\n c += b/d\n ROT(a, c, d)\n b += d ^ 2\n a += c * d\n end\n h1 = hessian_backback(test, (0.0, 2.0, 1.0, 3.0); iloss=1)\n h2 = hessian_numeric(test, (0.0, 2.0, 1.0, 3.0); iloss=1)\n @show h2\n @test isapprox(h1, h2, atol=1e-8)\nend\n" }, { "path": "test/autodiff/instructs.jl", "content": "using NiLang, NiLang.AD\nusing Test\n\n@testset \"check grad\" begin\n for opm in [PlusEq, MinusEq]\n @test check_grad(opm(identity), (1.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(*), (1.0, 2.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(+), (1.0, 2.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(-), (1.0, 2.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(^), (1.0, 2.0, 2); verbose=true, iloss=1)\n @test check_grad(opm(^), (1.0, 2.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(inv), (1.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(sqrt), (1.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(abs), (1.0, -2.0); verbose=true, iloss=1)\n @test check_grad(opm(abs2), (1.0, -2.0); verbose=true, iloss=1)\n @test check_grad(opm(exp), (1.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(log), (1.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(sin), (1.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(sinh), (1.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(asin), (1.0, 0.2); verbose=true, iloss=1)\n @test check_grad(opm(cos), (1.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(cosh), (1.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(acos), (1.0, 0.2); verbose=true, iloss=1)\n @test check_grad(opm(tan), (1.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(tanh), (1.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(atan), (1.0, -2.0); verbose=true, iloss=1)\n @test check_grad(opm(atan), (1.0, -2.0, 1.5); verbose=true, iloss=1)\n @test check_grad(opm(convert), (Fixed43(0.5), 2.0); verbose=true, iloss=1)\n @test check_grad(opm(/), (1.0, 2.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(min), (1.0, 2.0, 3.0); verbose=true, iloss=1)\n @test check_grad(opm(max), (1.0, 2.0, 3.0); verbose=true, iloss=1)\n @test check_grad(opm(min), (1.0, 3.0, 2.0); verbose=true, iloss=1)\n @test check_grad(opm(max), (1.0, 3.0, 2.0); verbose=true, iloss=1)\n @test_broken check_grad(opm(÷), (1.0, 2.0, 2.0); verbose=true, iloss=1)\n @test gradient(opm(sqrt), (1.0, 0.0); iloss=1)[2] == 0\n end\n @test check_grad(NEG, (1.0,); verbose=true, iloss=1)\n @test check_grad(INV, (3.0,); verbose=true, iloss=1)\n @test check_grad(AddConst(2.0), (3.0,); verbose=true, iloss=1)\n @test check_grad(SubConst(2.0), (3.0,); verbose=true, iloss=1)\n @test check_grad(INC, (1.0,); verbose=true, iloss=1)\n @test check_grad(DEC, (1.0,); verbose=true, iloss=1)\n @test check_grad(ROT, (1.0, 2.0, 2.0); verbose=true, iloss=1)\n @test check_grad(ROT, (1.0, 2.0, 2.0); verbose=true, iloss=2)\n @test check_grad(IROT, (1.0, 2.0, 2.0); verbose=true, iloss=1)\n @test check_grad(IROT, (1.0, 2.0, 2.0); verbose=true, iloss=2)\n @test check_grad(HADAMARD, (3.0, 2.0); verbose=true, iloss=1)\n @test check_grad(HADAMARD, (3.0, 2.0); verbose=true, iloss=2)\nend\n\n@testset \"partial gvar\" begin\n @i function testf1(f, a, b)\n\tf(a, b, 2.0)\n end\n @i function testf2(f, a, b)\n\tf(a, 2.0, b)\n end\n for testf in [testf1, testf2]\n \tfor opm in [PlusEq, MinusEq]\n @test check_grad(testf, (opm(*), 1.0, 2.0); verbose=true, iloss=2)\n @test check_grad(testf, (opm(+), 1.0, 2.0); verbose=true, iloss=2)\n @test check_grad(testf, (opm(-), 1.0, 2.0); verbose=true, iloss=2)\n @test check_grad(testf, (opm(^), 1.0, 2.0); verbose=true, iloss=2)\n @test check_grad(testf, (opm(atan), 1.0, -2.0); verbose=true, iloss=2)\n @test check_grad(testf, (opm(/), 1.0, 2.0); verbose=true, iloss=2)\n\tend\n end\n @test check_grad(testf1, (ROT, 1.0, 2.0); verbose=true, iloss=2)\n @test check_grad(testf1, (ROT, 1.0, 2.0); verbose=true, iloss=3)\n @test check_grad(testf1, (IROT, 1.0, 2.0); verbose=true, iloss=2)\n @test check_grad(testf1, (IROT, 1.0, 2.0); verbose=true, iloss=3)\n # ROT and HADAMARD does not allow different types of rotation elements\nend\n\n@testset \"sincos\" begin\n @i function f(s, c, x)\n (s, c) += sincos(x)\n end\n @test check_grad(f, (1.0, 2.0, 2.0); verbose=true, iloss=1)\n @test check_grad(f, (1.0, 2.0, 2.0); verbose=true, iloss=2)\nend\n\n@testset \"AD over pop\" begin\n @i function mean(out!::T, x) where T\n anc ← zero(out!)\n for i=1:length(x)\n anc += x[i]\n end\n out! += anc / (@const length(x))\n FLOAT64_STACK[end+1] ↔ anc::T\n end\n\n @test check_grad(mean, (0.0, [1,2,3.0, 4.0]); iloss=1)\nend\n\n@testset \"AD over pipe\" begin\n @i function mean(out!, anc, x)\n for i=1:length(x)\n PlusEq(identity)(anc, x[i])\n SWAP(anc, x[i])\n end\n out! += anc / (@const length(x))\n end\n\n @test check_grad(mean, (0.0, 0.0, [1,2,3.0, 4.0]); iloss=1, verbose=true)\nend\n\n@testset \"push, load data\" begin\n stack = []\n val = [1,2,3]\n @instr PUSH!(stack, val)\n @test val == Int[]\n val = 3.0\n @instr PUSH!(stack, val)\n @test val == 0.0\n val = 3.0\n @instr PUSH!(stack, val)\n x = GVar(3.0)\n #@test_throws InvertibilityError @instr POP!(stack, x)\n z = 3.0\n @instr PUSH!(stack, z)\n z = GVar(0.0)\n @instr POP!(stack, z)\n @test z == GVar(3.0)\n x = [1.0, 2.0, 3.0]\n @instr PUSH!(stack, x)\n y = empty(x)\n @instr POP!(stack, y)\n @test y == GVar.([1,2,3.0])\n x = [1.0, 2.0, 3.0]\n @instr PUSH!(stack, x)\n y = empty(x)\n @instr POP!(stack, y)\n @test y == [1,2,3.0]\nend\n\n@testset \"dataviews\" begin\n @i function f(z, y, x)\n y += cos(x |> INV)\n z += tan(y |> AddConst(4.0))\n z += y * (x |> NEG |> SubConst(0.5) |> INV)\n z += sin(x |> INV)\n end\n @test check_grad(f, (0.2, 0.5, 0.8); iloss=1)\nend\n\n@testset \"additive identity\" begin\n struct TestAdd2{T}\n x::T\n y::Vector{T}\n end\n x = TestAdd2(GVar(1.0, 2.0), [GVar(2.0, 1.2)])\n y = TestAdd2(GVar(6.0, 3.0), [GVar(4.0, 4.1)])\n @test getfield.(MinusEq(identity)(x, y), :x) == getfield.((TestAdd2(GVar(-5.0, 2.0), [GVar(-2.0, 1.2)]), TestAdd2(GVar(6.0, 5.0), [GVar(4.0, 5.3)])), :x)\n x = TestAdd2(GVar(1.0, 2.0), [GVar(2.0, 1.2)])\n y = TestAdd2(GVar(6.0, 3.0), [GVar(4.0, 4.1)])\n @test getfield.(MinusEq(identity)(x, y), :y) == getfield.((TestAdd2(GVar(-5.0, 2.0), [GVar(-2.0, 1.2)]), TestAdd2(GVar(6.0, 5.0), [GVar(4.0, 5.3)])), :y)\nend\n" }, { "path": "test/autodiff/jacobian.jl", "content": "using NiLang, NiLang.AD\nusing Test\nusing NiLang.AD: wrap_bcastgrad\n\n@i function asarrayfunc(params; f, kwargs...)\n if (length(params) == 1, ~)\n f(params[1]; kwargs...)\n elseif (length(params) == 2, ~)\n f(params[1], params[2]; kwargs...)\n elseif (length(params) == 3, ~)\n f(params[1], params[2], params[3]; kwargs...)\n end\nend\n\n@testset \"bcastgrad\" begin\n T = AutoBcast{Int, 4}\n @test wrap_bcastgrad(T, ones(10)) == [GVar(1.0, AutoBcast(ones(4))) for i=1:10]\n @test wrap_bcastgrad(T, 3) == 3\n @test wrap_bcastgrad(T, NoGrad(3.0)) == 3.0\n @test wrap_bcastgrad(T, 3.0) == GVar(3.0, AutoBcast(ones(4)))\n @test wrap_bcastgrad(T, (3.0,)) == (GVar(3.0, AutoBcast(ones(4))),)\n @test wrap_bcastgrad(T, exp) == exp\n @test wrap_bcastgrad(T, Inv(exp)) == Inv(exp)\nend\n\n@testset \"jacobians\" begin\n for op in [PlusEq(*), PlusEq(/), PlusEq(^), ROT]\n j1 = NiLang.AD.jacobian(asarrayfunc, [0.3, 0.4, 2.0]; iin=1, f=op)\n j2 = NiLang.AD.jacobian_repeat(asarrayfunc, [0.3, 0.4, 2.0]; iin=1, f=op)\n @test j1 ≈ j2\n end\n\n for op in [PlusEq(identity), PlusEq(abs), SWAP, PlusEq(exp), PlusEq(log), PlusEq(sin), PlusEq(cos)]\n j1 = NiLang.AD.jacobian(asarrayfunc, [0.3, 0.4]; iin=1, f=op)\n j2 = NiLang.AD.jacobian_repeat(asarrayfunc, [0.3, 0.4]; iin=1, f=op)\n @test j1 ≈ j2\n end\n\n for op in [-, NEG]\n j1 = NiLang.AD.jacobian(asarrayfunc, [0.3]; iin=1, f=op)\n j2 = NiLang.AD.jacobian_repeat(asarrayfunc, [0.3]; iin=1, f=op)\n @test j1 ≈ j2\n end\nend\n\n@testset \"nograd\" begin\n @test AddConst(3.0)(NoGrad(2.0)) == NoGrad(5.0)\n @test SWAP(NoGrad(2.0), NoGrad(3.0)) == (NoGrad(3.0), NoGrad(2.0))\n @test PlusEq(*)(NoGrad(2.0), NoGrad(3.0), NoGrad(4.0)) == (NoGrad(14.0), NoGrad(3.0), NoGrad(4.0))\nend\n" }, { "path": "test/autodiff/manual.jl", "content": "using NiLang, Test\nusing NiLang.AD\n\ntest_func(x) = exp(x)\nNiLang.AD.primitive_grad(::typeof(test_func), x) = exp(x)\n\ntest_g(x, y; k=0) = x^k * y\nfunction NiLang.AD.primitive_grad(::typeof(test_g), x, y; k=0)\n return k*x^(k-1)*y, x^k\nend\n\n@testset \"primitive grad\" begin\n @test check_grad(PlusEq(test_func), (1.0, 1.0), iloss=1)\n @test check_grad(PlusEq(test_g), (1.0, 3.0, 2.0), k=2, iloss=1)\nend\n" }, { "path": "test/autodiff/stack.jl", "content": "using NiLang, Test, NiLang.AD\n\n@testset \"loaddata\" begin\n @test NiLang.loaddata(GVar(0.1), 0.3) == GVar(0.3)\n @test NiLang.loaddata(Complex(GVar(0.1, 0.2), GVar(0.2)), 0.3+0.6im) == Complex(GVar(0.3), GVar(0.6))\n @test NiLang.loaddata(typeof(Complex(GVar(0.1, 0.2), GVar(0.2))), 0.3+0.6im) == Complex(GVar(0.3), GVar(0.6))\n @test NiLang.loaddata(GVar(0.2, AutoBcast{Float64,3}(zeros(3))), 0.3) == GVar(0.3, AutoBcast{Float64,3}(zeros(3)))\n @test NiLang.loaddata((GVar(0.2, AutoBcast{Float64,3}(zeros(3))), 7), (0.3, 4)) == (GVar(0.3, AutoBcast{Float64,3}(zeros(3))), 4)\n @test NiLang.loaddata(typeof((GVar(0.2, AutoBcast{Float64,3}(zeros(3))), 7)), (0.3, 4)) == (GVar(0.3, AutoBcast{Float64,3}(zeros(3))), 4)\n @test NiLang.loaddata(4, 2.0) == 2\nend\n\n@testset \"push load\" begin\n x = (0.3, 3.0, [1,2,3.0])\n @instr PUSH!(x)\n t = (0.0, 0.0, Float64[])\n @test x == t && typeof(x) == typeof(t)\n y = (0.0, GVar(0.0), GVar{Float64,Float64}[])\n @instr POP!(y)\n t = (0.3, GVar(3.0), GVar([1,2, 3.0]))\n @test y == t && typeof(y) == typeof(t)\n\n x = [0.3, 3.0, [1,2,3.0]]\n @instr PUSH!(x)\n t = []\n @test x == t && typeof(x) == typeof(t)\n y = []\n @instr POP!(y)\n t = [0.3, GVar(3.0), GVar([1,2, 3.0])]\n @test y == t && typeof(y) == typeof(t)\n\n x = (0.3, 3.0, [1,2,3.0])\n @instr @invcheckoff PUSH!(x)\n t = (0.0, 0.0, Float64[])\n @test x == t && typeof(x) == typeof(t)\n y = (0.0, GVar(0.0), GVar(zeros(0)))\n @instr @invcheckoff POP!(y)\n t = (0.3, GVar(3.0), GVar([1,2, 3.0]))\n @test y == t && typeof(y) == typeof(t)\n\n x = (0.3, 3.0, [1,2,3.0])\n @instr @invcheckoff COPYPUSH!(x)\n t = (0.3, 3.0, [1,2,3.0])\n @test x == t && typeof(x) == typeof(t)\n y = (0.3, GVar(t[2]), GVar(t[3]))\n @instr @invcheckoff COPYPOP!(y)\n t = (0.3, GVar(3.0), GVar([1,2, 3.0]))\n @test y == t && typeof(y) == typeof(t)\n\n x = (0.3, 3.0, [1,2,3.0])\n @instr COPYPUSH!(x)\n t = (0.3, 3.0, [1,2,3.0])\n @test x == t && typeof(x) == typeof(t)\n y = (0.3, GVar(t[2]), GVar(t[3]))\n @instr COPYPOP!(y)\n t = (0.3, GVar(3.0), GVar([1,2, 3.0]))\n @test y == t && typeof(y) == typeof(t)\n\n x = [0.3, 3.0, [1,2,3.0]]\n @instr COPYPUSH!(x)\n t = [0.3, 3.0, [1,2,3.0]]\n @test x == t && typeof(x) == typeof(t)\n y = [0.3, GVar(t[2]), GVar(t[3])]\n @instr COPYPOP!(y)\n t = [0.3, GVar(3.0), GVar([1,2, 3.0])]\n @test y == t && typeof(y) == typeof(t)\nend" }, { "path": "test/autodiff/ulog.jl", "content": "using NiLang, NiLang.AD\nusing Test, Random\nusing FixedPointNumbers\nusing NiLangCore: default_constructor\nusing FiniteDifferences\n\n@testset \"ULogarithmic\" begin\n\t@test check_grad(PlusEq(gaussian_log), (1.0, 2.0); iloss=1)\n\tfunction muleq(f, x::T, y::T, z::T) where T\n x = default_constructor(ULogarithmic{T}, x)\n y = default_constructor(ULogarithmic{T}, y)\n z = default_constructor(ULogarithmic{T}, z)\n\t\tx *= f(y, z)\n\t\tx.log\n\tend\n\tg1, = FiniteDifferences.grad(central_fdm(5,1), arr->muleq(+, arr...), [7.0, 5.0, 3.0])\n x, y, z = default_constructor(ULogarithmic{Float64}, 7.0),\n default_constructor(ULogarithmic{Float64}, 5.0),\n default_constructor(ULogarithmic{Float64}, 3.0)\n\t@instr (MulEq(+))(x, y, z)\n\t@instr GVar(x)\n\t@instr GVar(y)\n\t@instr GVar(z)\n\t@instr x.log.g += 1\n\t@instr (~MulEq(+))(x, y, z)\n\t@test grad(x.log) ≈ g1[1]\n\t@test grad(y.log) ≈ g1[2]\n\t@test grad(z.log) ≈ g1[3]\n\n\tg2, = FiniteDifferences.grad(central_fdm(5,1), arr->muleq(-, arr...), [7.0, 5.0, 3.0])\n x, y, z = default_constructor(ULogarithmic{Float64}, 2.0),\n default_constructor(ULogarithmic{Float64}, 5.0),\n default_constructor(ULogarithmic{Float64}, 3.0)\n\t@instr (MulEq(-))(x, y, z)\n\t@instr GVar(x)\n\t@instr GVar(y)\n\t@instr GVar(z)\n\t@instr x.log.g += 1\n\t@instr (~MulEq(-))(x, y, z)\n\t@test grad(x.log) ≈ g2[1]\n\t@test grad(y.log) ≈ g2[2]\n\t@test grad(z.log) ≈ g2[3]\nend\n\n@testset \"iexp\" begin\n\t@i function i_exp(y!::T, x::T) where T<:Union{Fixed, GVar{<:Fixed}}\n\t @invcheckoff begin\n\t @routine begin\n\t s ← one(ULogarithmic{T})\n\t lx ← one(ULogarithmic{T})\n\t k ← 0\n\t end\n\t lx *= convert(x)\n\t y! += convert(s)\n\t @from k==0 while s.log > -20\n\t k += 1\n\t s *= lx / k\n\t y! += convert(s)\n\t end\n\t ~(@from k==0 while s.log > -20\n\t k += 1\n\t s *= x / k\n\t end)\n\t lx /= convert(x)\n\t ~@routine\n\t end\n\tend\n\n\tx = Fixed43(3.5)\n\tres = i_exp(Fixed43(0.0), x)[1]\n\tgx = grad(Grad(i_exp)(Val(1), Fixed43(0.0), x)[3])\n\t@test res ≈ exp(3.5)\n\t@test gx ≈ exp(3.5)\nend\n" }, { "path": "test/autodiff/vars.jl", "content": "using NiLang, NiLang.AD\nusing Test\n\n@testset \"gvar\" begin\n g1 = GVar(0.0)\n @test (~GVar)(g1) === 0.0\n @assign (g1 |> grad) 0.5\n @test g1 === GVar(0.0, 0.5)\n @test_throws InvertibilityError (~GVar)(g1)\n @test !almost_same(GVar(0.0), GVar(0.0, 1.0))\n @test zero(GVar(3.0, 2.0)) == GVar(0.0)\n @test one(GVar(3.0, 2.0)) == GVar(1.0)\n @test iszero(GVar(0.0, 2.0))\n @test zero(GVar(2, AutoBcast([1, 0, 0]))) == GVar(0, AutoBcast([0, 0, 0]))\n @test GVar(true) == true\n @test grad(\"x\") == \"\"\n @test grad((1.0, GVar(1.0, 2.0))) == (0.0,2.0)\n @test grad(grad) == 0\n @test grad((1.0, 2.0)) == (0.0,0.0)\n @test grad([1.0, 2.0]) == [0.0,0.0]\n @test grad([GVar(1.0, 3.0), GVar(2.0, 1.0)]) == [3.0,1.0]\n @test grad(Complex(GVar(1.0, 3.0), GVar(2.0, 1.0))) == Complex(3.0,1.0)\n @test grad(Complex(1.0, 2.0)) == Complex(0.0,0.0)\nend\n\n\n@testset \"assign\" begin\n arg = (1,2,GVar(3.0))\n @assign (arg.:3).g 4.0\n @test arg[3].g == 4.0\n gv = GVar(1.0, GVar(0.0))\n @test gv.g.g === 0.0\n @assign gv.g.g 7.0\n @test gv.g.g === 7.0\n gv = GVar(1.0, GVar(0.0))\n @assign gv |> grad |> grad 0.0\n @test gv.g.g === 0.0\n args = (GVar(0.0, 1.0),)\n @assign (args.:1 |> grad) 0.0\n @test args[1].g == 0.0\n arr = [1.0]\n arr0 = arr\n @assign arr[] 0.0\n @test arr[] == 0.0\n @test arr === arr0\nend\n\n@testset \"assign tuple\" begin\n x = 0.3\n @instr for i=1:length(x) GVar(x) end\n @test x === GVar(0.3)\nend\n\n\n@testset \"assign bcast func\" begin\n # vector bcast\n x = [GVar(0.1, 0.1), GVar(0.2, 0.2)]\n res = [1.0, 2.0]\n @assign (x .|> value) res\n @test x == [GVar(1.0, 0.1), GVar(2.0, 0.2)]\n\n # tuple bcast\n x = (GVar(0.1, 0.1), GVar(0.2, 0.2))\n res = (1.0, 2.0)\n @assign (x .|> value) res\n @test x == (GVar(1.0, 0.1), GVar(2.0, 0.2))\nend\n\n@testset \"GVar over general type\" begin\n struct ABC{T1, T2}\n a::T1\n b::T1\n c::T2\n end\n x = ABC(1, 2, 3.0)\n @test GVar(x) == ABC(1, 2, GVar(3.0))\n @test GVar(x, x) == ABC(GVar(1, 1), GVar(2, 2), GVar(3.0, 3.0))\n @test (~GVar)(ABC(1, 2, GVar(3.0))) == x\n @test grad(ABC(1, 2, GVar(3.0, 2.0))) == ABC(0, 0, 2.0)\n @test GVar(1.0 + 2.0im , 2.0im + 4.0im) == Complex(GVar(1.0, 2.0), GVar(2.0, 4.0))\n @test GVar((1.0, 2.0im) , (2.0im, 4.0im)) == (GVar(1.0, 2.0), Complex(GVar(0.0), GVar(2.0, 4.0)))\n\n # without type parameter\n struct EFG\n x\n end\n @test GVar(EFG) == EFG\n @test grad(EFG(GVar(2.0, 3.0))) == EFG(3.0)\nend\n\n@testset \"dict\" begin\n @i function f()\n d ← Dict(1=>GVar(1.0, 2.0))\n d → Dict(1=>GVar(1.0))\n end\n @test f() == ()\nend\n\n@testset \"NoGrad\" begin\n a = NoGrad(0.5)\n @test a isa NoGrad\n @test zero(a) == NoGrad(0.0)\n @test (~NoGrad)(a) === 0.5\n @test -NoGrad(0.5) == NoGrad(-0.5)\n\n a2 = NoGrad{Float64}(a)\n @test a2 === a\n println(a2)\n @test chfield(a2, NoGrad, NoGrad(0.4)) === 0.4\n\n @test unwrap(NoGrad(a)) == 0.5\n @test NoGrad(a) < 0.6\n @test NoGrad(a) <= 0.6\n @test NoGrad(a) >= 0.4\n @test a ≈ 0.5\n @test a == 0.5\n @test a > 0.4\n @test isless(a, 0.6)\nend\n\n" }, { "path": "test/complex.jl", "content": "using Test, NiLang\n\n@testset \"complex\" begin\n a = 1.0+ 2im\n @instr (a |> real) += 2\n @instr (a |> imag) += 2\n @test a == 3.0 + 4im\n\n a = 1.0+ 2im\n @instr a += complex(2.0, 2.0)\n @test a == 3.0 + 4.0im\n @i function f(loss, a::Complex{T}, b) where T\n @routine begin\n c ← zero(a)\n sq ← zero(T)\n c += a * b\n sq += (c |> real) ^ 2\n sq += (c |> imag) ^ 2\n end\n loss += sq ^ 0.5\n ~@routine\n end\n a = 1.0 + 2.0im\n b = 2.0 + 1.0im\n loss = 0.0\n @instr f(loss, a, b)\n @test loss ≈ abs(a*b)\nend\n\n@testset \"complex arithmetics\" begin\n for op in [exp, log, identity]\n x, y = 2.0+1.0im, 0.5+0.2im\n @instr x += op(y)\n @test x ≈ 2.0+1.0im + op(0.5+0.2im)\n end\n for op in [SWAP, HADAMARD]\n x, y = 2.0+1.0im, 0.5+0.2im\n @instr op(x, y)\n @test x ≈ op(2.0+1.0im, 0.5+0.2im)[1]\n @test y ≈ op(2.0+1.0im, 0.5+0.2im)[2]\n end\n for op in [NEG, INC, DEC]\n x = 2.0+1.0im\n @instr op(x)\n @test x ≈ op(2.0+1.0im)\n end\n for op in [^, /, +, -]\n x, y, z = 2.0+1.0im, 0.5+0.2im, 0.8-2.0im\n @instr PlusEq(op)(x, y, z)\n @test x ≈ 2.0+1.0im + op(0.5+0.2im, 0.8-2.0im)\n end\nend\n\n" }, { "path": "test/instructs.jl", "content": "using NiLang\nusing Test\n\n@testset \"identity\" begin\n x, y = 0.2, 0.5\n @instr x += identity(y)\n @test x == 0.7 && y==0.5\nend\n\n@testset \"*, /\" begin\n x, y, out = 2.0, 2.0, 1.0\n @instr out += x * y\n @test x == 2.0 && y == 2.0 && out == 5.0\n x, y, out = 2.0, 2.0, 1.0\n @instr out += x / y\n @test x == 2.0 && y == 2.0 && out == 2.0\n\n out = Fixed43(0.0)\n x = 1\n @instr out += x/2\n @test out === Fixed43(0.5)\n @instr out -= x/2\n @test out === Fixed43(0.0)\nend\n\n@testset \"SWAP\" begin\n x, y = 1, 2\n @instr SWAP(x, y)\n @test x == 2 && y == 1\nend\n\n@testset \"NEG\" begin\n x = 0.3\n @instr NEG(x)\n @test x == -0.3\n @test check_inv(NEG, (x,))\nend\n\n@testset \"INV\" begin\n x = 0.2\n @instr INV(x)\n @test x == 5.0\n @test check_inv(INV, (x,))\nend\n\n@testset \"AddConst\" begin\n x = 0.3\n @instr AddConst(4.0)(x)\n @test x == 4.3\n @test check_inv(AddConst(4.0), (x,))\n\n x = 0.3\n @instr SubConst(4.0)(x)\n @test x == -3.7\n @test check_inv(SubConst(4.0), (x,))\nend\n\n@testset \"FLIP\" begin\n x = false\n @instr FLIP(x)\n @test x == true\n @test check_inv(FLIP, (x,))\nend\n\n@testset \"ROT\" begin\n x, y, θ = 0.0, 1.0, π\n @test check_inv(ROT, (x, y, θ); verbose=true)\n @test check_inv(IROT, (x, y, θ); verbose=true)\nend\n\n@testset \"INC, DEC\" begin\n x = Int32(2)\n @instr INC(x)\n @test x === Int32(3)\n @instr DEC(x)\n @test x === Int32(2)\nend\n\n@testset \"HADAMARD\" begin\n x = 0.5\n y = 0.8\n @test check_inv(HADAMARD, (x, y))\nend\n\n@testset \"dataviews\" begin\n @i function f(z, y, x)\n y += cos(x |> INV)\n z += tan(y |> AddConst(4.0))\n z += y * (x |> NEG |> SubConst(0.5) |> INV)\n z += sin(x |> INV)\n end\n @test check_inv(f, (0.2, 0.5, 0.8))\nend\n\n@testset \"fixed point arithmetics\" begin\n for op in [exp, log, sin, sinh, asin, cos, cosh, acos, tan, tanh, atan]\n x, y = Fixed43(2.0), Fixed43(0.5)\n @instr x += op(y)\n @test x ≈ 2.0 + op(0.5)\n end\n for op in [SWAP, HADAMARD]\n x, y = Fixed43(2.0), Fixed43(0.5)\n @instr op(x, y)\n @test x ≈ op(2.0, 0.5)[1]\n @test y ≈ op(2.0, 0.5)[2]\n end\n for op in [NEG, INC, DEC]\n x = Fixed43(2.0)\n @instr op(x)\n @test x ≈ op(2.0)\n end\n for op in [^, /]\n x, y, z = Fixed43(2.0), Fixed43(0.5), Fixed43(0.8)\n @instr PlusEq(op)(x, y, z)\n @test x ≈ 2.0 + op(0.5, 0.8)\n end\nend\n\n@testset \"additive identity\" begin\n struct TestAdd{T}\n x::T\n y::Vector{T}\n end\n @test getfield.(PlusEq(identity)(TestAdd(1, [2]), TestAdd(10, [2])), :x) == (TestAdd(11, [4]).x, TestAdd(10, [2]).x)\n @test getfield.(PlusEq(identity)(TestAdd(1, [2]), TestAdd(10, [2])), :y) == (TestAdd(11, [4]).y, TestAdd(10, [2]).y)\nend" }, { "path": "test/macros.jl", "content": "using Test, NiLang\nusing NiLang: auto_alloc, auto_expand\n\nNiLang.alloc(::typeof(NiLang.i_sum), x::AbstractArray{T}) where T = zero(T)\n\n@testset begin\n @test auto_alloc(:(y = exp(x))) == Expr(:block, :(y ← $alloc(exp, x)), :(y += exp(x)))\n ex1 = :(PlusEq(sin)(z, sin(x + 2y)))\n ex2 = auto_expand(ex1)\n @test length(ex2.args) == 13\n @i function test(x, y, z)\n #@auto_expand z += sin(x + 2y)\n @invcheckoff @auto_expand z += sin(x + 2y)\n end\n x, y, z = 1.0, 2.0, 3.0\n @test test(x, y, z)[3] == z + sin(x + 2y)\n @test check_inv(test, (x, y, z))\n @i function test(x, y, z, a)\n @auto_expand PlusEq(sin)(Complex{}(x, y), Complex{}(z, sin(a)))\n end\n x, y, z, a = 1.0, 2.0, 3.0, 4.0\n @test Complex(test(x, y, z, a)[1:2]...) == 1+im*y + sin(z+im*sin(a))\n @test check_inv(test, (x, y, z, a))\n\n @i function test2(y, x)\n @routine begin\n @auto_alloc i_sum(z, x)\n end\n y += z\n ~@routine\n end\n @test test2(1.0, [2,3.0])[1] == 6.0\n @test check_inv(test2, (1.0, [2,3.0]))\nend" }, { "path": "test/runtests.jl", "content": "using NiLang\nusing Test\n\n@testset \"vars.jl\" begin\n include(\"vars.jl\")\nend\n\n@testset \"utils.jl\" begin\n include(\"utils.jl\")\nend\n\n@testset \"wrappers.jl\" begin\n include(\"wrappers.jl\")\nend\n\n@testset \"instructs.jl\" begin\n include(\"instructs.jl\")\nend\n\n@testset \"complex.jl\" begin\n include(\"complex.jl\")\nend\n\n@testset \"ulog.jl\" begin\n include(\"ulog.jl\")\nend\n\n@testset \"macros.jl\" begin\n include(\"macros.jl\")\nend\n\n@testset \"autobcast.jl\" begin\n include(\"autobcast.jl\")\nend\n\n@testset \"autodiff\" begin\n include(\"autodiff/autodiff.jl\")\nend\n\n@testset \"stdlib\" begin\n include(\"stdlib/stdlib.jl\")\nend\n" }, { "path": "test/stdlib/base.jl", "content": "using NiLang, NiLang.AD\nusing Test\n\n@testset \"sqdistance\" begin\n @test i_sqdistance(0.0, [1.0, 0.0], [0.0, 1.0])[1] == 2.0\nend\n" }, { "path": "test/stdlib/bennett.jl", "content": "using Test\nusing NiLang, NiLang.AD\n\n@testset \"integrate\" begin\n FT = Float64\n n = 100\n h = FT(π/n)\n dt = FT(0.01)\n α = FT(4e-2)\n @i function step!(dest::AbstractArray{T}, src::AbstractArray{T}; α, h, dt) where T\n n ← length(dest)\n @invcheckoff for i=1:n\n @routine begin\n @zeros T cum g h2 αcum\n cum += src[mod1(i+1, n)] + src[mod1(i-1, n)]\n cum -= 2*src[i]\n αcum += cum * α\n h2 += h^2\n g += αcum/h2\n end\n dest[i] += src[i]\n dest[i] += dt*g\n ~@routine\n end\n n → length(dest)\n end\n x = zeros(FT, n)\n x[n÷2] = 1\n #state = Dict{Int,Vector{FT}}()\n k = 4\n N = 100\n x_last = NiLang.direct_emulate(step!, FT.(x); N=N, α=α, h=h, dt=dt)\n log1 = NiLang.BennettLog()\n log2 = NiLang.BennettLog()\n log3 = NiLang.BennettLog()\n _, x_last_b, _ = bennett(step!, zero(FT.(x)), FT.(x); k=k, N=N, α=α, h=h, dt=dt, logger=log1)\n _, x_last_b2 = bennett!(step!, Dict(1=>FT.(x)); k=k, N=N, α=α, h=h, dt=dt, logger=log2)\n _, x_last_b3 = bennett!([step! for _=1:100], Dict(1=>FT.(x)); k=k, N=N, α=α, h=h, dt=dt, logger=log3)\n @test sum(x_last_b) ≈ 1\n @test x_last ≈ x_last_b\n @test x_last ≈ x_last_b2[N+1]\n @test x_last ≈ x_last_b3[N+1]\n @test length(log1.fcalls) > length(log2.fcalls)\n @test length(log1.fcalls) < 2*length(log2.fcalls)\n @test length(log3.fcalls) == length(log2.fcalls)\n\n @i function loss(out, step, y, x; kwargs...)\n bennett((@skip! step), y, x; kwargs...)\n out += y[n÷2]\n end\n @i function loss2(out, step, d; N, kwargs...)\n bennett!((@skip! step), d; N, kwargs...)\n out += d[N+1][n÷2]\n end\n _, _, _, gx = NiLang.AD.gradient(loss, (0.0, step!, zero(x), copy(x)); iloss=1, k=k, N=N, α=α, h=h, dt=dt)\n _, _, gx2 = NiLang.AD.gradient(loss2, (0.0, step!, Dict(1=>copy(x))); iloss=1, k=k, N=N, α=α, h=h, dt=dt)\n x_last_2 = NiLang.direct_emulate(step!, (x2=copy(x); x2[n÷2]+=1e-5; FT.(x2)); N=N, α=α, h=h, dt=dt)\n @test gx[n÷2] ≈ (x_last_2 - x_last)[n÷2]/1e-5\n @test gx2[1][n÷2] ≈ (x_last_2 - x_last)[n÷2]/1e-5\nend" }, { "path": "test/stdlib/blas.jl", "content": "using Test\nusing LinearAlgebra\nusing NiLang, NiLang.AD\n\n@testset \"i_norm2, dot\" begin\n out = 0.0im\n vec = [1.0im, 2.0, 3.0]\n vec2 = [1.0, 2.0im, 5.0]\n @instr i_norm2(out, vec)\n @test out ≈ norm(vec)^2\n @test check_inv(i_norm2, (out, vec))\n\n out = 0.0im\n vec = [1.0im, 2.0, 3.0]\n vec2 = [1.0, 2.0im, 5.0]\n @instr i_dot(out, vec, vec2)\n @test out ≈ dot(vec, vec2)\n @test check_inv(i_dot, (out, vec, vec2))\n\n out = 0.0\n vec = [1.0, 2.0, 3.0]\n vec2 = [1.0, 2.0, 5.0]\n @test check_grad(i_norm2, (out, vec); verbose=true, iloss=1)\n\n out = 0.0\n @instr i_dot(out, vec, vec2)\n @test out ≈ dot(vec, vec2)\n @test check_inv(i_dot, (out, vec, vec2))\n\n @test check_grad(i_dot, (0.0, vec, vec2); verbose=true, iloss=1)\n\n m = randn(4,4)\n n = randn(4,4)\n out = 0.0\n @instr i_dot(out, m[:,2], n[:,4])\n @test out ≈ dot(m[:,2], n[:,4])\n @test check_inv(i_dot, (out, m[:,2], n[:,4]))\n\n @test check_grad(i_dot, (0.0, vec, vec2); verbose=true, iloss=1)\nend\n\nfunction naive_umm!(x, params)\n N = size(x, 1)\n k = 0\n for j=1:N\n for i=N-1:-1:j\n k += 1\n a, b = rot(x[i],x[i+1],params[k])\n x[i], x[i+1] = a, b\n end\n end\nend\n\nfunction inv_naive_umm!(x, params)\n N = size(x, 1)\n k = N*(N-1) ÷ 2\n for j=N:-1:1\n for i=j:N-1\n a, b = rot(x[i],x[i+1],-params[k])\n x[i], x[i+1] = a, b\n k -= 1\n end\n end\nend\n@testset \"naive unitary\" begin\n x = randn(200)\n params = randn(100*199).*2π\n\n x0 = copy(x)\n params0 = copy(params)\n naive_umm!(x, params)\n inv_naive_umm!(x, params)\n @test params ≈ params0\n @test x ≈ x0\nend\n\n\n\n@testset \"unitary\" begin\n x = randn(20)\n params = randn(10*19) * 2π\n\n x0 = copy(x)\n params0 = copy(params)\n Nx = length(x)\n @instr i_umm!(x, params)\n x1 = copy(x0)\n params1 = copy(params0)\n naive_umm!(x1, params1)\n @test params ≈ params1\n @test x ≈ x1\n @instr (~i_umm!)(x, params)\n @test params ≈ params0\n @test x ≈ x0\n @test check_inv(i_umm!, (x, params))\nend\n" }, { "path": "test/stdlib/linalg.jl", "content": "using Test\nusing NiLang, NiLang.AD\nusing LinearAlgebra\nusing Random\n\n@testset \"inv\" begin\n Random.seed!(2)\n\n id = [1 0 0; 0 1 0; 0 0 1.0]\n @test check_inv(i_inv!, (randn(3, 3), randn(3, 3)))\n @test check_inv(PlusEq(det), (0.3, randn(3, 3)))\n @test check_inv(PlusEq(logdet), (0.3, rand(3, 3) .+ id))\n @test check_grad(PlusEq(det), (0.3, randn(3, 3)), iloss=1)\n @test check_grad(PlusEq(logdet), (0.3, rand(3, 3) .+ id), iloss=1)\n\n @i function loss(out!, y, A)\n i_inv!(y, A)\n out! += y[1,1]\n end\n @test check_grad(loss, (0.0, randn(3, 3), randn(3, 3)); iloss=1)\nend\n\n@testset \"affine\" begin\n Random.seed!(2)\n A = randn(5, 5)\n b = randn(5)\n x = randn(5)\n y! = zeros(5)\n @test i_affine!(y!, A, b, x)[1] ≈ A*x + b\nend\n" }, { "path": "test/stdlib/mapreduce.jl", "content": "using NiLang, Test\n\n@testset \"filter and mapfoldl\" begin\n @i function f(z, y, x)\n i_filter!((@skip! x -> x < 0), y, x)\n i_mapfoldl((@skip! exp), (@skip! PlusEq(identity)), z, y)\n end\n @test f(0.0, Float64[], [-1, -0.5, 3])[1] == exp(-0.5) + exp(-1)\nend\n" }, { "path": "test/stdlib/nnlib.jl", "content": "using Test, Random\nusing NiLang, NiLang.AD\n\nfunction _sce(x::AbstractArray{T,N}, p) where {T,N}\n x = x .- maximum(x; dims=N) # avoid data overflow\n rho = exp.(x)\n Z = sum(rho; dims=N)\n return dropdims(sum((log.(Z) .- x) .* p; dims=N), dims=N)\nend\n\n\n@testset \"softmax_crossentropy\" begin\n Random.seed!(2)\n x = randn(10)\n x0 = copy(x)\n p = randn(10); p=p./maximum(p)\n res = _sce(x, p)\n imax = 0\n Z = 0.0\n out = 0.0\n xmax = 0.0\n x_ = x\n p_ = p\n @instr i_softmax_crossentropy(x_, p_, imax, xmax, Z, out)\n @show Z\n @test isapprox(imax, argmax(x0), atol=1e-8)\n @test isapprox(out, res[], atol=1e-8)\n @instr (~i_softmax_crossentropy)(x_, p_, imax, xmax, Z, out)\n args = x_, p_, imax, xmax, Z, out\n @test check_inv(i_softmax_crossentropy, args)\n args = x_, p_, imax, xmax, Z, out\n @test check_grad(i_softmax_crossentropy, args; iloss=6, verbose=true)\nend\n\n@testset \"logsumexp\" begin\n\tfunction logsumexp2(x)\n\t \tmx = maximum(x)\n\t \tlog.(sum(exp.(x .- mx))) .+ mx\n\tend\n\n\tx = randn(100)\n\t@test i_ascending!(Float64[], Int[], x)[1][end] == maximum(x)\n\t@test i_logsumexp(0.0, 0.0, Float64[], Int[], x)[1] ≈ logsumexp2(x)\n\t@test check_inv(i_logsumexp, (0.0, 0.0, Float64[], Int[], x))\nend\n" }, { "path": "test/stdlib/sparse.jl", "content": "using NiLang\nusing SparseArrays\nusing Test, Random\n\n@testset \"dot\" begin\n Random.seed!(2)\n sp1 = sprand(10, 10,0.3)\n sp2 = sprand(10, 10,0.3)\n @test SparseArrays.dot(sp1, sp2) ≈ i_dot(0.0, sp1, sp2)[1]\nend\n\n@testset \"mul!\" begin\n Random.seed!(2)\n sp1 = sprand(10, 10,0.3)\n v = randn(10)\n out = zero(v)\n @test SparseArrays.mul!(copy(out), sp1, v, 0.5, 1) ≈ i_mul!(copy(out), sp1, v, 0.5, 1)[1]\nend\n" }, { "path": "test/stdlib/statistics.jl", "content": "import Statistics\nusing Test, Random\nusing NiLang, NiLang.AD\nusing Distributions\n\n@testset \"statistics\" begin\n x = randn(100)\n y = randn(100)\n @test i_mean_sum(0.0, 0.0, x)[1] ≈ Statistics.mean(x)\n info = VarianceInfo(Float64)\n @test almost_same(i_var_mean_sum(info, copy(x))[1], VarianceInfo(Statistics.var(x), Statistics.var(x)*99, Statistics.mean(x), sum(x)))\n @test almost_same((~i_var_mean_sum)(i_var_mean_sum(info, copy(x))...), (info, x))\n @test almost_same(i_cor_cov(0.0, 0.0, copy(x), copy(y)), (Statistics.cor(x,y), Statistics.cov(x,y), x, y))\n @test almost_same((~i_cor_cov)(i_cor_cov(0.0, 0.0, copy(x), copy(y))...), (0.0, 0.0, x, y))\nend\n\n@testset \"normal log pdf\" begin\n out = 0.0\n x = 1.0\n μ = 0.3\n σ = 1.5\n l1 = i_normal_logpdf(out, x, μ, σ)\n distri = Normal(μ, σ)\n l2 = logpdf(distri, x)\n @test l1[1] ≈ l2\n @test check_inv(i_normal_logpdf, (out, x, μ, σ))\nend\n" }, { "path": "test/stdlib/stdlib.jl", "content": "include(\"base.jl\")\ninclude(\"blas.jl\")\ninclude(\"linalg.jl\")\ninclude(\"statistics.jl\")\ninclude(\"nnlib.jl\")\ninclude(\"sparse.jl\")\ninclude(\"mapreduce.jl\")\ninclude(\"bennett.jl\")\n" }, { "path": "test/ulog.jl", "content": "using NiLang, NiLang.AD\nusing Test, Random\nusing NiLangCore: default_constructor\n\n@testset \"basic instructions, ULogarithmic\" begin\n x, y = default_constructor(ULogarithmic{Int}, 1),\n default_constructor(ULogarithmic{Int}, 2)\n\t@instr x *= y\n @test x == default_constructor(ULogarithmic{Int}, 3)\n @test y == default_constructor(ULogarithmic{Int}, 2)\n\n\t@test PlusEq(gaussian_log)(1.0, 2.0) == (1.0+log(1+exp(2.0)), 2.0)\n\n x, y, z = default_constructor(ULogarithmic{Float64}, 7.0),\n default_constructor(ULogarithmic{Float64}, 2.0),\n default_constructor(ULogarithmic{Float64}, 3.0)\n\t@instr x *= y + z\n\t@test check_inv(MulEq(+), (x, y, z))\n\t@test x.log ≈ log(exp(7.0) * (exp(2.0) + exp(3.0)))\n x, y, z = default_constructor(ULogarithmic{Float64}, 7.0),\n default_constructor(ULogarithmic{Float64}, 5.0),\n default_constructor(ULogarithmic{Float64}, 3.0)\n\t@instr x *= y - z\n\t@test x.log ≈ log(exp(7.0) * (exp(5.0) - exp(3.0)))\n\n x, y, z = default_constructor(ULogarithmic{Float64}, 7.0),\n default_constructor(ULogarithmic{Float64}, 5.0),\n default_constructor(ULogarithmic{Float64}, 3.0)\n\t@instr x *= y^3.4\n\t@test x.log ≈ log(exp(5.0)^3.4 * exp(7.0))\n\n x, y, z = default_constructor(ULogarithmic{Float64}, 7.0),\n default_constructor(ULogarithmic{Float64}, 5.0),\n default_constructor(ULogarithmic{Float64}, 3.0)\n\t@instr x *= 3\n\t@test x.log ≈ log(exp(7.0) * 3)\nend\n\n\n@testset \"error on += and -=\" begin\n @i function f(x::ULogarithmic)\n x += ULogarithmic(3.0)\n end\n \n @test_throws MethodError f(ULogarithmic(2.0))\n @test_throws MethodError (~f)(ULogarithmic(2.0))\nend\n" }, { "path": "test/utils.jl", "content": "using NiLang\nusing Test\n\n@testset \"vec dataview\" begin\n @i function f(x::AbstractVector, y::AbstractMatrix)\n x .+= (y |> vec)\n vec(y)[5] += x[4]\n end\n x = zeros(25)\n y = ones(5,5)\n z = ones(5,5)\n z[5] = 2.0\n @instr f(x, y)\n @test y == z\nend\n" }, { "path": "test/vars.jl", "content": "using Test, NiLang, NiLangCore\n\n@testset \"@zeros\" begin\n @test (@macroexpand @zeros Float64 a b c) == :(begin\n a ← zero(Float64)\n b ← zero(Float64)\n c ← zero(Float64)\n end) |> NiLangCore.rmlines\n\n @test (@macroexpand @ones Float64 a b c) == :(begin\n a ← one(Float64)\n b ← one(Float64)\n c ← one(Float64)\n end) |> NiLangCore.rmlines\nend\n" }, { "path": "test/wrappers.jl", "content": "using NiLang, Test\n\n@testset \"partial\" begin\n x = Partial{:im}(3+2im)\n println(x)\n @test x === Partial{:im,Complex{Int64},Int64}(3+2im)\n @test value(x) == 2\n @test chfield(x, value, 4) == Partial{:im}(3+4im)\n @test zero(x) == Partial{:im}(0.0+0.0im)\n @test (~Partial{:im})(x) == 3+2im\nend\n\n@testset \"value\" begin\n x = 1.0\n @test value(x) === 1.0\n @assign (x |> value) 0.2\n @test x == 0.2\nend\n\nstruct NiTypeTest{T} <: IWrapper{T}\n x::T\n g::T\nend\nNiTypeTest(x) = NiTypeTest(x, zero(x))\n@fieldview NiLang.value(invtype::NiTypeTest) = invtype.x\n@fieldview gg(invtype::NiTypeTest) = invtype.g\n\n@testset \"inv type\" begin\n it = NiTypeTest(0.5)\n @test eps(typeof(it)) === eps(Float64)\n @test value(it) == 0.5\n @test it ≈ NiTypeTest(0.5)\n @test it > 0.4\n @test it < NiTypeTest(0.6)\n @test it < 7\n @test 0.4 < it\n @test 7 > it\n @test chfield(it, value, 0.3) == NiTypeTest(0.3)\n it = chfield(it, Val(:g), 0.2)\n @test almost_same(NiTypeTest(0.5+1e-15), NiTypeTest(0.5))\n @test !almost_same(NiTypeTest(1.0), NiTypeTest(1))\n it = NiTypeTest(0.5)\n @test chfield(it, gg, 0.3) == NiTypeTest(0.5, 0.3)\nend\n" } ]